diff options
Diffstat (limited to 'drivers/gpu/drm/xe/xe_sched_job.c')
| -rw-r--r-- | drivers/gpu/drm/xe/xe_sched_job.c | 246 |
1 files changed, 246 insertions, 0 deletions
diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c new file mode 100644 index 000000000000..ab81bfe17e8a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_sched_job.h" + +#include <linux/dma-fence-array.h> +#include <linux/slab.h> + +#include "xe_device_types.h" +#include "xe_engine.h" +#include "xe_gt.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_trace.h" +#include "xe_vm.h" + +static struct kmem_cache *xe_sched_job_slab; +static struct kmem_cache *xe_sched_job_parallel_slab; + +int __init xe_sched_job_module_init(void) +{ + xe_sched_job_slab = + kmem_cache_create("xe_sched_job", + sizeof(struct xe_sched_job) + + sizeof(u64), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_sched_job_slab) + return -ENOMEM; + + xe_sched_job_parallel_slab = + kmem_cache_create("xe_sched_job_parallel", + sizeof(struct xe_sched_job) + + sizeof(u64) * + XE_HW_ENGINE_MAX_INSTANCE , 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_sched_job_parallel_slab) { + kmem_cache_destroy(xe_sched_job_slab); + return -ENOMEM; + } + + return 0; +} + +void xe_sched_job_module_exit(void) +{ + kmem_cache_destroy(xe_sched_job_slab); + kmem_cache_destroy(xe_sched_job_parallel_slab); +} + +static struct xe_sched_job *job_alloc(bool parallel) +{ + return kmem_cache_zalloc(parallel ? xe_sched_job_parallel_slab : + xe_sched_job_slab, GFP_KERNEL); +} + +bool xe_sched_job_is_migration(struct xe_engine *e) +{ + return e->vm && (e->vm->flags & XE_VM_FLAG_MIGRATION) && + !(e->flags & ENGINE_FLAG_WA); +} + +static void job_free(struct xe_sched_job *job) +{ + struct xe_engine *e = job->engine; + bool is_migration = xe_sched_job_is_migration(e); + + kmem_cache_free(xe_engine_is_parallel(job->engine) || is_migration ? + xe_sched_job_parallel_slab : xe_sched_job_slab, job); +} + +struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, + u64 *batch_addr) +{ + struct xe_sched_job *job; + struct dma_fence **fences; + bool is_migration = xe_sched_job_is_migration(e); + int err; + int i, j; + u32 width; + + /* Migration and kernel engines have their own locking */ + if (!(e->flags & (ENGINE_FLAG_KERNEL | ENGINE_FLAG_VM | + ENGINE_FLAG_WA))) { + lockdep_assert_held(&e->vm->lock); + if (!xe_vm_no_dma_fences(e->vm)) + xe_vm_assert_held(e->vm); + } + + job = job_alloc(xe_engine_is_parallel(e) || is_migration); + if (!job) + return ERR_PTR(-ENOMEM); + + job->engine = e; + kref_init(&job->refcount); + xe_engine_get(job->engine); + + err = drm_sched_job_init(&job->drm, e->entity, 1, NULL); + if (err) + goto err_free; + + if (!xe_engine_is_parallel(e)) { + job->fence = xe_lrc_create_seqno_fence(e->lrc); + if (IS_ERR(job->fence)) { + err = PTR_ERR(job->fence); + goto err_sched_job; + } + } else { + struct dma_fence_array *cf; + + fences = kmalloc_array(e->width, sizeof(*fences), GFP_KERNEL); + if (!fences) { + err = -ENOMEM; + goto err_sched_job; + } + + for (j = 0; j < e->width; ++j) { + fences[j] = xe_lrc_create_seqno_fence(e->lrc + j); + if (IS_ERR(fences[j])) { + err = PTR_ERR(fences[j]); + goto err_fences; + } + } + + cf = dma_fence_array_create(e->width, fences, + e->parallel.composite_fence_ctx, + e->parallel.composite_fence_seqno++, + false); + if (!cf) { + --e->parallel.composite_fence_seqno; + err = -ENOMEM; + goto err_fences; + } + + /* Sanity check */ + for (j = 0; j < e->width; ++j) + XE_BUG_ON(cf->base.seqno != fences[j]->seqno); + + job->fence = &cf->base; + } + + width = e->width; + if (is_migration) + width = 2; + + for (i = 0; i < width; ++i) + job->batch_addr[i] = batch_addr[i]; + + trace_xe_sched_job_create(job); + return job; + +err_fences: + for (j = j - 1; j >= 0; --j) { + --e->lrc[j].fence_ctx.next_seqno; + dma_fence_put(fences[j]); + } + kfree(fences); +err_sched_job: + drm_sched_job_cleanup(&job->drm); +err_free: + xe_engine_put(e); + job_free(job); + return ERR_PTR(err); +} + +/** + * xe_sched_job_destroy - Destroy XE schedule job + * @ref: reference to XE schedule job + * + * Called when ref == 0, drop a reference to job's xe_engine + fence, cleanup + * base DRM schedule job, and free memory for XE schedule job. + */ +void xe_sched_job_destroy(struct kref *ref) +{ + struct xe_sched_job *job = + container_of(ref, struct xe_sched_job, refcount); + + xe_engine_put(job->engine); + dma_fence_put(job->fence); + drm_sched_job_cleanup(&job->drm); + job_free(job); +} + +void xe_sched_job_set_error(struct xe_sched_job *job, int error) +{ + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) + return; + + dma_fence_set_error(job->fence, error); + + if (dma_fence_is_array(job->fence)) { + struct dma_fence_array *array = + to_dma_fence_array(job->fence); + struct dma_fence **child = array->fences; + unsigned int nchild = array->num_fences; + + do { + struct dma_fence *current_fence = *child++; + + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + ¤t_fence->flags)) + continue; + dma_fence_set_error(current_fence, error); + } while (--nchild); + } + + trace_xe_sched_job_set_error(job); + + dma_fence_enable_sw_signaling(job->fence); + xe_hw_fence_irq_run(job->engine->fence_irq); +} + +bool xe_sched_job_started(struct xe_sched_job *job) +{ + struct xe_lrc *lrc = job->engine->lrc; + + return xe_lrc_start_seqno(lrc) >= xe_sched_job_seqno(job); +} + +bool xe_sched_job_completed(struct xe_sched_job *job) +{ + struct xe_lrc *lrc = job->engine->lrc; + + /* + * Can safely check just LRC[0] seqno as that is last seqno written when + * parallel handshake is done. + */ + + return xe_lrc_seqno(lrc) >= xe_sched_job_seqno(job); +} + +void xe_sched_job_arm(struct xe_sched_job *job) +{ + drm_sched_job_arm(&job->drm); +} + +void xe_sched_job_push(struct xe_sched_job *job) +{ + xe_sched_job_get(job); + trace_xe_sched_job_exec(job); + drm_sched_entity_push_job(&job->drm); + xe_sched_job_put(job); +} |