diff options
Diffstat (limited to 'drivers/gpu/drm/v3d')
| -rw-r--r-- | drivers/gpu/drm/v3d/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_drv.c | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_drv.h | 83 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_gem.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_irq.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_perfmon.c | 189 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_power.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_sched.c | 103 | ||||
| -rw-r--r-- | drivers/gpu/drm/v3d/v3d_submit.c | 952 |
9 files changed, 853 insertions, 513 deletions
diff --git a/drivers/gpu/drm/v3d/Kconfig b/drivers/gpu/drm/v3d/Kconfig index ce62c5908e1d..6a33e0ab30de 100644 --- a/drivers/gpu/drm/v3d/Kconfig +++ b/drivers/gpu/drm/v3d/Kconfig @@ -5,6 +5,7 @@ config DRM_V3D depends on DRM depends on COMMON_CLK depends on MMU + select DRM_EXEC select DRM_SCHED select DRM_GEM_SHMEM_HELPER help diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index fc81dd1247e3..8d914dbc3315 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -4,7 +4,11 @@ /** * DOC: Broadcom V3D Graphics Driver * - * This driver supports the Broadcom V3D 3.3 and 4.1 OpenGL ES GPUs. + * This driver supports the Broadcom V3D 4.2 and 7.1 GPUs. + * + * Support for V3D 3.3 and 4.1 GPUs is deprecated and it will be removed + * in the next kernel release. + * * For V3D 2.x support, see the VC4 driver. * * The V3D GPU includes a tiled render (composed of a bin and render @@ -414,13 +418,25 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) */ WARN_ON(v3d->ver != gen); + /* V3D 3.3 and V3D 4.1 has had no in-tree userspace since Mesa + * dropped support in 2024 (MR#25851) and they have no known users. + * Due to that, support is scheduled for removal in the next release. + */ + if (v3d->ver <= V3D_GEN_41) { + dev_warn(dev, + "V3D %u.%u support is deprecated and will be removed " + "in the next kernel release. If you rely on this hardware, " + "please report it to dri-devel@lists.freedesktop.org.\n", + v3d->ver / 10, v3d->ver % 10); + } + v3d->cores = V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_NCORES); WARN_ON(v3d->cores > 1); /* multicore not yet implemented */ ident3 = V3D_READ(V3D_HUB_IDENT3); v3d->rev = V3D_GET_FIELD(ident3, V3D_HUB_IDENT3_IPREV); - pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_set_autosuspend_delay(dev, 50); pm_runtime_use_autosuspend(dev); ret = drm_dev_register(drm, 0); diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 423bcfe331a0..f28b19237c0b 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -7,7 +7,8 @@ #include <linux/spinlock_types.h> #include <linux/workqueue.h> -#include <drm/drm_encoder.h> +#include <drm/drm_device.h> +#include <drm/drm_exec.h> #include <drm/drm_gem.h> #include <drm/drm_gem_shmem_helper.h> #include <drm/gpu_scheduler.h> @@ -74,11 +75,13 @@ struct v3d_queue_state { spinlock_t queue_lock; }; -/* Performance monitor object. The perform lifetime is controlled by userspace - * using perfmon related ioctls. A perfmon can be attached to a submit_cl - * request, and when this is the case, HW perf counters will be activated just - * before the submit_cl is submitted to the GPU and disabled when the job is - * done. This way, only events related to a specific job will be counted. +/* Performance monitor object + * + * The performance monitor (perfmon) lifetime is controlled by userspace using + * perfmon related ioctls. A perfmon can be attached to a CL or CSD submission + * request, and when it is, HW performance counters will be activated just + * before the job is submitted to the GPU and disabled when the job is done. + * This way, only events related to a specific submission will be counted. */ struct v3d_perfmon { /* Tracks the number of users of the perfmon, when this counter reaches @@ -86,9 +89,6 @@ struct v3d_perfmon { */ refcount_t refcnt; - /* Protects perfmon stop, as it can be invoked from multiple places. */ - struct mutex lock; - /* Number of counters activated in this perfmon instance * (should be less than DRM_V3D_MAX_PERF_COUNTERS). */ @@ -170,8 +170,32 @@ struct v3d_dev { struct v3d_queue_state queue[V3D_MAX_QUEUES]; - /* Used to track the active perfmon if any. */ - struct v3d_perfmon *active_perfmon; + /* + * Tracks the performance monitor state and consistency. + * + * When a non-global perfmon is attached to a job, the scheduler must + * not run any other job on the HW concurrently (otherwise, the + * counters would be polluted by unrelated work). + */ + struct { + /* Protects @active. */ + spinlock_t lock; + + /* Perfmon currently programmed in HW (or NULL if none). */ + struct v3d_perfmon *active; + + /* Finished fence of the most recently submitted job that + * opened a serialization window (i.e. a job with a non-global + * perfmon attached). + */ + struct dma_fence *fence; + + /* Finished fence of the most recently submitted job on each HW + * queue. Used so that a new perfmon-carrying job can depend on + * every job currently in-flight across all queues. + */ + struct dma_fence *last_hw_fence[V3D_MAX_QUEUES]; + } perfmon_state; /* Protects bo_stats */ struct mutex bo_lock; @@ -288,6 +312,27 @@ to_v3d_fence(struct dma_fence *fence) #define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset) #define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset) +#define V3D_MAX_JOBS_PER_SUBMISSION 3 + +/* Per-ioctl submission context */ +struct v3d_submit { + struct v3d_dev *v3d; + + struct drm_file *file_priv; + + /* DRM exec context for this submission. */ + struct drm_exec exec; + + /* Ordered array of jobs forming the submission chain. Jobs are + * appended via v3d_submit_add_job(), then chained and pushed to + * the scheduler by v3d_submit_jobs(). + */ + struct v3d_job *jobs[V3D_MAX_JOBS_PER_SUBMISSION]; + + /* Number of jobs currently in @jobs. */ + u32 job_count; +}; + struct v3d_job { struct drm_sched_job base; @@ -295,6 +340,9 @@ struct v3d_job { struct v3d_dev *v3d; + /* The queue that the job was submitted on. */ + enum v3d_queue queue; + /* This is the array of BOs that were looked up at the start * of submission. */ @@ -401,8 +449,10 @@ struct v3d_indirect_csd_info { /* Indirect CSD */ struct v3d_csd_job *job; - /* Clean cache job associated to the Indirect CSD job */ - struct v3d_job *clean_job; + /* Indirect CSD args, stashed by the extension parser and later used + * to create the CSD job from them. + */ + struct drm_v3d_submit_csd args; /* Offset within the BO where the workgroup counts are stored */ u32 offset; @@ -417,9 +467,6 @@ struct v3d_indirect_csd_info { /* Indirect BO */ struct drm_gem_object *indirect; - - /* Context of the Indirect CSD job */ - struct ww_acquire_ctx acquire_ctx; }; struct v3d_timestamp_query_info { @@ -645,6 +692,10 @@ void v3d_perfmon_put(struct v3d_perfmon *perfmon); void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon); void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, bool capture); +void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, + bool capture); +void v3d_perfmon_suspend(struct v3d_dev *v3d); +void v3d_perfmon_resume(struct v3d_dev *v3d); struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id); void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv); void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv); diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c index 23a0db9575a5..bc88ebdcdbea 100644 --- a/drivers/gpu/drm/v3d/v3d_gem.c +++ b/drivers/gpu/drm/v3d/v3d_gem.c @@ -149,7 +149,8 @@ v3d_reset(struct v3d_dev *v3d) v3d_mmu_set_page_table(v3d); v3d_irq_reset(v3d); - v3d_perfmon_stop(v3d, v3d->active_perfmon, false); + /* Re-arm the global perfmon HW counters that the reset zeroed. */ + v3d_perfmon_resume(v3d); trace_v3d_reset_end(dev); } @@ -319,6 +320,7 @@ v3d_gem_init(struct drm_device *dev) } spin_lock_init(&v3d->mm_lock); + spin_lock_init(&v3d->perfmon_state.lock); ret = drmm_mutex_init(dev, &v3d->bo_lock); if (ret) goto err_stats; @@ -381,8 +383,11 @@ v3d_gem_destroy(struct drm_device *dev) for (q = 0; q < V3D_MAX_QUEUES; q++) { WARN_ON(v3d->queue[q].active_job); v3d_stats_put(v3d->queue[q].stats); + dma_fence_put(v3d->perfmon_state.last_hw_fence[q]); } + dma_fence_put(v3d->perfmon_state.fence); + drm_mm_takedown(&v3d->mm); dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt, diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c index 86efaef2722c..963d711dc16b 100644 --- a/drivers/gpu/drm/v3d/v3d_irq.c +++ b/drivers/gpu/drm/v3d/v3d_irq.c @@ -90,9 +90,12 @@ v3d_irq_signal_fence(struct v3d_dev *v3d, enum v3d_queue q, void (*trace_irq)(struct drm_device *, uint64_t)) { struct v3d_queue_state *queue = &v3d->queue[q]; - struct v3d_fence *fence = to_v3d_fence(queue->active_job->irq_fence); + struct v3d_job *job = queue->active_job; + struct v3d_fence *fence = to_v3d_fence(job->irq_fence); - v3d_job_update_stats(queue->active_job); + v3d_perfmon_stop(v3d, job->perfmon, true); + + v3d_job_update_stats(job); trace_irq(&v3d->drm, fence->seqno); queue->active_job = NULL; diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c index 48ae748247be..07dab7fb3060 100644 --- a/drivers/gpu/drm/v3d/v3d_perfmon.c +++ b/drivers/gpu/drm/v3d/v3d_perfmon.c @@ -217,26 +217,15 @@ void v3d_perfmon_get(struct v3d_perfmon *perfmon) void v3d_perfmon_put(struct v3d_perfmon *perfmon) { - if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) { - mutex_destroy(&perfmon->lock); + if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) kfree(perfmon); - } } -void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) +static void v3d_perfmon_hw_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) { + u8 ncounters = perfmon->ncounters; + u32 mask = GENMASK(ncounters - 1, 0); unsigned int i; - u32 mask; - u8 ncounters; - - if (WARN_ON_ONCE(!perfmon || v3d->active_perfmon)) - return; - - if (!pm_runtime_get_if_active(v3d->drm.dev)) - return; - - ncounters = perfmon->ncounters; - mask = GENMASK(ncounters - 1, 0); for (i = 0; i < ncounters; i++) { u32 source = i / 4; @@ -258,39 +247,112 @@ void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, mask); V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask); V3D_CORE_WRITE(0, V3D_PCTR_0_OVERFLOW, mask); +} + +static void v3d_perfmon_hw_capture(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) +{ + u32 mask = GENMASK(perfmon->ncounters - 1, 0); - v3d->active_perfmon = perfmon; + for (int i = 0; i < perfmon->ncounters; i++) + perfmon->values[i] += V3D_CORE_READ(0, V3D_PCTR_0_PCTRX(i)); - v3d_pm_runtime_put(v3d); + V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask); } -void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, - bool capture) +static void v3d_perfmon_hw_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, + bool capture) { - unsigned int i; + if (capture) + v3d_perfmon_hw_capture(v3d, perfmon); + + V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0); +} - if (!perfmon || !v3d->active_perfmon) +void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) +{ + guard(spinlock_irqsave)(&v3d->perfmon_state.lock); + + if (!perfmon || v3d->global_perfmon) return; - mutex_lock(&perfmon->lock); - if (perfmon != v3d->active_perfmon) - goto out; + /* Cross-queue serialization should have drained any previous perfmon + * job before this one runs. + */ + if (WARN_ON_ONCE(v3d->perfmon_state.active)) + return; if (!pm_runtime_get_if_active(v3d->drm.dev)) - goto out_clear; + return; - if (capture) - for (i = 0; i < perfmon->ncounters; i++) - perfmon->values[i] += V3D_CORE_READ(0, V3D_PCTR_0_PCTRX(i)); + v3d_perfmon_hw_start(v3d, perfmon); + v3d->perfmon_state.active = perfmon; - V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0); + v3d_pm_runtime_put(v3d); +} +static void v3d_perfmon_capture_locked(struct v3d_dev *v3d, + struct v3d_perfmon *perfmon) +{ + lockdep_assert_held(&v3d->perfmon_state.lock); + + if (!perfmon || perfmon != v3d->perfmon_state.active) + return; + + if (!pm_runtime_get_if_active(v3d->drm.dev)) + return; + + v3d_perfmon_hw_capture(v3d, perfmon); v3d_pm_runtime_put(v3d); +} + +void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, + bool capture) +{ + lockdep_assert_held(&v3d->perfmon_state.lock); -out_clear: - v3d->active_perfmon = NULL; -out: - mutex_unlock(&perfmon->lock); + if (!perfmon || perfmon != v3d->perfmon_state.active) + return; + + v3d->perfmon_state.active = NULL; + + /* If the device is suspended, the HW has already stopped counting. */ + if (!pm_runtime_get_if_active(v3d->drm.dev)) + return; + + v3d_perfmon_hw_stop(v3d, perfmon, capture); + v3d_pm_runtime_put(v3d); +} + +void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, + bool capture) +{ + if (!perfmon) + return; + + guard(spinlock_irqsave)(&v3d->perfmon_state.lock); + v3d_perfmon_stop_locked(v3d, perfmon, capture); +} + +void +v3d_perfmon_suspend(struct v3d_dev *v3d) +{ + guard(spinlock_irqsave)(&v3d->perfmon_state.lock); + + if (!v3d->perfmon_state.active) + return; + + v3d_perfmon_hw_stop(v3d, v3d->perfmon_state.active, true); +} + +void +v3d_perfmon_resume(struct v3d_dev *v3d) +{ + guard(spinlock_irqsave)(&v3d->perfmon_state.lock); + + if (!v3d->perfmon_state.active) + return; + + v3d_perfmon_hw_start(v3d, v3d->perfmon_state.active); } struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id) @@ -316,14 +378,17 @@ static void v3d_perfmon_delete(struct v3d_file_priv *v3d_priv, struct v3d_dev *v3d = v3d_priv->v3d; /* If the active perfmon is being destroyed, stop it first */ - if (perfmon == v3d->active_perfmon) - v3d_perfmon_stop(v3d, perfmon, false); + scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { + v3d_perfmon_stop_locked(v3d, perfmon, false); - /* If the global perfmon is being destroyed, clean it and release - * the reference stashed in v3d_perfmon_set_global_ioctl(). - */ - if (cmpxchg(&v3d->global_perfmon, perfmon, NULL) == perfmon) - v3d_perfmon_put(perfmon); + /* If the global perfmon is being destroyed, clean it and release + * the reference stashed in v3d_perfmon_set_global_ioctl(). + */ + if (v3d->global_perfmon == perfmon) { + v3d_perfmon_put(v3d->global_perfmon); + v3d->global_perfmon = NULL; + } + } v3d_perfmon_put(perfmon); } @@ -371,12 +436,10 @@ int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data, perfmon->ncounters = req->ncounters; refcount_set(&perfmon->refcnt, 1); - mutex_init(&perfmon->lock); ret = xa_alloc(&v3d_priv->perfmons, &id, perfmon, xa_limit_32b, GFP_KERNEL); if (ret < 0) { - mutex_destroy(&perfmon->lock); kfree(perfmon); return ret; } @@ -408,7 +471,9 @@ int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data, struct v3d_dev *v3d = to_v3d_dev(dev); struct v3d_file_priv *v3d_priv = file_priv->driver_priv; struct drm_v3d_perfmon_get_values *req = data; + u64 values[DRM_V3D_MAX_PERF_COUNTERS]; struct v3d_perfmon *perfmon; + size_t size; int ret = 0; if (req->pad != 0) @@ -418,10 +483,14 @@ int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data, if (!perfmon) return -EINVAL; - v3d_perfmon_stop(v3d, perfmon, true); + size = perfmon->ncounters * sizeof(u64); + + scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { + v3d_perfmon_capture_locked(v3d, perfmon); + memcpy(values, perfmon->values, size); + } - if (copy_to_user(u64_to_user_ptr(req->values_ptr), perfmon->values, - perfmon->ncounters * sizeof(u64))) + if (copy_to_user(u64_to_user_ptr(req->values_ptr), values, size)) ret = -EFAULT; v3d_perfmon_put(perfmon); @@ -482,18 +551,36 @@ int v3d_perfmon_set_global_ioctl(struct drm_device *dev, void *data, */ v3d_perfmon_put(perfmon); - old = xchg(&v3d->global_perfmon, NULL); - if (!old) - return -EINVAL; + scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { + old = v3d->global_perfmon; + if (!old) + return -EINVAL; + + v3d_perfmon_stop_locked(v3d, old, true); + v3d->global_perfmon = NULL; + } v3d_perfmon_put(old); return 0; } - if (cmpxchg(&v3d->global_perfmon, NULL, perfmon)) { - v3d_perfmon_put(perfmon); - return -EBUSY; + scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { + if (v3d->perfmon_state.active || v3d->global_perfmon) { + v3d_perfmon_put(perfmon); + return -EBUSY; + } + + v3d->global_perfmon = perfmon; + v3d->perfmon_state.active = perfmon; + + /* If the device is suspended, v3d_perfmon_resume() will + * program the HW on the next resume. + */ + if (pm_runtime_get_if_active(v3d->drm.dev)) { + v3d_perfmon_hw_start(v3d, perfmon); + v3d_pm_runtime_put(v3d); + } } return 0; diff --git a/drivers/gpu/drm/v3d/v3d_power.c b/drivers/gpu/drm/v3d/v3d_power.c index c53146316079..f608e6d5516a 100644 --- a/drivers/gpu/drm/v3d/v3d_power.c +++ b/drivers/gpu/drm/v3d/v3d_power.c @@ -50,6 +50,8 @@ int v3d_power_suspend(struct device *dev) struct v3d_dev *v3d = to_v3d_dev(drm); int ret; + v3d_perfmon_suspend(v3d); + v3d_irq_disable(v3d); v3d_clean_caches(v3d); @@ -92,5 +94,7 @@ int v3d_power_resume(struct device *dev) v3d_mmu_set_page_table(v3d); v3d_irq_enable(v3d); + v3d_perfmon_resume(v3d); + return 0; } diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index c35f9f70cf8d..c49b71b6e6ee 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -126,24 +126,6 @@ v3d_performance_query_info_free(struct v3d_performance_query_info *query_info, } static void -v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job) -{ - struct v3d_perfmon *perfmon = v3d->global_perfmon; - - if (!perfmon) - perfmon = job->perfmon; - - if (perfmon == v3d->active_perfmon) - return; - - if (perfmon != v3d->active_perfmon) - v3d_perfmon_stop(v3d, v3d->active_perfmon, true); - - if (perfmon && v3d->active_perfmon != perfmon) - v3d_perfmon_start(v3d, perfmon); -} - -static void v3d_stats_start(struct v3d_stats *stats, u64 now) { raw_write_seqcount_begin(&stats->lock); @@ -189,15 +171,11 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) struct v3d_dev *v3d = job->base.v3d; struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; unsigned long irqflags; - if (unlikely(job->base.base.s_fence->finished.error)) { - spin_lock_irqsave(&queue->queue_lock, irqflags); - queue->active_job = NULL; - spin_unlock_irqrestore(&queue->queue_lock, irqflags); - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* Lock required around bin_job update vs * v3d_overflow_mem_work(). @@ -214,7 +192,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_BIN); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -224,7 +202,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) job->start, job->end); v3d_job_start_stats(&job->base); - v3d_switch_perfmon(v3d, &job->base); + v3d_perfmon_start(v3d, job->base.perfmon); /* Set the current and end address of the control list. * Writing the end register is what starts the job. @@ -242,6 +220,12 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end); return fence; + +out_clean_job: + spin_lock_irqsave(&queue->queue_lock, irqflags); + queue->active_job = NULL; + spin_unlock_irqrestore(&queue->queue_lock, irqflags); + return fence; } static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) @@ -249,12 +233,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) struct v3d_render_job *job = to_render_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_RENDER].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_RENDER].active_job = &job->base; @@ -268,7 +250,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_RENDER); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -278,7 +260,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) job->start, job->end); v3d_job_start_stats(&job->base); - v3d_switch_perfmon(v3d, &job->base); + v3d_perfmon_start(v3d, job->base.perfmon); /* XXX: Set the QCFG */ @@ -289,6 +271,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end); return fence; + +out_clean_job: + v3d->queue[V3D_RENDER].active_job = NULL; + return fence; } static struct dma_fence * @@ -297,18 +283,16 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) struct v3d_tfu_job *job = to_tfu_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_TFU].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_TFU].active_job = &job->base; fence = v3d_fence_create(v3d, V3D_TFU); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -336,6 +320,10 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC); return fence; + +out_clean_job: + v3d->queue[V3D_TFU].active_job = NULL; + return fence; } static struct dma_fence * @@ -344,13 +332,11 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) struct v3d_csd_job *job = to_csd_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; int i, csd_cfg0_reg; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_CSD].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* The HW interprets a workgroup size of 0 as 65536; however, the * user-space driver exposes a maximum of 65535. Therefore, a 0 in @@ -368,7 +354,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_CSD); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -377,7 +363,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno); v3d_job_start_stats(&job->base); - v3d_switch_perfmon(v3d, &job->base); + v3d_perfmon_start(v3d, job->base.perfmon); csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver); for (i = 1; i <= 6; i++) @@ -395,6 +381,10 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]); return fence; + +out_clean_job: + v3d->queue[V3D_CSD].active_job = NULL; + return fence; } static void @@ -597,15 +587,8 @@ v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data, v3d_perfmon_stop(v3d, perfmon, true); - if (job->copy.do_64bit) { - for (j = 0; j < perfmon->ncounters; j++) - write_to_buffer_64(data, offset + j, - perfmon->values[j]); - } else { - for (j = 0; j < perfmon->ncounters; j++) - write_to_buffer_32(data, offset + j, - perfmon->values[j]); - } + for (j = 0; j < perfmon->ncounters; j++) + write_to_buffer(data, offset + j, job->copy.do_64bit, perfmon->values[j]); v3d_perfmon_put(perfmon); } @@ -660,6 +643,9 @@ v3d_cpu_job_run(struct drm_sched_job *sched_job) struct v3d_cpu_job *job = to_cpu_job(sched_job); struct v3d_dev *v3d = job->base.v3d; + if (unlikely(job->base.base.s_fence->finished.error)) + return NULL; + if (job->job_type >= ARRAY_SIZE(cpu_job_function)) { drm_dbg(&v3d->drm, "Unknown CPU job: %d\n", job->job_type); return NULL; @@ -683,6 +669,9 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job) struct v3d_job *job = to_v3d_job(sched_job); struct v3d_dev *v3d = job->v3d; + if (unlikely(job->base.s_fence->finished.error)) + return NULL; + v3d_job_start_stats(job); v3d_clean_caches(v3d); @@ -709,6 +698,8 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job, if (sched_job) drm_sched_increase_karma(sched_job); + v3d_perfmon_stop(v3d, job->perfmon, false); + /* get the GPU back into the init state */ v3d_reset(v3d); diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 7682b24f13ec..119a81c9e1c4 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -4,6 +4,9 @@ * Copyright (C) 2023 Raspberry Pi */ +#include <linux/dma-fence-unwrap.h> +#include <linux/overflow.h> + #include <drm/drm_print.h> #include <drm/drm_syncobj.h> @@ -20,33 +23,51 @@ * to v3d, so we don't attach dma-buf fences to them. */ static int -v3d_lock_bo_reservations(struct v3d_job *job, - struct ww_acquire_ctx *acquire_ctx) +v3d_submit_lock_reservations(struct v3d_submit *submit) { - int i, ret; - - ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); - if (ret) - return ret; - - for (i = 0; i < job->bo_count; i++) { - ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); + int i, j, ret; + + drm_exec_init(&submit->exec, + DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); + drm_exec_until_all_locked(&submit->exec) { + for (i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + ret = drm_exec_prepare_array(&submit->exec, job->bo, + job->bo_count, 1); + if (ret) + break; + } + drm_exec_retry_on_contention(&submit->exec); if (ret) goto fail; + } - ret = drm_sched_job_add_implicit_dependencies(&job->base, - job->bo[i], true); - if (ret) - goto fail; + for (i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + for (j = 0; j < job->bo_count; j++) { + ret = drm_sched_job_add_implicit_dependencies(&job->base, + job->bo[j], + true); + if (ret) + goto fail; + } } return 0; fail: - drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); + drm_exec_fini(&submit->exec); return ret; } +static void +v3d_submit_unlock_reservations(struct v3d_submit *submit) +{ + drm_exec_fini(&submit->exec); +} + /** * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects * referenced by the job. @@ -64,25 +85,23 @@ fail: * failure, because that will happen at `v3d_job_free()`. */ static int -v3d_lookup_bos(struct drm_device *dev, - struct drm_file *file_priv, - struct v3d_job *job, - u64 bo_handles, - u32 bo_count) +v3d_lookup_bos(struct v3d_submit *submit, u64 bo_handles, u32 bo_count) { - job->bo_count = bo_count; + struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; - if (!job->bo_count) { + last_job->bo_count = bo_count; + + if (!last_job->bo_count) { /* See comment on bo_index for why we have to check * this. */ - drm_warn(dev, "Rendering requires BOs\n"); + drm_warn(&submit->v3d->drm, "Rendering requires BOs\n"); return -EINVAL; } - return drm_gem_objects_lookup(file_priv, + return drm_gem_objects_lookup(submit->file_priv, (void __user *)(uintptr_t)bo_handles, - job->bo_count, &job->bo); + last_job->bo_count, &last_job->bo); } static void @@ -162,74 +181,87 @@ void v3d_job_put(struct v3d_job *job) } static int -v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size) +v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv, + u32 in_sync, struct v3d_submit_ext *se) { - *container = kcalloc(1, size, GFP_KERNEL); - if (!*container) { - drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n"); - return -ENOMEM; + bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); + struct v3d_dev *v3d = job->v3d; + int ret = 0; + + if (!has_multisync) { + /* Ignore syncobj if its handle is zero */ + if (in_sync) + ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, + in_sync, 0); + return ret; + } + + if (se->in_sync_count && se->wait_stage == job->queue) { + struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); + + for (int i = 0; i < se->in_sync_count; i++) { + struct drm_v3d_sem in; + + if (copy_from_user(&in, handle++, sizeof(in))) { + drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); + return -EFAULT; + } + + /* Ignore syncobj if its handle is zero */ + if (in.handle) { + ret = drm_sched_job_add_syncobj_dependency(&job->base, + file_priv, in.handle, 0); + if (ret) + return ret; + } + } } return 0; } -static void -v3d_job_deallocate(void **container) -{ - kfree(*container); - *container = NULL; -} +static const struct { + size_t size; + void (*free)(struct kref *ref); +} v3d_job_types[] = { + [V3D_BIN] = { sizeof(struct v3d_bin_job), v3d_job_free }, + [V3D_RENDER] = { sizeof(struct v3d_render_job), v3d_render_job_free }, + [V3D_TFU] = { sizeof(struct v3d_tfu_job), v3d_job_free }, + [V3D_CSD] = { sizeof(struct v3d_csd_job), v3d_job_free }, + [V3D_CACHE_CLEAN] = { sizeof(struct v3d_job), v3d_job_free }, + [V3D_CPU] = { sizeof(struct v3d_cpu_job), v3d_cpu_job_free }, +}; -static int -v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, - struct v3d_job *job, void (*free)(struct kref *ref), - u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) +static struct v3d_job * +v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue) { - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; - bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); - int ret, i; + struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; + struct v3d_dev *v3d = submit->v3d; + struct v3d_job *job; + int ret; + + if (queue >= V3D_MAX_QUEUES) + return ERR_PTR(-EINVAL); + + job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL); + if (!job) + return ERR_PTR(-ENOMEM); job->v3d = v3d; - job->free = free; + job->queue = queue; job->file_priv = v3d_priv; + job->free = v3d_job_types[queue].free; ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], - 1, v3d_priv, file_priv->client_id); + 1, v3d_priv, submit->file_priv->client_id); if (ret) - return ret; - - if (has_multisync) { - if (se->in_sync_count && se->wait_stage == queue) { - struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); - - for (i = 0; i < se->in_sync_count; i++) { - struct drm_v3d_sem in; - - if (copy_from_user(&in, handle++, sizeof(in))) { - ret = -EFAULT; - drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); - goto fail_job_init; - } - ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); - - // TODO: Investigate why this was filtered out for the IOCTL. - if (ret && ret != -ENOENT) - goto fail_job_init; - } - } - } else { - ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); - - // TODO: Investigate why this was filtered out for the IOCTL. - if (ret && ret != -ENOENT) - goto fail_job_init; - } + goto fail_free; /* CPU jobs don't require hardware resources */ if (queue != V3D_CPU) { ret = v3d_pm_runtime_get(v3d); if (ret) - goto fail_job_init; + goto fail_sched_job; job->has_pm_ref = true; } @@ -238,52 +270,160 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); job->global_stats = v3d_stats_get(v3d->queue[queue].stats); - return 0; + submit->jobs[submit->job_count++] = job; -fail_job_init: + return job; + +fail_sched_job: drm_sched_job_cleanup(&job->base); - return ret; +fail_free: + kfree(job); + return ERR_PTR(ret); } static void -v3d_push_job(struct v3d_job *job) +v3d_submit_put_jobs(struct v3d_submit *submit) { - drm_sched_job_arm(&job->base); + for (int i = 0; i < submit->job_count; i++) + v3d_job_put(submit->jobs[i]); +} - job->done_fence = dma_fence_get(&job->base.s_fence->finished); +static void +v3d_submit_cleanup_jobs(struct v3d_submit *submit) +{ + for (int i = 0; i < submit->job_count; i++) + v3d_job_cleanup(submit->jobs[i]); +} - /* put by scheduler job completion */ - kref_get(&job->refcount); +static int +v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id) +{ + struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; + struct v3d_dev *v3d = submit->v3d; + struct v3d_perfmon *perfmon; + + if (!perfmon_id) + return 0; - drm_sched_entity_push_job(&job->base); + scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { + if (v3d->global_perfmon) + return -EAGAIN; + } + + perfmon = v3d_perfmon_find(v3d_priv, perfmon_id); + if (!perfmon) + return -ENOENT; + + for (int i = 0; i < submit->job_count; i++) { + submit->jobs[i]->perfmon = perfmon; + if (i != 0) + v3d_perfmon_get(perfmon); + } + + return 0; } -static void -v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, - struct v3d_job *job, - struct ww_acquire_ctx *acquire_ctx, - u32 out_sync, - struct v3d_submit_ext *se, - struct dma_fence *done_fence) +/* + * Prepare fences to enforce job serialization when a perfmon is active. A job + * that carries a non-global perfmon must wait for every job currently in-flight + * across all HW queues to finish, otherwise concurrent unrelated work on the + * same core would pollute the performance counters. Symmetrically, while such a + * job is still in-flight, all subsequently submitted jobs must wait for it. + * + * We don't serialize the jobs when using a global perfmon as it's expected to + * track concurrent activity from all jobs. + */ +static int +v3d_serialize_for_perfmon(struct v3d_job *job) { - struct drm_syncobj *sync_out; - bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); - int i; + struct v3d_dev *v3d = job->v3d; + struct dma_fence *merged; + bool is_global_perfmon; + int ret; + + lockdep_assert_held(&v3d->sched_lock); + + scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) + is_global_perfmon = !!v3d->global_perfmon; + + if (is_global_perfmon) + goto publish; + + if (job->perfmon) { + for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) { + struct dma_fence *f = v3d->perfmon_state.last_hw_fence[q]; + + if (!f || dma_fence_is_signaled(f)) + continue; + + ret = drm_sched_job_add_dependency(&job->base, dma_fence_get(f)); + if (ret) + return ret; + } + } else if (v3d->perfmon_state.fence && + !dma_fence_is_signaled(v3d->perfmon_state.fence)) { + ret = drm_sched_job_add_dependency(&job->base, + dma_fence_get(v3d->perfmon_state.fence)); + if (ret) + return ret; + } + +publish: + /* + * Accumulate every in-flight job on this queue into one merged fence. + * A HW queue is fed by several scheduler entities (one per-fd), so jobs + * on it can complete out of order. + */ + merged = dma_fence_unwrap_merge(v3d->perfmon_state.last_hw_fence[job->queue], + job->done_fence); + if (!merged) + return -ENOMEM; + + dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]); + v3d->perfmon_state.last_hw_fence[job->queue] = merged; - for (i = 0; i < job->bo_count; i++) { - /* XXX: Use shared fences for read-only objects. */ - dma_resv_add_fence(job->bo[i]->resv, job->done_fence, - DMA_RESV_USAGE_WRITE); + if (job->perfmon && !is_global_perfmon) { + dma_fence_put(v3d->perfmon_state.fence); + v3d->perfmon_state.fence = dma_fence_get(job->done_fence); + } + + return 0; +} + +static void +v3d_submit_attach_object_fences(struct v3d_submit *submit) +{ + struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; + + /* The submission's last fence covers the entire submission. Attach it + * to every BO touched by any job in the submission. + */ + for (int i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + for (int j = 0; j < job->bo_count; j++) { + /* XXX: Use shared fences for read-only objects. */ + dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence, + DMA_RESV_USAGE_WRITE); + } } +} - drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); +static void +v3d_submit_process_post_deps(struct v3d_submit *submit, struct drm_syncobj *sync_out, + struct v3d_submit_ext *se) +{ + bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); + struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; + + /* Make sure single syncobj and multisync are mutually exclusive */ + WARN_ON_ONCE(sync_out && has_multisync); /* Update the return sync object for the job */ /* If it only supports a single signal semaphore*/ if (!has_multisync) { - sync_out = drm_syncobj_find(file_priv, out_sync); if (sync_out) { - drm_syncobj_replace_fence(sync_out, done_fence); + drm_syncobj_replace_fence(sync_out, last_job->done_fence); drm_syncobj_put(sync_out); } return; @@ -291,9 +431,9 @@ v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, /* If multiple semaphores extension is supported */ if (se->out_sync_count) { - for (i = 0; i < se->out_sync_count; i++) { + for (int i = 0; i < se->out_sync_count; i++) { drm_syncobj_replace_fence(se->out_syncs[i].syncobj, - done_fence); + last_job->done_fence); drm_syncobj_put(se->out_syncs[i].syncobj); } kvfree(se->out_syncs); @@ -301,53 +441,100 @@ v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, } static int -v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, - struct v3d_dev *v3d, - struct drm_v3d_submit_csd *args, - struct v3d_csd_job **job, - struct v3d_job **clean_job, - struct v3d_submit_ext *se, - struct ww_acquire_ctx *acquire_ctx) +v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out, + struct v3d_submit_ext *se) { - int ret; + struct v3d_dev *v3d = submit->v3d; + int ret = 0; - ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job)); - if (ret) - return ret; + mutex_lock(&v3d->sched_lock); - ret = v3d_job_init(v3d, file_priv, &(*job)->base, - v3d_job_free, args->in_sync, se, V3D_CSD); - if (ret) { - v3d_job_deallocate((void *)job); - return ret; + for (int i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + drm_sched_job_arm(&job->base); + job->done_fence = dma_fence_get(&job->base.s_fence->finished); + + /* put by scheduler job completion */ + kref_get(&job->refcount); } - ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job)); - if (ret) - return ret; + for (int i = 1; i < submit->job_count; i++) { + ret = drm_sched_job_add_dependency(&submit->jobs[i]->base, + dma_fence_get(submit->jobs[i - 1]->done_fence)); + if (ret) + goto err; + } - ret = v3d_job_init(v3d, file_priv, *clean_job, - v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); - if (ret) { - v3d_job_deallocate((void *)clean_job); - return ret; + for (int i = 0; i < submit->job_count; i++) { + ret = v3d_serialize_for_perfmon(submit->jobs[i]); + if (ret) + goto err; } - (*job)->args = *args; + for (int i = 0; i < submit->job_count; i++) + drm_sched_entity_push_job(&submit->jobs[i]->base); + + mutex_unlock(&v3d->sched_lock); + + v3d_submit_attach_object_fences(submit); + v3d_submit_unlock_reservations(submit); + v3d_submit_process_post_deps(submit, sync_out, se); + + v3d_submit_put_jobs(submit); + + return 0; - ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, - args->bo_handles, args->bo_handle_count); +err: + /* Mark every armed job as failed so run_job() skips execution */ + for (int i = 0; i < submit->job_count; i++) + dma_fence_set_error(&submit->jobs[i]->base.s_fence->finished, ret); + + for (int i = 0; i < submit->job_count; i++) + drm_sched_entity_push_job(&submit->jobs[i]->base); + + mutex_unlock(&v3d->sched_lock); + + v3d_submit_unlock_reservations(submit); + v3d_submit_put_jobs(submit); + + return ret; +} + +static int +v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit, + struct drm_v3d_submit_csd *args, + struct v3d_submit_ext *se) +{ + struct v3d_csd_job *job; + struct v3d_job *clean_job; + int ret; + + job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD); + if (IS_ERR(job)) + return PTR_ERR(job); + + ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se); if (ret) return ret; - return v3d_lock_bo_reservations(*clean_job, acquire_ctx); + job->args = *args; + + clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN); + if (IS_ERR(clean_job)) + return PTR_ERR(clean_job); + + return v3d_lookup_bos(submit, args->bo_handles, args->bo_handle_count); } static void -v3d_put_multisync_post_deps(struct v3d_submit_ext *se) +v3d_submit_put_post_deps(struct drm_syncobj *sync_out, struct v3d_submit_ext *se) { unsigned int i; + if (sync_out) + drm_syncobj_put(sync_out); + if (!(se && se->out_sync_count)) return; @@ -489,6 +676,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, } job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; + info->args = indirect_csd.submit; info->offset = indirect_csd.offset; info->wg_size = indirect_csd.wg_size; memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, @@ -498,9 +686,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, if (!info->indirect) return -ENOENT; - return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, - &info->job, &info->clean_job, - NULL, &info->acquire_ctx); + return 0; } /* Get data for the query timestamp job submission. */ @@ -907,18 +1093,16 @@ int v3d_submit_cl_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_cl *args = data; + struct drm_syncobj *sync_out = NULL; struct v3d_submit_ext se = {0}; struct v3d_bin_job *bin = NULL; - struct v3d_render_job *render = NULL; - struct v3d_job *clean_job = NULL; - struct v3d_job *last_job; - struct ww_acquire_ctx acquire_ctx; - int ret = 0; + struct v3d_render_job *render; + struct v3d_job *clean_job; + int ret; - trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); + trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end); if (args->pad) return -EINVAL; @@ -938,30 +1122,17 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render)); - if (ret) - return ret; - - ret = v3d_job_init(v3d, file_priv, &render->base, - v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); - if (ret) { - v3d_job_deallocate((void *)&render); - goto fail; + /* If multisync is configured, give priority to it and ignore out_sync. */ + if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { + sync_out = drm_syncobj_find(file_priv, args->out_sync); + if (!sync_out) + return -ENOENT; } - render->start = args->rcl_start; - render->end = args->rcl_end; - INIT_LIST_HEAD(&render->unref_list); - if (args->bcl_start != args->bcl_end) { - ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin)); - if (ret) - goto fail; - - ret = v3d_job_init(v3d, file_priv, &bin->base, - v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); - if (ret) { - v3d_job_deallocate((void *)&bin); + bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN); + if (IS_ERR(bin)) { + ret = PTR_ERR(bin); goto fail; } @@ -970,101 +1141,60 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data, bin->qma = args->qma; bin->qms = args->qms; bin->qts = args->qts; - bin->render = render; - } - if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { - ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job)); + ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl, + &se); if (ret) goto fail; - - ret = v3d_job_init(v3d, file_priv, clean_job, - v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); - if (ret) { - v3d_job_deallocate((void *)&clean_job); - goto fail; - } - - last_job = clean_job; - } else { - last_job = &render->base; } - ret = v3d_lookup_bos(dev, file_priv, last_job, - args->bo_handles, args->bo_handle_count); - if (ret) + render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER); + if (IS_ERR(render)) { + ret = PTR_ERR(render); goto fail; + } - ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); - if (ret) - goto fail; + INIT_LIST_HEAD(&render->unref_list); + render->start = args->rcl_start; + render->end = args->rcl_end; - if (args->perfmon_id) { - if (v3d->global_perfmon) { - ret = -EAGAIN; - goto fail_perfmon; - } + if (bin) + bin->render = render; - render->base.perfmon = v3d_perfmon_find(v3d_priv, - args->perfmon_id); + ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se); + if (ret) + goto fail; - if (!render->base.perfmon) { - ret = -ENOENT; - goto fail_perfmon; + if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { + clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN); + if (IS_ERR(clean_job)) { + ret = PTR_ERR(clean_job); + goto fail; } } - mutex_lock(&v3d->sched_lock); - if (bin) { - bin->base.perfmon = render->base.perfmon; - v3d_perfmon_get(bin->base.perfmon); - v3d_push_job(&bin->base); - - ret = drm_sched_job_add_dependency(&render->base.base, - dma_fence_get(bin->base.done_fence)); - if (ret) - goto fail_unreserve; - } - - v3d_push_job(&render->base); - - if (clean_job) { - struct dma_fence *render_fence = - dma_fence_get(render->base.done_fence); - ret = drm_sched_job_add_dependency(&clean_job->base, - render_fence); - if (ret) - goto fail_unreserve; - clean_job->perfmon = render->base.perfmon; - v3d_perfmon_get(clean_job->perfmon); - v3d_push_job(clean_job); - } + ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); + if (ret) + goto fail; - mutex_unlock(&v3d->sched_lock); + ret = v3d_lookup_bos(&submit, args->bo_handles, args->bo_handle_count); + if (ret) + goto fail; - v3d_attach_fences_and_unlock_reservation(file_priv, - last_job, - &acquire_ctx, - args->out_sync, - &se, - last_job->done_fence); + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - v3d_job_put(&bin->base); - v3d_job_put(&render->base); - v3d_job_put(clean_job); + ret = v3d_submit_jobs(&submit, sync_out, &se); + if (ret) + goto fail_submit; return 0; -fail_unreserve: - mutex_unlock(&v3d->sched_lock); -fail_perfmon: - drm_gem_unlock_reservations(last_job->bo, - last_job->bo_count, &acquire_ctx); fail: - v3d_job_cleanup((void *)bin); - v3d_job_cleanup((void *)render); - v3d_job_cleanup(clean_job); - v3d_put_multisync_post_deps(&se); + v3d_submit_cleanup_jobs(&submit); +fail_submit: + v3d_submit_put_post_deps(sync_out, &se); return ret; } @@ -1082,14 +1212,14 @@ int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_tfu *args = data; + struct drm_syncobj *sync_out = NULL; struct v3d_submit_ext se = {0}; - struct v3d_tfu_job *job = NULL; - struct ww_acquire_ctx acquire_ctx; + struct v3d_tfu_job *job; int ret = 0; - trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); + trace_v3d_submit_tfu_ioctl(dev, args->iia); if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { drm_dbg(dev, "invalid flags: %d\n", args->flags); @@ -1104,17 +1234,23 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job)); - if (ret) - return ret; + /* If multisync is configured, give priority to it and ignore out_sync. */ + if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { + sync_out = drm_syncobj_find(file_priv, args->out_sync); + if (!sync_out) + return -ENOENT; + } - ret = v3d_job_init(v3d, file_priv, &job->base, - v3d_job_free, args->in_sync, &se, V3D_TFU); - if (ret) { - v3d_job_deallocate((void *)&job); + job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU); + if (IS_ERR(job)) { + ret = PTR_ERR(job); goto fail; } + ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se); + if (ret) + goto fail; + job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); if (!job->base.bo) { ret = -ENOMEM; @@ -1142,27 +1278,20 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, job->base.bo[job->base.bo_count] = bo; } - ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); + ret = v3d_submit_lock_reservations(&submit); if (ret) goto fail; - mutex_lock(&v3d->sched_lock); - v3d_push_job(&job->base); - mutex_unlock(&v3d->sched_lock); - - v3d_attach_fences_and_unlock_reservation(file_priv, - &job->base, &acquire_ctx, - args->out_sync, - &se, - job->base.done_fence); - - v3d_job_put(&job->base); + ret = v3d_submit_jobs(&submit, sync_out, &se); + if (ret) + goto fail_submit; return 0; fail: - v3d_job_cleanup((void *)job); - v3d_put_multisync_post_deps(&se); + v3d_submit_cleanup_jobs(&submit); +fail_submit: + v3d_submit_put_post_deps(sync_out, &se); return ret; } @@ -1180,21 +1309,18 @@ int v3d_submit_csd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_csd *args = data; + struct drm_syncobj *sync_out = NULL; struct v3d_submit_ext se = {0}; - struct v3d_csd_job *job = NULL; - struct v3d_job *clean_job = NULL; - struct ww_acquire_ctx acquire_ctx; int ret; - trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); + trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]); if (args->pad) return -EINVAL; - if (!v3d_has_csd(v3d)) { + if (!v3d_has_csd(submit.v3d)) { drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); return -EINVAL; } @@ -1212,58 +1338,35 @@ v3d_submit_csd_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, - &job, &clean_job, &se, - &acquire_ctx); - if (ret) - goto fail; - - if (args->perfmon_id) { - if (v3d->global_perfmon) { - ret = -EAGAIN; - goto fail_perfmon; - } - - job->base.perfmon = v3d_perfmon_find(v3d_priv, - args->perfmon_id); - if (!job->base.perfmon) { - ret = -ENOENT; - goto fail_perfmon; - } + /* If multisync is configured, give priority to it and ignore out_sync. */ + if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) { + sync_out = drm_syncobj_find(file_priv, args->out_sync); + if (!sync_out) + return -ENOENT; } - mutex_lock(&v3d->sched_lock); - v3d_push_job(&job->base); - - ret = drm_sched_job_add_dependency(&clean_job->base, - dma_fence_get(job->base.done_fence)); + ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se); if (ret) - goto fail_unreserve; + goto fail; - v3d_push_job(clean_job); - mutex_unlock(&v3d->sched_lock); + ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); + if (ret) + goto fail; - v3d_attach_fences_and_unlock_reservation(file_priv, - clean_job, - &acquire_ctx, - args->out_sync, - &se, - clean_job->done_fence); + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - v3d_job_put(&job->base); - v3d_job_put(clean_job); + ret = v3d_submit_jobs(&submit, sync_out, &se); + if (ret) + goto fail_submit; return 0; -fail_unreserve: - mutex_unlock(&v3d->sched_lock); -fail_perfmon: - drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, - &acquire_ctx); fail: - v3d_job_cleanup((void *)job); - v3d_job_cleanup(clean_job); - v3d_put_multisync_post_deps(&se); + v3d_submit_cleanup_jobs(&submit); +fail_submit: + v3d_submit_put_post_deps(sync_out, &se); return ret; } @@ -1277,6 +1380,126 @@ static const unsigned int cpu_job_bo_handle_count[] = { [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, }; +/* Reject offset + (count - 1) * stride + write_size if it leaves the BO. */ +static int +v3d_check_copy_extent(struct drm_device *dev, size_t bo_size, + u32 offset, u32 stride, u32 count, u64 write_size) +{ + u64 last; + + if (!count) + return 0; + + /* + * The executors walk a u8 * cursor, so the furthest written byte is + * offset + (count - 1) * stride + write_size, matching the pointer + * arithmetic in v3d_copy_query_results()/v3d_copy_performance_query(). + * (count - 1) * stride is a u32 * u32 product that is exact in u64, + * and offset + write_size stays far below the u64 range, so a single + * overflow check guards the total. + */ + last = write_size + offset; + if (check_add_overflow((u64)(count - 1) * stride, last, &last) || + last > bo_size) { + drm_dbg(dev, "CPU job copy buffer exceeds the destination BO.\n"); + return -EINVAL; + } + + return 0; +} + +/* Reject a query CPU job whose writes would land outside their BO. */ +static int +v3d_cpu_job_bounds_check(struct v3d_cpu_job *job) +{ + struct drm_device *dev = &job->base.v3d->drm; + struct v3d_timestamp_query_info *tquery = &job->timestamp_query; + struct v3d_copy_query_results_info *copy = &job->copy; + u32 elem = copy->do_64bit ? sizeof(u64) : sizeof(u32); + struct v3d_bo *dst, *src; + u64 slots, write_size; + u32 i; + + switch (job->job_type) { + case V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY: + case V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY: + /* Each query writes one u64 timestamp slot into bo[0]. */ + dst = to_v3d_bo(job->base.bo[0]); + + for (i = 0; i < tquery->count; i++) { + if ((u64)tquery->queries[i].offset + sizeof(u64) > + dst->base.base.size) + goto err_range; + } + return 0; + case V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY: + /* Copies one u64 per query from bo[1] into bo[0]. */ + dst = to_v3d_bo(job->base.bo[0]); + src = to_v3d_bo(job->base.bo[1]); + + for (i = 0; i < tquery->count; i++) { + if ((u64)tquery->queries[i].offset + sizeof(u64) > + src->base.base.size) + goto err_range; + } + + write_size = (copy->availability_bit ? 2 : 1) * elem; + return v3d_check_copy_extent(dev, dst->base.base.size, + copy->offset, copy->stride, + tquery->count, write_size); + case V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY: + /* + * Each query writes nperfmons * DRM_V3D_MAX_PERF_COUNTERS + * counter slots into bo[0], plus an availability slot at + * index ncounters. nperfmons and ncounters are user values, + * so the slot count is computed overflow-safe. + */ + dst = to_v3d_bo(job->base.bo[0]); + + slots = (u64)job->performance_query.nperfmons * + DRM_V3D_MAX_PERF_COUNTERS; + if (copy->availability_bit) + slots = max(slots, + (u64)job->performance_query.ncounters + 1); + + write_size = slots * elem; + return v3d_check_copy_extent(dev, dst->base.base.size, + copy->offset, copy->stride, + job->performance_query.count, + write_size); + case V3D_CPU_JOB_TYPE_INDIRECT_CSD: { + struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd; + + /* 3 is the three dimensions (x, y, z) of the workgroup counts. */ + src = to_v3d_bo(job->base.bo[0]); + if ((u64)indirect_csd->offset + 3 * sizeof(u32) > + src->base.base.size) + goto err_range; + + dst = to_v3d_bo(indirect_csd->indirect); + for (i = 0; i < 3; i++) { + u32 uidx = indirect_csd->wg_uniform_offsets[i]; + + /* + * 0xffffffff means "skip this rewrite", so the exec + * path never writes that index and it needs no check. + */ + if (uidx != 0xffffffff && + (u64)uidx * sizeof(u32) + sizeof(u32) > + dst->base.base.size) + goto err_range; + } + return 0; + } + default: + return 0; + } + +err_range: + drm_dbg(dev, "CPU job query offset exceeds the BO.\n"); + return -EINVAL; +} + /** * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. * @dev: DRM device @@ -1290,14 +1513,10 @@ int v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_cpu *args = data; struct v3d_submit_ext se = {0}; - struct v3d_submit_ext *out_se = NULL; struct v3d_cpu_job *cpu_job = NULL; - struct v3d_csd_job *csd_job = NULL; - struct v3d_job *clean_job = NULL; - struct ww_acquire_ctx acquire_ctx; int ret; if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { @@ -1305,9 +1524,9 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job)); - if (ret) - return ret; + cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU); + if (IS_ERR(cpu_job)) + return PTR_ERR(cpu_job); if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); @@ -1330,92 +1549,55 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, goto fail; } - trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); + trace_v3d_submit_cpu_ioctl(dev, cpu_job->job_type); - ret = v3d_job_init(v3d, file_priv, &cpu_job->base, - v3d_cpu_job_free, 0, &se, V3D_CPU); - if (ret) { - v3d_job_deallocate((void *)&cpu_job); + ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se); + if (ret) goto fail; - } - - clean_job = cpu_job->indirect_csd.clean_job; - csd_job = cpu_job->indirect_csd.job; + /* Look up the CPU jobs' BOs before v3d_setup_csd_jobs_and_bos() appends + * the CSD and clean jobs in the case of indirect CSD job. + */ if (args->bo_handle_count) { - ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, - args->bo_handles, args->bo_handle_count); + ret = v3d_lookup_bos(&submit, args->bo_handles, args->bo_handle_count); if (ret) goto fail; - ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); + ret = v3d_cpu_job_bounds_check(cpu_job); if (ret) goto fail; } - mutex_lock(&v3d->sched_lock); - v3d_push_job(&cpu_job->base); - - switch (cpu_job->job_type) { - case V3D_CPU_JOB_TYPE_INDIRECT_CSD: - ret = drm_sched_job_add_dependency(&csd_job->base.base, - dma_fence_get(cpu_job->base.done_fence)); - if (ret) - goto fail_unreserve; - - v3d_push_job(&csd_job->base); - - ret = drm_sched_job_add_dependency(&clean_job->base, - dma_fence_get(csd_job->base.done_fence)); + if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) { + ret = v3d_setup_csd_jobs_and_bos(&submit, &cpu_job->indirect_csd.args, + NULL); if (ret) - goto fail_unreserve; + goto fail; - v3d_push_job(clean_job); + /* The CSD job was appended at jobs[1] */ + if (WARN_ON(submit.jobs[1]->queue != V3D_CSD)) { + ret = -EINVAL; + goto fail; + } - break; - default: - break; + cpu_job->indirect_csd.job = container_of(submit.jobs[1], struct v3d_csd_job, + base); } - mutex_unlock(&v3d->sched_lock); - - out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; - v3d_attach_fences_and_unlock_reservation(file_priv, - &cpu_job->base, - &acquire_ctx, 0, - out_se, cpu_job->base.done_fence); - - switch (cpu_job->job_type) { - case V3D_CPU_JOB_TYPE_INDIRECT_CSD: - v3d_attach_fences_and_unlock_reservation(file_priv, - clean_job, - &cpu_job->indirect_csd.acquire_ctx, - 0, &se, clean_job->done_fence); - break; - default: - break; - } + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - v3d_job_put(&cpu_job->base); - v3d_job_put(&csd_job->base); - v3d_job_put(clean_job); + ret = v3d_submit_jobs(&submit, NULL, &se); + if (ret) + goto fail_submit; return 0; -fail_unreserve: - mutex_unlock(&v3d->sched_lock); - - drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, - &acquire_ctx); - - drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, - &cpu_job->indirect_csd.acquire_ctx); - fail: - v3d_job_cleanup((void *)cpu_job); - v3d_job_cleanup((void *)csd_job); - v3d_job_cleanup(clean_job); - v3d_put_multisync_post_deps(&se); + v3d_submit_cleanup_jobs(&submit); +fail_submit: + v3d_submit_put_post_deps(NULL, &se); return ret; } |