diff options
54 files changed, 792 insertions, 338 deletions
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 4b3a62aa8798..b713a57b3a3b 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -665,6 +665,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) struct amdxdna_dev *xdna = client->xdna; const struct drm_sched_init_args args = { .ops = &sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = HWCTX_MAX_CMDS, .timeout = tdr_timeout_ms ? msecs_to_jiffies(tdr_timeout_ms) : @@ -1052,6 +1053,16 @@ again: found = false; down_write(&xdna->notifier_lock); list_for_each_entry(mapp, &abo->mem.umap_list, node) { + /* + * Skip entries that have already been unmapped. + * + * If userspace unmaps the address and later submits I/O using + * it, the IOMMU will reject the access and report a fault. + * Ignore such entries here. + */ + if (mapp->unmapped) + continue; + if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { found = true; break; @@ -1059,6 +1070,12 @@ again: } if (!found) { + /* + * This also covers the case where all mappings have been + * removed. There are no invalid mappings left to process. + * Any subsequent I/O using the unmapped address will be + * rejected by the IOMMU. + */ abo->mem.map_invalid = false; up_write(&xdna->notifier_lock); return 0; diff --git a/drivers/accel/ethosu/ethosu_job.c b/drivers/accel/ethosu/ethosu_job.c index b76924645aaa..1e2465279aae 100644 --- a/drivers/accel/ethosu/ethosu_job.c +++ b/drivers/accel/ethosu/ethosu_job.c @@ -296,6 +296,7 @@ int ethosu_job_init(struct ethosu_device *edev) struct device *dev = edev->base.dev; struct drm_sched_init_args args = { .ops = ðosu_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 1, .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), .name = dev_name(dev), diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c index 2f1861f960cc..ac51bff39833 100644 --- a/drivers/accel/rocket/rocket_job.c +++ b/drivers/accel/rocket/rocket_job.c @@ -437,6 +437,7 @@ int rocket_job_init(struct rocket_core *core) { struct drm_sched_init_args args = { .ops = &rocket_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 1, .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), .name = dev_name(core->dev), diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index f8bf0f6b5097..35d76cfbc88a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -248,6 +248,10 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, if (size < sizeof(struct drm_amdgpu_cs_chunk_fence)) goto free_partial_kdata; + /* Only a single user fence is allowed to simplify handling. */ + if (p->uf_bo) + goto free_partial_kdata; + ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata, &uf_offset); if (ret) @@ -1123,8 +1127,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) if (p->gang_size > 1 && !adev->vm_manager.concurrent_flush) { for (i = 0; i < p->gang_size; ++i) { struct drm_sched_entity *entity = p->entities[i]; - struct drm_gpu_scheduler *sched = - container_of(entity->rq, typeof(*sched), rq); + struct drm_gpu_scheduler *sched = entity->rq->sched; struct amdgpu_ring *ring = to_amdgpu_ring(sched); if (amdgpu_vmid_uses_reserved(vm, ring->vm_hub)) @@ -1241,8 +1244,7 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) return r; } - sched = container_of(p->gang_leader->base.entity->rq, typeof(*sched), - rq); + sched = p->gang_leader->base.entity->rq->sched; while ((fence = amdgpu_sync_get_fence(&p->sync))) { struct drm_sched_fence *s_fence = to_drm_sched_fence(fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index bff3e06a756c..e84b75c03c6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1370,6 +1370,31 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) #endif } +/* + * Some dGPUs expose their display endpoint below an internal PCIe switch. + * Use the switch upstream port to query the host-facing link. + */ +static struct pci_dev *amdgpu_device_get_aspm_pdev(struct amdgpu_device *adev) +{ + struct pci_dev *swds, *swus; + + swds = pci_upstream_bridge(adev->pdev); + if (!swds || + (swds->vendor != PCI_VENDOR_ID_ATI && + swds->vendor != PCI_VENDOR_ID_AMD) || + pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM) + return adev->pdev; + + swus = pci_upstream_bridge(swds); + if (!swus || + (swus->vendor != PCI_VENDOR_ID_ATI && + swus->vendor != PCI_VENDOR_ID_AMD) || + pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM) + return adev->pdev; + + return swus; +} + /** * amdgpu_device_should_use_aspm - check if the device should program ASPM * @@ -1382,6 +1407,9 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) */ bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) { + struct pci_dev *aspm_pdev, *parent; + bool enabled; + switch (amdgpu_aspm) { case -1: break; @@ -1396,7 +1424,27 @@ bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) return false; if (amdgpu_device_aspm_support_quirk(adev)) return false; - return pcie_aspm_enabled(adev->pdev); + + /* + * pcie_aspm_enabled() checks the link between its argument and + * the immediate upstream bridge. Use SWUS for dGPUs with an + * internal switch so that this is the host-facing link. + */ + aspm_pdev = amdgpu_device_get_aspm_pdev(adev); + parent = pci_upstream_bridge(aspm_pdev); + if (!parent) { + dev_dbg(adev->dev, "ASPM: no upstream PCIe link for %s\n", + pci_name(aspm_pdev)); + return false; + } + + enabled = pcie_aspm_enabled(aspm_pdev); + /* Report the exact link used for the automatic ASPM decision. */ + dev_dbg(adev->dev, "ASPM: link %s <-> %s is %s\n", + pci_name(parent), pci_name(aspm_pdev), + enabled ? "enabled" : "disabled"); + + return enabled; } /* if we get transitioned to only one device, take VGA back */ @@ -2250,6 +2298,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) { struct drm_sched_init_args args = { .ops = &amdgpu_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .timeout_wq = adev->reset_domain->wq, .dev = adev->dev, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 5e83edbd313b..47e0680ed7b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -311,6 +311,19 @@ static int amdgpu_discovery_get_tmr_info(struct amdgpu_device *adev, goto out; } } else { + if (adev->discovery.offset) { + u32 signature; + + /* If VRAM holds a valid discovery signature at the default + * discovery offset, use it as-is. + */ + amdgpu_device_vram_access(adev, adev->discovery.offset, + &signature, sizeof(signature), + false); + if (le32_to_cpu(signature) == BINARY_SIGNATURE) + goto out; + } + tmr_size = RREG32(mmDRIVER_SCRATCH_2); if (tmr_size) { /* It's preferred to transition to PSP mailbox reg interface diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 6a0699746fbc..f754a4a3a1c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -397,6 +397,25 @@ const struct drm_gem_object_funcs amdgpu_gem_object_funcs = { .vm_ops = &amdgpu_gem_vm_ops, }; +static bool amdgpu_gem_are_domains_valid(u32 domains) +{ + u32 normal = AMDGPU_GEM_DOMAIN_CPU | + AMDGPU_GEM_DOMAIN_GTT | + AMDGPU_GEM_DOMAIN_VRAM; + /* Treat all non CPU/GTT/VRAM domains as special domains. */ + u32 special = AMDGPU_GEM_DOMAIN_MASK & ~normal; + u32 normal_mask = domains & normal; + u32 special_mask = domains & special; + + if (!special_mask) + return true; + + if (normal_mask) + return false; + + return !(special_mask & (special_mask - 1)); +} + /* * GEM ioctls. */ @@ -421,6 +440,8 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, /* reject invalid gem domains */ if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK) return -EINVAL; + if (!amdgpu_gem_are_domains_valid(args->in.domains)) + return -EINVAL; if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) { DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 9ecc6387c1eb..07771721af9d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -388,9 +388,7 @@ static struct dma_fence * amdgpu_job_prepare_job(struct drm_sched_job *sched_job, struct drm_sched_entity *s_entity) { - struct drm_gpu_scheduler *sched = - container_of(s_entity->rq, typeof(*sched), rq); - struct amdgpu_ring *ring = to_amdgpu_ring(sched); + struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched); struct amdgpu_job *job = to_amdgpu_job(sched_job); struct dma_fence *fence; int r; @@ -483,22 +481,25 @@ drm_sched_entity_queue_pop(struct drm_sched_entity *entity) void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) { - struct drm_sched_rq *rq = &sched->rq; - struct drm_sched_entity *s_entity; struct drm_sched_job *s_job; + struct drm_sched_entity *s_entity = NULL; + int i; /* Signal all jobs not yet scheduled */ - spin_lock(&rq->lock); - list_for_each_entry(s_entity, &rq->entities, list) { - while ((s_job = drm_sched_entity_queue_pop(s_entity))) { - struct drm_sched_fence *s_fence = s_job->s_fence; - - dma_fence_signal(&s_fence->scheduled); - dma_fence_set_error(&s_fence->finished, -EHWPOISON); - dma_fence_signal(&s_fence->finished); + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) { + while ((s_job = drm_sched_entity_queue_pop(s_entity))) { + struct drm_sched_fence *s_fence = s_job->s_fence; + + dma_fence_signal(&s_fence->scheduled); + dma_fence_set_error(&s_fence->finished, -EHWPOISON); + dma_fence_signal(&s_fence->finished); + } } + spin_unlock(&rq->lock); } - spin_unlock(&rq->lock); /* Signal all jobs already scheduled to HW */ list_for_each_entry(s_job, &sched->pending_list, list) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index e70a1117b812..56a88e14a044 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -107,10 +107,7 @@ struct amdgpu_job { static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job) { - struct drm_gpu_scheduler *sched = - container_of(job->base.entity->rq, typeof(*sched), rq); - - return to_amdgpu_ring(sched); + return to_amdgpu_ring(job->base.entity->rq->sched); } int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index 85724ec6aaf8..d13e64a69e25 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -145,7 +145,6 @@ TRACE_EVENT(amdgpu_cs, struct amdgpu_ib *ib), TP_ARGS(p, job, ib), TP_STRUCT__entry( - __field(struct drm_gpu_scheduler *, sched) __field(struct amdgpu_bo_list *, bo_list) __field(u32, ring) __field(u32, dw) @@ -153,14 +152,11 @@ TRACE_EVENT(amdgpu_cs, ), TP_fast_assign( - __entry->sched = container_of(job->base.entity->rq, - typeof(*__entry->sched), - rq); __entry->bo_list = p->bo_list; - __entry->ring = to_amdgpu_ring(__entry->sched)->idx; + __entry->ring = to_amdgpu_ring(job->base.entity->rq->sched)->idx; __entry->dw = ib->length_dw; __entry->fences = amdgpu_fence_count_emitted( - to_amdgpu_ring(__entry->sched)); + to_amdgpu_ring(job->base.entity->rq->sched)); ), TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u", __entry->bo_list, __entry->ring, __entry->dw, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 23383ac5323f..de3dbc95e376 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -646,17 +646,15 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, unsigned int height = msg[7]; unsigned int dpb_size = msg[9]; unsigned int pitch = msg[28]; - unsigned int level = msg[57]; unsigned int width_in_mb = width / 16; unsigned int height_in_mb = ALIGN(height / 16, 2); - unsigned int fs_in_mb = width_in_mb * height_in_mb; unsigned int image_size, tmp, min_dpb_size, num_dpb_buffer; unsigned int min_ctx_size = ~0; - /* Reject invalid dimensions to prevent division by zero */ - if (width < 16 || height < 16) { + /* Reject invalid dimensions */ + if (width < 16 || height < 16 || width > 4096 || height > 4096) { dev_WARN_ONCE(adev->dev, 1, "Invalid UVD decoding dimensions (%dx%d)!\n", width, height); @@ -669,35 +667,9 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, switch (stream_type) { case 0: /* H264 */ - switch (level) { - case 30: - num_dpb_buffer = 8100 / fs_in_mb; - break; - case 31: - num_dpb_buffer = 18000 / fs_in_mb; - break; - case 32: - num_dpb_buffer = 20480 / fs_in_mb; - break; - case 41: - num_dpb_buffer = 32768 / fs_in_mb; - break; - case 42: - num_dpb_buffer = 34816 / fs_in_mb; - break; - case 50: - num_dpb_buffer = 110400 / fs_in_mb; - break; - case 51: - num_dpb_buffer = 184320 / fs_in_mb; - break; - default: - num_dpb_buffer = 184320 / fs_in_mb; - break; - } - num_dpb_buffer++; + num_dpb_buffer = ((msg[61] >> 16) & 0xff) + 1; if (num_dpb_buffer > 17) - num_dpb_buffer = 17; + return -EINVAL; /* reference picture buffer */ min_dpb_size = image_size * num_dpb_buffer; @@ -747,35 +719,9 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, break; case 7: /* H264 Perf */ - switch (level) { - case 30: - num_dpb_buffer = 8100 / fs_in_mb; - break; - case 31: - num_dpb_buffer = 18000 / fs_in_mb; - break; - case 32: - num_dpb_buffer = 20480 / fs_in_mb; - break; - case 41: - num_dpb_buffer = 32768 / fs_in_mb; - break; - case 42: - num_dpb_buffer = 34816 / fs_in_mb; - break; - case 50: - num_dpb_buffer = 110400 / fs_in_mb; - break; - case 51: - num_dpb_buffer = 184320 / fs_in_mb; - break; - default: - num_dpb_buffer = 184320 / fs_in_mb; - break; - } - num_dpb_buffer++; + num_dpb_buffer = ((msg[61] >> 16) & 0xff) + 1; if (num_dpb_buffer > 17) - num_dpb_buffer = 17; + return -EINVAL; /* reference picture buffer */ min_dpb_size = image_size * num_dpb_buffer; @@ -803,6 +749,9 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, image_size = ALIGN(image_size, 256); num_dpb_buffer = (le32_to_cpu(msg[59]) & 0xff) + 2; + if (num_dpb_buffer > 17) + return -EINVAL; + min_dpb_size = image_size * num_dpb_buffer; min_ctx_size = ((width + 255) / 16) * ((height + 255) / 16) * 16 * num_dpb_buffer + 52 * 1024; @@ -813,7 +762,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, return -EINVAL; } - if (width > pitch) { + if (width > pitch || pitch > 4096) { DRM_ERROR("Invalid UVD decoding target pitch!\n"); return -EINVAL; } @@ -825,7 +774,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, } buf_sizes[0x1] = dpb_size; - buf_sizes[0x2] = image_size; + buf_sizes[0x2] = (pitch * height) * 3 / 2; buf_sizes[0x4] = min_ctx_size; /* store image width to adjust nb memory pstate */ adev->uvd.decode_image_width = width; @@ -972,15 +921,16 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx) ctx->buf_sizes[cmd]); return -EINVAL; } + } else if (cmd == 0x204 || cmd == 0x206) { + unsigned int min_size = ctx->buf_sizes[cmd == 0x204 ? 5 : 4]; - } else if (cmd == 0x206) { - if ((end - start) < ctx->buf_sizes[4]) { + if ((end - start) < min_size) { DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, (unsigned int)(end - start), - ctx->buf_sizes[4]); + min_size); return -EINVAL; } - } else if ((cmd != 0x100) && (cmd != 0x204)) { + } else if ((cmd != 0x100)) { DRM_ERROR("invalid UVD command %X!\n", cmd); return -EINVAL; } @@ -1110,11 +1060,12 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, { struct amdgpu_uvd_cs_ctx ctx = {}; unsigned int buf_sizes[] = { - [0x00000000] = 2048, + [0x00000000] = 3556, [0x00000001] = 0xFFFFFFFF, [0x00000002] = 0xFFFFFFFF, [0x00000003] = 2048, [0x00000004] = 0xFFFFFFFF, + [0x00000005] = 992, }; int r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index dc6a9d7dd0b2..1baad7624f1f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -800,6 +800,7 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, mutex_unlock(&id_mgr->lock); gds_switch_needed &= !!ring->funcs->emit_gds_switch; + spm_update_needed &= !!adev->gfx.rlc.funcs->update_spm_vmid; vm_flush_needed &= !!ring->funcs->emit_vm_flush && job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && @@ -811,7 +812,7 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, &job->base.s_fence->scheduled == isolation->spearhead; if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync && - !cleaner_shader_needed) + !cleaner_shader_needed && !spm_update_needed) return; amdgpu_ring_ib_begin(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c index fd09a2b5a147..2eb64df6daa9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c @@ -106,13 +106,13 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, struct dma_fence **fence) { - struct drm_gpu_scheduler *sched = - container_of(p->vm->delayed.rq, typeof(*sched), rq); - struct amdgpu_ring *ring = - container_of(sched, struct amdgpu_ring, sched); struct amdgpu_ib *ib = p->job->ibs; + struct amdgpu_ring *ring; struct dma_fence *f; + ring = container_of(p->vm->delayed.rq->sched, struct amdgpu_ring, + sched); + WARN_ON(ib->length_dw == 0); amdgpu_ring_pad_ib(ring, ib); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c index 409e103ffe8c..42be8ee155dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c @@ -466,15 +466,15 @@ int amdgpu_xcp_open_device(struct amdgpu_device *adev, void amdgpu_xcp_release_sched(struct amdgpu_device *adev, struct amdgpu_ctx_entity *entity) { - struct drm_gpu_scheduler *sched = - container_of(entity->entity.rq, typeof(*sched), rq); + struct drm_gpu_scheduler *sched; + struct amdgpu_ring *ring; if (!adev->xcp_mgr) return; + sched = entity->entity.rq->sched; if (drm_sched_wqueue_ready(sched)) { - struct amdgpu_ring *ring = to_amdgpu_ring(sched); - + ring = to_amdgpu_ring(entity->entity.rq->sched); atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); } } diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c index 000516b5845a..61eb0513dc97 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c @@ -500,7 +500,6 @@ static u32 nbif_v6_3_1_get_rom_offset(struct amdgpu_device *adev) static void nbif_v6_3_1_program_ltr(struct amdgpu_device *adev) { uint32_t def, data; - u16 devctl2; def = RREG32_SOC15(NBIO, 0, regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL); data = 0x35EB; @@ -514,15 +513,8 @@ static void nbif_v6_3_1_program_ltr(struct amdgpu_device *adev) if (def != data) WREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_BIF_STRAP2, data); - pcie_capability_read_word(adev->pdev, PCI_EXP_DEVCTL2, &devctl2); - - if (adev->pdev->ltr_path == (devctl2 & PCI_EXP_DEVCTL2_LTR_EN)) - return; - - if (adev->pdev->ltr_path) - pcie_capability_set_word(adev->pdev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_LTR_EN); - else - pcie_capability_clear_word(adev->pdev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_LTR_EN); + pcie_capability_set_word(adev->pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_LTR_EN); } #endif @@ -530,7 +522,7 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) { #ifdef CONFIG_PCIEASPM uint32_t def, data; - u16 devctl2, ltr; + u16 ltr; def = data = RREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL); data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK; @@ -560,11 +552,8 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_BIF_STRAP5, data); - pcie_capability_read_word(adev->pdev, PCI_EXP_DEVCTL2, &devctl2); - data = def = devctl2; - data &= ~PCI_EXP_DEVCTL2_LTR_EN; - if (def != data) - pcie_capability_set_word(adev->pdev, PCI_EXP_DEVCTL2, (u16)data); + pcie_capability_clear_word(adev->pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_LTR_EN); ltr = pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_LTR); @@ -572,15 +561,13 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) pci_write_config_dword(adev->pdev, ltr + PCI_LTR_MAX_SNOOP_LAT, 0x10011001); } -#if 0 - /* regPSWUSP0_PCIE_LC_CNTL2 should be replace by PCIE_LC_CNTL2 or someone else ? */ - def = data = RREG32_SOC15(NBIO, 0, regPSWUSP0_PCIE_LC_CNTL2); - data |= PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | - PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; - data &= ~PSWUSP0_PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK; + def = data = RREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL2); + data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | + PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; + data &= ~PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK; if (def != data) - WREG32_SOC15(NBIO, 0, regPSWUSP0_PCIE_LC_CNTL2, data); -#endif + WREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL2, data); + def = data = RREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL4); data |= PCIE_LC_CNTL4__LC_L1_POWERDOWN_MASK; if (def != data) @@ -591,7 +578,12 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_SOC15(PCIE, 0, regPCIE_LC_RXRECOVER_RXSTANDBY_CNTL, data); - nbif_v6_3_1_program_ltr(adev); + /* + * Do not enable endpoint LTR unless the Root Complex and every + * upstream switch support it. + */ + if (adev->pdev->ltr_path) + nbif_v6_3_1_program_ltr(adev); def = data = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_BIF_STRAP3); data |= 0x5DE0 << RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index c69f7d82060f..a3eae7d4b57e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -875,6 +875,23 @@ static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, ib->length_dw); } +static void vce_v3_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, + u64 seq, unsigned flags) +{ + WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT); + + amdgpu_ring_write(ring, VCE_CMD_FENCE); + amdgpu_ring_write(ring, addr); + amdgpu_ring_write(ring, upper_32_bits(addr)); + amdgpu_ring_write(ring, seq); + amdgpu_ring_write(ring, VCE_CMD_TRAP); +} + +static void vce_v3_0_ring_insert_end(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, VCE_CMD_END); +} + static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring, unsigned int vmid, uint64_t pd_addr) { @@ -884,7 +901,6 @@ static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring, amdgpu_ring_write(ring, VCE_CMD_FLUSH_TLB); amdgpu_ring_write(ring, vmid); - amdgpu_ring_write(ring, VCE_CMD_END); } static void vce_v3_0_emit_pipeline_sync(struct amdgpu_ring *ring) @@ -953,17 +969,19 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = { .set_wptr = vce_v3_0_ring_set_wptr, .patch_cs_in_place = amdgpu_vce_ring_parse_cs_vm, .emit_frame_size = - 6 + /* vce_v3_0_emit_vm_flush */ + 5 + /* vce_v3_0_emit_vm_flush */ 4 + /* vce_v3_0_emit_pipeline_sync */ - 6 + 6, /* amdgpu_vce_ring_emit_fence x2 vm fence */ + 5 + 5 + /* vce_v3_0_ring_emit_fence x2 vm fence */ + 1, /* vce_v3_0_ring_insert_end */ .emit_ib_size = 5, /* vce_v3_0_ring_emit_ib */ .emit_ib = vce_v3_0_ring_emit_ib, .emit_vm_flush = vce_v3_0_emit_vm_flush, .emit_pipeline_sync = vce_v3_0_emit_pipeline_sync, - .emit_fence = amdgpu_vce_ring_emit_fence, + .emit_fence = vce_v3_0_ring_emit_fence, .test_ring = amdgpu_vce_ring_test_ring, .test_ib = amdgpu_vce_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .insert_end = vce_v3_0_ring_insert_end, .pad_ib = amdgpu_ring_generic_pad_ib, .begin_use = amdgpu_vce_ring_begin_use, .end_use = amdgpu_vce_ring_end_use, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 06598273d481..56bf907f1f6c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -257,7 +257,7 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable) irq_type = amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id); - if (enable) { + if (enable && acrtc_state->stream) { struct dc *dc = adev->dm.dc; struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); struct psr_settings *psr = &acrtc_state->stream->link->psr_settings; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c index 88446817a71f..7f362c3926c6 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c @@ -58,7 +58,8 @@ enum dc_color_space_type { COLOR_SPACE_RGB_LIMITED_TYPE, COLOR_SPACE_YCBCR601_TYPE, COLOR_SPACE_YCBCR709_TYPE, - COLOR_SPACE_YCBCR2020_TYPE, + COLOR_SPACE_YCBCR2020_LIMITED_TYPE, + COLOR_SPACE_YCBCR2020_FULL_TYPE, COLOR_SPACE_YCBCR601_LIMITED_TYPE, COLOR_SPACE_YCBCR709_LIMITED_TYPE, COLOR_SPACE_YCBCR709_BLACK_TYPE, @@ -110,9 +111,15 @@ static const struct out_csc_color_matrix_type output_csc_matrix[] = { { 0xE00, 0xF349, 0xFEB7, 0x1000, 0x6CE, 0x16E3, 0x24F, 0x200, 0xFCCB, 0xF535, 0xE00, 0x1000} }, - { COLOR_SPACE_YCBCR2020_TYPE, + /* Corrected. Not included in the TODO above. */ + { COLOR_SPACE_YCBCR2020_LIMITED_TYPE, + { 0x0E04, 0xF31D, 0xFEDF, 0x1004, + 0x0733, 0x1294, 0x01A0, 0x0201, + 0xFC16, 0xF5E6, 0x0E04, 0x1004} }, + /* Corrected. Not included in the TODO above. */ + { COLOR_SPACE_YCBCR2020_FULL_TYPE, { 0x1000, 0xF149, 0xFEB7, 0x1004, - 0x0868, 0x15B2, 0x01E6, 0x201, + 0x0868, 0x15B2, 0x01E6, 0, 0xFB88, 0xF478, 0x1000, 0x1004} }, { COLOR_SPACE_YCBCR709_BLACK_TYPE, { 0x0000, 0x0000, 0x0000, 0x1000, @@ -179,14 +186,14 @@ static bool is_ycbcr709_type( return ret; } -static bool is_ycbcr2020_type( - enum dc_color_space color_space) +static bool is_ycbcr2020_limited_type(enum dc_color_space color_space) { - bool ret = false; + return color_space == COLOR_SPACE_2020_YCBCR_LIMITED; +} - if (color_space == COLOR_SPACE_2020_YCBCR_LIMITED || color_space == COLOR_SPACE_2020_YCBCR_FULL) - ret = true; - return ret; +static bool is_ycbcr2020_full_type(enum dc_color_space color_space) +{ + return color_space == COLOR_SPACE_2020_YCBCR_FULL; } static bool is_ycbcr709_limited_type( @@ -215,8 +222,10 @@ static enum dc_color_space_type get_color_space_type(enum dc_color_space color_s type = COLOR_SPACE_YCBCR601_LIMITED_TYPE; else if (is_ycbcr709_limited_type(color_space)) type = COLOR_SPACE_YCBCR709_LIMITED_TYPE; - else if (is_ycbcr2020_type(color_space)) - type = COLOR_SPACE_YCBCR2020_TYPE; + else if (is_ycbcr2020_limited_type(color_space)) + type = COLOR_SPACE_YCBCR2020_LIMITED_TYPE; + else if (is_ycbcr2020_full_type(color_space)) + type = COLOR_SPACE_YCBCR2020_FULL_TYPE; else if (color_space == COLOR_SPACE_YCBCR709) type = COLOR_SPACE_YCBCR709_BLACK_TYPE; else if (color_space == COLOR_SPACE_YCBCR709_BLACK) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c index 12c85c3afd6a..a51c9b282055 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c @@ -115,10 +115,11 @@ static const struct out_csc_color_matrix global_color_matrix[] = { { 0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} }, { COLOR_SPACE_2020_RGB_LIMITEDRANGE, { 0x1B67, 0, 0, 0x201, 0, 0x1B67, 0, 0x201, 0, 0, 0x1B67, 0x201} }, -{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, - 0x15B2, 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} }, +/* COLOR_SPACE_2020_YCBCR_* values corrected. Not included in the TODO above. */ +{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x0E04, 0xF31D, 0xFEDF, 0x1004, 0x0733, + 0x1294, 0x01A0, 0x201, 0xFC16, 0xF5E6, 0x0E04, 0x1004} }, { COLOR_SPACE_2020_YCBCR_FULL, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, 0x15B2, - 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} } + 0x01E6, 0, 0xFB88, 0xF478, 0x1000, 0x1004} } }; static bool setup_scaling_configuration( diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c index 1ed018aaa4bb..f5f8cd2d47a5 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c @@ -93,10 +93,11 @@ static const struct out_csc_color_matrix global_color_matrix[] = { { 0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} }, { COLOR_SPACE_2020_RGB_LIMITEDRANGE, { 0x1B67, 0, 0, 0x201, 0, 0x1B67, 0, 0x201, 0, 0, 0x1B67, 0x201} }, -{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, - 0x15B2, 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} }, +/* COLOR_SPACE_2020_YCBCR_* values corrected. Not included in the TODO above. */ +{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x0E04, 0xF31D, 0xFEDF, 0x1004, 0x0733, + 0x1294, 0x01A0, 0x201, 0xFC16, 0xF5E6, 0x0E04, 0x1004} }, { COLOR_SPACE_2020_YCBCR_FULL, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, 0x15B2, - 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} } + 0x01E6, 0, 0xFB88, 0xF478, 0x1000, 0x1004} } }; enum csc_color_mode { diff --git a/drivers/gpu/drm/clients/drm_log.c b/drivers/gpu/drm/clients/drm_log.c index 8d21b785bead..467cdce57dd3 100644 --- a/drivers/gpu/drm/clients/drm_log.c +++ b/drivers/gpu/drm/clients/drm_log.c @@ -160,6 +160,9 @@ static void drm_log_draw_kmsg_record(struct drm_log_scanout *scanout, { u32 prefix_len = 0; + if (!len) + return; + if (len > TS_PREFIX_LEN && s[0] == '[' && s[6] == '.' && s[TS_PREFIX_LEN] == ']') prefix_len = TS_PREFIX_LEN + 1; @@ -215,6 +218,12 @@ static int drm_log_setup_modeset(struct drm_client_dev *client, scanout->scaled_font_w = scanout->font->width * scale; scanout->rows = height / scanout->scaled_font_h; scanout->columns = width / scanout->scaled_font_w; + if (!scanout->rows || !scanout->columns) { + drm_client_buffer_delete(scanout->buffer); + scanout->buffer = NULL; + mode_set->fb = NULL; + return -EINVAL; + } scanout->front_color = drm_draw_color_from_xrgb8888(0xffffff, format); scanout->prefix_color = drm_draw_color_from_xrgb8888(0x4e9a06, format); return 0; @@ -419,6 +428,9 @@ void drm_log_register(struct drm_device *dev) { struct drm_log *new; + if (!scale) + scale = 1; + new = kzalloc_obj(*new); if (!new) goto err_warn; diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c index 4867edbf2622..cae0d85fb440 100644 --- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c +++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c @@ -1096,7 +1096,8 @@ drm_atomic_helper_connector_hdmi_update_audio_infoframe(struct drm_connector *co mutex_lock(&connector->hdmi.infoframes.lock); - memcpy(&infoframe->data, frame, sizeof(infoframe->data)); + BUILD_BUG_ON(sizeof(*frame) > sizeof(infoframe->data)); + memcpy(&infoframe->data, frame, sizeof(*frame)); infoframe->set = true; ret = write_infoframe(connector, &funcs->audio, "Audio", infoframe); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 139e6e38784b..3cc50d697c89 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -144,6 +144,7 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu) { const struct drm_sched_init_args args = { .ops = &etnaviv_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = etnaviv_hw_jobs_limit, .hang_limit = etnaviv_job_hang_limit, .timeout = msecs_to_jiffies(500), diff --git a/drivers/gpu/drm/imagination/pvr_queue.c b/drivers/gpu/drm/imagination/pvr_queue.c index d13726401d92..2d52ee321367 100644 --- a/drivers/gpu/drm/imagination/pvr_queue.c +++ b/drivers/gpu/drm/imagination/pvr_queue.c @@ -1285,6 +1285,7 @@ struct pvr_queue *pvr_queue_create(struct pvr_context *ctx, const struct drm_sched_init_args sched_args = { .ops = &pvr_queue_sched_ops, .submit_wq = pvr_dev->sched_wq, + .num_rqs = 1, .credit_limit = 64 * 1024, .hang_limit = 1, .timeout = msecs_to_jiffies(500), diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c index 0a01213c4878..9a1e6b9ecbe5 100644 --- a/drivers/gpu/drm/lima/lima_sched.c +++ b/drivers/gpu/drm/lima/lima_sched.c @@ -521,6 +521,7 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name) lima_sched_timeout_ms : 10000; const struct drm_sched_init_args args = { .ops = &lima_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 1, .hang_limit = lima_job_hang_limit, .timeout = msecs_to_jiffies(timeout), diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index 3ed05ab0eeef..c4cfe036066b 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -841,6 +841,7 @@ msm_gem_vm_create(struct drm_device *drm, struct msm_mmu *mmu, const char *name, if (!managed) { struct drm_sched_init_args args = { .ops = &msm_vm_bind_ops, + .num_rqs = 1, .credit_limit = 1, .timeout = MAX_SCHEDULE_TIMEOUT, .name = "msm-vm-bind", diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index 0d14c31bd4e4..2d6b930b766e 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -67,6 +67,7 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, { struct drm_sched_init_args args = { .ops = &msm_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = num_hw_submissions, .timeout = MAX_SCHEDULE_TIMEOUT, .dev = gpu->dev->dev, diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.c b/drivers/gpu/drm/nouveau/nouveau_sched.c index 2cbae003d6de..8b9f935afe09 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sched.c +++ b/drivers/gpu/drm/nouveau/nouveau_sched.c @@ -405,6 +405,7 @@ nouveau_sched_init(struct nouveau_sched *sched, struct nouveau_drm *drm, struct drm_sched_entity *entity = &sched->entity; struct drm_sched_init_args args = { .ops = &nouveau_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = credit_limit, .timeout = msecs_to_jiffies(NOUVEAU_SCHED_JOB_TIMEOUT_MS), .name = "nouveau_sched", diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 2d12b83e900a..d59b4863b8ad 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -850,6 +850,7 @@ int panfrost_jm_init(struct panfrost_device *pfdev) { struct drm_sched_init_args args = { .ops = &panfrost_sched_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 2, .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), .dev = pfdev->base.dev, diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index e592a8ebb478..e10dbd18d8cf 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -2732,6 +2732,7 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu, const struct drm_sched_init_args sched_args = { .ops = &panthor_vm_bind_ops, .submit_wq = ptdev->mmu->vm.wq, + .num_rqs = 1, .credit_limit = 1, /* Bind operations are synchronous for now, no timeout needed. */ .timeout = MAX_SCHEDULE_TIMEOUT, diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index 298b046c95ed..369ee06434bb 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -3502,6 +3502,7 @@ group_create_queue(struct panthor_group *group, struct drm_sched_init_args sched_args = { .ops = &panthor_queue_sched_ops, .submit_wq = group->ptdev->scheduler->wq, + .num_rqs = 1, /* * The credit limit argument tells us the total number of * instructions across all CS slots in the ringbuffer, with diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c index dc43fd790a9c..4a99c09f4164 100644 --- a/drivers/gpu/drm/radeon/radeon_kms.c +++ b/drivers/gpu/drm/radeon/radeon_kms.c @@ -71,6 +71,7 @@ void radeon_driver_unload_kms(struct drm_device *dev) if (radeon_is_px(dev)) { pm_runtime_get_sync(dev->dev); pm_runtime_forbid(dev->dev); + pm_runtime_dont_use_autosuspend(dev->dev); } radeon_acpi_fini(rdev); diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 4ebb513255ed..672b5c57ed8e 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -133,11 +133,39 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, entity->guilty = guilty; entity->priority = priority; entity->last_user = current->group_leader; + entity->rq_priority = drm_sched_policy == DRM_SCHED_POLICY_FAIR ? + DRM_SCHED_PRIORITY_KERNEL : priority; entity->num_sched_list = num_sched_list; entity->sched_list = num_sched_list > 1 ? sched_list : NULL; - entity->rq = &sched_list[0]->rq; RCU_INIT_POINTER(entity->last_scheduled, NULL); RB_CLEAR_NODE(&entity->rb_tree_node); + + if (!sched_list[0]->sched_rq) { + /* Since every entry covered by num_sched_list + * should be non-NULL and therefore we warn drivers + * not to do this and to fix their DRM calling order. + */ + pr_warn("%s: called with uninitialized scheduler\n", __func__); + } else { + enum drm_sched_priority p = entity->priority; + + /* + * The "priority" of an entity cannot exceed the number of + * run-queues of a scheduler. Protect against num_rqs being 0, + * by converting to signed. Choose the lowest priority + * available. + */ + if (p >= sched_list[0]->num_user_rqs) { + dev_err(sched_list[0]->dev, "entity with out-of-bounds priority:%u num_user_rqs:%u\n", + p, sched_list[0]->num_user_rqs); + p = max_t(s32, + (s32)sched_list[0]->num_user_rqs - 1, + (s32)DRM_SCHED_PRIORITY_KERNEL); + entity->priority = p; + } + entity->rq = sched_list[0]->sched_rq[entity->rq_priority]; + } + init_completion(&entity->entity_idle); /* We start in an idle state. */ @@ -336,7 +364,7 @@ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) if (!entity->rq) return 0; - sched = container_of(entity->rq, typeof(*sched), rq); + sched = entity->rq->sched; /* * The client will not queue more jobs during this fini - consume * existing queued ones, or discard them on SIGKILL. @@ -417,12 +445,10 @@ static void drm_sched_entity_wakeup(struct dma_fence *f, { struct drm_sched_entity *entity = container_of(cb, struct drm_sched_entity, cb); - struct drm_gpu_scheduler *sched = - container_of(entity->rq, typeof(*sched), rq); entity->dependency = NULL; dma_fence_put(f); - drm_sched_wakeup(sched); + drm_sched_wakeup(entity->rq->sched); } /** @@ -449,8 +475,7 @@ EXPORT_SYMBOL(drm_sched_entity_set_priority); static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity, struct drm_sched_job *sched_job) { - struct drm_gpu_scheduler *sched = - container_of(entity->rq, typeof(*sched), rq); + struct drm_gpu_scheduler *sched = entity->rq->sched; struct dma_fence *fence = entity->dependency; struct drm_sched_fence *s_fence; @@ -584,7 +609,7 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) spin_lock(&entity->lock); sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list); - rq = sched ? &sched->rq : NULL; + rq = sched ? sched->sched_rq[entity->rq_priority] : NULL; if (rq != entity->rq) { drm_sched_rq_remove_entity(entity->rq, entity); entity->rq = rq; @@ -608,9 +633,8 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) void drm_sched_entity_push_job(struct drm_sched_job *sched_job) { struct drm_sched_entity *entity = sched_job->entity; - struct drm_gpu_scheduler *sched = - container_of(entity->rq, typeof(*sched), rq); bool first; + ktime_t submit_ts; trace_drm_sched_job_queue(sched_job, entity); @@ -621,18 +645,22 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) xa_for_each(&sched_job->dependencies, index, entry) trace_drm_sched_job_add_dep(sched_job, entry); } - atomic_inc(sched->score); + atomic_inc(entity->rq->sched->score); WRITE_ONCE(entity->last_user, current->group_leader); /* * After the sched_job is pushed into the entity queue, it may be * completed and freed up at any time. We can no longer access it. + * Make sure to set the submit_ts first, to avoid a race. */ + sched_job->submit_ts = submit_ts = ktime_get(); first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); /* first job wakes up scheduler */ if (first) { - sched = drm_sched_rq_add_entity(entity); + struct drm_gpu_scheduler *sched; + + sched = drm_sched_rq_add_entity(entity, submit_ts); if (sched) drm_sched_wakeup(sched); } diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c index 85ee3d694dc9..096fe28aa9c9 100644 --- a/drivers/gpu/drm/scheduler/sched_fence.c +++ b/drivers/gpu/drm/scheduler/sched_fence.c @@ -227,7 +227,7 @@ void drm_sched_fence_init(struct drm_sched_fence *fence, { unsigned seq; - fence->sched = container_of(entity->rq, typeof(*fence->sched), rq); + fence->sched = entity->rq->sched; seq = atomic_inc_return(&entity->fence_seq); dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled, &fence->lock, entity->fence_context, seq); diff --git a/drivers/gpu/drm/scheduler/sched_internal.h b/drivers/gpu/drm/scheduler/sched_internal.h index 13ecb771d7a2..a901801fce85 100644 --- a/drivers/gpu/drm/scheduler/sched_internal.h +++ b/drivers/gpu/drm/scheduler/sched_internal.h @@ -31,20 +31,29 @@ struct drm_sched_entity_stats { struct ewma_drm_sched_avgtime avg_job_us; }; +/* Used to choose between FIFO and RR job-scheduling */ +extern int drm_sched_policy; + +#define DRM_SCHED_POLICY_RR 0 +#define DRM_SCHED_POLICY_FIFO 1 +#define DRM_SCHED_POLICY_FAIR 2 + bool drm_sched_can_queue(struct drm_gpu_scheduler *sched, struct drm_sched_entity *entity); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_rq_init(struct drm_sched_rq *rq); +void drm_sched_rq_init(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq); struct drm_gpu_scheduler * -drm_sched_rq_add_entity(struct drm_sched_entity *entity); +drm_sched_rq_add_entity(struct drm_sched_entity *entity, ktime_t ts); void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, struct drm_sched_entity *entity); void drm_sched_rq_pop_entity(struct drm_sched_entity *entity); struct drm_sched_entity * -drm_sched_select_entity(struct drm_gpu_scheduler *sched); +drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq); void drm_sched_entity_select_rq(struct drm_sched_entity *entity); struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity); diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 818d3d4434b5..214ed02cb496 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -84,6 +84,15 @@ #define CREATE_TRACE_POINTS #include "gpu_scheduler_trace.h" +int drm_sched_policy = DRM_SCHED_POLICY_FIFO; + +/** + * DOC: sched_policy (int) + * Used to override default entities scheduling policy in a run queue. + */ +MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default), " __stringify(DRM_SCHED_POLICY_FAIR) " = Fair (experimental)."); +module_param_named(sched_policy, drm_sched_policy, int, 0444); + static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched) { u32 credits; @@ -648,7 +657,7 @@ void drm_sched_job_arm(struct drm_sched_job *job) BUG_ON(!entity); drm_sched_entity_select_rq(entity); - sched = container_of(entity->rq, typeof(*sched), rq); + sched = entity->rq->sched; job->sched = sched; job->s_priority = entity->priority; @@ -872,6 +881,34 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched) } /** + * drm_sched_select_entity - Select next entity to process + * + * @sched: scheduler instance + * + * Return an entity to process or NULL if none are found. + * + * Note, that we break out of the for-loop when "entity" is non-null, which can + * also be an error-pointer--this assures we don't process lower priority + * run-queues. See comments in the respectively called functions. + */ +static struct drm_sched_entity * +drm_sched_select_entity(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_entity *entity = NULL; + int i; + + /* Start with the highest priority. + */ + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + entity = drm_sched_rq_select_entity(sched, sched->sched_rq[i]); + if (entity) + break; + } + + return IS_ERR(entity) ? NULL : entity; +} + +/** * drm_sched_get_finished_job - fetch the next finished job to be destroyed * * @sched: scheduler instance @@ -996,7 +1033,7 @@ static void drm_sched_run_job_work(struct work_struct *w) /* Find entity with a ready job */ entity = drm_sched_select_entity(sched); - if (IS_ERR_OR_NULL(entity)) { + if (!entity) { /* * Either no more work to do, or the next ready job needs more * credits than the scheduler has currently available. @@ -1072,6 +1109,8 @@ static struct workqueue_struct *drm_sched_alloc_wq(const char *name) */ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_args *args) { + int i; + sched->ops = args->ops; sched->credit_limit = args->credit_limit; sched->name = args->name; @@ -1081,6 +1120,21 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_ sched->score = args->score ? args->score : &sched->_score; sched->dev = args->dev; + if (args->num_rqs > DRM_SCHED_PRIORITY_COUNT) { + /* This is a gross violation--tell drivers what the problem is. + */ + dev_err(sched->dev, "%s: num_rqs cannot be greater than DRM_SCHED_PRIORITY_COUNT\n", + __func__); + return -EINVAL; + } else if (sched->sched_rq) { + /* Not an error, but warn anyway so drivers can + * fine-tune their DRM calling order, and return all + * is good. + */ + dev_warn(sched->dev, "%s: scheduler already initialized!\n", __func__); + return 0; + } + if (args->submit_wq) { sched->submit_wq = args->submit_wq; sched->own_submit_wq = false; @@ -1092,7 +1146,19 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_ sched->own_submit_wq = true; } - drm_sched_rq_init(&sched->rq); + sched->num_user_rqs = args->num_rqs; + sched->num_rqs = drm_sched_policy != DRM_SCHED_POLICY_FAIR ? + args->num_rqs : 1; + sched->sched_rq = kzalloc_objs(*sched->sched_rq, args->num_rqs); + if (!sched->sched_rq) + goto Out_check_own; + + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + sched->sched_rq[i] = kzalloc_obj(*sched->sched_rq[i]); + if (!sched->sched_rq[i]) + goto Out_unroll; + drm_sched_rq_init(sched, sched->sched_rq[i]); + } init_waitqueue_head(&sched->job_scheduled); INIT_LIST_HEAD(&sched->pending_list); @@ -1108,6 +1174,17 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_ sched->ready = true; return 0; +Out_unroll: + for (--i ; i >= DRM_SCHED_PRIORITY_KERNEL; i--) + kfree(sched->sched_rq[i]); + + kfree(sched->sched_rq); + sched->sched_rq = NULL; +Out_check_own: + if (sched->own_submit_wq) + destroy_workqueue(sched->submit_wq); + dev_err(sched->dev, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); + return -ENOMEM; } EXPORT_SYMBOL(drm_sched_init); @@ -1138,8 +1215,13 @@ static void drm_sched_cancel_remaining_jobs(struct drm_gpu_scheduler *sched) */ void drm_sched_fini(struct drm_gpu_scheduler *sched) { + int i; + drm_sched_wqueue_stop(sched); + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) + kfree(sched->sched_rq[i]); + /* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */ wake_up_all(&sched->job_scheduled); @@ -1153,6 +1235,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) if (sched->own_submit_wq) destroy_workqueue(sched->submit_wq); sched->ready = false; + kfree(sched->sched_rq); + sched->sched_rq = NULL; if (!list_empty(&sched->pending_list)) dev_warn(sched->dev, "Tearing down scheduler while jobs are pending!\n"); @@ -1170,28 +1254,35 @@ EXPORT_SYMBOL(drm_sched_fini); */ void drm_sched_increase_karma(struct drm_sched_job *bad) { + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; struct drm_gpu_scheduler *sched = bad->sched; - struct drm_sched_entity *entity, *tmp; - struct drm_sched_rq *rq = &sched->rq; /* don't change @bad's karma if it's from KERNEL RQ, * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) * corrupt but keep in mind that kernel jobs always considered good. */ - if (bad->s_priority == DRM_SCHED_PRIORITY_KERNEL) - return; - - atomic_inc(&bad->karma); - - spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; } } - spin_unlock(&rq->lock); } EXPORT_SYMBOL(drm_sched_increase_karma); diff --git a/drivers/gpu/drm/scheduler/sched_rq.c b/drivers/gpu/drm/scheduler/sched_rq.c index 044546bcb5f8..0464d324d98d 100644 --- a/drivers/gpu/drm/scheduler/sched_rq.c +++ b/drivers/gpu/drm/scheduler/sched_rq.c @@ -49,7 +49,7 @@ static void drm_sched_rq_update_prio(struct drm_sched_rq *rq) rq->head_prio = prio; } -static void drm_sched_rq_remove_tree_locked(struct drm_sched_entity *entity, +static void drm_sched_rq_remove_fifo_locked(struct drm_sched_entity *entity, struct drm_sched_rq *rq) { lockdep_assert_held(&entity->lock); @@ -62,7 +62,7 @@ static void drm_sched_rq_remove_tree_locked(struct drm_sched_entity *entity, } } -static void drm_sched_rq_update_tree_locked(struct drm_sched_entity *entity, +static void drm_sched_rq_update_fifo_locked(struct drm_sched_entity *entity, struct drm_sched_rq *rq, ktime_t ts) { @@ -74,7 +74,7 @@ static void drm_sched_rq_update_tree_locked(struct drm_sched_entity *entity, lockdep_assert_held(&entity->lock); lockdep_assert_held(&rq->lock); - drm_sched_rq_remove_tree_locked(entity, rq); + drm_sched_rq_remove_fifo_locked(entity, rq); entity->oldest_job_waiting = ts; @@ -85,15 +85,18 @@ static void drm_sched_rq_update_tree_locked(struct drm_sched_entity *entity, /** * drm_sched_rq_init - initialize a given run queue struct + * @sched: scheduler instance to associate with this run queue * @rq: scheduler run queue * * Initializes a scheduler runqueue. */ -void drm_sched_rq_init(struct drm_sched_rq *rq) +void drm_sched_rq_init(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq) { spin_lock_init(&rq->lock); INIT_LIST_HEAD(&rq->entities); rq->rb_tree_root = RB_ROOT_CACHED; + rq->sched = sched; rq->head_prio = DRM_SCHED_PRIORITY_INVALID; } @@ -162,8 +165,7 @@ drm_sched_entity_restore_vruntime(struct drm_sched_entity *entity, enum drm_sched_priority rq_prio) { struct drm_sched_entity_stats *stats = entity->stats; - struct drm_gpu_scheduler *sched = - container_of(entity->rq, typeof(*sched), rq); + struct drm_gpu_scheduler *sched = entity->rq->sched; enum drm_sched_priority prio = entity->priority; unsigned long avg_us, sched_avg_us; ktime_t vruntime; @@ -237,9 +239,15 @@ static ktime_t drm_sched_entity_update_vruntime(struct drm_sched_entity *entity) return runtime; } +static ktime_t drm_sched_entity_get_job_ts(struct drm_sched_entity *entity) +{ + return drm_sched_entity_update_vruntime(entity); +} + /** * drm_sched_rq_add_entity - add an entity * @entity: scheduler entity + * @ts: submission timestamp * * Adds a scheduler entity to the run queue. * @@ -247,11 +255,10 @@ static ktime_t drm_sched_entity_update_vruntime(struct drm_sched_entity *entity) * been stopped and cannot be submitted to. */ struct drm_gpu_scheduler * -drm_sched_rq_add_entity(struct drm_sched_entity *entity) +drm_sched_rq_add_entity(struct drm_sched_entity *entity, ktime_t ts) { struct drm_gpu_scheduler *sched; struct drm_sched_rq *rq; - ktime_t ts; /* Add the entity to the run queue */ spin_lock(&entity->lock); @@ -263,17 +270,23 @@ drm_sched_rq_add_entity(struct drm_sched_entity *entity) } rq = entity->rq; - sched = container_of(rq, typeof(*sched), rq); spin_lock(&rq->lock); + sched = rq->sched; if (list_empty(&entity->list)) { atomic_inc(sched->score); list_add_tail(&entity->list, &rq->entities); } - ts = drm_sched_rq_get_min_vruntime(rq); - ts = drm_sched_entity_restore_vruntime(entity, ts, rq->head_prio); - drm_sched_rq_update_tree_locked(entity, rq, ts); + if (drm_sched_policy == DRM_SCHED_POLICY_FAIR) { + ts = drm_sched_rq_get_min_vruntime(rq); + ts = drm_sched_entity_restore_vruntime(entity, ts, + rq->head_prio); + } else if (drm_sched_policy == DRM_SCHED_POLICY_RR) { + ts = entity->rr_ts; + } + + drm_sched_rq_update_fifo_locked(entity, rq, ts); spin_unlock(&rq->lock); spin_unlock(&entity->lock); @@ -291,8 +304,6 @@ drm_sched_rq_add_entity(struct drm_sched_entity *entity) void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, struct drm_sched_entity *entity) { - struct drm_gpu_scheduler *sched = container_of(rq, typeof(*sched), rq); - lockdep_assert_held(&entity->lock); if (list_empty(&entity->list)) @@ -300,14 +311,30 @@ void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, spin_lock(&rq->lock); - atomic_dec(sched->score); + atomic_dec(rq->sched->score); list_del_init(&entity->list); - drm_sched_rq_remove_tree_locked(entity, rq); + drm_sched_rq_remove_fifo_locked(entity, rq); spin_unlock(&rq->lock); } +static ktime_t +drm_sched_rq_next_rr_ts(struct drm_sched_rq *rq, + struct drm_sched_entity *entity) +{ + ktime_t ts; + + lockdep_assert_held(&entity->lock); + lockdep_assert_held(&rq->lock); + + ts = ktime_add_ns(rq->rr_ts, 1); + entity->rr_ts = ts; + rq->rr_ts = ts; + + return ts; +} + /** * drm_sched_rq_pop_entity - pops an entity * @entity: scheduler entity @@ -330,22 +357,32 @@ void drm_sched_rq_pop_entity(struct drm_sched_entity *entity) if (next_job) { ktime_t ts; - ts = drm_sched_entity_update_vruntime(entity); - drm_sched_rq_update_tree_locked(entity, rq, ts); + if (drm_sched_policy == DRM_SCHED_POLICY_FAIR) + ts = drm_sched_entity_get_job_ts(entity); + else if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) + ts = next_job->submit_ts; + else + ts = drm_sched_rq_next_rr_ts(rq, entity); + + drm_sched_rq_update_fifo_locked(entity, rq, ts); } else { - ktime_t min_vruntime; + drm_sched_rq_remove_fifo_locked(entity, rq); - drm_sched_rq_remove_tree_locked(entity, rq); - min_vruntime = drm_sched_rq_get_min_vruntime(rq); - drm_sched_entity_save_vruntime(entity, min_vruntime); + if (drm_sched_policy == DRM_SCHED_POLICY_FAIR) { + ktime_t min_vruntime; + + min_vruntime = drm_sched_rq_get_min_vruntime(rq); + drm_sched_entity_save_vruntime(entity, min_vruntime); + } } spin_unlock(&rq->lock); spin_unlock(&entity->lock); } /** - * drm_sched_select_entity - Select an entity which provides a job to run + * drm_sched_rq_select_entity - Select an entity which provides a job to run * @sched: the gpu scheduler + * @rq: scheduler run queue to check. * * Find oldest waiting ready entity. * @@ -354,9 +391,9 @@ void drm_sched_rq_pop_entity(struct drm_sched_entity *entity) * its job; return NULL, if no ready entity was found. */ struct drm_sched_entity * -drm_sched_select_entity(struct drm_gpu_scheduler *sched) +drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched, + struct drm_sched_rq *rq) { - struct drm_sched_rq *rq = &sched->rq; struct rb_node *rb; spin_lock(&rq->lock); diff --git a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c index 14403a762335..8e9ae7d980eb 100644 --- a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c +++ b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c @@ -290,6 +290,7 @@ struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout) { struct drm_sched_init_args args = { .ops = &drm_mock_scheduler_ops, + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = U32_MAX, .hang_limit = 1, .timeout = timeout, diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 63dd95b828c8..bea46298b69e 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -835,6 +835,7 @@ v3d_queue_sched_init(struct v3d_dev *v3d, const struct drm_sched_backend_ops *op enum v3d_queue queue, const char *name) { struct drm_sched_init_args args = { + .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 1, .timeout = msecs_to_jiffies(500), .timeout_wq = v3d->reset_wq, diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 5f4a0cd8deca..73469ea5f333 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -164,31 +164,14 @@ static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, XE_PAGE_SIZE); - if (IS_DGFX(xe)) - dpt = xe_bo_create_pin_map_at_novm(xe, tile0, - dpt_size, ~0ull, - ttm_bo_type_kernel, - XE_BO_FLAG_VRAM0 | - XE_BO_FLAG_GGTT | - XE_BO_FLAG_PAGETABLE, - pin_params->alignment, false); - else - dpt = xe_bo_create_pin_map_at_novm(xe, tile0, - dpt_size, ~0ull, - ttm_bo_type_kernel, - XE_BO_FLAG_STOLEN | - XE_BO_FLAG_GGTT | - XE_BO_FLAG_PAGETABLE, - pin_params->alignment, false); - if (IS_ERR(dpt)) - dpt = xe_bo_create_pin_map_at_novm(xe, tile0, - dpt_size, ~0ull, - ttm_bo_type_kernel, - XE_BO_FLAG_SYSTEM | - XE_BO_FLAG_GGTT | - XE_BO_FLAG_PAGETABLE | - XE_BO_FLAG_FORCE_WC, - pin_params->alignment, false); + dpt = xe_bo_create_pin_map_at_novm(xe, tile0, + dpt_size, ~0ull, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile0) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_PAGETABLE | + XE_BO_FLAG_FORCE_WC, + pin_params->alignment, false); if (IS_ERR(dpt)) return PTR_ERR(dpt); diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 7ae1d9ac0574..57039cf42ea7 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -6,6 +6,7 @@ #ifndef _XE_BO_H_ #define _XE_BO_H_ +#include <drm/drm_prime.h> #include <drm/ttm/ttm_tt.h> #include "xe_bo_types.h" @@ -548,6 +549,19 @@ void xe_bo_dev_fini(struct xe_bo_dev *bo_device); struct sg_table *xe_bo_sg(struct xe_bo *bo); +/** + * xe_bo_sg_is_contiguous() - Check if a BO's DMA address space is contiguous. + * @bo: the BO to check (must have a valid sg table, i.e. !xe_bo_is_vram()) + * @len: required contiguous length in bytes + * + * Returns true if the first @len bytes of the BO are mapped to a contiguous + * DMA address range. + */ +static inline bool xe_bo_sg_is_contiguous(struct xe_bo *bo, size_t len) +{ + return drm_prime_get_contiguous_size(xe_bo_sg(bo)) >= len; +} + /* * xe_sg_segment_size() - Provides upper limit for sg segment size. * @dev: device pointer diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c index 004aac8b89e6..51d99fee9aa5 100644 --- a/drivers/gpu/drm/xe/xe_dep_scheduler.c +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c @@ -78,6 +78,7 @@ xe_dep_scheduler_create(struct xe_device *xe, const struct drm_sched_init_args args = { .ops = &sched_ops, .submit_wq = submit_wq, + .num_rqs = 1, .credit_limit = job_limit, .timeout = MAX_SCHEDULE_TIMEOUT, .name = name, diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index dcb48caa485d..a2b076b1dc60 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1102,7 +1102,11 @@ int xe_device_probe(struct xe_device *xe) if (err) goto err_unregister_display; - return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); + err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); + if (err) + goto err_unregister_display; + + return 0; err_unregister_display: xe_display_unregister(xe); diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 9fb99c038ea8..0fe4fb226ef4 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -328,6 +328,7 @@ static int execlist_exec_queue_init(struct xe_exec_queue *q) struct drm_gpu_scheduler *sched; const struct drm_sched_init_args args = { .ops = &drm_sched_ops, + .num_rqs = 1, .credit_limit = xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, .hang_limit = XE_SCHED_HANG_LIMIT, .timeout = XE_SCHED_JOB_TIMEOUT, diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c index 67d8ce368486..9c8004d5dd91 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.c +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c @@ -66,6 +66,7 @@ int xe_sched_init(struct xe_gpu_scheduler *sched, const struct drm_sched_init_args args = { .ops = ops, .submit_wq = submit_wq, + .num_rqs = 1, .credit_limit = hw_submission, .hang_limit = hang_limit, .timeout = timeout, diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index c98454545a85..886bafd31f45 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -63,10 +63,14 @@ ads_to_map(struct xe_guc_ads *ads) /* * The Additional Data Struct (ADS) has pointers for different buffers used by - * the GuC. One single gem object contains the ADS struct itself (guc_ads) and - * all the extra buffers indirectly linked via the ADS struct's entries. + * the GuC. One gem object (ads->bo) contains the ADS struct itself (guc_ads) + * and most of the extra buffers linked via the ADS struct's entries. The UM + * fault queues (PAGE_FAULT, PAGE_FAULT_RESPONSE, ACCESS_COUNTER rings) are + * kept in a separate BO (ads->um_queue_bo) so that the full memset of ads->bo + * performed on every GT reset does not discard fault descriptors already + * written into the rings by the GPU. * - * Layout of the ADS blob allocated for the GuC: + * Layout of the ADS blob (ads->bo): * * +---------------------------------------+ <== base * | guc_ads | @@ -98,10 +102,6 @@ ads_to_map(struct xe_guc_ads *ads) * +---------------------------------------+ * | padding | * +---------------------------------------+ <== 4K aligned - * | UM queues | - * +---------------------------------------+ - * | padding | - * +---------------------------------------+ <== 4K aligned * | private data | * +---------------------------------------+ * | padding | @@ -155,16 +155,6 @@ static size_t guc_ads_capture_size(struct xe_guc_ads *ads) return PAGE_ALIGN(ads->capture_size); } -static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads) -{ - struct xe_device *xe = ads_to_xe(ads); - - if (!xe->info.has_usm) - return 0; - - return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX; -} - static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) { return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); @@ -205,22 +195,12 @@ static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) return PAGE_ALIGN(offset); } -static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads) -{ - u32 offset; - - offset = guc_ads_capture_offset(ads) + - guc_ads_capture_size(ads); - - return PAGE_ALIGN(offset); -} - static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) { size_t offset; - offset = guc_ads_um_queues_offset(ads) + - guc_ads_um_queues_size(ads); + offset = guc_ads_capture_offset(ads) + + guc_ads_capture_size(ads); return PAGE_ALIGN(offset); } @@ -409,6 +389,49 @@ int xe_guc_ads_init(struct xe_guc_ads *ads) ads->bo = bo; + if (xe->info.has_usm) { + /* + * Allocate a separate BO for the HW fault ring (UM queues). + * + * Round the size up to the next power of two so that on iGPU + * (system memory, no IOMMU) the TTM pool issues a single + * alloc_pages(order=N) call, maximising the chance of getting + * a physically contiguous block. GuC requires contiguous DPA. + */ + size_t um_size = IS_DGFX(xe) ? + GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX : + roundup_pow_of_two(GUC_UM_QUEUE_SIZE * + GUC_UM_HW_QUEUE_MAX); + + u32 um_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE | + XE_BO_FLAG_NEEDS_UC; + + bo = xe_managed_bo_create_pin_map(xe, tile, um_size, um_flags); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + /* + * On pre-Xe3p platforms, GAM (not GuC) accesses the UM queue + * ring via base_dpa, which must be a contiguous DMA address + * range. Verify that the allocated pages are contiguous in + * DMA address space. + */ + if (!xe_bo_is_vram(bo) && + !xe_guc_using_main_gamctrl_queues(ads_to_guc(ads)) && + unlikely(!xe_bo_sg_is_contiguous(bo, + GUC_UM_QUEUE_SIZE * + GUC_UM_HW_QUEUE_MAX))) { + drm_err(&xe->drm, + "UM fault queue memory is not contiguous in DMA address space; GAM requires contiguous DPA\n"); + return -ENOMEM; + } + + ads->um_queue_bo = bo; + } + return 0; } ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */ @@ -820,7 +843,7 @@ static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) static void guc_um_init_params(struct xe_guc_ads *ads) { - u32 um_queue_offset = guc_ads_um_queues_offset(ads); + struct xe_bo *um_bo = ads->um_queue_bo; struct xe_guc *guc = ads_to_guc(ads); struct xe_device *xe = ads_to_xe(ads); u64 base_dpa; @@ -830,8 +853,14 @@ static void guc_um_init_params(struct xe_guc_ads *ads) with_dpa = !xe_guc_using_main_gamctrl_queues(guc); - base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset; - base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset; + if (um_bo) { + /* All USM platforms: UM queues in dedicated um_queue_bo */ + base_ggtt = xe_bo_ggtt_addr(um_bo); + base_dpa = xe_bo_main_addr(um_bo, PAGE_SIZE); + } else { + /* Platform does not support USM: no UM queues, nothing to do */ + return; + } for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { /* diff --git a/drivers/gpu/drm/xe/xe_guc_ads_types.h b/drivers/gpu/drm/xe/xe_guc_ads_types.h index 48a8e092023f..845c1fbd93a4 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads_types.h +++ b/drivers/gpu/drm/xe/xe_guc_ads_types.h @@ -16,6 +16,11 @@ struct xe_bo; struct xe_guc_ads { /** @bo: Xe BO for GuC ads blob */ struct xe_bo *bo; + /** + * @um_queue_bo: Dedicated BO for the HW fault ring (UM queues). + * NULL if the platform does not support USM. + */ + struct xe_bo *um_queue_bo; /** @golden_lrc_size: golden LRC size */ size_t golden_lrc_size; /** @regset_size: size of register set passed to GuC for save/restore */ diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 59f2fa79ad42..7cf8f4858598 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -911,6 +911,7 @@ static bool pc_needs_min_freq_change(struct xe_guc_pc *pc) static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) { int ret; + u32 min_freq; lockdep_assert_held(&pc->freq_lock); @@ -933,8 +934,14 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) * Same thing happens for Server platforms where min is listed as * RPMax */ - if (pc_get_min_freq(pc) > pc->rp0_freq) + min_freq = pc_get_min_freq(pc); + if (min_freq > pc->rp0_freq) { ret = pc_set_min_freq(pc, pc->rp0_freq); + if (ret) + goto out; + + min_freq = pc->rp0_freq; + } /* * Setting GT RP min frequency to 1.2GHz by default for @@ -947,8 +954,8 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) * we aren't expecting high power output across board * */ - if (pc_needs_min_freq_change(pc)) - ret = pc_set_min_freq(pc, max(BMG_MIN_FREQ, pc_get_min_freq(pc))); + if (pc_needs_min_freq_change(pc) && min_freq < BMG_MIN_FREQ) + ret = pc_set_min_freq(pc, BMG_MIN_FREQ); out: return ret; diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index a4292a11391d..9f8217ff1904 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1096,7 +1096,7 @@ static void xe_lrc_finish(struct xe_lrc *lrc) * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and * store it in the PPHSWP. */ -#define CONTEXT_ACTIVE 1ULL +#define CONTEXT_ACTIVE XE_LRC_CTX_TIMESTAMP_ACTIVE static ssize_t setup_utilization_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *batch, @@ -1849,6 +1849,13 @@ void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size) __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop)); } + + /* + * The ring and the LRC context image are both WC, so the ring tail + * update which publishes these writes can become visible to the device + * first. Ensure the ring contents are visible before returning. + */ + xe_device_wmb(xe); } u64 xe_lrc_descriptor(struct xe_lrc *lrc) @@ -2720,21 +2727,27 @@ static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) { u64 reg_ts, new_ts = lrc->ctx_timestamp; + u64 stored; /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ if (IS_SRIOV_VF(lrc_to_xe(lrc))) return xe_lrc_ctx_timestamp(lrc); - if (context_active(lrc) && - !get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) + /* Safely read CTX_TIMESTAMP: Avoid TOCTOU on LRC-stored CONTEXT_ACTIVE sentinel */ + stored = xe_lrc_ctx_timestamp(lrc); + if (stored != CONTEXT_ACTIVE) + return stored; + + /* Context is active: read the live timestamp from the engine's MMIO register */ + if (!get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) new_ts = reg_ts; - /* - * If context swicthed out while we were here, just return the latest - * LRC CTX TIMESTAMP value. + /* If the context switched out prefer using the value + * from context-save over the stale MMIO read. */ - if (!context_active(lrc)) - return xe_lrc_ctx_timestamp(lrc); + stored = xe_lrc_ctx_timestamp(lrc); + if (stored != CONTEXT_ACTIVE) + return stored; return new_ts; } diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 0a3a611391ee..7be5e3da8bc8 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -9,6 +9,13 @@ #include "xe_lrc_types.h" +/* + * Sentinel value stored in lrc->ctx_timestamp while a context is starting. + * The hardware hasn't yet written the real CTX_TIMESTAMP, so this is not a + * valid elapsed-time sample and must not be used as one. + */ +#define XE_LRC_CTX_TIMESTAMP_ACTIVE 1ULL + struct drm_printer; struct xe_bb; struct xe_device; diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 2dce6a47202c..18d990c5d4ec 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1594,6 +1594,10 @@ static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) config = xchg(&stream->oa_config, config); drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", stream->oa_config->uuid); + } else { + while (param.num_syncs--) + xe_sync_entry_cleanup(¶m.syncs[param.num_syncs]); + kfree(param.syncs); } err_config_put: @@ -2713,9 +2717,7 @@ static int xe_oa_init_gt(struct xe_gt *gt) __xe_oa_init_oa_units(gt); - drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock); - - return 0; + return drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock); } static void xe_oa_print_gt_oa_units(struct xe_gt *gt) @@ -2855,7 +2857,10 @@ int xe_oa_init(struct xe_device *xe) oa->xe = xe; oa->oa_formats = oa_formats; - drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock); + ret = drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock); + if (ret) + goto exit; + idr_init_base(&oa->metrics_idr, 1); ret = xe_oa_init_oa_units(oa); diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index 968b7e70b3f9..d17ab6e59df5 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -8,6 +8,8 @@ #include <drm/drm_managed.h> #include <uapi/drm/xe_drm.h> +#include <linux/device.h> + #include "xe_bo.h" #include "xe_bo_types.h" #include "xe_device_types.h" @@ -164,16 +166,9 @@ static void mark_termination_in_progress(struct xe_pxp *pxp) pxp->status = XE_PXP_TERMINATION_IN_PROGRESS; } -static void pxp_terminate(struct xe_pxp *pxp) +static bool pxp_prep_for_termination(struct xe_pxp *pxp) { - int ret = 0; - struct xe_device *xe = pxp->xe; - - if (!wait_for_completion_timeout(&pxp->activation, - msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) - drm_err(&xe->drm, "failed to wait for PXP start before termination\n"); - - mutex_lock(&pxp->mutex); + lockdep_assert_held(&pxp->mutex); if (pxp->status == XE_PXP_ACTIVE) pxp->key_instance++; @@ -182,10 +177,8 @@ static void pxp_terminate(struct xe_pxp *pxp) * we'll mark the status as needing termination on resume, so no need to * emit a termination now. */ - if (pxp->status == XE_PXP_SUSPENDED) { - mutex_unlock(&pxp->mutex); - return; - } + if (pxp->status == XE_PXP_SUSPENDED) + return false; /* * If we have a termination already in progress, we need to wait for @@ -195,15 +188,44 @@ static void pxp_terminate(struct xe_pxp *pxp) */ if (pxp->status == XE_PXP_TERMINATION_IN_PROGRESS) { pxp->status = XE_PXP_NEEDS_ADDITIONAL_TERMINATION; - mutex_unlock(&pxp->mutex); - return; + return false; } mark_termination_in_progress(pxp); - mutex_unlock(&pxp->mutex); + return true; +} + +static void pxp_terminate(struct xe_pxp *pxp, bool hw_only) +{ + struct xe_device *xe = pxp->xe; + int ret = 0; - pxp_invalidate_queues(pxp); + if (!wait_for_completion_timeout(&pxp->activation, + msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS))) + drm_err(&xe->drm, "failed to wait for PXP start before termination\n"); + + if (!hw_only) { + bool prep_ok; + + mutex_lock(&pxp->mutex); + + prep_ok = pxp_prep_for_termination(pxp); + + mutex_unlock(&pxp->mutex); + + if (!prep_ok) + return; + + pxp_invalidate_queues(pxp); + } else { + /* + * The caller of the HW-only termination should have already + * called pxp_prep_for_termination and marked the termination as + * in progress. + */ + xe_assert(xe, !completion_done(&pxp->termination)); + } ret = pxp_terminate_hw(pxp); if (ret) { @@ -249,33 +271,46 @@ static void pxp_terminate_complete(struct xe_pxp *pxp) mutex_unlock(&pxp->mutex); } -static void pxp_irq_work(struct work_struct *work) +static void pxp_events_work(struct work_struct *work) { - struct xe_pxp *pxp = container_of(work, typeof(*pxp), irq.work); + struct xe_pxp *pxp = container_of(work, typeof(*pxp), events.work); struct xe_device *xe = pxp->xe; + bool hw_only = false; u32 events = 0; - spin_lock_irq(&xe->irq.lock); - events = pxp->irq.events; - pxp->irq.events = 0; - spin_unlock_irq(&xe->irq.lock); + events = atomic_xchg(&pxp->events.pending, 0); if (!events) return; /* - * If we're processing a termination irq while suspending then don't - * bother, we're going to re-init everything on resume anyway. + * If the termination request comes from an irq while we're suspending, + * then we can defer it to the resume path instead of waking the device + * up. + * In the case of the termination on resume the pm reference is taken + * in xe_pxp_pm_resume() and released here. + * Note that we do not expect both events to be set at the same time, + * but if it does happen due to a spurious interrupt we want to behave + * as if the only request we got was the one from the resume path; this + * is because the termination prep has already been done in + * xe_pxp_pm_resume() and it is impossible for any PXP operations to + * occur between the prep and the termination completion, so there is no + * need for a new SW prep. */ - if ((events & PXP_TERMINATION_REQUEST) && !xe_pm_runtime_get_if_active(xe)) + if (events & PXP_TERMINATION_REQUEST_ON_RESUME) { + events &= ~PXP_TERMINATION_REQUEST_IRQ; + hw_only = true; + } + + if ((events & PXP_TERMINATION_REQUEST_IRQ) && !xe_pm_runtime_get_if_active(xe)) return; if (events & PXP_TERMINATION_REQUEST) { - events &= ~PXP_TERMINATION_COMPLETE; - pxp_terminate(pxp); + events &= ~PXP_TERMINATION_COMPLETE_IRQ; + pxp_terminate(pxp, hw_only); } - if (events & PXP_TERMINATION_COMPLETE) + if (events & PXP_TERMINATION_COMPLETE_IRQ) pxp_terminate_complete(pxp); if (events & PXP_TERMINATION_REQUEST) @@ -296,20 +331,18 @@ void xe_pxp_irq_handler(struct xe_device *xe, u16 iir) return; } - lockdep_assert_held(&xe->irq.lock); - if (unlikely(!iir)) return; if (iir & (KCR_PXP_STATE_TERMINATED_INTERRUPT | KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT)) - pxp->irq.events |= PXP_TERMINATION_REQUEST; + atomic_or(PXP_TERMINATION_REQUEST_IRQ, &pxp->events.pending); if (iir & KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT) - pxp->irq.events |= PXP_TERMINATION_COMPLETE; + atomic_or(PXP_TERMINATION_COMPLETE_IRQ, &pxp->events.pending); - if (pxp->irq.events) - queue_work(pxp->irq.wq, &pxp->irq.work); + if (atomic_read(&pxp->events.pending)) + queue_work(pxp->events.wq, &pxp->events.work); } static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable) @@ -340,7 +373,7 @@ static void pxp_fini(void *arg) { struct xe_pxp *pxp = arg; - destroy_workqueue(pxp->irq.wq); + destroy_workqueue(pxp->events.wq); xe_pxp_destroy_execution_resources(pxp); /* no need to explicitly disable KCR since we're going to do an FLR */ @@ -402,7 +435,7 @@ int xe_pxp_init(struct xe_device *xe) INIT_LIST_HEAD(&pxp->queues.list); spin_lock_init(&pxp->queues.lock); - INIT_WORK(&pxp->irq.work, pxp_irq_work); + INIT_WORK(&pxp->events.work, pxp_events_work); pxp->xe = xe; pxp->gt = gt; @@ -421,8 +454,8 @@ int xe_pxp_init(struct xe_device *xe) mutex_init(&pxp->mutex); - pxp->irq.wq = alloc_ordered_workqueue("pxp-wq", 0); - if (!pxp->irq.wq) { + pxp->events.wq = alloc_ordered_workqueue("pxp-wq", 0); + if (!pxp->events.wq) { err = -ENOMEM; goto out_free; } @@ -442,7 +475,7 @@ int xe_pxp_init(struct xe_device *xe) out_kcr_disable: kcr_pxp_disable(pxp); out_wq: - destroy_workqueue(pxp->irq.wq); + destroy_workqueue(pxp->events.wq); out_free: drmm_kfree(&xe->drm, pxp); out: @@ -889,6 +922,7 @@ wait_for_activation: fallthrough; case XE_PXP_ACTIVE: pxp->key_instance++; + pxp->needs_termination_on_resume = true; needs_queue_inval = true; break; } @@ -924,6 +958,7 @@ wait_for_activation: */ void xe_pxp_pm_resume(struct xe_pxp *pxp) { + bool has_pm = false; int err; if (!xe_pxp_is_enabled(pxp)) @@ -931,14 +966,57 @@ void xe_pxp_pm_resume(struct xe_pxp *pxp) err = kcr_pxp_enable(pxp); + /* + * We want to avoid the device runtime suspending before we're done with + * the termination queued below, so we need a runtime PM reference; we + * can't call the rpm functions from within the PXP lock, so we take the + * ref here. Note that we don't want the rpm resume code to actually run + * here as that would call back into this function, but as long as we + * don't enable DPM_FLAG_SMART_SUSPEND (which we currently do not) we're + * guaranteed to not be runtime suspended at this point, so we can + * safely use the get_noresume variant. + */ + if (pxp->needs_termination_on_resume) { + has_pm = true; + + xe_assert(pxp->xe, !dev_pm_smart_suspend(pxp->xe->drm.dev)); + xe_pm_runtime_get_noresume(pxp->xe); + } + mutex_lock(&pxp->mutex); xe_assert(pxp->xe, pxp->status == XE_PXP_SUSPENDED); - if (err) + if (err) { pxp->status = XE_PXP_ERROR; - else + } else { pxp->status = XE_PXP_NEEDS_TERMINATION; + if (pxp->needs_termination_on_resume) { + pxp->needs_termination_on_resume = false; + + /* + * We can't call pxp_terminate_hw directly from here + * because we're not allowed to do allocations within + * the rpm resume call, so we defer the termination to + * the worker that we use for the termination irqs. + * However, we do not want any PXP ops to go through + * between the suspend completing and the worker + * starting, so we need to do the termination prep + * immediately, which will mark the termination as in + * progress and stall PXP ops. + */ + if (pxp_prep_for_termination(pxp)) { + has_pm = false; /* move PM ref ownership to worker */ + + atomic_or(PXP_TERMINATION_REQUEST_ON_RESUME, &pxp->events.pending); + queue_work(pxp->events.wq, &pxp->events.work); + } + } + } + mutex_unlock(&pxp->mutex); + + if (has_pm) + xe_pm_runtime_put(pxp->xe); } diff --git a/drivers/gpu/drm/xe/xe_pxp_types.h b/drivers/gpu/drm/xe/xe_pxp_types.h index ec86306e16f4..8132a9750b6e 100644 --- a/drivers/gpu/drm/xe/xe_pxp_types.h +++ b/drivers/gpu/drm/xe/xe_pxp_types.h @@ -85,17 +85,20 @@ struct xe_pxp { /** @gsc_res: kernel-owned objects for PXP submissions to the GSCCS */ struct xe_pxp_gsc_client_resources gsc_res; - /** @irq: wrapper for the worker and queue used for PXP irq support */ + /** @events: wrapper for the worker and queue used for PXP event handling */ struct { - /** @irq.work: worker that manages irq events. */ + /** @events.work: worker that manages termination events. */ struct work_struct work; - /** @irq.wq: workqueue on which to queue the irq work. */ + /** @events.wq: workqueue on which to queue the work. */ struct workqueue_struct *wq; - /** @irq.events: pending events, protected with xe->irq.lock. */ - u32 events; -#define PXP_TERMINATION_REQUEST BIT(0) -#define PXP_TERMINATION_COMPLETE BIT(1) - } irq; + /** @events.pending: pending events */ + atomic_t pending; +#define PXP_TERMINATION_REQUEST_IRQ BIT(0) +#define PXP_TERMINATION_REQUEST_ON_RESUME BIT(1) +#define PXP_TERMINATION_REQUEST (PXP_TERMINATION_REQUEST_IRQ | \ + PXP_TERMINATION_REQUEST_ON_RESUME) +#define PXP_TERMINATION_COMPLETE_IRQ BIT(2) + } events; /** @mutex: protects the pxp status and the queue list */ struct mutex mutex; @@ -130,6 +133,14 @@ struct xe_pxp { * suspend cycles. */ u32 last_suspend_key_instance; + /** + * @needs_termination_on_resume: indicates if PXP termination is needed + * on resume. This is set if PXP was active when we suspend and it is + * cleared when we queue the termination on resume. Since the suspend + * and resume calls cannot execute at the same time, this variable does + * not need to be protected by the PXP lock. + */ + bool needs_termination_on_resume; }; #endif /* _XE_PXP_TYPES_H_ */ diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index d61c19e78182..790f7ecb6b85 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -100,7 +100,8 @@ struct drm_sched_entity { * @lock: * * Lock protecting the run-queue (@rq) to which this entity belongs, - * @priority and the list of schedulers (@sched_list, @num_sched_list). + * @priority, the list of schedulers (@sched_list, @num_sched_list) and + * the @rr_ts field. */ spinlock_t lock; @@ -154,6 +155,18 @@ struct drm_sched_entity { enum drm_sched_priority priority; /** + * @rq_priority: Run-queue priority + */ + enum drm_sched_priority rq_priority; + + /** + * @rr_ts: + * + * Fake timestamp of the last popped job from the entity. + */ + ktime_t rr_ts; + + /** * @job_queue: the list of jobs of this entity. */ struct spsc_queue job_queue; @@ -249,7 +262,9 @@ struct drm_sched_entity { /** * struct drm_sched_rq - queue of entities to be scheduled. * - * @lock: protects @entities, @rb_tree_root and @head_prio. + * @sched: the scheduler to which this rq belongs to. + * @lock: protects @entities, @rb_tree_root, @rr_ts and @head_prio. + * @rr_ts: monotonically incrementing fake timestamp for RR mode. * @entities: list of the entities to be scheduled. * @rb_tree_root: root of time based priority queue of entities for FIFO scheduling * @head_prio: priority of the top tree element. @@ -259,8 +274,11 @@ struct drm_sched_entity { * the next entity to emit commands from. */ struct drm_sched_rq { + struct drm_gpu_scheduler *sched; + spinlock_t lock; /* Following members are protected by the @lock: */ + ktime_t rr_ts; struct list_head entities; struct rb_root_cached rb_tree_root; enum drm_sched_priority head_prio; @@ -347,6 +365,13 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); */ struct drm_sched_job { /** + * @submit_ts: + * + * When the job was pushed into the entity queue. + */ + ktime_t submit_ts; + + /** * @sched: * * The scheduler this job is or will be scheduled on. Gets set by @@ -549,7 +574,11 @@ struct drm_sched_backend_ops { * @credit_count: the current credit count of this scheduler * @timeout: the time after which a job is removed from the scheduler. * @name: name of the ring for which this scheduler is being used. - * @rq: Scheduler run queue. + * @num_user_rqs: Number of run-queues. This is at most + * DRM_SCHED_PRIORITY_COUNT, as there's usually one run-queue per + * priority, but could be less. + * @num_rqs: Equal to @num_user_rqs for FIFO and RR and 1 for the FAIR policy. + * @sched_rq: An allocated array of run-queues of size @num_rqs; * @job_scheduled: once drm_sched_entity_flush() is called the scheduler * waits on this wait queue until all the scheduled jobs are * finished. @@ -581,7 +610,9 @@ struct drm_gpu_scheduler { atomic_t credit_count; long timeout; const char *name; - struct drm_sched_rq rq; + u32 num_rqs; + u32 num_user_rqs; + struct drm_sched_rq **sched_rq; wait_queue_head_t job_scheduled; atomic64_t job_id_count; struct workqueue_struct *submit_wq; @@ -608,6 +639,8 @@ struct drm_gpu_scheduler { * @ops: backend operations provided by the driver * @submit_wq: workqueue to use for submission. If NULL, an ordered wq is * allocated and used. + * @num_rqs: Number of run-queues. This may be at most DRM_SCHED_PRIORITY_COUNT, + * as there's usually one run-queue per priority, but may be less. * @credit_limit: the number of credits this scheduler can hold from all jobs * @hang_limit: number of times to allow a job to hang before dropping it. * This mechanism is DEPRECATED. Set it to 0. @@ -621,6 +654,7 @@ struct drm_sched_init_args { const struct drm_sched_backend_ops *ops; struct workqueue_struct *submit_wq; struct workqueue_struct *timeout_wq; + u32 num_rqs; u32 credit_limit; unsigned int hang_limit; long timeout; |