diff options
Diffstat (limited to 'drivers/gpu/drm/amd')
109 files changed, 1945 insertions, 1120 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 4c732e0f776e..9014678d75ab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -508,6 +508,7 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) u32 val = 0; u32 count = 0; struct amdgpu_device *adev = ip_block->adev; + int ret = 0; /* return early if no ACP */ if (!adev->acp.acp_genpd) { @@ -529,7 +530,8 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) break; if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } udelay(100); } @@ -546,21 +548,24 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) break; if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } udelay(100); } - +out: device_for_each_child(adev->acp.parent, NULL, acp_genpd_remove_device); mfd_remove_devices(adev->acp.parent); kfree(adev->acp.i2s_pdata); kfree(adev->acp.acp_res); + pm_genpd_remove(&adev->acp.acp_genpd->gpd); kfree(adev->acp.acp_genpd); + adev->acp.acp_genpd = NULL; kfree(adev->acp.acp_cell); - return 0; + return ret; } static int acp_suspend(struct amdgpu_ip_block *ip_block) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 9783a3cefb04..da325863ad76 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -558,7 +558,7 @@ uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev) int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, struct amdgpu_device **dmabuf_adev, - uint64_t *bo_size, void *metadata_buffer, + uint64_t *bo_size, void **metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags, int8_t *xcp_id) { @@ -593,9 +593,24 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, *dmabuf_adev = adev; if (bo_size) *bo_size = amdgpu_bo_size(bo); - if (metadata_buffer) - r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size, - metadata_size, &metadata_flags); + if (metadata_buffer) { + /* first get metadata_size by buffer = NULL */ + r = amdgpu_bo_get_metadata(bo, NULL, 0, + metadata_size, NULL); + + /* user buf_size is bigger than bo metadata_size + * allocate a buf at kernel space and copy */ + if (*metadata_size <= buffer_size) { + *metadata_buffer = kzalloc(*metadata_size, GFP_KERNEL); + + if (!*metadata_buffer) + return -ENOMEM; + + r = amdgpu_bo_get_metadata(bo, *metadata_buffer, *metadata_size, + NULL, &metadata_flags); + } else + r = -EINVAL; + } if (flags) { *flags = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? KFD_IOC_ALLOC_MEM_FLAGS_VRAM diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 5333e052d56d..e443a7277299 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -262,7 +262,7 @@ uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev); uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev); int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, struct amdgpu_device **dmabuf_adev, - uint64_t *bo_size, void *metadata_buffer, + uint64_t *bo_size, void **metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags, int8_t *xcp_id); int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index d54794e5b18b..35fe2c974699 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1914,13 +1914,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( mutex_lock(&mem->lock); - /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */ - if (mem->alloc_flags & - (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | - KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { - amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); - } - mapped_to_gpu_memory = mem->mapped_to_gpu_memory; is_imported = mem->is_imported; mutex_unlock(&mem->lock); @@ -1934,6 +1927,15 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( return -EBUSY; } + /* At this point the BO is guaranteed to be freed, so unpin the + * MMIO/DOORBELL BOs that were pinned during allocation. + */ + if (mem->alloc_flags & + (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | + KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { + amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); + } + /* Make sure restore workers don't access the BO any more */ mutex_lock(&process_info->lock); if (!list_empty(&mem->validate_list)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index aa039e148a5e..8525c45ab209 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -372,19 +372,59 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev) } #ifdef CONFIG_ACPI +/** + * amdgpu_acpi_vfct_match() - Check if a VFCT entry matches the device + * @adev: AMDGPU device + * @vhdr: VFCT image header to check + * + * VFCT entries contain the PCI bus number as recorded during BIOS POST. + * On systems where the kernel renumbers PCI buses (e.g. pci=realloc or + * resource conflicts), the runtime bus number may differ from the POST + * value. Match by device identity (vendor + device + function) and use + * the bus number as a preference: exact bus match is preferred, but when + * the bus numbers disagree we accept the entry if the device identity + * matches. + * + * Returns: 0 on match, -ENODEV on no match + */ +static int amdgpu_acpi_vfct_match(struct amdgpu_device *adev, + VFCT_IMAGE_HEADER *vhdr) +{ + /* Vendor and device IDs must always match */ + if (vhdr->VendorID != adev->pdev->vendor || + vhdr->DeviceID != adev->pdev->device) + return -ENODEV; + + if (vhdr->PCIDevice != PCI_SLOT(adev->pdev->devfn) || + vhdr->PCIFunction != PCI_FUNC(adev->pdev->devfn)) + return -ENODEV; + + /* Exact bus number match - preferred */ + if (vhdr->PCIBus == adev->pdev->bus->number) + return 0; + + /* Bus mismatch but device identity matches (PCI renumbering case) */ + dev_notice(adev->dev, + "VFCT bus number mismatch: table %u != runtime %u, matching by device identity (vendor 0x%04x device 0x%04x)\n", + vhdr->PCIBus, adev->pdev->bus->number, + adev->pdev->vendor, adev->pdev->device); + return 0; +} + static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) { struct acpi_table_header *hdr; acpi_size tbl_size; UEFI_ACPI_VFCT *vfct; unsigned int offset; + bool r = false; if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr))) return false; tbl_size = hdr->length; if (tbl_size < sizeof(UEFI_ACPI_VFCT)) { dev_info(adev->dev, "ACPI VFCT table present but broken (too short #1),skipping\n"); - return false; + goto out; } vfct = (UEFI_ACPI_VFCT *)hdr; @@ -397,36 +437,36 @@ static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) offset += sizeof(VFCT_IMAGE_HEADER); if (offset > tbl_size) { dev_info(adev->dev, "ACPI VFCT image header truncated,skipping\n"); - return false; + goto out; } offset += vhdr->ImageLength; if (offset > tbl_size) { dev_info(adev->dev, "ACPI VFCT image truncated,skipping\n"); - return false; + goto out; } if (vhdr->ImageLength && - vhdr->PCIBus == adev->pdev->bus->number && - vhdr->PCIDevice == PCI_SLOT(adev->pdev->devfn) && - vhdr->PCIFunction == PCI_FUNC(adev->pdev->devfn) && - vhdr->VendorID == adev->pdev->vendor && - vhdr->DeviceID == adev->pdev->device) { + !amdgpu_acpi_vfct_match(adev, vhdr)) { adev->bios = kmemdup(&vbios->VbiosContent, vhdr->ImageLength, GFP_KERNEL); if (!check_atom_bios(adev, vhdr->ImageLength)) { amdgpu_bios_release(adev); - return false; + goto out; } adev->bios_size = vhdr->ImageLength; - return true; + r = true; + goto out; } } dev_info(adev->dev, "ACPI VFCT table present but broken (too short #2),skipping\n"); - return false; + +out: + acpi_put_table(hdr); + return r; } #else static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 115b134b4cd1..e714cee2997a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -247,13 +247,17 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, goto free_partial_kdata; break; + case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: + if (size < sizeof(struct drm_amdgpu_cs_chunk_cp_gfx_shadow)) + goto free_partial_kdata; + break; + case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_IN: case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: - case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: break; default: @@ -1318,7 +1322,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, e->range = NULL; } - if (r || !list_empty(&vm->individual.moved)) { + if (r || !list_empty(&vm->individual.needs_update)) { r = -EAGAIN; mutex_unlock(&p->adev->notifier_lock); return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 0d7f6cd74f79..ce35b415093d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -326,7 +326,6 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, struct drm_file *filp, struct amdgpu_ctx *ctx) { struct amdgpu_fpriv *fpriv = filp->driver_priv; - u32 current_stable_pstate; int r; r = amdgpu_ctx_priority_permit(filp, priority); @@ -344,36 +343,21 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, ctx->generation = amdgpu_vm_generation(mgr->adev, &fpriv->vm); ctx->init_priority = priority; ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET; - - r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); - if (r) - return r; - - if (mgr->adev->pm.stable_pstate_ctx) - ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate; - else - ctx->stable_pstate = current_stable_pstate; + ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; return 0; } -static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, - u32 stable_pstate) +static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, + u32 stable_pstate) { struct amdgpu_device *adev = ctx->mgr->adev; enum amd_dpm_forced_level level; + struct amdgpu_ctx *current_ctx; u32 current_stable_pstate; - int r; + int r = 0; - mutex_lock(&adev->pm.stable_pstate_ctx_lock); - if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) { - r = -EBUSY; - goto done; - } - - r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); - if (r || (stable_pstate == current_stable_pstate)) - goto done; + lockdep_assert_held(&adev->pm.stable_pstate_ctx_lock); switch (stable_pstate) { case AMDGPU_CTX_STABLE_PSTATE_NONE: @@ -392,17 +376,41 @@ static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; break; default: - r = -EINVAL; - goto done; + return -EINVAL; } + current_ctx = adev->pm.stable_pstate_ctx; + if (current_ctx && current_ctx != ctx) + return -EBUSY; + + r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); + if (r || current_stable_pstate == stable_pstate) + return r; + r = amdgpu_dpm_force_performance_level(adev, level); + if (r) + return r; - if (level == AMD_DPM_FORCED_LEVEL_AUTO) - adev->pm.stable_pstate_ctx = NULL; - else + if (!current_ctx) { adev->pm.stable_pstate_ctx = ctx; -done: + /* + * Serialized by context taking ownership for the first time + * while holding adev->pm.stable_pstate_ctx_lock). + */ + WRITE_ONCE(ctx->stable_pstate, current_stable_pstate); + } + + return 0; +} + +static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, + u32 stable_pstate) +{ + struct amdgpu_device *adev = ctx->mgr->adev; + int r; + + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + r = __amdgpu_ctx_set_stable_pstate(ctx, stable_pstate); mutex_unlock(&adev->pm.stable_pstate_ctx_lock); return r; @@ -428,7 +436,12 @@ static void amdgpu_ctx_fini(struct kref *ref) } if (drm_dev_enter(adev_to_drm(adev), &idx)) { - amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + if (adev->pm.stable_pstate_ctx == ctx) { + __amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); + adev->pm.stable_pstate_ctx = NULL; + } + mutex_unlock(&adev->pm.stable_pstate_ctx_lock); drm_dev_exit(idx); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c index 27830518a230..6480a344006d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c @@ -22,8 +22,9 @@ * */ -#include <generated/utsrelease.h> #include <linux/devcoredump.h> +#include <linux/utsname.h> +#include <drm/drm_exec.h> #include "amdgpu_dev_coredump.h" #include "atom.h" @@ -207,28 +208,146 @@ static void amdgpu_devcoredump_fw_info(struct amdgpu_device *adev, } } +static void +amdgpu_devcoredump_print_ibs(struct drm_printer *p, + struct amdgpu_coredump_info *coredump, + bool sizing_pass) +{ + struct amdgpu_device *adev = coredump->adev; + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_bo *abo; + struct drm_exec exec; + struct amdgpu_vm *vm; + u32 *ib_content; + u64 va_start, offset; + u8 *kptr; + u32 off; + int r; + + /* + * On the sizing pass there is no VM to look up and no BO to lock; the + * size estimate doesn't depend on whether the IB BOs are reachable. + * Just emit the per-IB headers (the content is not written anywhere). + */ + if (sizing_pass) { + for (int i = 0; i < coredump->num_ibs; i++) { + drm_printf(p, "\nIB #%d 0x%llx %d dw\n", i, + coredump->ibs[i].gpu_addr, + coredump->ibs[i].ib_size_dw); + + for (int j = 0; j < coredump->ibs[i].ib_size_dw; j++) + drm_printf(p, "0xffffffff\n"); + } + return; + } + + /* + * Lock the VM root PD and every IB BO together in a single drm_exec + * ticket. Reserving the IB BOs one by one while the root PD is held + * would be a recursive reservation_ww_class_mutex acquire without a + * ww_acquire_ctx, which trips lockdep and self-deadlocks for IB BOs + * that share their dma_resv with the root PD (always-valid BOs). + */ + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 1 + coredump->num_ibs); + drm_exec_until_all_locked(&exec) { + vm = amdgpu_vm_lock_by_pasid(adev, coredump->pasid, &exec); + if (!vm) + goto unlock; + + for (int i = 0; i < coredump->num_ibs; i++) { + u64 pfn = (coredump->ibs[i].gpu_addr & + AMDGPU_GMC_HOLE_MASK) / AMDGPU_GPU_PAGE_SIZE; + + mapping = amdgpu_vm_bo_lookup_mapping(vm, pfn); + if (!mapping) + continue; + + abo = mapping->bo_va->base.bo; + r = drm_exec_lock_obj(&exec, &abo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (r) + goto unlock; + } + } + + for (int i = 0; i < coredump->num_ibs; i++) { + bool emit_content = false; + + ib_content = kvmalloc_array(coredump->ibs[i].ib_size_dw, 4, + GFP_KERNEL); + if (!ib_content) + continue; + + va_start = coredump->ibs[i].gpu_addr & AMDGPU_GMC_HOLE_MASK; + mapping = amdgpu_vm_bo_lookup_mapping(vm, + va_start / AMDGPU_GPU_PAGE_SIZE); + if (!mapping) + goto output_ib_content; + + abo = mapping->bo_va->base.bo; + offset = va_start - mapping->start * AMDGPU_GPU_PAGE_SIZE; + + if (abo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) { + struct amdgpu_res_cursor cursor; + + off = 0; + + if (abo->tbo.resource->mem_type != TTM_PL_VRAM) + goto output_ib_content; + + amdgpu_res_first(abo->tbo.resource, offset, + coredump->ibs[i].ib_size_dw * 4, &cursor); + while (cursor.remaining) { + amdgpu_device_mm_access(adev, cursor.start / 4, + &ib_content[off], cursor.size / 4, + false); + off += cursor.size; + amdgpu_res_next(&cursor, cursor.size); + } + emit_content = true; + } else { + r = ttm_bo_kmap(&abo->tbo, 0, PFN_UP(abo->tbo.base.size), + &abo->kmap); + if (r) + goto output_ib_content; + + kptr = amdgpu_bo_kptr(abo); + kptr += offset; + memcpy(ib_content, kptr, coredump->ibs[i].ib_size_dw * 4); + + amdgpu_bo_kunmap(abo); + emit_content = true; + } + +output_ib_content: + drm_printf(p, "\nIB #%d 0x%llx %d dw\n", i, + coredump->ibs[i].gpu_addr, coredump->ibs[i].ib_size_dw); + if (emit_content) { + for (int j = 0; j < coredump->ibs[i].ib_size_dw; j++) + drm_printf(p, "0x%08x\n", ib_content[j]); + } + kvfree(ib_content); + } + +unlock: + drm_exec_fini(&exec); +} + static ssize_t amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_info *coredump) { - struct amdgpu_device *adev = coredump->adev; struct drm_printer p; struct drm_print_iterator iter; struct amdgpu_vm_fault_info *fault_info; - struct amdgpu_bo_va_mapping *mapping; struct amdgpu_ip_block *ip_block; - struct amdgpu_res_cursor cursor; - struct amdgpu_bo *abo, *root; - uint64_t va_start, offset; struct amdgpu_ring *ring; - struct amdgpu_vm *vm; - u32 *ib_content; - uint8_t *kptr; - int ver, i, j, r; + int ver, i, j; u32 ring_idx, off; bool sizing_pass; sizing_pass = buffer == NULL; iter.data = buffer; + iter.start = 0; iter.offset = 0; iter.remain = count; @@ -236,13 +355,17 @@ amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_inf drm_printf(&p, "**** AMDGPU Device Coredump ****\n"); drm_printf(&p, "version: " AMDGPU_COREDUMP_VERSION "\n"); - drm_printf(&p, "kernel: " UTS_RELEASE "\n"); + drm_printf(&p, "kernel: %s\n", init_utsname()->release); drm_printf(&p, "module: " KBUILD_MODNAME "\n"); drm_printf(&p, "time: %ptSp\n", &coredump->reset_time); + drm_printf(&p, "pasid: %u\n", coredump->pasid); + drm_printf(&p, "vmid: %u\n", coredump->vmid); if (coredump->reset_task_info.task.pid) - drm_printf(&p, "process_name: %s PID: %d\n", + drm_printf(&p, "process_name: %s TGID: %d thread: %s PID: %d\n", coredump->reset_task_info.process_name, + coredump->reset_task_info.tgid, + coredump->reset_task_info.task.comm, coredump->reset_task_info.task.pid); /* SOC Information */ @@ -342,86 +465,8 @@ amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_inf else if (coredump->reset_vram_lost) drm_printf(&p, "VRAM is lost due to GPU reset!\n"); - if (coredump->num_ibs) { - /* Don't try to lookup the VM or map the BOs when calculating the - * size required to store the devcoredump. - */ - if (sizing_pass) - vm = NULL; - else - vm = amdgpu_vm_lock_by_pasid(adev, &root, coredump->pasid); - - for (int i = 0; i < coredump->num_ibs && (sizing_pass || vm); i++) { - ib_content = kvmalloc_array(coredump->ibs[i].ib_size_dw, 4, - GFP_KERNEL); - if (!ib_content) - continue; - - /* vm=NULL can only happen when 'sizing_pass' is true. Skip to the - * drm_printf() calls (ib_content doesn't need to be initialized - * as its content won't be written anywhere). - */ - if (!vm) - goto output_ib_content; - - va_start = coredump->ibs[i].gpu_addr & AMDGPU_GMC_HOLE_MASK; - mapping = amdgpu_vm_bo_lookup_mapping(vm, va_start / AMDGPU_GPU_PAGE_SIZE); - if (!mapping) - goto free_ib_content; - - offset = va_start - (mapping->start * AMDGPU_GPU_PAGE_SIZE); - abo = amdgpu_bo_ref(mapping->bo_va->base.bo); - r = amdgpu_bo_reserve(abo, false); - if (r) - goto free_ib_content; - - if (abo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) { - off = 0; - - if (abo->tbo.resource->mem_type != TTM_PL_VRAM) - goto unreserve_abo; - - amdgpu_res_first(abo->tbo.resource, offset, - coredump->ibs[i].ib_size_dw * 4, - &cursor); - while (cursor.remaining) { - amdgpu_device_mm_access(adev, cursor.start / 4, - &ib_content[off], cursor.size / 4, - false); - off += cursor.size; - amdgpu_res_next(&cursor, cursor.size); - } - } else { - r = ttm_bo_kmap(&abo->tbo, 0, - PFN_UP(abo->tbo.base.size), - &abo->kmap); - if (r) - goto unreserve_abo; - - kptr = amdgpu_bo_kptr(abo); - kptr += offset; - memcpy(ib_content, kptr, - coredump->ibs[i].ib_size_dw * 4); - - amdgpu_bo_kunmap(abo); - } - -output_ib_content: - drm_printf(&p, "\nIB #%d 0x%llx %d dw\n", - i, coredump->ibs[i].gpu_addr, coredump->ibs[i].ib_size_dw); - for (int j = 0; j < coredump->ibs[i].ib_size_dw; j++) - drm_printf(&p, "0x%08x\n", ib_content[j]); -unreserve_abo: - if (vm) - amdgpu_bo_unreserve(abo); -free_ib_content: - kvfree(ib_content); - } - if (vm) { - amdgpu_bo_unreserve(root); - amdgpu_bo_unref(&root); - } - } + if (coredump->num_ibs) + amdgpu_devcoredump_print_ibs(&p, coredump, sizing_pass); return count - iter.remain; } @@ -524,6 +569,7 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check, amdgpu_vm_put_task_info(ti); } coredump->pasid = job->pasid; + coredump->vmid = job->vmid; coredump->num_ibs = job->num_ibs; for (i = 0; i < job->num_ibs; ++i) { coredump->ibs[i].gpu_addr = job->ibs[i].gpu_addr; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h index 2371e20fc68b..63f27337c09a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h @@ -63,6 +63,7 @@ struct amdgpu_coredump_info { char *formatted; unsigned int pasid; + unsigned int vmid; int num_ibs; struct amdgpu_coredump_ib_info ibs[] __counted_by(num_ibs); }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 942f0251c748..e5f26e5892ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1323,6 +1323,15 @@ static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device if (c->x86_vendor == X86_VENDOR_INTEL) return false; + + /* + * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't + * support PCIe dynamic speed switching. + * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436 + */ + if (c->x86_vendor == X86_VENDOR_AMD && c->x86 == 0x17 && + c->x86_model == 0x08) + return false; #endif return true; } @@ -1333,7 +1342,8 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) * It's unclear if this is a platform-specific or GPU-specific issue. * Disable ASPM on SI for the time being. */ - if (adev->family == AMDGPU_FAMILY_SI) + if (adev->family == AMDGPU_FAMILY_SI || + (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK) && adev->family == AMDGPU_FAMILY_VI)) return true; #if IS_ENABLED(CONFIG_X86) @@ -3043,7 +3053,7 @@ static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); for (i = adev->num_ip_blocks - 1; i >= 0; i--) { - if (!adev->ip_blocks[i].status.valid) + if (!adev->ip_blocks[i].status.valid || !adev->ip_blocks[i].status.hw) continue; /* displays are handled in phase1 */ if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) @@ -3771,6 +3781,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->gfx.workload_profile_mutex); mutex_init(&adev->vcn.workload_profile_mutex); + spin_lock_init(&adev->irq.lock); + amdgpu_device_init_apu_flags(adev); r = amdgpu_device_check_arguments(adev); @@ -4182,8 +4194,6 @@ static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) iounmap(adev->rmmio); adev->rmmio = NULL; - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); adev->mman.aper_base_kaddr = NULL; /* Memory manager related */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index be5069642a90..2860f12915c0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -2119,6 +2119,8 @@ static int amdgpu_discovery_set_common_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &soc21_common_ip_block); break; case IP_VERSION(12, 0, 0): @@ -2180,6 +2182,8 @@ static int amdgpu_discovery_set_gmc_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &gmc_v11_0_ip_block); break; case IP_VERSION(12, 0, 0): @@ -2300,6 +2304,7 @@ static int amdgpu_discovery_set_psp_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &psp_v14_0_ip_block); break; case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): amdgpu_device_ip_block_add(adev, &psp_v15_0_ip_block); break; case IP_VERSION(15, 0, 8): @@ -2371,6 +2376,7 @@ static int amdgpu_discovery_set_smu_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(15, 0, 0): case IP_VERSION(15, 0, 5): case IP_VERSION(15, 0, 8): + case IP_VERSION(15, 0, 9): amdgpu_device_ip_block_add(adev, &smu_v15_0_ip_block); break; default: @@ -2506,6 +2512,8 @@ static int amdgpu_discovery_set_gc_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &gfx_v11_0_ip_block); break; case IP_VERSION(12, 0, 0): @@ -2719,6 +2727,8 @@ static int amdgpu_discovery_set_mes_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &mes_v11_0_ip_block); adev->enable_mes = true; adev->enable_mes_kiq = true; @@ -3129,6 +3139,10 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 6): adev->family = AMDGPU_FAMILY_GC_11_5_0; break; + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): + adev->family = AMDGPU_FAMILY_GC_11_5_4; + break; case IP_VERSION(12, 0, 0): case IP_VERSION(12, 0, 1): case IP_VERSION(12, 1, 0): @@ -3156,6 +3170,8 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->flags |= AMD_IS_APU; break; default: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index bf4260269681..4c0c77eafbd1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -3196,6 +3196,14 @@ static void __exit amdgpu_exit(void) amdgpu_sync_fini(); mmu_notifier_synchronize(); amdgpu_xcp_drv_release(); + + /* + * Flush outstanding call_rcu() callbacks before the + * module text is freed. Otherwise a grace period elapsing after + * unload invokes a callback in already-freed module memory and + * faults in rcu_do_batch(). + */ + rcu_barrier(); } module_init(amdgpu_init); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 212c14d99f6b..6a0699746fbc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -535,6 +535,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, bo = gem_to_amdgpu_bo(gobj); bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + bo->parent = amdgpu_bo_ref(fpriv->vm.root.bo); r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags); if (r) goto release_object; @@ -1094,6 +1095,11 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, * If that number is larger than the size of the array, the ioctl must * be retried. */ + if (!bo_va) { + r = -ENOENT; + goto out_exec; + } + if (args->num_entries > INT_MAX / sizeof(*vm_entries)) { r = -EINVAL; goto out_exec; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 1e190fb54a97..85372af1216d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -1664,12 +1664,13 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct drm_gpu_scheduler *sched = &ring->sched; struct drm_sched_entity entity; + unsigned int ib_size_dw = 16; static atomic_t counter; struct dma_fence *f; struct amdgpu_job *job; struct amdgpu_ib *ib; void *owner; - int i, r; + int r; /* Initialize the scheduler entity */ r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL, @@ -1687,7 +1688,7 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) owner = (void *)(unsigned long)atomic_inc_return(&counter); r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner, - 64, 0, &job, + ib_size_dw * sizeof(uint32_t), 0, &job, AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER); if (r) goto err; @@ -1697,9 +1698,8 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) job->run_cleaner_shader = true; ib = &job->ibs[0]; - for (i = 0; i <= ring->funcs->align_mask; ++i) - ib->ptr[i] = ring->funcs->nop; - ib->length_dw = ring->funcs->align_mask + 1; + memset32(ib->ptr, ring->funcs->nop, ib_size_dw); + ib->length_dw = ib_size_dw; f = amdgpu_job_submit(job); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 5f7745143f56..5d6149ba7ab7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -977,6 +977,8 @@ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): /* Don't enable it by default yet. */ if (amdgpu_tmz < 1) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index d23a91d029aa..0ea32561c4bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -272,7 +272,20 @@ static bool amdgpu_gtt_mgr_intersects(struct ttm_resource_manager *man, const struct ttm_place *place, size_t size) { - return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res); + const struct drm_mm_node *const node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + const u32 num_pages = PFN_UP(size); + + if (!place->lpfn) + return true; + + if (!amdgpu_gtt_mgr_has_gart_addr(res)) + return false; + + if (place->fpfn >= (node->start + num_pages) || + (place->lpfn && place->lpfn <= node->start)) + return false; + + return true; } /** @@ -290,7 +303,20 @@ static bool amdgpu_gtt_mgr_compatible(struct ttm_resource_manager *man, const struct ttm_place *place, size_t size) { - return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res); + const struct drm_mm_node *const node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + const u32 num_pages = PFN_UP(size); + + if (!place->lpfn) + return true; + + if (!amdgpu_gtt_mgr_has_gart_addr(res)) + return false; + + if (node->start < place->fpfn || + (place->lpfn && (node->start + num_pages) > place->lpfn)) + return false; + + return true; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c index 99bc9ad67d5b..a7d13e337d84 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c @@ -67,7 +67,6 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, { struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - struct amdgpu_bo *vm_root = bo->vm_bo->vm->root.bo; long r; if (!mmu_notifier_range_blockable(range)) @@ -78,7 +77,7 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, mmu_interval_set_seq(mni, cur_seq); amdgpu_vm_bo_invalidate(bo, false); - r = dma_resv_wait_timeout(vm_root->tbo.base.resv, + r = dma_resv_wait_timeout(bo->parent->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, false, MAX_SCHEDULE_TIMEOUT); mutex_unlock(&adev->notifier_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 254a4e983f40..53be764968e4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -309,8 +309,6 @@ int amdgpu_irq_init(struct amdgpu_device *adev) unsigned int irq, flags; int r; - spin_lock_init(&adev->irq.lock); - /* Enable MSI if not disabled by module parameter */ adev->irq.msi_enabled = false; @@ -547,7 +545,7 @@ void amdgpu_irq_delegate(struct amdgpu_device *adev, unsigned int num_dw) { amdgpu_ih_ring_write(adev, &adev->irq.ih_soft, entry->iv_entry, num_dw); - schedule_work(&adev->irq.ih_soft_work); + queue_work(system_unbound_wq, &adev->irq.ih_soft_work); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c index d5d71fd7c70d..61450af539a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c @@ -16,6 +16,17 @@ #ifdef CONFIG_LOCKDEP +struct amdgpu_lockdep_dummy_locks { + struct mutex reset_lock; + struct mutex userq_sch_mutex; + struct mutex userq_mutex; + struct mutex notifier_lock; + struct mutex vram_lock; + struct mutex srbm_mutex; + struct mutex grbm_idx_mutex; + spinlock_t mmio_idx_lock; +}; + /* Lock class keys for associating with real driver locks */ static struct lock_class_key amdgpu_userq_sch_mutex_key; static struct lock_class_key amdgpu_userq_mutex_key; @@ -84,72 +95,65 @@ void amdgpu_lockdep_set_class(struct amdgpu_device *adev) int amdgpu_lockdep_init(void) { struct amdgpu_reset_domain *reset_domain = NULL; - struct amdgpu_reset_control reset_ctl; - struct mutex userq_sch_mutex; - struct mutex userq_mutex; - struct mutex notifier_lock; - struct mutex vram_lock; - struct mutex srbm_mutex; - struct mutex grbm_idx_mutex; - spinlock_t mmio_idx_lock; + struct amdgpu_lockdep_dummy_locks *locks; unsigned long flags; + locks = kzalloc(sizeof(*locks), GFP_KERNEL); + if (!locks) + return -ENOMEM; + /* * Initialize dummy reset domain */ reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "lockdep_test"); - if (!reset_domain) + if (!reset_domain) { + kfree(locks); return -ENOMEM; - + } /* Initialize dummy locks */ - mutex_init(&userq_sch_mutex); - mutex_init(&userq_mutex); - mutex_init(¬ifier_lock); - mutex_init(&vram_lock); - mutex_init(&reset_ctl.reset_lock); - mutex_init(&srbm_mutex); - mutex_init(&grbm_idx_mutex); - spin_lock_init(&mmio_idx_lock); + mutex_init(&locks->userq_sch_mutex); + mutex_init(&locks->userq_mutex); + mutex_init(&locks->notifier_lock); + mutex_init(&locks->vram_lock); + mutex_init(&locks->reset_lock); + mutex_init(&locks->srbm_mutex); + mutex_init(&locks->grbm_idx_mutex); + spin_lock_init(&locks->mmio_idx_lock); /* * Associate dummy locks with the same class keys used for real * driver locks. This ensures lockdep connects the ordering learned * here with the actual locks used at runtime. */ - lockdep_set_class(&userq_sch_mutex, &amdgpu_userq_sch_mutex_key); - lockdep_set_class(&userq_mutex, &amdgpu_userq_mutex_key); - lockdep_set_class(¬ifier_lock, &amdgpu_notifier_lock_key); - lockdep_set_class(&vram_lock, &amdgpu_vram_lock_key); + lockdep_set_class(&locks->userq_sch_mutex, &amdgpu_userq_sch_mutex_key); + lockdep_set_class(&locks->userq_mutex, &amdgpu_userq_mutex_key); + lockdep_set_class(&locks->notifier_lock, &amdgpu_notifier_lock_key); + lockdep_set_class(&locks->vram_lock, &amdgpu_vram_lock_key); lockdep_set_class(&reset_domain->sem, &amdgpu_reset_sem_key); - lockdep_set_class(&reset_ctl.reset_lock, &amdgpu_reset_lock_key); - lockdep_set_class(&srbm_mutex, &amdgpu_srbm_lock_key); - lockdep_set_class(&grbm_idx_mutex, &amdgpu_grbm_lock_key); - lockdep_set_class(&mmio_idx_lock, &amdgpu_mmio_lock_key); - + lockdep_set_class(&locks->reset_lock, &amdgpu_reset_lock_key); + lockdep_set_class(&locks->srbm_mutex, &amdgpu_srbm_lock_key); + lockdep_set_class(&locks->grbm_idx_mutex, &amdgpu_grbm_lock_key); + lockdep_set_class(&locks->mmio_idx_lock, &amdgpu_mmio_lock_key); /* * Take locks in the correct order to train lockdep. * This establishes the dependency chain. */ /* Level 1: Global userq scheduler mutex (outermost) */ - mutex_lock(&userq_sch_mutex); + mutex_lock(&locks->userq_sch_mutex); /* Level 2: Per-context userq mutex */ - mutex_lock(&userq_mutex); - + mutex_lock(&locks->userq_mutex); /* Level 3: MMU notifier lock */ - mutex_lock(¬ifier_lock); - + mutex_lock(&locks->notifier_lock); /* Level 4: VRAM allocator lock */ - mutex_lock(&vram_lock); - + mutex_lock(&locks->vram_lock); /* Level 5: Reset domain semaphore */ down_read(&reset_domain->sem); /* Level 6: Reset control lock */ - mutex_lock(&reset_ctl.reset_lock); - + mutex_lock(&locks->reset_lock); /* * Mark potential memory reclaim boundary. * GPU operations might trigger memory allocation/reclaim. @@ -157,36 +161,35 @@ int amdgpu_lockdep_init(void) fs_reclaim_acquire(GFP_KERNEL); /* Level 7: SRBM register access */ - mutex_lock(&srbm_mutex); - + mutex_lock(&locks->srbm_mutex); /* Level 8: GRBM index access */ - mutex_lock(&grbm_idx_mutex); + mutex_lock(&locks->grbm_idx_mutex); /* Level 9: MMIO index access (innermost lock, spinlock) */ - spin_lock_irqsave(&mmio_idx_lock, flags); - + spin_lock_irqsave(&locks->mmio_idx_lock, flags); /* * All locks acquired in order. * Lockdep has now learned the valid dependency chain. */ /* Release in reverse order */ - spin_unlock_irqrestore(&mmio_idx_lock, flags); - mutex_unlock(&grbm_idx_mutex); - mutex_unlock(&srbm_mutex); - + spin_unlock_irqrestore(&locks->mmio_idx_lock, flags); + mutex_unlock(&locks->grbm_idx_mutex); + mutex_unlock(&locks->srbm_mutex); fs_reclaim_release(GFP_KERNEL); - mutex_unlock(&reset_ctl.reset_lock); + mutex_unlock(&locks->reset_lock); up_read(&reset_domain->sem); - mutex_unlock(&vram_lock); - mutex_unlock(¬ifier_lock); - mutex_unlock(&userq_mutex); - mutex_unlock(&userq_sch_mutex); + + mutex_unlock(&locks->vram_lock); + mutex_unlock(&locks->notifier_lock); + mutex_unlock(&locks->userq_mutex); + mutex_unlock(&locks->userq_sch_mutex); /* Cleanup */ amdgpu_reset_put_reset_domain(reset_domain); + kfree(locks); pr_info("AMDGPU: Lockdep annotations initialized (9 lock levels)\n"); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index fdd06a17520a..1aae49f4df49 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -302,12 +302,14 @@ struct mes_suspend_gang_input { uint64_t gang_context_addr; uint64_t suspend_fence_addr; uint32_t suspend_fence_value; + uint32_t doorbell_offset; }; struct mes_resume_gang_input { uint32_t xcc_id; bool resume_all_gangs; uint64_t gang_context_addr; + uint32_t doorbell_offset; }; struct mes_reset_queue_input { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index f98bfba59a2c..718937777e53 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -276,10 +276,12 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev, goto error_free; } - r = amdgpu_bo_pin(*bo_ptr, domain); - if (r) { - dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); - goto error_unreserve; + if (free) { + r = amdgpu_bo_pin(*bo_ptr, domain); + if (r) { + dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); + goto error_unreserve; + } } r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); @@ -302,7 +304,8 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev, return 0; error_unpin: - amdgpu_bo_unpin(*bo_ptr); + if (free) + amdgpu_bo_unpin(*bo_ptr); error_unreserve: amdgpu_bo_unreserve(*bo_ptr); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 96e1b72b9e1c..e0c0d7872e45 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -275,6 +275,7 @@ static int psp_early_init(struct amdgpu_ip_block *ip_block) psp->boot_time_tmr = false; break; case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): psp_v15_0_0_set_psp_funcs(psp); psp->boot_time_tmr = false; break; @@ -3475,7 +3476,9 @@ static int psp_load_non_psp_fw(struct psp_context *psp) amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 0) || amdgpu_ip_version(adev, MP0_HWIP, 0) == - IP_VERSION(15, 0, 8)) && + IP_VERSION(15, 0, 8) || + amdgpu_ip_version(adev, MP0_HWIP, 0) == + IP_VERSION(15, 0, 9)) && (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 2740de94e93c..eb8bbfc7e6d9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -208,9 +208,10 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity, void *cpu_addr; uint64_t flags; int r; + const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT); BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < - AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); + GTT_MAX_PAGES * AMDGPU_GPU_PAGES_IN_CPU_PAGE * 8); if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) return -EINVAL; @@ -230,7 +231,7 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity, offset = mm_cur->start & ~PAGE_MASK; num_pages = PFN_UP(*size + offset); - num_pages = min_t(uint32_t, num_pages, AMDGPU_GTT_MAX_TRANSFER_SIZE); + num_pages = min_t(uint32_t, num_pages, GTT_MAX_PAGES); *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); @@ -515,6 +516,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, if (new_mem->mem_type == TTM_PL_TT || new_mem->mem_type == AMDGPU_PL_PREEMPT) { + if (old_mem && (old_mem->mem_type == TTM_PL_TT || + old_mem->mem_type == AMDGPU_PL_PREEMPT)) { + r = ttm_bo_wait_ctx(bo, ctx); + if (r) + return r; + + amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); + } + r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; @@ -549,6 +559,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, ttm_bo_assign_mem(bo, new_mem); return 0; } + if ((old_mem->mem_type == TTM_PL_TT || + old_mem->mem_type == AMDGPU_PL_PREEMPT) && + (new_mem->mem_type == TTM_PL_TT || + new_mem->mem_type == AMDGPU_PL_PREEMPT)) { + amdgpu_bo_move_notify(bo, evict, new_mem); + ttm_resource_free(bo, &bo->resource); + ttm_bo_assign_mem(bo, new_mem); + return 0; + } if (old_mem->mem_type == AMDGPU_PL_GDS || old_mem->mem_type == AMDGPU_PL_GWS || @@ -2015,6 +2034,7 @@ static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr, u32 num_gart_windows) { int i, r, num_pages; + const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT); r = drm_sched_entity_init(&entity->base, prio, scheds, num_schedulers, NULL); if (r) @@ -2027,7 +2047,7 @@ static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr, if (num_gart_windows == 0) return 0; - num_pages = num_gart_windows * AMDGPU_GTT_MAX_TRANSFER_SIZE; + num_pages = num_gart_windows * GTT_MAX_PAGES; r = amdgpu_gtt_mgr_alloc_entries(mgr, &entity->gart_node, num_pages, DRM_MM_INSERT_BEST); if (r) { @@ -2038,7 +2058,7 @@ static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr, for (i = 0; i < num_gart_windows; i++) { entity->gart_window_offs[i] = amdgpu_gtt_node_to_byte_offset(&entity->gart_node) + - i * AMDGPU_GTT_MAX_TRANSFER_SIZE * PAGE_SIZE; + i * GTT_MAX_PAGES * PAGE_SIZE; } return 0; @@ -2100,18 +2120,23 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) /* Change the size here instead of the init above so only lpfn is affected */ amdgpu_ttm_disable_buffer_funcs(adev); #ifdef CONFIG_64BIT -#ifdef CONFIG_X86 - if (adev->gmc.xgmi.connected_to_cpu) - adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, - adev->gmc.visible_vram_size); - - else if (adev->gmc.is_app_apu) + if (adev->gmc.xgmi.connected_to_cpu) { + void *kaddr = devm_memremap(adev->dev, adev->gmc.aper_base, + adev->gmc.visible_vram_size, + MEMREMAP_WB); + if (IS_ERR(kaddr)) + return PTR_ERR(kaddr); + adev->mman.aper_base_kaddr = (__force void __iomem *)kaddr; + } else if (adev->gmc.is_app_apu) { DRM_DEBUG_DRIVER( "No need to ioremap when real vram size is 0\n"); - else -#endif - adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, - adev->gmc.visible_vram_size); + } else { + adev->mman.aper_base_kaddr = devm_ioremap_wc(adev->dev, + adev->gmc.aper_base, + adev->gmc.visible_vram_size); + if (!adev->mman.aper_base_kaddr) + return -ENOMEM; + } #endif amdgpu_ttm_init_vram_resv_regions(adev); @@ -2228,8 +2253,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) */ void amdgpu_ttm_fini(struct amdgpu_device *adev) { - int idx; - if (!adev->mman.initialized) return; @@ -2252,14 +2275,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_FW_VRAM_USAGE); amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_DRV_VRAM_USAGE); - if (drm_dev_enter(adev_to_drm(adev), &idx)) { - - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); - adev->mman.aper_base_kaddr = NULL; - - drm_dev_exit(idx); - } + adev->mman.aper_base_kaddr = NULL; if (!adev->gmc.is_app_apu) amdgpu_vram_mgr_fini(adev); @@ -2668,12 +2684,22 @@ void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev, return; } - /* Navi1x's workaround requires us to limit to a single SDMA sched - * for ttm. - */ hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; - adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ? - 1 : n; + + /* + * Allow using multiple SDMA schedulers only on GPUs where + * we are allowed to do concurrent VM flushes. + * This consideration is necessary because all GART windows + * are mapped in VMID 0 (the kernel VMID) so each buffer + * entity would flush VMID 0 concurrently. + * + * Also consider the SDMA invalidation workaround on + * Navi 1x GPUs, which also prevents us from using + * multiple SDMA engines on VMID 0 at the same time. + */ + adev->mman.num_buffer_funcs_scheds = + (adev->vm_manager.concurrent_flush && + !hub->sdma_invalidation_workaround) ? n : 1; } #if defined(CONFIG_DEBUG_FS) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index 2d72fa217274..b5d938b31383 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -39,7 +39,7 @@ #define AMDGPU_PL_MMIO_REMAP (TTM_PL_PRIV + 5) #define __AMDGPU_PL_NUM (TTM_PL_PRIV + 6) -#define AMDGPU_GTT_MAX_TRANSFER_SIZE 1024 +#define AMDGPU_GTT_MAX_TRANSFER_SIZE (1ULL << 22) extern const struct attribute_group amdgpu_vram_mgr_attr_group; extern const struct attribute_group amdgpu_gtt_mgr_attr_group; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 91554e7c092c..572f2949cb64 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -523,6 +523,15 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que amdgpu_userq_cleanup(queue); mutex_unlock(&uq_mgr->userq_mutex); + /* + * A failed unmap means MES could not remove the hung queue and is now + * unresponsive. Recover the GPU here so the wedged MES does not fail + * the next, unrelated queue submission and trigger a reset attributed + * to an innocent workload. + */ + if (r) + queue_work(adev->reset_domain->wq, &uq_mgr->reset_work); + cancel_delayed_work_sync(&queue->hang_detect_work); uq_funcs->mqd_destroy(queue); queue->userq_mgr = NULL; @@ -680,8 +689,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) /* Update VM owner at userq submit-time for page-fault attribution. */ amdgpu_vm_set_task_info(&fpriv->vm); - r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, - GFP_KERNEL)); + r = xa_insert_irq(&adev->userq_doorbell_xa, index, queue, + GFP_KERNEL); if (r) goto clean_mqd; @@ -1367,16 +1376,19 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev) /* TODO: We probably need a new lock for the queue state */ xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { - if (queue->state != AMDGPU_USERQ_STATE_MAPPED) - continue; - - userq_funcs = adev->userq_funcs[queue->queue_type]; - userq_funcs->unmap(queue); - /* just mark all queues as hung at this point. - * if unmap succeeds, we could map again - * in amdgpu_userq_post_reset() if vram is not lost + if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { + userq_funcs = adev->userq_funcs[queue->queue_type]; + userq_funcs->unmap(queue); + /* just mark all queues as hung at this point. + * if unmap succeeds, we could map again + * in amdgpu_userq_post_reset() if vram is not lost + */ + queue->state = AMDGPU_USERQ_STATE_HUNG; + } + /* Force-complete any pending fence regardless of queue state so + * that eviction/suspend and queue teardown waiters don't block + * forever on a fence that will never signal after the reset. */ - queue->state = AMDGPU_USERQ_STATE_HUNG; amdgpu_userq_fence_driver_force_completion(queue); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 3a3bc0d370fa..23383ac5323f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -135,7 +135,7 @@ MODULE_FIRMWARE(FIRMWARE_VEGA12); MODULE_FIRMWARE(FIRMWARE_VEGA20); static void amdgpu_uvd_idle_work_handler(struct work_struct *work); -static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo); +static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *abo); static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev, uint32_t size, @@ -158,7 +158,7 @@ static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev, amdgpu_bo_kunmap(bo); amdgpu_bo_unpin(bo); amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); - amdgpu_uvd_force_into_uvd_segment(bo); + amdgpu_uvd_force_into_vcpu_segment(bo); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) goto err; @@ -188,6 +188,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) const struct common_firmware_header *hdr; unsigned int family_id; int i, j, r; + u32 vcpu_bo_domain; INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler); @@ -319,12 +320,20 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); + /* UVD 5.0 and newer HW can use 64 bit addressing. */ + adev->uvd.address_64_bit = + !amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0); + + vcpu_bo_domain = AMDGPU_GEM_DOMAIN_VRAM; + if (adev->uvd.address_64_bit) + vcpu_bo_domain |= AMDGPU_GEM_DOMAIN_GTT; + for (j = 0; j < adev->uvd.num_uvd_inst; j++) { if (adev->uvd.harvest_config & (1 << j)) continue; + r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + vcpu_bo_domain, &adev->uvd.inst[j].vcpu_bo, &adev->uvd.inst[j].gpu_addr, &adev->uvd.inst[j].cpu_addr); @@ -339,10 +348,6 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) adev->uvd.filp[i] = NULL; } - /* from uvd v5.0 HW addressing capacity increased to 64 bits */ - if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0)) - adev->uvd.address_64_bit = true; - r = amdgpu_uvd_create_msg_bo_helper(adev, 128 << 10, &adev->uvd.ib_bo); if (r) return r; @@ -545,6 +550,24 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) } } +static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_bo *vcpu_bo = adev->uvd.inst[0].vcpu_bo; + struct amdgpu_res_cursor vcpu_cur; + + amdgpu_res_first(vcpu_bo->tbo.resource, 0, + amdgpu_bo_size(vcpu_bo), &vcpu_cur); + + bo->placement.num_placement = 1; + bo->placement.placement = &bo->placements[0]; + bo->placements[0].fpfn = ALIGN_DOWN(vcpu_cur.start, SZ_256M) >> PAGE_SHIFT; + bo->placements[0].lpfn = bo->placements[0].fpfn + (SZ_256M >> PAGE_SHIFT); + bo->placements[0].mem_type = vcpu_bo->tbo.resource->mem_type; + if (bo->placements[0].mem_type == TTM_PL_VRAM) + bo->placements[0].flags |= TTM_PL_FLAG_CONTIGUOUS; +} + static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo) { int i; @@ -595,13 +618,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx) if (!ctx->parser->adev->uvd.address_64_bit) { /* check if it's a message or feedback command */ cmd = amdgpu_ib_get_value(ctx->ib, ctx->idx) >> 1; - if (cmd == 0x0 || cmd == 0x3) { - /* yes, force it into VRAM */ - uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM; - - amdgpu_bo_placement_from_domain(bo, domain); - } - amdgpu_uvd_force_into_uvd_segment(bo); + if (cmd == 0x0 || cmd == 0x3) + amdgpu_uvd_force_into_vcpu_segment(bo); + else + amdgpu_uvd_force_into_uvd_segment(bo); r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx); } @@ -635,6 +655,14 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, unsigned int image_size, tmp, min_dpb_size, num_dpb_buffer; unsigned int min_ctx_size = ~0; + /* Reject invalid dimensions to prevent division by zero */ + if (width < 16 || height < 16) { + dev_WARN_ONCE(adev->dev, 1, + "Invalid UVD decoding dimensions (%dx%d)!\n", + width, height); + return -EINVAL; + } + image_size = width * height; image_size += image_size / 2; image_size = ALIGN(image_size, 1024); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index efdebd9c0a1f..eef3c9853a5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -877,9 +877,20 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, goto out; } - *size = amdgpu_ib_get_value(ib, idx + 8) * - amdgpu_ib_get_value(ib, idx + 10) * - 8 * 3 / 2; + uint32_t width, height; + width = amdgpu_ib_get_value(ib, idx + 8); + height = amdgpu_ib_get_value(ib, idx + 10); + + if (width == 0 || height == 0 || + width > 4096 || height > 2304) { + DRM_ERROR("invalid VCE image size: %ux%u\n", + width, height); + r = -EINVAL; + goto out; + } + + *size = width * height * 8 * 3 / 2; + break; case 0x04000001: /* config extension */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 7d51880b4860..f224d33c4bc4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -142,7 +142,7 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm) static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists) { INIT_LIST_HEAD(&lists->evicted); - INIT_LIST_HEAD(&lists->moved); + INIT_LIST_HEAD(&lists->needs_update); INIT_LIST_HEAD(&lists->idle); } @@ -211,14 +211,14 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) amdgpu_vm_bo_unlock_lists(vm_bo); } /** - * amdgpu_vm_bo_moved - vm_bo is moved + * amdgpu_vm_bo_needs_update - vm_bo needs pagetable update * - * @vm_bo: vm_bo which is moved + * @vm_bo: vm_bo which is out of date * - * State for vm_bo objects meaning the underlying BO was moved but the new - * location not yet reflected in the page tables. + * State for vm_bo objects meaning the underlying BO had mapping changes (move, PRT bind/unbind) + * but the new location is not yet reflected in the page tables. */ -static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) +static void amdgpu_vm_bo_needs_update(struct amdgpu_vm_bo_base *vm_bo) { struct amdgpu_vm_bo_status *lists; struct amdgpu_bo *bo = vm_bo->bo; @@ -232,8 +232,7 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) vm_bo->moved = false; list_move(&vm_bo->vm_status, &lists->idle); } else { - vm_bo->moved = true; - list_move(&vm_bo->vm_status, &lists->moved); + list_move(&vm_bo->vm_status, &lists->needs_update); } amdgpu_vm_bo_unlock_lists(vm_bo); } @@ -274,14 +273,14 @@ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm) */ amdgpu_vm_assert_locked(vm); list_for_each_entry_safe(vm_bo, tmp, &vm->kernel.idle, vm_status) - amdgpu_vm_bo_moved(vm_bo); + amdgpu_vm_bo_needs_update(vm_bo); list_for_each_entry_safe(vm_bo, tmp, &vm->always_valid.idle, vm_status) - amdgpu_vm_bo_moved(vm_bo); + amdgpu_vm_bo_needs_update(vm_bo); spin_lock(&vm->individual_lock); list_for_each_entry_safe(vm_bo, tmp, &vm->individual.idle, vm_status) { vm_bo->moved = true; - list_move(&vm_bo->vm_status, &vm->individual.moved); + list_move(&vm_bo->vm_status, &vm->individual.needs_update); } spin_unlock(&vm->individual_lock); } @@ -436,7 +435,7 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, */ if (bo->preferred_domains & amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) - amdgpu_vm_bo_moved(base); + amdgpu_vm_bo_needs_update(base); else amdgpu_vm_bo_evicted(base); } @@ -608,7 +607,8 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, return r; vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo)); - amdgpu_vm_bo_moved(bo_base); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); } /* @@ -625,7 +625,8 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (r) return r; - amdgpu_vm_bo_moved(bo_base); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); } if (!ticket) @@ -645,7 +646,8 @@ restart: if (r) return r; - amdgpu_vm_bo_moved(bo_base); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); /* It's a bit inefficient to always jump back to the start, but * we would need to re-structure the KFD for properly fixing @@ -853,12 +855,10 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, job->oa_size); } - if (vm_flush_needed || pasid_mapping_needed || cleaner_shader_needed) { - amdgpu_fence_emit(ring, job->hw_vm_fence, 0); - fence = &job->hw_vm_fence->base; - /* get a ref for the job */ - dma_fence_get(fence); - } + amdgpu_fence_emit(ring, job->hw_vm_fence, 0); + fence = &job->hw_vm_fence->base; + /* get a ref for the job */ + dma_fence_get(fence); if (vm_flush_needed) { mutex_lock(&id_mgr->lock); @@ -979,7 +979,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, amdgpu_vm_assert_locked(vm); - if (list_empty(&vm->kernel.moved)) + if (list_empty(&vm->kernel.needs_update)) return 0; if (!drm_dev_enter(adev_to_drm(adev), &idx)) @@ -995,7 +995,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (r) goto error; - list_for_each_entry(entry, &vm->kernel.moved, vm_status) { + list_for_each_entry(entry, &vm->kernel.needs_update, vm_status) { /* vm_flush_needed after updating moved PDEs */ flush_tlb_needed |= entry->moved; @@ -1011,7 +1011,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (flush_tlb_needed) atomic64_inc(&vm->tlb_seq); - list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status) + list_for_each_entry_safe(entry, tmp, &vm->kernel.needs_update, + vm_status) amdgpu_vm_bo_idle(entry); error: @@ -1615,7 +1616,7 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, bool clear, unlock; int r; - list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved, + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.needs_update, base.vm_status) { /* Per VM BOs never need to bo cleared in the page tables */ r = amdgpu_vm_bo_update(adev, bo_va, false); @@ -1624,8 +1625,8 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, } spin_lock(&vm->individual_lock); - while (!list_empty(&vm->individual.moved)) { - bo_va = list_first_entry(&vm->individual.moved, + while (!list_empty(&vm->individual.needs_update)) { + bo_va = list_first_entry(&vm->individual.needs_update, typeof(*bo_va), base.vm_status); bo = bo_va->base.bo; resv = bo->tbo.base.resv; @@ -1786,7 +1787,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, amdgpu_vm_prt_get(adev); if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved) - amdgpu_vm_bo_moved(&bo_va->base); + amdgpu_vm_bo_needs_update(&bo_va->base); trace_amdgpu_vm_bo_map(bo_va, mapping); } @@ -2095,7 +2096,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, if (amdgpu_vm_is_bo_always_valid(vm, bo) && !before->bo_va->base.moved) - amdgpu_vm_bo_moved(&before->bo_va->base); + amdgpu_vm_bo_needs_update(&before->bo_va->base); } else { kfree(before); } @@ -2110,7 +2111,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, if (amdgpu_vm_is_bo_always_valid(vm, bo) && !after->bo_va->base.moved) - amdgpu_vm_bo_moved(&after->bo_va->base); + amdgpu_vm_bo_needs_update(&after->bo_va->base); } else { kfree(after); } @@ -2284,7 +2285,8 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted) if (bo_base->moved) continue; - amdgpu_vm_bo_moved(bo_base); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); } } @@ -2456,19 +2458,6 @@ static void amdgpu_vm_destroy_task_info(struct kref *kref) kfree(ti); } -static inline struct amdgpu_vm * -amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid) -{ - struct amdgpu_vm *vm; - unsigned long flags; - - xa_lock_irqsave(&adev->vm_manager.pasids, flags); - vm = xa_load(&adev->vm_manager.pasids, pasid); - xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); - - return vm; -} - /** * amdgpu_vm_put_task_info - reference down the vm task_info ptr * @@ -2515,8 +2504,16 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm) struct amdgpu_task_info * amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid) { - return amdgpu_vm_get_task_info_vm( - amdgpu_vm_get_vm_from_pasid(adev, pasid)); + struct amdgpu_task_info *ti; + struct amdgpu_vm *vm; + unsigned long flags; + + xa_lock_irqsave(&adev->vm_manager.pasids, flags); + vm = xa_load(&adev->vm_manager.pasids, pasid); + ti = amdgpu_vm_get_task_info_vm(vm); + xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); + + return ti; } static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm) @@ -2920,47 +2917,56 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) } /** - * amdgpu_vm_lock_by_pasid - return an amdgpu_vm and its root bo from a pasid, if possible. + * amdgpu_vm_lock_by_pasid - look up a VM by PASID and lock its root PD * @adev: amdgpu device pointer - * @root: root BO of the VM * @pasid: PASID of the VM - * The caller needs to unreserve and unref the root bo on success. + * @exec: drm_exec context to lock the root PD in + * + * Must be called from within a drm_exec_until_all_locked() loop; the caller + * runs drm_exec_retry_on_contention() afterwards. The drm_exec context holds + * a reference on the root BO until it is finalised. + * + * Return: the VM on success, or NULL if the PASID has no VM, the VM is being + * torn down, or locking the root PD failed. */ struct amdgpu_vm *amdgpu_vm_lock_by_pasid(struct amdgpu_device *adev, - struct amdgpu_bo **root, u32 pasid) + u32 pasid, struct drm_exec *exec) { unsigned long irqflags; + struct amdgpu_bo *root; struct amdgpu_vm *vm; int r; xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); vm = xa_load(&adev->vm_manager.pasids, pasid); - *root = vm ? amdgpu_bo_ref(vm->root.bo) : NULL; + root = vm ? amdgpu_bo_ref(vm->root.bo) : NULL; xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); - if (!*root) + if (!root) return NULL; - r = amdgpu_bo_reserve(*root, true); - if (r) - goto error_unref; + r = drm_exec_lock_obj(exec, &root->tbo.base); + if (r) { + amdgpu_bo_unref(&root); + return NULL; + } /* Double check that the VM still exists */ xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); vm = xa_load(&adev->vm_manager.pasids, pasid); - if (vm && vm->root.bo != *root) + if (vm && vm->root.bo != root) vm = NULL; xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); - if (!vm) - goto error_unlock; + if (!vm) { + drm_exec_unlock_obj(exec, &root->tbo.base); + amdgpu_bo_unref(&root); + return NULL; + } - return vm; -error_unlock: - amdgpu_bo_unreserve(*root); + /* The drm_exec context holds its own reference on the root BO. */ + amdgpu_bo_unref(&root); -error_unref: - amdgpu_bo_unref(root); - return NULL; + return vm; } /** @@ -2982,33 +2988,51 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, uint64_t ts, bool write_fault) { bool is_compute_context = false; - struct amdgpu_bo *root; + struct drm_exec exec; uint64_t value, flags; struct amdgpu_vm *vm; int r; - vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid); - if (!vm) + drm_exec_init(&exec, 0, 1); + drm_exec_until_all_locked(&exec) { + vm = amdgpu_vm_lock_by_pasid(adev, pasid, &exec); + drm_exec_retry_on_contention(&exec); + if (!vm) + break; + } + if (!vm) { + drm_exec_fini(&exec); return false; + } is_compute_context = vm->is_compute_context; if (is_compute_context) { - /* Unreserve root since svm_range_restore_pages might try to reserve it. */ - /* TODO: rework svm_range_restore_pages so that this isn't necessary. */ - amdgpu_bo_unreserve(root); + __label__ drm_exec_retry; + + /* Release the root PD lock since svm_range_restore_pages + * might try to take it. + * TODO: rework svm_range_restore_pages so that this isn't + * necessary. + */ + drm_exec_fini(&exec); if (!svm_range_restore_pages(adev, pasid, vmid, - node_id, addr >> PAGE_SHIFT, ts, write_fault)) { - amdgpu_bo_unref(&root); + node_id, addr >> PAGE_SHIFT, ts, write_fault)) return true; - } - amdgpu_bo_unref(&root); /* Re-acquire the VM lock, could be that the VM was freed in between. */ - vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid); - if (!vm) + drm_exec_init(&exec, 0, 1); + drm_exec_until_all_locked(&exec) { + vm = amdgpu_vm_lock_by_pasid(adev, pasid, &exec); + drm_exec_retry_on_contention(&exec); + if (!vm) + break; + } + if (!vm) { + drm_exec_fini(&exec); return false; + } } addr /= AMDGPU_GPU_PAGE_SIZE; @@ -3032,7 +3056,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, value = 0; } - r = dma_resv_reserve_fences(root->tbo.base.resv, 1); + r = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); if (r) { pr_debug("failed %d to reserve fence slot\n", r); goto error_unlock; @@ -3046,12 +3070,10 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, r = amdgpu_vm_update_pdes(adev, vm, true); error_unlock: - amdgpu_bo_unreserve(root); + drm_exec_fini(&exec); if (r < 0) dev_err(adev->dev, "Can't handle page fault (%d)\n", r); - amdgpu_bo_unref(&root); - return false; } @@ -3075,7 +3097,7 @@ static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m, id = 0; seq_puts(m, "\tMoved BOs:\n"); - list_for_each_entry(base, &lists->moved, vm_status) { + list_for_each_entry(base, &lists->needs_update, vm_status) { if (!base->bo) continue; @@ -3084,7 +3106,7 @@ static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m, id = 0; seq_puts(m, "\tIdle BOs:\n"); - list_for_each_entry(base, &lists->moved, vm_status) { + list_for_each_entry(base, &lists->needs_update, vm_status) { if (!base->bo) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 3695299f1a03..5822836fa4a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -212,7 +212,8 @@ struct amdgpu_vm_bo_base { * protected by vm BO being reserved */ bool shared; - /* protected by the BO being reserved */ + /* if the BO was moved and all mappings are invalid + * protected by the BO being reserved */ bool moved; }; @@ -220,14 +221,14 @@ struct amdgpu_vm_bo_base { * The following status lists contain amdgpu_vm_bo_base objects for * either PD/PTs, per VM BOs or BOs with individual resv object. * - * The state transits are: evicted -> moved -> idle + * The state transits are: evicted -> needs_update -> idle */ struct amdgpu_vm_bo_status { /* BOs evicted which need to move into place again */ struct list_head evicted; - /* BOs which moved but new location hasn't been updated in the PDs/PTs */ - struct list_head moved; + /* BOs whose mappings changed but PDs/PTs haven't been updated */ + struct list_head needs_update; /* BOs done with the state machine and need no further action */ struct list_head idle; @@ -592,7 +593,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, bool write_fault); struct amdgpu_vm *amdgpu_vm_lock_by_pasid(struct amdgpu_device *adev, - struct amdgpu_bo **root, u32 pasid); + u32 pasid, struct drm_exec *exec); void amdgpu_vm_set_task_info(struct amdgpu_vm *vm); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 0780c5e5de4f..a9961d504833 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4022,7 +4022,7 @@ static void gfx_v10_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -5350,6 +5350,15 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev) gfx_v10_0_get_tcc_info(adev); adev->gfx.config.pa_sc_tile_steering_override = gfx_v10_0_init_pa_sc_tile_steering_override(adev); + /* Program DB_RING_CONTROL for multiple GFX pipes + * Default power up value is 1. + * Possible values: + * 0 - split occlusion counters between gfx pipes + * 1 - all occlusion counters to pipe 0 + * 2 - all occlusion counters to pipe 1 + */ + WREG32_FIELD15(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, + (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ @@ -8658,7 +8667,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -8693,7 +8702,7 @@ static void gfx_v10_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -8726,9 +8735,9 @@ static void gfx_v10_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -8776,9 +8785,6 @@ static void gfx_v10_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index f856b0cf5bec..3b12eb27a253 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -133,6 +133,14 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_pfp.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_me.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_mec.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_rlc.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_pfp.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_me.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_mec.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_rlc.bin"); static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), @@ -546,7 +554,7 @@ static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -1128,6 +1136,8 @@ static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->gfx.config.max_hw_contexts = 8; adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; adev->gfx.config.sc_prim_fifo_size_backend = 0x100; @@ -1612,6 +1622,8 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->gfx.me.num_me = 1; adev->gfx.me.num_pipe_per_me = 1; adev->gfx.me.num_queue_per_pipe = 2; @@ -3085,7 +3097,9 @@ static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) || amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) || - amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6)) + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6) || + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 0) || + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 1)) bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); else @@ -5758,6 +5772,8 @@ static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); break; default: @@ -5798,6 +5814,8 @@ static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): if (!enable) amdgpu_gfx_off_ctrl(adev, false); @@ -5834,6 +5852,8 @@ static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): gfx_v11_0_update_gfx_clock_gating(adev, state == AMD_CG_STATE_GATE); break; @@ -5997,7 +6017,7 @@ static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -6032,7 +6052,7 @@ static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -6065,9 +6085,9 @@ static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -6121,9 +6141,6 @@ static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | @@ -6510,25 +6527,33 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { u32 doorbell_offset = entry->src_data[0]; - u8 me_id, pipe_id, queue_id; - struct amdgpu_ring *ring; - int i; DRM_DEBUG("IH: CP EOP\n"); - if (adev->enable_mes && doorbell_offset) { - amdgpu_userq_process_fence_irq(adev, doorbell_offset); - } else { - me_id = (entry->ring_id & 0x0c) >> 2; - pipe_id = (entry->ring_id & 0x03) >> 0; - queue_id = (entry->ring_id & 0x70) >> 4; + if (!adev->gfx.disable_kq) { + u8 me_id = (entry->ring_id & 0x0c) >> 2; + u8 pipe_id = (entry->ring_id & 0x03) >> 0; + u8 queue_id = (entry->ring_id & 0x70) >> 4; + struct amdgpu_ring *ring; + int i; switch (me_id) { case 0: - if (pipe_id == 0) - amdgpu_fence_process(&adev->gfx.gfx_ring[0]); - else - amdgpu_fence_process(&adev->gfx.gfx_ring[1]); + /* + * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, + * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). + * Require a strict (me,pipe,queue) match so userq gfx + * EOPs fall through to amdgpu_userq_process_fence_irq(). + */ + for (i = 0; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; + if ((ring->me == me_id) && + (ring->pipe == pipe_id) && + (ring->queue == queue_id)) { + amdgpu_fence_process(ring); + return 0; + } + } break; case 1: case 2: @@ -6540,13 +6565,20 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, */ if ((ring->me == me_id) && (ring->pipe == pipe_id) && - (ring->queue == queue_id)) + (ring->queue == queue_id)) { amdgpu_fence_process(ring); + return 0; + } } break; + default: + break; } } + if (adev->enable_mes && doorbell_offset) + amdgpu_userq_process_fence_irq(adev, doorbell_offset); + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index f66293fc675e..da668a8d6abd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -440,7 +440,7 @@ static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -3519,10 +3519,19 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev) gfx_v12_0_cp_gfx_enable(adev, true); } - if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) + if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) { r = amdgpu_mes_kiq_hw_init(adev, 0); - else + /* + * With MES, GFX KIQ ring is owned by the MES and is never + * initialized/used directly by the driver, so it must + * not be left flagged as ready. mes_v12_0_hw_init() clears + * but clear here if MES init fails + */ + if (r) + adev->gfx.kiq[0].ring.sched.ready = false; + } else { r = gfx_v12_0_kiq_resume(adev); + } if (r) return r; @@ -4493,7 +4502,7 @@ static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, control |= ib->length_dw | (vmid << 24); amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -4512,7 +4521,7 @@ static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring, u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -4543,9 +4552,9 @@ static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -4593,9 +4602,6 @@ static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | @@ -4838,25 +4844,33 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { u32 doorbell_offset = entry->src_data[0]; - u8 me_id, pipe_id, queue_id; - struct amdgpu_ring *ring; - int i; DRM_DEBUG("IH: CP EOP\n"); - if (adev->enable_mes && doorbell_offset) { - amdgpu_userq_process_fence_irq(adev, doorbell_offset); - } else { - me_id = (entry->ring_id & 0x0c) >> 2; - pipe_id = (entry->ring_id & 0x03) >> 0; - queue_id = (entry->ring_id & 0x70) >> 4; + if (!adev->gfx.disable_kq) { + u8 me_id = (entry->ring_id & 0x0c) >> 2; + u8 pipe_id = (entry->ring_id & 0x03) >> 0; + u8 queue_id = (entry->ring_id & 0x70) >> 4; + struct amdgpu_ring *ring; + int i; switch (me_id) { case 0: - if (pipe_id == 0) - amdgpu_fence_process(&adev->gfx.gfx_ring[0]); - else - amdgpu_fence_process(&adev->gfx.gfx_ring[1]); + /* + * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, + * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). + * Require a strict (me,pipe,queue) match so userq gfx + * EOPs fall through to amdgpu_userq_process_fence_irq(). + */ + for (i = 0; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; + if ((ring->me == me_id) && + (ring->pipe == pipe_id) && + (ring->queue == queue_id)) { + amdgpu_fence_process(ring); + return 0; + } + } break; case 1: case 2: @@ -4868,13 +4882,20 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev, */ if ((ring->me == me_id) && (ring->pipe == pipe_id) && - (ring->queue == queue_id)) + (ring->queue == queue_id)) { amdgpu_fence_process(ring); + return 0; + } } break; + default: + break; } } + if (adev->enable_mes && doorbell_offset) + amdgpu_userq_process_fence_irq(adev, doorbell_offset); + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index 61c3577f829f..e7e9f11b9754 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -248,7 +248,7 @@ static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, PACKET3_WAIT_REG_MEM__FUNCTION(3))); /* equal */ if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -2547,10 +2547,19 @@ static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id); - if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) + if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) { r = amdgpu_mes_kiq_hw_init(adev, xcc_id); - else + /* + * With MES, GFX KIQ ring is owned by the MES and is never + * initialized/used directly by the driver, so it must + * not be left flagged as ready. mes_v12_0_hw_init() clears + * but clear here if MES init fails + */ + if (r) + adev->gfx.kiq[xcc_id].ring.sched.ready = false; + } else { r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id); + } if (r) return r; @@ -3433,7 +3442,7 @@ static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -3466,9 +3475,9 @@ static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -3515,9 +3524,6 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1))); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 130196859ff3..70ba81e6b4d4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -6256,9 +6256,6 @@ static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned int flags) { - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 47721d0c3781..3370f542e990 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1183,7 +1183,7 @@ static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -4071,6 +4071,41 @@ err_priv_inst: return r; } +static void gfx_v9_0_deactivate_kcq_hqd(struct amdgpu_device *adev) +{ + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); + for (int i = 0; i < adev->gfx.num_compute_rings; i++) { + u32 tmp; + struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; + + mutex_lock(&adev->srbm_mutex); + soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); + /* disable the queue if it's active */ + if (tmp & CP_HQD_ACTIVE__ACTIVE_MASK) { + int j; + + WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); + for (j = 0; j < adev->usec_timeout; j++) { + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); + if (!(tmp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + udelay(1); + } + if (j == AMDGPU_MAX_USEC_TIMEOUT) { + DRM_DEBUG("comp_%u_%u_%u dequeue request failed.\n", + ring->me, ring->pipe, ring->queue); + /* Manual disable if dequeue request times out */ + WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0); + } + WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0); + } + soc15_grbm_select(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); + } + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); +} + static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; @@ -4095,6 +4130,10 @@ static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) return 0; } + if ((adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev) && + amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_MODE2) + gfx_v9_0_deactivate_kcq_hqd(adev); + /* Use deinitialize sequence from CAIL when unbinding device from driver, * otherwise KIQ is hanging when binding back */ @@ -5435,7 +5474,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5531,7 +5570,7 @@ static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5572,9 +5611,9 @@ static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index 510266ba0c38..2a36647b975a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -405,7 +405,7 @@ static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -2944,7 +2944,7 @@ static void gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -2978,9 +2978,9 @@ static void gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -3040,9 +3040,6 @@ static void gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 8eb9847d9e1e..c40d9c467204 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -606,6 +606,8 @@ static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->gfxhub.funcs = &gfxhub_v11_5_0_funcs; break; default: @@ -781,6 +783,8 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); /* diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c index 95b3f4e55ec3..699c274d357e 100644 --- a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c @@ -790,7 +790,7 @@ static void ih_v6_1_set_interrupt_funcs(struct amdgpu_device *adev) const struct amdgpu_ip_block_version ih_v6_1_ip_block = { .type = AMD_IP_BLOCK_TYPE_IH, .major = 6, - .minor = 0, + .minor = 1, .rev = 0, .funcs = &ih_v6_1_ip_funcs, }; diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c b/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c index f5927c3553ce..05b164f38c97 100644 --- a/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c @@ -43,6 +43,8 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_2_imu.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_3_imu.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_imu.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_6_imu.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_imu.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_imu.bin"); static int imu_v11_0_init_microcode(struct amdgpu_device *adev) { diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c index 0c746580de11..d8204fbc198d 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c @@ -1010,7 +1010,7 @@ void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count) static bool jpeg_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - bool ret = false; + bool ret = true; int i, j; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c index 250316704dfa..ae3afc7ab326 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c @@ -657,7 +657,7 @@ static void jpeg_v5_0_1_dec_ring_set_wptr(struct amdgpu_ring *ring) static bool jpeg_v5_0_1_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - bool ret = false; + bool ret = true; int i, j; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index ac6d4f277336..1b071a3de173 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -60,6 +60,10 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_4_mes_2.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_mes1.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_6_mes_2.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_6_mes1.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_mes_2.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_mes1.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_mes_2.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_mes1.bin"); static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block); static int mes_v11_0_hw_fini(struct amdgpu_ip_block *ip_block); @@ -559,6 +563,7 @@ static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes, mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v11_0_submit_pkt_and_poll_completion(mes, &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt), @@ -578,6 +583,7 @@ static int mes_v11_0_resume_gang(struct amdgpu_mes *mes, mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v11_0_submit_pkt_and_poll_completion(mes, &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt), diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index 7453fb11289e..b6cbc25e1ab4 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -592,6 +592,7 @@ static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes, mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE, &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt), @@ -611,6 +612,7 @@ static int mes_v12_0_resume_gang(struct amdgpu_mes *mes, mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE, &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt), diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index 8a90ad5a51b8..e13535d94c51 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -484,6 +484,7 @@ static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes, mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset; /* Suspend gang is handled by master MES */ return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE, @@ -504,6 +505,7 @@ static int mes_v12_1_resume_gang(struct amdgpu_mes *mes, mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset; /* Resume gang is handled by master MES */ return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c index 2a8582e87f2b..2a4d91368ac6 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c @@ -33,6 +33,8 @@ MODULE_FIRMWARE("amdgpu/psp_15_0_0_toc.bin"); MODULE_FIRMWARE("amdgpu/psp_15_0_0_ta.bin"); +MODULE_FIRMWARE("amdgpu/psp_15_0_9_toc.bin"); +MODULE_FIRMWARE("amdgpu/psp_15_0_9_ta.bin"); static int psp_v15_0_0_init_microcode(struct psp_context *psp) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index 88428b88e00f..8652928861ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -457,7 +457,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 /* write the fence */ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -467,7 +467,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 addr += 4; amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index fa02907217e0..b809942b1eb7 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -527,7 +527,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -538,7 +538,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index f6ecbc524c9b..87c1e29fd298 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -377,7 +377,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -388,7 +388,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index d894b7599c18..d7537888e60c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -361,7 +361,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -372,7 +372,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index f154b68dda70..49c57a38151b 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -363,7 +363,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -374,7 +374,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c index cd9668605a50..b06001f6b536 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c @@ -331,7 +331,7 @@ static void sdma_v7_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -342,7 +342,7 @@ static void sdma_v7_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index 963659deeaff..e0b80abcd075 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -406,6 +406,7 @@ soc21_asic_reset_method(struct amdgpu_device *adev) case IP_VERSION(14, 0, 4): case IP_VERSION(14, 0, 5): case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): return AMD_RESET_METHOD_MODE2; default: if (amdgpu_dpm_is_baco_supported(adev)) @@ -838,6 +839,60 @@ static int soc21_common_early_init(struct amdgpu_ip_block *ip_block) adev->pg_flags = 0; adev->external_rev_id = adev->rev_id + 0xd0; break; + case IP_VERSION(11, 7, 0): + adev->cg_flags = AMD_CG_SUPPORT_VCN_MGCG | + AMD_CG_SUPPORT_JPEG_MGCG | + AMD_CG_SUPPORT_GFX_CGCG | + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_FGCG | + AMD_CG_SUPPORT_REPEATER_FGCG | + AMD_CG_SUPPORT_GFX_PERF_CLK | + AMD_CG_SUPPORT_GFX_3D_CGCG | + AMD_CG_SUPPORT_GFX_3D_CGLS | + AMD_CG_SUPPORT_MC_MGCG | + AMD_CG_SUPPORT_MC_LS | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD | + AMD_CG_SUPPORT_ATHUB_MGCG | + AMD_CG_SUPPORT_ATHUB_LS | + AMD_CG_SUPPORT_IH_CG | + AMD_CG_SUPPORT_BIF_MGCG | + AMD_CG_SUPPORT_BIF_LS; + adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG | + AMD_PG_SUPPORT_VCN | + AMD_PG_SUPPORT_JPEG | + AMD_PG_SUPPORT_GFX_PG; + adev->external_rev_id = adev->rev_id + 0xF; + break; + case IP_VERSION(11, 7, 1): + adev->cg_flags = AMD_CG_SUPPORT_VCN_MGCG | + AMD_CG_SUPPORT_JPEG_MGCG | + AMD_CG_SUPPORT_GFX_CGCG | + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_FGCG | + AMD_CG_SUPPORT_REPEATER_FGCG | + AMD_CG_SUPPORT_GFX_PERF_CLK | + AMD_CG_SUPPORT_GFX_3D_CGCG | + AMD_CG_SUPPORT_GFX_3D_CGLS | + AMD_CG_SUPPORT_MC_MGCG | + AMD_CG_SUPPORT_MC_LS | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD | + AMD_CG_SUPPORT_ATHUB_MGCG | + AMD_CG_SUPPORT_ATHUB_LS | + AMD_CG_SUPPORT_IH_CG | + AMD_CG_SUPPORT_BIF_MGCG | + AMD_CG_SUPPORT_BIF_LS; + adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG | + AMD_PG_SUPPORT_VCN | + AMD_PG_SUPPORT_JPEG | + AMD_PG_SUPPORT_GFX_PG; + adev->external_rev_id = adev->rev_id + 0x40; + break; default: /* FIXME: not supported yet */ return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c index 265db9331d0b..9dce30d2bb8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc24.c +++ b/drivers/gpu/drm/amd/amdgpu/soc24.c @@ -496,8 +496,36 @@ static int soc24_common_suspend(struct amdgpu_ip_block *ip_block) return soc24_common_hw_fini(ip_block); } +static bool soc24_need_reset_on_resume(struct amdgpu_device *adev) +{ + u32 sol_reg1, sol_reg2; + + /* Will reset for the following suspend abort cases. + * 1) Only reset dGPU side. + * 2) S3 suspend got aborted and TOS is active. + * As for dGPU suspend abort cases the SOL value + * will be kept as zero at this resume point. + */ + if (!(adev->flags & AMD_IS_APU) && adev->in_s3) { + sol_reg1 = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81); + msleep(100); + sol_reg2 = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81); + + return (sol_reg1 != sol_reg2); + } + + return false; +} + static int soc24_common_resume(struct amdgpu_ip_block *ip_block) { + struct amdgpu_device *adev = ip_block->adev; + + if (soc24_need_reset_on_resume(adev)) { + dev_info(adev->dev, "S3 suspend aborted, resetting..."); + soc24_asic_reset(adev); + } + return soc24_common_hw_init(ip_block); } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index ff7269bafae8..894780669f9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -1927,14 +1927,17 @@ out: #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003 /* return the offset in ib if id is found, -1 otherwise */ -static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start) +static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start, uint32_t *length) { int i; uint32_t len; for (i = start; (len = amdgpu_ib_get_value(ib, i)) >= 8; i += len / 4) { - if (amdgpu_ib_get_value(ib, i + 1) == id) + if (amdgpu_ib_get_value(ib, i + 1) == id) { + if (length) + *length = len; return i; + } } return -1; } @@ -1944,14 +1947,14 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, struct amdgpu_ib *ib) { struct amdgpu_ring *ring = amdgpu_job_ring(job); - uint32_t val; + uint32_t val, len; int idx = 0, sidx; /* The first instance can decode anything */ if (!ring->me) return 0; - while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx)) >= 0) { + while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx, &len)) >= 0) { val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */ if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { uint32_t valid_buf_flag = amdgpu_ib_get_value(ib, idx + 6); @@ -1964,12 +1967,12 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, amdgpu_ib_get_value(ib, idx + 8); return vcn_v4_0_dec_msg(p, job, msg_buffer_addr); } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) { - sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx); + sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx, NULL); if (sidx >= 0 && amdgpu_ib_get_value(ib, sidx + 2) == RENCODE_ENCODE_STANDARD_AV1) return vcn_v4_0_limit_sched(p, job); } - idx += amdgpu_ib_get_value(ib, idx) / 4; + idx += len / 4; } return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index a2b100d14425..c7edebd2fd8a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1299,18 +1299,11 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep, return -EINVAL; } - devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr), - GFP_KERNEL); - if (!devices_arr) - return -ENOMEM; + devices_arr = memdup_array_user((void *)args->device_ids_array_ptr, + args->n_devices, sizeof(*devices_arr)); - err = copy_from_user(devices_arr, - (void __user *)args->device_ids_array_ptr, - args->n_devices * sizeof(*devices_arr)); - if (err != 0) { - err = -EFAULT; - goto copy_from_user_failed; - } + if (IS_ERR(devices_arr)) + return PTR_ERR(devices_arr); mutex_lock(&p->mutex); pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle)); @@ -1391,7 +1384,6 @@ get_mem_obj_from_handle_failed: map_memory_to_gpu_failed: sync_memory_failed: mutex_unlock(&p->mutex); -copy_from_user_failed: kfree(devices_arr); return err; @@ -1416,18 +1408,11 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep, return -EINVAL; } - devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr), - GFP_KERNEL); - if (!devices_arr) - return -ENOMEM; + devices_arr = memdup_array_user((void *)args->device_ids_array_ptr, + args->n_devices, sizeof(*devices_arr)); - err = copy_from_user(devices_arr, - (void __user *)args->device_ids_array_ptr, - args->n_devices * sizeof(*devices_arr)); - if (err != 0) { - err = -EFAULT; - goto copy_from_user_failed; - } + if (IS_ERR(devices_arr)) + return PTR_ERR(devices_arr); mutex_lock(&p->mutex); pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle)); @@ -1493,7 +1478,6 @@ get_mem_obj_from_handle_failed: unmap_memory_from_gpu_failed: sync_memory_failed: mutex_unlock(&p->mutex); -copy_from_user_failed: kfree(devices_arr); return err; } @@ -1562,16 +1546,10 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep, if (!dev) return -EINVAL; - if (args->metadata_ptr) { - metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL); - if (!metadata_buffer) - return -ENOMEM; - } - /* Get dmabuf info from KGD */ r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd, &dmabuf_adev, &args->size, - metadata_buffer, args->metadata_size, + &metadata_buffer, args->metadata_size, &args->metadata_size, &flags, &xcp_id); if (r) goto exit; @@ -1583,7 +1561,7 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep, args->flags = flags; /* Copy metadata buffer to user mode */ - if (metadata_buffer) { + if (metadata_buffer && args->metadata_ptr) { r = copy_to_user((void __user *)args->metadata_ptr, metadata_buffer, args->metadata_size); if (r != 0) @@ -1936,13 +1914,13 @@ static int criu_checkpoint_devices(struct kfd_process *p, struct kfd_criu_device_bucket *device_buckets = NULL; int ret = 0, i; - device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL); + device_buckets = kvcalloc(num_devices, sizeof(*device_buckets), GFP_KERNEL); if (!device_buckets) { ret = -ENOMEM; goto exit; } - device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL); + device_priv = kvcalloc(num_devices, sizeof(*device_priv), GFP_KERNEL); if (!device_priv) { ret = -ENOMEM; goto exit; @@ -2062,17 +2040,17 @@ static int criu_checkpoint_bos(struct kfd_process *p, int ret = 0, pdd_index, bo_index = 0, id; void *mem; - bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL); + bo_buckets = kvcalloc(num_bos, sizeof(*bo_buckets), GFP_KERNEL); if (!bo_buckets) return -ENOMEM; - bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL); + bo_privs = kvcalloc(num_bos, sizeof(*bo_privs), GFP_KERNEL); if (!bo_privs) { ret = -ENOMEM; goto exit; } - files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL); + files = kvcalloc(num_bos, sizeof(struct file *), GFP_KERNEL); if (!files) { ret = -ENOMEM; goto exit; @@ -2359,17 +2337,11 @@ static int criu_restore_devices(struct kfd_process *p, if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size) return -EINVAL; - device_buckets = kmalloc_objs(*device_buckets, args->num_devices); - if (!device_buckets) - return -ENOMEM; + device_buckets = memdup_array_user((void *)args->devices, + args->num_devices, sizeof(*device_buckets)); - ret = copy_from_user(device_buckets, (void __user *)args->devices, - args->num_devices * sizeof(*device_buckets)); - if (ret) { - pr_err("Failed to copy devices buckets from user\n"); - ret = -EFAULT; - goto exit; - } + if (IS_ERR(device_buckets)) + return PTR_ERR(device_buckets); for (i = 0; i < args->num_devices; i++) { struct kfd_node *dev; @@ -2609,7 +2581,7 @@ static int criu_restore_bos(struct kfd_process *p, if (!bo_buckets) return -ENOMEM; - files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL); + files = kvcalloc(args->num_bos, sizeof(struct file *), GFP_KERNEL); if (!files) { ret = -ENOMEM; goto exit; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index f28259d13818..2a239f45fc24 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -1715,6 +1715,8 @@ int kfd_get_gpu_cache_info(struct kfd_node *kdev, struct kfd_gpu_cache_info **pc case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): /* Cacheline size not available in IP discovery for gc11. * kfd_fill_gpu_cache_info_from_gfx_config to hard code it */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 5eb863dec8f4..008a0719fe1f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -169,6 +169,8 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): kfd->device_info.event_interrupt_class = &event_interrupt_class_v11; break; case IP_VERSION(12, 0, 0): @@ -451,6 +453,14 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) gfx_target_version = 110504; f2g = &gfx_v11_kfd2kgd; break; + case IP_VERSION(11, 7, 0): + gfx_target_version = 110700; + f2g = &gfx_v11_kfd2kgd; + break; + case IP_VERSION(11, 7, 1): + gfx_target_version = 110701; + f2g = &gfx_v11_kfd2kgd; + break; case IP_VERSION(12, 0, 0): gfx_target_version = 120000; f2g = &gfx_v12_kfd2kgd; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 2e010c1f8828..67137679a901 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -3103,6 +3103,7 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) { struct device_queue_manager *dqm; + int i; pr_debug("Loading device queue manager\n"); @@ -3231,6 +3232,9 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd); out_free: + for (i = 0; i < KFD_MQD_TYPE_MAX; i++) + kfree(dqm->mqd_mgrs[i]); + kfree(dqm); return NULL; } @@ -3818,6 +3822,12 @@ out: dqm_unlock(dqm); return r; } + +size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, enum kfd_queue_type type) +{ + return dqm->mqd_mgrs[get_mqd_type_from_queue_type(type)]->mqd_size; +} + #if defined(CONFIG_DEBUG_FS) static void seq_reg_dump(struct seq_file *m, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index e0b6a47e7722..641b8ada82a0 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -333,6 +333,8 @@ int debug_refresh_runlist(struct device_queue_manager *dqm); bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, struct qcm_process_device *qpd, int doorbell_off, u32 *queue_format); +size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, + enum kfd_queue_type type); static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c index 05c74887fd6f..fdcf7f2d1b5b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c @@ -153,14 +153,16 @@ void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, u32 inx; mutex_lock(&kfd->doorbell_mutex); + inx = find_first_zero_bit(kfd->doorbell_bitmap, PAGE_SIZE / sizeof(u32)); + if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { + mutex_unlock(&kfd->doorbell_mutex); + return NULL; + } __set_bit(inx, kfd->doorbell_bitmap); mutex_unlock(&kfd->doorbell_mutex); - if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) - return NULL; - *doorbell_off = amdgpu_doorbell_index_on_bar(kfd->adev, kfd->doorbells, inx, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 81900b49d9d5..2e97da597b3d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -107,6 +107,9 @@ static int allocate_event_notification_slot(struct kfd_process *p, } if (restore_id) { + if (*restore_id >= KFD_SIGNAL_EVENT_LIMIT) + return -EINVAL; + id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, GFP_KERNEL); } else { @@ -204,7 +207,7 @@ static int create_signal_event(struct file *devkfd, struct kfd_process *p, ret = allocate_event_notification_slot(p, ev, restore_id); if (ret) { - pr_warn("Signal event wasn't created because out of kernel memory\n"); + pr_warn("Failed to create signal event notification slot\n"); return ret; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 28dc6886c1ff..7cd236c1ff75 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -128,7 +128,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys, enum MIGRATION_COPY_DIR direction, struct dma_fence **mfence) { - const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE; + const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT); struct amdgpu_ring *ring; struct amdgpu_ttm_buffer_entity *entity; u64 gart_s, gart_d; @@ -424,7 +424,7 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange, migrate.dst = migrate.src + npages; scratch = (dma_addr_t *)(migrate.dst + npages); - kfd_smi_event_migration_start(node, p->lead_thread->pid, + kfd_smi_event_migration_start(node, p->lead_thread, start >> PAGE_SHIFT, end >> PAGE_SHIFT, 0, node->id, prange->prefetch_loc, prange->preferred_loc, trigger); @@ -462,7 +462,7 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange, out_free: kvfree(buf); - kfd_smi_event_migration_end(node, p->lead_thread->pid, + kfd_smi_event_migration_end(node, p->lead_thread, start >> PAGE_SHIFT, end >> PAGE_SHIFT, 0, node->id, trigger, r); out: @@ -727,7 +727,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange, migrate.fault_page = fault_page; scratch = (dma_addr_t *)(migrate.dst + npages); - kfd_smi_event_migration_start(node, p->lead_thread->pid, + kfd_smi_event_migration_start(node, p->lead_thread, start >> PAGE_SHIFT, end >> PAGE_SHIFT, node->id, 0, prange->prefetch_loc, prange->preferred_loc, trigger); @@ -766,7 +766,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange, out_free: kvfree(buf); - kfd_smi_event_migration_end(node, p->lead_thread->pid, + kfd_smi_event_migration_end(node, p->lead_thread, start >> PAGE_SHIFT, end >> PAGE_SHIFT, node->id, 0, trigger, r); out: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h index 06ca6235ff1b..63ea70e5c0e6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h @@ -127,6 +127,7 @@ struct mqd_manager { struct mutex mqd_mutex; struct kfd_node *dev; uint32_t mqd_size; + uint32_t ctl_stack_size; }; struct mqd_user_context_save_area_header { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c index 8e8ec266ca46..e034da638c07 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c @@ -203,8 +203,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c index fff137e00b5e..350fcbbba4b2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c @@ -241,8 +241,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c index 8c815f129614..7c387fa90076 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c @@ -216,8 +216,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c index 475589b924e9..431a940f91f3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c @@ -294,8 +294,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 17bfb419b202..be99f0d53b18 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -27,6 +27,7 @@ #include <linux/uaccess.h> #include "kfd_priv.h" #include "kfd_mqd_manager.h" +#include "kfd_topology.h" #include "v9_structs.h" #include "gc/gc_9_0_offset.h" #include "gc/gc_9_0_sh_mask.h" @@ -411,8 +412,11 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd, static int get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size) { struct v9_mqd *m = get_mqd(mqd); + u32 per_xcc_size; - if (check_mul_overflow(m->cp_hqd_cntl_stack_size, NUM_XCC(mm->dev->xcc_mask), ctl_stack_size)) + per_xcc_size = min_t(u32, m->cp_hqd_cntl_stack_size, mm->ctl_stack_size); + + if (check_mul_overflow(per_xcc_size, NUM_XCC(mm->dev->xcc_mask), ctl_stack_size)) return -EINVAL; return 0; @@ -421,13 +425,15 @@ static int get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst) { struct v9_mqd *m; + u32 ctl_stack_copy_size; /* Control stack is located one page after MQD. */ void *ctl_stack = (void *)((uintptr_t)mqd + AMDGPU_GPU_PAGE_SIZE); m = get_mqd(mqd); + ctl_stack_copy_size = min_t(u32, m->cp_hqd_cntl_stack_size, mm->ctl_stack_size); memcpy(mqd_dst, m, sizeof(struct v9_mqd)); - memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size); + memcpy(ctl_stack_dst, ctl_stack, ctl_stack_copy_size); } static void checkpoint_mqd_v9_4_3(struct mqd_manager *mm, @@ -436,15 +442,19 @@ static void checkpoint_mqd_v9_4_3(struct mqd_manager *mm, void *ctl_stack_dst) { struct v9_mqd *m; + u32 ctl_stack_stride; int xcc; uint64_t size = get_mqd(mqd)->cp_mqd_stride_size; + ctl_stack_stride = min_t(u32, get_mqd(mqd)->cp_hqd_cntl_stack_size, + mm->ctl_stack_size); + for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) { m = get_mqd(mqd + size * xcc); checkpoint_mqd(mm, m, (uint8_t *)mqd_dst + sizeof(*m) * xcc, - (uint8_t *)ctl_stack_dst + m->cp_hqd_cntl_stack_size * xcc); + (uint8_t *)ctl_stack_dst + ctl_stack_stride * xcc); } } @@ -998,6 +1008,15 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, mqd->is_occupied = kfd_is_occupied_cp; mqd->get_checkpoint_info = get_checkpoint_info; mqd->mqd_size = sizeof(struct v9_mqd); + if (dev->kfd->cwsr_enabled) { + struct kfd_topology_device *topo_dev; + + topo_dev = kfd_topology_device_by_id(dev->id); + if (topo_dev) + mqd->ctl_stack_size = + ALIGN(topo_dev->node_props.ctl_stack_size, + AMDGPU_GPU_PAGE_SIZE); + } mqd->mqd_stride = mqd_stride_v9; #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c index c86779af323b..60b87a500698 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c @@ -214,8 +214,8 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control |= min(0xA, - order_base_2(q->eop_ring_buffer_size / 4) - 1); + m->cp_hqd_eop_control |= q->eop_ring_buffer_size ? min(0xA, + order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index acd0e41e744c..4d65f94da4d8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -440,7 +440,8 @@ enum kfd_queue_type { KFD_QUEUE_TYPE_SDMA, KFD_QUEUE_TYPE_HIQ, KFD_QUEUE_TYPE_SDMA_XGMI, - KFD_QUEUE_TYPE_SDMA_BY_ENG_ID + KFD_QUEUE_TYPE_SDMA_BY_ENG_ID, + KFD_QUEUE_TYPE_MAX, }; enum kfd_queue_format { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 368283d53077..5fb3679e4e85 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -910,7 +910,7 @@ static void kfd_process_free_id(struct kfd_process *process) { struct kfd_process *primary_process; - if (process->context_id != KFD_CONTEXT_ID_PRIMARY) + if (process->context_id == KFD_CONTEXT_ID_PRIMARY) return; primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm); @@ -1175,10 +1175,12 @@ static void kfd_process_remove_sysfs(struct kfd_process *p) if (!p->kobj) return; - sysfs_remove_file(p->kobj, &p->attr_pasid); - kobject_del(p->kobj_queues); - kobject_put(p->kobj_queues); - p->kobj_queues = NULL; + if (p->kobj_queues) { + sysfs_remove_file(p->kobj, &p->attr_pasid); + kobject_del(p->kobj_queues); + kobject_put(p->kobj_queues); + p->kobj_queues = NULL; + } for (i = 0; i < p->n_pdds; i++) { pdd = p->pdds[i]; @@ -1186,17 +1188,21 @@ static void kfd_process_remove_sysfs(struct kfd_process *p) sysfs_remove_file(p->kobj, &pdd->attr_vram); sysfs_remove_file(p->kobj, &pdd->attr_sdma); - sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); - if (pdd->dev->kfd2kgd->get_cu_occupancy) - sysfs_remove_file(pdd->kobj_stats, - &pdd->attr_cu_occupancy); - kobject_del(pdd->kobj_stats); - kobject_put(pdd->kobj_stats); - pdd->kobj_stats = NULL; + if (pdd->kobj_stats) { + sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); + if (pdd->dev->kfd2kgd->get_cu_occupancy) + sysfs_remove_file(pdd->kobj_stats, + &pdd->attr_cu_occupancy); + kobject_del(pdd->kobj_stats); + kobject_put(pdd->kobj_stats); + pdd->kobj_stats = NULL; + } } for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) { pdd = p->pdds[i]; + if (!pdd->kobj_counters) + continue; sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); @@ -1254,6 +1260,13 @@ static void kfd_process_wq_release(struct work_struct *work) kfd_debugfs_remove_process(p); + /* + * Remove the proc/sysfs entries before destroying PDDs. The removal path + * walks the PDD array and sysfs callbacks dereference PDD fields, so the + * backing data must remain valid until sysfs removal has completed. + */ + kfd_process_remove_sysfs(p); + kfd_process_kunmap_signal_bo(p); kfd_process_free_outstanding_kfd_bos(p); svm_range_list_fini(p); @@ -1267,11 +1280,6 @@ static void kfd_process_wq_release(struct work_struct *work) put_task_struct(p->lead_thread); - /* the last step is removing process entries under /sys - * to indicate the process has been terminated. - */ - kfd_process_remove_sysfs(p); - kfree(p); } @@ -1969,7 +1977,7 @@ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger) struct kfd_process_device *pdd = p->pdds[i]; struct device *dev = pdd->dev->adev->dev; - kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid, + kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread, trigger); r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, @@ -1999,7 +2007,7 @@ fail: if (n_evicted == 0) break; - kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); + kfd_smi_event_queue_restore(pdd->dev, p->lead_thread); if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, &pdd->qpd)) @@ -2022,7 +2030,7 @@ int kfd_process_restore_queues(struct kfd_process *p) struct kfd_process_device *pdd = p->pdds[i]; struct device *dev = pdd->dev->adev->dev; - kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); + kfd_smi_event_queue_restore(pdd->dev, p->lead_thread); r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, &pdd->qpd); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 44e39ce222b7..9ccbc6e5b27b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -265,6 +265,11 @@ static int init_user_queue(struct process_queue_manager *pqm, (*q)->process = pqm->process; if (dev->kfd->shared_resources.enable_mes) { + if (!q_properties->wptr_bo) { + pr_debug("Queue initialization with shared MES requires queue buffers to be initialized\n"); + return -EINVAL; + } + retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, AMDGPU_MES_GANG_CTX_SIZE, AMDGPU_GEM_DOMAIN_GTT, @@ -962,8 +967,8 @@ static void set_queue_properties_from_criu(struct queue_properties *qp, qp->priority = q_data->priority; qp->queue_address = q_data->q_address; qp->queue_size = q_data->q_size; - qp->read_ptr = (uint32_t *) q_data->read_ptr_addr; - qp->write_ptr = (uint32_t *) q_data->write_ptr_addr; + qp->read_ptr = (void __user *)q_data->read_ptr_addr; + qp->write_ptr = (void __user *)q_data->write_ptr_addr; qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address; qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size; qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address; @@ -1003,6 +1008,23 @@ int kfd_criu_restore_queue(struct kfd_process *p, goto exit; } + pdd = kfd_process_device_data_by_id(p, q_data->gpu_id); + if (!pdd) { + pr_err("Failed to get pdd\n"); + ret = -EINVAL; + goto exit; + } + + if (q_data->type >= KFD_QUEUE_TYPE_MAX) { + ret = -EINVAL; + goto exit; + } + + if (q_data->mqd_size != mqd_size_from_queue_type(pdd->dev->dqm, q_data->type)) { + ret = -EINVAL; + goto exit; + } + *priv_data_offset += sizeof(*q_data); q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size; @@ -1025,13 +1047,6 @@ int kfd_criu_restore_queue(struct kfd_process *p, *priv_data_offset += q_extra_data_size; - pdd = kfd_process_device_data_by_id(p, q_data->gpu_id); - if (!pdd) { - pr_err("Failed to get pdd\n"); - ret = -EINVAL; - goto exit; - } - /* * data stored in this order: * mqd[xcc0], mqd[xcc1],..., ctl_stack[xcc0], ctl_stack[xcc1]... diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index 28354a4e5dd5..98a5512b701b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -23,6 +23,7 @@ */ #include <linux/slab.h> +#include <linux/overflow.h> #include "kfd_priv.h" #include "kfd_topology.h" #include "kfd_svm.h" @@ -235,7 +236,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope struct kfd_topology_device *topo_dev; u64 expected_queue_size; struct amdgpu_vm *vm; - u32 total_cwsr_size; + u64 total_cwsr_size; int err; topo_dev = kfd_topology_device_by_id(pdd->dev->id); @@ -308,8 +309,14 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope goto out_err_unreserve; } - total_cwsr_size = (properties->ctx_save_restore_area_size + - topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask); + total_cwsr_size = (u64)properties->ctx_save_restore_area_size + + topo_dev->node_props.debug_memory_size; + if (check_mul_overflow(total_cwsr_size, + NUM_XCC(pdd->dev->xcc_mask), + &total_cwsr_size)) { + err = -EINVAL; + goto out_err_unreserve; + } total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address, @@ -344,7 +351,7 @@ out_err_release: int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties) { struct kfd_topology_device *topo_dev; - u32 total_cwsr_size; + u64 total_cwsr_size; kfd_queue_buffer_put(&properties->wptr_bo); kfd_queue_buffer_put(&properties->rptr_bo); @@ -355,8 +362,12 @@ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_prope topo_dev = kfd_topology_device_by_id(pdd->dev->id); if (!topo_dev) return -EINVAL; - total_cwsr_size = (properties->ctx_save_restore_area_size + - topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask); + total_cwsr_size = (u64)properties->ctx_save_restore_area_size + + topo_dev->node_props.debug_memory_size; + if (check_mul_overflow(total_cwsr_size, + NUM_XCC(pdd->dev->xcc_mask), + &total_cwsr_size)) + return -EINVAL; total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c index dfbde5a571f6..e659cd50eb0b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c @@ -195,17 +195,35 @@ static void add_event_to_kfifo(pid_t pid, struct kfd_node *dev, rcu_read_unlock(); } +/** + * kfd_smi_task_to_pid - Convert task to namespace-aware PID + * @task: task_struct pointer (typically p->lead_thread) + * + * Returns the PID as it appears in the task's own PID namespace. + * For containerized processes, this returns the container-local PID + * (what getpid() returns), not the global host PID. + * + * Returns 0 if task is NULL. + */ +static inline pid_t kfd_smi_task_to_pid(struct task_struct *task) +{ + return task ? task_tgid_nr_ns(task, task_active_pid_ns(task)) : 0; +} + __printf(4, 5) -static void kfd_smi_event_add(pid_t pid, struct kfd_node *dev, +static void kfd_smi_event_add(struct task_struct *task, struct kfd_node *dev, unsigned int event, char *fmt, ...) { char fifo_in[KFD_SMI_EVENT_MSG_SIZE]; int len; va_list args; + pid_t pid; if (list_empty(&dev->smi_clients)) return; + pid = kfd_smi_task_to_pid(task); + len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event); va_start(args, fmt); @@ -234,14 +252,15 @@ void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset, amdgpu_reset_get_desc(reset_context, reset_cause, sizeof(reset_cause)); - kfd_smi_event_add(0, dev, event, KFD_EVENT_FMT_UPDATE_GPU_RESET( + kfd_smi_event_add(NULL, dev, event, KFD_EVENT_FMT_UPDATE_GPU_RESET( dev->reset_seq_num, reset_cause)); } void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev, uint64_t throttle_bitmask) { - kfd_smi_event_add(0, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, KFD_EVENT_FMT_THERMAL_THROTTLING( + kfd_smi_event_add(NULL, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, + KFD_EVENT_FMT_THERMAL_THROTTLING( throttle_bitmask, amdgpu_dpm_get_thermal_throttling_counter(dev->adev))); } @@ -254,67 +273,67 @@ void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid) if (task_info) { /* Report VM faults from user applications, not retry from kernel */ if (task_info->task.pid) - kfd_smi_event_add(task_info->tgid, dev, - KFD_SMI_EVENT_VMFAULT, - KFD_EVENT_FMT_VMFAULT(task_info->task.pid, - task_info->task.comm)); + kfd_smi_event_add(NULL, dev, KFD_SMI_EVENT_VMFAULT, KFD_EVENT_FMT_VMFAULT( + task_info->task.pid, task_info->task.comm)); amdgpu_vm_put_task_info(task_info); } } -void kfd_smi_event_page_fault_start(struct kfd_node *node, pid_t pid, +void kfd_smi_event_page_fault_start(struct kfd_node *node, struct task_struct *task, unsigned long address, bool write_fault, ktime_t ts) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_PAGE_FAULT_START, - KFD_EVENT_FMT_PAGEFAULT_START(ktime_to_ns(ts), pid, - address, node->id, write_fault ? 'W' : 'R')); + kfd_smi_event_add(task, node, KFD_SMI_EVENT_PAGE_FAULT_START, + KFD_EVENT_FMT_PAGEFAULT_START(ktime_to_ns(ts), + kfd_smi_task_to_pid(task), address, node->id, + write_fault ? 'W' : 'R')); } -void kfd_smi_event_page_fault_end(struct kfd_node *node, pid_t pid, +void kfd_smi_event_page_fault_end(struct kfd_node *node, struct task_struct *task, unsigned long address, bool migration) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_PAGE_FAULT_END, + kfd_smi_event_add(task, node, KFD_SMI_EVENT_PAGE_FAULT_END, KFD_EVENT_FMT_PAGEFAULT_END(ktime_get_boottime_ns(), - pid, address, node->id, migration ? 'M' : 'U')); + kfd_smi_task_to_pid(task), address, node->id, + migration ? 'M' : 'U')); } -void kfd_smi_event_migration_start(struct kfd_node *node, pid_t pid, +void kfd_smi_event_migration_start(struct kfd_node *node, struct task_struct *task, unsigned long start, unsigned long end, uint32_t from, uint32_t to, uint32_t prefetch_loc, uint32_t preferred_loc, uint32_t trigger) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_MIGRATE_START, - KFD_EVENT_FMT_MIGRATE_START( - ktime_get_boottime_ns(), pid, start, end - start, - from, to, prefetch_loc, preferred_loc, trigger)); + kfd_smi_event_add(task, node, KFD_SMI_EVENT_MIGRATE_START, + KFD_EVENT_FMT_MIGRATE_START(ktime_get_boottime_ns(), + kfd_smi_task_to_pid(task), start, end - start, from, + to, prefetch_loc, preferred_loc, trigger)); } -void kfd_smi_event_migration_end(struct kfd_node *node, pid_t pid, +void kfd_smi_event_migration_end(struct kfd_node *node, struct task_struct *task, unsigned long start, unsigned long end, uint32_t from, uint32_t to, uint32_t trigger, int error_code) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_MIGRATE_END, - KFD_EVENT_FMT_MIGRATE_END( - ktime_get_boottime_ns(), pid, start, end - start, - from, to, trigger, error_code)); + kfd_smi_event_add(task, node, KFD_SMI_EVENT_MIGRATE_END, + KFD_EVENT_FMT_MIGRATE_END(ktime_get_boottime_ns(), + kfd_smi_task_to_pid(task), start, end - start, from, + to, trigger, error_code)); } -void kfd_smi_event_queue_eviction(struct kfd_node *node, pid_t pid, +void kfd_smi_event_queue_eviction(struct kfd_node *node, struct task_struct *task, uint32_t trigger) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_EVICTION, - KFD_EVENT_FMT_QUEUE_EVICTION(ktime_get_boottime_ns(), pid, - node->id, trigger)); + kfd_smi_event_add(task, node, KFD_SMI_EVENT_QUEUE_EVICTION, + KFD_EVENT_FMT_QUEUE_EVICTION(ktime_get_boottime_ns(), + kfd_smi_task_to_pid(task), node->id, trigger)); } -void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid) +void kfd_smi_event_queue_restore(struct kfd_node *node, struct task_struct *task) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_RESTORE, - KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), pid, - node->id, '0')); + kfd_smi_event_add(task, node, KFD_SMI_EVENT_QUEUE_RESTORE, + KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), + kfd_smi_task_to_pid(task), node->id, '0')); } void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) @@ -329,21 +348,23 @@ void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) for (i = 0; i < p->n_pdds; i++) { struct kfd_process_device *pdd = p->pdds[i]; - kfd_smi_event_add(p->lead_thread->pid, pdd->dev, + kfd_smi_event_add(p->lead_thread, pdd->dev, KFD_SMI_EVENT_QUEUE_RESTORE, KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), - p->lead_thread->pid, pdd->dev->id, 'R')); + kfd_smi_task_to_pid(p->lead_thread), + pdd->dev->id, 'R')); } kfd_unref_process(p); } -void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, pid_t pid, +void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, struct task_struct *task, unsigned long address, unsigned long last, uint32_t trigger) { - kfd_smi_event_add(pid, node, KFD_SMI_EVENT_UNMAP_FROM_GPU, + kfd_smi_event_add(task, node, KFD_SMI_EVENT_UNMAP_FROM_GPU, KFD_EVENT_FMT_UNMAP_FROM_GPU(ktime_get_boottime_ns(), - pid, address, last - address + 1, node->id, trigger)); + kfd_smi_task_to_pid(task), address, + last - address + 1, node->id, trigger)); } void kfd_smi_event_process(struct kfd_process_device *pdd, bool start) @@ -358,7 +379,7 @@ void kfd_smi_event_process(struct kfd_process_device *pdd, bool start) task_info = amdgpu_vm_get_task_info_vm(avm); if (task_info) { - kfd_smi_event_add(task_info->tgid, pdd->dev, + kfd_smi_event_add(NULL, pdd->dev, start ? KFD_SMI_EVENT_PROCESS_START : KFD_SMI_EVENT_PROCESS_END, KFD_EVENT_FMT_PROCESS(task_info->task.pid, @@ -387,7 +408,7 @@ int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd) spin_lock_init(&client->lock); client->events = 0; client->dev = dev; - client->pid = current->tgid; + client->pid = kfd_smi_task_to_pid(current); client->suser = capable(CAP_SYS_ADMIN); spin_lock(&dev->smi_lock); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h index bb4d72b57387..afa93d7cfa7f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h @@ -32,25 +32,25 @@ void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev, uint64_t throttle_bitmask); void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset, struct amdgpu_reset_context *reset_context); -void kfd_smi_event_page_fault_start(struct kfd_node *node, pid_t pid, +void kfd_smi_event_page_fault_start(struct kfd_node *node, struct task_struct *task, unsigned long address, bool write_fault, ktime_t ts); -void kfd_smi_event_page_fault_end(struct kfd_node *node, pid_t pid, +void kfd_smi_event_page_fault_end(struct kfd_node *node, struct task_struct *task, unsigned long address, bool migration); -void kfd_smi_event_migration_start(struct kfd_node *node, pid_t pid, +void kfd_smi_event_migration_start(struct kfd_node *node, struct task_struct *task, unsigned long start, unsigned long end, uint32_t from, uint32_t to, uint32_t prefetch_loc, uint32_t preferred_loc, uint32_t trigger); -void kfd_smi_event_migration_end(struct kfd_node *node, pid_t pid, +void kfd_smi_event_migration_end(struct kfd_node *node, struct task_struct *task, unsigned long start, unsigned long end, uint32_t from, uint32_t to, uint32_t trigger, int error_code); -void kfd_smi_event_queue_eviction(struct kfd_node *node, pid_t pid, +void kfd_smi_event_queue_eviction(struct kfd_node *node, struct task_struct *task, uint32_t trigger); -void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid); +void kfd_smi_event_queue_restore(struct kfd_node *node, struct task_struct *task); void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm); -void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, pid_t pid, +void kfd_smi_event_unmap_from_gpu(struct kfd_node *node, struct task_struct *task, unsigned long address, unsigned long last, uint32_t trigger); void kfd_smi_event_process(struct kfd_process_device *pdd, bool start); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 3841943da5ec..0900bb23349e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1144,7 +1144,7 @@ static int svm_range_split_tail(struct svm_range *prange, uint64_t new_last, struct list_head *insert_list, struct list_head *remap_list) { - unsigned long last_align_down = ALIGN_DOWN(prange->last, 512); + unsigned long last_align_down = ALIGN_DOWN(prange->last + 1, 512); unsigned long start_align = ALIGN(prange->start, 512); bool huge_page_mapping = last_align_down > start_align; struct svm_range *tail = NULL; @@ -1168,7 +1168,7 @@ static int svm_range_split_head(struct svm_range *prange, uint64_t new_start, struct list_head *insert_list, struct list_head *remap_list) { - unsigned long last_align_down = ALIGN_DOWN(prange->last, 512); + unsigned long last_align_down = ALIGN_DOWN(prange->last + 1, 512); unsigned long start_align = ALIGN(prange->start, 512); bool huge_page_mapping = last_align_down > start_align; struct svm_range *head = NULL; @@ -1181,8 +1181,8 @@ svm_range_split_head(struct svm_range *prange, uint64_t new_start, list_add(&head->list, insert_list); - if (huge_page_mapping && head->last + 1 > start_align && - head->last + 1 < last_align_down && (!IS_ALIGNED(head->last, 512))) + if (huge_page_mapping && new_start > start_align && + new_start < last_align_down && !IS_ALIGNED(new_start, 512)) list_add(&head->update_list, remap_list); return 0; @@ -1408,7 +1408,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, return -EINVAL; } - kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid, + kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread, start, last, trigger); r = svm_range_unmap_from_gpu(pdd->dev->adev, @@ -3205,7 +3205,7 @@ retry_write_locked: svms, prange->start, prange->last, best_loc, prange->actual_loc); - kfd_smi_event_page_fault_start(node, p->lead_thread->pid, addr, + kfd_smi_event_page_fault_start(node, p->lead_thread, addr, write_fault, timestamp); /* Align migration range start and size to granularity size */ @@ -3248,7 +3248,7 @@ retry_write_locked: r, svms, start, last); out_migrate_fail: - kfd_smi_event_page_fault_end(node, p->lead_thread->pid, addr, + kfd_smi_event_page_fault_end(node, p->lead_thread, addr, migration); out_unlock_range: @@ -4115,6 +4115,7 @@ exit: list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) { pr_debug("freeing criu_svm_md[]\n\tstart: 0x%llx\n", criu_svm_md->data.start_addr); + list_del(&criu_svm_md->list); kfree(criu_svm_md); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 97ab1e83b318..9c564cd5edee 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -580,89 +580,25 @@ static void schedule_dc_vmin_vmax(struct amdgpu_device *adev, queue_work(system_percpu_wq, &offload_work->work); } -static void dm_vupdate_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_crtc *acrtc; - struct drm_device *drm_dev; - struct drm_vblank_crtc *vblank; - ktime_t frame_duration_ns, previous_timestamp; - unsigned long flags; - int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); - - if (acrtc) { - vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); - drm_dev = acrtc->base.dev; - vblank = drm_crtc_vblank_crtc(&acrtc->base); - previous_timestamp = atomic64_read(&irq_params->previous_timestamp); - frame_duration_ns = vblank->time - previous_timestamp; - - if (frame_duration_ns > 0) { - trace_amdgpu_refresh_rate_track(acrtc->base.index, - frame_duration_ns, - ktime_divns(NSEC_PER_SEC, frame_duration_ns)); - atomic64_set(&irq_params->previous_timestamp, vblank->time); - } - - drm_dbg_vbl(drm_dev, - "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, - vrr_active); - - /* Core vblank handling is done here after end of front-porch in - * vrr mode, as vblank timestamping will give valid results - * while now done after front-porch. This will also deliver - * page-flip completion events that have been queued to us - * if a pageflip happened inside front-porch. - */ - if (vrr_active && acrtc->dm_irq_params.stream) { - bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; - bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; - bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state - == VRR_STATE_ACTIVE_VARIABLE; - - amdgpu_dm_crtc_handle_vblank(acrtc); - - /* BTR processing for pre-DCE12 ASICs */ - if (adev->family < AMDGPU_FAMILY_AI) { - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - mod_freesync_handle_v_update( - adev->dm.freesync_module, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params); - - if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { - schedule_dc_vmin_vmax(adev, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - } - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - } - } - } -} - /** - * dm_crtc_high_irq() - Handles CRTC interrupt - * @interrupt_params: used for determining the CRTC instance + * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling + * @adev: amdgpu device + * @acrtc: the CRTC to service * - * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK - * event handler. + * Performs writeback completion, vblank event handling, CRC processing, VRR BTR + * updates and pageflip completion delivery. + * + * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from + * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank + * from dm_crtc_high_irq(). */ -static void dm_crtc_high_irq(void *interrupt_params) +static void dm_crtc_high_irq_handler(struct amdgpu_device *adev, + struct amdgpu_crtc *acrtc) { - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; struct drm_writeback_job *job; - struct amdgpu_crtc *acrtc; unsigned long flags; int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); - if (!acrtc) - return; + bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0; if (acrtc->wb_conn) { spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags); @@ -699,12 +635,17 @@ static void dm_crtc_high_irq(void *interrupt_params) vrr_active, acrtc->dm_irq_params.active_planes); /** - * Core vblank handling at start of front-porch is only possible - * in non-vrr mode, as only there vblank timestamping will give - * valid results while done in front-porch. Otherwise defer it - * to dm_vupdate_high_irq after end of front-porch. + * Core vblank handling. + * + * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch + * point, which is the correct place to timestamp both VRR and non-VRR + * vblanks. + * + * On DCE this handler runs at the start of front-porch, where only + * non-VRR timestamping is valid; VRR vblank is deferred to + * dm_vupdate_high_irq() after end of front-porch. */ - if (!vrr_active) + if (is_dcn || !vrr_active) amdgpu_dm_crtc_handle_vblank(acrtc); /** @@ -737,18 +678,33 @@ static void dm_crtc_high_irq(void *interrupt_params) } /* - * If there aren't any active_planes then DCH HUBP may be clock-gated. - * In that case, pageflip completion interrupts won't fire and pageflip - * completion events won't get delivered. Prevent this by sending - * pending pageflip events from here if a flip is still pending. + * Deliver pageflip completion events (DCN only). * - * If any planes are enabled, use dm_pflip_high_irq() instead, to - * avoid race conditions between flip programming and completion, - * which could cause too early flip completion events. + * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch + * point. Deliver any pending pageflip completion event from here, + * once HW has consumed the new address (the OTG no longer reports a + * pending flip). + * + * Also handle the case here where there aren't any active planes and + * DCN HUBP may be clock-gated, so the flip-pending status may be + * undefined. */ - if (adev->family >= AMDGPU_FAMILY_RV && - acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && - acrtc->dm_irq_params.active_planes == 0) { + if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && + acrtc->event) { + + if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) { + drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); + acrtc->event = NULL; + drm_crtc_vblank_put(&acrtc->base); + acrtc->pflip_status = AMDGPU_FLIP_NONE; + } + /* + * If the flip is still pending, leave it armed and + * retry on the next vupdate. + */ + } else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && + acrtc->dm_irq_params.active_planes == 0) { + if (acrtc->event) { drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); acrtc->event = NULL; @@ -760,6 +716,104 @@ static void dm_crtc_high_irq(void *interrupt_params) spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } +static void dm_vupdate_high_irq(void *interrupt_params) +{ + struct common_irq_params *irq_params = interrupt_params; + struct amdgpu_device *adev = irq_params->adev; + struct amdgpu_crtc *acrtc; + struct drm_device *drm_dev; + struct drm_vblank_crtc *vblank; + ktime_t frame_duration_ns, previous_timestamp; + unsigned long flags; + int vrr_active; + + acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); + if (!acrtc) + return; + + vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); + drm_dev = acrtc->base.dev; + vblank = drm_crtc_vblank_crtc(&acrtc->base); + previous_timestamp = atomic64_read(&irq_params->previous_timestamp); + frame_duration_ns = vblank->time - previous_timestamp; + + if (frame_duration_ns > 0) { + trace_amdgpu_refresh_rate_track(acrtc->base.index, + frame_duration_ns, + ktime_divns(NSEC_PER_SEC, frame_duration_ns)); + atomic64_set(&irq_params->previous_timestamp, vblank->time); + } + + drm_dbg_vbl(drm_dev, + "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, + vrr_active); + + /* + * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver + * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are + * not used. Run the full handler here. + */ + if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) { + dm_crtc_high_irq_handler(adev, acrtc); + return; + } + + /* DCE only below. */ + + /* Core vblank handling is done here after end of front-porch in + * vrr mode, as vblank timestamping will give valid results + * while now done after front-porch. This will also deliver + * page-flip completion events that have been queued to us + * if a pageflip happened inside front-porch. + */ + if (vrr_active && acrtc->dm_irq_params.stream) { + bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; + bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; + bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state + == VRR_STATE_ACTIVE_VARIABLE; + + amdgpu_dm_crtc_handle_vblank(acrtc); + + /* BTR processing for pre-DCE12 ASICs */ + if (adev->family < AMDGPU_FAMILY_AI) { + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); + mod_freesync_handle_v_update( + adev->dm.freesync_module, + acrtc->dm_irq_params.stream, + &acrtc->dm_irq_params.vrr_params); + + if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { + schedule_dc_vmin_vmax(adev, + acrtc->dm_irq_params.stream, + &acrtc->dm_irq_params.vrr_params.adjust); + } + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); + } + } +} + +/** + * dm_crtc_high_irq() - Handles CRTC interrupt + * @interrupt_params: used for determining the CRTC instance + * + * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler. + * + * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is + * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq(). + */ +static void dm_crtc_high_irq(void *interrupt_params) +{ + struct common_irq_params *irq_params = interrupt_params; + struct amdgpu_device *adev = irq_params->adev; + struct amdgpu_crtc *acrtc; + + acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); + if (!acrtc) + return; + + dm_crtc_high_irq_handler(adev, acrtc); +} + #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) /** * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for @@ -3298,6 +3352,13 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, */ if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis"); + + } else if (acrtc && state->stream_status[i].plane_count != 0) { + /* DCN only needs to toggle VUPDATE_NO_LOCK */ + rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, enable); + if (rc) + drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n", + enable ? "en" : "dis"); } } @@ -4069,6 +4130,8 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps; caps->aux_support = false; + panel_backlight_quirk = drm_get_panel_backlight_quirk(aconnector->drm_edid); + if (caps->ext_caps->bits.oled == 1 /* * || @@ -4081,6 +4144,9 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) caps->aux_support = false; else if (amdgpu_backlight == 1) caps->aux_support = true; + else if (!IS_ERR_OR_NULL(panel_backlight_quirk) && + panel_backlight_quirk->force_pwm) + caps->aux_support = false; if (caps->aux_support) aconnector->dc_link->backlight_control_type = BACKLIGHT_CONTROL_AMD_AUX; @@ -4096,8 +4162,6 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) else caps->aux_min_input_signal = 1; - panel_backlight_quirk = - drm_get_panel_backlight_quirk(aconnector->drm_edid); if (!IS_ERR_OR_NULL(panel_backlight_quirk)) { if (panel_backlight_quirk->min_brightness) { caps->min_input_signal = @@ -4863,38 +4927,6 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev) * for acknowledging and handling. */ - /* Use VSTARTUP interrupt */ - for (i = DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP; - i <= DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP + adev->mode_info.num_crtc - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->crtc_irq); - - if (r) { - drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - /* Use otg vertical line interrupt */ #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) { @@ -4966,37 +4998,6 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev) return -ENOMEM; } - /* Use GRPH_PFLIP interrupt */ - for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT; - i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq); - if (r) { - drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - /* HPD */ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT, &adev->hpd_irq); @@ -5563,11 +5564,11 @@ amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) caps = &dm->backlight_caps[aconnector->bl_idx]; if (get_brightness_range(caps, &min, &max)) { if (power_supply_is_system_supplied() > 0) - props.brightness = DIV_ROUND_CLOSEST((max - min) * caps->ac_level, 100); + props.brightness = DIV_ROUND_CLOSEST(max * caps->ac_level, 100); else - props.brightness = DIV_ROUND_CLOSEST((max - min) * caps->dc_level, 100); + props.brightness = DIV_ROUND_CLOSEST(max * caps->dc_level, 100); /* min is zero, so max needs to be adjusted */ - props.max_brightness = max - min; + props.max_brightness = max; drm_dbg(drm, "Backlight caps: min: %d, max: %d, ac %d, dc %d\n", min, max, caps->ac_level, caps->dc_level); } else @@ -6614,8 +6615,8 @@ static void fill_dc_dirty_rects(struct drm_plane *plane, { struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); struct rect *dirty_rects = flip_addrs->dirty_rects; - u32 num_clips; - struct drm_mode_rect *clips; + u32 num_clips = 0; + struct drm_mode_rect *clips = NULL; bool bb_changed; bool fb_changed; u32 i = 0; @@ -6631,8 +6632,10 @@ static void fill_dc_dirty_rects(struct drm_plane *plane, if (new_plane_state->rotation != DRM_MODE_ROTATE_0) goto ffu; - num_clips = drm_plane_get_damage_clips_count(new_plane_state); - clips = drm_plane_get_damage_clips(new_plane_state); + if (!new_plane_state->ignore_damage_clips) { + num_clips = drm_plane_get_damage_clips_count(new_plane_state); + clips = drm_plane_get_damage_clips(new_plane_state); + } if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 && is_psr_su))) @@ -9672,21 +9675,9 @@ static void manage_dm_interrupts(struct amdgpu_device *adev, if (acrtc_state) { timing = &acrtc_state->stream->timing; - if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= - IP_VERSION(3, 2, 0) && - !(adev->flags & AMD_IS_APU)) { - /* - * DGPUs NV3x and newer that support idle optimizations - * experience intermittent flip-done timeouts on cursor - * updates. Restore 5s offdelay behavior for now. - * - * Discussion on the issue: - * https://lore.kernel.org/amd-gfx/20260217191632.1243826-1-sysdadmin@m1k.cloud/ - */ - config.offdelay_ms = 5000; - config.disable_immediate = false; - } else if (amdgpu_ip_version(adev, DCE_HWIP, 0) < - IP_VERSION(3, 5, 0)) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) < + IP_VERSION(3, 5, 0) || + !(adev->flags & AMD_IS_APU)) { /* * Older HW and DGPU have issues with instant off; * use a 2 frame offdelay. @@ -9705,14 +9696,22 @@ static void manage_dm_interrupts(struct amdgpu_device *adev, drm_crtc_vblank_on_config(&acrtc->base, &config); - /* Allow RX6xxx, RX7700, RX7800 GPUs to call amdgpu_irq_get.*/ + /* + * Since pflip_high_irq is no longer registered for DCN, grab an + * extra reference to vupdate irq instead to workaround this + * issue: + * https://gitlab.freedesktop.org/drm/amd/-/work_items/3936 + * + * The callbacks to drm_vblank_on/off should really take care of + * this though. + */ switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { case IP_VERSION(3, 0, 0): case IP_VERSION(3, 0, 2): case IP_VERSION(3, 0, 3): case IP_VERSION(3, 2, 0): - if (amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type)) - drm_err(dev, "DM_IRQ: Cannot get pageflip irq!\n"); + if (amdgpu_irq_get(adev, &adev->vupdate_irq, irq_type)) + drm_err(dev, "DM_IRQ: Cannot get vupdate irq!\n"); #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) if (amdgpu_irq_get(adev, &adev->vline0_irq, irq_type)) drm_err(dev, "DM_IRQ: Cannot get vline0 irq!\n"); @@ -9730,8 +9729,8 @@ static void manage_dm_interrupts(struct amdgpu_device *adev, if (amdgpu_irq_put(adev, &adev->vline0_irq, irq_type)) drm_err(dev, "DM_IRQ: Cannot put vline0 irq!\n"); #endif - if (amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type)) - drm_err(dev, "DM_IRQ: Cannot put pageflip irq!\n"); + if (amdgpu_irq_put(adev, &adev->vupdate_irq, irq_type)) + drm_err(dev, "DM_IRQ: Cannot put vupdate irq!\n"); } drm_crtc_vblank_off(&acrtc->base); @@ -9744,6 +9743,10 @@ static void dm_update_pflip_irq_state(struct amdgpu_device *adev, int irq_type = amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id); + /* GRPH_PFLIP is not used on DCN; nothing to reapply. */ + if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) + return; + /** * This reads the current state for the IRQ and force reapplies * the setting to hardware. @@ -10076,9 +10079,13 @@ static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm, struct dm_crtc_state *old_state, struct dm_crtc_state *new_state) { + struct amdgpu_device *adev = drm_to_adev(new_state->base.crtc->dev); bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state); bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state); + /* Only DCE gates vupdate on VRR, keep it enabled for DCN */ + bool vrr_gates_vupdate = amdgpu_ip_version(adev, DCE_HWIP, 0) == 0; + if (!old_vrr_active && new_vrr_active) { /* Transition VRR inactive -> active: * While VRR is active, we must not disable vblank irq, as a @@ -10088,7 +10095,8 @@ static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm, * We also need vupdate irq for the actual core vblank handling * at end of vblank. */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0); + if (vrr_gates_vupdate) + WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0); WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0); drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR off->on: Get vblank ref\n", __func__, new_state->base.crtc->base.id); @@ -10104,7 +10112,8 @@ static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm, /* Transition VRR active -> inactive: * Allow vblank irq disable again for fixed refresh rate. */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0); + if (vrr_gates_vupdate) + WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0); drm_crtc_vblank_put(new_state->base.crtc); drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR on->off: Drop vblank ref\n", __func__, new_state->base.crtc->base.id); @@ -10253,6 +10262,28 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm, } } +static void dm_arm_vblank_event(struct amdgpu_crtc *acrtc, + struct dm_crtc_state *acrtc_state, + bool pflip_update, + bool cursor_update) +{ + assert_spin_locked(&acrtc->base.dev->event_lock); + + if (!acrtc->base.state->event || acrtc_state->active_planes == 0) + return; + + if (pflip_update) { + drm_crtc_vblank_get(&acrtc->base); + WARN_ON(acrtc->pflip_status != AMDGPU_FLIP_NONE); + /* Arm flip completion handling and event delivery after programming. */ + prepare_flip_isr(acrtc); + } else if (cursor_update) { + drm_crtc_vblank_get(&acrtc->base); + acrtc->event = acrtc->base.state->event; + acrtc->base.state->event = NULL; + } +} + static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state, struct drm_device *dev, struct amdgpu_display_manager *dm, @@ -10275,6 +10306,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state, bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state); bool cursor_update = false; bool pflip_present = false; + bool immediate_flip = false; + bool flip_latched_during_prog = false; bool dirty_rects_changed = false; bool updated_planes_and_streams = false; struct { @@ -10439,6 +10472,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state, acrtc_state->update_type == UPDATE_TYPE_FAST && get_mem_type(old_plane_state->fb) == get_mem_type(fb); + immediate_flip |= bundle->flip_addrs[planes_count].flip_immediate; + timestamp_ns = ktime_get_ns(); bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000); bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count]; @@ -10507,39 +10542,24 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state, usleep_range(1000, 1100); } - /** - * Prepare the flip event for the pageflip interrupt to handle. - * - * This only works in the case where we've already turned on the - * appropriate hardware blocks (eg. HUBP) so in the transition case - * from 0 -> n planes we have to skip a hardware generated event - * and rely on sending it from software. - */ - if (acrtc_attach->base.state->event && - acrtc_state->active_planes > 0) { - drm_crtc_vblank_get(pcrtc); - - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - - WARN_ON(acrtc_attach->pflip_status != AMDGPU_FLIP_NONE); - prepare_flip_isr(acrtc_attach); - - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - if (acrtc_state->stream) { if (acrtc_state->freesync_vrr_info_changed) bundle->stream_update.vrr_infopacket = &acrtc_state->stream->vrr_infopacket; } - } else if (cursor_update && acrtc_state->active_planes > 0) { - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - if (acrtc_attach->base.state->event) { - drm_crtc_vblank_get(pcrtc); - acrtc_attach->event = acrtc_attach->base.state->event; - acrtc_attach->base.state->event = NULL; + } + + /* + * DCE depends on a combination of GRPH_FLIP, VLINE0, and VUPDATE for + * event delivery. Only GRPH_FLIP handler can send pflip events, and it + * only fires if HW latched to the flip. Maintain legacy behavior by + * arming event before programming. + */ + if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) == 0) { + scoped_guard(spinlock_irqsave, &pcrtc->dev->event_lock) { + dm_arm_vblank_event(acrtc_attach, acrtc_state, + pflip_present, cursor_update); } - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); } /* Update the planes if changed or disable if we don't have any. */ @@ -10631,6 +10651,115 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state, acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) amdgpu_dm_commit_cursors(state); + /* + * DCN specific vblank handling + * ============================ + * + * With the event_lock held, arm the vblank event, and determine whether + * deliver it immediately, or in VUPDATE_NO_LOCK IRQ (i.e. HW latch + * point) handler. Do this *after* programming so that the IRQ handler + * will not deliver the event before HW laches onto the programmed + * values: + * + * Commit thread IRQ handler HW + * ----------------------------------------------------------------- + * arm_vblank_event() + * vupdate() + * vupdate_handler() + * cook_timestamp() + * # prev flip already latched, + * # so flip_latched == true. + * if event_armed && flip_latched: + * send_vblank_event() + * # sent before latch, **BAD!** + * hw_program() + * vupdate() + * **latch** + * + * There's a consequence of arming after: it's possible for HW to latch + * between start of HW programming and acrtc->event/pflip_status arming. + * When this happens, the IRQ handler will send the event on the next + * immediate latch point, even though HW has already latched. This is + * handled by optimistically checking for HW latch after programming, + * and if latched, send the event immediately: + * + * Commit thread IRQ handler HW + * ----------------------------------------------------------------- + * hw_program() + * vupdate() + * **latch** + * vupdate_handler() + * cook_timestamp() + * # event_armed == false + * # **no event sent!** + * arm_vblank_event() + * if flip_latched: + * **send_vblank_event()** + * disarm_vblank_event() + * + * The IRQ handler is expected to cook the timestamp, but we need to + * cook the timestamp before optimistic sending as well. That's because + * the following sequence is possible: + * + * Commit thread IRQ handler HW + * ----------------------------------------------------------------- + * hw_program() + * arm_vblank_event() + * vupdate() + * **latch** + * if flip_latched: + * # Need cook before send! + * **cook_timestamp()** + * send_vblank_event() + * disarm_vblank_event() + * vupdate_handler() + * cook_timestamp() + * # event_armed == false + * # no event sent! + * + * Cooking twice is OK, since DRM scanout accurate timestamps report A) + * the previous vactive start if currently in vactive, or B) the next + * vactive start if currently in vblank (see &get_vblank_counter). 'A)' + * is what we want for the optimistic send, and for 'B)', we'll cook a + * timestamp no later than the next IRQ handler run. + * + * The more correct fix is to wrap programming and arming with the + * event_lock and thus serializing it with the IRQ handler. However, + * there are various sleep-waits within + * update_planes_and_stream_adapter() that makes spin locking illegal. + * And on full updates, it can take 1-2 frame-times to return (see + * commit_planes_for_stream). + * + * On DCE, GRPH_PFLIP IRQ is used and takes care of this. + */ + if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) { + spin_lock_irqsave(&pcrtc->dev->event_lock, flags); + + if (updated_planes_and_streams) { + flip_latched_during_prog = + !dc_get_flip_pending_on_otg(dm->dc, acrtc_attach->otg_inst); + } + + dm_arm_vblank_event(acrtc_attach, acrtc_state, + pflip_present, cursor_update); + + /* + * Deliver the event immediately on immediate flip, or on a + * update that has already latched. + */ + if ((immediate_flip || flip_latched_during_prog) && + acrtc_attach->pflip_status == AMDGPU_FLIP_SUBMITTED && + acrtc_attach->event) { + drm_crtc_accurate_vblank_count(&acrtc_attach->base); + drm_crtc_send_vblank_event(&acrtc_attach->base, + acrtc_attach->event); + acrtc_attach->event = NULL; + drm_crtc_vblank_put(&acrtc_attach->base); + acrtc_attach->pflip_status = AMDGPU_FLIP_NONE; + } + spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); + } + cleanup: kfree(bundle); } @@ -12096,6 +12225,7 @@ skip_modeset: /* Release extra reference */ if (new_stream) dc_stream_release(new_stream); + new_stream = NULL; /* * We want to do dc stream updates that do not require a @@ -12812,10 +12942,15 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, /* Overlay cursor not supported on HW before DCN * DCN401/420 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions * as previous DCN generations, so enable native mode on DCN401/420 + * + * Always set native cursor mode when the CRTC is disabled, + * to make sure it doesn't cause atomic commits to fail when + * they are trying to disable the CRTC. */ if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1) || amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0) || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1)) { + amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1) || + !dm_crtc_state->base.enable) { *cursor_mode = DM_CURSOR_NATIVE_MODE; return 0; } @@ -12942,13 +13077,11 @@ static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev, struct drm_plane_state *new_plane_state, *old_plane_state; drm_for_each_plane_mask(plane, dev, crtc_state->plane_mask) { - new_plane_state = drm_atomic_get_plane_state(state, plane); - old_plane_state = drm_atomic_get_plane_state(state, plane); + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + old_plane_state = drm_atomic_get_old_plane_state(state, plane); - if (IS_ERR(new_plane_state) || IS_ERR(old_plane_state)) { - drm_err(dev, "Failed to get plane state for plane %s\n", plane->name); - return false; - } + if (!old_plane_state || !new_plane_state) + continue; if (old_plane_state->fb && new_plane_state->fb && get_mem_type(old_plane_state->fb) != get_mem_type(new_plane_state->fb)) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 3dcedaa67ed8..b43cd68cde67 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -274,7 +274,14 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable) drm_crtc_vblank_restore(crtc); } - if (dc_supports_vrr(dm->dc->ctx->dce_version)) { + /* + * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver + * vblank and pageflip completion events, so enable it whenever vblank + * is enabled. On DCE, vupdate is only needed in VRR mode. + */ + if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) { + rc = amdgpu_dm_crtc_set_vupdate_irq(crtc, enable); + } else if (dc_supports_vrr(dm->dc->ctx->dce_version)) { if (enable) { /* vblank irq on -> Only need vupdate irq in vrr mode */ if (amdgpu_dm_crtc_vrr_active(acrtc_state)) @@ -288,36 +295,43 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable) if (rc) return rc; - /* crtc vblank or vstartup interrupt */ - if (enable) { - rc = amdgpu_irq_get(adev, &adev->crtc_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Get crtc_irq ret=%d\n", rc); - } else { - rc = amdgpu_irq_put(adev, &adev->crtc_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Put crtc_irq ret=%d\n", rc); - } - - if (rc) - return rc; - /* - * hubp surface flip interrupt - * - * We have no guarantee that the frontend index maps to the same - * backend index - some even map to more than one. - * - * TODO: Use a different interrupt or check DC itself for the mapping. + * VLINE0 (crtc_irq) and GRPH_PFLIP (pageflip_irq) are only used on + * DCE. On DCN, vblank and pageflip completion are delivered from + * VUPDATE_NO_LOCK (enabled above), so don't touch them here. */ - if (enable) { - rc = amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Get pageflip_irq ret=%d\n", rc); - } else { - rc = amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Put pageflip_irq ret=%d\n", rc); - } + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { + /* crtc vblank or vstartup interrupt */ + if (enable) { + rc = amdgpu_irq_get(adev, &adev->crtc_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Get crtc_irq ret=%d\n", rc); + } else { + rc = amdgpu_irq_put(adev, &adev->crtc_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Put crtc_irq ret=%d\n", rc); + } - if (rc) - return rc; + if (rc) + return rc; + + /* + * hubp surface flip interrupt + * + * We have no guarantee that the frontend index maps to the same + * backend index - some even map to more than one. + * + * TODO: Use a different interrupt or check DC itself for the mapping. + */ + if (enable) { + rc = amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Get pageflip_irq ret=%d\n", rc); + } else { + rc = amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Put pageflip_irq ret=%d\n", rc); + } + + if (rc) + return rc; + } #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) /* crtc vline0 interrupt, only available on DCN+ */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index f257ea91a34d..6be7f6edd0b2 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -95,8 +95,11 @@ static u32 edid_extract_panel_id(struct edid *edid) (u32)EDID_PRODUCT_ID(edid); } -static void apply_edid_quirks(struct drm_device *dev, struct edid *edid, struct dc_edid_caps *edid_caps) +static void apply_edid_quirks(struct dc_link *link, struct edid *edid, + struct dc_edid_caps *edid_caps) { + struct amdgpu_dm_connector *aconnector = link->priv; + struct drm_device *dev = aconnector->base.dev; uint32_t panel_id = edid_extract_panel_id(edid); switch (panel_id) { @@ -126,6 +129,11 @@ static void apply_edid_quirks(struct drm_device *dev, struct edid *edid, struct drm_dbg_driver(dev, "Disabling VSC on monitor with panel id %X\n", panel_id); edid_caps->panel_patch.disable_colorimetry = true; break; + /* Workaround for monitors that get corrupted by the PHY SSC reduction */ + case drm_edid_encode_panel_id('D', 'E', 'L', 0x4147): + drm_dbg_driver(dev, "Skip PHY SSC reduction on panel id %X\n", panel_id); + link->wa_flags.skip_phy_ssc_reduction = true; + break; default: return; } @@ -147,7 +155,6 @@ enum dc_edid_status dm_helpers_parse_edid_caps( { struct amdgpu_dm_connector *aconnector = link->priv; struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; struct edid *edid_buf = edid ? (struct edid *) edid->raw_edid : NULL; struct cea_sad *sads; int sad_count = -1; @@ -188,7 +195,7 @@ enum dc_edid_status dm_helpers_parse_edid_caps( edid_caps->frl_dsc_max_frl_rate, edid_caps->frl_dsc_total_chunk_kbytes); } - apply_edid_quirks(dev, edid_buf, edid_caps); + apply_edid_quirks(link, edid_buf, edid_caps); sad_count = drm_edid_to_sad((struct edid *) edid->raw_edid, &sads); if (sad_count <= 0) @@ -1194,11 +1201,25 @@ enum dc_edid_status dm_helpers_read_local_edid( continue; edid = drm_edid_raw(drm_edid); // FIXME: Get rid of drm_edid_raw() - if (!edid || - edid->extensions >= sizeof(sink->dc_edid.raw_edid) / EDID_LENGTH) + /* + * Use the length of the EDID property blob populated by + * drm_edid_connector_update() above. It reflects the true number + * of EDID blocks, including any HDMI Forum EDID Extension Override + * Data Block (HF-EEODB) count, which the raw byte 0x7e extension + * count can hide (e.g. HDMI 8K sinks). + */ + if (!edid || !connector->edid_blob_ptr || + connector->edid_blob_ptr->length > sizeof(sink->dc_edid.raw_edid)) return EDID_BAD_INPUT; - sink->dc_edid.length = EDID_LENGTH * (edid->extensions + 1); + /* + * FIXME: amdgpu_dm today does not consider the HF-EEODB, which + * may contain additional mode info for sinks. This is a + * workaround until dc_edid is refactored out from DC into + * amdgpu_dm's ownership, allowing amdgpu_dm to use drm_edid + * directly + */ + sink->dc_edid.length = connector->edid_blob_ptr->length; memmove(sink->dc_edid.raw_edid, (uint8_t *)edid, sink->dc_edid.length); /* We don't need the original edid anymore */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index e957657b06c7..c7f8e08feaf4 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1859,6 +1859,7 @@ static const struct drm_plane_funcs dm_plane_funcs = { .atomic_duplicate_state = amdgpu_dm_plane_drm_plane_duplicate_state, .atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state, .format_mod_supported = amdgpu_dm_plane_format_mod_supported, + .format_mod_supported_async = amdgpu_dm_plane_format_mod_supported, #ifdef AMD_PRIVATE_COLOR .atomic_set_property = dm_atomic_plane_set_property, .atomic_get_property = dm_atomic_plane_get_property, diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c index 00c4be7c3aa4..ff47af3854b6 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c @@ -158,7 +158,6 @@ void dcn31_update_clocks(struct clk_mgr *clk_mgr_base, if (new_clocks->zstate_support != DCN_ZSTATE_SUPPORT_DISALLOW && new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) { dcn31_smu_set_zstate_support(clk_mgr, new_clocks->zstate_support); - dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, true); clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; } @@ -184,7 +183,6 @@ void dcn31_update_clocks(struct clk_mgr *clk_mgr_base, if (new_clocks->zstate_support == DCN_ZSTATE_SUPPORT_DISALLOW && new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) { dcn31_smu_set_zstate_support(clk_mgr, DCN_ZSTATE_SUPPORT_DISALLOW); - dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, false); clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c index dd6f11ecb9c9..24f6304011ae 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c @@ -230,7 +230,6 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, if (new_clocks->zstate_support != DCN_ZSTATE_SUPPORT_DISALLOW && new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) { dcn314_smu_set_zstate_support(clk_mgr, new_clocks->zstate_support); - dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, true); clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; } @@ -255,7 +254,6 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, if (new_clocks->zstate_support == DCN_ZSTATE_SUPPORT_DISALLOW && new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) { dcn314_smu_set_zstate_support(clk_mgr, DCN_ZSTATE_SUPPORT_DISALLOW); - dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, false); clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c index 103013e2a0de..a69824e1eb26 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c @@ -419,6 +419,7 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base, if (new_clocks->zstate_support != DCN_ZSTATE_SUPPORT_DISALLOW && new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) { dcn35_smu_set_zstate_support(clk_mgr, new_clocks->zstate_support); + dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, true); clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; } @@ -438,6 +439,7 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base, if (new_clocks->zstate_support == DCN_ZSTATE_SUPPORT_DISALLOW && new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) { dcn35_smu_set_zstate_support(clk_mgr, DCN_ZSTATE_SUPPORT_DISALLOW); + dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, false); clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index bcdbf3471039..e25b94b65dac 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1509,7 +1509,7 @@ static void disable_vbios_mode_if_required( struct dc *dc_create(const struct dc_init_data *init_params) { - struct dc *dc = kzalloc_obj(*dc); + struct dc *dc = kvzalloc_obj(*dc); unsigned int full_pipe_count; if (!dc) @@ -1557,7 +1557,7 @@ struct dc *dc_create(const struct dc_init_data *init_params) destruct_dc: dc_destruct(dc); - kfree(dc); + kvfree(dc); return NULL; } @@ -1606,7 +1606,7 @@ void dc_deinit_callbacks(struct dc *dc) void dc_destroy(struct dc **dc) { dc_destruct(*dc); - kfree(*dc); + kvfree(*dc); *dc = NULL; } @@ -4077,8 +4077,6 @@ static void commit_planes_do_stream_update_sequence(struct dc *dc, { int j; struct block_sequence_state seq_state = { .steps = block_sequence, .num_steps = num_steps }; - struct dsc_config dsc_cfgs[MAX_PIPES]; - struct dsc_optc_config dsc_optc_cfgs[MAX_PIPES]; unsigned int dsc_cfg_index = 0; *num_steps = 0; // Initialize to 0 @@ -4150,11 +4148,13 @@ static void commit_planes_do_stream_update_sequence(struct dc *dc, if (stream_update->dsc_config) if (dsc_cfg_index < MAX_PIPES) { + struct dsc_config dsc_cfg; + struct dsc_optc_config dsc_optc_cfg; + add_link_update_dsc_config_sequence(&seq_state, pipe_ctx, - &dsc_cfgs[dsc_cfg_index], - &dsc_optc_cfgs[dsc_cfg_index]); - dsc_cfg_index++; + &dsc_cfg, + &dsc_optc_cfg); } if (stream_update->mst_bw_update) { @@ -6165,6 +6165,51 @@ void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src) dal_irq_service_ack(dc->res_pool->irqs, src); } +/* + * dc_get_flip_pending_on_otg() - Check if a GRPH_FLIP is still pending on OTG + * + * @dc: display core context @otg_inst: OTG instance to query + * + * Reads the HUBP flip-pending status for the pipe(s) bound to @otg_inst, + * returning true if any of them has not yet latched its programmed surface + * address. + * + * Unlike dc_plane_get_status(), this does not take or mutate a dc_plane_state, + * so it is safe to call from interrupt context without racing a concurrent + * commit that may be updating plane state. + * + * Return: true if a flip is still pending on the OTG, false otherwise. + */ +bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst) +{ + bool flip_pending = false; + int i; + + if (!dc || !dc->current_state) + return false; + + dc_exit_ips_for_hw_access(dc); + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i]; + struct hubp *hubp = pipe_ctx->plane_res.hubp; + + if (!pipe_ctx->plane_state || !pipe_ctx->stream_res.tg) + continue; + + if (pipe_ctx->stream_res.tg->inst != otg_inst) + continue; + + if (hubp && hubp->funcs->hubp_is_flip_pending && + hubp->funcs->hubp_is_flip_pending(hubp)) { + flip_pending = true; + break; + } + } + + return flip_pending; +} + void dc_power_down_on_boot(struct dc *dc) { if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW && diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 82d02ebbd829..1dc85f6b6689 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1815,6 +1815,8 @@ struct dc_scratch_space { bool dp_skip_DID2; bool dp_skip_reset_segment; bool dp_skip_fs_144hz; + /* Some DP bridges don't work with RBR and must use HBR. */ + bool dp_skip_rbr; bool dp_mot_reset_segment; /* Some USB4 docks do not handle turning off MST DSC once it has been enabled. */ bool dpia_mst_dsc_always_on; @@ -2881,6 +2883,7 @@ enum dc_irq_source dc_interrupt_to_irq_source( uint32_t ext_id); bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable); void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src); +bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst); enum dc_irq_source dc_get_hpd_irq_source_at_index( struct dc *dc, uint32_t link_index); diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h index 2076565b1caa..a2b17ed11bdb 100644 --- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h +++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h @@ -46,6 +46,7 @@ DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DCCG_FIFO_ERRDET_STATE, mask_sh),\ DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DCCG_FIFO_ERRDET_OVR_EN, mask_sh),\ DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_CHG_FWD_CORR_DISABLE, mask_sh),\ + DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, RESYNC_FIFO_LEVEL_ADJUST_EN, mask_sh),\ DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_PHASE, mask_sh),\ DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_MODULO, mask_sh),\ DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\ @@ -56,34 +57,24 @@ DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_SRC_SEL, mask_sh),\ DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_EN, mask_sh),\ DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_SRC_SEL, mask_sh),\ - DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_EN, mask_sh),\ - DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_SRC_SEL, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK0_EN, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK1_EN, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_EN, mask_sh),\ - DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_EN, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK0_SRC_SEL, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK1_SRC_SEL, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_SRC_SEL, mask_sh),\ - DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_SRC_SEL, mask_sh),\ DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN, mask_sh),\ DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE2_SRC_SEL, mask_sh),\ - DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE3_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_EN, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_EN, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE2_EN, mask_sh),\ - DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE3_EN, mask_sh),\ DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE0_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE1_SRC_SEL, mask_sh),\ - DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE2_SRC_SEL, mask_sh),\ - DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE3_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE0_EN, mask_sh),\ DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE1_EN, mask_sh),\ - DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE2_EN, mask_sh),\ - DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE3_EN, mask_sh),\ DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 0, mask_sh),\ DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 1, mask_sh),\ DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 2, mask_sh),\ @@ -121,7 +112,6 @@ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GLOBAL_FGCG_REP_CNTL, DCCG_GLOBAL_FGCG_REP_DIS, mask_sh),\ DCCG_SFII(OTG, PIXEL_RATE_CNTL, DP_DTO, ENABLE, 0, mask_sh),\ DCCG_SFII(OTG, PIXEL_RATE_CNTL, DP_DTO, ENABLE, 1, mask_sh),\ @@ -134,7 +124,6 @@ DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK0_EN, mask_sh),\ DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_EN, mask_sh),\ DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_EN, mask_sh),\ - DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK3_EN, mask_sh),\ DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_PHASE, mask_sh),\ DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_MODULO, mask_sh),\ DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_PHASE, mask_sh),\ @@ -147,36 +136,26 @@ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKA_FE_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKB_FE_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKC_FE_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKA_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKB_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKC_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE0_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE1_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE2_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE3_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE0_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE1_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE2_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE3_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE0_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE1_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE2_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE3_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE0_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE1_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE2_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE3_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL4, HDMICHARCLK0_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYA_REFCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYB_REFCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYC_REFCLK_ROOT_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYD_REFCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, mask_sh),\ @@ -184,19 +163,15 @@ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_FE_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_FE_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_FE_ROOT_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_FE_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_ROOT_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_ROOT_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_GATE_DISABLE, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK0_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK1_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK2_ROOT_GATE_DISABLE, mask_sh),\ @@ -208,26 +183,38 @@ DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_CLOCK_ENABLE, mask_sh),\ DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_CLOCK_ENABLE, mask_sh),\ DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_CLOCK_ENABLE, mask_sh),\ - DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_CLOCK_ENABLE, mask_sh),\ DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_FE_EN, mask_sh),\ DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_FE_EN, mask_sh),\ DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_FE_EN, mask_sh),\ - DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_EN, mask_sh),\ DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_FE_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_FE_SRC_SEL, mask_sh),\ - DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_FE_SRC_SEL, mask_sh),\ - DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_SRC_SEL, mask_sh) + DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_FE_SRC_SEL, mask_sh) #define DCCG_MASK_SH_LIST_DCN42(mask_sh) \ DCCG_MASK_SH_LIST_DCN42_COMMON(mask_sh),\ + DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_EN, mask_sh),\ + DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_SRC_SEL, mask_sh),\ DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_EN, mask_sh),\ DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_SRC_SEL, mask_sh),\ DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_PHASE, mask_sh),\ DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_MODULO, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh),\ DCCG_SF(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_PHASE, mask_sh),\ DCCG_SF(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_MODULO, mask_sh),\ - DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE3_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE2_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE3_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE3_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE2_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE3_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_FE_ROOT_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_ROOT_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_ROOT_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYD_REFCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYE_REFCLK_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_FE_ROOT_GATE_DISABLE, mask_sh),\ DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_ROOT_GATE_DISABLE, mask_sh),\ @@ -236,11 +223,22 @@ DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_SRC_SEL, mask_sh),\ + DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_CLOCK_ENABLE, mask_sh),\ + DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_EN, mask_sh),\ + DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_CLOCK_ENABLE, mask_sh),\ DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_FE_EN, mask_sh),\ DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_FE_SRC_SEL, mask_sh),\ - DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, RESYNC_FIFO_LEVEL_ADJUST_EN, mask_sh) + DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE3_SRC_SEL, mask_sh),\ + DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE3_EN, mask_sh),\ + DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE2_SRC_SEL, mask_sh),\ + DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE3_SRC_SEL, mask_sh),\ + DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE2_EN, mask_sh),\ + DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE3_EN, mask_sh),\ + DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_EN, mask_sh),\ + DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_SRC_SEL, mask_sh),\ + DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK3_EN, mask_sh) void dccg42_otg_add_pixel(struct dccg *dccg, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c index 7c293917e6fd..ecb8493ec523 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c @@ -1229,9 +1229,9 @@ static bool get_dp_dto_frequency_100hz( */ modulo_hz = REG_READ(MODULO[inst]); if (modulo_hz) { - temp = div_u64((uint64_t)clock_hz * dp_dto_ref_khz * 10, modulo_hz); - ASSERT(temp / 100 <= 0xFFFFFFFFUL); - *pixel_clk_100hz = (unsigned int)(temp / 100); + temp = clock_hz * dp_dto_ref_khz * 10; + ASSERT(temp <= UINT_MAX * modulo_hz * 100ULL); + *pixel_clk_100hz = div_u64(temp, modulo_hz * 100); } else *pixel_clk_100hz = 0; } else { @@ -1285,13 +1285,12 @@ static bool dcn401_get_dp_dto_frequency_100hz(const struct clock_source *clock_s * - target pix_clk_hz = (DPDTO INTEGER * DPDTO MODULO + DPDTO PHASE) */ temp = (unsigned long long)dp_dto_integer * modulo_hz + phase_hz; - - if (temp / 100 > 0xFFFFFFFFUL) { + if (temp > (UINT_MAX * 100ULL)) { /* pixel rate 100hz should never be this high, if it is, throw an assert and return 0 */ BREAK_TO_DEBUGGER(); *pixel_clk_100hz = 0; } else { - *pixel_clk_100hz = (unsigned int)(temp / 100); + *pixel_clk_100hz = div_u64(temp, 100); } return true; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c index ed407e779c12..2c3a20d35fe9 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c @@ -271,7 +271,6 @@ static void dce110_stream_encoder_dp_set_stream_attribute( bool use_vsc_sdp_for_colorimetry, uint32_t enable_sdp_splitting) { - (void)use_vsc_sdp_for_colorimetry; (void)enable_sdp_splitting; uint32_t h_active_start; uint32_t v_active_start; @@ -334,6 +333,16 @@ static void dce110_stream_encoder_dp_set_stream_attribute( if (REG(DP_MSA_MISC)) misc1 = REG_READ(DP_MSA_MISC); + /* For YCbCr420 and BT2020 Colorimetry Formats, VSC SDP shall be used. + * When MISC1, bit 6, is Set to 1, a Source device uses a VSC SDP to indicate the + * Pixel Encoding/Colorimetry Format and that a Sink device shall ignore MISC1, bit 7, + * and MISC0, bits 7:1 (MISC1, bit 7, and MISC0, bits 7:1, become "don't care"). + */ + if (use_vsc_sdp_for_colorimetry) + misc1 = misc1 | 0x40; + else + misc1 = misc1 & ~0x40; + /* set color depth */ switch (hw_crtc_timing.display_color_depth) { @@ -499,6 +508,10 @@ static void dce110_stream_encoder_dp_set_stream_attribute( hw_crtc_timing.h_addressable + hw_crtc_timing.h_border_right, DP_MSA_VHEIGHT, hw_crtc_timing.v_border_top + hw_crtc_timing.v_addressable + hw_crtc_timing.v_border_bottom); + } else { + /* DCE-only path */ + if (REG(DP_MSA_MISC)) + REG_WRITE(DP_MSA_MISC, misc1); /* MSA_MISC1 */ } } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h index 342c0afe6a94..88d6044904d1 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h @@ -96,7 +96,8 @@ #define SE_COMMON_REG_LIST(id)\ SE_COMMON_REG_LIST_DCE_BASE(id), \ - SRI(AFMT_CNTL, DIG, id) + SRI(AFMT_CNTL, DIG, id), \ + SRI(DP_MSA_MISC, DP, id) #define SE_DCN_REG_LIST(id)\ SE_COMMON_REG_LIST_BASE(id),\ diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c index a3212fd151d1..29dbb5e410d8 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c @@ -623,7 +623,7 @@ static bool detect_dp(struct dc_link *link, link->dpcd_caps.sink_count.bits.SINK_COUNT = 1; /* NUTMEG requires that we use HBR, doesn't work with RBR. */ if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_00001A) - link->preferred_link_setting.link_rate = LINK_RATE_HIGH; + link->wa_flags.dp_skip_rbr = true; } return true; @@ -1164,8 +1164,11 @@ static bool detect_link_and_local_sink(struct dc_link *link, link->link_enc->features.flags.bits.DP_IS_USB_C == 1) { /* if alt mode times out, return false */ - if (!wait_for_entering_dp_alt_mode(link)) + if (!wait_for_entering_dp_alt_mode(link)) { + if (prev_sink) + dc_sink_release(prev_sink); return false; + } } if (!detect_dp(link, &sink_caps, reason)) { diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c index 47abb4066709..1cd17a0272bc 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c @@ -750,8 +750,10 @@ static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_setting if (req_bw > dp_link_bandwidth_kbps(link, &link->verified_link_cap)) return false; - if (link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN) - initial_link_setting.link_rate = link->preferred_link_setting.link_rate; + if (link->wa_flags.dp_skip_rbr) { + initial_link_setting.link_rate = LINK_RATE_HIGH; + current_link_setting.link_rate = LINK_RATE_HIGH; + } /* search for the minimum link setting that: * 1. is supported according to the link training result diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c index b92d4f378d60..97ea22af5d2b 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c @@ -992,6 +992,11 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link( for (i = 0; i < pool->stream_enc_count; i++) { if (!res_ctx->is_stream_enc_acquired[i] && pool->stream_enc[i]) { + /* DP/MST needs a digital encoder; skip analog/no-DP encoders */ + if (dc_is_dp_signal(stream->signal) && + (!pool->stream_enc[i]->funcs || + !pool->stream_enc[i]->funcs->dp_set_stream_attribute)) + continue; /* Store first available for MST second display * in daisy chain use case */ @@ -1014,7 +1019,7 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link( * required for non DP connectors. */ - if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT) + if (j >= 0 && dc_is_dp_signal(stream->signal)) return pool->stream_enc[j]; return NULL; diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c index 7de12b16d7ad..a8241036def2 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c @@ -2142,6 +2142,7 @@ static bool dcn42_resource_construct( dc->config.use_pipe_ctx_sync_logic = true; dc->config.dc_mode_clk_limit_support = false; dc->config.enable_windowed_mpo_odm = true; + dc->config.set_pipe_unlock_order = true; /* Need to ensure DET gets freed before allocating */ /* Use psp mailbox to enable assr */ dc->config.use_assr_psp_message = true; /* dcn42 and afterward always support external panel replay */ diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c index 527d17f29f3b..94e166c0a9b0 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c @@ -22,6 +22,7 @@ #include "dcn35/dcn35_resource.h" #include "dcn321/dcn321_resource.h" #include "dcn401/dcn401_resource.h" +#include "dcn42/dcn42_resource.h" #include "dcn42/dcn42_resource_fpu.h" #include "dcn10/dcn10_ipp.h" @@ -116,6 +117,23 @@ #define regAPG9_APG_DBG_GEN_CONTROL 0x38ae #define regAPG9_APG_DBG_GEN_CONTROL_BASE_IDX 2 +#define regHUBP0_HUBPREQ_DEBUG_DB 0x05f8 +#define regHUBP0_HUBPREQ_DEBUG_DB_BASE_IDX 2 +#define regHUBP0_HUBPREQ_DEBUG 0x05f9 +#define regHUBP0_HUBPREQ_DEBUG_BASE_IDX 2 +#define regHUBP1_HUBPREQ_DEBUG_DB 0x06d4 +#define regHUBP1_HUBPREQ_DEBUG_DB_BASE_IDX 2 +#define regHUBP1_HUBPREQ_DEBUG 0x06d5 +#define regHUBP1_HUBPREQ_DEBUG_BASE_IDX 2 +#define regHUBP2_HUBPREQ_DEBUG_DB 0x07b0 +#define regHUBP2_HUBPREQ_DEBUG_DB_BASE_IDX 2 +#define regHUBP2_HUBPREQ_DEBUG 0x07b1 +#define regHUBP2_HUBPREQ_DEBUG_BASE_IDX 2 +#define regHUBP3_HUBPREQ_DEBUG_DB 0x088c +#define regHUBP3_HUBPREQ_DEBUG_DB_BASE_IDX 2 +#define regHUBP3_HUBPREQ_DEBUG 0x088d +#define regHUBP3_HUBPREQ_DEBUG_BASE_IDX 2 + enum dcn401_clk_src_array_id { DCN401_CLK_SRC_PLL0, DCN401_CLK_SRC_PLL1, @@ -461,7 +479,7 @@ static const struct dcn_optc_mask optc_mask = { OPTC_COMMON_MASK_SH_LIST_DCN42B(_MASK)}; #define hubp_regs_init(id) \ - HUBP_REG_LIST_DCN42B_RI(id) + HUBP_REG_LIST_DCN42_RI(id) static struct dcn_hubp2_registers hubp_regs[4]; @@ -1882,9 +1900,7 @@ static struct resource_funcs dcn42b_res_pool_funcs = { .update_soc_for_wm_a = dcn30_update_soc_for_wm_a, .add_phantom_pipes = dcn32_add_phantom_pipes, .calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes, -#ifdef CONFIG_DRM_AMD_DC_DML21 .prepare_mcache_programming = dcn42b_prepare_mcache_programming, -#endif .build_pipe_pix_clk_params = dcn42b_build_pipe_pix_clk_params, .get_power_profile = dcn401_get_power_profile, .get_vstartup_for_pipe = dcn401_get_vstartup_for_pipe, @@ -2087,6 +2103,7 @@ static bool dcn42b_resource_construct( dc->config.use_pipe_ctx_sync_logic = true; dc->config.dc_mode_clk_limit_support = false; dc->config.enable_windowed_mpo_odm = true; + dc->config.set_pipe_unlock_order = true; /* Need to ensure DET gets freed before allocating */ /* Use psp mailbox to enable assr */ dc->config.use_assr_psp_message = true; /* dcn42 and afterward always support external panel replay */ diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h index 2da3e3c8304a..2824a0e1acc9 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h @@ -344,7 +344,6 @@ * DCCG_SRII(PHASE, DP_DTO, 3), * DCCG_SRII(MODULO, DP_DTO, 3), * SR(DSCCLK3_DTO_PARAM), - * SR(HDMISTREAMCLK_CNTL), * SR(SYMCLKD_CLOCK_ENABLE), * SR(SYMCLKE_CLOCK_ENABLE) */ @@ -360,6 +359,7 @@ SR(PHYBSYMCLK_CLOCK_CNTL), \ SR(PHYCSYMCLK_CLOCK_CNTL), \ SR(DPSTREAMCLK_CNTL), \ + SR(HDMISTREAMCLK_CNTL), \ SR(SYMCLK32_SE_CNTL), \ SR(SYMCLK32_LE_CNTL), \ DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0), \ @@ -542,120 +542,6 @@ SRI_ARR(DC_ABM1_ACE_OFFSET_SLOPE_DATA, ABM, id), \ SRI_ARR(DC_ABM1_ACE_PWL_CNTL, ABM, id) -/* HUBP */ -/* Not in DCN42B: HUBPREQ_DEBUG_DB and HUBPREQ_DEBUG */ -#define HUBP_REG_LIST_DCN42B_RI(id) \ - SRI_ARR(DCN_DMDATA_VM_CNTL, HUBPREQ, id), \ - SRI_ARR(FLIP_PARAMETERS_3, HUBPREQ, id), \ - SRI_ARR(FLIP_PARAMETERS_4, HUBPREQ, id), \ - SRI_ARR(FLIP_PARAMETERS_5, HUBPREQ, id), \ - SRI_ARR(FLIP_PARAMETERS_6, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_5, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_6, HUBPREQ, id), \ - HUBP_REG_LIST_DCN_VM_RI(id), \ - SRI_ARR(PREFETCH_SETTINGS, HUBPREQ, id), \ - SRI_ARR(PREFETCH_SETTINGS_C, HUBPREQ, id), \ - SRI_ARR(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, HUBPREQ, id), \ - SRI_ARR(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, HUBPREQ, id), \ - SRI_ARR(CURSOR_SETTINGS, HUBPREQ, id), \ - SRI_ARR(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id), \ - SRI_ARR(CURSOR_SURFACE_ADDRESS, CURSOR0_, id), \ - SRI_ARR(CURSOR_SIZE, CURSOR0_, id), \ - SRI_ARR(CURSOR_CONTROL, CURSOR0_, id), \ - SRI_ARR(CURSOR_POSITION, CURSOR0_, id), \ - SRI_ARR(CURSOR_HOT_SPOT, CURSOR0_, id), \ - SRI_ARR(CURSOR_DST_OFFSET, CURSOR0_, id), \ - SRI_ARR(DMDATA_ADDRESS_HIGH, CURSOR0_, id), \ - SRI_ARR(DMDATA_ADDRESS_LOW, CURSOR0_, id), \ - SRI_ARR(DMDATA_CNTL, CURSOR0_, id), \ - SRI_ARR(DMDATA_SW_CNTL, CURSOR0_, id), \ - SRI_ARR(DMDATA_QOS_CNTL, CURSOR0_, id), \ - SRI_ARR(DMDATA_SW_DATA, CURSOR0_, id), \ - SRI_ARR(DMDATA_STATUS, CURSOR0_, id), \ - SRI_ARR(FLIP_PARAMETERS_0, HUBPREQ, id), \ - SRI_ARR(FLIP_PARAMETERS_1, HUBPREQ, id), \ - SRI_ARR(FLIP_PARAMETERS_2, HUBPREQ, id), \ - SRI_ARR(DCN_CUR1_TTU_CNTL0, HUBPREQ, id), \ - SRI_ARR(DCN_CUR1_TTU_CNTL1, HUBPREQ, id), \ - SRI_ARR(DCSURF_FLIP_CONTROL2, HUBPREQ, id), \ - SRI_ARR(VMID_SETTINGS_0, HUBPREQ, id), \ - SRI_ARR(DCHUBP_CNTL, HUBP, id), \ - SRI_ARR(DCSURF_ADDR_CONFIG, HUBP, id), \ - SRI_ARR(DCSURF_TILING_CONFIG, HUBP, id), \ - SRI_ARR(DCSURF_SURFACE_PITCH, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_PITCH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_CONFIG, HUBP, id), \ - SRI_ARR(DCSURF_FLIP_CONTROL, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION, HUBP, id), \ - SRI_ARR(DCSURF_PRI_VIEWPORT_START, HUBP, id), \ - SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION, HUBP, id), \ - SRI_ARR(DCSURF_SEC_VIEWPORT_START, HUBP, id), \ - SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION_C, HUBP, id), \ - SRI_ARR(DCSURF_PRI_VIEWPORT_START_C, HUBP, id), \ - SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION_C, HUBP, id), \ - SRI_ARR(DCSURF_SEC_VIEWPORT_START_C, HUBP, id), \ - SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_INUSE, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_INUSE_HIGH, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_INUSE_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_INUSE_HIGH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_CONTROL, HUBPREQ, id), \ - SRI_ARR(DCSURF_SURFACE_FLIP_INTERRUPT, HUBPREQ, id), \ - SRI_ARR(HUBPRET_CONTROL, HUBPRET, id), \ - SRI_ARR(HUBPRET_READ_LINE_STATUS, HUBPRET, id), \ - SRI_ARR(DCN_EXPANSION_MODE, HUBPREQ, id), \ - SRI_ARR(DCHUBP_REQ_SIZE_CONFIG, HUBP, id), \ - SRI_ARR(DCHUBP_REQ_SIZE_CONFIG_C, HUBP, id), \ - SRI_ARR(BLANK_OFFSET_0, HUBPREQ, id), \ - SRI_ARR(BLANK_OFFSET_1, HUBPREQ, id), \ - SRI_ARR(DST_DIMENSIONS, HUBPREQ, id), \ - SRI_ARR(DST_AFTER_SCALER, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_0, HUBPREQ, id), \ - SRI_ARR(REF_FREQ_TO_PIX_FREQ, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_1, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_3, HUBPREQ, id), \ - SRI_ARR(NOM_PARAMETERS_4, HUBPREQ, id), \ - SRI_ARR(NOM_PARAMETERS_5, HUBPREQ, id), \ - SRI_ARR(PER_LINE_DELIVERY_PRE, HUBPREQ, id), \ - SRI_ARR(PER_LINE_DELIVERY, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_2, HUBPREQ, id), \ - SRI_ARR(VBLANK_PARAMETERS_4, HUBPREQ, id), \ - SRI_ARR(NOM_PARAMETERS_6, HUBPREQ, id), \ - SRI_ARR(NOM_PARAMETERS_7, HUBPREQ, id), \ - SRI_ARR(DCN_TTU_QOS_WM, HUBPREQ, id), \ - SRI_ARR(DCN_GLOBAL_TTU_CNTL, HUBPREQ, id), \ - SRI_ARR(DCN_SURF0_TTU_CNTL0, HUBPREQ, id), \ - SRI_ARR(DCN_SURF0_TTU_CNTL1, HUBPREQ, id), \ - SRI_ARR(DCN_SURF1_TTU_CNTL0, HUBPREQ, id), \ - SRI_ARR(DCN_SURF1_TTU_CNTL1, HUBPREQ, id), \ - SRI_ARR(DCN_CUR0_TTU_CNTL0, HUBPREQ, id), \ - SRI_ARR(DCN_CUR0_TTU_CNTL1, HUBPREQ, id), \ - SRI_ARR(HUBP_CLK_CNTL, HUBP, id), \ - SRI_ARR(HUBPRET_READ_LINE_VALUE, HUBPRET, id), \ - SRI_ARR(DCHUBP_MALL_CONFIG, HUBP, id), \ - SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id), \ - SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id), \ - SRI_ARR(HUBP_3DLUT_DLG_PARAM, CURSOR0_, id), \ - HUBP_3DLUT_FL_REG_LIST_DCN401(id) struct dcn42b_resource_pool { struct resource_pool base; }; diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c index 1164fd96b714..f0f8e280ed30 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c @@ -33,22 +33,28 @@ void mod_hdcp_dump_binary_message(uint8_t *msg, uint32_t msg_size, byte_size = 3, newline_size = 1, terminator_size = 1; - uint32_t line_count = msg_size / bytes_per_line, - trailing_bytes = msg_size % bytes_per_line; - uint32_t target_size = (byte_size * bytes_per_line + newline_size) * line_count + - byte_size * trailing_bytes + newline_size + terminator_size; uint32_t buf_pos = 0; uint32_t i = 0; - if (buf_size >= target_size) { - for (i = 0; i < msg_size; i++) { - if (i % bytes_per_line == 0) - buf[buf_pos++] = '\n'; - sprintf((char *)&buf[buf_pos], "%02X ", msg[i]); - buf_pos += byte_size; - } - buf[buf_pos++] = '\0'; + /* Need room for at least the terminator. */ + if (buf_size < terminator_size) + return; + + for (i = 0; i < msg_size; i++) { + uint32_t needed = byte_size + terminator_size; + + if (i % bytes_per_line == 0) + needed += newline_size; + + if (buf_pos + needed > buf_size) + break; + + if (i % bytes_per_line == 0) + buf[buf_pos++] = '\n'; + sprintf((char *)&buf[buf_pos], "%02X ", msg[i]); + buf_pos += byte_size; } + buf[buf_pos++] = '\0'; } void mod_hdcp_log_ddc_trace(struct mod_hdcp *hdcp) diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h index f9629d42ada2..7808147ada38 100644 --- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h +++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h @@ -427,6 +427,7 @@ union MESAPI__SUSPEND { uint32_t suspend_fence_value; struct MES_API_STATUS api_status; + uint32_t doorbell_offset; }; uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS]; @@ -444,6 +445,7 @@ union MESAPI__RESUME { uint64_t gang_context_addr; struct MES_API_STATUS api_status; + uint32_t doorbell_offset; }; uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS]; diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index f43d09769320..97da01aff76c 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -41,6 +41,8 @@ #define DEVICE_ATTR_IS(_name) (attr_id == device_attr_id__##_name) +#define power_2_mwatt(power) (((power) >> 8) * 1000 + ((power) & 0xff)) + struct od_attribute { struct kobj_attribute attribute; struct list_head entry; @@ -2696,6 +2698,11 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ gc_ver != IP_VERSION(9, 4, 3)) || gc_ver < IP_VERSION(9, 0, 0)) *states = ATTR_STATE_UNSUPPORTED; + + if (adev->scpm_enabled) { + dev_attr->attr.mode &= ~S_IWUGO; + dev_attr->store = NULL; + } } else if (DEVICE_ATTR_IS(gpu_metrics)) { if (gc_ver < IP_VERSION(9, 1, 0)) *states = ATTR_STATE_UNSUPPORTED; @@ -3349,7 +3356,6 @@ static int amdgpu_hwmon_get_power(struct device *dev, enum amd_pp_sensors sensor) { struct amdgpu_device *adev = dev_get_drvdata(dev); - unsigned int uw; u32 query = 0; int r; @@ -3358,9 +3364,7 @@ static int amdgpu_hwmon_get_power(struct device *dev, return r; /* convert to microwatts */ - uw = (query >> 8) * 1000000 + (query & 0xff) * 1000; - - return uw; + return power_2_mwatt(query) * 1000; } static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev, @@ -4903,7 +4907,7 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a { uint32_t mp1_ver = amdgpu_ip_version(adev, MP1_HWIP, 0); uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); - uint32_t value; + uint32_t value, mwatt, centiwatt; uint64_t value64 = 0; uint32_t query = 0; int size; @@ -4928,17 +4932,21 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a seq_printf(m, "\t%u mV (VDDNB)\n", value); size = sizeof(uint32_t); if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_AVG_POWER, (void *)&query, &size)) { + mwatt = power_2_mwatt(query); + centiwatt = DIV_ROUND_CLOSEST(mwatt, 10); if (adev->flags & AMD_IS_APU) - seq_printf(m, "\t%u.%02u W (average SoC including CPU)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (average SoC including CPU)\n", centiwatt / 100, centiwatt % 100); else - seq_printf(m, "\t%u.%02u W (average SoC)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (average SoC)\n", centiwatt / 100, centiwatt % 100); } size = sizeof(uint32_t); if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_INPUT_POWER, (void *)&query, &size)) { + mwatt = power_2_mwatt(query); + centiwatt = DIV_ROUND_CLOSEST(mwatt, 10); if (adev->flags & AMD_IS_APU) - seq_printf(m, "\t%u.%02u W (current SoC including CPU)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (current SoC including CPU)\n", centiwatt / 100, centiwatt % 100); else - seq_printf(m, "\t%u.%02u W (current SoC)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (current SoC)\n", centiwatt / 100, centiwatt % 100); } size = sizeof(value); seq_printf(m, "\n"); diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index 8079da7c5335..012227c70600 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -3892,13 +3892,16 @@ static void si_notify_hw_of_powersource(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - /* Check if the platform already manages the AC/DC switch via dedicated GPIO. */ - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) - return; - - /* The SMU automatically notices DC, but needs to be notified when switching to AC. */ - if (adev->pm.ac_power) + /* + * Check if the platform already manages the AC/DC switch via dedicated GPIO. + * Otherwise SMU automatically notices DC, but needs to be notified of AC. + */ + if (adev->pm.ac_power && + (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)) amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_RunningOnAC); + + /* Recompute clocks with updated max_limits. */ + amdgpu_legacy_dpm_compute_clocks(adev); } static PPSMC_Result si_send_msg_to_smc_with_parameter(struct amdgpu_device *adev, @@ -7689,7 +7692,7 @@ static int si_dpm_process_interrupt(struct amdgpu_device *adev, break; } - if (queue_thermal) + if (queue_thermal && amdgpu_dpm) schedule_work(&adev->pm.dpm.thermal.work); return 0; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c index 1d6e30269d56..4d553be56396 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c @@ -106,11 +106,8 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr) hwmgr->od_enabled = false; switch (hwmgr->chip_id) { case CHIP_BONAIRE: - /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM - * R7 260X cards with old MC ucode: MCLK DPM is unstable - */ - if (adev->pdev->subsystem_vendor == 0x106B || - adev->pdev->device == 0x6658) { + /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM */ + if (adev->pdev->subsystem_vendor == 0x106B) { dev_info(adev->dev, "disabling MCLK DPM on quirky ASIC"); adev->pm.pp_feature &= ~PP_MCLK_DPM_MASK; hwmgr->feature_mask &= ~PP_MCLK_DPM_MASK; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 95bf187f02a5..bc82ba2e4c9b 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -5857,15 +5857,19 @@ static int smu7_power_off_asic(struct pp_hwmgr *hwmgr) static void smu7_notify_ac_dc(struct pp_hwmgr *hwmgr) { struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev); + const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; - /* Check if the platform already manages the AC/DC switch via dedicated GPIO. */ - if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + /* + * Check if the platform already manages the AC/DC switch via dedicated GPIO. + * Otherwise SMU automatically notices DC, but needs to be notified of AC. + */ + if (adev->pm.ac_power && + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_AutomaticDCTransition)) - return; - - /* The SMU automatically notices DC, but needs to be notified when switching to AC. */ - if (adev->pm.ac_power) smum_send_msg_to_smc(hwmgr, PPSMC_MSG_RunningOnAC, NULL); + + /* Recompute clocks with updated max_limits. */ + pp_funcs->pm_compute_clocks(adev->powerplay.pp_handle); } static const struct pp_hwmgr_func smu7_hwmgr_funcs = { diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 208a2fba6d40..3969a7670482 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -802,6 +802,7 @@ static int smu_set_funcs(struct amdgpu_device *adev) break; case IP_VERSION(15, 0, 0): case IP_VERSION(15, 0, 5): + case IP_VERSION(15, 0, 9): smu_v15_0_0_set_ppt_funcs(smu); break; case IP_VERSION(15, 0, 8): @@ -1366,6 +1367,14 @@ static void smu_feature_cap_init(struct smu_context *smu) bitmap_zero(fea_cap->cap_map, SMU_FEATURE_CAP_ID__COUNT); } +static int smu_set_power_dep(struct smu_context *smu, bool enable) +{ + if (!smu->ppt_funcs->set_power_dep) + return 0; + + return smu->ppt_funcs->set_power_dep(smu, enable); +} + static int smu_sw_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; @@ -1427,6 +1436,8 @@ static int smu_sw_init(struct amdgpu_ip_block *ip_block) if (!smu->ppt_funcs->get_fan_control_mode) smu->adev->pm.no_fan = true; + smu_set_power_dep(smu, true); + return 0; } @@ -1449,6 +1460,8 @@ static int smu_sw_fini(struct amdgpu_ip_block *ip_block) smu_fini_microcode(smu); + smu_set_power_dep(smu, false); + return 0; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h index d76e0b005308..e3a89e9a9df4 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h @@ -749,6 +749,9 @@ struct smu_context { bool pm_enabled; bool is_apu; + /* Power dependency link from an integrated xHCI controller to the GPU */ + struct device_link *usb_power_link; + uint32_t smc_driver_if_version; uint32_t smc_fw_if_version; uint32_t smc_fw_version; @@ -1648,12 +1651,19 @@ struct pptable_funcs { int (*ras_send_msg)(struct smu_context *smu, enum smu_message_type msg, uint32_t param, uint32_t *read_arg); - /** * @get_ras_smu_drv: Get RAS smu driver interface * Return: ras_smu_drv * */ int (*get_ras_smu_drv)(struct smu_context *smu, const struct ras_smu_drv **ras_smu_drv); + + /** + * @set_power_dep: Create or destroy a power dependency link + * from an integrated xHCI controller to the GPU so that the GPU is + * resumed before the USB controller during PM resume. @enable is true + * to create the link and false to tear it down. + */ + int (*set_power_dep)(struct smu_context *smu, bool enable); }; typedef enum { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 7f8d4bb47d02..acbd7046d8a5 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -2403,11 +2403,14 @@ static int smu_v13_0_0_get_power_limit(struct smu_context *smu, uint32_t pp_limit = smu->adev->pm.ac_power ? skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - uint32_t power_limit = 0, od_percent_upper = 0, od_percent_lower = 0; + uint32_t msg_limit = skutable->MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; + uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit); + uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit); + uint32_t od_percent_upper = 0, od_percent_lower = 0; int ret; if (current_power_limit) { - ret = smu_v13_0_get_current_power_limit(smu, &power_limit); + ret = smu_v13_0_get_current_power_limit(smu, current_power_limit); if (ret) *current_power_limit = pp_limit; } @@ -2430,12 +2433,12 @@ static int smu_v13_0_0_get_power_limit(struct smu_context *smu, od_percent_upper, od_percent_lower, pp_limit); if (max_power_limit) { - *max_power_limit = pp_limit * (100 + od_percent_upper); + *max_power_limit = max_limit * (100 + od_percent_upper); *max_power_limit /= 100; } if (min_power_limit) { - *min_power_limit = pp_limit * (100 - od_percent_lower); + *min_power_limit = min_limit * (100 - od_percent_lower); *min_power_limit /= 100; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index 0f774b0920ce..42c9ceeb4f7d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -2385,15 +2385,16 @@ static int smu_v13_0_7_get_power_limit(struct smu_context *smu, uint32_t pp_limit = smu->adev->pm.ac_power ? skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - uint32_t power_limit = 0, od_percent_upper = 0, od_percent_lower = 0; + uint32_t msg_limit = skutable->MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; + uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit); + uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit); + uint32_t od_percent_upper = 0, od_percent_lower = 0; int ret; if (current_power_limit) { - ret = smu_v13_0_get_current_power_limit(smu, &power_limit); + ret = smu_v13_0_get_current_power_limit(smu, current_power_limit); if (ret) - power_limit = pp_limit; - - *current_power_limit = power_limit; + *current_power_limit = pp_limit; } if (default_power_limit) @@ -2414,12 +2415,12 @@ static int smu_v13_0_7_get_power_limit(struct smu_context *smu, od_percent_upper, od_percent_lower, pp_limit); if (max_power_limit) { - *max_power_limit = pp_limit * (100 + od_percent_upper); + *max_power_limit = max_limit * (100 + od_percent_upper); *max_power_limit /= 100; } if (min_power_limit) { - *min_power_limit = pp_limit * (100 - od_percent_lower); + *min_power_limit = min_limit * (100 - od_percent_lower); *min_power_limit /= 100; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c index 75719c47a41e..3d73f2050bbe 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c @@ -1701,6 +1701,50 @@ static int smu_v14_0_0_restore_user_od_settings(struct smu_context *smu) return 0; } +/* + * Link any xHCI controller sharing the GPU's PCIe root port as a consumer + * of the GPU so the GPU resumes first, avoiding an xHCI resume race. + */ +static int smu_v14_0_0_set_power_dep(struct smu_context *smu, bool enable) +{ + struct amdgpu_device *adev = smu->adev; + struct pci_dev *gpu_pdev = adev->pdev; + struct pci_dev *root_port, *usb_pdev = NULL; + struct device_link *link; + + if (!enable) { + if (smu->usb_power_link) { + device_link_del(smu->usb_power_link); + smu->usb_power_link = NULL; + } + return 0; + } + + root_port = pcie_find_root_port(gpu_pdev); + while ((usb_pdev = pci_get_class(PCI_CLASS_SERIAL_USB_XHCI, usb_pdev))) { + struct pci_dev *usb_root; + + usb_root = pcie_find_root_port(usb_pdev); + if (usb_root != root_port) + continue; + + /* Create device link: USB (consumer) depends on GPU (supplier) */ + link = device_link_add(&usb_pdev->dev, &gpu_pdev->dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + if (link) { + smu->usb_power_link = link; + drm_info(adev_to_drm(adev), "USB controller %s D0 power state depends on %s\n", + pci_name(usb_pdev), pci_name(gpu_pdev)); + /* Only create one link for the first USB controller found */ + break; + } + } + + pci_dev_put(usb_pdev); + + return 0; +} + static const struct pptable_funcs smu_v14_0_0_ppt_funcs = { .check_fw_status = smu_v14_0_check_fw_status, .check_fw_version = smu_cmn_check_fw_version, @@ -1734,6 +1778,7 @@ static const struct pptable_funcs smu_v14_0_0_ppt_funcs = { .dpm_set_umsch_mm_enable = smu_v14_0_0_set_umsch_mm_enable, .get_dpm_clock_table = smu_v14_0_common_get_dpm_table, .set_mall_enable = smu_v14_0_common_set_mall_enable, + .set_power_dep = smu_v14_0_0_set_power_dep, }; static void smu_v14_0_0_init_msg_ctl(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c index fdc1456b885c..a6a88e7b2668 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c @@ -1621,19 +1621,23 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu, table_context->power_play_table; PPTable_t *pptable = table_context->driver_pptable; CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - int16_t od_percent_upper = 0, od_percent_lower = 0; + uint32_t pp_limit = smu->adev->pm.ac_power ? + skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : + skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - uint32_t power_limit; + uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit); + uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit); + int16_t od_percent_upper = 0, od_percent_lower = 0; + int ret; - if (smu_v14_0_get_current_power_limit(smu, &power_limit)) - power_limit = smu->adev->pm.ac_power ? - skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : - skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; + if (current_power_limit) { + ret = smu_v14_0_get_current_power_limit(smu, current_power_limit); + if (ret) + *current_power_limit = pp_limit; + } - if (current_power_limit) - *current_power_limit = power_limit; if (default_power_limit) - *default_power_limit = power_limit; + *default_power_limit = pp_limit; if (powerplay_table) { if (smu->od_enabled && @@ -1647,15 +1651,15 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu, } dev_dbg(smu->adev->dev, "od percent upper:%d, od percent lower:%d (default power: %d)\n", - od_percent_upper, od_percent_lower, power_limit); + od_percent_upper, od_percent_lower, pp_limit); if (max_power_limit) { - *max_power_limit = msg_limit * (100 + od_percent_upper); + *max_power_limit = max_limit * (100 + od_percent_upper); *max_power_limit /= 100; } if (min_power_limit) { - *min_power_limit = power_limit * (100 + od_percent_lower); + *min_power_limit = min_limit * (100 + od_percent_lower); *min_power_limit /= 100; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c index a1318409e4b5..8fc99e93ac53 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c @@ -664,6 +664,7 @@ int smu_v15_0_gfx_off_control(struct smu_context *smu, bool enable) switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) { case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) return 0; if (enable) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c index fb1145691410..bb8d09e73c7d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c @@ -227,9 +227,14 @@ static int smu_v15_0_0_system_features_control(struct smu_context *smu, bool en) struct amdgpu_device *adev = smu->adev; int ret = 0; - if (!en && !adev->in_s0ix) + if (!en && !adev->in_s0ix) { ret = smu_cmn_send_smc_msg(smu, SMU_MSG_PrepareMp1ForUnload, NULL); + /* SMU resets BIF_FB_EN to zero, re-enable MC access on APUs with SMU V15 */ + if (!ret && adev->nbio.funcs && adev->nbio.funcs->mc_access_enable) + adev->nbio.funcs->mc_access_enable(adev, true); + } + return ret; } @@ -1172,7 +1177,8 @@ static int smu_v15_0_common_get_dpm_profile_freq(struct smu_context *smu, smu_v15_0_common_get_dpm_ultimate_freq(smu, SMU_SOCCLK, NULL, &clk_limit); break; case SMU_FCLK: - if (amdgpu_ip_version(smu->adev, MP1_HWIP, 0) == IP_VERSION(15, 0, 0)) + if (amdgpu_ip_version(smu->adev, MP1_HWIP, 0) == IP_VERSION(15, 0, 0) || + amdgpu_ip_version(smu->adev, MP1_HWIP, 0) == IP_VERSION(15, 0, 9)) smu_v15_0_common_get_dpm_ultimate_freq(smu, SMU_FCLK, NULL, &clk_limit); else clk_limit = SMU_15_0_UMD_PSTATE_FCLK; |