diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 194 |
1 files changed, 186 insertions, 8 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index ab3b2e7be9bd..5cba592ba941 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -71,6 +71,12 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, struct queue *q, const uint32_t *restore_sdma_id); static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); +static int resume_all_queues_mes(struct device_queue_manager *dqm); +static int suspend_all_queues_mes(struct device_queue_manager *dqm); +static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, + u32 doorbell_offset); +static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, + struct qcm_process_device *qpd); static inline enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) @@ -273,13 +279,19 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, return r; } -static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, - struct qcm_process_device *qpd) +static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q, + struct qcm_process_device *qpd, + bool is_for_reset, + bool flush_mes_queue) { struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; int r; struct mes_remove_queue_input queue_input; + /* queue was already removed during reset */ + if (q->properties.is_reset) + return 0; + if (!dqm->sched_running || dqm->sched_halt) return 0; if (!down_read_trylock(&adev->reset_domain->sem)) @@ -288,6 +300,7 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); queue_input.doorbell_offset = q->properties.doorbell_off; queue_input.gang_context_addr = q->gang_ctx_gpu_addr; + queue_input.remove_queue_after_reset = flush_mes_queue; queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; amdgpu_mes_lock(&adev->mes); @@ -295,7 +308,13 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, amdgpu_mes_unlock(&adev->mes); up_read(&adev->reset_domain->sem); + if (is_for_reset) + return r; + if (r) { + if (!suspend_all_queues_mes(dqm)) + return resume_all_queues_mes(dqm); + dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", q->properties.doorbell_off); dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); @@ -305,6 +324,35 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, return r; } +static void set_perfcount(struct device_queue_manager *dqm, int enable) +{ + struct device_process_node *cur; + struct qcm_process_device *qpd; + struct queue *q; + struct mqd_update_info minfo = { 0 }; + + if (!dqm) + return; + + minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE : + UPDATE_FLAG_PERFCOUNT_DISABLE); + dqm_lock(dqm); + list_for_each_entry(cur, &dqm->queues, list) { + qpd = cur->qpd; + list_for_each_entry(q, &qpd->queues_list, list) { + pqm_update_mqd(qpd->pqm, q->properties.queue_id, + &minfo); + } + } + dqm_unlock(dqm); +} + +static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, + struct qcm_process_device *qpd) +{ + return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false); +} + static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm) { struct device_process_node *cur; @@ -359,6 +407,92 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) return retval; } +static int reset_queues_mes(struct device_queue_manager *dqm) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; + int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; + int num_hung = 0, r = 0, i, pipe, queue, queue_type; + u32 *hung_array = dqm->hung_db_array; + struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info; + struct kfd_process_device *pdd; + struct queue *q; + + if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) { + r = -ENOTRECOVERABLE; + goto fail; + } + + /* reset should be used only in dqm locked queue reset */ + if (WARN_ON(dqm->detect_hang_count > 0)) + return 0; + + if (!amdgpu_gpu_recovery) { + r = -ENOTRECOVERABLE; + goto fail; + } + + if (!hung_array || !hqd_info) { + r = -ENOMEM; + goto fail; + } + + memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info)); + + /* + * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called + * post suspend_all as reset & detect will return all hung queue types. + * + * Passed parameter is for targeting queues not scheduled by MES add_queue. + */ + r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE, + false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1); + + if (!num_hung || r) { + r = -ENOTRECOVERABLE; + goto fail; + } + + /* MES resets queue/pipe and cleans up internally */ + for (i = 0; i < num_hung; i++) { + hqd_info[i].bit0_31 = hung_array[i + hqd_info_size]; + pipe = hqd_info[i].pipe_index; + queue = hqd_info[i].queue_index; + queue_type = hqd_info[i].queue_type; + + if (queue_type != MES_QUEUE_TYPE_COMPUTE && + queue_type != MES_QUEUE_TYPE_SDMA) { + pr_warn("Unsupported hung queue reset type: %d\n", queue_type); + hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET; + continue; + } + + q = find_queue_by_doorbell_offset(dqm, hung_array[i]); + if (!q) { + r = -ENOTRECOVERABLE; + goto fail; + } + + pdd = kfd_get_process_device_data(q->device, q->process); + if (!pdd) { + r = -ENODEV; + goto fail; + } + + pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n", + hung_array[i], pipe, queue, queue_type); + /* Proceed remove_queue with reset=true */ + remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false); + set_queue_as_reset(dqm, q, &pdd->qpd); + } + + dqm->detect_hang_count = num_hung; + kfd_signal_reset_event(dqm->dev); + +fail: + dqm->detect_hang_count = 0; + return r; +} + static int suspend_all_queues_mes(struct device_queue_manager *dqm) { struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; @@ -367,10 +501,13 @@ static int suspend_all_queues_mes(struct device_queue_manager *dqm) if (!down_read_trylock(&adev->reset_domain->sem)) return -EIO; - r = amdgpu_mes_suspend(adev); + r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1); up_read(&adev->reset_domain->sem); if (r) { + if (!reset_queues_mes(dqm)) + return 0; + dev_err(adev->dev, "failed to suspend gangs from MES\n"); dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); kfd_hws_hang(dqm); @@ -387,7 +524,7 @@ static int resume_all_queues_mes(struct device_queue_manager *dqm) if (!down_read_trylock(&adev->reset_domain->sem)) return -EIO; - r = amdgpu_mes_resume(adev); + r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1); up_read(&adev->reset_domain->sem); if (r) { @@ -475,6 +612,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd, } else { /* For CP queues on SOC15 */ if (restore_id) { + if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) + return -EINVAL; + /* make sure that ID is free */ if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) return -EINVAL; @@ -572,7 +712,7 @@ static int allocate_vmid(struct device_queue_manager *dqm, qpd->vmid, qpd->page_table_base); /* invalidate the VM context after pasid and vmid mapping is set up */ - kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); + kfd_flush_tlb(qpd_to_pdd(qpd)); if (dqm->dev->kfd2kgd->set_scratch_backing_va) dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, @@ -610,7 +750,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm, if (flush_texture_cache_nocpsch(q->device, qpd)) dev_err(dev, "Failed to flush TC\n"); - kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); + kfd_flush_tlb(qpd_to_pdd(qpd)); /* Release the vmid mapping */ set_pasid_vmid_mapping(dqm, 0, qpd->vmid); @@ -1284,7 +1424,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, dqm->dev->adev, qpd->vmid, qpd->page_table_base); - kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); + kfd_flush_tlb(pdd); } /* Take a safe reference to the mm_struct, which may otherwise @@ -1587,6 +1727,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, } if (restore_sdma_id) { + if (*restore_sdma_id >= get_num_sdma_queues(dqm)) + return -EINVAL; + /* Re-use existing sdma_id */ if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { dev_err(dev, "SDMA queue already in use\n"); @@ -1613,6 +1756,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, return -ENOMEM; } if (restore_sdma_id) { + if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm)) + return -EINVAL; + /* Re-use existing sdma_id */ if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { dev_err(dev, "SDMA queue already in use\n"); @@ -1797,10 +1943,11 @@ static int halt_cpsch(struct device_queue_manager *dqm) static int unhalt_cpsch(struct device_queue_manager *dqm) { int ret = 0; + struct amdgpu_device *adev = dqm->dev->adev; dqm_lock(dqm); if (!dqm->sched_running || !dqm->sched_halt) { - WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n"); + dev_dbg(adev->dev, "Scheduling is not on halt.\n"); dqm_unlock(dqm); return 0; } @@ -1821,6 +1968,9 @@ static int start_cpsch(struct device_queue_manager *dqm) { struct device *dev = dqm->dev->adev->dev; int retval, num_hw_queue_slots; + struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; + int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev); + int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; dqm_lock(dqm); @@ -1870,6 +2020,11 @@ static int start_cpsch(struct device_queue_manager *dqm) goto fail_detect_hang_buffer; } + dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL); + dqm->hqd_info = kzalloc( + hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info), + GFP_KERNEL); + dqm_unlock(dqm); return 0; @@ -1910,6 +2065,9 @@ static int stop_cpsch(struct device_queue_manager *dqm) pm_uninit(&dqm->packet_mgr); kfree(dqm->detect_hang_info); dqm->detect_hang_info = NULL; + kfree(dqm->hung_db_array); + kfree(dqm->hqd_info); + dqm_unlock(dqm); return ret; @@ -2137,6 +2295,7 @@ static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q q->properties.queue_id, pdd->process->lead_thread->pid); pdd->has_reset_queue = true; + q->properties.is_reset = true; if (q->properties.is_active) { q->properties.is_active = false; decrement_queue_count(dqm, qpd, q); @@ -2203,6 +2362,23 @@ static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uin return NULL; } +static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset) +{ + struct device_process_node *cur; + struct qcm_process_device *qpd; + struct queue *q; + + list_for_each_entry(cur, &dqm->queues, list) { + qpd = cur->qpd; + list_for_each_entry(q, &qpd->queues_list, list) { + if (doorbell_offset == q->properties.doorbell_off) + return q; + } + } + + return NULL; +} + static int reset_hung_queues(struct device_queue_manager *dqm) { int r = 0, reset_count = 0, i; @@ -2970,6 +3146,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) dqm->ops.reset_queues = reset_queues_cpsch; dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; dqm->ops.checkpoint_mqd = checkpoint_mqd; + dqm->ops.set_perfcount = set_perfcount; break; case KFD_SCHED_POLICY_NO_HWS: /* initialize dqm for no cp scheduling */ @@ -2990,6 +3167,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) dqm->ops.get_wave_state = get_wave_state; dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; dqm->ops.checkpoint_mqd = checkpoint_mqd; + dqm->ops.set_perfcount = set_perfcount; break; default: dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); |