diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
22 files changed, 397 insertions, 328 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 531e20748198..411ee894f623 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1783,9 +1783,6 @@ static int kfd_ptl_control(struct kfd_process_device *pdd, bool enable) uint32_t ptl_state = enable ? 1 : 0; int ret; - if (!ptl->hw_supported) - return -EOPNOTSUPP; - if (!pdd->dev->kfd2kgd || !pdd->dev->kfd2kgd->ptl_ctrl) return -EOPNOTSUPP; @@ -1804,6 +1801,9 @@ int kfd_ptl_disable_request(struct kfd_process_device *pdd, struct amdgpu_ptl *ptl = &adev->psp.ptl; int ret = 0; + if (!ptl->hw_supported) + return -EOPNOTSUPP; + mutex_lock(&ptl->mutex); if (pdd->ptl_disable_req) @@ -1833,6 +1833,9 @@ int kfd_ptl_disable_release(struct kfd_process_device *pdd, struct amdgpu_ptl *ptl = &adev->psp.ptl; int ret = 0; + if (!ptl->hw_supported) + return -EOPNOTSUPP; + mutex_lock(&ptl->mutex); if (!pdd->ptl_disable_req) @@ -1914,13 +1917,13 @@ static int criu_checkpoint_devices(struct kfd_process *p, struct kfd_criu_device_bucket *device_buckets = NULL; int ret = 0, i; - device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL); + device_buckets = kvcalloc(num_devices, sizeof(*device_buckets), GFP_KERNEL); if (!device_buckets) { ret = -ENOMEM; goto exit; } - device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL); + device_priv = kvcalloc(num_devices, sizeof(*device_priv), GFP_KERNEL); if (!device_priv) { ret = -ENOMEM; goto exit; @@ -2040,17 +2043,17 @@ static int criu_checkpoint_bos(struct kfd_process *p, int ret = 0, pdd_index, bo_index = 0, id; void *mem; - bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL); + bo_buckets = kvcalloc(num_bos, sizeof(*bo_buckets), GFP_KERNEL); if (!bo_buckets) return -ENOMEM; - bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL); + bo_privs = kvcalloc(num_bos, sizeof(*bo_privs), GFP_KERNEL); if (!bo_privs) { ret = -ENOMEM; goto exit; } - files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL); + files = kvcalloc(num_bos, sizeof(struct file *), GFP_KERNEL); if (!files) { ret = -ENOMEM; goto exit; @@ -2581,7 +2584,7 @@ static int criu_restore_bos(struct kfd_process *p, if (!bo_buckets) return -ENOMEM; - files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL); + files = kvcalloc(args->num_bos, sizeof(struct file *), GFP_KERNEL); if (!files) { ret = -ENOMEM; goto exit; @@ -3734,7 +3737,8 @@ static int kfd_mmap(struct file *filep, struct vm_area_struct *vma) return kfd_doorbell_mmap(dev, process, vma); case KFD_MMAP_TYPE_EVENTS: - return kfd_event_mmap(process, vma); + pr_warn("KFD_MMAP_TYPE_EVENTS is no longer supported\n"); + return -EINVAL; case KFD_MMAP_TYPE_RESERVED_MEM: pr_warn("KFD_MMAP_TYPE_RESERVED_MEM is no longer supported\n"); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index f28259d13818..2a239f45fc24 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -1715,6 +1715,8 @@ int kfd_get_gpu_cache_info(struct kfd_node *kdev, struct kfd_gpu_cache_info **pc case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): /* Cacheline size not available in IP discovery for gc11. * kfd_fill_gpu_cache_info_from_gfx_config to hard code it */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 5eb863dec8f4..586e640f13dc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -169,6 +169,8 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): kfd->device_info.event_interrupt_class = &event_interrupt_class_v11; break; case IP_VERSION(12, 0, 0): @@ -451,6 +453,14 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) gfx_target_version = 110504; f2g = &gfx_v11_kfd2kgd; break; + case IP_VERSION(11, 7, 0): + gfx_target_version = 110700; + f2g = &gfx_v11_kfd2kgd; + break; + case IP_VERSION(11, 7, 1): + gfx_target_version = 110701; + f2g = &gfx_v11_kfd2kgd; + break; case IP_VERSION(12, 0, 0): gfx_target_version = 120000; f2g = &gfx_v12_kfd2kgd; @@ -1796,6 +1806,30 @@ void kgd2kfd_teardown_processes(struct amdgpu_device *adev) cond_resched(); } +int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id, + int queue_type, int pipe, int queue, + unsigned int db) +{ + struct kfd_node *node; + int ret; + + if (!kfd->init_complete) + return 0; + + if (node_id >= kfd->num_nodes) { + dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n", + node_id, kfd->num_nodes - 1); + return -EINVAL; + } + node = kfd->nodes[node_id]; + + ret = kfd_reset_queue_mes(node->dqm, queue_type, pipe, queue, db); + if (ret) + dev_err(kfd_device, "Error resetting queue\n"); + + return ret; +} + #if defined(CONFIG_DEBUG_FS) /* This function will send a package to HIQ to hang the HWS diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 2e010c1f8828..97402e6c8f83 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -37,7 +37,8 @@ #include "amdgpu_amdkfd.h" #include "amdgpu_reset.h" #include "amdgpu_sdma.h" -#include "mes_v11_api_def.h" +#include "amdgpu_ring.h" +#include "amdgpu_mes.h" #include "kfd_debug.h" /* Size of the per-pipe EOP queue */ @@ -71,12 +72,11 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, struct queue *q, const uint32_t *restore_sdma_id); static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); -static int resume_all_queues_mes(struct device_queue_manager *dqm); -static int suspend_all_queues_mes(struct device_queue_manager *dqm); static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset); static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, struct qcm_process_device *qpd); +static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q); static inline enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) @@ -184,24 +184,24 @@ static void kfd_hws_hang(struct device_queue_manager *dqm) amdgpu_amdkfd_gpu_reset(dqm->dev->adev); } -static int convert_to_mes_queue_type(int queue_type) +static int convert_to_amdgpu_ring_type(int queue_type) { - int mes_queue_type; + int amdgpu_ring_type; switch (queue_type) { case KFD_QUEUE_TYPE_COMPUTE: - mes_queue_type = MES_QUEUE_TYPE_COMPUTE; + amdgpu_ring_type = AMDGPU_RING_TYPE_COMPUTE; break; case KFD_QUEUE_TYPE_SDMA: - mes_queue_type = MES_QUEUE_TYPE_SDMA; + amdgpu_ring_type = AMDGPU_RING_TYPE_SDMA; break; default: WARN(1, "Invalid queue type %d", queue_type); - mes_queue_type = -EINVAL; + amdgpu_ring_type = -EINVAL; break; } - return mes_queue_type; + return amdgpu_ring_type; } static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, @@ -251,7 +251,7 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, (qpd->pqm->process->debug_trap_enabled || kfd_dbg_has_ttmps_always_setup(q->device)); - queue_type = convert_to_mes_queue_type(q->properties.type); + queue_type = convert_to_amdgpu_ring_type(q->properties.type); if (queue_type < 0) { dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", q->properties.type); @@ -300,6 +300,7 @@ static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, st memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); queue_input.doorbell_offset = q->properties.doorbell_off; queue_input.gang_context_addr = q->gang_ctx_gpu_addr; + queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type); queue_input.remove_queue_after_reset = flush_mes_queue; queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; @@ -308,14 +309,14 @@ static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, st amdgpu_mes_unlock(&adev->mes); up_read(&adev->reset_domain->sem); - if (is_for_reset) + /* If is_for_reset set, it is a mes internal cleanup */ + if (!r || is_for_reset) return r; - if (r) { - if (!suspend_all_queues_mes(dqm)) - return resume_all_queues_mes(dqm); - - dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", + /* remove_hw_queue failure indicates a queue hang. reset the queue */ + r = reset_queues_mes(dqm, q); + if (r && amdgpu_gpu_recovery) { + dev_err(adev->dev, "failed to remove queue from MES, doorbell=0x%x\n", q->properties.doorbell_off); dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); kfd_hws_hang(dqm); @@ -407,16 +408,50 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) return retval; } -static int reset_queues_mes(struct device_queue_manager *dqm) +static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q, + int queue_type, int pipe, int queue, unsigned int db) { struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; - int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; - int num_hung = 0, r = 0, i, pipe, queue, queue_type; - u32 *hung_array = dqm->hung_db_array; - struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info; struct kfd_process_device *pdd; + bool use_mmio = adev->gfx.mec.use_mmio_for_reset; + int r; + + pdd = kfd_get_process_device_data(q->device, q->process); + if (!pdd) + return -ENODEV; + + if (use_mmio) + r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, + ffs(dqm->dev->xcc_mask) - 1); + else + r = amdgpu_mes_reset_user_queue(adev, queue_type, db, + ffs(dqm->dev->xcc_mask) - 1); + if (r) + return r; + /* Proceed remove_queue with reset=true */ + remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true); + set_queue_as_reset(dqm, q, &pdd->qpd); + return 0; +} + +int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type, + int pipe, int queue, unsigned int db) +{ struct queue *q; + q = find_queue_by_doorbell_offset(dqm, db); + if (!q) + return 0; + return reset_queue_mes(dqm, q, queue_type, pipe, queue, db); +} + +static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; + unsigned int num_hung = 0; + int r = 0; + struct mes_remove_queue_input queue_input; + if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) { r = -ENOTRECOVERABLE; goto fail; @@ -431,111 +466,29 @@ static int reset_queues_mes(struct device_queue_manager *dqm) goto fail; } - if (!hung_array || !hqd_info) { - r = -ENOMEM; - goto fail; - } - - memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info)); - - /* - * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called - * post suspend_all as reset & detect will return all hung queue types. - * - * Passed parameter is for targeting queues not scheduled by MES add_queue. - */ - r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE, - false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1); - - if (!num_hung || r) { - r = -ENOTRECOVERABLE; + memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); + queue_input.doorbell_offset = q->properties.doorbell_off; + queue_input.gang_context_addr = q->gang_ctx_gpu_addr; + queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type); + queue_input.remove_queue_after_reset = false; + queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; + /* pass the known bad queue info to the reset function */ + r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input); + if (r) goto fail; - } - - /* MES resets queue/pipe and cleans up internally */ - for (i = 0; i < num_hung; i++) { - hqd_info[i].bit0_31 = hung_array[i + hqd_info_size]; - pipe = hqd_info[i].pipe_index; - queue = hqd_info[i].queue_index; - queue_type = hqd_info[i].queue_type; - - if (queue_type != MES_QUEUE_TYPE_COMPUTE && - queue_type != MES_QUEUE_TYPE_SDMA) { - pr_warn("Unsupported hung queue reset type: %d\n", queue_type); - hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET; - continue; - } - - q = find_queue_by_doorbell_offset(dqm, hung_array[i]); - if (!q) { - r = -ENOTRECOVERABLE; - goto fail; - } - - pdd = kfd_get_process_device_data(q->device, q->process); - if (!pdd) { - r = -ENODEV; - goto fail; - } - - pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n", - hung_array[i], pipe, queue, queue_type); - /* Proceed remove_queue with reset=true */ - remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false); - set_queue_as_reset(dqm, q, &pdd->qpd); - } dqm->detect_hang_count = num_hung; - kfd_signal_reset_event(dqm->dev); + /* When MES doesn't detect any queue hang, no reset happens. Don't signal reset + * event. + */ + if (dqm->detect_hang_count) + kfd_signal_reset_event(dqm->dev); fail: dqm->detect_hang_count = 0; return r; } -static int suspend_all_queues_mes(struct device_queue_manager *dqm) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; - int r = 0; - - if (!down_read_trylock(&adev->reset_domain->sem)) - return -EIO; - - r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1); - up_read(&adev->reset_domain->sem); - - if (r) { - if (!reset_queues_mes(dqm)) - return 0; - - dev_err(adev->dev, "failed to suspend gangs from MES\n"); - dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); - kfd_hws_hang(dqm); - } - - return r; -} - -static int resume_all_queues_mes(struct device_queue_manager *dqm) -{ - struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; - int r = 0; - - if (!down_read_trylock(&adev->reset_domain->sem)) - return -EIO; - - r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1); - up_read(&adev->reset_domain->sem); - - if (r) { - dev_err(adev->dev, "failed to resume gangs from MES\n"); - dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); - kfd_hws_hang(dqm); - } - - return r; -} - static void increment_queue_count(struct device_queue_manager *dqm, struct qcm_process_device *qpd, struct queue *q) @@ -1064,8 +1017,15 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm, /* Get the SDMA queue stats */ if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { - retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, - &sdma_val); + if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) + retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter( + dqm->dev->adev, q->mqd, + dqm->dev->kfd->device_info.num_sdma_queues_per_engine, + &sdma_val); + else + retval = read_sdma_queue_counter( + (uint64_t __user *)q->properties.read_ptr, + &sdma_val); if (retval) dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", q->properties.queue_id); @@ -2703,8 +2663,16 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, /* Get the SDMA queue stats */ if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { - retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, - &sdma_val); + if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) + retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter( + dqm->dev->adev, q->mqd, + dqm->dev->kfd->device_info.num_sdma_queues_per_engine, + &sdma_val); + else + retval = read_sdma_queue_counter( + (uint64_t __user *)q->properties.read_ptr, + &sdma_val); + if (retval) dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", q->properties.queue_id); @@ -3244,12 +3212,12 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm) kfree(dqm); } +/* bad queue notified by interrupt from CP */ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) { struct kfd_process_device *pdd = NULL; struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd); struct device_queue_manager *dqm = knode->dqm; - struct device *dev = dqm->dev->adev->dev; struct qcm_process_device *qpd; struct queue *q = NULL; int ret = 0; @@ -3264,19 +3232,10 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel list_for_each_entry(q, &qpd->queues_list, list) { if (q->doorbell_id == doorbell_id && q->properties.is_active) { - /* suspend all queues will save any good queues and mark the rest as bad */ - suspend_all_queues_mes(dqm); - + reset_queues_mes(dqm, q); q->properties.is_evicted = true; q->properties.is_active = false; decrement_queue_count(dqm, qpd, q); - - /* this will remove the bad queue and sched a GPU reset if needed */ - ret = remove_queue_mes(dqm, q, qpd); - if (ret) - dev_err(dev, "Removing bad queue failed"); - /* resume the good queues */ - resume_all_queues_mes(dqm); break; } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index e0b6a47e7722..2229f8b2f446 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -333,6 +333,8 @@ int debug_refresh_runlist(struct device_queue_manager *dqm); bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, struct qcm_process_device *qpd, int doorbell_off, u32 *queue_format); +int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type, + int pipe, int queue, unsigned int db); static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 81900b49d9d5..dae01e2bb464 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -29,10 +29,12 @@ #include <linux/uaccess.h> #include <linux/mman.h> #include <linux/memory.h> +#include <linux/workqueue.h> #include "kfd_priv.h" #include "kfd_events.h" #include "kfd_device_queue_manager.h" #include <linux/device.h> +#include <drm/amdgpu_drm.h> /* * Wrapper around wait_queue_entry_t @@ -44,67 +46,19 @@ struct kfd_event_waiter { bool event_age_enabled; /* set to true when last_event_age is non-zero */ }; -/* - * Each signal event needs a 64-bit signal slot where the signaler will write - * a 1 before sending an interrupt. (This is needed because some interrupts - * do not contain enough spare data bits to identify an event.) - * We get whole pages and map them to the process VA. - * Individual signal events use their event_id as slot index. - */ -struct kfd_signal_page { - uint64_t *kernel_address; - uint64_t __user *user_address; - bool need_to_free_pages; -}; - -static uint64_t *page_slots(struct kfd_signal_page *page) -{ - return page->kernel_address; -} - -static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p) -{ - void *backing_store; - struct kfd_signal_page *page; - - page = kzalloc_obj(*page); - if (!page) - return NULL; - - backing_store = (void *) __get_free_pages(GFP_KERNEL, - get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); - if (!backing_store) - goto fail_alloc_signal_store; - - /* Initialize all events to unsignaled */ - memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, - KFD_SIGNAL_EVENT_LIMIT * 8); - - page->kernel_address = backing_store; - page->need_to_free_pages = true; - pr_debug("Allocated new event signal page at %p, for process %p\n", - page, p); - - return page; - -fail_alloc_signal_store: - kfree(page); - return NULL; -} - static int allocate_event_notification_slot(struct kfd_process *p, struct kfd_event *ev, const int *restore_id) { int id; - if (!p->signal_page) { - p->signal_page = allocate_signal_page(p); - if (!p->signal_page) - return -ENOMEM; - /* Oldest user mode expects 256 event slots */ - p->signal_mapped_size = 256*8; - } + /* + * The signal page is allocated in user mode and mapped to the kernel + * via the event_page_offset of the create event IOCTL. Without it no + * signal events can be created. + */ + if (!p->signal_page) + return -ENOMEM; if (restore_id) { id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, @@ -123,7 +77,7 @@ static int allocate_event_notification_slot(struct kfd_process *p, return id; ev->event_id = id; - page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT; + p->signal_page[id] = UNSIGNALED_EVENT_SLOT; return 0; } @@ -169,7 +123,7 @@ static struct kfd_event *lookup_signaled_event_by_partial_id( */ if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) { if (signal_mailbox_updated && - page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) + p->signal_page[id] == UNSIGNALED_EVENT_SLOT) return NULL; return idr_find(&p->event_idr, id); @@ -179,7 +133,7 @@ static struct kfd_event *lookup_signaled_event_by_partial_id( * and find the first one that has signaled. */ for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) { - if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) + if (p->signal_page[id] == UNSIGNALED_EVENT_SLOT) continue; ev = idr_find(&p->event_idr, id); @@ -210,10 +164,8 @@ static int create_signal_event(struct file *devkfd, struct kfd_process *p, p->signal_event_count++; - ev->user_signal_address = &p->signal_page->user_address[ev->event_id]; - pr_debug("Signal event number %zu created with id %d, address %p\n", - p->signal_event_count, ev->event_id, - ev->user_signal_address); + pr_debug("Signal event number %zu created with id %d\n", + p->signal_event_count, ev->event_id); return 0; } @@ -293,26 +245,9 @@ static void destroy_events(struct kfd_process *p) mutex_destroy(&p->event_mutex); } -/* - * We assume that the process is being destroyed and there is no need to - * unmap the pages or keep bookkeeping data in order. - */ -static void shutdown_signal_page(struct kfd_process *p) -{ - struct kfd_signal_page *page = p->signal_page; - - if (page) { - if (page->need_to_free_pages) - free_pages((unsigned long)page->kernel_address, - get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); - kfree(page); - } -} - void kfd_event_free_process(struct kfd_process *p) { destroy_events(p); - shutdown_signal_page(p); } static bool event_can_be_gpu_signaled(const struct kfd_event *ev) @@ -329,8 +264,6 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev) static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, uint64_t size, uint64_t user_handle) { - struct kfd_signal_page *page; - if (p->signal_page) return -EBUSY; @@ -340,17 +273,11 @@ static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, return -EINVAL; } - page = kzalloc_obj(*page); - if (!page) - return -ENOMEM; - /* Initialize all events to unsignaled */ memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT, KFD_SIGNAL_EVENT_LIMIT * 8); - page->kernel_address = kernel_address; - - p->signal_page = page; + p->signal_page = kernel_address; p->signal_mapped_size = size; p->signal_handle = user_handle; return 0; @@ -387,7 +314,8 @@ int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset) return -EINVAL; } - err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(mem, &kern_addr, &size); + err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel(mem, &kern_addr, &size, + AMDGPU_GEM_DOMAIN_GTT); if (err) { pr_err("Failed to map event page to kernel\n"); return err; @@ -396,7 +324,7 @@ int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset) err = kfd_event_page_set(p, kern_addr, size, event_page_offset); if (err) { pr_err("Failed to set event page\n"); - amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); + amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem); return err; } return err; @@ -717,7 +645,7 @@ unlock_rcu: static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) { - WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT); + WRITE_ONCE(p->signal_page[ev->event_id], UNSIGNALED_EVENT_SLOT); } static void set_event_from_interrupt(struct kfd_process *p, @@ -760,7 +688,7 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, * in the interrupt payload was invalid and do an * exhaustive search of signaled events. */ - uint64_t *slots = page_slots(p->signal_page); + uint64_t *slots = p->signal_page; uint32_t id; if (valid_id_bits) @@ -1068,51 +996,6 @@ out: return ret; } -int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) -{ - unsigned long pfn; - struct kfd_signal_page *page; - int ret; - - /* check required size doesn't exceed the allocated size */ - if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) < - get_order(vma->vm_end - vma->vm_start)) { - pr_err("Event page mmap requested illegal size\n"); - return -EINVAL; - } - - page = p->signal_page; - if (!page) { - /* Probably KFD bug, but mmap is user-accessible. */ - pr_debug("Signal page could not be found\n"); - return -EINVAL; - } - - pfn = __pa(page->kernel_address); - pfn >>= PAGE_SHIFT; - - vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE - | VM_DONTDUMP | VM_PFNMAP); - - pr_debug("Mapping signal page\n"); - pr_debug(" start user address == 0x%08lx\n", vma->vm_start); - pr_debug(" end user address == 0x%08lx\n", vma->vm_end); - pr_debug(" pfn == 0x%016lX\n", pfn); - pr_debug(" vm_flags == 0x%08lX\n", vma->vm_flags); - pr_debug(" size == 0x%08lX\n", - vma->vm_end - vma->vm_start); - - page->user_address = (uint64_t __user *)vma->vm_start; - - /* mapping the page to user process */ - ret = remap_pfn_range(vma, vma->vm_start, pfn, - vma->vm_end - vma->vm_start, vma->vm_page_prot); - if (!ret) - p->signal_mapped_size = vma->vm_end - vma->vm_start; - - return ret; -} - /* * Assumes that p is not going away. */ @@ -1338,6 +1221,71 @@ void kfd_signal_reset_event(struct kfd_node *dev) srcu_read_unlock(&kfd_processes_srcu, idx); } +/* + * Per-process opt-in for poison-consumption SIGBUS handling. + * + * Default: kernel sends SIGBUS to the process immediately when poison is + * consumed, in addition to delivering the KFD HW/MEMORY exception events. + * + * Userspace (ROCr) can opt-in per-process via the + * DRM_IOCTL_AMDGPU_PROC_OPTIONS / AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY + * option. This lets the app's registered system-event callback handle the + * RAS error first, instead of being killed by SIGBUS. + * + * Encoded value (stored on the kfd_process): + * 0 - default: SIGBUS immediately (no opt-in) + * 0xFFFFFFFF - opt-in, never escalate to SIGBUS + * N (other) - opt-in, escalate to SIGBUS after N ms if app does not + * handle the error in time (safety timeout) + */ + +void kfd_signal_sigbus_delayed_fn(struct work_struct *work) +{ + struct kfd_process *p = container_of(to_delayed_work(work), + struct kfd_process, signal_work); + + if (p->lead_thread) + send_sig(SIGBUS, p->lead_thread, 0); + + kfd_unref_process(p); +} + +static void kfd_signal_sigbus_with_delay(struct kfd_node *dev, + struct kfd_process *p) +{ + u32 delay_ms = atomic_read(&p->kfd_sigbus_delay_ms); + + if (delay_ms == AMDGPU_PROC_OPTIONS_KFD_SIGBUS_DELAY_DISABLED) { + dev_info(dev->adev->dev, + "SIGBUS suppressed for process %s(pid:%d): app opted in to handle RAS error\n", + p->lead_thread->comm, p->lead_thread->pid); + return; + } + + if (delay_ms == 0) + goto send_now; + + /* + * Take an extra reference for the delayed worker. If the work is + * already pending (e.g. another device of this process consumed poison + * just before), drop the reference and skip rescheduling - the process + * only needs to be notified once. + */ + kref_get(&p->ref); + if (!schedule_delayed_work(&p->signal_work, msecs_to_jiffies(delay_ms))) { + kfd_unref_process(p); + return; + } + + dev_info(dev->adev->dev, + "Deferring SIGBUS to process %s(pid:%d) by %u ms (RAS error opt-in safety timeout)\n", + p->lead_thread->comm, p->lead_thread->pid, delay_ms); + return; + +send_now: + send_sig(SIGBUS, p->lead_thread, 0); +} + void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid) { struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, NULL); @@ -1392,7 +1340,7 @@ void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid) rcu_read_unlock(); /* user application will handle SIGBUS signal */ - send_sig(SIGBUS, p->lead_thread, 0); + kfd_signal_sigbus_with_delay(dev, p); kfd_unref_process(p); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h index 1dc21c13833b..827a2c7d7721 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h @@ -49,7 +49,6 @@ #define UNSIGNALED_EVENT_SLOT ((uint64_t)-1) struct kfd_event_waiter; -struct signal_page; struct kfd_event { u32 event_id; @@ -63,9 +62,6 @@ struct kfd_event { spinlock_t lock; wait_queue_head_t wq; /* List of event waiters. */ - /* Only for signal events. */ - uint64_t __user *user_signal_address; - /* type specific data */ union { struct kfd_hsa_memory_exception_data memory_exception_data; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 226e76ae0be7..7cd236c1ff75 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -128,7 +128,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys, enum MIGRATION_COPY_DIR direction, struct dma_fence **mfence) { - const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE; + const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT); struct amdgpu_ring *ring; struct amdgpu_ttm_buffer_entity *entity; u64 gart_s, gart_d; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c index 723b725d20b8..9b7859a77950 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c @@ -315,3 +315,19 @@ bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id, return false; } + +bool mqd_on_vram(struct amdgpu_device *adev) +{ + if (adev->apu_prefer_gtt) + return false; + + switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { + case IP_VERSION(9, 4, 2): + case IP_VERSION(9, 4, 3): + case IP_VERSION(9, 4, 4): + case IP_VERSION(9, 5, 0): + return true; + default: + return false; + } +} diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h index 06ca6235ff1b..59eff3389d39 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h @@ -127,6 +127,7 @@ struct mqd_manager { struct mutex mqd_mutex; struct kfd_node *dev; uint32_t mqd_size; + uint32_t ctl_stack_size; }; struct mqd_user_context_save_area_header { @@ -201,4 +202,6 @@ uint64_t kfd_mqd_stride(struct mqd_manager *mm, struct queue_properties *q); bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id, uint32_t inst); +bool mqd_on_vram(struct amdgpu_device *adev); + #endif /* KFD_MQD_MANAGER_H_ */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c index 8e8ec266ca46..e034da638c07 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c @@ -203,8 +203,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c index fff137e00b5e..350fcbbba4b2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c @@ -241,8 +241,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c index 8c815f129614..7c387fa90076 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c @@ -216,8 +216,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c index 475589b924e9..431a940f91f3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c @@ -294,8 +294,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control = min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1); + m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 17bfb419b202..75e5a9f67d50 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -27,6 +27,7 @@ #include <linux/uaccess.h> #include "kfd_priv.h" #include "kfd_mqd_manager.h" +#include "kfd_topology.h" #include "v9_structs.h" #include "gc/gc_9_0_offset.h" #include "gc/gc_9_0_sh_mask.h" @@ -115,20 +116,6 @@ static void set_priority(struct v9_mqd *m, struct queue_properties *q) m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; } -static bool mqd_on_vram(struct amdgpu_device *adev) -{ - if (adev->apu_prefer_gtt) - return false; - - switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { - case IP_VERSION(9, 4, 3): - case IP_VERSION(9, 5, 0): - return true; - default: - return false; - } -} - static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, struct queue_properties *q) { @@ -411,8 +398,11 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd, static int get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size) { struct v9_mqd *m = get_mqd(mqd); + u32 per_xcc_size; + + per_xcc_size = min_t(u32, m->cp_hqd_cntl_stack_size, mm->ctl_stack_size); - if (check_mul_overflow(m->cp_hqd_cntl_stack_size, NUM_XCC(mm->dev->xcc_mask), ctl_stack_size)) + if (check_mul_overflow(per_xcc_size, NUM_XCC(mm->dev->xcc_mask), ctl_stack_size)) return -EINVAL; return 0; @@ -421,13 +411,15 @@ static int get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst) { struct v9_mqd *m; + u32 ctl_stack_copy_size; /* Control stack is located one page after MQD. */ void *ctl_stack = (void *)((uintptr_t)mqd + AMDGPU_GPU_PAGE_SIZE); m = get_mqd(mqd); + ctl_stack_copy_size = min_t(u32, m->cp_hqd_cntl_stack_size, mm->ctl_stack_size); memcpy(mqd_dst, m, sizeof(struct v9_mqd)); - memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size); + memcpy(ctl_stack_dst, ctl_stack, ctl_stack_copy_size); } static void checkpoint_mqd_v9_4_3(struct mqd_manager *mm, @@ -436,15 +428,19 @@ static void checkpoint_mqd_v9_4_3(struct mqd_manager *mm, void *ctl_stack_dst) { struct v9_mqd *m; + u32 ctl_stack_stride; int xcc; uint64_t size = get_mqd(mqd)->cp_mqd_stride_size; + ctl_stack_stride = min_t(u32, get_mqd(mqd)->cp_hqd_cntl_stack_size, + mm->ctl_stack_size); + for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) { m = get_mqd(mqd + size * xcc); checkpoint_mqd(mm, m, (uint8_t *)mqd_dst + sizeof(*m) * xcc, - (uint8_t *)ctl_stack_dst + m->cp_hqd_cntl_stack_size * xcc); + (uint8_t *)ctl_stack_dst + ctl_stack_stride * xcc); } } @@ -998,6 +994,15 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, mqd->is_occupied = kfd_is_occupied_cp; mqd->get_checkpoint_info = get_checkpoint_info; mqd->mqd_size = sizeof(struct v9_mqd); + if (dev->kfd->cwsr_enabled) { + struct kfd_topology_device *topo_dev; + + topo_dev = kfd_topology_device_by_id(dev->id); + if (topo_dev) + mqd->ctl_stack_size = + ALIGN(topo_dev->node_props.ctl_stack_size, + AMDGPU_GPU_PAGE_SIZE); + } mqd->mqd_stride = mqd_stride_v9; #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c index c86779af323b..60b87a500698 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c @@ -214,8 +214,8 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd, * more than (EOP entry count - 1) so a queue size of 0x800 dwords * is safe, giving a maximum field value of 0xA. */ - m->cp_hqd_eop_control |= min(0xA, - order_base_2(q->eop_ring_buffer_size / 4) - 1); + m->cp_hqd_eop_control |= q->eop_ring_buffer_size ? min(0xA, + order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index acd0e41e744c..90f010cbe54e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -952,11 +952,32 @@ struct kfd_process { struct idr event_idr; /* Event page */ u64 signal_handle; - struct kfd_signal_page *signal_page; + /* + * Each signal event needs a 64-bit signal slot where the signaler will + * write a 1 before sending an interrupt. (This is needed because some + * interrupts do not contain enough spare data bits to identify an + * event.) The signal page is allocated in user mode and mapped to the + * kernel; individual signal events use their event_id as slot index. + */ + uint64_t *signal_page; size_t signal_mapped_size; size_t signal_event_count; bool signal_event_limit_reached; + /** + * @kfd_sigbus_delay_ms: Per-process KFD SIGBUS delivery option for + * poison/RAS events (set via DRM_IOCTL_AMDGPU_PROC_OPTIONS / + * AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY). + * + * 0 - send SIGBUS immediately (default) + * 0xFFFFFFFF - suppress SIGBUS delivery + * other - delay SIGBUS delivery by this many milliseconds + */ + atomic_t kfd_sigbus_delay_ms; + + /* Delayed signal delivery to user */ + struct delayed_work signal_work; + /* Information used for memory eviction */ void *kgd_process_info; /* Eviction fence that is attached to all the BOs of this process. The @@ -1525,7 +1546,6 @@ extern const struct kfd_device_global_init_class device_global_init_class_cik; int kfd_event_init_process(struct kfd_process *p); void kfd_event_free_process(struct kfd_process *p); -int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma); int kfd_wait_on_events(struct kfd_process *p, uint32_t num_events, void __user *data, bool all, uint32_t *user_timeout_ms, @@ -1554,6 +1574,7 @@ void kfd_signal_vm_fault_event(struct kfd_process_device *pdd, void kfd_signal_reset_event(struct kfd_node *dev); void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid); +void kfd_signal_sigbus_delayed_fn(struct work_struct *work); void kfd_signal_process_terminate_event(struct kfd_process *p); static inline void kfd_flush_tlb(struct kfd_process_device *pdd) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index ca71fa726e32..767c2cc8e29e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -153,6 +153,21 @@ static void kfd_sdma_activity_worker(struct work_struct *work) (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) continue; + if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) { + val = 0; + ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter( + dqm->dev->adev, q->mqd, + dqm->dev->kfd->device_info.num_sdma_queues_per_engine, + &val); + + if (ret) + pr_debug("Failed to read SDMA queue active counter %i\n", ret); + else + workarea->sdma_activity_counter += val; + + continue; + } + sdma_q = kzalloc_obj(struct temp_sdma_queue_list); if (!sdma_q) { dqm_unlock(dqm); @@ -171,7 +186,7 @@ static void kfd_sdma_activity_worker(struct work_struct *work) * count */ if (list_empty(&sdma_q_list.list)) { - workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; + workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; dqm_unlock(dqm); return; } @@ -721,7 +736,7 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem, struct kfd_node *dev = pdd->dev; if (kptr && *kptr) { - amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); + amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem); *kptr = NULL; } @@ -761,10 +776,16 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, } if (kptr) { - err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel( - (struct kgd_mem *)*mem, kptr, NULL); + u32 domain; + + if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) + domain = AMDGPU_GEM_DOMAIN_VRAM; + else + domain = AMDGPU_GEM_DOMAIN_GTT; + err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel((struct kgd_mem *)*mem, + kptr, NULL, domain); if (err) { - pr_debug("Map GTT BO to kernel failed\n"); + pr_debug("Map BO to kernel failed err %d\n", err); goto sync_memory_failed; } } @@ -986,6 +1007,33 @@ out: return process; } +/** + * amdgpu_amdkfd_set_sigbus_delay - Set per-process KFD SIGBUS delay + * @task: task in the target process + * @ms: encoded delay value (0 = immediate, 0xFFFFFFFF = suppress, + * otherwise delay in milliseconds) + * + * Stores the SIGBUS delivery option on the kfd_process associated with + * @task. If the calling process has not opened /dev/kfd yet (no + * kfd_process exists), this is a no-op - the option only applies to + * processes that actually use KFD. + */ +int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms) +{ + struct kfd_process *p; + + if (!task->mm) + return -EINVAL; + + p = kfd_lookup_process_by_mm(task->mm); + if (!p) + return 0; + + atomic_set(&p->kfd_sigbus_delay_ms, ms); + kfd_unref_process(p); + return 0; +} + static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) { struct kfd_process *process; @@ -1092,7 +1140,7 @@ static void kfd_process_kunmap_signal_bo(struct kfd_process *p) if (!mem) goto out; - amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); + amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem); out: mutex_unlock(&p->mutex); @@ -1330,6 +1378,11 @@ void kfd_process_notifier_release_internal(struct kfd_process *p) kfd_process_table_remove(p); cancel_delayed_work_sync(&p->eviction_work); cancel_delayed_work_sync(&p->restore_work); + /* + * If work pending, cancel it and drop the extra ref + */ + if (cancel_delayed_work_sync(&p->signal_work)) + kfd_unref_process(p); /* * Dequeue and destroy user queues, it is not safe for GPU to access @@ -1436,8 +1489,7 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) { struct kfd_node *dev = pdd->dev; struct qcm_process_device *qpd = &pdd->qpd; - uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT - | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE + u32 flags = KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; struct kgd_mem *mem; void *kaddr; @@ -1446,7 +1498,12 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) return 0; - /* cwsr_base is only set for dGPU */ + if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt) + flags |= KFD_IOC_ALLOC_MEM_FLAGS_VRAM; + else + flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; + + /* Allocate CWSR TBA/TMA buffers */ ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr); if (ret) @@ -1586,6 +1643,7 @@ struct kfd_process *create_process(const struct task_struct *thread, bool primar INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); + INIT_DELAYED_WORK(&process->signal_work, kfd_signal_sigbus_delayed_fn); process->last_restore_timestamp = get_jiffies_64(); err = kfd_event_init_process(process); if (err) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 0ac35789b239..acbdca91cde5 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -1040,7 +1040,13 @@ int kfd_criu_restore_queue(struct kfd_process *p, ctl_stack = mqd + q_data->mqd_size; memset(&qp, 0, sizeof(qp)); - set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->adev->gfx.xcc_mask)); + set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->xcc_mask)); + + ret = kfd_queue_acquire_buffers(pdd, &qp); + if (ret) { + pr_debug("failed to acquire user queue buffers for CRIU\n"); + goto exit; + } ret = kfd_queue_acquire_buffers(pdd, &qp); if (ret) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c index e659cd50eb0b..6a7b4d959541 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c @@ -224,10 +224,10 @@ static void kfd_smi_event_add(struct task_struct *task, struct kfd_node *dev, pid = kfd_smi_task_to_pid(task); - len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event); + len = scnprintf(fifo_in, sizeof(fifo_in), "%x ", event); va_start(args, fmt); - len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args); + len += vscnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args); va_end(args); add_event_to_kfifo(pid, dev, event, fifo_in, len); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 0900bb23349e..30ad10bbd47e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -3473,7 +3473,13 @@ svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size) unsigned long start_unchg = start; start <<= PAGE_SHIFT; - end = start + (size << PAGE_SHIFT); + + if (size == 0) + return -EINVAL; + + if (check_add_overflow(start, size << PAGE_SHIFT, &end)) + return -EOVERFLOW; + do { vma = vma_lookup(p->mm, start); if (!vma || (vma->vm_flags & device_vma)) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 00517c3d0e6a..35b3abe57b80 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -2013,12 +2013,21 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev) dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; - if (!amdgpu_sriov_vf(dev->gpu->adev)) + if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 3) && + !amdgpu_sriov_vf(dev->gpu->adev)) dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED; } else { dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 | HSA_DBG_WATCH_ADDR_MASK_HI_BIT; + /* gfx11 dGPU and gfx12.0 */ + if ((KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 0) || + KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 2) || + KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 3) || + KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 0, 0) || + KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 0, 1)) && + !amdgpu_sriov_vf(dev->gpu->adev)) + dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED; if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 0, 0)) dev->node_props.capability |= |