summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/amd/amdkfd
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_chardev.c273
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device.c53
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c194
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h6
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c19
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_events.c5
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_migrate.c3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c16
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c14
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c8
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c6
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c26
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c11
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h37
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_process.c91
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_svm.c15
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_topology.c4
17 files changed, 615 insertions, 166 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 55ea5145a28a..81bcb16eb6dd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -21,10 +21,12 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
+#include <linux/capability.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/file.h>
+#include <linux/overflow.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
@@ -43,6 +45,7 @@
#include "kfd_smi_events.h"
#include "amdgpu_dma_buf.h"
#include "kfd_debug.h"
+#include "amdgpu_ptl.h"
static long kfd_ioctl(struct file *, unsigned int, unsigned long);
static int kfd_open(struct inode *, struct file *);
@@ -66,6 +69,21 @@ static const struct class kfd_class = {
.name = kfd_dev_name,
};
+/*
+ * Cache the address space of the chardev on first open so that the reset
+ * path can drop all userspace mappings of doorbell and MMIO ranges via
+ * unmap_mapping_range().
+ */
+static struct address_space *kfd_dev_mapping;
+
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen)
+{
+ struct address_space *mapping = READ_ONCE(kfd_dev_mapping);
+
+ if (mapping)
+ unmap_mapping_range(mapping, holebegin, holelen, 1);
+}
+
static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
{
struct kfd_process_device *pdd;
@@ -132,6 +150,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
if (iminor(inode) != 0)
return -ENODEV;
+ /*
+ * /dev/kfd is a single chardev so all opens share one inode. Cache
+ * its address_space on the first open for use by the reset path.
+ */
+ if (!READ_ONCE(kfd_dev_mapping))
+ cmpxchg(&kfd_dev_mapping, NULL, inode->i_mapping);
+
is_32bit_user_mode = in_compat_syscall();
if (is_32bit_user_mode) {
@@ -146,11 +171,6 @@ static int kfd_open(struct inode *inode, struct file *filep)
if (IS_ERR(process))
return PTR_ERR(process);
- if (kfd_process_init_cwsr_apu(process, filep)) {
- kfd_unref_process(process);
- return -EFAULT;
- }
-
/* filep now owns the reference returned by kfd_create_process */
filep->private_data = process;
@@ -1359,7 +1379,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
if (WARN_ON_ONCE(!peer_pdd))
continue;
- kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(peer_pdd);
}
kfree(devices_arr);
@@ -1454,7 +1474,7 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
if (WARN_ON_ONCE(!peer_pdd))
continue;
if (flush_tlb)
- kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
+ kfd_flush_tlb(peer_pdd);
/* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, peer_pdd->drm_priv);
@@ -1695,6 +1715,16 @@ static int kfd_ioctl_smi_events(struct file *filep,
return kfd_smi_event_open(pdd->dev, &args->anon_fd);
}
+static int kfd_ioctl_svm_validate(void *kdata, unsigned int usize)
+{
+ struct kfd_ioctl_svm_args *args = kdata;
+ size_t expected = struct_size(args, attrs, args->nattr);
+
+ if (expected == SIZE_MAX || usize < expected)
+ return -EINVAL;
+ return 0;
+}
+
#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
static int kfd_ioctl_set_xnack_mode(struct file *filep,
@@ -1766,6 +1796,108 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
}
#endif
+static int kfd_ptl_control(struct kfd_process_device *pdd, bool enable)
+{
+ struct amdgpu_device *adev = pdd->dev->adev;
+ struct amdgpu_ptl *ptl = &adev->psp.ptl;
+ enum amdgpu_ptl_fmt pref_format1 = ptl->fmt1;
+ enum amdgpu_ptl_fmt pref_format2 = ptl->fmt2;
+ uint32_t ptl_state = enable ? 1 : 0;
+ int ret;
+
+ if (!ptl->hw_supported)
+ return -EOPNOTSUPP;
+
+ if (!pdd->dev->kfd2kgd || !pdd->dev->kfd2kgd->ptl_ctrl)
+ return -EOPNOTSUPP;
+
+ ret = pdd->dev->kfd2kgd->ptl_ctrl(adev, PSP_PTL_PERF_MON_SET,
+ &ptl_state,
+ &pref_format1,
+ &pref_format2);
+
+ return ret;
+}
+
+int kfd_ptl_disable_request(struct kfd_process_device *pdd,
+ struct kfd_process *p)
+{
+ struct amdgpu_device *adev = pdd->dev->adev;
+ struct amdgpu_ptl *ptl = &adev->psp.ptl;
+ int ret = 0;
+
+ mutex_lock(&ptl->mutex);
+
+ if (pdd->ptl_disable_req)
+ goto out;
+
+ if (atomic_inc_return(&ptl->disable_ref) == 1) {
+ ret = kfd_ptl_control(pdd, false);
+ if (ret) {
+ atomic_dec(&ptl->disable_ref);
+ dev_warn(pdd->dev->adev->dev,
+ "failed to disable PTL\n");
+ goto out;
+ }
+ }
+ set_bit(AMDGPU_PTL_DISABLE_PROFILER, ptl->disable_bitmap);
+ pdd->ptl_disable_req = true;
+
+out:
+ mutex_unlock(&ptl->mutex);
+ return ret;
+}
+
+int kfd_ptl_disable_release(struct kfd_process_device *pdd,
+ struct kfd_process *p)
+{
+ struct amdgpu_device *adev = pdd->dev->adev;
+ struct amdgpu_ptl *ptl = &adev->psp.ptl;
+ int ret = 0;
+
+ mutex_lock(&ptl->mutex);
+
+ if (!pdd->ptl_disable_req)
+ goto out;
+
+ if (atomic_dec_return(&ptl->disable_ref) == 0) {
+ clear_bit(AMDGPU_PTL_DISABLE_PROFILER, ptl->disable_bitmap);
+ ret = kfd_ptl_control(pdd, true);
+ if (ret) {
+ atomic_inc(&ptl->disable_ref);
+ set_bit(AMDGPU_PTL_DISABLE_PROFILER, ptl->disable_bitmap);
+ dev_warn(adev->dev, "Failed to enable PTL on release: %d\n", ret);
+ goto out;
+ }
+ }
+ pdd->ptl_disable_req = false;
+
+out:
+ mutex_unlock(&ptl->mutex);
+ return ret;
+}
+
+static int kfd_profiler_ptl_control(struct kfd_process *p,
+ struct kfd_ioctl_ptl_control *args)
+{
+ struct kfd_process_device *pdd;
+ int ret;
+
+ mutex_lock(&p->mutex);
+ pdd = kfd_process_device_data_by_id(p, args->gpu_id);
+ mutex_unlock(&p->mutex);
+
+ if (!pdd || !pdd->dev || !pdd->dev->kfd)
+ return -EINVAL;
+
+ if (args->enable == 0)
+ ret = kfd_ptl_disable_request(pdd, p);
+ else
+ ret = kfd_ptl_disable_release(pdd, p);
+
+ return ret;
+}
+
static int criu_checkpoint_process(struct kfd_process *p,
uint8_t __user *user_priv_data,
uint64_t *priv_offset)
@@ -2329,6 +2461,9 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
const bool criu_resume = true;
u64 offset;
+ if (bo_priv->idr_handle > INT_MAX)
+ return -EINVAL;
+
if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
if (bo_bucket->size !=
kfd_doorbell_process_slice(pdd->dev->kfd))
@@ -3207,9 +3342,114 @@ static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, v
return 0;
}
+static inline uint32_t profile_lock_device(struct kfd_process *p,
+ uint32_t gpu_id, uint32_t op)
+{
+ struct kfd_process_device *pdd;
+ struct kfd_dev *kfd;
+ int status = -EINVAL;
+ struct amdgpu_ptl *ptl;
+
+ if (!p)
+ return -EINVAL;
+
+ mutex_lock(&p->mutex);
+ pdd = kfd_process_device_data_by_id(p, gpu_id);
+ mutex_unlock(&p->mutex);
+
+ if (!pdd || !pdd->dev || !pdd->dev->kfd)
+ return -EINVAL;
+
+ kfd = pdd->dev->kfd;
+ ptl = &pdd->dev->adev->psp.ptl;
+
+ mutex_lock(&kfd->profiler_lock);
+ if (op == 1) {
+ if (!kfd->profiler_process) {
+ kfd->profiler_process = p;
+ status = 0;
+ mutex_unlock(&kfd->profiler_lock);
+ if (ptl->hw_supported) {
+ status = kfd_ptl_disable_request(pdd, p);
+ if (status != 0)
+ dev_err(kfd_device,
+ "Failed to lock device %d for profiling, error %d\n",
+ gpu_id, status);
+ }
+ return status;
+ } else if (kfd->profiler_process == p) {
+ status = -EALREADY;
+ } else {
+ status = -EBUSY;
+ }
+ } else if (op == 0 && kfd->profiler_process == p) {
+ kfd->profiler_process = NULL;
+ status = 0;
+ mutex_unlock(&kfd->profiler_lock);
+
+ if (ptl->hw_supported) {
+ status = kfd_ptl_disable_release(pdd, p);
+ if (status)
+ dev_err(kfd_device,
+ "Failed to unlock device %d for profiling, error %d\n",
+ gpu_id, status);
+ }
+ return status;
+ }
+ mutex_unlock(&kfd->profiler_lock);
+
+ return status;
+}
+
+static inline int kfd_profiler_pmc(struct kfd_process *p,
+ struct kfd_ioctl_pmc_settings *args)
+{
+ struct kfd_process_device *pdd;
+ struct device_queue_manager *dqm;
+ int status;
+
+ /* Check if we have the correct permissions. */
+ if (!perfmon_capable())
+ return -EPERM;
+
+ /* Lock/Unlock the device based on the parameter given in OP */
+ status = profile_lock_device(p, args->gpu_id, args->lock);
+ if (status != 0)
+ return status;
+
+ /* Enable/disable perfcount if requested */
+ mutex_lock(&p->mutex);
+ pdd = kfd_process_device_data_by_id(p, args->gpu_id);
+ dqm = pdd->dev->dqm;
+ mutex_unlock(&p->mutex);
+
+ dqm->ops.set_perfcount(dqm, args->perfcount_enable);
+ return status;
+}
+
+static int kfd_ioctl_profiler(struct file *filep, struct kfd_process *p, void *data)
+{
+ struct kfd_ioctl_profiler_args *args = data;
+
+ switch (args->op) {
+ case KFD_IOC_PROFILER_VERSION:
+ args->version = KFD_IOC_PROFILER_VERSION_NUM;
+ return 0;
+ case KFD_IOC_PROFILER_PMC:
+ return kfd_profiler_pmc(p, &args->pmc);
+ case KFD_IOC_PROFILER_PTL_CONTROL:
+ return kfd_profiler_ptl_control(p, &args->ptl);
+ }
+ return -EINVAL;
+}
+
#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
- .cmd_drv = 0, .name = #ioctl}
+ .validate = NULL, .cmd_drv = 0, .name = #ioctl}
+
+#define AMDKFD_IOCTL_DEF_V(ioctl, _func, _validate, _flags) \
+ [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
+ .validate = _validate, .cmd_drv = 0, .name = #ioctl}
/** Ioctl table */
static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
@@ -3306,7 +3546,8 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
kfd_ioctl_smi_events, 0),
- AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
+ AMDKFD_IOCTL_DEF_V(AMDKFD_IOC_SVM, kfd_ioctl_svm,
+ kfd_ioctl_svm_validate, 0),
AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
kfd_ioctl_set_xnack_mode, 0),
@@ -3328,6 +3569,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_PROCESS,
kfd_ioctl_create_process, 0),
+
+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_PROFILER,
+ kfd_ioctl_profiler, 0),
};
#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
@@ -3431,6 +3675,12 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
memset(kdata, 0, usize);
}
+ if (ioctl->validate) {
+ retcode = ioctl->validate(kdata, usize);
+ if (retcode)
+ goto err_i1;
+ }
+
retcode = func(filep, process, kdata);
if (cmd & IOC_OUT)
@@ -3515,9 +3765,8 @@ static int kfd_mmap(struct file *filep, struct vm_area_struct *vma)
return kfd_event_mmap(process, vma);
case KFD_MMAP_TYPE_RESERVED_MEM:
- if (!dev)
- return -ENODEV;
- return kfd_reserved_mem_mmap(dev, process, vma);
+ pr_warn("KFD_MMAP_TYPE_RESERVED_MEM is no longer supported\n");
+ return -EINVAL;
case KFD_MMAP_TYPE_MMIO:
if (!dev)
return -ENODEV;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 8ff97bf7d95a..c2c59781feee 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -936,6 +936,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
svm_range_set_max_pages(kfd->adev);
+ kfd->profiler_process = NULL;
+ mutex_init(&kfd->profiler_lock);
+
kfd->init_complete = true;
dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
kfd->adev->pdev->device);
@@ -971,6 +974,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
ida_destroy(&kfd->doorbell_ida);
kfd_gtt_sa_fini(kfd);
amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem);
+ mutex_destroy(&kfd->profiler_lock);
}
kfree(kfd);
@@ -1647,6 +1651,22 @@ int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd)
return 0;
}
+int amdgpu_amdkfd_stop_sched_all(struct amdgpu_device *adev)
+{
+ if (!adev->kfd.init_complete)
+ return 0;
+
+ return kgd2kfd_stop_sched_all_nodes(adev->kfd.dev);
+}
+
+int amdgpu_amdkfd_start_sched_all(struct amdgpu_device *adev)
+{
+ if (!adev->kfd.init_complete)
+ return 0;
+
+ return kgd2kfd_start_sched_all_nodes(adev->kfd.dev);
+}
+
bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id)
{
struct kfd_node *node;
@@ -1737,37 +1757,6 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
return false;
}
-/* check if there is kfd process still uses adev */
-static bool kgd2kfd_check_device_idle(struct amdgpu_device *adev)
-{
- struct kfd_process *p;
- struct hlist_node *p_temp;
- unsigned int temp;
- struct kfd_node *dev;
-
- mutex_lock(&kfd_processes_mutex);
-
- if (hash_empty(kfd_processes_table)) {
- mutex_unlock(&kfd_processes_mutex);
- return true;
- }
-
- /* check if there is device still use adev */
- hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
- for (int i = 0; i < p->n_pdds; i++) {
- dev = p->pdds[i]->dev;
- if (dev->adev == adev) {
- mutex_unlock(&kfd_processes_mutex);
- return false;
- }
- }
- }
-
- mutex_unlock(&kfd_processes_mutex);
-
- return true;
-}
-
/** kgd2kfd_teardown_processes - gracefully tear down existing
* kfd processes that use adev
*
@@ -1800,7 +1789,7 @@ void kgd2kfd_teardown_processes(struct amdgpu_device *adev)
mutex_unlock(&kfd_processes_mutex);
/* wait all kfd processes use adev terminate */
- while (!kgd2kfd_check_device_idle(adev))
+ while (!!atomic_read(&adev->kfd.dev->kfd_processes_count))
cond_resched();
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ab3b2e7be9bd..5cba592ba941 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -71,6 +71,12 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
struct queue *q, const uint32_t *restore_sdma_id);
static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
+static int resume_all_queues_mes(struct device_queue_manager *dqm);
+static int suspend_all_queues_mes(struct device_queue_manager *dqm);
+static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm,
+ u32 doorbell_offset);
+static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd);
static inline
enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
@@ -273,13 +279,19 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
return r;
}
-static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
- struct qcm_process_device *qpd)
+static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd,
+ bool is_for_reset,
+ bool flush_mes_queue)
{
struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
int r;
struct mes_remove_queue_input queue_input;
+ /* queue was already removed during reset */
+ if (q->properties.is_reset)
+ return 0;
+
if (!dqm->sched_running || dqm->sched_halt)
return 0;
if (!down_read_trylock(&adev->reset_domain->sem))
@@ -288,6 +300,7 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
queue_input.doorbell_offset = q->properties.doorbell_off;
queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
+ queue_input.remove_queue_after_reset = flush_mes_queue;
queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
amdgpu_mes_lock(&adev->mes);
@@ -295,7 +308,13 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
amdgpu_mes_unlock(&adev->mes);
up_read(&adev->reset_domain->sem);
+ if (is_for_reset)
+ return r;
+
if (r) {
+ if (!suspend_all_queues_mes(dqm))
+ return resume_all_queues_mes(dqm);
+
dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
q->properties.doorbell_off);
dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
@@ -305,6 +324,35 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
return r;
}
+static void set_perfcount(struct device_queue_manager *dqm, int enable)
+{
+ struct device_process_node *cur;
+ struct qcm_process_device *qpd;
+ struct queue *q;
+ struct mqd_update_info minfo = { 0 };
+
+ if (!dqm)
+ return;
+
+ minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE :
+ UPDATE_FLAG_PERFCOUNT_DISABLE);
+ dqm_lock(dqm);
+ list_for_each_entry(cur, &dqm->queues, list) {
+ qpd = cur->qpd;
+ list_for_each_entry(q, &qpd->queues_list, list) {
+ pqm_update_mqd(qpd->pqm, q->properties.queue_id,
+ &minfo);
+ }
+ }
+ dqm_unlock(dqm);
+}
+
+static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd)
+{
+ return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+}
+
static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
{
struct device_process_node *cur;
@@ -359,6 +407,92 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
return retval;
}
+static int reset_queues_mes(struct device_queue_manager *dqm)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
+ int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
+ int num_hung = 0, r = 0, i, pipe, queue, queue_type;
+ u32 *hung_array = dqm->hung_db_array;
+ struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info;
+ struct kfd_process_device *pdd;
+ struct queue *q;
+
+ if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) {
+ r = -ENOTRECOVERABLE;
+ goto fail;
+ }
+
+ /* reset should be used only in dqm locked queue reset */
+ if (WARN_ON(dqm->detect_hang_count > 0))
+ return 0;
+
+ if (!amdgpu_gpu_recovery) {
+ r = -ENOTRECOVERABLE;
+ goto fail;
+ }
+
+ if (!hung_array || !hqd_info) {
+ r = -ENOMEM;
+ goto fail;
+ }
+
+ memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info));
+
+ /*
+ * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called
+ * post suspend_all as reset & detect will return all hung queue types.
+ *
+ * Passed parameter is for targeting queues not scheduled by MES add_queue.
+ */
+ r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
+ false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1);
+
+ if (!num_hung || r) {
+ r = -ENOTRECOVERABLE;
+ goto fail;
+ }
+
+ /* MES resets queue/pipe and cleans up internally */
+ for (i = 0; i < num_hung; i++) {
+ hqd_info[i].bit0_31 = hung_array[i + hqd_info_size];
+ pipe = hqd_info[i].pipe_index;
+ queue = hqd_info[i].queue_index;
+ queue_type = hqd_info[i].queue_type;
+
+ if (queue_type != MES_QUEUE_TYPE_COMPUTE &&
+ queue_type != MES_QUEUE_TYPE_SDMA) {
+ pr_warn("Unsupported hung queue reset type: %d\n", queue_type);
+ hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET;
+ continue;
+ }
+
+ q = find_queue_by_doorbell_offset(dqm, hung_array[i]);
+ if (!q) {
+ r = -ENOTRECOVERABLE;
+ goto fail;
+ }
+
+ pdd = kfd_get_process_device_data(q->device, q->process);
+ if (!pdd) {
+ r = -ENODEV;
+ goto fail;
+ }
+
+ pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n",
+ hung_array[i], pipe, queue, queue_type);
+ /* Proceed remove_queue with reset=true */
+ remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false);
+ set_queue_as_reset(dqm, q, &pdd->qpd);
+ }
+
+ dqm->detect_hang_count = num_hung;
+ kfd_signal_reset_event(dqm->dev);
+
+fail:
+ dqm->detect_hang_count = 0;
+ return r;
+}
+
static int suspend_all_queues_mes(struct device_queue_manager *dqm)
{
struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
@@ -367,10 +501,13 @@ static int suspend_all_queues_mes(struct device_queue_manager *dqm)
if (!down_read_trylock(&adev->reset_domain->sem))
return -EIO;
- r = amdgpu_mes_suspend(adev);
+ r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
up_read(&adev->reset_domain->sem);
if (r) {
+ if (!reset_queues_mes(dqm))
+ return 0;
+
dev_err(adev->dev, "failed to suspend gangs from MES\n");
dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
kfd_hws_hang(dqm);
@@ -387,7 +524,7 @@ static int resume_all_queues_mes(struct device_queue_manager *dqm)
if (!down_read_trylock(&adev->reset_domain->sem))
return -EIO;
- r = amdgpu_mes_resume(adev);
+ r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1);
up_read(&adev->reset_domain->sem);
if (r) {
@@ -475,6 +612,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd,
} else {
/* For CP queues on SOC15 */
if (restore_id) {
+ if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
+ return -EINVAL;
+
/* make sure that ID is free */
if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
return -EINVAL;
@@ -572,7 +712,7 @@ static int allocate_vmid(struct device_queue_manager *dqm,
qpd->vmid,
qpd->page_table_base);
/* invalidate the VM context after pasid and vmid mapping is set up */
- kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(qpd_to_pdd(qpd));
if (dqm->dev->kfd2kgd->set_scratch_backing_va)
dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
@@ -610,7 +750,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
if (flush_texture_cache_nocpsch(q->device, qpd))
dev_err(dev, "Failed to flush TC\n");
- kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(qpd_to_pdd(qpd));
/* Release the vmid mapping */
set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
@@ -1284,7 +1424,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
dqm->dev->adev,
qpd->vmid,
qpd->page_table_base);
- kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(pdd);
}
/* Take a safe reference to the mm_struct, which may otherwise
@@ -1587,6 +1727,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
}
if (restore_sdma_id) {
+ if (*restore_sdma_id >= get_num_sdma_queues(dqm))
+ return -EINVAL;
+
/* Re-use existing sdma_id */
if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
dev_err(dev, "SDMA queue already in use\n");
@@ -1613,6 +1756,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
return -ENOMEM;
}
if (restore_sdma_id) {
+ if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm))
+ return -EINVAL;
+
/* Re-use existing sdma_id */
if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
dev_err(dev, "SDMA queue already in use\n");
@@ -1797,10 +1943,11 @@ static int halt_cpsch(struct device_queue_manager *dqm)
static int unhalt_cpsch(struct device_queue_manager *dqm)
{
int ret = 0;
+ struct amdgpu_device *adev = dqm->dev->adev;
dqm_lock(dqm);
if (!dqm->sched_running || !dqm->sched_halt) {
- WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");
+ dev_dbg(adev->dev, "Scheduling is not on halt.\n");
dqm_unlock(dqm);
return 0;
}
@@ -1821,6 +1968,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
{
struct device *dev = dqm->dev->adev->dev;
int retval, num_hw_queue_slots;
+ struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
+ int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
+ int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
dqm_lock(dqm);
@@ -1870,6 +2020,11 @@ static int start_cpsch(struct device_queue_manager *dqm)
goto fail_detect_hang_buffer;
}
+ dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL);
+ dqm->hqd_info = kzalloc(
+ hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info),
+ GFP_KERNEL);
+
dqm_unlock(dqm);
return 0;
@@ -1910,6 +2065,9 @@ static int stop_cpsch(struct device_queue_manager *dqm)
pm_uninit(&dqm->packet_mgr);
kfree(dqm->detect_hang_info);
dqm->detect_hang_info = NULL;
+ kfree(dqm->hung_db_array);
+ kfree(dqm->hqd_info);
+
dqm_unlock(dqm);
return ret;
@@ -2137,6 +2295,7 @@ static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q
q->properties.queue_id, pdd->process->lead_thread->pid);
pdd->has_reset_queue = true;
+ q->properties.is_reset = true;
if (q->properties.is_active) {
q->properties.is_active = false;
decrement_queue_count(dqm, qpd, q);
@@ -2203,6 +2362,23 @@ static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uin
return NULL;
}
+static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset)
+{
+ struct device_process_node *cur;
+ struct qcm_process_device *qpd;
+ struct queue *q;
+
+ list_for_each_entry(cur, &dqm->queues, list) {
+ qpd = cur->qpd;
+ list_for_each_entry(q, &qpd->queues_list, list) {
+ if (doorbell_offset == q->properties.doorbell_off)
+ return q;
+ }
+ }
+
+ return NULL;
+}
+
static int reset_hung_queues(struct device_queue_manager *dqm)
{
int r = 0, reset_count = 0, i;
@@ -2970,6 +3146,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
dqm->ops.reset_queues = reset_queues_cpsch;
dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
dqm->ops.checkpoint_mqd = checkpoint_mqd;
+ dqm->ops.set_perfcount = set_perfcount;
break;
case KFD_SCHED_POLICY_NO_HWS:
/* initialize dqm for no cp scheduling */
@@ -2990,6 +3167,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
dqm->ops.get_wave_state = get_wave_state;
dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
dqm->ops.checkpoint_mqd = checkpoint_mqd;
+ dqm->ops.set_perfcount = set_perfcount;
break;
default:
dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 3272328da11f..e0b6a47e7722 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -32,7 +32,6 @@
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
-
#define VMID_NUM 16
#define KFD_MES_PROCESS_QUANTUM 100000
@@ -200,6 +199,8 @@ struct device_queue_manager_ops {
const struct queue *q,
void *mqd,
void *ctl_stack);
+ void (*set_perfcount)(struct device_queue_manager *dqm,
+ int enable);
};
struct device_queue_manager_asic_ops {
@@ -285,6 +286,9 @@ struct device_queue_manager {
struct dqm_detect_hang_info *detect_hang_info;
size_t detect_hang_info_size;
int detect_hang_count;
+ /* for per-queue reset with mes */
+ u32 *hung_db_array;
+ struct amdgpu_mes_hung_queue_hqd_info *hqd_info;
};
void device_queue_manager_init_cik(
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
index 9e70a5f8a50b..525989371378 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
@@ -54,6 +54,7 @@ static int update_qpd_v12_1(struct device_queue_manager *dqm,
struct kfd_process_device *pdd;
struct amdgpu_device *adev = dqm->dev->adev;
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+ bool xnack_enabled;
pdd = qpd_to_pdd(qpd);
qpd->vm_cntx_cntl = hub->vm_cntx_cntl;
@@ -71,16 +72,18 @@ static int update_qpd_v12_1(struct device_queue_manager *dqm,
qpd->sh_mem_ape1_base = 0;
}
- if (KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev)) {
- if (!pdd->process->xnack_enabled) {
- qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
- qpd->vm_cntx_cntl &=
+ xnack_enabled = KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev) ?
+ pdd->process->xnack_enabled :
+ !pdd->dev->kfd->noretry;
+
+ if (!xnack_enabled) {
+ qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
+ qpd->vm_cntx_cntl &=
~(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
- } else {
- qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
- qpd->vm_cntx_cntl |=
+ } else {
+ qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
+ qpd->vm_cntx_cntl |=
(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
- }
}
qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 44150a71ffd5..a11c4ab3aafd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -483,6 +483,11 @@ int kfd_criu_restore_event(struct file *devkfd,
}
*priv_data_offset += sizeof(*ev_priv);
+ if (ev_priv->event_id > INT_MAX) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
if (ev_priv->user_handle) {
ret = kfd_kmap_event_page(p, ev_priv->user_handle);
if (ret)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 964efa325908..28dc6886c1ff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -129,13 +129,14 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
struct dma_fence **mfence)
{
const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
- struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+ struct amdgpu_ring *ring;
struct amdgpu_ttm_buffer_entity *entity;
u64 gart_s, gart_d;
struct dma_fence *next;
u64 size;
int r;
+ ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
entity = &adev->mman.move_entities[0];
mutex_lock(&entity->lock);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 77fb41e2486a..8e8ec266ca46 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -123,10 +123,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
*/
m->cp_hqd_hq_scheduler0 = 1 << 14;
- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- }
if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
@@ -141,6 +140,12 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -220,6 +225,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
if (mm->dev->kfd->cwsr_enabled)
m->cp_hqd_ctx_save_control = 0;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
+
update_cu_mask(mm, mqd, minfo);
set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index a1e3cf2384dd..7568e7ed5244 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -163,10 +163,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
m->cp_hqd_hq_status0 |= 1 << 29;
- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- }
if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
@@ -181,6 +180,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -258,6 +262,12 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
}
if (mm->dev->kfd->cwsr_enabled)
m->cp_hqd_ctx_save_control = 0;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
update_cu_mask(mm, mqd, minfo);
set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
index b3e122d7876e..8c815f129614 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
@@ -138,10 +138,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
m->cp_hqd_hq_status0 |= 1 << 29;
- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- }
if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
@@ -156,6 +155,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index c90c0d99b1e3..475589b924e9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -32,6 +32,10 @@
#include "amdgpu_amdkfd.h"
#include "kfd_device_queue_manager.h"
+static void update_mqd(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q,
+ struct mqd_update_info *minfo);
+
static inline struct v12_1_compute_mqd *get_mqd(void *mqd)
{
return (struct v12_1_compute_mqd *)mqd;
@@ -215,7 +219,7 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
*mqd = m;
if (gart_addr)
*gart_addr = addr;
- mm->update_mqd(mm, m, q, NULL);
+ update_mqd(mm, m, q, NULL);
}
static int load_mqd(struct mqd_manager *mm, void *mqd,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index e8f97de9d6e4..17bfb419b202 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -227,10 +227,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- if (q->tba_addr) {
+ if (q->tba_addr)
m->compute_pgm_rsrc2 |=
(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
- }
if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
m->cp_hqd_persistent_state |=
@@ -245,6 +244,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -327,6 +331,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
m->cp_hqd_ctx_save_control = 0;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
+
if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3) &&
KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 4) &&
KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 5, 0))
@@ -364,11 +375,15 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
{
struct v9_mqd *m;
struct kfd_context_save_area_header header;
+ u32 cntl_stack_size;
+ u32 cntl_stack_offset;
/* Control stack is located one page after MQD. */
void *mqd_ctl_stack = (void *)((uintptr_t)mqd + AMDGPU_GPU_PAGE_SIZE);
m = get_mqd(mqd);
+ cntl_stack_size = min_t(u32, m->cp_hqd_cntl_stack_size, q->ctl_stack_size);
+ cntl_stack_offset = min_t(u32, m->cp_hqd_cntl_stack_offset, cntl_stack_size);
*ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
m->cp_hqd_cntl_stack_offset;
@@ -384,9 +399,10 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state)))
return -EFAULT;
- if (copy_to_user(ctl_stack + m->cp_hqd_cntl_stack_offset,
- mqd_ctl_stack + m->cp_hqd_cntl_stack_offset,
- *ctl_stack_used_size))
+ *ctl_stack_used_size = cntl_stack_size - cntl_stack_offset;
+
+ if (copy_to_user(ctl_stack + cntl_stack_offset, mqd_ctl_stack + cntl_stack_offset,
+ *ctl_stack_used_size))
return -EFAULT;
return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 431a20323146..c86779af323b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -148,6 +148,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
m->cp_hqd_wg_state_offset = q->ctl_stack_size;
}
+ mutex_lock(&mm->dev->kfd->profiler_lock);
+ if (mm->dev->kfd->profiler_process != NULL)
+ m->compute_perfcount_enable = 1;
+ mutex_unlock(&mm->dev->kfd->profiler_lock);
+
*mqd = m;
if (gart_addr)
*gart_addr = addr;
@@ -230,6 +235,12 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd,
m->cp_hqd_ctx_save_control =
atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
+ if (minfo) {
+ if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+ m->compute_perfcount_enable = 1;
+ else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+ m->compute_perfcount_enable = 0;
+ }
update_cu_mask(mm, mqd, minfo);
set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 6e333bfa17d6..acd0e41e744c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -215,8 +215,7 @@ enum cache_policy {
((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4)) || \
- (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) || \
- (KFD_GC_VERSION(dev) == IP_VERSION(12, 1, 0)))
+ (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)))
struct kfd_node;
@@ -384,6 +383,11 @@ struct kfd_dev {
int kfd_dev_lock;
atomic_t kfd_processes_count;
+
+ /* Lock for profiler process */
+ struct mutex profiler_lock;
+ /* Process currently holding the lock */
+ struct kfd_process *profiler_process;
};
enum kfd_mempool {
@@ -395,6 +399,7 @@ enum kfd_mempool {
/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen);
/**
* enum kfd_unmap_queues_filter - Enum for queue filters.
@@ -523,6 +528,7 @@ struct queue_properties {
uint32_t pm4_target_xcc;
bool is_dbg_wa;
bool is_user_cu_masked;
+ bool is_reset;
/* Not relevant for user mode queues in cp scheduling */
unsigned int vmid;
/* Relevant only for sdma queues*/
@@ -556,6 +562,8 @@ enum mqd_update_flag {
UPDATE_FLAG_DBG_WA_ENABLE = 1,
UPDATE_FLAG_DBG_WA_DISABLE = 2,
UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
+ UPDATE_FLAG_PERFCOUNT_ENABLE = 5,
+ UPDATE_FLAG_PERFCOUNT_DISABLE = 6,
};
struct mqd_update_info {
@@ -865,6 +873,8 @@ struct kfd_process_device {
bool has_reset_queue;
u32 pasid;
+ /* Indicates this process has requested PTL stay disabled */
+ bool ptl_disable_req;
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
@@ -986,9 +996,6 @@ struct kfd_process {
struct kobject *kobj_queues;
struct attribute attr_pasid;
- /* Keep track cwsr init */
- bool has_cwsr;
-
/* Exception code enable mask and status */
uint64_t exception_enable_mask;
uint64_t exception_status;
@@ -1047,10 +1054,13 @@ extern struct srcu_struct kfd_processes_srcu;
typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
void *data);
+typedef int amdkfd_ioctl_validate_t(void *kdata, unsigned int usize);
+
struct amdkfd_ioctl_desc {
unsigned int cmd;
int flags;
amdkfd_ioctl_t *func;
+ amdkfd_ioctl_validate_t *validate;
unsigned int cmd_drv;
const char *name;
};
@@ -1101,8 +1111,6 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
-int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
- struct vm_area_struct *vma);
void kfd_process_notifier_release_internal(struct kfd_process *p);
/* KFD process API for creating and translating handles */
@@ -1219,9 +1227,6 @@ void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
bool enabled);
-/* CWSR initialization */
-int kfd_process_init_cwsr_apu(struct kfd_process *process, struct file *filep);
-
/* CRIU */
/*
* Need to increment KFD_CRIU_PRIV_VERSION each time a change is made to any of the CRIU private
@@ -1551,13 +1556,13 @@ void kfd_signal_reset_event(struct kfd_node *dev);
void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
void kfd_signal_process_terminate_event(struct kfd_process *p);
-static inline void kfd_flush_tlb(struct kfd_process_device *pdd,
- enum TLB_FLUSH_TYPE type)
+static inline void kfd_flush_tlb(struct kfd_process_device *pdd)
{
struct amdgpu_device *adev = pdd->dev->adev;
struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
- amdgpu_vm_flush_compute_tlb(adev, vm, type, pdd->dev->xcc_mask);
+ amdgpu_vm_flush_compute_tlb(adev, vm, TLB_FLUSH_HEAVYWEIGHT,
+ pdd->dev->xcc_mask);
}
static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)
@@ -1601,6 +1606,12 @@ static inline bool kfd_is_first_node(struct kfd_node *node)
return (node == node->kfd->nodes[0]);
}
+/* PTL support */
+int kfd_ptl_disable_request(struct kfd_process_device *pdd,
+ struct kfd_process *p);
+int kfd_ptl_disable_release(struct kfd_process_device *pdd,
+ struct kfd_process *p);
+
/* Debugfs */
#if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index d28ca581cad0..368283d53077 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1106,6 +1106,16 @@ static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
kfd_process_device_free_bos(p->pdds[i]);
}
+static void kfd_process_profiler_release(struct kfd_process *p, struct kfd_process_device *pdd)
+{
+ mutex_lock(&pdd->dev->kfd->profiler_lock);
+ if (pdd->dev->kfd->profiler_process == p) {
+ pdd->qpd.dqm->ops.set_perfcount(pdd->qpd.dqm, 0);
+ pdd->dev->kfd->profiler_process = NULL;
+ }
+ mutex_unlock(&pdd->dev->kfd->profiler_lock);
+}
+
static void kfd_process_destroy_pdds(struct kfd_process *p)
{
int i;
@@ -1117,6 +1127,11 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
pdd->dev->id, p->lead_thread->pid);
+ kfd_process_profiler_release(p, pdd);
+
+ if (pdd->ptl_disable_req)
+ kfd_ptl_disable_release(pdd, p);
+
kfd_process_device_destroy_cwsr_dgpu(pdd);
kfd_process_device_destroy_ib_mem(pdd);
@@ -1409,50 +1424,6 @@ void kfd_cleanup_processes(void)
mmu_notifier_synchronize();
}
-int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
-{
- unsigned long offset;
- int i;
-
- if (p->has_cwsr)
- return 0;
-
- for (i = 0; i < p->n_pdds; i++) {
- struct kfd_node *dev = p->pdds[i]->dev;
- struct qcm_process_device *qpd = &p->pdds[i]->qpd;
-
- if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
- continue;
-
- offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
- qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
- KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
- MAP_SHARED, offset);
-
- if (IS_ERR_VALUE(qpd->tba_addr)) {
- int err = qpd->tba_addr;
-
- dev_err(dev->adev->dev,
- "Failure to set tba address. error %d.\n", err);
- qpd->tba_addr = 0;
- qpd->cwsr_kaddr = NULL;
- return err;
- }
-
- memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
-
- kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
-
- qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
- pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
- qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
- }
-
- p->has_cwsr = true;
-
- return 0;
-}
-
static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
{
struct kfd_node *dev = pdd->dev;
@@ -2228,38 +2199,6 @@ int kfd_resume_all_processes(void)
return ret;
}
-int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
- struct vm_area_struct *vma)
-{
- struct kfd_process_device *pdd;
- struct qcm_process_device *qpd;
-
- if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
- dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
- return -EINVAL;
- }
-
- pdd = kfd_get_process_device_data(dev, process);
- if (!pdd)
- return -EINVAL;
- qpd = &pdd->qpd;
-
- qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
- get_order(KFD_CWSR_TBA_TMA_SIZE));
- if (!qpd->cwsr_kaddr) {
- dev_err(dev->adev->dev,
- "Error allocating per process CWSR buffer.\n");
- return -ENOMEM;
- }
-
- vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
- | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
- /* Mapping pages to user process */
- return remap_pfn_range(vma, vma->vm_start,
- PFN_DOWN(__pa(qpd->cwsr_kaddr)),
- KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
-}
-
/* assumes caller holds process lock. */
int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
{
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index b120fdb0ef77..35ec67d9739b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1366,6 +1366,12 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
pr_debug("CPU[0x%llx 0x%llx] -> GPU[0x%llx 0x%llx]\n", start, last,
gpu_start, gpu_end);
+
+ if (!amdgpu_vm_ready(vm)) {
+ pr_debug("VM not ready, canceling unmap\n");
+ return -EINVAL;
+ }
+
return amdgpu_vm_update_range(adev, vm, false, true, true, false, NULL, gpu_start,
gpu_end, init_pte_value, 0, 0, NULL, NULL,
fence);
@@ -1418,7 +1424,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
if (r)
break;
}
- kfd_flush_tlb(pdd, TLB_FLUSH_HEAVYWEIGHT);
+ kfd_flush_tlb(pdd);
}
return r;
@@ -1443,6 +1449,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms,
last_start, last_start + npages - 1, readonly);
+ if (!amdgpu_vm_ready(vm)) {
+ pr_debug("VM not ready, canceling map\n");
+ return -EINVAL;
+ }
+
for (i = offset; i < offset + npages; i++) {
uint64_t gpu_start;
uint64_t gpu_end;
@@ -1560,7 +1571,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
}
}
- kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(pdd);
}
return r;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 29dee26261ab..630f46091dc7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -514,7 +514,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
dev->node_props.capability |=
HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
- if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) &&
+ if ((KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) ||
+ KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 1, 0)) &&
(dev->gpu->adev->sdma.supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE))
dev->node_props.capability2 |= HSA_CAP2_PER_SDMA_QUEUE_RESET_SUPPORTED;
@@ -2027,6 +2028,7 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev)
if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 1, 0)) {
dev->node_props.capability |=
HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED;
+ dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED;
dev->node_props.capability2 |=
HSA_CAP2_TRAP_DEBUG_LDS_OUT_OF_ADDR_RANGE_SUPPORTED;
}