summaryrefslogtreecommitdiff
path: root/drivers/accel/amdxdna
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/accel/amdxdna')
-rw-r--r--drivers/accel/amdxdna/aie.c39
-rw-r--r--drivers/accel/amdxdna/aie.h4
-rw-r--r--drivers/accel/amdxdna/aie2_ctx.c132
-rw-r--r--drivers/accel/amdxdna/aie2_error.c9
-rw-r--r--drivers/accel/amdxdna/aie2_message.c19
-rw-r--r--drivers/accel/amdxdna/aie2_msg_priv.h1
-rw-r--r--drivers/accel/amdxdna/aie2_pci.c30
-rw-r--r--drivers/accel/amdxdna/aie2_pci.h8
-rw-r--r--drivers/accel/amdxdna/aie4_pci.c8
-rw-r--r--drivers/accel/amdxdna/amdxdna_error.h10
-rw-r--r--drivers/accel/amdxdna/amdxdna_gem.c461
-rw-r--r--drivers/accel/amdxdna/amdxdna_gem.h6
-rw-r--r--drivers/accel/amdxdna/amdxdna_mailbox.c221
-rw-r--r--drivers/accel/amdxdna/amdxdna_mailbox.h9
-rw-r--r--drivers/accel/amdxdna/amdxdna_mailbox_helper.c2
-rw-r--r--drivers/accel/amdxdna/amdxdna_mailbox_helper.h1
-rw-r--r--drivers/accel/amdxdna/amdxdna_pci_drv.c6
-rw-r--r--drivers/accel/amdxdna/amdxdna_pci_drv.h9
-rw-r--r--drivers/accel/amdxdna/npu4_regs.c1
19 files changed, 706 insertions, 270 deletions
diff --git a/drivers/accel/amdxdna/aie.c b/drivers/accel/amdxdna/aie.c
index dd6f36f222c7..bf8eb66baab8 100644
--- a/drivers/accel/amdxdna/aie.c
+++ b/drivers/accel/amdxdna/aie.c
@@ -118,6 +118,45 @@ void amdxdna_vbnv_init(struct amdxdna_dev *xdna)
amdxdna_update_vbnv(xdna, info->rev_vbnv_tbl, rev);
}
+void amdxdna_io_stats_job_start(struct amdxdna_client *client)
+{
+ int depth;
+
+ guard(spinlock)(&client->io_stats.lock);
+
+ depth = client->io_stats.job_depth++;
+ if (!depth)
+ client->io_stats.start_time = ktime_get_ns();
+}
+
+void amdxdna_io_stats_job_done(struct amdxdna_client *client)
+{
+ u64 busy_ns;
+ int depth;
+
+ guard(spinlock)(&client->io_stats.lock);
+
+ depth = --client->io_stats.job_depth;
+ if (!depth) {
+ busy_ns = ktime_get_ns() - client->io_stats.start_time;
+ client->io_stats.start_time = 0;
+ client->io_stats.busy_time += busy_ns;
+ }
+}
+
+u64 amdxdna_io_stats_busy_time_ns(struct amdxdna_client *client)
+{
+ u64 busy_ns;
+
+ guard(spinlock)(&client->io_stats.lock);
+
+ busy_ns = client->io_stats.busy_time;
+ if (client->io_stats.job_depth)
+ busy_ns += ktime_get_ns() - client->io_stats.start_time;
+
+ return busy_ns;
+}
+
int amdxdna_get_metadata(struct aie_device *aie,
struct amdxdna_client *client,
struct amdxdna_drm_get_info *args)
diff --git a/drivers/accel/amdxdna/aie.h b/drivers/accel/amdxdna/aie.h
index 0483d582b7f8..1fc5ddfe696a 100644
--- a/drivers/accel/amdxdna/aie.h
+++ b/drivers/accel/amdxdna/aie.h
@@ -117,4 +117,8 @@ void aie_smu_fini(struct smu_device *smu);
int aie_smu_set_clocks(struct smu_device *smu, u32 *npuclk, u32 *hclk);
int aie_smu_set_dpm(struct smu_device *smu, u32 dpm_level);
+void amdxdna_io_stats_job_start(struct amdxdna_client *client);
+void amdxdna_io_stats_job_done(struct amdxdna_client *client);
+u64 amdxdna_io_stats_busy_time_ns(struct amdxdna_client *client);
+
#endif /* _AIE_H_ */
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index baf9a8b90a4d..9c6b7353cbe9 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -7,6 +7,7 @@
#include <drm/drm_device.h>
#include <drm/drm_gem.h>
#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_managed.h>
#include <drm/drm_print.h>
#include <drm/drm_syncobj.h>
#include <linux/hmm.h>
@@ -99,9 +100,9 @@ static void aie2_job_put(struct amdxdna_sched_job *job)
static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx,
struct drm_sched_job *bad_job)
{
- drm_sched_stop(&hwctx->priv->sched, bad_job);
+ drm_sched_stop(hwctx->priv->sched, bad_job);
aie2_destroy_context(xdna->dev_handle, hwctx);
- drm_sched_start(&hwctx->priv->sched, 0);
+ drm_sched_start(hwctx->priv->sched, 0);
}
static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx)
@@ -289,6 +290,7 @@ aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size)
int ret = 0;
u32 status;
+ amdxdna_io_stats_job_done(job->hwctx->client);
cmd_abo = job->cmd_bo;
if (unlikely(job->job_timeout)) {
@@ -321,6 +323,8 @@ aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size)
struct amdxdna_sched_job *job = handle;
int ret = 0;
+ amdxdna_io_stats_job_done(job->hwctx->client);
+
if (unlikely(!data || size != sizeof(u32))) {
job->drv_cmd->result = U32_MAX;
ret = -EINVAL;
@@ -343,6 +347,7 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size)
u32 cmd_status;
int ret = 0;
+ amdxdna_io_stats_job_done(job->hwctx->client);
cmd_abo = job->cmd_bo;
if (unlikely(job->job_timeout)) {
@@ -404,6 +409,7 @@ aie2_sched_job_run(struct drm_sched_job *sched_job)
kref_get(&job->refcnt);
fence = dma_fence_get(job->fence);
+ amdxdna_io_stats_job_start(job->hwctx->client);
if (job->drv_cmd) {
switch (job->drv_cmd->opcode) {
case SYNC_DEBUG_BO:
@@ -431,6 +437,7 @@ aie2_sched_job_run(struct drm_sched_job *sched_job)
out:
if (ret) {
+ amdxdna_io_stats_job_done(job->hwctx->client);
dma_fence_put(job->fence);
aie2_job_put(job);
mmput(job->mm);
@@ -659,10 +666,19 @@ static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx)
drm_syncobj_put(hwctx->priv->syncobj);
}
-int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
+void aie2_hwctx_sched_fini(struct amdxdna_dev_hdl *ndev)
{
- struct amdxdna_client *client = hwctx->client;
- struct amdxdna_dev *xdna = client->xdna;
+ int i;
+
+ for (i = 0; i < ndev->priv->hwctx_limit; i++)
+ drm_sched_fini(&ndev->hwctx_sched[i]);
+
+ ida_destroy(&ndev->hwctx_sched_ida);
+}
+
+int aie2_hwctx_sched_init(struct amdxdna_dev_hdl *ndev)
+{
+ struct amdxdna_dev *xdna = ndev->aie.xdna;
const struct drm_sched_init_args args = {
.ops = &sched_ops,
.num_rqs = DRM_SCHED_PRIORITY_COUNT,
@@ -673,7 +689,55 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
.name = "amdxdna_js",
.dev = xdna->ddev.dev,
};
- struct drm_gpu_scheduler *sched;
+ int i, ret;
+
+ ndev->hwctx_sched = drmm_kcalloc(&xdna->ddev, ndev->priv->hwctx_limit,
+ sizeof(*ndev->hwctx_sched), GFP_KERNEL);
+ if (!ndev->hwctx_sched)
+ return -ENOMEM;
+
+ for (i = 0; i < ndev->priv->hwctx_limit; i++) {
+ ret = drm_sched_init(&ndev->hwctx_sched[i], &args);
+ if (ret) {
+ XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);
+ goto fini_sched;
+ }
+ }
+
+ ida_init(&ndev->hwctx_sched_ida);
+
+ return 0;
+
+fini_sched:
+ while (--i >= 0)
+ drm_sched_fini(&ndev->hwctx_sched[i]);
+
+ return ret;
+}
+
+static struct drm_gpu_scheduler *aie2_hwctx_sched_alloc(struct amdxdna_dev_hdl *ndev)
+{
+ int ret;
+
+ ret = ida_alloc_range(&ndev->hwctx_sched_ida, 0,
+ ndev->priv->hwctx_limit - 1, GFP_KERNEL);
+ if (ret < 0)
+ return ERR_PTR(ret);
+
+ return &ndev->hwctx_sched[ret];
+}
+
+static void aie2_hwctx_sched_free(struct amdxdna_dev_hdl *ndev,
+ struct drm_gpu_scheduler *sched)
+{
+ ida_free(&ndev->hwctx_sched_ida, sched - ndev->hwctx_sched);
+}
+
+int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
+{
+ struct amdxdna_client *client = hwctx->client;
+ struct amdxdna_dev *xdna = client->xdna;
+ struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
struct amdxdna_hwctx_priv *priv;
struct amdxdna_gem_obj *heap;
int i, ret;
@@ -683,13 +747,26 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
return -ENOMEM;
hwctx->priv = priv;
+ priv->sched = aie2_hwctx_sched_alloc(ndev);
+ if (IS_ERR(priv->sched)) {
+ ret = PTR_ERR(priv->sched);
+ goto free_priv;
+ }
+
+ ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL,
+ &priv->sched, 1, NULL);
+ if (ret) {
+ XDNA_ERR(xdna, "Failed to initial sched entity. ret %d", ret);
+ goto free_sched;
+ }
+
mutex_lock(&client->mm_lock);
heap = xa_load(&client->dev_heap_xa, 0);
if (!heap) {
XDNA_ERR(xdna, "The client dev heap object not exist");
mutex_unlock(&client->mm_lock);
ret = -ENOENT;
- goto free_priv;
+ goto free_entity;
}
drm_gem_object_get(to_gobj(heap));
mutex_unlock(&client->mm_lock);
@@ -722,30 +799,16 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
priv->cmd_buf[i] = abo;
}
- sched = &priv->sched;
mutex_init(&priv->io_lock);
fs_reclaim_acquire(GFP_KERNEL);
might_lock(&priv->io_lock);
fs_reclaim_release(GFP_KERNEL);
- ret = drm_sched_init(sched, &args);
- if (ret) {
- XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);
- goto free_cmd_bufs;
- }
-
- ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL,
- &sched, 1, NULL);
- if (ret) {
- XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret);
- goto free_sched;
- }
-
ret = aie2_hwctx_col_list(hwctx);
if (ret) {
XDNA_ERR(xdna, "Create col list failed, ret %d", ret);
- goto free_entity;
+ goto free_cmd_bufs;
}
ret = amdxdna_pm_resume_get_locked(xdna);
@@ -791,19 +854,20 @@ suspend_put:
amdxdna_pm_suspend_put(xdna);
free_col_list:
kfree(hwctx->col_list);
-free_entity:
- drm_sched_entity_destroy(&priv->entity);
-free_sched:
- drm_sched_fini(&priv->sched);
free_cmd_bufs:
for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {
if (!priv->cmd_buf[i])
continue;
+ amdxdna_gem_heap_free(client, priv->cmd_buf[i]);
drm_gem_object_put(to_gobj(priv->cmd_buf[i]));
}
amdxdna_gem_unpin(heap);
put_heap:
drm_gem_object_put(to_gobj(heap));
+free_entity:
+ drm_sched_entity_destroy(&priv->entity);
+free_sched:
+ aie2_hwctx_sched_free(ndev, priv->sched);
free_priv:
kfree(priv);
return ret;
@@ -811,18 +875,20 @@ free_priv:
void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
{
+ struct amdxdna_dev_hdl *ndev;
struct amdxdna_dev *xdna;
int idx;
xdna = hwctx->client->xdna;
+ ndev = xdna->dev_handle;
XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq);
aie2_hwctx_wait_for_idle(hwctx);
/* Request fw to destroy hwctx and cancel the rest pending requests */
- drm_sched_stop(&hwctx->priv->sched, NULL);
+ drm_sched_stop(hwctx->priv->sched, NULL);
aie2_release_resource(hwctx);
- drm_sched_start(&hwctx->priv->sched, 0);
+ drm_sched_start(hwctx->priv->sched, 0);
mutex_unlock(&xdna->dev_lock);
drm_sched_entity_destroy(&hwctx->priv->entity);
@@ -833,11 +899,17 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
atomic64_read(&hwctx->job_free_cnt));
mutex_lock(&xdna->dev_lock);
- drm_sched_fini(&hwctx->priv->sched);
+ aie2_hwctx_sched_free(ndev, hwctx->priv->sched);
aie2_ctx_syncobj_destroy(hwctx);
- for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++)
+ for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) {
+ /*
+ * The open/close will never be called for driver allocated
+ * dev bo. Call amdxdna_gem_heap_free explicitly.
+ */
+ amdxdna_gem_heap_free(hwctx->client, hwctx->priv->cmd_buf[idx]);
drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx]));
+ }
amdxdna_gem_unpin(hwctx->priv->heap);
drm_gem_object_put(to_gobj(hwctx->priv->heap));
diff --git a/drivers/accel/amdxdna/aie2_error.c b/drivers/accel/amdxdna/aie2_error.c
index babdac0157ab..8516574e33eb 100644
--- a/drivers/accel/amdxdna/aie2_error.c
+++ b/drivers/accel/amdxdna/aie2_error.c
@@ -343,15 +343,14 @@ void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev)
kfree(events);
}
-int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
+int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev, u32 num_events)
{
struct amdxdna_dev *xdna = ndev->aie.xdna;
- u32 total_col = ndev->total_col;
- u32 total_size = ASYNC_BUF_SIZE * total_col;
+ u32 total_size = ASYNC_BUF_SIZE * num_events;
struct async_events *events;
int i, ret;
- events = kzalloc_flex(*events, event, total_col);
+ events = kzalloc_flex(*events, event, num_events);
if (!events)
return -ENOMEM;
@@ -361,7 +360,7 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
goto free_events;
}
events->size = total_size;
- events->event_cnt = total_col;
+ events->event_cnt = num_events;
events->wq = alloc_ordered_workqueue("async_wq", 0);
if (!events->wq) {
diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c
index b4c49259a1a2..56b919b350ac 100644
--- a/drivers/accel/amdxdna/aie2_message.c
+++ b/drivers/accel/amdxdna/aie2_message.c
@@ -261,8 +261,9 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
goto del_ctx_req;
}
+ /* +1 for the config_cu message that can be in-flight concurrently */
ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x,
- intr_reg, ret);
+ intr_reg, ret, HWCTX_MAX_CMDS + 1);
if (ret) {
XDNA_ERR(xdna, "Not able to create channel");
ret = -EINVAL;
@@ -277,6 +278,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
free_channel:
xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
+ hwctx->priv->mbox_chann = NULL;
del_ctx_req:
aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
return ret;
@@ -485,7 +487,7 @@ int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr,
req.buf_size = size;
XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size);
- return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg, TX_TIMEOUT);
+ return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg);
}
int aie2_config_cu(struct amdxdna_hwctx *hwctx,
@@ -544,7 +546,8 @@ int aie2_config_cu(struct amdxdna_hwctx *hwctx,
msg.handle = hwctx;
msg.opcode = MSG_OP_CONFIG_CU;
msg.notify_cb = notify_cb;
- return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
+
+ return xdna_mailbox_send_msg(chann, &msg);
}
static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
@@ -949,7 +952,7 @@ int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req,
0x40, false);
- ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
+ ret = xdna_mailbox_send_msg(chann, &msg);
if (ret) {
XDNA_ERR(xdna, "Send message failed");
return ret;
@@ -1038,7 +1041,7 @@ int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
msg.send_size = sizeof(req);
print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
&req, msg.send_size, false);
- ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
+ ret = xdna_mailbox_send_msg(chann, &msg);
if (ret) {
XDNA_ERR(xdna, "Send message failed");
return ret;
@@ -1085,7 +1088,7 @@ int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
msg.send_size = sizeof(req);
print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
&req, msg.send_size, false);
- ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
+ ret = xdna_mailbox_send_msg(chann, &msg);
if (ret) {
XDNA_ERR(hwctx->client->xdna, "Send message failed");
return ret;
@@ -1121,7 +1124,7 @@ int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
msg.send_size = sizeof(req);
msg.opcode = MSG_OP_SYNC_BO;
- ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
+ ret = xdna_mailbox_send_msg(chann, &msg);
if (ret) {
XDNA_ERR(xdna, "Send message failed");
return ret;
@@ -1156,7 +1159,7 @@ int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *
msg.send_size = sizeof(req);
msg.opcode = MSG_OP_CONFIG_DEBUG_BO;
- return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
+ return xdna_mailbox_send_msg(chann, &msg);
}
int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h
index fd65a4236d49..8ed2b5701dcc 100644
--- a/drivers/accel/amdxdna/aie2_msg_priv.h
+++ b/drivers/accel/amdxdna/aie2_msg_priv.h
@@ -530,6 +530,7 @@ enum aie2_dev_revision {
AIE2_DEV_REVISION_GPT1,
AIE2_DEV_REVISION_GPT2,
AIE2_DEV_REVISION_GPT3,
+ AIE2_DEV_REVISION_GRGH,
AIE2_DEV_REVISION_UNKN,
};
diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c
index daec1f6b4907..d2fa46e4d3a4 100644
--- a/drivers/accel/amdxdna/aie2_pci.c
+++ b/drivers/accel/amdxdna/aie2_pci.c
@@ -24,6 +24,7 @@
#include "aie2_pci.h"
#include "aie2_solver.h"
#include "amdxdna_ctx.h"
+#include "amdxdna_error.h"
#include "amdxdna_gem.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
@@ -385,11 +386,21 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
}
xdna_mailbox_intr_reg = ndev->aie.mgmt_i2x.mb_head_ptr_reg + 4;
+ /*
+ * At most AMDXDNA_MAX_ASYNC_EVENT_BUFS async event messages plus one
+ * management command can be unresponded. The async slots stay busy
+ * until firmware reports an error. Every management command holds
+ * dev_lock across aie_send_mgmt_msg_wait(), and the response clears
+ * the slot before that wait returns, so a second ioctl blocks on
+ * dev_lock rather than observing -ENOBUFS. Async re-registration
+ * takes the same lock after the RX path has freed its slot.
+ */
ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann,
&ndev->aie.mgmt_x2i,
&ndev->aie.mgmt_i2x,
xdna_mailbox_intr_reg,
- mgmt_mb_irq);
+ mgmt_mb_irq,
+ AMDXDNA_MAX_ASYNC_EVENT_BUFS + 1);
if (ret) {
XDNA_ERR(xdna, "failed to start management mailbox channel");
ret = -EINVAL;
@@ -414,7 +425,7 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
goto stop_fw;
}
- ret = aie2_error_async_events_alloc(ndev);
+ ret = aie2_error_async_events_alloc(ndev, AMDXDNA_MAX_ASYNC_EVENT_BUFS);
if (ret) {
XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret);
goto stop_fw;
@@ -503,10 +514,16 @@ static int aie2_init(struct amdxdna_dev *xdna)
ndev->priv = xdna->dev_info->dev_priv;
ndev->aie.xdna = xdna;
+ ret = aie2_hwctx_sched_init(ndev);
+ if (ret)
+ return ret;
+
for (i = 0; i < ARRAY_SIZE(npu_fw); i++) {
fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]);
- if (!fw_full_path)
- return -ENOMEM;
+ if (!fw_full_path) {
+ ret = -ENOMEM;
+ goto free_hwctx_sched;
+ }
ret = firmware_request_nowarn(&fw, fw_full_path, &pdev->dev);
kfree(fw_full_path);
@@ -519,7 +536,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
if (ret) {
XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
ndev->priv->fw_path, ret);
- return ret;
+ goto free_hwctx_sched;
}
ret = pcim_enable_device(pdev);
@@ -623,6 +640,8 @@ stop_hw:
aie2_hw_stop(xdna);
release_fw:
release_firmware(fw);
+free_hwctx_sched:
+ aie2_hwctx_sched_fini(ndev);
return ret;
}
@@ -631,6 +650,7 @@ static void aie2_fini(struct amdxdna_dev *xdna)
{
amdxdna_pm_fini(xdna);
aie2_hw_stop(xdna);
+ aie2_hwctx_sched_fini(xdna->dev_handle);
}
static int aie2_get_aie_status(struct amdxdna_client *client,
diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h
index ea1dac106400..a5fd71329b93 100644
--- a/drivers/accel/amdxdna/aie2_pci.h
+++ b/drivers/accel/amdxdna/aie2_pci.h
@@ -107,7 +107,7 @@ struct amdxdna_hwctx_priv {
struct amdxdna_gem_obj *heap;
void *mbox_chann;
- struct drm_gpu_scheduler sched;
+ struct drm_gpu_scheduler *sched;
struct drm_sched_entity entity;
struct mutex io_lock; /* protect seq and cmd order */
@@ -152,6 +152,8 @@ struct amdxdna_dev_hdl {
u32 total_col;
struct amdxdna_drm_query_aie_version version;
struct aie2_exec_msg_ops *exec_msg_ops;
+ struct drm_gpu_scheduler *hwctx_sched;
+ struct ida hwctx_sched_ida;
/* power management and clock*/
enum amdxdna_power_mode_type pw_mode;
@@ -242,7 +244,7 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type
int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
/* aie2_error.c */
-int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev);
+int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev, u32 num_events);
void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev);
int aie2_error_async_msg_thread(void *data);
int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev,
@@ -294,6 +296,8 @@ int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us);
/* aie2_hwctx.c */
int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);
void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx);
+int aie2_hwctx_sched_init(struct amdxdna_dev_hdl *ndev);
+void aie2_hwctx_sched_fini(struct amdxdna_dev_hdl *ndev);
int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size);
int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl);
void aie2_hwctx_suspend(struct amdxdna_client *client);
diff --git a/drivers/accel/amdxdna/aie4_pci.c b/drivers/accel/amdxdna/aie4_pci.c
index a58a83af42a4..e99ad35870ef 100644
--- a/drivers/accel/amdxdna/aie4_pci.c
+++ b/drivers/accel/amdxdna/aie4_pci.c
@@ -12,6 +12,7 @@
#include "aie.h"
#include "aie4_msg_priv.h"
#include "aie4_pci.h"
+#include "amdxdna_error.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_mailbox_helper.h"
#include "amdxdna_pci_drv.h"
@@ -181,11 +182,16 @@ static int aie4_mailbox_start(struct amdxdna_dev *xdna,
goto free_channel;
}
+ /*
+ * At any given time, at most AMDXDNA_MAX_ASYNC_EVENT_BUFS async event
+ * messages plus 1 other management command can be unresponded.
+ */
ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann,
&ndev->aie.mgmt_x2i,
&ndev->aie.mgmt_i2x,
NO_IOHUB,
- mgmt_mb_irq);
+ mgmt_mb_irq,
+ AMDXDNA_MAX_ASYNC_EVENT_BUFS + 1);
if (ret) {
XDNA_ERR(xdna, "failed to start management mailbox channel");
ret = -EINVAL;
diff --git a/drivers/accel/amdxdna/amdxdna_error.h b/drivers/accel/amdxdna/amdxdna_error.h
index c51de86ec12b..326d7153b17d 100644
--- a/drivers/accel/amdxdna/amdxdna_error.h
+++ b/drivers/accel/amdxdna/amdxdna_error.h
@@ -56,4 +56,14 @@ enum amdxdna_error_module {
(FIELD_PREP(AMDXDNA_EXTRA_ERR_COL_MASK, col) | \
FIELD_PREP(AMDXDNA_EXTRA_ERR_ROW_MASK, row))
+/*
+ * Maximum number of async event buffers. This value is used in two places:
+ * (1) to size the management mailbox channel, which must happen before
+ * total_col is known from firmware; (2) to allocate the async event pool,
+ * which runs after total_col is known. Both counts must match. The channel
+ * is sized AMDXDNA_MAX_ASYNC_EVENT_BUFS + 1 (one extra for other management
+ * commands), so the pool cannot exceed AMDXDNA_MAX_ASYNC_EVENT_BUFS slots.
+ */
+#define AMDXDNA_MAX_ASYNC_EVENT_BUFS 4
+
#endif /* _AMDXDNA_ERROR_H_ */
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 4b0d58d0329b..f14484bc31ec 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -14,6 +14,7 @@
#include <linux/dma-direct.h>
#include <linux/iosys-map.h>
#include <linux/pagemap.h>
+#include <linux/swap.h>
#include <linux/vmalloc.h>
#include "amdxdna_cbuf.h"
@@ -37,6 +38,7 @@ amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
struct amdxdna_gem_obj *heap;
u64 heap_addr, exp_heap_uva;
u32 heap_id;
+ int ret;
if (xa_empty(&client->dev_heap_xa)) {
XDNA_DBG(xdna, "Empty heap xa");
@@ -58,24 +60,32 @@ amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
heap = xa_load(&client->dev_heap_xa, heap_id);
exp_heap_uva = amdxdna_gem_uva(heap);
heap_addr = amdxdna_gem_dev_addr(heap);
- dev_bo->heap_start_id = heap_id;
dev_bo->mem.uva = dev_bo->mm_node.start - heap_addr + exp_heap_uva;
for (; heap_id < client->dev_heap_nid; heap_id++) {
heap = xa_load(&client->dev_heap_xa, heap_id);
if (!heap) {
XDNA_ERR(xdna, "Failed to load heap %d", heap_id);
- return -EINVAL;
+ ret = -EINVAL;
+ goto cleanup_heap_xa;
}
heap_addr = amdxdna_gem_uva(heap);
if (heap_addr == AMDXDNA_INVALID_ADDR) {
XDNA_ERR(xdna, "Heap %d is not mapped", heap_id);
- return -EAGAIN;
+ ret = -EAGAIN;
+ goto cleanup_heap_xa;
}
if (heap_addr != exp_heap_uva) {
XDNA_ERR(xdna, "Heap %d uva is not contiguous", heap_id);
- return -EINVAL;
+ ret = -EINVAL;
+ goto cleanup_heap_xa;
+ }
+
+ ret = xa_insert(&dev_bo->heap_xa, heap_id, heap, GFP_KERNEL);
+ if (ret) {
+ ret = -ENOMEM;
+ goto cleanup_heap_xa;
}
if (heap->dev_addr + heap->mem.size >=
@@ -87,12 +97,15 @@ amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
if (heap_id == client->dev_heap_nid) {
XDNA_DBG(xdna, "Can not find heap end");
- return -EAGAIN;
+ ret = -EAGAIN;
+ goto cleanup_heap_xa;
}
- dev_bo->heap_end_id = heap_id;
-
return 0;
+
+cleanup_heap_xa:
+ xa_destroy(&dev_bo->heap_xa);
+ return ret;
}
static int
@@ -132,8 +145,7 @@ amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
}
client->heap_usage += mem->size;
- xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
- abo->heap_start_id, abo->heap_end_id)
+ xa_for_each(&abo->heap_xa, heap_id, heap)
drm_gem_object_get(to_gobj(heap));
unlock_out:
@@ -142,48 +154,16 @@ unlock_out:
return ret;
}
-static void
-amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo)
+void amdxdna_gem_heap_free(struct amdxdna_client *client, struct amdxdna_gem_obj *abo)
{
- struct amdxdna_client *client = abo->client;
- struct amdxdna_gem_obj *heap;
- unsigned long heap_id;
-
mutex_lock(&client->mm_lock);
drm_mm_remove_node(&abo->mm_node);
client->heap_usage -= abo->mem.size;
- xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
- abo->heap_start_id, abo->heap_end_id)
- drm_gem_object_put(to_gobj(heap));
-
mutex_unlock(&client->mm_lock);
}
-static struct amdxdna_gem_obj *
-amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
-{
- struct amdxdna_gem_obj *abo;
-
- abo = kzalloc_obj(*abo);
- if (!abo)
- return ERR_PTR(-ENOMEM);
-
- abo->pinned = false;
- abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
- mutex_init(&abo->lock);
-
- abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
- abo->mem.uva = AMDXDNA_INVALID_ADDR;
- abo->mem.size = size;
- abo->open_ref = 0;
- abo->internal = false;
- INIT_LIST_HEAD(&abo->mem.umap_list);
-
- return abo;
-}
-
static void
amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo)
{
@@ -253,13 +233,16 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
struct amdxdna_gem_obj *abo = mapp->abo;
struct amdxdna_dev *xdna;
+ if (!mmu_notifier_range_blockable(range))
+ return false;
+
+ if (mapp->unmapped)
+ return true;
+
xdna = to_xdna_dev(to_gobj(abo)->dev);
XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d",
mapp->range.start, mapp->range.end, abo->type);
- if (!mmu_notifier_range_blockable(range))
- return false;
-
down_write(&xdna->notifier_lock);
abo->mem.map_invalid = true;
mapp->invalid = true;
@@ -271,10 +254,8 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
if (range->event == MMU_NOTIFY_UNMAP) {
down_write(&xdna->notifier_lock);
- if (!mapp->unmapped) {
- queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
- mapp->unmapped = true;
- }
+ mapp->unmapped = true;
+ queue_work(xdna->notifier_wq, &abo->hmm_unreg_work);
up_write(&xdna->notifier_lock);
}
@@ -301,33 +282,42 @@ static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo,
down_write(&xdna->notifier_lock);
list_for_each_entry(mapp, &abo->mem.umap_list, node) {
- if (!vma || compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) {
- if (!mapp->unmapped) {
- queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
- mapp->unmapped = true;
- }
- if (vma)
- break;
- }
+ if (!compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end))
+ continue;
+
+ mapp->unmapped = true;
+ queue_work(xdna->notifier_wq, &abo->hmm_unreg_work);
}
up_write(&xdna->notifier_lock);
}
-static void amdxdna_umap_release(struct kref *ref)
+static void amdxdna_hmm_unreg_umaps(struct amdxdna_gem_obj *abo, bool force)
{
- struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
- struct amdxdna_gem_obj *abo = mapp->abo;
- struct amdxdna_dev *xdna;
-
- mmu_interval_notifier_remove(&mapp->notifier);
+ struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
+ struct amdxdna_umap *mapp, *tmp;
+ LIST_HEAD(dead);
- xdna = to_xdna_dev(to_gobj(mapp->abo)->dev);
down_write(&xdna->notifier_lock);
- list_del(&mapp->node);
+ list_for_each_entry_safe(mapp, tmp, &abo->mem.umap_list, node) {
+ if (!force && !mapp->unmapped)
+ continue;
+
+ mapp->unmapped = true;
+ list_move(&mapp->node, &dead);
+ }
if (list_empty(&abo->mem.umap_list))
abo->mem.uva = AMDXDNA_INVALID_ADDR;
up_write(&xdna->notifier_lock);
+ list_for_each_entry_safe(mapp, tmp, &dead, node)
+ amdxdna_umap_put(mapp);
+}
+
+static void amdxdna_umap_release(struct kref *ref)
+{
+ struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
+
+ mmu_interval_notifier_remove(&mapp->notifier);
kvfree(mapp->range.hmm_pfns);
kfree(mapp);
}
@@ -339,10 +329,10 @@ void amdxdna_umap_put(struct amdxdna_umap *mapp)
static void amdxdna_hmm_unreg_work(struct work_struct *work)
{
- struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap,
- hmm_unreg_work);
+ struct amdxdna_gem_obj *abo = container_of(work, struct amdxdna_gem_obj,
+ hmm_unreg_work);
- amdxdna_umap_put(mapp);
+ amdxdna_hmm_unreg_umaps(abo, false);
}
static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
@@ -387,6 +377,21 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
goto free_map;
}
+ mapp->range.notifier = &mapp->notifier;
+ mapp->range.start = vma->vm_start;
+ mapp->range.end = vma->vm_end;
+ /*
+ * Access permissions are fixed at mmap() time. Changing them later
+ * with mprotect() is not supported: the range keeps requesting the
+ * original permissions, so the application may see a fault failure
+ * or an IOMMU fault.
+ */
+ mapp->range.default_flags = HMM_PFN_REQ_FAULT;
+ if (vma->vm_flags & VM_WRITE)
+ mapp->range.default_flags |= HMM_PFN_REQ_WRITE;
+ mapp->abo = abo;
+ kref_init(&mapp->refcnt);
+
ret = mmu_interval_notifier_insert_locked(&mapp->notifier,
current->mm,
addr,
@@ -397,15 +402,6 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
goto free_pfns;
}
- mapp->range.notifier = &mapp->notifier;
- mapp->range.start = vma->vm_start;
- mapp->range.end = vma->vm_end;
- mapp->range.default_flags = HMM_PFN_REQ_FAULT;
- mapp->abo = abo;
- kref_init(&mapp->refcnt);
-
- INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work);
-
down_write(&xdna->notifier_lock);
if (list_empty(&abo->mem.umap_list))
abo->mem.uva = addr;
@@ -421,17 +417,47 @@ free_map:
return ret;
}
+static struct amdxdna_gem_obj *
+amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
+{
+ struct amdxdna_gem_obj *abo;
+
+ abo = kzalloc_obj(*abo);
+ if (!abo)
+ return ERR_PTR(-ENOMEM);
+
+ abo->pinned = false;
+ abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
+ mutex_init(&abo->lock);
+
+ abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
+ abo->mem.uva = AMDXDNA_INVALID_ADDR;
+ abo->mem.size = size;
+ abo->open_ref = 0;
+ abo->internal = false;
+ INIT_LIST_HEAD(&abo->mem.umap_list);
+ xa_init_flags(&abo->heap_xa, XA_FLAGS_ALLOC);
+ INIT_WORK(&abo->hmm_unreg_work, amdxdna_hmm_unreg_work);
+
+ return abo;
+}
+
static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj)
{
struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+ struct amdxdna_gem_obj *heap;
+ unsigned long heap_id;
XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, amdxdna_gem_dev_addr(abo));
if (abo->pinned)
amdxdna_gem_unpin(abo);
amdxdna_gem_vunmap(abo);
- amdxdna_gem_heap_free(abo);
+ xa_for_each(&abo->heap_xa, heap_id, heap)
+ drm_gem_object_put(to_gobj(heap));
+ xa_destroy(&abo->heap_xa);
+
drm_gem_object_release(gobj);
amdxdna_gem_destroy_obj(abo);
}
@@ -458,16 +484,16 @@ static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo,
{
struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
unsigned long num_pages = vma_pages(vma);
- unsigned long offset = 0;
int ret;
- if (!is_import_bo(abo)) {
- ret = drm_gem_shmem_mmap(&abo->base, vma);
- if (ret) {
- XDNA_ERR(xdna, "Failed shmem mmap %d", ret);
- return ret;
- }
- } else {
+ /*
+ * Until today there is not any use case to mmap with non-zero
+ * offset. Put an explicit check here.
+ */
+ if (vma->vm_pgoff - drm_vma_node_start(&to_gobj(abo)->vma_node))
+ return -EINVAL;
+
+ if (is_import_bo(abo)) {
vma->vm_private_data = NULL;
vma->vm_ops = NULL;
ret = dma_buf_mmap(abo->dma_buf, vma, 0);
@@ -476,23 +502,28 @@ static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo,
return ret;
}
+ amdxdna_mark_mapp_invalid(abo, vma);
+
/* Drop the reference drm_gem_mmap_obj() acquired.*/
drm_gem_object_put(to_gobj(abo));
+ return 0;
}
- do {
- vm_fault_t fault_ret;
-
- fault_ret = handle_mm_fault(vma, vma->vm_start + offset,
- FAULT_FLAG_WRITE, NULL);
- if (fault_ret & VM_FAULT_ERROR) {
- XDNA_ERR(xdna, "Fault in page failed");
- amdxdna_mark_mapp_invalid(abo, vma);
- break;
- }
+ ret = drm_gem_shmem_mmap(&abo->base, vma);
+ if (ret) {
+ XDNA_ERR(xdna, "Failed shmem mmap %d", ret);
+ return ret;
+ }
- offset += PAGE_SIZE;
- } while (--num_pages);
+ vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
+ ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, &num_pages);
+ if (ret) {
+ XDNA_ERR(xdna, "Failed to insert pages %d", ret);
+ dma_resv_lock(to_gobj(abo)->resv, NULL);
+ drm_gem_shmem_put_pages_locked(&abo->base);
+ dma_resv_unlock(to_gobj(abo)->resv);
+ return ret;
+ }
return 0;
}
@@ -504,6 +535,10 @@ static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
int ret;
+ XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx",
+ drm_vma_node_offset_addr(&gobj->vma_node), abo->type,
+ vma->vm_start, gobj->size);
+
ret = amdxdna_hmm_register(abo, vma);
if (ret)
return ret;
@@ -514,9 +549,6 @@ static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
goto hmm_unreg;
}
- XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx",
- drm_vma_node_offset_addr(&gobj->vma_node), abo->type,
- vma->vm_start, gobj->size);
return 0;
hmm_unreg:
@@ -524,14 +556,99 @@ hmm_unreg:
return ret;
}
+/*
+ * VM operations for amdxdna shmem VMAs that use VM_MIXEDMAP.
+ *
+ * drm_gem_shmem_vm_ops cannot be used on VM_MIXEDMAP VMAs because its fault
+ * handler calls vmf_insert_pfn() → vmf_insert_pfn_prot() which contains:
+ * BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn))
+ * All amdxdna shmem pages are ordinary struct pages so pfn_valid() is always
+ * true, making the combination fatal.
+ *
+ * These ops use vmf_insert_page() (struct-page based) instead, which is the
+ * correct API for VM_MIXEDMAP VMAs backed by real struct pages. The open and
+ * close handlers replicate drm_gem_shmem_vm_open/close using only exported
+ * symbols.
+ */
+static vm_fault_t amdxdna_gem_mixedmap_fault(struct vm_fault *vmf)
+{
+ struct vm_area_struct *vma = vmf->vma;
+ struct drm_gem_object *gobj = vma->vm_private_data;
+ struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(gobj);
+ loff_t num_pages = gobj->size >> PAGE_SHIFT;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
+ pgoff_t page_offset;
+ struct page *page;
+
+ /*
+ * Partial free of vma is unexpected. Otherwise, the wrong page
+ * will be faulted in and the user application may crash itself.
+ */
+ page_offset = linear_page_delta(vma, vmf->address);
+
+ dma_resv_lock(gobj->resv, NULL);
+
+ if (!shmem->pages || shmem->madv < 0 || page_offset >= num_pages)
+ goto out;
+
+ page = shmem->pages[page_offset];
+ if (WARN_ON_ONCE(!page))
+ goto out;
+
+ /*
+ * Use vmf_insert_page() (struct-page path) not vmf_insert_pfn()
+ * (PFN path) because this VMA carries VM_MIXEDMAP.
+ */
+ ret = vmf_insert_page(vma, vmf->address, page);
+ if (ret == VM_FAULT_NOPAGE)
+ folio_mark_accessed(page_folio(page));
+
+out:
+ dma_resv_unlock(gobj->resv);
+ return ret;
+}
+
+static void amdxdna_gem_mixedmap_vm_open(struct vm_area_struct *vma)
+{
+ struct drm_gem_object *gobj = vma->vm_private_data;
+ struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(gobj);
+
+ /*
+ * Bump pages_use_count so the page array stays alive for the new
+ * mapping copy created by fork(). Mirrors drm_gem_shmem_vm_open().
+ */
+ dma_resv_lock(gobj->resv, NULL);
+ drm_WARN_ON_ONCE(gobj->dev, !refcount_inc_not_zero(&shmem->pages_use_count));
+ dma_resv_unlock(gobj->resv);
+
+ drm_gem_vm_open(vma);
+}
+
+static void amdxdna_gem_mixedmap_vm_close(struct vm_area_struct *vma)
+{
+ struct drm_gem_object *gobj = vma->vm_private_data;
+ struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(gobj);
+
+ dma_resv_lock(gobj->resv, NULL);
+ drm_gem_shmem_put_pages_locked(shmem);
+ dma_resv_unlock(gobj->resv);
+
+ drm_gem_vm_close(vma);
+}
+
+static const struct vm_operations_struct amdxdna_gem_mixedmap_vm_ops = {
+ .fault = amdxdna_gem_mixedmap_fault,
+ .open = amdxdna_gem_mixedmap_vm_open,
+ .close = amdxdna_gem_mixedmap_vm_close,
+};
+
static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
{
struct drm_gem_object *gobj = dma_buf->priv;
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
- unsigned long num_pages = vma_pages(vma);
int ret;
- vma->vm_ops = &drm_gem_shmem_vm_ops;
+ vma->vm_ops = &amdxdna_gem_mixedmap_vm_ops;
vma->vm_private_data = gobj;
drm_gem_object_get(gobj);
@@ -541,16 +658,9 @@ static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struc
/* The buffer is based on memory pages. Fix the flag. */
vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
- ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages,
- &num_pages);
- if (ret)
- goto close_vma;
return 0;
-close_vma:
- vma->vm_ops->close(vma);
- return ret;
put_obj:
drm_gem_object_put(gobj);
return ret;
@@ -609,10 +719,8 @@ amdxdna_gem_skip_bo_usage(struct amdxdna_gem_obj *abo)
}
static void
-amdxdna_gem_add_bo_usage(struct amdxdna_gem_obj *abo)
+amdxdna_gem_add_bo_usage(struct amdxdna_client *client, struct amdxdna_gem_obj *abo)
{
- struct amdxdna_client *client = abo->client;
-
if (amdxdna_gem_skip_bo_usage(abo))
return;
@@ -624,10 +732,8 @@ amdxdna_gem_add_bo_usage(struct amdxdna_gem_obj *abo)
}
static void
-amdxdna_gem_del_bo_usage(struct amdxdna_gem_obj *abo)
+amdxdna_gem_del_bo_usage(struct amdxdna_client *client, struct amdxdna_gem_obj *abo)
{
- struct amdxdna_client *client = abo->client;
-
if (amdxdna_gem_skip_bo_usage(abo))
return;
@@ -643,8 +749,9 @@ static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
- amdxdna_hmm_unregister(abo, NULL);
- flush_workqueue(xdna->notifier_wq);
+ /* No notifier survives this, so no new work can be queued. */
+ amdxdna_hmm_unreg_umaps(abo, true);
+ cancel_work_sync(&abo->hmm_unreg_work);
if (abo->pinned)
amdxdna_gem_unpin(abo);
@@ -663,12 +770,20 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
{
struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+ struct amdxdna_client *client;
int ret;
- guard(mutex)(&abo->lock);
+ mutex_lock(&abo->lock);
+ if (abo->open_ref > 0 && filp->driver_priv != abo->client) {
+ mutex_unlock(&abo->lock);
+ return -EPERM;
+ }
+
abo->open_ref++;
- if (abo->open_ref > 1)
+ if (abo->open_ref > 1) {
+ mutex_unlock(&abo->lock);
return 0;
+ }
/* Attached to the client when first opened by it. */
abo->client = filp->driver_priv;
@@ -679,26 +794,34 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
if (ret) {
abo->open_ref--;
abo->client = NULL;
+ mutex_unlock(&abo->lock);
return ret;
}
}
+ client = abo->client;
+ mutex_unlock(&abo->lock);
- amdxdna_gem_add_bo_usage(abo);
+ amdxdna_gem_add_bo_usage(client, abo);
return 0;
}
static void amdxdna_gem_obj_close(struct drm_gem_object *gobj, struct drm_file *filp)
{
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+ struct amdxdna_client *client = NULL;
- guard(mutex)(&abo->lock);
+ mutex_lock(&abo->lock);
abo->open_ref--;
if (abo->open_ref == 0) {
- amdxdna_gem_del_bo_usage(abo);
/* Detach from the client when last closed by it. */
+ client = abo->client;
abo->client = NULL;
}
+ mutex_unlock(&abo->lock);
+
+ if (client)
+ amdxdna_gem_del_bo_usage(client, abo);
}
static int amdxdna_gem_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
@@ -734,19 +857,72 @@ static void amdxdna_gem_obj_vunmap(struct drm_gem_object *obj, struct iosys_map
drm_gem_shmem_object_vunmap(obj, map);
}
+static int amdxdna_gem_dev_obj_open(struct drm_gem_object *gobj, struct drm_file *filp)
+{
+ struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+
+ guard(mutex)(&abo->lock);
+ if (filp->driver_priv != abo->client)
+ return -EPERM;
+ abo->open_ref++;
+
+ return 0;
+}
+
+static void amdxdna_gem_dev_obj_close(struct drm_gem_object *gobj, struct drm_file *filp)
+{
+ struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+ struct amdxdna_client *client = NULL;
+
+ mutex_lock(&abo->lock);
+ abo->open_ref--;
+
+ /*
+ * Freeing the heap allocation here, when the handle is closed, is
+ * intentional. DEV BOs are carved out of a per-client drm_mm heap;
+ * any subsequent allocation that lands on the same device address will
+ * also belong to the same client. If the user closes the handle while
+ * a job is still in flight the only consequence is self-inflicted
+ * corruption within their own context -- it cannot affect other
+ * processes. The GEM reference held by the in-flight job keeps the
+ * amdxdna_gem_obj struct alive until the job completes; it does not
+ * prevent the device address from being reclaimed by the allocator.
+ *
+ * Cross-process handle creation for DEV BOs is rejected in
+ * amdxdna_gem_dev_obj_open(), which prevents the following UAF:
+ * if a second process shared the handle via GEM flink and the
+ * original creator exited (freeing client), the importer would later
+ * reach open_ref == 0 here and call amdxdna_gem_heap_free() with a
+ * dangling abo->client pointer. Because cross-process opens are
+ * rejected, the process arriving here is always the owning client,
+ * which is still alive. abo->client is nulled out afterwards so that
+ * any code path running on a lingering GEM reference (e.g. an
+ * in-flight job) cannot silently dereference a stale pointer.
+ */
+ if (abo->open_ref == 0) {
+ client = abo->client;
+ abo->client = NULL;
+ }
+ mutex_unlock(&abo->lock);
+
+ if (client)
+ amdxdna_gem_heap_free(client, abo);
+}
+
static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
{
struct amdxdna_gem_obj *abo = to_xdna_obj(obj);
struct amdxdna_gem_obj *heap;
+ unsigned long index = 0;
void *base;
u64 offset;
- /* vmap dev bo which is across more than 1 heap is not allowed */
- if (abo->heap_start_id != abo->heap_end_id)
+ heap = xa_find(&abo->heap_xa, &index, ULONG_MAX, XA_PRESENT);
+ if (!heap)
return -ENOMEM;
- heap = xa_load(&abo->client->dev_heap_xa, abo->heap_start_id);
- if (!heap)
+ /* vmap dev bo which is across more than 1 heap is not allowed */
+ if (xa_find_after(&abo->heap_xa, &index, ULONG_MAX, XA_PRESENT))
return -ENOMEM;
base = amdxdna_gem_vmap(heap);
@@ -765,6 +941,8 @@ static struct dma_buf *amdxdna_gem_dev_obj_export(struct drm_gem_object *gobj, i
static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
.free = amdxdna_gem_dev_obj_free,
+ .open = amdxdna_gem_dev_obj_open,
+ .close = amdxdna_gem_dev_obj_close,
.vmap = amdxdna_gem_dev_obj_vmap,
.export = amdxdna_gem_dev_obj_export,
};
@@ -780,7 +958,7 @@ static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
.vmap = amdxdna_gem_obj_vmap,
.vunmap = amdxdna_gem_obj_vunmap,
.mmap = amdxdna_gem_obj_mmap,
- .vm_ops = &drm_gem_shmem_vm_ops,
+ .vm_ops = &amdxdna_gem_mixedmap_vm_ops,
.export = amdxdna_gem_prime_export,
};
@@ -962,6 +1140,11 @@ amdxdna_drm_create_dev_heap_bo(struct drm_device *dev,
struct amdxdna_gem_obj *abo;
int ret;
+ if (!xdna->dev_info->dev_mem_size) {
+ XDNA_DBG(xdna, "Device heap is not supported");
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+
WARN_ON(!is_power_of_2(xdna->dev_info->dev_mem_size));
XDNA_DBG(xdna, "Requested dev heap size 0x%llx", args->size);
if (!args->size || !IS_ALIGNED(args->size, xdna->dev_info->dev_mem_size)) {
@@ -1103,6 +1286,8 @@ int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_f
args->handle, args->type, amdxdna_gem_uva(abo),
amdxdna_gem_dev_addr(abo), abo->mem.size);
put_obj:
+ if (ret && abo->type == AMDXDNA_BO_DEV)
+ amdxdna_gem_heap_free(client, abo);
/* Dereference object reference. Handle holds it now. */
drm_gem_object_put(to_gobj(abo));
return ret;
@@ -1136,7 +1321,6 @@ static void amdxdna_bo_unpin(struct amdxdna_gem_obj *abo)
int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
{
- struct amdxdna_client *client = abo->client;
struct amdxdna_gem_obj *heap;
unsigned long heap_id, last = ULONG_MAX;
int ret = 0;
@@ -1144,17 +1328,15 @@ int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
if (abo->type != AMDXDNA_BO_DEV)
return amdxdna_bo_pin(abo);
- xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
- abo->heap_start_id, abo->heap_end_id) {
+ xa_for_each(&abo->heap_xa, heap_id, heap) {
ret = amdxdna_bo_pin(heap);
if (ret)
break;
last = heap_id;
}
- if (ret && last <= abo->heap_end_id) {
- xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
- abo->heap_start_id, last)
+ if (ret && last != ULONG_MAX) {
+ xa_for_each_range(&abo->heap_xa, heap_id, heap, 0, last)
amdxdna_bo_unpin(heap);
}
@@ -1179,8 +1361,7 @@ void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo)
struct amdxdna_gem_obj *heap;
unsigned long heap_id;
- xa_for_each_range(&abo->client->dev_heap_xa, heap_id, heap,
- abo->heap_start_id, abo->heap_end_id)
+ xa_for_each(&abo->heap_xa, heap_id, heap)
amdxdna_bo_unpin(heap);
} else {
amdxdna_bo_unpin(abo);
@@ -1300,8 +1481,7 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
u64 flush_start = bo_start + args->offset;
u64 flush_end = flush_start + args->size;
- xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
- abo->heap_start_id, abo->heap_end_id) {
+ xa_for_each(&abo->heap_xa, heap_id, heap) {
u64 heap_start = amdxdna_gem_dev_addr(heap);
u64 heap_end = heap_start + heap->mem.size;
u64 start = max(flush_start, heap_start);
@@ -1328,7 +1508,7 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
amdxdna_gem_unpin(abo);
if (ret) {
- drm_WARN(&xdna->ddev, 1, "Can not get flush memory");
+ XDNA_DBG(xdna, "Flush BO %d failed, ret %d", args->handle, ret);
goto put_obj;
}
}
@@ -1336,7 +1516,8 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n",
args->handle, args->offset, args->size);
- if (args->direction == SYNC_DIRECT_FROM_DEVICE)
+ if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE &&
+ args->direction == SYNC_DIRECT_FROM_DEVICE)
ret = amdxdna_hwctx_sync_debug_bo(client, args->handle);
put_obj:
diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h
index 1e90e32bf3cd..9b4aa21a37c9 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.h
+++ b/drivers/accel/amdxdna/amdxdna_gem.h
@@ -14,7 +14,6 @@
struct amdxdna_umap {
struct mmu_interval_notifier notifier;
struct hmm_range range;
- struct work_struct hmm_unreg_work;
struct amdxdna_gem_obj *abo;
struct list_head node;
struct kref refcnt;
@@ -43,11 +42,11 @@ struct amdxdna_gem_obj {
struct mutex lock; /* Protects: pinned, mem.kva, open_ref */
struct amdxdna_mem mem;
int open_ref;
+ struct work_struct hmm_unreg_work;
/* Below members are initialized when needed */
struct drm_mm_node mm_node; /* For AMDXDNA_BO_DEV */
- u32 heap_start_id;
- u32 heap_end_id;
+ struct xarray heap_xa;
u64 dev_addr; /* For heap bo */
u32 assigned_hwctx;
struct dma_buf *dma_buf;
@@ -104,6 +103,7 @@ static inline u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *abo)
}
void amdxdna_umap_put(struct amdxdna_umap *mapp);
+void amdxdna_gem_heap_free(struct amdxdna_client *client, struct amdxdna_gem_obj *abo);
struct drm_gem_object *
amdxdna_gem_create_shmem_object_cb(struct drm_device *dev, size_t size);
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c
index cc8865f4e79c..c4701b2703d7 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox.c
+++ b/drivers/accel/amdxdna/amdxdna_mailbox.c
@@ -9,7 +9,6 @@
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/slab.h>
-#include <linux/xarray.h>
#define CREATE_TRACE_POINTS
#include <trace/events/amdxdna.h>
@@ -35,9 +34,12 @@
(_chann)->msix_irq, ##args); \
})
+#define MSG_BUF_SZ(chann) \
+ (mailbox_get_ringbuf_size(chann, CHAN_RES_X2I) + \
+ sizeof(struct mailbox_msg))
+
#define MAGIC_VAL 0x1D000000U
#define MAGIC_VAL_MASK 0xFF000000
-#define MAX_MSG_ID_ENTRIES 256
#define MSG_RX_TIMER 200 /* milliseconds */
#define MAILBOX_NAME "xdna_mailbox"
@@ -57,8 +59,6 @@ struct mailbox_channel {
struct xdna_mailbox_chann_res res[CHAN_RES_NUM];
int msix_irq;
u32 iohub_int_addr;
- struct xarray chan_xa;
- u32 next_msgid;
u32 x2i_tail;
/* Received msg related fields */
@@ -66,6 +66,12 @@ struct mailbox_channel {
struct work_struct rx_work;
u32 i2x_head;
bool bad_state;
+
+ /* Pre-allocated message buffer pool */
+ void *msg_buf;
+ u32 msg_buf_num;
+ u32 next_slot; /* next slot index to try for allocation */
+ struct mutex lock;
};
#define MSG_BODY_SZ GENMASK(10, 0)
@@ -90,12 +96,36 @@ struct mailbox_pkg {
#define TOMBSTONE 0xDEADFACE
struct mailbox_msg {
+ bool busy;
void *handle;
int (*notify_cb)(void *handle, void __iomem *data, size_t size);
size_t pkg_size; /* package size in bytes */
struct mailbox_pkg pkg;
};
+/*
+ * Clear busy before invoking the callback so that a new submission can
+ * reuse the slot immediately - by the time notify_cb returns, the caller
+ * may already have submitted a new job that needs this slot.
+ *
+ * notify_cb and handle are cached in locals first. smp_store_release()
+ * ensures those loads complete before busy is cleared - without it the
+ * compiler or CPU could sink the loads past the store, letting a
+ * concurrent sender overwrite notify_cb/handle before we read them.
+ * Paired with smp_load_acquire() in mailbox_acquire_msgid().
+ */
+static int mailbox_msg_done(struct mailbox_msg *mb_msg, void __iomem *data, size_t size)
+{
+ int (*notify_cb)(void *, void __iomem *, size_t) = mb_msg->notify_cb;
+ void *handle = mb_msg->handle;
+
+ /* Pairs with smp_load_acquire() in mailbox_acquire_msgid(). */
+ smp_store_release(&mb_msg->busy, false);
+ if (notify_cb)
+ return notify_cb(handle, data, size);
+ return 0;
+}
+
static void mailbox_reg_write(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 data)
{
struct xdna_mailbox_res *mb_res = &mb_chann->mb->res;
@@ -161,38 +191,57 @@ static inline int mailbox_validate_msgid(int msg_id)
return (msg_id & MAGIC_VAL_MASK) == MAGIC_VAL;
}
-static int mailbox_acquire_msgid(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg)
+static struct mailbox_msg *mailbox_msg_ptr(struct mailbox_channel *mb_chann, u32 slot)
{
- u32 msg_id;
- int ret;
+ return mb_chann->msg_buf + slot * MSG_BUF_SZ(mb_chann);
+}
- ret = xa_alloc_cyclic_irq(&mb_chann->chan_xa, &msg_id, mb_msg,
- XA_LIMIT(0, MAX_MSG_ID_ENTRIES - 1),
- &mb_chann->next_msgid, GFP_NOWAIT);
- if (ret < 0)
- return ret;
+static int mailbox_acquire_msgid(struct mailbox_channel *mb_chann)
+{
+ struct mailbox_msg *mb_msg;
+ u32 slot;
+ u32 i;
/*
- * Add MAGIC_VAL to the higher bits.
+ * msg_id = slot | MAGIC_VAL. Scan slots starting from next_slot.
+ * smp_load_acquire() pairs with smp_store_release() in mailbox_msg_done()
+ * to ensure that once we observe busy == false, notify_cb and handle are
+ * also visible as their post-completion values before we overwrite them.
*/
- msg_id |= MAGIC_VAL;
- return msg_id;
-}
+ for (i = 0; i < mb_chann->msg_buf_num; i++) {
+ slot = (mb_chann->next_slot + i) % mb_chann->msg_buf_num;
+ mb_msg = mailbox_msg_ptr(mb_chann, slot);
+ /* Pairs with smp_store_release() in mailbox_msg_done(). */
+ if (!smp_load_acquire(&mb_msg->busy)) {
+ mb_chann->next_slot = (slot + 1) % mb_chann->msg_buf_num;
+ return slot | MAGIC_VAL;
+ }
+ }
-static void mailbox_release_msgid(struct mailbox_channel *mb_chann, int msg_id)
-{
- msg_id &= ~MAGIC_VAL_MASK;
- xa_erase_irq(&mb_chann->chan_xa, msg_id);
+ return -ENOBUFS;
}
-static void mailbox_release_msg(struct mailbox_channel *mb_chann,
- struct mailbox_msg *mb_msg)
+static struct mailbox_msg *mailbox_get_msg_buf(struct mailbox_channel *mb_chann,
+ size_t msg_size)
{
- MB_DBG(mb_chann, "msg_id 0x%x msg opcode 0x%x",
- mb_msg->pkg.header.id, mb_msg->pkg.header.opcode);
- if (mb_msg->notify_cb)
- mb_msg->notify_cb(mb_msg->handle, NULL, 0);
- kfree(mb_msg);
+ struct mailbox_msg *mb_msg;
+ int msg_id;
+ u32 slot;
+
+ /* msg_id = slot | MAGIC_VAL */
+ msg_id = mailbox_acquire_msgid(mb_chann);
+ if (msg_id < 0) {
+ MB_ERR(mb_chann, "No free message slot");
+ return NULL;
+ }
+
+ slot = msg_id & ~MAGIC_VAL_MASK;
+ mb_msg = mailbox_msg_ptr(mb_chann, slot);
+ memset(mb_msg, 0, msg_size + sizeof(*mb_msg));
+ mb_msg->busy = true;
+ mb_msg->pkg.header.id = msg_id;
+
+ return mb_msg;
}
static int
@@ -224,7 +273,7 @@ check_again:
if (tail < head && tmp_tail >= head) {
ret = read_poll_timeout(mailbox_get_headptr, head,
tmp_tail < head || tail >= head,
- 1, 100, false, mb_chann, CHAN_RES_X2I);
+ 1, 2000000, false, mb_chann, CHAN_RES_X2I);
if (ret)
return ret;
@@ -248,8 +297,9 @@ mailbox_get_resp(struct mailbox_channel *mb_chann, struct xdna_msg_header *heade
void __iomem *data)
{
struct mailbox_msg *mb_msg;
- int msg_id;
int ret = 0;
+ int msg_id;
+ u32 slot;
msg_id = header->id;
if (!mailbox_validate_msgid(msg_id)) {
@@ -257,22 +307,32 @@ mailbox_get_resp(struct mailbox_channel *mb_chann, struct xdna_msg_header *heade
return -EINVAL;
}
- msg_id &= ~MAGIC_VAL_MASK;
- mb_msg = xa_erase_irq(&mb_chann->chan_xa, msg_id);
- if (!mb_msg) {
- MB_ERR(mb_chann, "Cannot find msg 0x%x", msg_id);
+ slot = msg_id & ~MAGIC_VAL_MASK;
+ if (unlikely(slot >= mb_chann->msg_buf_num)) {
+ MB_ERR(mb_chann, "Invalid msg_id 0x%x", msg_id);
return -EINVAL;
}
+ mb_msg = mailbox_msg_ptr(mb_chann, slot);
MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x",
header->opcode, header->total_size, header->id);
- if (mb_msg->notify_cb) {
- ret = mb_msg->notify_cb(mb_msg->handle, data, header->total_size);
- if (unlikely(ret))
- MB_ERR(mb_chann, "Message callback ret %d", ret);
+
+ /*
+ * msg_id is slot | MAGIC_VAL and is reused as soon as the slot is
+ * freed. The management firmware protocol guarantees exactly one
+ * response for each request, so a response cannot refer to a previous
+ * user of a reused slot. Keep the busy check as a defensive guard
+ * against an unexpected response for an idle slot.
+ */
+ if (unlikely(!mb_msg->busy)) {
+ MB_WARN_ONCE(mb_chann, "Unexpected response for idle slot 0x%x", msg_id);
+ return 0;
}
- kfree(mb_msg);
+ ret = mailbox_msg_done(mb_msg, data, header->total_size);
+ if (unlikely(ret))
+ MB_ERR(mb_chann, "Message callback ret %d", ret);
+
return ret;
}
@@ -399,7 +459,7 @@ again:
}
int xdna_mailbox_send_msg(struct mailbox_channel *mb_chann,
- const struct xdna_mailbox_msg *msg, u64 tx_timeout)
+ const struct xdna_mailbox_msg *msg)
{
struct xdna_msg_header *header;
struct mailbox_msg *mb_msg;
@@ -428,9 +488,12 @@ int xdna_mailbox_send_msg(struct mailbox_channel *mb_chann,
return -EPIPE;
}
- mb_msg = kzalloc(sizeof(*mb_msg) + pkg_size, GFP_KERNEL);
- if (!mb_msg)
- return -ENOMEM;
+ mutex_lock(&mb_chann->lock);
+ mb_msg = mailbox_get_msg_buf(mb_chann, pkg_size);
+ if (!mb_msg) {
+ mutex_unlock(&mb_chann->lock);
+ return -ENOBUFS;
+ }
mb_msg->handle = msg->handle;
mb_msg->notify_cb = msg->notify_cb;
@@ -447,28 +510,18 @@ int xdna_mailbox_send_msg(struct mailbox_channel *mb_chann,
header->opcode = msg->opcode;
memcpy(mb_msg->pkg.payload, msg->send_data, msg->send_size);
- ret = mailbox_acquire_msgid(mb_chann, mb_msg);
- if (unlikely(ret < 0)) {
- MB_ERR(mb_chann, "mailbox_acquire_msgid failed");
- goto msg_id_failed;
- }
- header->id = ret;
-
MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x",
header->opcode, header->total_size, header->id);
ret = mailbox_send_msg(mb_chann, mb_msg);
if (ret) {
MB_DBG(mb_chann, "Error in mailbox send msg, ret %d", ret);
- goto release_id;
+ mb_msg->busy = false;
+ mutex_unlock(&mb_chann->lock);
+ return ret;
}
+ mutex_unlock(&mb_chann->lock);
- return 0;
-
-release_id:
- mailbox_release_msgid(mb_chann, header->id);
-msg_id_failed:
- kfree(mb_msg);
return ret;
}
@@ -481,12 +534,13 @@ struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb)
return NULL;
INIT_WORK(&mb_chann->rx_work, mailbox_rx_worker);
- mb_chann->work_q = create_singlethread_workqueue(MAILBOX_NAME);
+ mb_chann->work_q = alloc_ordered_workqueue(MAILBOX_NAME, WQ_MEM_RECLAIM);
if (!mb_chann->work_q) {
MB_ERR(mb_chann, "Create workqueue failed");
goto free_chann;
}
mb_chann->mb = mb;
+ mutex_init(&mb_chann->lock);
return mb_chann;
@@ -500,7 +554,9 @@ void xdna_mailbox_free_channel(struct mailbox_channel *mb_chann)
if (!mb_chann)
return;
+ kfree(mb_chann->msg_buf);
destroy_workqueue(mb_chann->work_q);
+ mutex_destroy(&mb_chann->lock);
kfree(mb_chann);
}
@@ -509,21 +565,32 @@ xdna_mailbox_start_channel(struct mailbox_channel *mb_chann,
const struct xdna_mailbox_chann_res *x2i,
const struct xdna_mailbox_chann_res *i2x,
u32 iohub_int_addr,
- int mb_irq)
+ int mb_irq, u32 n_msg)
{
int ret;
- if (!is_power_of_2(x2i->rb_size) || !is_power_of_2(i2x->rb_size)) {
- pr_err("Ring buf size must be power of 2\n");
- return -EINVAL;
- }
-
+ /*
+ * The ring buffer geometry, rb_start_addr and rb_size for both the
+ * x2i and the i2x channel, comes from AMD signed firmware, through
+ * the management or mailbox information block, or through the
+ * CREATE_CONTEXT response. The driver trusts those values by design
+ * and does not bound them against the mapped region.
+ */
mb_chann->msix_irq = mb_irq;
mb_chann->iohub_int_addr = iohub_int_addr;
memcpy(&mb_chann->res[CHAN_RES_X2I], x2i, sizeof(*x2i));
memcpy(&mb_chann->res[CHAN_RES_I2X], i2x, sizeof(*i2x));
- xa_init_flags(&mb_chann->chan_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
+ /*
+ * msg_buf is a flat array of n_msg slots, separate from the hardware
+ * ring buffer. Each slot is rb_size bytes - the combined size of the
+ * mailbox_msg metadata and the package payload must not exceed rb_size.
+ */
+ mb_chann->msg_buf = kcalloc(n_msg, MSG_BUF_SZ(mb_chann), GFP_KERNEL);
+ if (!mb_chann->msg_buf)
+ return -ENOMEM;
+
+ mb_chann->msg_buf_num = n_msg;
mb_chann->x2i_tail = mailbox_get_tailptr(mb_chann, CHAN_RES_X2I);
mb_chann->i2x_head = mailbox_get_headptr(mb_chann, CHAN_RES_I2X);
@@ -531,6 +598,8 @@ xdna_mailbox_start_channel(struct mailbox_channel *mb_chann,
ret = request_irq(mb_irq, mailbox_irq_handler, 0, MAILBOX_NAME, mb_chann);
if (ret) {
MB_ERR(mb_chann, "Failed to request irq %d ret %d", mb_irq, ret);
+ kfree(mb_chann->msg_buf);
+ mb_chann->msg_buf = NULL;
return ret;
}
@@ -544,7 +613,7 @@ xdna_mailbox_start_channel(struct mailbox_channel *mb_chann,
void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann)
{
struct mailbox_msg *mb_msg;
- unsigned long msg_id;
+ u32 i;
if (!mb_chann)
return;
@@ -555,12 +624,22 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann)
/* Cancel RX work and wait for it to finish */
drain_workqueue(mb_chann->work_q);
- /* We can clean up and release resources */
- xa_for_each_start(&mb_chann->chan_xa, msg_id, mb_msg, mb_chann->next_msgid)
- mailbox_release_msg(mb_chann, mb_msg);
- xa_for_each_range(&mb_chann->chan_xa, msg_id, mb_msg, 0, mb_chann->next_msgid - 1)
- mailbox_release_msg(mb_chann, mb_msg);
- xa_destroy(&mb_chann->chan_xa);
+ /* Release any in-flight messages in the pool
+ *
+ * Drain in submission order: next_slot points to the slot after
+ * the most recently assigned one, so start from next_slot and
+ * wrap around so callbacks fire oldest-first.
+ */
+ for (i = 0; i < mb_chann->msg_buf_num; i++) {
+ u32 slot = (mb_chann->next_slot + i) % mb_chann->msg_buf_num;
+
+ mb_msg = mailbox_msg_ptr(mb_chann, slot);
+ if (!mb_msg->busy)
+ continue;
+ MB_DBG(mb_chann, "msg_id 0x%x msg opcode 0x%x",
+ mb_msg->pkg.header.id, mb_msg->pkg.header.opcode);
+ mailbox_msg_done(mb_msg, NULL, 0);
+ }
MB_DBG(mb_chann, "Mailbox channel stopped, irq: %d", mb_chann->msix_irq);
}
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.h b/drivers/accel/amdxdna/amdxdna_mailbox.h
index 2908404303ae..7f4a40b4230b 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox.h
+++ b/drivers/accel/amdxdna/amdxdna_mailbox.h
@@ -88,6 +88,10 @@ struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb);
* @i2x: firmware to host mailbox resources
* @xdna_mailbox_intr_reg: register addr of MSI-X interrupt
* @mb_irq: Linux IRQ number associated with mailbox MSI-X interrupt vector index
+ * @n_msg: number of message slots to pre-allocate; must be non-zero for
+ * the PCI transport (the platform transport ignores this argument).
+ * Each slot is rb_size bytes. The caller must ensure at most n_msg
+ * messages are in flight at any time.
*
* Return: If success, return a handle of mailbox channel. Otherwise, return NULL.
*/
@@ -96,7 +100,7 @@ xdna_mailbox_start_channel(struct mailbox_channel *mb_chann,
const struct xdna_mailbox_chann_res *x2i,
const struct xdna_mailbox_chann_res *i2x,
u32 xdna_mailbox_intr_reg,
- int mb_irq);
+ int mb_irq, u32 n_msg);
/*
* xdna_mailbox_free_channel() -- free mailbox channel
@@ -117,11 +121,10 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mailbox_chann);
*
* @mailbox_chann: Mailbox channel handle
* @msg: message struct for message information
- * @tx_timeout: the timeout value for sending the message in ms.
*
* Return: If success return 0, otherwise, return error code
*/
int xdna_mailbox_send_msg(struct mailbox_channel *mailbox_chann,
- const struct xdna_mailbox_msg *msg, u64 tx_timeout);
+ const struct xdna_mailbox_msg *msg);
#endif /* _AIE_MAILBOX_ */
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox_helper.c b/drivers/accel/amdxdna/amdxdna_mailbox_helper.c
index 6d0c24513476..ee33a34852f0 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox_helper.c
+++ b/drivers/accel/amdxdna/amdxdna_mailbox_helper.c
@@ -44,7 +44,7 @@ int xdna_send_msg_wait(struct amdxdna_dev *xdna, struct mailbox_channel *chann,
struct xdna_notify *hdl = msg->handle;
int ret;
- ret = xdna_mailbox_send_msg(chann, msg, TX_TIMEOUT);
+ ret = xdna_mailbox_send_msg(chann, msg);
if (ret) {
XDNA_ERR(xdna, "Send message failed, ret %d", ret);
return ret;
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox_helper.h b/drivers/accel/amdxdna/amdxdna_mailbox_helper.h
index 556c712cad0a..11924d38f2fe 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox_helper.h
+++ b/drivers/accel/amdxdna/amdxdna_mailbox_helper.h
@@ -6,7 +6,6 @@
#ifndef _AMDXDNA_MAILBOX_HELPER_H
#define _AMDXDNA_MAILBOX_HELPER_H
-#define TX_TIMEOUT 2000 /* milliseconds */
#define RX_TIMEOUT 5000 /* milliseconds */
struct amdxdna_dev;
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c
index bb339e641416..67a02d6b7f97 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.c
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c
@@ -14,6 +14,7 @@
#include <linux/iommu.h>
#include <linux/pci.h>
+#include "aie.h"
#include "amdxdna_cbuf.h"
#include "amdxdna_ctx.h"
#include "amdxdna_debugfs.h"
@@ -151,6 +152,8 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
filp->driver_priv = client;
client->filp = filp;
+ spin_lock_init(&client->io_stats.lock);
+
XDNA_DBG(xdna, "pid %d opened", client->pid);
return 0;
@@ -287,6 +290,9 @@ static void amdxdna_show_fdinfo(struct drm_printer *p, struct drm_file *filp)
size_t heap_usage, external_usage, internal_usage;
char *drv_name = filp->minor->dev->driver->name;
+ drm_printf(p, "drm-engine-npu:\t%llu ns\n",
+ amdxdna_io_stats_busy_time_ns(client));
+
mutex_lock(&client->mm_lock);
heap_usage = client->heap_usage;
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h
index a997d27a504d..510356a31532 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.h
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h
@@ -143,6 +143,13 @@ struct amdxdna_device_id {
const struct amdxdna_dev_info *dev_info;
};
+struct amdxdna_io_stats {
+ spinlock_t lock; /* protect io stats */
+ int job_depth;
+ u64 start_time;
+ u64 busy_time;
+};
+
/*
* struct amdxdna_client - amdxdna client
* A per fd data structure for managing context and other user process stuffs.
@@ -169,6 +176,8 @@ struct amdxdna_client {
size_t heap_usage;
size_t total_bo_usage;
size_t total_int_bo_usage;
+
+ struct amdxdna_io_stats io_stats;
};
#define amdxdna_for_each_hwctx(client, hwctx_id, entry) \
diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c
index 15a161384625..c9648c8032ec 100644
--- a/drivers/accel/amdxdna/npu4_regs.c
+++ b/drivers/accel/amdxdna/npu4_regs.c
@@ -153,6 +153,7 @@ const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[] = {
{ AIE2_DEV_REVISION_GPT1, "NPU Gorgon Point 1" },
{ AIE2_DEV_REVISION_GPT2, "NPU Gorgon Point 2" },
{ AIE2_DEV_REVISION_GPT3, "NPU Gorgon Point 3" },
+ { AIE2_DEV_REVISION_GRGH, "NPU Gorgon Halo" },
{ 0 }
};