From 2365f8d7f609cff766b150028372bc28d8b436f7 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:19 +0300 Subject: RDMA/mlx5: Add mlx5_data_direct_supported() helper The data direct capability FW checks are duplicated inline in mlx5_ib_data_direct_init() and mlx5_ib_data_direct_cleanup(). Wrap it in a mlx5_data_direct_supported() helper and use it in both places, in preparation for moving the data direct matching code to mlx5_core. For the same reason put it in driver.h instead of data_direct.h. No functional change. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-3-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- include/linux/mlx5/driver.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index b1871c0821d0..ee29f95ead18 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1395,4 +1395,10 @@ static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) #define MLX5_SW_IMAGE_GUID_MAX_BYTES 9 +static inline bool mlx5_data_direct_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, data_direct) && + MLX5_CAP_GEN_2(mdev, query_vuid); +} + #endif /* MLX5_DRIVER_H */ -- cgit v1.2.3 From ebbb9f87689998157e7310e96f585bc293f6fcb0 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:20 +0300 Subject: RDMA/mlx5: Make the data direct VUID query self-contained mlx5_cmd_query_vuid() in cmd.c is generic over the data_direct flag but its only caller is the data direct init path, which always passes data_direct=true. Move it into data_direct.c as mlx5_data_direct_query_vuid(), dropping the always-true parameter and hardcoding data_direct=1. This groups the query with the rest of the data direct code so that it travels together when the data direct matching code is relocated to mlx5_core, and lets it become static there once the query is folded into the core bind path. This function will become private in a subsequent patch. No functional change. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-4-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/cmd.c | 21 --------------------- drivers/infiniband/hw/mlx5/cmd.h | 2 -- drivers/infiniband/hw/mlx5/data_direct.c | 20 ++++++++++++++++++++ drivers/infiniband/hw/mlx5/data_direct.h | 3 +++ drivers/infiniband/hw/mlx5/main.c | 3 ++- include/linux/mlx5/data_direct.h | 13 +++++++++++++ 6 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 include/linux/mlx5/data_direct.h (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c index 7c08e3008927..895b62cc528d 100644 --- a/drivers/infiniband/hw/mlx5/cmd.c +++ b/drivers/infiniband/hw/mlx5/cmd.c @@ -245,24 +245,3 @@ int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid) MLX5_SET(dealloc_uar_in, in, uid, uid); return mlx5_cmd_exec_in(dev, dealloc_uar, in); } - -int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct, - char *out_vuid) -{ - u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + - MLX5_ST_SZ_BYTES(array1024_auto)] = {}; - u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; - char *vuid; - int err; - - MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); - MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); - MLX5_SET(query_vuid_in, in, data_direct, data_direct); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (err) - return err; - - vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); - memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); - return 0; -} diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h index e6c88b6ebd0d..e5cd31270443 100644 --- a/drivers/infiniband/hw/mlx5/cmd.h +++ b/drivers/infiniband/hw/mlx5/cmd.h @@ -58,6 +58,4 @@ int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb, u16 opmod, u8 port); int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid); int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid); -int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct, - char *out_vuid); #endif /* MLX5_IB_CMD_H */ diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 04affdbdcb12..8d006b611c63 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -24,6 +24,26 @@ struct mlx5_data_direct_registration { struct blocking_notifier_head users; }; +int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid) +{ + u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + + MLX5_ST_SZ_BYTES(array1024_auto)] = {}; + u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; + char *vuid; + int err; + + MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); + MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); + MLX5_SET(query_vuid_in, in, data_direct, 1); + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); + memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); + return 0; +} + static const struct pci_device_id mlx5_data_direct_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ { 0, } diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index a9bb471949e2..fa3a145c06c6 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -7,7 +7,9 @@ #define _MLX5_IB_DATA_DIRECT_H #include +#include +struct mlx5_core_dev; struct mlx5_ib_dev; enum mlx5_data_direct_event { @@ -21,6 +23,7 @@ struct mlx5_data_direct_dev { struct list_head list; }; +int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, struct notifier_block *nb); void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 0c57cb39ae7e..8d6914de1442 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -4039,7 +4040,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid); + ret = mlx5_data_direct_query_vuid(dev->mdev, vuid); if (ret) return ret; diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h new file mode 100644 index 000000000000..4556e58c6d03 --- /dev/null +++ b/include/linux/mlx5/data_direct.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _MLX5_DATA_DIRECT_H +#define _MLX5_DATA_DIRECT_H + +struct mlx5_core_dev; + +int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); + +#endif -- cgit v1.2.3 From 1c879b1e9f2ed620523e17ca239683bc8029e143 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:23 +0300 Subject: RDMA/mlx5: Add new registration stage for data direct users For the upcoming move of data direct to mlx5_core driver, the API needs to be split into init and register stages. That's because only the core will do the init phase but both IB and netdev devices will need to register. While data direct still lives in mlx5_ib, do all steps in mlx5_ib_data_direct_init(). While at it, make mlx5_data_direct_query_vuid() private as it is no longer needed. This patch has no functional changes. It is a preparation for moving data direct infrastructure to mlx5_core. For this reason the new functions are added to the new header. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-7-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 114 +++++++++++++++++++++++++------ drivers/infiniband/hw/mlx5/data_direct.h | 5 -- drivers/infiniband/hw/mlx5/main.c | 41 +++++------ drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 - include/linux/mlx5/data_direct.h | 10 ++- 5 files changed, 118 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index cc3f741b4f81..f871e3ce9f73 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -24,7 +24,8 @@ struct mlx5_data_direct_registration { struct blocking_notifier_head users; }; -int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid) +static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, + char *out_vuid) { u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + MLX5_ST_SZ_BYTES(array1024_auto)] = {}; @@ -182,25 +183,65 @@ void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) memset(&dev->ddr, 0, sizeof(dev->ddr)); } -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, - struct notifier_block *nb) +static struct mlx5_data_direct_registration * +mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev) +{ + struct mlx5_data_direct_registration *reg; + + list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) + if (reg->ibdev == ibdev) + return reg; + return NULL; +} + +static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev, + struct mlx5_data_direct_dev *dev) +{ + mutex_lock(&ibdev->data_direct_lock); + ibdev->data_direct_dev = dev; + mutex_unlock(&ibdev->data_direct_lock); +} + +static void +mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) +{ + struct mlx5_ib_dev *ibdev = reg->ibdev; + + mutex_lock(&ibdev->data_direct_lock); + blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, + NULL); + ibdev->data_direct_dev = NULL; + mutex_unlock(&ibdev->data_direct_lock); +} + +int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) { struct mlx5_data_direct_registration *reg; struct mlx5_data_direct_dev *dev; + int err; + + if (!mlx5_data_direct_supported(ibdev->mdev)) + return 0; reg = kzalloc_obj(*reg); if (!reg) return -ENOMEM; reg->ibdev = ibdev; - strscpy(reg->vuid, vuid); BLOCKING_INIT_NOTIFIER_HEAD(®->users); - blocking_notifier_chain_register(®->users, nb); + + err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); + if (err) { + mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n", + err); + kfree(reg); + return err; + } mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { - if (strcmp(dev->vuid, vuid) == 0) { - mlx5_ib_data_direct_bind(ibdev, dev); + if (strcmp(dev->vuid, reg->vuid) == 0) { + mlx5_data_direct_bind(ibdev, dev); break; } } @@ -213,25 +254,56 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, return 0; } -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, - struct notifier_block *nb) +void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) { struct mlx5_data_direct_registration *reg; + if (!mlx5_data_direct_supported(ibdev->mdev)) + return; + mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (reg->ibdev == ibdev) { - blocking_notifier_chain_unregister(®->users, nb); - list_del(®->list); - kfree(reg); - goto end; - } + reg = mlx5_data_direct_get_reg(ibdev); + if (reg) { + list_del(®->list); + mlx5_data_direct_do_unbind(reg); } + mutex_unlock(&mlx5_data_direct_mutex); - WARN_ON(true); -end: + kfree(reg); +} + +int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(ibdev->mdev)) + return 0; + + mutex_lock(&mlx5_data_direct_mutex); + reg = mlx5_data_direct_get_reg(ibdev); + if (reg) + blocking_notifier_chain_register(®->users, nb); + mutex_unlock(&mlx5_data_direct_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(mlx5_data_direct_register); + +void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(ibdev->mdev)) + return; + + mutex_lock(&mlx5_data_direct_mutex); + reg = mlx5_data_direct_get_reg(ibdev); + if (reg) + blocking_notifier_chain_unregister(®->users, nb); mutex_unlock(&mlx5_data_direct_mutex); } +EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) { @@ -240,7 +312,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_ib_data_direct_bind(reg->ibdev, dev); + mlx5_data_direct_bind(reg->ibdev, dev); } /* Add the data direct device to the global list, further IB devices may @@ -259,9 +331,7 @@ static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) list_del(&dev->list); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - blocking_notifier_call_chain(®->users, - MLX5_DATA_DIRECT_UNBIND, - NULL); + mlx5_data_direct_do_unbind(reg); } mutex_unlock(&mlx5_data_direct_mutex); } diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index 90e0565d7333..f207239461bf 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -33,11 +33,6 @@ struct mlx5_data_direct_resources { int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); -int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, - struct notifier_block *nb); -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, - struct notifier_block *nb); int mlx5_data_direct_driver_register(void); void mlx5_data_direct_driver_unregister(void); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 0bece3e55109..d0141509870a 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3945,33 +3945,41 @@ static int mlx5_ib_data_direct_event(struct notifier_block *nb, struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev, data_direct_nb); - if (action == MLX5_DATA_DIRECT_UNBIND) - mlx5_ib_data_direct_unbind(dev); + if (action != MLX5_DATA_DIRECT_UNBIND) + return NOTIFY_DONE; + + mlx5_ib_revoke_data_direct_mrs(dev); return NOTIFY_OK; } static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) { - char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {}; int ret; if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_data_direct_query_vuid(dev->mdev, vuid); + ret = mlx5_data_direct_init(dev); if (ret) return ret; ret = mlx5_data_direct_create_resources(dev); if (ret) - return ret; + goto err_resources; INIT_LIST_HEAD(&dev->data_direct_mr_list); dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; - ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb); + ret = mlx5_data_direct_register(dev, &dev->data_direct_nb); if (ret) - mlx5_data_direct_free_resources(dev); + goto err_register; + + return ret; + +err_register: + mlx5_data_direct_free_resources(dev); +err_resources: + mlx5_data_direct_cleanup(dev); return ret; } @@ -3981,8 +3989,9 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return; - mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb); + mlx5_data_direct_unregister(dev, &dev->data_direct_nb); mlx5_data_direct_free_resources(dev); + mlx5_data_direct_cleanup(dev); } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) @@ -5008,22 +5017,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev) cancel_work_sync(&devr->ports[port].pkey_change_work); } -void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, - struct mlx5_data_direct_dev *dev) -{ - mutex_lock(&ibdev->data_direct_lock); - ibdev->data_direct_dev = dev; - mutex_unlock(&ibdev->data_direct_lock); -} - -void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev) -{ - mutex_lock(&ibdev->data_direct_lock); - mlx5_ib_revoke_data_direct_mrs(ibdev); - ibdev->data_direct_dev = NULL; - mutex_unlock(&ibdev->data_direct_lock); -} - void __mlx5_ib_remove(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile, int stage) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index ff026985e293..e56f3eee7ba1 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1396,9 +1396,6 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, struct ib_dm_mr_attr *attr, struct uverbs_attr_bundle *attrs); -void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, - struct mlx5_data_direct_dev *dev); -void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev); void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev); #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index 4556e58c6d03..71f47547aebb 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -7,7 +7,15 @@ #define _MLX5_DATA_DIRECT_H struct mlx5_core_dev; +struct mlx5_ib_dev; +struct notifier_block; -int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); +int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); +void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); + +int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb); +void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb); #endif -- cgit v1.2.3 From f6c452b311f0e4a8fc2a0c287dde298d5492112a Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:25 +0300 Subject: RDMA/mlx5: Consolidate data direct state into one object There are three per device data direct objects: mlx5_data_direct_dev, mlx5_data_direct_resources and mlx5_data_direct_registration with separate allocations. This patch consolidates all of them into the registration object. For this, a few things needed to happen: - mlx5_ib_dev keeps a pointer to mlx5_data_direct. - mlx5_data_direct_dev is now accessible from mlx5_data_direct. mlx5_data_direct_get_dev() is added as a helper to access the device pointer. - Since mlx5_data_direct is part of mlx5_data_direct_registration, getting to the registration is a simple container_of. No need for searching the list. Since mlx5_data_direct_resources no longer has an accurate name, this patch renames it. This is a preparation for the upcoming move of data_direct to mlx5_core. This patch has no functional changes. Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-9-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 73 +++++++++++++------------------- drivers/infiniband/hw/mlx5/data_direct.h | 7 --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +- drivers/infiniband/hw/mlx5/mr.c | 6 +-- drivers/infiniband/hw/mlx5/odp.c | 2 +- drivers/infiniband/hw/mlx5/std_types.c | 2 +- drivers/infiniband/hw/mlx5/umr.c | 11 +++-- include/linux/mlx5/data_direct.h | 17 ++++++++ 8 files changed, 59 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 16afacdd3601..f58a144d95e3 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -18,10 +18,10 @@ static LIST_HEAD(mlx5_data_direct_reg_list); static DEFINE_MUTEX(mlx5_data_direct_mutex); struct mlx5_data_direct_registration { - struct mlx5_ib_dev *ibdev; char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; struct list_head list; struct blocking_notifier_head users; + struct mlx5_data_direct dd; }; static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, @@ -139,8 +139,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) if (err) goto err_mkey; - dev->ddr.mkey = mkey; - dev->ddr.pdn = pdn; + dev->data_direct->mkey = mkey; + dev->data_direct->pdn = pdn; /* create another mkey with RO support */ if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { @@ -157,8 +157,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); /* RO is defined as best effort */ if (!err) { - dev->ddr.mkey_ro = mkey; - dev->ddr.mkey_ro_valid = true; + dev->data_direct->mkey_ro = mkey; + dev->data_direct->mkey_ro_valid = true; } } @@ -174,38 +174,25 @@ err: void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) { - if (dev->ddr.mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro); + if (dev->data_direct->mkey_ro_valid) + mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro); - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn); + mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey); + mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn); - memset(&dev->ddr, 0, sizeof(dev->ddr)); + memset(dev->data_direct, 0, sizeof(*dev->data_direct)); } -static struct mlx5_data_direct_registration * -mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev) -{ - struct mlx5_data_direct_registration *reg; - - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) - if (reg->ibdev == ibdev) - return reg; - return NULL; -} - -static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev, +static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, struct mlx5_data_direct_dev *dev) { - WRITE_ONCE(ibdev->data_direct_dev, dev); + WRITE_ONCE(reg->dd.dev, dev); } static void mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) { - struct mlx5_ib_dev *ibdev = reg->ibdev; - - WRITE_ONCE(ibdev->data_direct_dev, NULL); + WRITE_ONCE(reg->dd.dev, NULL); blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, NULL); } @@ -223,7 +210,6 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) if (!reg) return -ENOMEM; - reg->ibdev = ibdev; BLOCKING_INIT_NOTIFIER_HEAD(®->users); err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); @@ -234,10 +220,12 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) return err; } + ibdev->data_direct = ®->dd; + mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) { - mlx5_data_direct_bind(ibdev, dev); + mlx5_data_direct_bind(reg, dev); break; } } @@ -257,14 +245,14 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) if (!mlx5_data_direct_supported(ibdev->mdev)) return; + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) { - list_del(®->list); - mlx5_data_direct_do_unbind(reg); - } + list_del(®->list); + mlx5_data_direct_do_unbind(reg); mutex_unlock(&mlx5_data_direct_mutex); + ibdev->data_direct = NULL; kfree(reg); } @@ -276,11 +264,10 @@ int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, if (!mlx5_data_direct_supported(ibdev->mdev)) return 0; - mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) - blocking_notifier_chain_register(®->users, nb); - mutex_unlock(&mlx5_data_direct_mutex); + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_register(®->users, nb); + return 0; } EXPORT_SYMBOL_GPL(mlx5_data_direct_register); @@ -293,11 +280,9 @@ void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, if (!mlx5_data_direct_supported(ibdev->mdev)) return; - mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) - blocking_notifier_chain_unregister(®->users, nb); - mutex_unlock(&mlx5_data_direct_mutex); + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_unregister(®->users, nb); } EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); @@ -308,7 +293,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_data_direct_bind(reg->ibdev, dev); + mlx5_data_direct_bind(reg, dev); } /* Add the data direct device to the global list, further IB devices may diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index f207239461bf..de2730f00852 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -23,13 +23,6 @@ struct mlx5_data_direct_dev { struct list_head list; }; -struct mlx5_data_direct_resources { - u32 pdn; - u32 mkey; - u32 mkey_ro; - u8 mkey_ro_valid :1; -}; - int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 8792e10340c5..6952170f472b 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1091,7 +1091,7 @@ struct mlx5_macsec { struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; - struct mlx5_data_direct_dev *data_direct_dev; + struct mlx5_data_direct *data_direct; /* Protects data_direct_mr_list and serializes mr * registration/deregistration with data direct device unbind. */ @@ -1152,7 +1152,6 @@ struct mlx5_ib_dev { u16 pkey_table_len; u8 lag_ports; struct mlx5_special_mkeys mkeys; - struct mlx5_data_direct_resources ddr; #ifdef CONFIG_MLX5_MACSEC struct mlx5_macsec macsec; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 1fc61a680f8a..5efe78cb34c1 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -587,7 +587,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, populate ? pd : dev->umrc.pd); /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ if (umem->is_dmabuf && ksm_mode) - MLX5_SET(mkc, mkc, pd, dev->ddr.pdn); + MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode); @@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = READ_ONCE(dev->data_direct_dev); + data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); if (!data_direct_dev) { ret = -EINVAL; goto end; @@ -1006,7 +1006,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, /* If no device's 'data direct mkey' with RO flags exists * mask it out accordingly. */ - if (!dev->ddr.mkey_ro_valid) + if (!dev->data_direct->mkey_ro_valid) access_flags &= ~IB_ACCESS_RELAXED_ORDERING; crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, offset, length, virt_addr, fd, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1badec9bf527..7b9f73b76d0d 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) int ret; if (mr->data_direct) - pdn = dev->ddr.pdn; + pdn = dev->data_direct->pdn; else pdn = to_mpd(pd)->pdn; diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index df949fa634c9..3828100fdc6c 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = READ_ONCE(dev->data_direct_dev); + data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); if (!data_direct_dev) { ret = -ENODEV; goto end; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 48cae5cc1c1b..1ca83d2d8a0b 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -760,12 +760,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, } if (dd) { + struct mlx5_data_direct *mdd = dev->data_direct; + cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && - dev->ddr.mkey_ro_valid) - cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro); + mdd->mkey_ro_valid) + cur_ksm->key = cpu_to_be32(mdd->mkey_ro); else - cur_ksm->key = cpu_to_be32(dev->ddr.mkey); + cur_ksm->key = cpu_to_be32(mdd->mkey); if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP)) { cur_ksm->va = 0; @@ -810,7 +812,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + return _mlx5r_umr_update_mr_pas(mr, flags, + mr_to_mdev(mr)->data_direct->pdn, true, start_block, nblocks); } diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index 71f47547aebb..eaa2dc6bc607 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -6,10 +6,27 @@ #ifndef _MLX5_DATA_DIRECT_H #define _MLX5_DATA_DIRECT_H +#include + struct mlx5_core_dev; +struct mlx5_data_direct_dev; struct mlx5_ib_dev; struct notifier_block; +struct mlx5_data_direct { + struct mlx5_data_direct_dev *dev; + u32 pdn; + u32 mkey; + u32 mkey_ro; + u8 mkey_ro_valid :1; +}; + +static inline struct mlx5_data_direct_dev * +mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) +{ + return dd ? READ_ONCE(dd->dev) : NULL; +} + int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); -- cgit v1.2.3 From 00290b9ff59ef47b990d26a6b1a8b2ea6dedf5e9 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:27 +0300 Subject: mlx5: Move data direct implementation to mlx5_core As data direct will also be used by netdev devices, move the data direct code to mlx5_core. Now mlx5_core is responsible for data direct initialization and resource creation. Now data_direct is initialized in mlx5_load() before attaching consumers and clean it up in mlx5_unload() after detaching them. The data direct PCI driver is moved and registered in mlx5_core. IB specific work stays in mlx5_ib: notifier registration, mutex initialization and MR list initialization. Failure to query the VUID is now fatal because this is done only on devices that support data direct. Other than this, this patch has no additional functional changes. Signed-off-by: Dragos Tatulea Link: https://patch.msgid.link/20260930114227.139274-11-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/Makefile | 1 - drivers/infiniband/hw/mlx5/data_direct.c | 397 -------------------- drivers/infiniband/hw/mlx5/data_direct.h | 29 -- drivers/infiniband/hw/mlx5/main.c | 23 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +- drivers/infiniband/hw/mlx5/mr.c | 7 +- drivers/infiniband/hw/mlx5/odp.c | 2 +- drivers/infiniband/hw/mlx5/std_types.c | 3 +- drivers/infiniband/hw/mlx5/umr.c | 4 +- drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +- .../net/ethernet/mellanox/mlx5/core/data_direct.c | 400 +++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/main.c | 15 + .../net/ethernet/mellanox/mlx5/core/mlx5_core.h | 5 + include/linux/mlx5/data_direct.h | 23 +- include/linux/mlx5/driver.h | 2 + 15 files changed, 449 insertions(+), 467 deletions(-) delete mode 100644 drivers/infiniband/hw/mlx5/data_direct.c delete mode 100644 drivers/infiniband/hw/mlx5/data_direct.h create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/data_direct.c (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile index dd7bb377f491..35afc7bc57d2 100644 --- a/drivers/infiniband/hw/mlx5/Makefile +++ b/drivers/infiniband/hw/mlx5/Makefile @@ -6,7 +6,6 @@ mlx5_ib-y := ah.o \ cong.o \ counters.o \ cq.o \ - data_direct.o \ dm.o \ dmah.o \ doorbell.o \ diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c deleted file mode 100644 index 3878c4aebb2c..000000000000 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ /dev/null @@ -1,397 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB -/* - * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved - */ - -#include "mlx5_ib.h" - -#include - -#include "data_direct.h" - -static LIST_HEAD(mlx5_data_direct_dev_list); -static LIST_HEAD(mlx5_data_direct_reg_list); - -/* - * This mutex should be held when accessing either of the above lists - */ -static DEFINE_MUTEX(mlx5_data_direct_mutex); - -struct mlx5_data_direct_registration { - char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; - struct list_head list; - struct blocking_notifier_head users; - struct mlx5_data_direct dd; -}; - -static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, - char *out_vuid) -{ - u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + - MLX5_ST_SZ_BYTES(array1024_auto)] = {}; - u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; - char *vuid; - int err; - - MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); - MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); - MLX5_SET(query_vuid_in, in, data_direct, 1); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (err) - return err; - - vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); - memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); - return 0; -} - -static const struct pci_device_id mlx5_data_direct_pci_table[] = { - { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ - { 0, } -}; - -static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev) -{ - struct pci_dev *pdev = dev->pdev; - unsigned int vpd_size, kw_len; - u8 *vpd_data; - int start; - int ret; - - vpd_data = pci_vpd_alloc(pdev, &vpd_size); - if (IS_ERR(vpd_data)) { - pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data); - return PTR_ERR(vpd_data); - } - - start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len); - if (start < 0) { - ret = start; - pci_err(pdev, "VU keyword not found, err=%d\n", ret); - goto end; - } - - dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); - ret = dev->vuid ? 0 : -ENOMEM; - -end: - kfree(vpd_data); - return ret; -} - -static void mlx5_data_direct_shutdown(struct pci_dev *pdev) -{ - pci_disable_device(pdev); -} - -static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) -{ - int err; - - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (err) { - dev_warn(&pdev->dev, - "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err); - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err); - return err; - } - } - - dma_set_max_seg_size(&pdev->dev, SZ_2G); - return 0; -} - -static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) -{ - int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - struct mlx5_core_dev *mdev = dev->mdev; - bool ro_supp = false; - void *mkc; - u32 mkey; - u32 pdn; - u32 *in; - int err; - - err = mlx5_core_alloc_pd(mdev, &pdn); - if (err) - return err; - - in = kvzalloc(inlen, GFP_KERNEL); - if (!in) { - err = -ENOMEM; - goto err; - } - - MLX5_SET(create_mkey_in, in, data_direct, 1); - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); - MLX5_SET(mkc, mkc, lw, 1); - MLX5_SET(mkc, mkc, lr, 1); - MLX5_SET(mkc, mkc, rw, 1); - MLX5_SET(mkc, mkc, rr, 1); - MLX5_SET(mkc, mkc, a, 1); - MLX5_SET(mkc, mkc, pd, pdn); - MLX5_SET(mkc, mkc, length64, 1); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); - if (err) - goto err_mkey; - - dev->data_direct->mkey = mkey; - dev->data_direct->pdn = pdn; - - /* create another mkey with RO support */ - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { - MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); - ro_supp = true; - } - - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) { - MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); - ro_supp = true; - } - - if (ro_supp) { - err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); - /* RO is defined as best effort */ - if (!err) { - dev->data_direct->mkey_ro = mkey; - dev->data_direct->mkey_ro_valid = true; - } - } - - kvfree(in); - return 0; - -err_mkey: - kvfree(in); -err: - mlx5_core_dealloc_pd(mdev, pdn); - return err; -} - -static void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) -{ - if (dev->data_direct->mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro); - - mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn); -} - -static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, - struct mlx5_data_direct_dev *dev) -{ - WRITE_ONCE(reg->dd.dev, dev); -} - -static void -mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) -{ - WRITE_ONCE(reg->dd.dev, NULL); - blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, - NULL); -} - -int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) -{ - struct mlx5_data_direct_registration *reg; - struct mlx5_data_direct_dev *dev; - int err; - - if (!mlx5_data_direct_supported(ibdev->mdev)) - return 0; - - reg = kzalloc_obj(*reg); - if (!reg) - return -ENOMEM; - - BLOCKING_INIT_NOTIFIER_HEAD(®->users); - - err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); - if (err) { - mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n", - err); - kfree(reg); - return err; - } - - ibdev->data_direct = ®->dd; - - err = mlx5_data_direct_create_resources(ibdev); - if (err) - goto err_resources; - - mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { - if (strcmp(dev->vuid, reg->vuid) == 0) { - mlx5_data_direct_bind(reg, dev); - break; - } - } - - /* Add the registration to its global list, to be used upon bind/unbind - * of its affiliated data direct device - */ - list_add_tail(®->list, &mlx5_data_direct_reg_list); - mutex_unlock(&mlx5_data_direct_mutex); - return 0; - -err_resources: - ibdev->data_direct = NULL; - kfree(reg); - return err; -} - -void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) -{ - struct mlx5_data_direct_registration *reg; - - if (!mlx5_data_direct_supported(ibdev->mdev)) - return; - - reg = container_of(ibdev->data_direct, - struct mlx5_data_direct_registration, dd); - mutex_lock(&mlx5_data_direct_mutex); - list_del(®->list); - mlx5_data_direct_do_unbind(reg); - mutex_unlock(&mlx5_data_direct_mutex); - - mlx5_data_direct_free_resources(ibdev); - ibdev->data_direct = NULL; - kfree(reg); -} - -int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, - struct notifier_block *nb) -{ - struct mlx5_data_direct_registration *reg; - - if (!mlx5_data_direct_supported(ibdev->mdev)) - return 0; - - reg = container_of(ibdev->data_direct, - struct mlx5_data_direct_registration, dd); - blocking_notifier_chain_register(®->users, nb); - - return 0; -} -EXPORT_SYMBOL_GPL(mlx5_data_direct_register); - -void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, - struct notifier_block *nb) -{ - struct mlx5_data_direct_registration *reg; - - if (!mlx5_data_direct_supported(ibdev->mdev)) - return; - - reg = container_of(ibdev->data_direct, - struct mlx5_data_direct_registration, dd); - blocking_notifier_chain_unregister(®->users, nb); -} -EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); - -static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) -{ - struct mlx5_data_direct_registration *reg; - - mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_data_direct_bind(reg, dev); - } - - /* Add the data direct device to the global list, further IB devices may - * use it later as well - */ - list_add_tail(&dev->list, &mlx5_data_direct_dev_list); - mutex_unlock(&mlx5_data_direct_mutex); -} - -static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) -{ - struct mlx5_data_direct_registration *reg; - - mutex_lock(&mlx5_data_direct_mutex); - /* Prevent any further affiliations */ - list_del(&dev->list); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_data_direct_do_unbind(reg); - } - mutex_unlock(&mlx5_data_direct_mutex); -} - -static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - struct mlx5_data_direct_dev *dev; - int err; - - dev = kzalloc_obj(*dev); - if (!dev) - return -ENOMEM; - - dev->device = &pdev->dev; - dev->pdev = pdev; - - pci_set_drvdata(dev->pdev, dev); - err = pci_enable_device(pdev); - if (err) { - dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err); - goto err; - } - - pci_set_master(pdev); - err = mlx5_data_direct_set_dma_caps(pdev); - if (err) - goto err_disable; - - if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && - pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && - pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) - dev_dbg(dev->device, "Enabling pci atomics failed\n"); - - err = mlx5_data_direct_vpd_get_vuid(dev); - if (err) - goto err_disable; - - mlx5_data_direct_dev_reg(dev); - return 0; - -err_disable: - pci_disable_device(pdev); -err: - kfree(dev); - return err; -} - -static void mlx5_data_direct_remove(struct pci_dev *pdev) -{ - struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev); - - mlx5_data_direct_dev_unreg(dev); - pci_disable_device(pdev); - kfree(dev->vuid); - kfree(dev); -} - -static struct pci_driver mlx5_data_direct_driver = { - .name = "mlx5_dd", - .id_table = mlx5_data_direct_pci_table, - .probe = mlx5_data_direct_probe, - .remove = mlx5_data_direct_remove, - .shutdown = mlx5_data_direct_shutdown, -}; - -int mlx5_data_direct_driver_register(void) -{ - return pci_register_driver(&mlx5_data_direct_driver); -} - -void mlx5_data_direct_driver_unregister(void) -{ - pci_unregister_driver(&mlx5_data_direct_driver); -} diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h deleted file mode 100644 index 49cdd407018a..000000000000 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* - * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved - */ - -#ifndef _MLX5_IB_DATA_DIRECT_H -#define _MLX5_IB_DATA_DIRECT_H - -#include -#include - -struct mlx5_core_dev; -struct mlx5_ib_dev; - -enum mlx5_data_direct_event { - MLX5_DATA_DIRECT_UNBIND, -}; - -struct mlx5_data_direct_dev { - struct device *device; - struct pci_dev *pdev; - char *vuid; - struct list_head list; -}; - -int mlx5_data_direct_driver_register(void); -void mlx5_data_direct_driver_unregister(void); - -#endif diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 941928a782bd..dd2a3996b14e 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -52,7 +52,6 @@ #include #include #include "macsec.h" -#include "data_direct.h" #include "dmah.h" #define UVERBS_MODULE_NAME mlx5_ib @@ -3962,20 +3961,9 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_data_direct_init(dev); - if (ret) - return ret; - INIT_LIST_HEAD(&dev->data_direct_mr_list); dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; - ret = mlx5_data_direct_register(dev, &dev->data_direct_nb); - if (ret) - goto err_register; - - return ret; - -err_register: - mlx5_data_direct_cleanup(dev); + ret = mlx5_data_direct_register(dev->mdev, &dev->data_direct_nb); return ret; } @@ -3985,8 +3973,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return; - mlx5_data_direct_unregister(dev, &dev->data_direct_nb); - mlx5_data_direct_cleanup(dev); + mlx5_data_direct_unregister(dev->mdev, &dev->data_direct_nb); } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) @@ -5438,9 +5425,6 @@ static int __init mlx5_ib_init(void) ret = mlx5r_rep_init(); if (ret) goto rep_err; - ret = mlx5_data_direct_driver_register(); - if (ret) - goto dd_err; ret = auxiliary_driver_register(&mlx5r_mp_driver); if (ret) goto mp_err; @@ -5453,8 +5437,6 @@ static int __init mlx5_ib_init(void) drv_err: auxiliary_driver_unregister(&mlx5r_mp_driver); mp_err: - mlx5_data_direct_driver_unregister(); -dd_err: mlx5r_rep_cleanup(); rep_err: mlx5_ib_qp_event_cleanup(); @@ -5466,7 +5448,6 @@ qp_event_err: static void __exit mlx5_ib_cleanup(void) { - mlx5_data_direct_driver_unregister(); auxiliary_driver_unregister(&mlx5r_driver); auxiliary_driver_unregister(&mlx5r_mp_driver); mlx5r_rep_cleanup(); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 6952170f472b..1a3b9cb79184 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -24,7 +24,7 @@ #include #include -#include "data_direct.h" +#include #include "srq.h" #include "qp.h" #include "macsec.h" @@ -1091,7 +1091,6 @@ struct mlx5_macsec { struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; - struct mlx5_data_direct *data_direct; /* Protects data_direct_mr_list and serializes mr * registration/deregistration with data direct device unbind. */ diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 5efe78cb34c1..7463b36a36b6 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -44,7 +44,6 @@ #include "dm.h" #include "mlx5_ib.h" #include "umr.h" -#include "data_direct.h" #include "dmah.h" static int mkey_max_umr_order(struct mlx5_ib_dev *dev) @@ -587,7 +586,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, populate ? pd : dev->umrc.pd); /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ if (umem->is_dmabuf && ksm_mode) - MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn); + MLX5_SET(mkc, mkc, pd, dev->mdev->data_direct->pdn); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode); @@ -997,7 +996,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); + data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct); if (!data_direct_dev) { ret = -EINVAL; goto end; @@ -1006,7 +1005,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, /* If no device's 'data direct mkey' with RO flags exists * mask it out accordingly. */ - if (!dev->data_direct->mkey_ro_valid) + if (!dev->mdev->data_direct->mkey_ro_valid) access_flags &= ~IB_ACCESS_RELAXED_ORDERING; crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, offset, length, virt_addr, fd, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 7b9f73b76d0d..2e9a704acc44 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) int ret; if (mr->data_direct) - pdn = dev->data_direct->pdn; + pdn = dev->mdev->data_direct->pdn; else pdn = to_mpd(pd)->pdn; diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 3828100fdc6c..b705b88dc917 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -10,7 +10,6 @@ #include #include #include "mlx5_ib.h" -#include "data_direct.h" #define UVERBS_MODULE_NAME mlx5_ib #include @@ -203,7 +202,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); + data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct); if (!data_direct_dev) { ret = -ENODEV; goto end; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 1ca83d2d8a0b..a3bb8d03fd35 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -760,7 +760,7 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, } if (dd) { - struct mlx5_data_direct *mdd = dev->data_direct; + struct mlx5_data_direct *mdd = dev->mdev->data_direct; cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && @@ -813,7 +813,7 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, return -EINVAL; return _mlx5r_umr_update_mr_pas(mr, flags, - mr_to_mdev(mr)->data_direct->pdn, + mr_to_mdev(mr)->mdev->data_direct->pdn, true, start_block, nblocks); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 19e50f0d55af..a1871259210d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -18,7 +18,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o diag/rsc_dump.o \ diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o \ - lib/nv_param.o + lib/nv_param.o data_direct.o # # Netdev basic diff --git a/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c new file mode 100644 index 000000000000..5afb11a2a9f0 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#include +#include +#include +#include +#include +#include + +#include "mlx5_core.h" + +static LIST_HEAD(mlx5_data_direct_dev_list); +static LIST_HEAD(mlx5_data_direct_reg_list); + +/* + * This mutex should be held when accessing either of the above lists + */ +static DEFINE_MUTEX(mlx5_data_direct_mutex); + +struct mlx5_data_direct_registration { + char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; + struct list_head list; + struct blocking_notifier_head users; + struct mlx5_data_direct dd; +}; + +static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, + char *out_vuid) +{ + u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + + MLX5_ST_SZ_BYTES(array1024_auto)] = {}; + u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; + char *vuid; + int err; + + MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); + MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); + MLX5_SET(query_vuid_in, in, data_direct, 1); + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); + memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); + return 0; +} + +static const struct pci_device_id mlx5_data_direct_pci_table[] = { + { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ + { 0, } +}; + +static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev) +{ + struct pci_dev *pdev = dev->pdev; + unsigned int vpd_size, kw_len; + u8 *vpd_data; + int start; + int ret; + + vpd_data = pci_vpd_alloc(pdev, &vpd_size); + if (IS_ERR(vpd_data)) { + pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data); + return PTR_ERR(vpd_data); + } + + start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len); + if (start < 0) { + ret = start; + pci_err(pdev, "VU keyword not found, err=%d\n", ret); + goto end; + } + + dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); + ret = dev->vuid ? 0 : -ENOMEM; + +end: + kfree(vpd_data); + return ret; +} + +static void mlx5_data_direct_shutdown(struct pci_dev *pdev) +{ + pci_disable_device(pdev); +} + +static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) +{ + int err; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_warn(&pdev->dev, + "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err); + return err; + } + } + + dma_set_max_seg_size(&pdev->dev, SZ_2G); + return 0; +} + +static int mlx5_data_direct_create_resources(struct mlx5_core_dev *mdev) +{ + int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); + bool ro_supp = false; + void *mkc; + u32 mkey; + u32 pdn; + u32 *in; + int err; + + err = mlx5_core_alloc_pd(mdev, &pdn); + if (err) + return err; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err; + } + + MLX5_SET(create_mkey_in, in, data_direct, 1); + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); + MLX5_SET(mkc, mkc, lw, 1); + MLX5_SET(mkc, mkc, lr, 1); + MLX5_SET(mkc, mkc, rw, 1); + MLX5_SET(mkc, mkc, rr, 1); + MLX5_SET(mkc, mkc, a, 1); + MLX5_SET(mkc, mkc, pd, pdn); + MLX5_SET(mkc, mkc, length64, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); + if (err) + goto err_mkey; + + mdev->data_direct->mkey = mkey; + mdev->data_direct->pdn = pdn; + + /* create another mkey with RO support */ + if (MLX5_CAP_GEN(mdev, relaxed_ordering_write)) { + MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); + ro_supp = true; + } + + if (MLX5_CAP_GEN(mdev, relaxed_ordering_read)) { + MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); + ro_supp = true; + } + + if (ro_supp) { + err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); + /* RO is defined as best effort */ + if (!err) { + mdev->data_direct->mkey_ro = mkey; + mdev->data_direct->mkey_ro_valid = true; + } + } + + kvfree(in); + return 0; + +err_mkey: + kvfree(in); +err: + mlx5_core_dealloc_pd(mdev, pdn); + return err; +} + +static void mlx5_data_direct_free_resources(struct mlx5_core_dev *mdev) +{ + struct mlx5_data_direct *data_direct = mdev->data_direct; + + if (data_direct->mkey_ro_valid) + mlx5_core_destroy_mkey(mdev, data_direct->mkey_ro); + + mlx5_core_destroy_mkey(mdev, data_direct->mkey); + mlx5_core_dealloc_pd(mdev, data_direct->pdn); +} + +static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, + struct mlx5_data_direct_dev *dev) +{ + WRITE_ONCE(reg->dd.dev, dev); +} + +static void +mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) +{ + WRITE_ONCE(reg->dd.dev, NULL); + blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, + NULL); +} + +int mlx5_data_direct_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_data_direct_registration *reg; + struct mlx5_data_direct_dev *dev; + int err; + + if (!mlx5_data_direct_supported(mdev)) + return 0; + + reg = kzalloc_obj(*reg); + if (!reg) + return -ENOMEM; + + BLOCKING_INIT_NOTIFIER_HEAD(®->users); + + err = mlx5_data_direct_query_vuid(mdev, reg->vuid); + if (err) { + mlx5_core_warn(mdev, "Failed to query VUID, err=%d\n", err); + kfree(reg); + return -EINVAL; + } + + mdev->data_direct = ®->dd; + + err = mlx5_data_direct_create_resources(mdev); + if (err) + goto err_resources; + + mutex_lock(&mlx5_data_direct_mutex); + list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { + if (strcmp(dev->vuid, reg->vuid) == 0) { + mlx5_data_direct_bind(reg, dev); + break; + } + } + + /* Add the registration to its global list, to be used upon bind/unbind + * of its affiliated data direct device. + */ + list_add_tail(®->list, &mlx5_data_direct_reg_list); + mutex_unlock(&mlx5_data_direct_mutex); + return 0; + +err_resources: + mdev->data_direct = NULL; + kfree(reg); + return err; +} + +void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(mdev)) + return; + + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); + mutex_lock(&mlx5_data_direct_mutex); + list_del(®->list); + mlx5_data_direct_do_unbind(reg); + mutex_unlock(&mlx5_data_direct_mutex); + + mlx5_data_direct_free_resources(mdev); + mdev->data_direct = NULL; + kfree(reg); +} + +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(mdev)) + return 0; + + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_register(®->users, nb); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx5_data_direct_register); + +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(mdev)) + return; + + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_unregister(®->users, nb); +} +EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); + +static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) +{ + struct mlx5_data_direct_registration *reg; + + mutex_lock(&mlx5_data_direct_mutex); + list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { + if (strcmp(dev->vuid, reg->vuid) == 0) + mlx5_data_direct_bind(reg, dev); + } + + /* Add the data direct device to the global list, further mlx5 devices + * may use it later as well. + */ + list_add_tail(&dev->list, &mlx5_data_direct_dev_list); + mutex_unlock(&mlx5_data_direct_mutex); +} + +static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) +{ + struct mlx5_data_direct_registration *reg; + + mutex_lock(&mlx5_data_direct_mutex); + /* Prevent any further affiliations */ + list_del(&dev->list); + list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { + if (strcmp(dev->vuid, reg->vuid) == 0) + mlx5_data_direct_do_unbind(reg); + } + mutex_unlock(&mlx5_data_direct_mutex); +} + +static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct mlx5_data_direct_dev *dev; + int err; + + dev = kzalloc_obj(*dev); + if (!dev) + return -ENOMEM; + + dev->device = &pdev->dev; + dev->pdev = pdev; + + pci_set_drvdata(dev->pdev, dev); + err = pci_enable_device(pdev); + if (err) { + dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err); + goto err; + } + + pci_set_master(pdev); + err = mlx5_data_direct_set_dma_caps(pdev); + if (err) + goto err_disable; + + if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && + pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && + pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) + dev_dbg(dev->device, "Enabling pci atomics failed\n"); + + err = mlx5_data_direct_vpd_get_vuid(dev); + if (err) + goto err_disable; + + mlx5_data_direct_dev_reg(dev); + return 0; + +err_disable: + pci_disable_device(pdev); +err: + kfree(dev); + return err; +} + +static void mlx5_data_direct_remove(struct pci_dev *pdev) +{ + struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev); + + mlx5_data_direct_dev_unreg(dev); + pci_disable_device(pdev); + kfree(dev->vuid); + kfree(dev); +} + +static struct pci_driver mlx5_data_direct_driver = { + .name = "mlx5_dd", + .id_table = mlx5_data_direct_pci_table, + .probe = mlx5_data_direct_probe, + .remove = mlx5_data_direct_remove, + .shutdown = mlx5_data_direct_shutdown, +}; + +int mlx5_data_direct_driver_register(void) +{ + return pci_register_driver(&mlx5_data_direct_driver); +} + +void mlx5_data_direct_driver_unregister(void) +{ + pci_unregister_driver(&mlx5_data_direct_driver); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 643b4aac2033..34f0375b6851 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -1339,8 +1340,14 @@ static int mlx5_load(struct mlx5_core_dev *dev) if (err) goto err_traps_reg; + err = mlx5_data_direct_init(dev); + if (err) + goto err_data_direct; + return 0; +err_data_direct: + mlx5_devlink_traps_unregister(priv_to_devlink(dev)); err_traps_reg: mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); @@ -1371,6 +1378,7 @@ err_irq_table: static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_data_direct_cleanup(dev); mlx5_devlink_traps_unregister(priv_to_devlink(dev)); mlx5_vhca_event_stop(dev); mlx5_sf_dev_table_destroy(dev); @@ -2360,6 +2368,10 @@ static int __init mlx5_init(void) if (err) goto err_sf; + err = mlx5_data_direct_driver_register(); + if (err) + goto err_dd; + err = pci_register_driver(&mlx5_core_driver); if (err) goto err_pci; @@ -2367,6 +2379,8 @@ static int __init mlx5_init(void) return 0; err_pci: + mlx5_data_direct_driver_unregister(); +err_dd: mlx5_sf_driver_unregister(); err_sf: mlx5e_cleanup(); @@ -2378,6 +2392,7 @@ err_debug: static void __exit mlx5_cleanup(void) { pci_unregister_driver(&mlx5_core_driver); + mlx5_data_direct_driver_unregister(); mlx5_sf_driver_unregister(); mlx5e_cleanup(); mlx5_unregister_debugfs(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 09e669f83dba..fd93530a3938 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -530,4 +530,9 @@ static inline bool mlx5_pcie_cong_event_supported(struct mlx5_core_dev *dev) return true; } + +int mlx5_data_direct_init(struct mlx5_core_dev *mdev); +void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev); +int mlx5_data_direct_driver_register(void); +void mlx5_data_direct_driver_unregister(void); #endif /* __MLX5_CORE_H__ */ diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index eaa2dc6bc607..4cff5ff5489d 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -7,11 +7,23 @@ #define _MLX5_DATA_DIRECT_H #include +#include +struct device; struct mlx5_core_dev; -struct mlx5_data_direct_dev; -struct mlx5_ib_dev; struct notifier_block; +struct pci_dev; + +enum mlx5_data_direct_event { + MLX5_DATA_DIRECT_UNBIND, +}; + +struct mlx5_data_direct_dev { + struct device *device; + struct pci_dev *pdev; + char *vuid; + struct list_head list; +}; struct mlx5_data_direct { struct mlx5_data_direct_dev *dev; @@ -27,12 +39,9 @@ mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) return dd ? READ_ONCE(dd->dev) : NULL; } -int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); -void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); - -int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, struct notifier_block *nb); -void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, struct notifier_block *nb); #endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index ee29f95ead18..ed348a82576e 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -707,6 +707,7 @@ struct mlx5_rsvd_gids { struct mlx5_clock; struct mlx5_clock_dev_state; +struct mlx5_data_direct; struct mlx5_dm; struct mlx5_fw_tracer; struct mlx5_vxlan; @@ -801,6 +802,7 @@ struct mlx5_core_dev { /* sync write combining state */ struct mutex wc_state_lock; struct devlink *shd; + struct mlx5_data_direct *data_direct; }; struct mlx5_db { -- cgit v1.2.3