diff options
Diffstat (limited to 'drivers/net/ethernet/microsoft')
| -rw-r--r-- | drivers/net/ethernet/microsoft/Kconfig | 26 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/Makefile | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/Makefile | 9 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/gdma_cdx.c | 340 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/gdma_main.c | 1063 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/gdma_pci.c | 917 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/hw_channel.c | 24 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/mana_bpf.c | 5 | ||||
| -rw-r--r-- | drivers/net/ethernet/microsoft/mana/mana_en.c | 70 |
9 files changed, 1513 insertions, 943 deletions
diff --git a/drivers/net/ethernet/microsoft/Kconfig b/drivers/net/ethernet/microsoft/Kconfig index c4ad3d85875a..6ff6ab71bbce 100644 --- a/drivers/net/ethernet/microsoft/Kconfig +++ b/drivers/net/ethernet/microsoft/Kconfig @@ -15,15 +15,19 @@ config NET_VENDOR_MICROSOFT if NET_VENDOR_MICROSOFT +config MICROSOFT_GDMA_CORE + tristate + select AUXILIARY_BUS + select DIMLIB + select PAGE_POOL + select NET_SHAPER + config MICROSOFT_MANA tristate "Microsoft Azure Network Adapter (MANA) support" depends on PCI_MSI depends on X86_64 || ARM64 depends on PCI_HYPERV - select AUXILIARY_BUS - select DIMLIB - select PAGE_POOL - select NET_SHAPER + select MICROSOFT_GDMA_CORE help This driver supports Microsoft Azure Network Adapter (MANA). So far, the driver is only supported on X86_64. @@ -31,4 +35,18 @@ config MICROSOFT_MANA To compile this driver as a module, choose M here. The module will be called mana. +config MICROSOFT_MANA_CDX + tristate "Microsoft Azure Network Adapter (MANA) support for the CDX bus" + depends on CDX_BUS + depends on GENERIC_MSI_IRQ + depends on 64BIT && !CPU_BIG_ENDIAN + select MICROSOFT_GDMA_CORE + help + This driver supports the Microsoft Azure Network Adapter (MANA) on + systems where it is integrated into the SoC and presented on the CDX + bus rather than discovered over PCI. + + To compile this driver as a module, choose M here. + The module will be called mana_cdx. + endif #NET_VENDOR_MICROSOFT diff --git a/drivers/net/ethernet/microsoft/Makefile b/drivers/net/ethernet/microsoft/Makefile index d2ddc218135f..c3b6dafdf473 100644 --- a/drivers/net/ethernet/microsoft/Makefile +++ b/drivers/net/ethernet/microsoft/Makefile @@ -2,4 +2,4 @@ # Makefile for the Microsoft Azure network device driver. # -obj-$(CONFIG_MICROSOFT_MANA) += mana/ +obj-$(CONFIG_MICROSOFT_GDMA_CORE) += mana/ diff --git a/drivers/net/ethernet/microsoft/mana/Makefile b/drivers/net/ethernet/microsoft/mana/Makefile index e16a4221f571..64926015f734 100644 --- a/drivers/net/ethernet/microsoft/mana/Makefile +++ b/drivers/net/ethernet/microsoft/mana/Makefile @@ -2,5 +2,12 @@ # # Makefile for the Microsoft Azure Network Adapter driver +obj-$(CONFIG_MICROSOFT_GDMA_CORE) += gdma_core.o +gdma_core-y := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o \ + mana_bpf.o + obj-$(CONFIG_MICROSOFT_MANA) += mana.o -mana-objs := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o mana_bpf.o +mana-y := gdma_pci.o + +obj-$(CONFIG_MICROSOFT_MANA_CDX) += mana_cdx.o +mana_cdx-y := gdma_cdx.o diff --git a/drivers/net/ethernet/microsoft/mana/gdma_cdx.c b/drivers/net/ethernet/microsoft/mana/gdma_cdx.c new file mode 100644 index 000000000000..0d01e3d2f0d9 --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/gdma_cdx.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2025, Microsoft Corporation. */ +#include <linux/module.h> +#include <linux/bitmap.h> +#include <linux/cdx/cdx_bus.h> +#include <linux/msi.h> +#include <linux/xarray.h> + +#include <net/mana/mana.h> + +#define MANA_CDX_GDMA_REGS_SIZE (GDMA_REG_SHM_OFFSET + sizeof(u64)) + +static int mana_cdx_msix_virq(struct gdma_context *gc, int msi) +{ + unsigned int virq = msi_get_virq(gc->dev, msi); + + /* CDX allocates its whole vector pool in mana_cdx_setup_hwc_irqs(), + * so a vector with no Linux IRQ was never part of that pool. + */ + return virq ? virq : -EINVAL; +} + +static int mana_cdx_dev_reset(struct gdma_context *gc) +{ + return cdx_dev_reset(gc->dev); +} + +static int mana_cdx_setup_hwc_irqs(struct gdma_context *gc) +{ + struct cdx_device *cdx_dev = to_cdx_device(gc->dev); + struct gdma_irq_context *gic; + unsigned int max_irqs; + int nvec; + int err, i = 0, j; + + max_irqs = cdx_dev->num_msi; + + /* Need 1 IRQ for HWC and at least 1 for the data path queues */ + if (max_irqs < 2) { + dev_err(&cdx_dev->dev, "Not enough MSI vectors: %u\n", + max_irqs); + return -ENOSPC; + } + + /* Constrain the device to 32-bit DMA addresses, for both the MSI + * doorbells and the data path buffers, and never widen it afterwards. + * + * The MSI doorbell address has to land within the low 4GB. The data + * path carries a second constraint: the device does not perform + * writes to the final page of the 48-bit IOVA window, and + * iommu_dma_alloc_iova() allocates downwards from the top of that + * window, so the first queue created lands on the unusable page and + * its event queue stalls permanently. PCI devices are shielded from + * this by the low-4GB first attempt in iommu_dma_alloc_iova(), but + * that attempt is gated on dev_is_pci() and CDX devices do not + * qualify. + */ + err = dma_set_mask_and_coherent(gc->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&cdx_dev->dev, "Failed to set 32-bit DMA mask\n"); + return err; + } + + /* The PCI path also calls dma_set_max_seg_size(), but that is + * deliberately omitted here: the CDX core does not attach a + * dma_parms to the device, so the helper would only WARN, and MANA + * never builds scatter-gather lists, so the limit is never read. + */ + + err = cdx_enable_msi(cdx_dev); + if (err) { + dev_err(&cdx_dev->dev, "Failed to enable MSI\n"); + return err; + } + + err = msi_domain_alloc_irqs(gc->dev, MSI_DEFAULT_DOMAIN, max_irqs); + if (err) { + dev_err(&cdx_dev->dev, "Failed to alloc MSI IRQs\n"); + cdx_disable_msi(cdx_dev); + return err; + } + + nvec = max_irqs; + + xa_init(&gc->irq_contexts); + + /* No IRQ affinity hint is applied here. The PCI transport spreads its + * vectors over the NUMA node local to the device; the CDX platforms + * this driver targets are single-node, so there is nothing to spread + * over. Affinity management can be added when a multi-node CDX host + * exists to validate it against. + */ + for (i = 0; i < nvec; i++) { + int msi = i; + + gic = mana_gd_get_gic(gc, false, &msi); + if (IS_ERR(gic)) { + err = PTR_ERR(gic); + goto free_irq; + } + } + + gc->max_num_msix = nvec; + gc->num_msix_usable = nvec; + + /* CDX sizes its MSI pool at probe time and cannot grow it later, so + * never share vectors: vector 0 is reserved for the HWC and each of + * the remaining nvec - 1 vectors backs exactly one EQ. That bounds + * the queue count at nvec - 1 and guarantees the HWC vector is never + * handed out to a data queue. + */ + gc->msi_sharing = false; + + gc->msi_bitmap = bitmap_zalloc(nvec, GFP_KERNEL); + if (!gc->msi_bitmap) { + err = -ENOMEM; + goto free_irq; + } + + /* Reserve the HWC vector */ + set_bit(0, gc->msi_bitmap); + + return 0; + +free_irq: + for (j = i - 1; j >= 0; j--) + mana_gd_put_gic(gc, false, j); + + xa_destroy(&gc->irq_contexts); + msi_domain_free_irqs_all(gc->dev, MSI_DEFAULT_DOMAIN); + cdx_disable_msi(cdx_dev); + return err; +} + +static void mana_cdx_remove_irqs(struct gdma_context *gc) +{ + struct cdx_device *cdx_dev = to_cdx_device(gc->dev); + int i; + + if (gc->max_num_msix < 1) + return; + + for (i = 0; i < gc->max_num_msix; i++) { + if (!xa_load(&gc->irq_contexts, i)) + continue; + + mana_gd_put_gic(gc, false, i); + } + + WARN_ON(!xa_empty(&gc->irq_contexts)); + + xa_destroy(&gc->irq_contexts); + msi_domain_free_irqs_all(gc->dev, MSI_DEFAULT_DOMAIN); + cdx_disable_msi(cdx_dev); + + bitmap_free(gc->msi_bitmap); + gc->msi_bitmap = NULL; + gc->max_num_msix = 0; + gc->num_msix_usable = 0; +} + +static int mana_cdx_setup_remaining_irqs(struct gdma_context *gc) +{ + unsigned int max_queues_vport; + + /* mana_gd_query_max_resources() may turn MSI sharing back on and round + * the per-vPort queue count up to MANA_DEF_NUM_QUEUES. CDX sizes its + * vector pool at probe time and cannot grow it later, and the HWC + * vector must stay private, so give every vPort a private slice of the + * remaining vectors instead. gc->max_num_queues is already capped at + * num_msix_usable - 1 by the core. + */ + gc->msi_sharing = false; + max_queues_vport = (gc->num_msix_usable - 1) / gc->num_ports; + if (!max_queues_vport) { + dev_err(gc->dev, "%u MSI vectors cannot serve %u vPorts\n", + gc->num_msix_usable, gc->num_ports); + return -ENOSPC; + } + + gc->max_num_queues_vport = min(gc->max_num_queues, max_queues_vport); + + dev_dbg(gc->dev, "%u MSI vectors, %u vPorts, %u queues, no MSI sharing\n", + gc->num_msix_usable, gc->num_ports, gc->max_num_queues_vport); + + return 0; +} + +static const struct gdma_bus_ops mana_cdx_bus_ops = { + .bus_name = "cdx", + .adev_prefix = "mana_cdx", + .msix_virq = mana_cdx_msix_virq, + .setup_hwc_irqs = mana_cdx_setup_hwc_irqs, + .setup_remaining_irqs = mana_cdx_setup_remaining_irqs, + .remove_irqs = mana_cdx_remove_irqs, + .dev_reset = mana_cdx_dev_reset, +}; + +static int mana_cdx_gd_probe(struct cdx_device *cdx_dev) +{ + struct gdma_context *gc; + void __iomem *bar0_va; + int err; + + /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ + BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); + + err = cdx_dev_reset(&cdx_dev->dev); + if (err) + return err; + + err = cdx_set_master(cdx_dev); + if (err) + return err; + + gc = vzalloc(sizeof(*gc)); + if (!gc) { + err = -ENOMEM; + goto clear_master; + } + + mutex_init(&gc->eq_test_event_mutex); + mutex_init(&gc->gic_mutex); + dev_set_drvdata(&cdx_dev->dev, gc); + gc->bar0_pa = cdx_resource_start(cdx_dev, 0); + gc->bar0_size = cdx_resource_len(cdx_dev, 0); + if (gc->bar0_size < MANA_CDX_GDMA_REGS_SIZE) { + dev_err(&cdx_dev->dev, "BAR0 size %#llx is too small\n", + (u64)gc->bar0_size); + err = -EINVAL; + goto free_gc; + } + + bar0_va = ioremap(gc->bar0_pa, gc->bar0_size); + if (!bar0_va) { + err = -ENOMEM; + goto free_gc; + } + + gc->numa_node = dev_to_node(&cdx_dev->dev); + gc->bar0_va = bar0_va; + gc->dev = &cdx_dev->dev; + gc->bus_ops = &mana_cdx_bus_ops; + + /* gc->is_pf is intentionally left clear: this device uses the VF + * register layout, and the vPort and MAC filter registration that + * is_pf selects is not required on this bus. + */ + + err = mana_gd_setup(gc); + if (err) + goto unmap_bar; + + err = mana_probe(&gc->mana, false); + if (err) + goto cleanup_gd; + + err = mana_rdma_probe(&gc->mana_ib); + if (err) + goto remove_mana; + + /* If a hardware reset event arrived over HWC during probe, roll back: + * the device state the rest of probe observed is no longer valid. + */ + if (test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { + err = -EPROTO; + goto remove_rdma; + } + + return 0; + +remove_rdma: + mana_rdma_remove(&gc->mana_ib); +remove_mana: + mana_remove(&gc->mana, false); +cleanup_gd: + mana_gd_cleanup(gc); +unmap_bar: + iounmap(bar0_va); +free_gc: + dev_set_drvdata(&cdx_dev->dev, NULL); + vfree(gc); +clear_master: + cdx_clear_master(cdx_dev); + return err; +} + +static int mana_cdx_gd_remove(struct cdx_device *cdx_dev) +{ + struct gdma_context *gc = dev_get_drvdata(&cdx_dev->dev); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, false); + + mana_gd_cleanup(gc); + cdx_clear_master(cdx_dev); + + iounmap(gc->bar0_va); + + dev_set_drvdata(&cdx_dev->dev, NULL); + vfree(gc); + return 0; +} + +/* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ +static void mana_cdx_gd_shutdown(struct cdx_device *cdx_dev) +{ + struct gdma_context *gc = dev_get_drvdata(&cdx_dev->dev); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, true); + + mana_gd_cleanup(gc); + cdx_clear_master(cdx_dev); +} + +#define CDX_VENDOR_ID_MICROSOFT 0x1414 +static const struct cdx_device_id mana_cdx_table[] = { + { CDX_DEVICE(CDX_VENDOR_ID_MICROSOFT, MANA_CDX_DEVICE_ID) }, + {} +}; + +MODULE_DEVICE_TABLE(cdx, mana_cdx_table); + +static struct cdx_driver gdma_cdx_driver = { + .match_id_table = mana_cdx_table, + .probe = mana_cdx_gd_probe, + .remove = mana_cdx_gd_remove, + .shutdown = mana_cdx_gd_shutdown, + .driver = { + .name = "mana_cdx", + }, +}; + +module_driver(gdma_cdx_driver, cdx_driver_register, cdx_driver_unregister); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver for the CDX bus"); +MODULE_IMPORT_NS("CDX_BUS"); +MODULE_IMPORT_NS("NET_MANA"); diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 740005782fa3..8c915e008784 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -4,7 +4,6 @@ #include <linux/bitfield.h> #include <linux/debugfs.h> #include <linux/module.h> -#include <linux/pci.h> #include <linux/sizes.h> #include <linux/utsname.h> #include <linux/version.h> @@ -18,33 +17,20 @@ struct dentry *mana_debugfs_root; -struct mana_dev_recovery { - struct list_head list; - struct pci_dev *pdev; - enum gdma_eqe_type type; -}; - -static struct mana_dev_recovery_work { - struct list_head dev_list; - struct delayed_work work; - - /* Lock for dev_list above */ - spinlock_t lock; -} mana_dev_recovery_work; - -static u32 mana_gd_r32(struct gdma_context *g, u64 offset) +/* + * True if the underlying bus can allocate MSI-X vectors after probe time. + * Buses that size their vector pool at probe install no callback. + */ +static bool mana_gd_msix_can_alloc_dyn(struct gdma_context *gc) { - return readl(g->bar0_va + offset); -} + if (!gc->bus_ops || !gc->bus_ops->msix_can_alloc_dyn) + return false; -static u64 mana_gd_r64(struct gdma_context *g, u64 offset) -{ - return readq(g->bar0_va + offset); + return gc->bus_ops->msix_can_alloc_dyn(gc); } -static int mana_gd_init_pf_regs(struct pci_dev *pdev) +static int mana_gd_init_pf_regs(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); u64 remaining_barsize; u64 sriov_base_off; u64 sriov_shm_off; @@ -102,9 +88,8 @@ static int mana_gd_init_pf_regs(struct pci_dev *pdev) return 0; } -static int mana_gd_init_vf_regs(struct pci_dev *pdev) +static int mana_gd_init_vf_regs(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); u64 shm_off; gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; @@ -148,14 +133,12 @@ static int mana_gd_init_vf_regs(struct pci_dev *pdev) return 0; } -static int mana_gd_init_registers(struct pci_dev *pdev) +int mana_gd_init_registers(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); - if (gc->is_pf && !gc->is_pf2) - return mana_gd_init_pf_regs(pdev); + return mana_gd_init_pf_regs(gc); else - return mana_gd_init_vf_regs(pdev); + return mana_gd_init_vf_regs(gc); } /* Suppress logging when we set timeout to zero */ @@ -176,9 +159,8 @@ bool mana_need_log(struct gdma_context *gc, int err) return true; } -static int mana_gd_query_max_resources(struct pci_dev *pdev) +int mana_gd_query_max_resources(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); struct gdma_query_max_resources_resp resp = {}; struct gdma_general_req req = {}; unsigned int max_num_queues; @@ -194,7 +176,7 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) * MSI-X allocation; on non-dyn platforms msi_sharing is * unconditionally true (set in mana_gd_setup_hwc_irqs). */ - if (pci_msix_can_alloc_dyn(to_pci_dev(gc->dev))) + if (mana_gd_msix_can_alloc_dyn(gc)) gc->msi_sharing = false; mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, @@ -207,7 +189,11 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) return err ? err : -EPROTO; } - if (!pci_msix_can_alloc_dyn(pdev)) { + if (!mana_gd_msix_can_alloc_dyn(gc)) { + /* Buses that size their vector pool at probe time cannot grow + * it afterwards, so never raise num_msix_usable above what has + * already been allocated. + */ if (gc->num_msix_usable > resp.max_msix) gc->num_msix_usable = resp.max_msix; } else { @@ -221,20 +207,24 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) /* MSI-X vectors are allocated by index into the device MSI-X table, so * never ask for more than the table holds. It can be smaller than both - * resp.max_msix and the CPU count. + * resp.max_msix and the CPU count. A bus that cannot report a table + * size installs no callback and skips the clamp. */ - err = pci_msix_vec_count(pdev); - if (err <= 0) { - dev_err(gc->dev, "Failed to query MSI-X table size: %d\n", err); - return err < 0 ? err : -ENOSPC; - } - msix_vec_count = err; + if (gc->bus_ops && gc->bus_ops->msix_vec_count) { + err = gc->bus_ops->msix_vec_count(gc); + if (err <= 0) { + dev_err(gc->dev, + "Failed to query MSI-X table size: %d\n", err); + return err < 0 ? err : -ENOSPC; + } + msix_vec_count = err; - if (gc->num_msix_usable > msix_vec_count) { - dev_info(gc->dev, - "Limiting MSI-X vectors from %u to table size %u\n", - gc->num_msix_usable, msix_vec_count); - gc->num_msix_usable = msix_vec_count; + if (gc->num_msix_usable > msix_vec_count) { + dev_info(gc->dev, + "Limiting MSI-X vectors from %u to table size %u\n", + gc->num_msix_usable, msix_vec_count); + gc->num_msix_usable = msix_vec_count; + } } if (gc->num_msix_usable <= 1) @@ -286,6 +276,8 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) if (num_ports > MAX_PORTS_IN_MANA_DEV) num_ports = MAX_PORTS_IN_MANA_DEV; + gc->num_ports = num_ports; + /* * Adjust the per-vPort max queue count to allow dedicated * MSIx for each vPort. Prefer at least MANA_DEF_NUM_QUEUES, @@ -317,9 +309,8 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) return 0; } -static int mana_gd_query_hwc_timeout(struct pci_dev *pdev, u32 *timeout_val) +static int mana_gd_query_hwc_timeout(struct gdma_context *gc, u32 *timeout_val) { - struct gdma_context *gc = pci_get_drvdata(pdev); struct gdma_query_hwc_timeout_resp resp = {}; struct gdma_query_hwc_timeout_req req = {}; int err; @@ -336,9 +327,8 @@ static int mana_gd_query_hwc_timeout(struct pci_dev *pdev, u32 *timeout_val) return 0; } -static int mana_gd_detect_devices(struct pci_dev *pdev) +int mana_gd_detect_devices(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); struct gdma_list_devices_resp resp = {}; struct gdma_general_req req = {}; struct gdma_dev_id dev; @@ -667,187 +657,31 @@ void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, } EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA"); -#define MANA_SERVICE_PERIOD 10 - -static void mana_serv_rescan(struct pci_dev *pdev) -{ - struct pci_bus *parent; - - pci_lock_rescan_remove(); - - parent = pdev->bus; - if (!parent) { - dev_err(&pdev->dev, "MANA service: no parent bus\n"); - goto out; - } - - pci_stop_and_remove_bus_device(pdev); - pci_rescan_bus(parent); - -out: - pci_unlock_rescan_remove(); -} - -static void mana_serv_fpga(struct pci_dev *pdev) -{ - struct pci_bus *bus, *parent; - - pci_lock_rescan_remove(); - - bus = pdev->bus; - if (!bus) { - dev_err(&pdev->dev, "MANA service: no bus\n"); - goto out; - } - - parent = bus->parent; - if (!parent) { - dev_err(&pdev->dev, "MANA service: no parent bus\n"); - goto out; - } - - pci_stop_and_remove_bus_device(bus->self); - - msleep(MANA_SERVICE_PERIOD * 1000); - - pci_rescan_bus(parent); - -out: - pci_unlock_rescan_remove(); -} - -static void mana_serv_reset(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - struct hw_channel_context *hwc; - int ret; - - if (!gc) { - /* Perform PCI rescan on device if GC is not set up */ - dev_err(&pdev->dev, "MANA service: GC not setup, rescanning\n"); - mana_serv_rescan(pdev); - return; - } - - hwc = gc->hwc.driver_data; - if (!hwc) { - dev_err(&pdev->dev, "MANA service: no HWC\n"); - goto out; - } - - /* HWC is not responding in this case, so don't wait */ - hwc->hwc_timeout = 0; - - dev_info(&pdev->dev, "MANA reset cycle start\n"); - - mana_gd_suspend(pdev, PMSG_SUSPEND); - - msleep(MANA_SERVICE_PERIOD * 1000); - - ret = mana_gd_resume(pdev); - if (ret == -ETIMEDOUT || ret == -EPROTO) { - /* Perform PCI rescan on device if we failed on HWC */ - dev_err(&pdev->dev, "MANA service: resume failed, rescanning\n"); - mana_serv_rescan(pdev); - return; - } - - if (ret) - dev_info(&pdev->dev, "MANA reset cycle failed err %d\n", ret); - else - dev_info(&pdev->dev, "MANA reset cycle completed\n"); - -out: - clear_bit(GC_IN_SERVICE, &gc->flags); -} - -static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev) -{ - switch (type) { - case GDMA_EQE_HWC_FPGA_RECONFIG: - mana_serv_fpga(pdev); - break; - - case GDMA_EQE_HWC_RESET_REQUEST: - mana_serv_reset(pdev); - break; - - default: - dev_err(&pdev->dev, "MANA service: unknown type %d\n", type); - break; - } -} - -static void mana_recovery_delayed_func(struct work_struct *w) -{ - struct mana_dev_recovery_work *work; - struct mana_dev_recovery *dev; - unsigned long flags; - - work = container_of(w, struct mana_dev_recovery_work, work.work); - - spin_lock_irqsave(&work->lock, flags); - - while (!list_empty(&work->dev_list)) { - dev = list_first_entry(&work->dev_list, - struct mana_dev_recovery, list); - list_del(&dev->list); - spin_unlock_irqrestore(&work->lock, flags); - - mana_do_service(dev->type, dev->pdev); - pci_dev_put(dev->pdev); - kfree(dev); - - spin_lock_irqsave(&work->lock, flags); - } - - spin_unlock_irqrestore(&work->lock, flags); -} - -static void mana_serv_func(struct work_struct *w) +/* + * Queue device servicing or recovery on buses that support it. + * + * Servicing tears the device down and brings it back up, so it depends on + * bus-level facilities the GDMA core does not have. Buses that provide no + * servicing path install no callback and the request is rejected. + */ +int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type) { - struct mana_serv_work *mns_wk; - struct pci_dev *pdev; - - mns_wk = container_of(w, struct mana_serv_work, serv_work); - pdev = mns_wk->pdev; - - if (pdev) - mana_do_service(mns_wk->type, pdev); + if (!gc->bus_ops || !gc->bus_ops->schedule_serv_work) + return -EOPNOTSUPP; - pci_dev_put(pdev); - kfree(mns_wk); - module_put(THIS_MODULE); + return gc->bus_ops->schedule_serv_work(gc, type); } -int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type) +/* The servicing workqueue is owned by the GDMA core because the queueing + * sites live here and in mana_en.c, which every transport shares. Each bus + * driver creates it during setup and destroys it during cleanup. + */ +int mana_gd_alloc_service_wq(struct gdma_context *gc) { - struct mana_serv_work *mns_wk; - - if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) { - dev_info(gc->dev, "Already in service\n"); - return -EBUSY; - } - - if (!try_module_get(THIS_MODULE)) { - dev_info(gc->dev, "Module is unloading\n"); - clear_bit(GC_IN_SERVICE, &gc->flags); - return -ENODEV; - } - - mns_wk = kzalloc_obj(*mns_wk, GFP_ATOMIC); - if (!mns_wk) { - module_put(THIS_MODULE); - clear_bit(GC_IN_SERVICE, &gc->flags); + gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0); + if (!gc->service_wq) return -ENOMEM; - } - dev_info(gc->dev, "Start MANA service type:%d\n", type); - mns_wk->pdev = to_pci_dev(gc->dev); - mns_wk->type = type; - pci_dev_get(mns_wk->pdev); - INIT_WORK(&mns_wk->serv_work, mana_serv_func); - schedule_work(&mns_wk->serv_work); return 0; } @@ -914,6 +748,15 @@ ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf, return copied; } +void mana_gd_free_service_wq(struct gdma_context *gc) +{ + if (!gc->service_wq) + return; + + destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; +} + static void mana_gd_process_eqe(struct gdma_queue *eq) { u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); @@ -985,7 +828,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq) } } -static void mana_gd_process_eq_events(void *arg) +void mana_gd_process_eq_events(void *arg) { u32 owner_bits, new_bits, old_bits; union gdma_eqe_info eqe_info; @@ -1540,9 +1383,8 @@ void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) } EXPORT_SYMBOL_NS(mana_gd_destroy_queue, "NET_MANA"); -int mana_gd_verify_vf_version(struct pci_dev *pdev) +int mana_gd_verify_vf_version(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); struct gdma_verify_ver_resp resp = {}; struct gdma_verify_ver_req req = {}; struct hw_channel_context *hwc; @@ -1556,6 +1398,8 @@ int mana_gd_verify_vf_version(struct pci_dev *pdev) req.protocol_ver_max = GDMA_PROTOCOL_LAST; req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1; + if (gc->bus_ops) + req.gd_drv_cap_flags1 |= gc->bus_ops->drv_cap_flags1; req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2; req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3; req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4; @@ -1584,7 +1428,7 @@ int mana_gd_verify_vf_version(struct pci_dev *pdev) &gc->pf_cap_flags1); if (resp.pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG) { - err = mana_gd_query_hwc_timeout(pdev, &hwc->hwc_timeout); + err = mana_gd_query_hwc_timeout(gc, &hwc->hwc_timeout); if (err) { dev_err(gc->dev, "Failed to set the hwc timeout %d\n", err); return err; @@ -1906,7 +1750,7 @@ int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) } EXPORT_SYMBOL_NS(mana_gd_poll_cq, "NET_MANA"); -static irqreturn_t mana_gd_intr(int irq, void *arg) +irqreturn_t mana_gd_intr(int irq, void *arg) { struct gdma_irq_context *gic = arg; struct list_head *eq_list = &gic->eq_list; @@ -1921,11 +1765,25 @@ static irqreturn_t mana_gd_intr(int irq, void *arg) return IRQ_HANDLED; } +/* + * Reset the device using whatever mechanism the bus provides. + */ +int mana_gd_dev_reset(struct gdma_context *gc) +{ + if (!gc->bus_ops || !gc->bus_ops->dev_reset) + return -EOPNOTSUPP; + + return gc->bus_ops->dev_reset(gc); +} + +/* + * Release a reference on the IRQ context backing an MSI vector, freeing + * the vector once the last user is gone. + */ void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi) { - struct pci_dev *dev = to_pci_dev(gc->dev); + const struct gdma_bus_ops *ops = gc->bus_ops; struct gdma_irq_context *gic; - struct msi_map irq_map; int irq; mutex_lock(&gc->gic_mutex); @@ -1950,11 +1808,8 @@ void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi) irq_update_affinity_hint(irq, NULL); free_irq(irq, gic); - if (gic->dyn_msix) { - irq_map.virq = irq; - irq_map.index = msi; - pci_msix_free_irq(dev, irq_map); - } + if (gic->dyn_msix) + ops->msix_free(gc, msi, irq); xa_erase(&gc->irq_contexts, msi); kfree(gic); @@ -1977,12 +1832,10 @@ struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, bool use_msi_bitmap, int *msi_requested) { - struct pci_dev *dev = to_pci_dev(gc->dev); + const struct gdma_bus_ops *ops = gc->bus_ops; struct gdma_irq_context *gic; - struct msi_map irq_map = { }; - int irq; - int msi; - int err; + bool dyn_msix = false; + int msi, irq, err; mutex_lock(&gc->gic_mutex); @@ -2008,27 +1861,34 @@ struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, goto out; } - irq = pci_irq_vector(dev, msi); + irq = ops->msix_virq(gc, msi); if (irq == -EINVAL) { - irq_map = pci_msix_alloc_irq_at(dev, msi, NULL); - if (!irq_map.virq) { - err = irq_map.index; - dev_err(gc->dev, - "Failed to alloc irq_map msi %d err %d\n", - msi, err); - gic = ERR_PTR(err); + /* A bus that sizes its vector pool up front has nothing left + * to hand out once every vector has been claimed. + */ + if (!ops->msix_alloc_at) { + dev_err(gc->dev, "No IRQ for MSI %d\n", msi); + gic = ERR_PTR(-ENOENT); + goto out; + } + + irq = ops->msix_alloc_at(gc, &msi); + if (irq < 0) { + dev_err(gc->dev, "Failed to alloc irq msi %d err %d\n", + *msi_requested, irq); + gic = ERR_PTR(irq); goto out; } - irq = irq_map.virq; - msi = irq_map.index; + + dyn_msix = true; *msi_requested = msi; } gic = kzalloc_obj(*gic); if (!gic) { gic = ERR_PTR(-ENOMEM); - if (irq_map.virq) - pci_msix_free_irq(dev, irq_map); + if (dyn_msix) + ops->msix_free(gc, msi, irq); goto out; } @@ -2039,11 +1899,11 @@ struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, spin_lock_init(&gic->lock); if (!gic->msi) - snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s", - pci_name(dev)); + snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@%s:%s", + ops->bus_name, dev_name(gc->dev)); else - snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_msi%d@pci:%s", - gic->msi, pci_name(dev)); + snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_msi%d@%s:%s", + gic->msi, ops->bus_name, dev_name(gc->dev)); err = request_irq(irq, mana_gd_intr, 0, gic->name, gic); if (err) { @@ -2051,12 +1911,12 @@ struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, irq, gic->name); kfree(gic); gic = ERR_PTR(err); - if (irq_map.virq) - pci_msix_free_irq(dev, irq_map); + if (dyn_msix) + ops->msix_free(gc, msi, irq); goto out; } - gic->dyn_msix = !!irq_map.virq; + gic->dyn_msix = dyn_msix; refcount_set(&gic->refcount, 1); gic->bitmap_refs = use_msi_bitmap ? 1 : 0; @@ -2067,8 +1927,8 @@ struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, free_irq(irq, gic); kfree(gic); gic = ERR_PTR(err); - if (irq_map.virq) - pci_msix_free_irq(dev, irq_map); + if (dyn_msix) + ops->msix_free(gc, msi, irq); goto out; } @@ -2100,356 +1960,29 @@ void mana_gd_free_res_map(struct gdma_resource *r) r->size = 0; } -/* - * Spread on CPUs with the following heuristics: - * - * 1. No more than one IRQ per CPU, if possible; - * 2. NUMA locality is the second priority; - * 3. Sibling dislocality is the last priority. - * - * Let's consider this topology: - * - * Node 0 1 - * Core 0 1 2 3 - * CPU 0 1 2 3 4 5 6 7 - * - * The most performant IRQ distribution based on the above topology - * and heuristics may look like this: - * - * IRQ Nodes Cores CPUs - * 0 1 0 0-1 - * 1 1 1 2-3 - * 2 1 0 0-1 - * 3 1 1 2-3 - * 4 2 2 4-5 - * 5 2 3 6-7 - * 6 2 2 4-5 - * 7 2 3 6-7 - * - * The heuristics is implemented as follows. - * - * The outer for_each() loop resets the 'weight' to the actual number - * of CPUs in the hop. Then inner for_each() loop decrements it by the - * number of sibling groups (cores) while assigning first set of IRQs - * to each group. IRQs 0 and 1 above are distributed this way. - * - * Now, because NUMA locality is more important, we should walk the - * same set of siblings and assign 2nd set of IRQs (2 and 3), and it's - * implemented by the medium while() loop. We do like this unless the - * number of IRQs assigned on this hop will not become equal to number - * of CPUs in the hop (weight == 0). Then we switch to the next hop and - * do the same thing. +/* Bring up the GDMA context on a probed device: create the debugfs directory, + * map the shared-memory registers, start the hardware channel and size the + * interrupt pool. Every step here is common to all buses; the ones that are + * not are reached through gdma_bus_ops. */ - -static int mana_irq_setup_numa_aware(unsigned int *irqs, unsigned int len, - int node, bool skip_first_cpu) -{ - const struct cpumask *next, *prev = cpu_none_mask; - cpumask_var_t cpus __free(free_cpumask_var); - int cpu, weight; - - if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) - return -ENOMEM; - - rcu_read_lock(); - for_each_numa_hop_mask(next, node) { - weight = cpumask_weight_andnot(next, prev); - while (weight > 0) { - cpumask_andnot(cpus, next, prev); - for_each_cpu(cpu, cpus) { - cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); - --weight; - - if (unlikely(skip_first_cpu)) { - skip_first_cpu = false; - continue; - } - - if (len-- == 0) - goto done; - - irq_set_affinity_and_hint(*irqs++, topology_sibling_cpumask(cpu)); - } - } - prev = next; - } -done: - rcu_read_unlock(); - return 0; -} - -/* must be called with cpus_read_lock() held */ -static void mana_irq_setup_linear(unsigned int *irqs, unsigned int len) -{ - int cpu; - - for_each_online_cpu(cpu) { - if (len == 0) - break; - - irq_set_affinity_and_hint(*irqs++, cpumask_of(cpu)); - len--; - } -} - -static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - struct gdma_irq_context *gic; - int *irqs, err, i, msi; - - irqs = kmalloc_objs(int, nvec); - if (!irqs) - return -ENOMEM; - - /* - * In this function, num_msix_usable = HWC IRQ + Queue IRQ. - * nvec is only Queue IRQ (HWC already setup). - * While processing the next pci irq vector, we start with index 1, - * as IRQ vector at index 0 is already processed for HWC. - * However, the population of irqs array starts with index 0, to be - * further used in mana_irq_setup_numa_aware() - */ - for (i = 1; i <= nvec; i++) { - msi = i; - gic = mana_gd_get_gic(gc, false, &msi); - if (IS_ERR(gic)) { - err = PTR_ERR(gic); - goto free_irq; - } - - irqs[i - 1] = gic->irq; - } - - /* - * When calling mana_irq_setup_numa_aware() for dynamically added IRQs, - * if number of CPUs is more than or equal to allocated MSI-X, we need to - * skip the first CPU sibling group since they are already affinitized to - * HWC IRQ - */ - cpus_read_lock(); - if (gc->num_msix_usable <= num_online_cpus()) { - err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, - true); - if (err) { - cpus_read_unlock(); - goto free_irq; - } - } else { - /* - * When num_msix_usable are more than num_online_cpus, our - * queue IRQs should be equal to num of online vCPUs. - * We try to make sure queue IRQs spread across all vCPUs. - * In such a case NUMA or CPU core affinity does not matter. - * Note: in this case the total mana IRQ should always be - * num_online_cpus + 1. The first HWC IRQ is already handled - * in HWC setup calls - * However, if CPUs went offline since num_msix_usable was - * computed, queue IRQs will be more than num_online_cpus(). - * In such cases remaining extra IRQs will retain their default - * affinity. - */ - int first_unassigned = num_online_cpus(); - - if (nvec > first_unassigned) { - char buf[32]; - - if (first_unassigned == nvec - 1) - snprintf(buf, sizeof(buf), "%d", - first_unassigned); - else - snprintf(buf, sizeof(buf), "%d-%d", - first_unassigned, nvec - 1); - - dev_dbg(&pdev->dev, - "MANA IRQ indices #%s will retain the default CPU affinity\n", - buf); - } - - mana_irq_setup_linear(irqs, nvec); - } - - cpus_read_unlock(); - kfree(irqs); - return 0; - -free_irq: - for (i -= 1; i > 0; i--) - mana_gd_put_gic(gc, false, i); - kfree(irqs); - return err; -} - -static int mana_gd_setup_irqs(struct pci_dev *pdev, int nvec) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - struct gdma_irq_context *gic; - int *irqs, *start_irqs; - unsigned int cpu; - int err, i, msi; - - irqs = kmalloc_objs(int, nvec); - if (!irqs) - return -ENOMEM; - - start_irqs = irqs; - - for (i = 0; i < nvec; i++) { - msi = i; - gic = mana_gd_get_gic(gc, false, &msi); - if (IS_ERR(gic)) { - err = PTR_ERR(gic); - goto free_irq; - } - - irqs[i] = gic->irq; - } - - /* If number of IRQ is one extra than number of online CPUs, - * then we need to assign IRQ0 (hwc irq) and IRQ1 to - * same CPU. - * Else we will use different CPUs for IRQ0 and IRQ1. - * Also we are using cpumask_local_spread instead of - * cpumask_first for the node, because the node can be - * mem only. - */ - cpus_read_lock(); - if (nvec > num_online_cpus()) { - cpu = cpumask_local_spread(0, gc->numa_node); - irq_set_affinity_and_hint(irqs[0], cpumask_of(cpu)); - irqs++; - nvec -= 1; - } - - err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, false); - if (err) { - cpus_read_unlock(); - goto free_irq; - } - - cpus_read_unlock(); - kfree(start_irqs); - return 0; - -free_irq: - for (i -= 1; i >= 0; i--) - mana_gd_put_gic(gc, false, i); - - kfree(start_irqs); - return err; -} - -static int mana_gd_setup_hwc_irqs(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - unsigned int max_irqs, min_irqs; - int nvec, err; - - if (pci_msix_can_alloc_dyn(pdev)) { - max_irqs = 1; - min_irqs = 1; - } else { - /* Need 1 interrupt for HWC */ - max_irqs = min(num_online_cpus(), MANA_MAX_NUM_QUEUES) + 1; - min_irqs = 2; - gc->msi_sharing = true; - } - - nvec = pci_alloc_irq_vectors(pdev, min_irqs, max_irqs, PCI_IRQ_MSIX); - if (nvec < 0) - return nvec; - - err = mana_gd_setup_irqs(pdev, nvec); - if (err) { - pci_free_irq_vectors(pdev); - return err; - } - - gc->num_msix_usable = nvec; - gc->max_num_msix = nvec; - - return 0; -} - -static int mana_gd_setup_remaining_irqs(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - struct msi_map irq_map; - int max_irqs, i, err; - - if (!pci_msix_can_alloc_dyn(pdev)) - /* remain irqs are already allocated with HWC IRQ */ - return 0; - - /* allocate only remaining IRQs*/ - max_irqs = gc->num_msix_usable - 1; - - for (i = 1; i <= max_irqs; i++) { - irq_map = pci_msix_alloc_irq_at(pdev, i, NULL); - if (!irq_map.virq) { - err = irq_map.index; - /* caller will handle cleaning up all allocated - * irqs, after HWC is destroyed - */ - return err; - } - } - - err = mana_gd_setup_dyn_irqs(pdev, max_irqs); - if (err) - return err; - - gc->max_num_msix = gc->max_num_msix + max_irqs; - - return 0; -} - -static void mana_gd_remove_irqs(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - int i; - - if (gc->max_num_msix < 1) - return; - - for (i = 0; i < gc->max_num_msix; i++) { - if (!xa_load(&gc->irq_contexts, i)) - continue; - - mana_gd_put_gic(gc, false, i); - } - - WARN_ON(!xa_empty(&gc->irq_contexts)); - - pci_free_irq_vectors(pdev); - - bitmap_free(gc->msi_bitmap); - gc->msi_bitmap = NULL; - gc->max_num_msix = 0; - gc->num_msix_usable = 0; -} - -static int mana_gd_setup(struct pci_dev *pdev) +int mana_gd_setup(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); int err; - gc->mana_pci_debugfs = debugfs_create_dir(pci_name(pdev), + gc->mana_pci_debugfs = debugfs_create_dir(dev_name(gc->dev), mana_debugfs_root); - err = mana_gd_init_registers(pdev); + err = mana_gd_init_registers(gc); if (err) goto remove_debugfs; mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); - gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0); - if (!gc->service_wq) { - err = -ENOMEM; + err = mana_gd_alloc_service_wq(gc); + if (err) goto remove_debugfs; - } - err = mana_gd_setup_hwc_irqs(pdev); + err = gc->bus_ops->setup_hwc_irqs(gc); if (err) { dev_err(gc->dev, "Failed to setup IRQs for HWC creation: %d\n", err); @@ -2460,360 +1993,70 @@ static int mana_gd_setup(struct pci_dev *pdev) if (err) goto remove_irq; - err = mana_gd_verify_vf_version(pdev); + err = mana_gd_verify_vf_version(gc); if (err) goto destroy_hwc; - err = mana_gd_detect_devices(pdev); + err = mana_gd_detect_devices(gc); if (err) goto destroy_hwc; - err = mana_gd_query_max_resources(pdev); + err = mana_gd_query_max_resources(gc); if (err) goto destroy_hwc; - err = mana_gd_setup_remaining_irqs(pdev); - if (err) { - dev_err(gc->dev, "Failed to setup remaining IRQs: %d", err); + err = gc->bus_ops->setup_remaining_irqs(gc); + if (err) goto destroy_hwc; - } - - if (!gc->msi_sharing) { - gc->msi_bitmap = bitmap_zalloc(gc->num_msix_usable, GFP_KERNEL); - if (!gc->msi_bitmap) { - err = -ENOMEM; - goto destroy_hwc; - } - /* Set bit for HWC */ - set_bit(0, gc->msi_bitmap); - } - dev_dbg(&pdev->dev, "mana gdma setup successful\n"); + dev_dbg(gc->dev, "mana gdma setup successful\n"); return 0; destroy_hwc: mana_hwc_destroy_channel(gc); remove_irq: - mana_gd_remove_irqs(pdev); + gc->bus_ops->remove_irqs(gc); free_workqueue: - destroy_workqueue(gc->service_wq); - gc->service_wq = NULL; + mana_gd_free_service_wq(gc); remove_debugfs: debugfs_remove_recursive(gc->mana_pci_debugfs); gc->mana_pci_debugfs = NULL; - dev_err(&pdev->dev, "%s failed (error %d)\n", __func__, err); + dev_err(gc->dev, "%s failed (error %d)\n", __func__, err); return err; } +EXPORT_SYMBOL_NS(mana_gd_setup, "NET_MANA"); -static void mana_gd_cleanup_device(struct pci_dev *pdev) +void mana_gd_cleanup(struct gdma_context *gc) { - struct gdma_context *gc = pci_get_drvdata(pdev); - mana_hwc_destroy_channel(gc); - mana_gd_remove_irqs(pdev); + gc->bus_ops->remove_irqs(gc); - if (gc->service_wq) { - destroy_workqueue(gc->service_wq); - gc->service_wq = NULL; - } + mana_gd_free_service_wq(gc); debugfs_remove_recursive(gc->mana_pci_debugfs); gc->mana_pci_debugfs = NULL; - dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); -} - -static bool mana_is_pf(unsigned short dev_id) -{ - return dev_id == MANA_PF_DEVICE_ID || dev_id == MANA_PF2_DEVICE_ID; + dev_dbg(gc->dev, "mana gdma cleanup successful\n"); } +EXPORT_SYMBOL_NS(mana_gd_cleanup, "NET_MANA"); -static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +static int __init gdma_core_init(void) { - struct gdma_context *gc; - void __iomem *bar0_va; - int bar = 0; - int err; - - /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ - BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); - - err = pci_enable_device(pdev); - if (err) { - dev_err(&pdev->dev, "Failed to enable pci device (err=%d)\n", err); - return -ENXIO; - } - - pci_set_master(pdev); - - err = pci_request_regions(pdev, "mana"); - if (err) - goto disable_dev; - - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (err) { - dev_err(&pdev->dev, "DMA set mask failed: %d\n", err); - goto release_region; - } - dma_set_max_seg_size(&pdev->dev, UINT_MAX); - - err = -ENOMEM; - gc = vzalloc(sizeof(*gc)); - if (!gc) - goto release_region; - - mutex_init(&gc->eq_test_event_mutex); - mutex_init(&gc->gic_mutex); - pci_set_drvdata(pdev, gc); - gc->bar0_pa = pci_resource_start(pdev, 0); - gc->bar0_size = pci_resource_len(pdev, 0); - - bar0_va = pci_iomap(pdev, bar, 0); - if (!bar0_va) - goto free_gc; - - gc->numa_node = dev_to_node(&pdev->dev); - gc->is_pf = mana_is_pf(pdev->device); - gc->is_pf2 = (pdev->device == MANA_PF2_DEVICE_ID); - - gc->bar0_va = bar0_va; - gc->dev = &pdev->dev; - xa_init(&gc->irq_contexts); - - err = mana_gd_setup(pdev); - if (err) - goto unmap_bar; - - err = mana_probe(&gc->mana, false); - if (err) - goto cleanup_gd; - - err = mana_rdma_probe(&gc->mana_ib); - if (err) - goto cleanup_mana; - - /* - * If a hardware reset event has occurred over HWC during probe, - * rollback and perform hardware reset procedure. - */ - if (test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { - err = -EPROTO; - goto cleanup_mana_rdma; - } - - return 0; - -cleanup_mana_rdma: - mana_rdma_remove(&gc->mana_ib); -cleanup_mana: - mana_remove(&gc->mana, false); -cleanup_gd: - mana_gd_cleanup_device(pdev); -unmap_bar: - xa_destroy(&gc->irq_contexts); - pci_iounmap(pdev, bar0_va); -free_gc: - pci_set_drvdata(pdev, NULL); - vfree(gc); -release_region: - pci_release_regions(pdev); -disable_dev: - pci_disable_device(pdev); - dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err); - - /* - * Hardware could be in recovery mode and the HWC returns TIMEDOUT or - * EPROTO from mana_gd_setup(), mana_probe() or mana_rdma_probe(), or - * we received a hardware reset event over HWC interrupt. In this case, - * perform the device recovery procedure after MANA_SERVICE_PERIOD - * seconds. - */ - if (err == -ETIMEDOUT || err == -EPROTO) { - struct mana_dev_recovery *dev; - unsigned long flags; - - dev_info(&pdev->dev, "Start MANA recovery mode\n"); - - dev = kzalloc_obj(*dev); - if (!dev) - return err; - - dev->pdev = pci_dev_get(pdev); - dev->type = GDMA_EQE_HWC_RESET_REQUEST; - - spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); - list_add_tail(&dev->list, &mana_dev_recovery_work.dev_list); - spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); - - schedule_delayed_work(&mana_dev_recovery_work.work, - secs_to_jiffies(MANA_SERVICE_PERIOD)); - } - - return err; -} - -static void mana_gd_remove(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - - pci_disable_sriov(pdev); - - mana_rdma_remove(&gc->mana_ib); - mana_remove(&gc->mana, false); - - mana_gd_cleanup_device(pdev); - - xa_destroy(&gc->irq_contexts); - - pci_iounmap(pdev, gc->bar0_va); - - vfree(gc); - - pci_release_regions(pdev); - pci_disable_device(pdev); - - dev_dbg(&pdev->dev, "mana gdma remove successful\n"); -} - -/* The 'state' parameter is not used. */ -int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - - mana_rdma_remove(&gc->mana_ib); - mana_remove(&gc->mana, true); - - mana_gd_cleanup_device(pdev); - - return 0; -} - -int mana_gd_resume(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - int err; - - err = mana_gd_setup(pdev); - if (err) - return err; - - err = mana_probe(&gc->mana, true); - if (err) - goto cleanup_gd; - - err = mana_rdma_probe(&gc->mana_ib); - if (err) - mana_rdma_remove(&gc->mana_ib); - - return err; - -cleanup_gd: - mana_gd_cleanup_device(pdev); - return err; -} - -/* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ -static void mana_gd_shutdown(struct pci_dev *pdev) -{ - struct gdma_context *gc = pci_get_drvdata(pdev); - - dev_info(&pdev->dev, "Shutdown was called\n"); - - mana_rdma_remove(&gc->mana_ib); - mana_remove(&gc->mana, true); - - mana_gd_cleanup_device(pdev); - - pci_disable_device(pdev); -} - -static int mana_sriov_configure(struct pci_dev *pdev, int numvfs) -{ - int err = 0; - - dev_info(&pdev->dev, "Requested num VFs: %d\n", numvfs); - - if (numvfs > 0) { - err = pci_enable_sriov(pdev, numvfs); - } else { - if (pci_vfs_assigned(pdev)) { - dev_warn(&pdev->dev, - "Cannot disable SR-IOV while VFs are assigned\n"); - return -EPERM; - } - - pci_disable_sriov(pdev); - } - - return err ? err : numvfs; -} - -static const struct pci_device_id mana_id_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, - { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) }, - { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, - { } -}; - -static struct pci_driver mana_driver = { - .name = "mana", - .id_table = mana_id_table, - .probe = mana_gd_probe, - .remove = mana_gd_remove, - .suspend = mana_gd_suspend, - .resume = mana_gd_resume, - .shutdown = mana_gd_shutdown, - .sriov_configure = mana_sriov_configure, -}; - -static int __init mana_driver_init(void) -{ - int err; - - INIT_LIST_HEAD(&mana_dev_recovery_work.dev_list); - spin_lock_init(&mana_dev_recovery_work.lock); - INIT_DELAYED_WORK(&mana_dev_recovery_work.work, mana_recovery_delayed_func); - mana_debugfs_root = debugfs_create_dir("mana", NULL); - err = pci_register_driver(&mana_driver); - if (err) { - debugfs_remove(mana_debugfs_root); - mana_debugfs_root = NULL; - } - - return err; + return 0; } -static void __exit mana_driver_exit(void) +static void __exit gdma_core_exit(void) { - struct mana_dev_recovery *dev; - unsigned long flags; - - disable_delayed_work_sync(&mana_dev_recovery_work.work); - - spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); - while (!list_empty(&mana_dev_recovery_work.dev_list)) { - dev = list_first_entry(&mana_dev_recovery_work.dev_list, - struct mana_dev_recovery, list); - list_del(&dev->list); - pci_dev_put(dev->pdev); - kfree(dev); - } - spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); - - pci_unregister_driver(&mana_driver); - debugfs_remove(mana_debugfs_root); mana_debugfs_root = NULL; } -module_init(mana_driver_init); -module_exit(mana_driver_exit); - -MODULE_DEVICE_TABLE(pci, mana_id_table); +module_init(gdma_core_init); +module_exit(gdma_core_exit); MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver"); +MODULE_DESCRIPTION("Microsoft Azure Network Adapter GDMA core"); diff --git a/drivers/net/ethernet/microsoft/mana/gdma_pci.c b/drivers/net/ethernet/microsoft/mana/gdma_pci.c new file mode 100644 index 000000000000..ed8925296c8a --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/gdma_pci.c @@ -0,0 +1,917 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2025, Microsoft Corporation. */ + +#include <linux/bitmap.h> +#include <linux/cpumask.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/topology.h> +#include <linux/xarray.h> + +#include <net/mana/mana.h> +#include <net/mana/hw_channel.h> + +struct mana_dev_recovery { + struct list_head list; + struct pci_dev *pdev; + enum gdma_eqe_type type; +}; + +static struct mana_dev_recovery_work { + struct list_head dev_list; + struct delayed_work work; + + /* Lock for dev_list above */ + spinlock_t lock; +} mana_dev_recovery_work; + +static int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state); +static int mana_gd_resume(struct pci_dev *pdev); + +#define MANA_SERVICE_PERIOD 10 + +static void mana_serv_rescan(struct pci_dev *pdev) +{ + struct pci_bus *parent; + + pci_lock_rescan_remove(); + + parent = pdev->bus; + if (!parent) { + dev_err(&pdev->dev, "MANA service: no parent bus\n"); + goto out; + } + + pci_stop_and_remove_bus_device(pdev); + pci_rescan_bus(parent); + +out: + pci_unlock_rescan_remove(); +} + +static void mana_serv_fpga(struct pci_dev *pdev) +{ + struct pci_bus *bus, *parent; + + pci_lock_rescan_remove(); + + bus = pdev->bus; + if (!bus) { + dev_err(&pdev->dev, "MANA service: no bus\n"); + goto out; + } + + parent = bus->parent; + if (!parent) { + dev_err(&pdev->dev, "MANA service: no parent bus\n"); + goto out; + } + + pci_stop_and_remove_bus_device(bus->self); + + msleep(MANA_SERVICE_PERIOD * 1000); + + pci_rescan_bus(parent); + +out: + pci_unlock_rescan_remove(); +} + +static void mana_serv_reset(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct hw_channel_context *hwc; + int ret; + + if (!gc) { + /* Perform PCI rescan on device if GC is not set up */ + dev_err(&pdev->dev, "MANA service: GC not setup, rescanning\n"); + mana_serv_rescan(pdev); + return; + } + + hwc = gc->hwc.driver_data; + if (!hwc) { + dev_err(&pdev->dev, "MANA service: no HWC\n"); + goto out; + } + + /* HWC is not responding in this case, so don't wait */ + hwc->hwc_timeout = 0; + + dev_info(&pdev->dev, "MANA reset cycle start\n"); + + mana_gd_suspend(pdev, PMSG_SUSPEND); + + msleep(MANA_SERVICE_PERIOD * 1000); + + ret = mana_gd_resume(pdev); + if (ret == -ETIMEDOUT || ret == -EPROTO) { + /* Perform PCI rescan on device if we failed on HWC */ + dev_err(&pdev->dev, "MANA service: resume failed, rescanning\n"); + mana_serv_rescan(pdev); + return; + } + + if (ret) + dev_info(&pdev->dev, "MANA reset cycle failed err %d\n", ret); + else + dev_info(&pdev->dev, "MANA reset cycle completed\n"); + +out: + clear_bit(GC_IN_SERVICE, &gc->flags); +} + +static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev) +{ + switch (type) { + case GDMA_EQE_HWC_FPGA_RECONFIG: + mana_serv_fpga(pdev); + break; + + case GDMA_EQE_HWC_RESET_REQUEST: + mana_serv_reset(pdev); + break; + + default: + dev_err(&pdev->dev, "MANA service: unknown type %d\n", type); + break; + } +} + +static void mana_recovery_delayed_func(struct work_struct *w) +{ + struct mana_dev_recovery_work *work; + struct mana_dev_recovery *dev; + unsigned long flags; + + work = container_of(w, struct mana_dev_recovery_work, work.work); + + spin_lock_irqsave(&work->lock, flags); + + while (!list_empty(&work->dev_list)) { + dev = list_first_entry(&work->dev_list, + struct mana_dev_recovery, list); + list_del(&dev->list); + spin_unlock_irqrestore(&work->lock, flags); + + mana_do_service(dev->type, dev->pdev); + pci_dev_put(dev->pdev); + kfree(dev); + + spin_lock_irqsave(&work->lock, flags); + } + + spin_unlock_irqrestore(&work->lock, flags); +} + +static void mana_serv_func(struct work_struct *w) +{ + struct mana_serv_work *mns_wk; + struct pci_dev *pdev; + + mns_wk = container_of(w, struct mana_serv_work, serv_work); + pdev = mns_wk->pdev; + + if (pdev) + mana_do_service(mns_wk->type, pdev); + + pci_dev_put(pdev); + kfree(mns_wk); + module_put(THIS_MODULE); +} + +static int mana_pci_schedule_serv_work(struct gdma_context *gc, + enum gdma_eqe_type type) +{ + struct mana_serv_work *mns_wk; + + if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) { + dev_info(gc->dev, "Already in service\n"); + return -EBUSY; + } + + if (!try_module_get(THIS_MODULE)) { + dev_info(gc->dev, "Module is unloading\n"); + clear_bit(GC_IN_SERVICE, &gc->flags); + return -ENODEV; + } + + mns_wk = kzalloc(sizeof(*mns_wk), GFP_ATOMIC); + if (!mns_wk) { + module_put(THIS_MODULE); + clear_bit(GC_IN_SERVICE, &gc->flags); + return -ENOMEM; + } + + dev_info(gc->dev, "Start MANA service type:%d\n", type); + mns_wk->pdev = to_pci_dev(gc->dev); + mns_wk->type = type; + pci_dev_get(mns_wk->pdev); + INIT_WORK(&mns_wk->serv_work, mana_serv_func); + schedule_work(&mns_wk->serv_work); + return 0; +} + +static bool mana_pci_msix_can_alloc_dyn(struct gdma_context *gc) +{ + return pci_msix_can_alloc_dyn(to_pci_dev(gc->dev)); +} + +static int mana_pci_msix_virq(struct gdma_context *gc, int msi) +{ + return pci_irq_vector(to_pci_dev(gc->dev), msi); +} + +static int mana_pci_msix_alloc_at(struct gdma_context *gc, int *msi) +{ + struct msi_map irq_map; + + irq_map = pci_msix_alloc_irq_at(to_pci_dev(gc->dev), *msi, NULL); + if (!irq_map.virq) + return irq_map.index; + + *msi = irq_map.index; + return irq_map.virq; +} + +static void mana_pci_msix_free(struct gdma_context *gc, int msi, int irq) +{ + struct msi_map irq_map = { .virq = irq, .index = msi }; + + pci_msix_free_irq(to_pci_dev(gc->dev), irq_map); +} + +static int mana_pci_msix_vec_count(struct gdma_context *gc) +{ + return pci_msix_vec_count(to_pci_dev(gc->dev)); +} + +static int mana_pci_dev_reset(struct gdma_context *gc) +{ + return pcie_flr(to_pci_dev(gc->dev)); +} + +/* + * Spread on CPUs with the following heuristics: + * + * 1. No more than one IRQ per CPU, if possible; + * 2. NUMA locality is the second priority; + * 3. Sibling dislocality is the last priority. + * + * Let's consider this topology: + * + * Node 0 1 + * Core 0 1 2 3 + * CPU 0 1 2 3 4 5 6 7 + * + * The most performant IRQ distribution based on the above topology + * and heuristics may look like this: + * + * IRQ Nodes Cores CPUs + * 0 1 0 0-1 + * 1 1 1 2-3 + * 2 1 0 0-1 + * 3 1 1 2-3 + * 4 2 2 4-5 + * 5 2 3 6-7 + * 6 2 2 4-5 + * 7 2 3 6-7 + * + * The heuristics is implemented as follows. + * + * The outer for_each() loop resets the 'weight' to the actual number + * of CPUs in the hop. Then inner for_each() loop decrements it by the + * number of sibling groups (cores) while assigning first set of IRQs + * to each group. IRQs 0 and 1 above are distributed this way. + * + * Now, because NUMA locality is more important, we should walk the + * same set of siblings and assign 2nd set of IRQs (2 and 3), and it's + * implemented by the medium while() loop. We do like this unless the + * number of IRQs assigned on this hop will not become equal to number + * of CPUs in the hop (weight == 0). Then we switch to the next hop and + * do the same thing. + */ + +static int mana_irq_setup_numa_aware(unsigned int *irqs, unsigned int len, + int node, bool skip_first_cpu) +{ + const struct cpumask *next, *prev = cpu_none_mask; + cpumask_var_t cpus __free(free_cpumask_var); + int cpu, weight; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + rcu_read_lock(); + for_each_numa_hop_mask(next, node) { + weight = cpumask_weight_andnot(next, prev); + while (weight > 0) { + cpumask_andnot(cpus, next, prev); + for_each_cpu(cpu, cpus) { + cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); + --weight; + + if (unlikely(skip_first_cpu)) { + skip_first_cpu = false; + continue; + } + + if (len-- == 0) + goto done; + + irq_set_affinity_and_hint(*irqs++, topology_sibling_cpumask(cpu)); + } + } + prev = next; + } +done: + rcu_read_unlock(); + return 0; +} + +/* must be called with cpus_read_lock() held */ +static void mana_irq_setup_linear(unsigned int *irqs, unsigned int len) +{ + int cpu; + + for_each_online_cpu(cpu) { + if (len == 0) + break; + + irq_set_affinity_and_hint(*irqs++, cpumask_of(cpu)); + len--; + } +} + +static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_irq_context *gic; + int *irqs, err, i, msi; + + irqs = kmalloc_objs(int, nvec); + if (!irqs) + return -ENOMEM; + + /* + * In this function, num_msix_usable = HWC IRQ + Queue IRQ. + * nvec is only Queue IRQ (HWC already setup). + * While processing the next pci irq vector, we start with index 1, + * as IRQ vector at index 0 is already processed for HWC. + * However, the population of irqs array starts with index 0, to be + * further used in mana_irq_setup_numa_aware() + */ + for (i = 1; i <= nvec; i++) { + msi = i; + gic = mana_gd_get_gic(gc, false, &msi); + if (IS_ERR(gic)) { + err = PTR_ERR(gic); + goto free_irq; + } + + irqs[i - 1] = gic->irq; + } + + /* + * When calling mana_irq_setup_numa_aware() for dynamically added IRQs, + * if number of CPUs is more than or equal to allocated MSI-X, we need to + * skip the first CPU sibling group since they are already affinitized to + * HWC IRQ + */ + cpus_read_lock(); + if (gc->num_msix_usable <= num_online_cpus()) { + err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, + true); + if (err) { + cpus_read_unlock(); + goto free_irq; + } + } else { + /* + * When num_msix_usable are more than num_online_cpus, our + * queue IRQs should be equal to num of online vCPUs. + * We try to make sure queue IRQs spread across all vCPUs. + * In such a case NUMA or CPU core affinity does not matter. + * Note: in this case the total mana IRQ should always be + * num_online_cpus + 1. The first HWC IRQ is already handled + * in HWC setup calls + * However, if CPUs went offline since num_msix_usable was + * computed, queue IRQs will be more than num_online_cpus(). + * In such cases remaining extra IRQs will retain their default + * affinity. + */ + int first_unassigned = num_online_cpus(); + + if (nvec > first_unassigned) { + char buf[32]; + + if (first_unassigned == nvec - 1) + snprintf(buf, sizeof(buf), "%d", + first_unassigned); + else + snprintf(buf, sizeof(buf), "%d-%d", + first_unassigned, nvec - 1); + + dev_dbg(&pdev->dev, + "MANA IRQ indices #%s will retain the default CPU affinity\n", + buf); + } + + mana_irq_setup_linear(irqs, nvec); + } + + cpus_read_unlock(); + kfree(irqs); + return 0; + +free_irq: + for (i -= 1; i > 0; i--) + mana_gd_put_gic(gc, false, i); + kfree(irqs); + return err; +} + +static int mana_gd_setup_irqs(struct pci_dev *pdev, int nvec) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_irq_context *gic; + int *irqs, *start_irqs; + unsigned int cpu; + int err, i, msi; + + irqs = kmalloc_objs(int, nvec); + if (!irqs) + return -ENOMEM; + + start_irqs = irqs; + + for (i = 0; i < nvec; i++) { + msi = i; + gic = mana_gd_get_gic(gc, false, &msi); + if (IS_ERR(gic)) { + err = PTR_ERR(gic); + goto free_irq; + } + + irqs[i] = gic->irq; + } + + /* If number of IRQ is one extra than number of online CPUs, + * then we need to assign IRQ0 (hwc irq) and IRQ1 to + * same CPU. + * Else we will use different CPUs for IRQ0 and IRQ1. + * Also we are using cpumask_local_spread instead of + * cpumask_first for the node, because the node can be + * mem only. + */ + cpus_read_lock(); + if (nvec > num_online_cpus()) { + cpu = cpumask_local_spread(0, gc->numa_node); + irq_set_affinity_and_hint(irqs[0], cpumask_of(cpu)); + irqs++; + nvec -= 1; + } + + err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, false); + if (err) { + cpus_read_unlock(); + goto free_irq; + } + + cpus_read_unlock(); + kfree(start_irqs); + return 0; + +free_irq: + for (i -= 1; i >= 0; i--) + mana_gd_put_gic(gc, false, i); + + kfree(start_irqs); + return err; +} + +static int mana_gd_setup_hwc_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + unsigned int max_irqs, min_irqs; + int nvec, err; + + if (pci_msix_can_alloc_dyn(pdev)) { + max_irqs = 1; + min_irqs = 1; + } else { + /* Need 1 interrupt for HWC */ + max_irqs = min(num_online_cpus(), MANA_MAX_NUM_QUEUES) + 1; + min_irqs = 2; + gc->msi_sharing = true; + } + + nvec = pci_alloc_irq_vectors(pdev, min_irqs, max_irqs, PCI_IRQ_MSIX); + if (nvec < 0) + return nvec; + + err = mana_gd_setup_irqs(pdev, nvec); + if (err) { + pci_free_irq_vectors(pdev); + return err; + } + + gc->num_msix_usable = nvec; + gc->max_num_msix = nvec; + + return 0; +} + +static int mana_gd_setup_remaining_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct msi_map irq_map; + int max_irqs, i, err; + + if (!pci_msix_can_alloc_dyn(pdev)) + /* remain irqs are already allocated with HWC IRQ */ + return 0; + + /* allocate only remaining IRQs*/ + max_irqs = gc->num_msix_usable - 1; + + for (i = 1; i <= max_irqs; i++) { + irq_map = pci_msix_alloc_irq_at(pdev, i, NULL); + if (!irq_map.virq) { + err = irq_map.index; + /* caller will handle cleaning up all allocated + * irqs, after HWC is destroyed + */ + return err; + } + } + + err = mana_gd_setup_dyn_irqs(pdev, max_irqs); + if (err) + return err; + + gc->max_num_msix = gc->max_num_msix + max_irqs; + + return 0; +} + +static void mana_gd_remove_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + int i; + + if (gc->max_num_msix < 1) + return; + + for (i = 0; i < gc->max_num_msix; i++) { + if (!xa_load(&gc->irq_contexts, i)) + continue; + + mana_gd_put_gic(gc, false, i); + } + + WARN_ON(!xa_empty(&gc->irq_contexts)); + + pci_free_irq_vectors(pdev); + + bitmap_free(gc->msi_bitmap); + gc->msi_bitmap = NULL; + gc->max_num_msix = 0; + gc->num_msix_usable = 0; +} + +static int mana_pci_setup_hwc_irqs(struct gdma_context *gc) +{ + return mana_gd_setup_hwc_irqs(to_pci_dev(gc->dev)); +} + +static int mana_pci_setup_remaining_irqs(struct gdma_context *gc) +{ + int err; + + err = mana_gd_setup_remaining_irqs(to_pci_dev(gc->dev)); + if (err) { + dev_err(gc->dev, "Failed to setup remaining IRQs: %d", err); + return err; + } + + if (!gc->msi_sharing) { + gc->msi_bitmap = bitmap_zalloc(gc->num_msix_usable, GFP_KERNEL); + if (!gc->msi_bitmap) + return -ENOMEM; + /* Set bit for HWC */ + set_bit(0, gc->msi_bitmap); + } + + return 0; +} + +static void mana_pci_remove_irqs(struct gdma_context *gc) +{ + mana_gd_remove_irqs(to_pci_dev(gc->dev)); +} + +static const struct gdma_bus_ops mana_pci_bus_ops = { + .bus_name = "pci", + .adev_prefix = "mana", + .msix_can_alloc_dyn = mana_pci_msix_can_alloc_dyn, + .msix_virq = mana_pci_msix_virq, + .msix_alloc_at = mana_pci_msix_alloc_at, + .msix_free = mana_pci_msix_free, + .msix_vec_count = mana_pci_msix_vec_count, + .setup_hwc_irqs = mana_pci_setup_hwc_irqs, + .setup_remaining_irqs = mana_pci_setup_remaining_irqs, + .remove_irqs = mana_pci_remove_irqs, + .dev_reset = mana_pci_dev_reset, + .schedule_serv_work = mana_pci_schedule_serv_work, + .drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1_PCI, +}; + +static bool mana_is_pf(unsigned short dev_id) +{ + return dev_id == MANA_PF_DEVICE_ID || dev_id == MANA_PF2_DEVICE_ID; +} + +static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct gdma_context *gc; + void __iomem *bar0_va; + int bar = 0; + int err; + + /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ + BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "Failed to enable pci device (err=%d)\n", err); + return -ENXIO; + } + + pci_set_master(pdev); + + err = pci_request_regions(pdev, "mana"); + if (err) + goto disable_dev; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "DMA set mask failed: %d\n", err); + goto release_region; + } + dma_set_max_seg_size(&pdev->dev, UINT_MAX); + + err = -ENOMEM; + gc = vzalloc(sizeof(*gc)); + if (!gc) + goto release_region; + + mutex_init(&gc->eq_test_event_mutex); + mutex_init(&gc->gic_mutex); + pci_set_drvdata(pdev, gc); + gc->bar0_pa = pci_resource_start(pdev, 0); + gc->bar0_size = pci_resource_len(pdev, 0); + + bar0_va = pci_iomap(pdev, bar, 0); + if (!bar0_va) + goto free_gc; + + gc->numa_node = dev_to_node(&pdev->dev); + gc->is_pf = mana_is_pf(pdev->device); + gc->is_pf2 = (pdev->device == MANA_PF2_DEVICE_ID); + + gc->bar0_va = bar0_va; + gc->dev = &pdev->dev; + gc->bus_ops = &mana_pci_bus_ops; + xa_init(&gc->irq_contexts); + + err = mana_gd_setup(gc); + if (err) + goto unmap_bar; + + err = mana_probe(&gc->mana, false); + if (err) + goto cleanup_gd; + + err = mana_rdma_probe(&gc->mana_ib); + if (err) + goto cleanup_mana; + + /* + * If a hardware reset event has occurred over HWC during probe, + * rollback and perform hardware reset procedure. + */ + if (test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { + err = -EPROTO; + goto cleanup_mana_rdma; + } + + return 0; + +cleanup_mana_rdma: + mana_rdma_remove(&gc->mana_ib); +cleanup_mana: + mana_remove(&gc->mana, false); +cleanup_gd: + mana_gd_cleanup(gc); +unmap_bar: + xa_destroy(&gc->irq_contexts); + pci_iounmap(pdev, bar0_va); +free_gc: + pci_set_drvdata(pdev, NULL); + vfree(gc); +release_region: + pci_release_regions(pdev); +disable_dev: + pci_disable_device(pdev); + dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err); + + /* + * Hardware could be in recovery mode and the HWC returns TIMEDOUT or + * EPROTO from mana_gd_setup(), mana_probe() or mana_rdma_probe(), or + * we received a hardware reset event over HWC interrupt. In this case, + * perform the device recovery procedure after MANA_SERVICE_PERIOD + * seconds. + */ + if (err == -ETIMEDOUT || err == -EPROTO) { + struct mana_dev_recovery *dev; + unsigned long flags; + + dev_info(&pdev->dev, "Start MANA recovery mode\n"); + + dev = kzalloc_obj(*dev); + if (!dev) + return err; + + dev->pdev = pci_dev_get(pdev); + dev->type = GDMA_EQE_HWC_RESET_REQUEST; + + spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); + list_add_tail(&dev->list, &mana_dev_recovery_work.dev_list); + spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); + + schedule_delayed_work(&mana_dev_recovery_work.work, + secs_to_jiffies(MANA_SERVICE_PERIOD)); + } + + return err; +} + +static void mana_gd_remove(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + pci_disable_sriov(pdev); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, false); + + mana_gd_cleanup(gc); + + xa_destroy(&gc->irq_contexts); + + pci_iounmap(pdev, gc->bar0_va); + + vfree(gc); + + pci_release_regions(pdev); + pci_disable_device(pdev); + + dev_dbg(&pdev->dev, "mana gdma remove successful\n"); +} + +/* The 'state' parameter is not used. */ +static int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, true); + + mana_gd_cleanup(gc); + + return 0; +} + +static int mana_gd_resume(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + int err; + + err = mana_gd_setup(gc); + if (err) + return err; + + err = mana_probe(&gc->mana, true); + if (err) + goto cleanup_gd; + + err = mana_rdma_probe(&gc->mana_ib); + if (err) + mana_rdma_remove(&gc->mana_ib); + + return err; + +cleanup_gd: + mana_gd_cleanup(gc); + return err; +} + +/* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ +static void mana_gd_shutdown(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + dev_info(&pdev->dev, "Shutdown was called\n"); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, true); + + mana_gd_cleanup(gc); + + pci_disable_device(pdev); +} + +static int mana_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + int err = 0; + + dev_info(&pdev->dev, "Requested num VFs: %d\n", numvfs); + + if (numvfs > 0) { + err = pci_enable_sriov(pdev, numvfs); + } else { + if (pci_vfs_assigned(pdev)) { + dev_warn(&pdev->dev, + "Cannot disable SR-IOV while VFs are assigned\n"); + return -EPERM; + } + + pci_disable_sriov(pdev); + } + + return err ? err : numvfs; +} + +static const struct pci_device_id mana_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, + { } +}; + +static struct pci_driver mana_driver = { + .name = "mana", + .id_table = mana_id_table, + .probe = mana_gd_probe, + .remove = mana_gd_remove, + .suspend = mana_gd_suspend, + .resume = mana_gd_resume, + .shutdown = mana_gd_shutdown, + .sriov_configure = mana_sriov_configure, +}; + +MODULE_DEVICE_TABLE(pci, mana_id_table); + +static int __init mana_driver_init(void) +{ + INIT_LIST_HEAD(&mana_dev_recovery_work.dev_list); + spin_lock_init(&mana_dev_recovery_work.lock); + INIT_DELAYED_WORK(&mana_dev_recovery_work.work, + mana_recovery_delayed_func); + + return pci_register_driver(&mana_driver); +} + +static void __exit mana_driver_exit(void) +{ + struct mana_dev_recovery *dev; + unsigned long flags; + + disable_delayed_work_sync(&mana_dev_recovery_work.work); + + spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); + while (!list_empty(&mana_dev_recovery_work.dev_list)) { + dev = list_first_entry(&mana_dev_recovery_work.dev_list, + struct mana_dev_recovery, list); + list_del(&dev->list); + pci_dev_put(dev->pdev); + kfree(dev); + } + spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); + + pci_unregister_driver(&mana_driver); +} + +module_init(mana_driver_init); +module_exit(mana_driver_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver"); +MODULE_IMPORT_NS("NET_MANA"); diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index 263e7c4e2934..3bca4b683134 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -170,12 +170,12 @@ static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, hwc->txq->msg_buf->gpa_mkey = val; break; - case HWC_INIT_DATA_PF_DEST_RQ_ID: - hwc->pf_dest_vrq_id = val; + case HWC_INIT_DATA_DEST_RQ_ID: + hwc->dest_vrq_id = val; break; - case HWC_INIT_DATA_PF_DEST_CQ_ID: - hwc->pf_dest_vrcq_id = val; + case HWC_INIT_DATA_DEST_CQ_ID: + hwc->dest_vrcq_id = val; break; } @@ -855,13 +855,12 @@ void mana_hwc_destroy_channel(struct gdma_context *gc) int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, const void *req, u32 resp_len, void *resp) { - struct gdma_context *gc = hwc->gdma_dev->gdma_context; struct hwc_work_request *tx_wr; struct hwc_wq *txq = hwc->txq; struct gdma_req_hdr *req_msg; struct hwc_caller_ctx *ctx; - u32 dest_vrcq = 0; - u32 dest_vrq = 0; + u32 dest_vrcq; + u32 dest_vrq; u32 command; u16 msg_id; int err; @@ -890,10 +889,13 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, tx_wr->msg_size = req_len; command = req_msg->req.msg_type; - if (gc->is_pf) { - dest_vrq = hwc->pf_dest_vrq_id; - dest_vrcq = hwc->pf_dest_vrcq_id; - } + /* The hardware reports the HWC destination queues through + * HWC_INIT_DATA_DEST_RQ_ID and HWC_INIT_DATA_DEST_CQ_ID, and + * always supplies values that are valid for this function, so no + * PF-specific handling is needed here. + */ + dest_vrq = hwc->dest_vrq_id; + dest_vrcq = hwc->dest_vrcq_id; err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false); if (err) { diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c index b27fd9b8c4f6..7c55f8724441 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -21,7 +21,7 @@ void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev) ndevtxq = netdev_get_tx_queue(ndev, txq_idx); __netif_tx_lock(ndevtxq, smp_processor_id()); - rc = mana_start_xmit(skb, ndev); + rc = __mana_start_xmit(skb, ndev, true); __netif_tx_unlock(ndevtxq); @@ -254,7 +254,6 @@ err_dealloc_rxbuffs: int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf) { struct netlink_ext_ack *extack = bpf->extack; - int ret; switch (bpf->command) { case XDP_SETUP_PROG: @@ -263,6 +262,4 @@ int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf) default: return -EOPNOTSUPP; } - - return ret; } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index a1fb06b23947..6b7c027e497e 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -9,7 +9,6 @@ #include <linux/ethtool.h> #include <linux/filter.h> #include <linux/mm.h> -#include <linux/pci.h> #include <linux/export.h> #include <linux/skbuff.h> @@ -20,6 +19,7 @@ #include <net/xdp.h> #include <net/mana/mana.h> + #include <net/mana/mana_auxiliary.h> #include <net/mana/hw_channel.h> @@ -346,7 +346,14 @@ clear_flag: rtnl_unlock(); } -netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) +/* skb->cb of the SKBs pending a TX completion */ +struct mana_skb_cb { + struct gdma_posted_wqe_info wqe_info; + bool xdp; +}; + +netdev_tx_t __mana_start_xmit(struct sk_buff *skb, struct net_device *ndev, + bool xdp) { enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; struct mana_port_context *apc = netdev_priv(ndev); @@ -360,6 +367,7 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) struct gdma_queue *gdma_sq; int err, len, num_gso_seg; unsigned int csum_type; + struct mana_skb_cb *cb; struct mana_txq *txq; struct mana_cq *cq; @@ -520,14 +528,16 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) goto free_sgl_ptr; } + cb = (struct mana_skb_cb *)skb->cb; + cb->xdp = xdp; + skb_queue_tail(&txq->pending_skbs, skb); len = skb->len; num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; net_txq = netdev_get_tx_queue(ndev, txq_idx); - err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, - (struct gdma_posted_wqe_info *)skb->cb); + err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, &cb->wqe_info); if (!mana_can_tx(gdma_sq)) { netif_tx_stop_queue(net_txq); apc->eth_stats.stop_queue++; @@ -543,6 +553,14 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) err = NETDEV_TX_OK; atomic_inc(&txq->pending_sends); + /* Account the skb in BQL before the doorbell lets the hardware + * complete it. XDP frames bypass the qdisc and BQL can't hold them + * back, so leave them out: XDP traffic could otherwise keep the + * queue stopped, starving the stack and tripping the TX watchdog. + */ + if (!xdp) + netdev_tx_sent_queue(net_txq, len); + mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); /* skb may be freed after mana_gd_post_work_request. Do not use it. */ @@ -574,6 +592,11 @@ tx_drop: return NETDEV_TX_OK; } +netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) +{ + return __mana_start_xmit(skb, ndev, false); +} + #if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) static netdev_features_t mana_features_check(struct sk_buff *skb, struct net_device *ndev, @@ -1938,9 +1961,10 @@ void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) static void mana_poll_tx_cq(struct mana_cq *cq) { struct gdma_comp *completions = cq->gdma_comp_buf; - struct gdma_posted_wqe_info *wqe_info; unsigned int pkt_transmitted = 0; unsigned int wqe_unit_cnt = 0; + unsigned int bql_bytes = 0; + unsigned int bql_pkts = 0; unsigned int tx_bytes = 0; struct mana_txq *txq = cq->txq; struct mana_port_context *apc; @@ -1948,6 +1972,7 @@ static void mana_poll_tx_cq(struct mana_cq *cq) struct gdma_queue *gdma_wq; unsigned int avail_space; struct net_device *ndev; + struct mana_skb_cb *cb; struct sk_buff *skb; bool txq_stopped; int comp_read; @@ -2018,12 +2043,16 @@ static void mana_poll_tx_cq(struct mana_cq *cq) if (WARN_ON_ONCE(!skb)) return; - wqe_info = (struct gdma_posted_wqe_info *)skb->cb; - wqe_unit_cnt += wqe_info->wqe_size_in_bu; + cb = (struct mana_skb_cb *)skb->cb; + wqe_unit_cnt += cb->wqe_info.wqe_size_in_bu; mana_unmap_skb(skb, apc); tx_bytes += skb->len; + if (!cb->xdp) { + bql_pkts++; + bql_bytes += skb->len; + } napi_consume_skb(skb, cq->budget); @@ -2038,10 +2067,12 @@ static void mana_poll_tx_cq(struct mana_cq *cq) gdma_wq = txq->gdma_sq; avail_space = mana_gd_wq_avail_space(gdma_wq); + net_txq = txq->net_txq; + netdev_tx_completed_queue(net_txq, bql_pkts, bql_bytes); + /* Ensure tail updated before checking q stop */ smp_mb(); - net_txq = txq->net_txq; txq_stopped = netif_tx_queue_stopped(net_txq); /* Ensure checking txq_stopped before apc->port_is_up. */ @@ -2694,6 +2725,13 @@ static void mana_destroy_txq(struct mana_port_context *apc) apc->tx_qp[i]->txq.napi_initialized = false; } + /* Forget the SKBs freed without being completed after a drain + * timeout, now that NAPI can't complete any. Doing it here also + * covers the queues that a lower channel count won't recreate, + * which could otherwise stay stopped. + */ + netdev_tx_reset_queue(apc->tx_qp[i]->txq.net_txq); + if (apc->tx_qp[i]->tx_object != INVALID_MANA_HANDLE) mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i]->tx_object); @@ -3754,10 +3792,9 @@ static int mana_dealloc_queues(struct net_device *ndev) tsleep <<= 1; } if (atomic_read(&txq->pending_sends)) { - err = - pcie_flr(to_pci_dev(gd->gdma_context->dev)); + err = mana_gd_dev_reset(gd->gdma_context); if (err) { - netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n", + netdev_err(ndev, "device reset failed %d with %d pkts pending in txq %u\n", err, atomic_read(&txq->pending_sends), txq->gdma_txq_id); @@ -3987,7 +4024,9 @@ static int add_adev(struct gdma_dev *gd, const char *name) /* madev is owned by the auxiliary device */ madev = NULL; - ret = auxiliary_device_add(adev); + /* Keep match names independent of the common module's name. */ + ret = __auxiliary_device_add(adev, + gd->gdma_context->bus_ops->adev_prefix); if (ret) goto add_fail; @@ -4060,6 +4099,9 @@ int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type even return 0; } + if (!gc->service_wq) + return -EOPNOTSUPP; + serv_work = kzalloc_obj(*serv_work, GFP_ATOMIC); if (!serv_work) return -ENOMEM; @@ -4221,6 +4263,7 @@ out: return err; } +EXPORT_SYMBOL_NS(mana_probe, "NET_MANA"); void mana_remove(struct gdma_dev *gd, bool suspending) { @@ -4299,6 +4342,7 @@ void mana_remove(struct gdma_dev *gd, bool suspending) kfree(ac); dev_dbg(dev, "%s succeeded\n", __func__); } +EXPORT_SYMBOL_NS(mana_remove, "NET_MANA"); int mana_rdma_probe(struct gdma_dev *gd) { @@ -4334,6 +4378,7 @@ int mana_rdma_probe(struct gdma_dev *gd) return err; } +EXPORT_SYMBOL_NS(mana_rdma_probe, "NET_MANA"); void mana_rdma_remove(struct gdma_dev *gd) { @@ -4354,6 +4399,7 @@ void mana_rdma_remove(struct gdma_dev *gd) mana_gd_deregister_device(gd); } +EXPORT_SYMBOL_NS(mana_rdma_remove, "NET_MANA"); struct net_device *mana_get_primary_netdev(struct mana_context *ac, u32 port_index, |