// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * mlx5 VFIO selftest driver * * Programs mlx5 ConnectX VFs and PFs through the bare-metal command interface * and RDMA WRITE self-loopback to perform DMA. Implements vfio_pci_driver_ops * (probe/init/remove) and plugs into the VFIO selftest framework. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mlx5_hw.h" /* * Driver state — overlaid on device->driver.region.vaddr. * * Contains both software-only state and HW-visible DMA buffers. HW buffers need * strict IOVA alignment. */ struct mlx5st_device { /* Back pointer */ struct vfio_pci_device *device; /* BAR0 */ struct mlx5st_initial_seg __iomem *init_seg; void __iomem *bar0; /* Command interface */ struct mlx5st_cmd_queue_entry *cmd_lay; struct mlx5st_cmd_queue_entry *pages_cmd_lay; u8 cmd_log_stride; unsigned int pages_slot; u8 cmd_token; bool cmd_sig_enabled; /* PD */ u32 pdn; /* Global PA-mode MKEY */ u32 global_lkey; u32 global_rkey; u32 mkey_index; /* CQ */ u32 cqn; u32 cq_ci; u32 cq_arm_sn; /* UAR */ u32 uar_page; void __iomem *uar_base; unsigned int uar_bf_offset; /* EQ (cmd/pages events — polled, not interrupt-driven) */ u32 eqn; u32 eq_cons_index; bool have_eq; /* MSI EQ (CQ completion events — fires MSI-X) */ u32 msi_eqn; u32 msi_eq_cons_index; bool have_msi_eq; /* Async pages slot state */ bool pages_slot_in_use; bool pages_slot_is_reclaim; unsigned int pages_reclaim_npages; unsigned int pages_pending_give; unsigned int pages_pending_reclaim; u16 pages_pending_func_id; bool pages_func_id_seen; /* QP */ u32 qpn; u32 sq_pi; u32 sq_ci; /* FW pages bitmap */ u64 fw_pages_bitmap[MAX_FW_PAGES / 64]; u32 fw_pages_given; u16 fw_func_id; /* Capabilities */ bool fl_supported; u8 log_max_msg; /* Buffers used by send_msi() to trigger an interrupt */ u64 send_msi_src; u64 send_msi_dst; /* * HW-visible DMA buffers below — device reads/writes via DMA. */ struct mlx5st_cmd_queue_entry cmd_queue [MLX5_HW_PAGE_SIZE / sizeof(struct mlx5st_cmd_queue_entry)] __aligned(MLX5_HW_PAGE_SIZE); struct mlx5st_send_wqe sq_buf[SQ_WQE_CNT]; struct mlx5st_dbrec cq_dbrec; struct mlx5st_dbrec qp_dbrec; struct mlx5st_cqe64 cq_buf[CQ_CQE_CNT]; /* Slot 0 mailboxes (regular commands) */ struct mlx5st_mbox_entry cmd_in_mbox[CMD_MBOX_NENT]; struct mlx5st_mbox_entry cmd_out_mbox[CMD_MBOX_NENT]; /* Pages slot mailboxes (async MANAGE_PAGES) */ struct mlx5st_mbox_entry pages_in_mbox[CMD_MBOX_NENT]; struct mlx5st_mbox_entry pages_out_mbox[CMD_MBOX_NENT]; /* EQ does not support page_offset */ struct mlx5st_eqe eq_buf[EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE); /* MSI EQ buffer — CQ completions generate EQEs here -> MSI-X */ struct mlx5st_eqe msi_eq_buf[MSI_EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE); u8 fw_pages[MAX_FW_PAGES][MLX5_HW_PAGE_SIZE] __aligned(MLX5_HW_PAGE_SIZE); }; /* Check against HW limits on IOVA alignment */ static_assert(offsetof(struct mlx5st_device, cmd_in_mbox) % CMD_MBOX_STRIDE == 0, "cmd_in_mbox must be stride-aligned"); static_assert(offsetof(struct mlx5st_device, pages_in_mbox) % CMD_MBOX_STRIDE == 0, "pages_in_mbox must be stride-aligned"); static_assert(offsetof(struct mlx5st_device, cq_buf) % 64 == 0, "cq_buf must be 64-byte aligned"); static_assert(offsetof(struct mlx5st_device, sq_buf) % 64 == 0, "sq_buf must be 64-byte aligned"); static_assert(offsetof(struct mlx5st_device, cq_dbrec) % 64 == 0, "cq_dbrec must be 64-byte aligned"); static_assert(offsetof(struct mlx5st_device, qp_dbrec) % 64 == 0, "qp_dbrec must be 64-byte aligned"); static_assert(offsetof(struct mlx5st_device, eq_buf) % MLX5_HW_PAGE_SIZE == 0, "eq_buf must be page-aligned"); static_assert(offsetof(struct mlx5st_device, msi_eq_buf) % MLX5_HW_PAGE_SIZE == 0, "msi_eq_buf must be page-aligned"); static_assert(offsetof(struct mlx5st_device, fw_pages) % MLX5_HW_PAGE_SIZE == 0, "fw_pages must be page-aligned"); static struct mlx5st_device *to_mlx5st(struct vfio_pci_device *device) { return device->driver.region.vaddr; } /* * Fill a PAS (Physical Address Segment) for a buffer in the driver region. * Sets pas[0] to the page-aligned IOVA and returns the page_offset (the * buffer's byte offset within that page, in units of 64 bytes). */ static unsigned int mlx5st_fill_pas(struct vfio_pci_device *device, void *buf, __be64 *pas) { u64 iova = to_iova(device, buf); pas[0] = cpu_to_be64(iova & ~(u64)(MLX5_HW_PAGE_SIZE - 1)); return (iova & (MLX5_HW_PAGE_SIZE - 1)) / 64; } /* * Probe — match mlx5 devices by PCI vendor/device ID. */ static int mlx5st_probe(struct vfio_pci_device *device) { static const u16 mlx5st_pci_ids[] = { 0x1011, /* Connect-IB */ 0x1012, /* Connect-IB VF */ 0x1013, /* ConnectX-4 */ 0x1014, /* ConnectX-4 VF */ 0x1015, /* ConnectX-4LX */ 0x1016, /* ConnectX-4LX VF */ 0x1017, /* ConnectX-5 */ 0x1018, /* ConnectX-5 VF */ 0x1019, /* ConnectX-5 Ex */ 0x101a, /* ConnectX-5 Ex VF */ 0x101b, /* ConnectX-6 */ 0x101c, /* ConnectX-6 VF */ 0x101d, /* ConnectX-6 Dx */ 0x101e, /* ConnectX-6 Dx VF */ 0x101f, /* ConnectX-6 LX */ 0x1021, /* ConnectX-7 */ 0x1023, /* ConnectX-8 */ 0x1025, /* ConnectX-9 */ 0x1027, /* ConnectX-10 */ 0x2101, /* ConnectX-10 NVLink-C2C */ 0xa2d2, /* BlueField integrated ConnectX-5 */ 0xa2d3, /* BlueField integrated ConnectX-5 VF */ 0xa2d6, /* BlueField-2 integrated ConnectX-6 Dx */ 0xa2dc, /* BlueField-3 integrated ConnectX-7 */ 0xa2df, /* BlueField-4 integrated ConnectX-8 */ }; unsigned int i; u16 did; if (vfio_pci_config_readw(device, PCI_VENDOR_ID) != PCI_VENDOR_ID_MELLANOX) return -ENODEV; did = vfio_pci_config_readw(device, PCI_DEVICE_ID); for (i = 0; i < ARRAY_SIZE(mlx5st_pci_ids); i++) { if (mlx5st_pci_ids[i] == did) return 0; } return -ENODEV; } /* * Command interface */ static u8 xor8_buf(const void *buf, size_t offset, size_t len) { const u8 *p = buf; u8 sum = 0; size_t i; for (i = offset; i < offset + len; i++) sum ^= p[i]; return sum; } #define CMD_IF_BOX_CTRL_OFF MLX5_BYTE_OFF(cmd_if_box, reserved_at_1000) #define CMD_IF_BOX_CTRL_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, ctrl_signature) #define CMD_IF_BOX_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, signature) static void mlx5st_cmd_calc_block_sig(struct mlx5st_cmd_if_box *blk) { MLX5_SET(cmd_if_box, blk, ctrl_signature, ~xor8_buf(blk, CMD_IF_BOX_CTRL_OFF, CMD_IF_BOX_CTRL_SIG_OFF - CMD_IF_BOX_CTRL_OFF)); MLX5_SET(cmd_if_box, blk, signature, ~xor8_buf(blk, 0, CMD_IF_BOX_SIG_OFF)); } static int mlx5st_cmd_verify_block_sig(struct mlx5st_cmd_if_box *blk) { if (xor8_buf(blk, CMD_IF_BOX_CTRL_OFF, CMD_IF_BOX_SIG_OFF - CMD_IF_BOX_CTRL_OFF) != 0xff) return -1; if (xor8_buf(blk, 0, sizeof(struct mlx5st_cmd_if_box)) != 0xff) return -1; return 0; } static unsigned int mlx5st_cmd_setup_mbox_chain(struct vfio_pci_device *device, struct mlx5st_mbox_entry *mbox, unsigned int nblocks, u8 token) { unsigned int i; for (i = 0; i < nblocks; i++) { struct mlx5st_cmd_if_box *blk = &mbox[i].block; u64 next_iova; memset(blk, 0, sizeof(struct mlx5st_cmd_if_box)); MLX5_SET(cmd_if_box, blk, block_number, i); MLX5_SET(cmd_if_box, blk, token, token); if (i < nblocks - 1) { next_iova = to_iova(device, &mbox[i + 1]); MLX5_SET(cmd_if_box, blk, next_pointer_63_32, next_iova >> 32); MLX5_SET(cmd_if_box, blk, next_pointer_31_10, (u32)next_iova >> 10); } } return nblocks; } static void mlx5st_cmd_copy_to_mbox(struct mlx5st_mbox_entry *mbox, const void *data, unsigned int len) { const u8 *src = data; unsigned int i = 0; while (len > 0) { unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ? len : MLX5_CMD_DATA_BLOCK_SIZE; memcpy(MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data), src, chunk); src += chunk; len -= chunk; i++; } } static void mlx5st_cmd_copy_from_mbox(void *data, const struct mlx5st_mbox_entry *mbox, unsigned int len) { unsigned int i = 0; u8 *dst = data; while (len > 0) { unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ? len : MLX5_CMD_DATA_BLOCK_SIZE; memcpy(dst, MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data), chunk); dst += chunk; len -= chunk; i++; } } /* Forward declaration — cmd_exec polls events during command wait */ static void mlx5st_process_events(struct mlx5st_device *dev); static const char *mlx5st_cmd_name(u16 opcode) { switch (opcode) { case MLX5_CMD_OP_QUERY_HCA_CAP: return "QUERY_HCA_CAP"; case MLX5_CMD_OP_INIT_HCA: return "INIT_HCA"; case MLX5_CMD_OP_TEARDOWN_HCA: return "TEARDOWN_HCA"; case MLX5_CMD_OP_ENABLE_HCA: return "ENABLE_HCA"; case MLX5_CMD_OP_DISABLE_HCA: return "DISABLE_HCA"; case MLX5_CMD_OP_QUERY_PAGES: return "QUERY_PAGES"; case MLX5_CMD_OP_MANAGE_PAGES: return "MANAGE_PAGES"; case MLX5_CMD_OP_SET_HCA_CAP: return "SET_HCA_CAP"; case MLX5_CMD_OP_SET_ISSI: return "SET_ISSI"; case MLX5_CMD_OP_CREATE_MKEY: return "CREATE_MKEY"; case MLX5_CMD_OP_DESTROY_MKEY: return "DESTROY_MKEY"; case MLX5_CMD_OP_CREATE_EQ: return "CREATE_EQ"; case MLX5_CMD_OP_DESTROY_EQ: return "DESTROY_EQ"; case MLX5_CMD_OP_CREATE_CQ: return "CREATE_CQ"; case MLX5_CMD_OP_DESTROY_CQ: return "DESTROY_CQ"; case MLX5_CMD_OP_CREATE_QP: return "CREATE_QP"; case MLX5_CMD_OP_DESTROY_QP: return "DESTROY_QP"; case MLX5_CMD_OP_RST2INIT_QP: return "RST2INIT_QP"; case MLX5_CMD_OP_INIT2RTR_QP: return "INIT2RTR_QP"; case MLX5_CMD_OP_RTR2RTS_QP: return "RTR2RTS_QP"; case MLX5_CMD_OP_ALLOC_PD: return "ALLOC_PD"; case MLX5_CMD_OP_DEALLOC_PD: return "DEALLOC_PD"; case MLX5_CMD_OP_ALLOC_UAR: return "ALLOC_UAR"; case MLX5_CMD_OP_DEALLOC_UAR: return "DEALLOC_UAR"; default: return "UNKNOWN"; } } /* * Post a command on a given slot: fill the cmd_queue_entry, set up mailbox * chains, compute signatures, hand ownership to FW, and ring the doorbell. */ static void mlx5st_cmd_post(struct mlx5st_device *dev, struct mlx5st_cmd_queue_entry *cmd, struct mlx5st_mbox_entry *in_mbox, struct mlx5st_mbox_entry *out_mbox, void *in, unsigned int ilen, unsigned int olen, u32 doorbell) { struct vfio_pci_device *device = dev->device; unsigned int in_remain, out_remain, in_nblk, out_nblk; unsigned int i; void *cin, *cout; u8 token; /* Rotating non-zero token ties cmd entry to its mailbox blocks */ token = ++dev->cmd_token; if (!token) token = ++dev->cmd_token; in_remain = ilen > MLX5_CMD_INLINE_SZ ? ilen - MLX5_CMD_INLINE_SZ : 0; out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0; in_nblk = (in_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE; out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE; /* Set up mailbox chains */ if (in_nblk > 0) { mlx5st_cmd_setup_mbox_chain(device, in_mbox, in_nblk, token); mlx5st_cmd_copy_to_mbox(in_mbox, (u8 *)in + MLX5_CMD_INLINE_SZ, in_remain); } if (out_nblk > 0) mlx5st_cmd_setup_mbox_chain(device, out_mbox, out_nblk, token); /* Copy inline input */ cin = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_input_inline_data); memset(cin, 0, MLX5_CMD_INLINE_SZ); memcpy(cin, in, ilen < MLX5_CMD_INLINE_SZ ? ilen : MLX5_CMD_INLINE_SZ); MLX5_SET(cmd_queue_entry, cmd, input_length, ilen); MLX5_SET(cmd_queue_entry, cmd, token, token); /* Zero inline output */ cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); memset(cout, 0, MLX5_CMD_INLINE_SZ); MLX5_SET(cmd_queue_entry, cmd, output_length, olen); /* * Compute signatures: mailbox blocks first, then cmd_queue_entry. * The sig must cover the final state including ownership=0x1, but * we must not set ownership until after the sig is in place - * XOR in the 0x1 without storing it to memory. */ for (i = 0; i < in_nblk; i++) mlx5st_cmd_calc_block_sig(&in_mbox[i].block); for (i = 0; i < out_nblk; i++) mlx5st_cmd_calc_block_sig(&out_mbox[i].block); MLX5_SET(cmd_queue_entry, cmd, signature, 0); MLX5_SET(cmd_queue_entry, cmd, signature, ~(xor8_buf(cmd, 0, sizeof(struct mlx5st_cmd_queue_entry)) ^ 0x1)); /* Ensure all cmd data (including sig) is visible, then hand to FW */ dma_wmb(); MLX5_SET_ONCE(cmd_queue_entry, cmd, ownership, 1); /* Ring doorbell */ MLX5_SET_MMIO(initial_seg, dev->init_seg, command_doorbell_vector, doorbell); } static void mlx5st_cmd_exec(struct mlx5st_device *dev, void *in, unsigned int ilen, void *out, unsigned int olen) { struct mlx5st_cmd_queue_entry *cmd = dev->cmd_lay; unsigned int out_remain, out_nblk; struct timespec start, now; unsigned int elapsed; unsigned int i; void *cout; mlx5st_cmd_post(dev, cmd, dev->cmd_in_mbox, dev->cmd_out_mbox, in, ilen, olen, 1); out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0; out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE; /* Poll for completion — also process EQ events for PF page requests */ clock_gettime(CLOCK_MONOTONIC, &start); for (;;) { if (!MLX5_GET_ONCE(cmd_queue_entry, cmd, ownership)) break; if (dev->have_eq) mlx5st_process_events(dev); sched_yield(); clock_gettime(CLOCK_MONOTONIC, &now); elapsed = (now.tv_sec - start.tv_sec) * 1000 + (now.tv_nsec - start.tv_nsec) / 1000000; if (elapsed > MLX5_CMD_TIMEOUT_MS) VFIO_FAIL("cmd timeout after %d ms", elapsed); } /* Ensure output data reads happen after ownership is seen clear */ dma_rmb(); /* Verify output signatures when FW has checksums enabled */ if (dev->cmd_sig_enabled) { if (xor8_buf(cmd, 0, sizeof(struct mlx5st_cmd_queue_entry)) != 0xff) VFIO_FAIL("cmd output signature mismatch"); for (i = 0; i < out_nblk; i++) { if (mlx5st_cmd_verify_block_sig( &dev->cmd_out_mbox[i].block)) VFIO_FAIL("cmd output mailbox block %d signature mismatch", i); } } /* Copy output: inline first */ cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); memcpy(out, cout, olen < MLX5_CMD_INLINE_SZ ? olen : MLX5_CMD_INLINE_SZ); /* Copy remaining from output mailbox chain */ if (out_remain > 0) mlx5st_cmd_copy_from_mbox((u8 *)out + MLX5_CMD_INLINE_SZ, dev->cmd_out_mbox, out_remain); /* Check command status */ if (MLX5_GET(enable_hca_out, out, status) != MLX5_CMD_STAT_OK) VFIO_FAIL("%s: status=0x%x syndrome=0x%x", mlx5st_cmd_name(MLX5_GET(enable_hca_in, in, opcode)), MLX5_GET(enable_hca_out, out, status), MLX5_GET(enable_hca_out, out, syndrome)); } static struct mlx5st_cmd_queue_entry * mlx5st_cmd_slot_init(struct mlx5st_device *dev, unsigned int slot, struct mlx5st_mbox_entry *in_mbox, struct mlx5st_mbox_entry *out_mbox) { struct vfio_pci_device *device = dev->device; struct mlx5st_cmd_queue_entry *cmd = &dev->cmd_queue[(slot << dev->cmd_log_stride) / sizeof(struct mlx5st_cmd_queue_entry)]; u64 iova; MLX5_SET(cmd_queue_entry, cmd, type, MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT); iova = to_iova(device, in_mbox); MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_63_32, iova >> 32); MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_31_9, iova >> 9); iova = to_iova(device, out_mbox); MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_63_32, iova >> 32); MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_31_9, iova >> 9); return cmd; } static void mlx5st_cmd_init(struct mlx5st_device *dev) { struct mlx5st_initial_seg __iomem *seg = dev->init_seg; struct vfio_pci_device *device = dev->device; u16 cmdif_rev; u8 log_sz; u64 iova; cmdif_rev = MLX5_GET_MMIO(initial_seg, seg, cmd_interface_rev); VFIO_ASSERT_EQ(cmdif_rev, 5); /* Read command queue geometry from BAR */ log_sz = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_size); dev->cmd_log_stride = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_stride); dev->pages_slot = (1 << log_sz) - 1; VFIO_ASSERT_LE((unsigned int)(1 << log_sz), 32u); VFIO_ASSERT_GE((unsigned int)(1 << dev->cmd_log_stride), (unsigned int)sizeof(struct mlx5st_cmd_queue_entry)); VFIO_ASSERT_LE((unsigned int)((dev->pages_slot + 1) << dev->cmd_log_stride), (unsigned int)sizeof(dev->cmd_queue)); /* Set up slot 0 — regular commands */ dev->cmd_lay = mlx5st_cmd_slot_init(dev, 0, dev->cmd_in_mbox, dev->cmd_out_mbox); /* Set up pages slot — async MANAGE_PAGES */ dev->pages_cmd_lay = mlx5st_cmd_slot_init(dev, dev->pages_slot, dev->pages_in_mbox, dev->pages_out_mbox); /* Write command queue page address to BAR0 */ iova = to_iova(device, dev->cmd_queue); MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_63_32, iova >> 32); MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_31_12, iova >> 12); dev_dbg(device, "Command interface initialized (cmdif_rev=5, log_sz=%u, log_stride=%u, pages_slot=%u)\n", log_sz, dev->cmd_log_stride, dev->pages_slot); } /* * FW pages: bitmap allocator + MANAGE_PAGES */ static void mlx5st_fw_pages_alloc(struct mlx5st_device *dev, unsigned int npages, u64 *iovas) { struct vfio_pci_device *device = dev->device; unsigned int found = 0; unsigned int w, b; u64 word; for (w = 0; w < MAX_FW_PAGES / 64 && found < npages; w++) { word = dev->fw_pages_bitmap[w]; for (b = 0; b < 64 && found < npages; b++) { if (!(word & (1ULL << b))) { unsigned int idx = w * 64 + b; dev->fw_pages_bitmap[w] |= (1ULL << b); iovas[found++] = to_iova(device, dev->fw_pages[idx]); } } } VFIO_ASSERT_EQ(found, npages); dev->fw_pages_given += npages; } static void mlx5st_fw_pages_free(struct mlx5st_device *dev, unsigned int npages, const u64 *iovas) { struct vfio_pci_device *device = dev->device; unsigned int i, idx; u64 off; for (i = 0; i < npages; i++) { off = iovas[i] - to_iova(device, dev->fw_pages); idx = off / MLX5_HW_PAGE_SIZE; VFIO_ASSERT_TRUE(idx < MAX_FW_PAGES); dev->fw_pages_bitmap[idx / 64] &= ~(1ULL << (idx % 64)); } dev->fw_pages_given -= npages; } static void *mlx5st_build_manage_pages_give(u16 func_id, unsigned int npages, const u64 *iovas, unsigned int *out_inlen) { unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in) + npages * 8; unsigned int i; void *in; in = calloc(1, inlen); VFIO_ASSERT_NOT_NULL(in); MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); MLX5_SET(manage_pages_in, in, op_mod, MLX5_MANAGE_PAGES_IN_OP_MOD_ALLOCATION_SUCCESS); MLX5_SET(manage_pages_in, in, function_id, func_id); MLX5_SET(manage_pages_in, in, input_num_entries, npages); for (i = 0; i < npages; i++) MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, iovas[i]); *out_inlen = inlen; return in; } static void mlx5st_fw_pages_give_one(struct mlx5st_device *dev, u16 func_id, unsigned int npages, u64 *iovas) { u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {}; unsigned int inlen; void *in; in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen); mlx5st_cmd_exec(dev, in, inlen, out, sizeof(out)); free(in); } static void mlx5st_fw_pages_give(struct mlx5st_device *dev, u16 func_id, unsigned int npages) { unsigned int remaining = npages; u64 *iovas; if (!npages) return; iovas = calloc(npages, sizeof(u64)); VFIO_ASSERT_NOT_NULL(iovas); mlx5st_fw_pages_alloc(dev, npages, iovas); /* Batch into chunks that fit in one mailbox */ for (unsigned int off = 0; remaining > 0;) { unsigned int batch = remaining < MAX_FW_PAGES_PER_CMD ? remaining : MAX_FW_PAGES_PER_CMD; mlx5st_fw_pages_give_one(dev, func_id, batch, iovas + off); off += batch; remaining -= batch; } dev_dbg(dev->device, "MANAGE_PAGES GIVE: %d pages to func_id=%u\n", npages, func_id); free(iovas); } static void mlx5st_fw_pages_satisfy(struct mlx5st_device *dev, int boot) { u32 qo[MLX5_ST_SZ_DW(query_pages_out)] = {}; u32 qi[MLX5_ST_SZ_DW(query_pages_in)] = {}; u16 func_id; int npages; MLX5_SET(query_pages_in, qi, opcode, MLX5_CMD_OP_QUERY_PAGES); MLX5_SET(query_pages_in, qi, op_mod, boot ? 0x01 : 0x02); mlx5st_cmd_exec(dev, qi, sizeof(qi), qo, sizeof(qo)); npages = MLX5_GET(query_pages_out, qo, num_pages); func_id = MLX5_GET(query_pages_out, qo, function_id); dev_dbg(dev->device, "QUERY_PAGES (%s): %d pages (func_id=%u)\n", boot ? "boot" : "init", npages, func_id); if (npages > 0) { dev->fw_func_id = func_id; mlx5st_fw_pages_give(dev, func_id, npages); } } /* * Async MANAGE_PAGES on the pages command slot. * * On PFs, firmware sends PAGE_REQUEST events via the EQ during command * execution. We must respond with MANAGE_PAGES on a second command slot * before the first (regular) command can complete. */ static void mlx5st_pages_slot_post(struct mlx5st_device *dev, void *in, unsigned int ilen, unsigned int olen) { mlx5st_cmd_post(dev, dev->pages_cmd_lay, dev->pages_in_mbox, dev->pages_out_mbox, in, ilen, olen, 1 << dev->pages_slot); } static void mlx5st_pages_slot_give(struct mlx5st_device *dev, u16 func_id, unsigned int npages) { unsigned int inlen; u64 *iovas; void *in; iovas = calloc(npages, sizeof(u64)); VFIO_ASSERT_NOT_NULL(iovas); mlx5st_fw_pages_alloc(dev, npages, iovas); in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen); free(iovas); mlx5st_pages_slot_post(dev, in, inlen, MLX5_ST_SZ_BYTES(manage_pages_out)); dev->pages_slot_in_use = true; dev->pages_slot_is_reclaim = false; free(in); dev_dbg(dev->device, "PAGE_REQUEST: %d pages given async to func_id=%u\n", npages, func_id); } static void mlx5st_pages_slot_reclaim(struct mlx5st_device *dev, u16 func_id, unsigned int npages) { unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); unsigned int outlen = MLX5_ST_SZ_BYTES(manage_pages_out) + npages * 8; void *in; in = calloc(1, inlen); VFIO_ASSERT_NOT_NULL(in); MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); MLX5_SET(manage_pages_in, in, op_mod, MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES); MLX5_SET(manage_pages_in, in, function_id, func_id); MLX5_SET(manage_pages_in, in, input_num_entries, npages); mlx5st_pages_slot_post(dev, in, inlen, outlen); dev->pages_slot_in_use = true; dev->pages_slot_is_reclaim = true; dev->pages_reclaim_npages = npages; free(in); dev_dbg(dev->device, "PAGE_REQUEST: reclaim %d pages async from func_id=%u\n", npages, func_id); } static void mlx5st_pages_slot_kick(struct mlx5st_device *dev) { unsigned int batch; if (dev->pages_slot_in_use) return; if (dev->pages_pending_give) { batch = dev->pages_pending_give < MAX_FW_PAGES_PER_CMD ? dev->pages_pending_give : MAX_FW_PAGES_PER_CMD; dev->pages_pending_give -= batch; mlx5st_pages_slot_give(dev, dev->pages_pending_func_id, batch); } else if (dev->pages_pending_reclaim) { batch = dev->pages_pending_reclaim < MAX_FW_PAGES_PER_CMD ? dev->pages_pending_reclaim : MAX_FW_PAGES_PER_CMD; dev->pages_pending_reclaim -= batch; mlx5st_pages_slot_reclaim(dev, dev->pages_pending_func_id, batch); } } static void mlx5st_fw_pages_give_async(struct mlx5st_device *dev, u16 func_id, unsigned int npages) { if (!npages) return; dev->pages_pending_give += npages; dev->pages_pending_func_id = func_id; mlx5st_pages_slot_kick(dev); } static void mlx5st_fw_pages_reclaim_async(struct mlx5st_device *dev, u16 func_id, unsigned int npages) { dev->pages_pending_reclaim += npages; dev->pages_pending_func_id = func_id; mlx5st_pages_slot_kick(dev); } static void mlx5st_pages_slot_complete(struct mlx5st_device *dev) { struct mlx5st_cmd_queue_entry *cmd = dev->pages_cmd_lay; void *cout; dma_rmb(); cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); if (MLX5_GET(enable_hca_out, cout, status) != MLX5_CMD_STAT_OK) VFIO_FAIL("async MANAGE_PAGES failed: status=0x%x syndrome=0x%x", MLX5_GET(enable_hca_out, cout, status), MLX5_GET(enable_hca_out, cout, syndrome)); if (dev->pages_slot_is_reclaim) { unsigned int outlen = MLX5_ST_SZ_BYTES(manage_pages_out) + dev->pages_reclaim_npages * 8; unsigned int num_claimed; unsigned int i; void *out; u64 *iovas; out = calloc(1, outlen); iovas = calloc(dev->pages_reclaim_npages, sizeof(u64)); VFIO_ASSERT_NOT_NULL(out); VFIO_ASSERT_NOT_NULL(iovas); /* Copy inline output */ memcpy(out, cout, MLX5_CMD_INLINE_SZ); if (outlen > MLX5_CMD_INLINE_SZ) mlx5st_cmd_copy_from_mbox( (u8 *)out + MLX5_CMD_INLINE_SZ, dev->pages_out_mbox, outlen - MLX5_CMD_INLINE_SZ); num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); for (i = 0; i < num_claimed; i++) iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas, i); mlx5st_fw_pages_free(dev, num_claimed, iovas); dev_dbg(dev->device, "PAGE_REQUEST: reclaimed %d pages\n", num_claimed); free(iovas); free(out); } dev->pages_slot_in_use = false; mlx5st_pages_slot_kick(dev); } /* * UAR alloc/dealloc */ static void mlx5st_alloc_uar(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {}; MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->uar_page = MLX5_GET(alloc_uar_out, out, uar); dev->uar_base = (u8 __iomem*)dev->bar0 + dev->uar_page * MLX5_HW_PAGE_SIZE; dev->uar_bf_offset = MLX5_BF_OFFSET; dev_dbg(dev->device, "Allocated UAR page_id=%u, doorbell offset=0x%x\n", dev->uar_page, dev->uar_page * MLX5_HW_PAGE_SIZE + MLX5_BF_OFFSET); } static void mlx5st_dealloc_uar(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)] = {}; u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {}; MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR); MLX5_SET(dealloc_uar_in, in, uar, dev->uar_page); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } /* * EQ infrastructure */ static struct mlx5st_eqe *mlx5st_eq_get_eqe(struct mlx5st_device *dev, u32 cc) { u32 ci = dev->eq_cons_index + cc; struct mlx5st_eqe *eqe = &dev->eq_buf[ci % EQ_NENT]; u8 owner = MLX5_GET_ONCE(eqe, eqe, owner); u8 expected = !!(ci & EQ_NENT); if (owner != expected) return NULL; dma_rmb(); return eqe; } static void mlx5st_eq_update_ci(struct mlx5st_device *dev, u32 cc, bool arm) { u32 val; dev->eq_cons_index += cc; val = (dev->eq_cons_index & 0xffffff) | (dev->eqn << 24); iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET + (arm ? 0 : 8)); } static void mlx5st_create_eq(struct mlx5st_device *dev) { struct vfio_pci_device *device = dev->device; u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {}; u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {}; struct mlx5_ifc_eqc_bits *eqc; unsigned int i; __be64 *pas; /* Initialize EQE owner bits */ for (i = 0; i < EQ_NENT; i++) { struct mlx5st_eqe *eqe = &dev->eq_buf[i]; MLX5_SET_ONCE(eqe, eqe, owner, 1); } MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); /* Subscribe to CMD completions and PAGE_REQUEST events */ MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, 0, (1ULL << MLX5_EVENT_TYPE_CMD) | (1ULL << MLX5_EVENT_TYPE_PAGE_REQUEST)); eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); MLX5_SET(eqc, eqc, log_eq_size, LOG_EQ_SIZE); MLX5_SET(eqc, eqc, uar_page, dev->uar_page); pas = MLX5_ADDR_OF(create_eq_in, in, pas); VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->eq_buf, pas), 0u); MLX5_SET(eqc, eqc, log_page_size, 0); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->eqn = MLX5_GET(create_eq_out, out, eq_number); dev->eq_cons_index = 0; mlx5st_eq_update_ci(dev, 0, 0); dev->have_eq = true; dev_dbg(device, "Created EQ: eqn=%u, %d entries (CMD+PAGE_REQUEST)\n", dev->eqn, EQ_NENT); } static void mlx5st_destroy_eq(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); MLX5_SET(destroy_eq_in, in, eq_number, dev->eqn); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } /* * Drain all pending EQ events. Dispatches PAGE_REQUEST to the async pages * slot and CMD completions to the pages slot completion handler. */ static void mlx5st_process_events(struct mlx5st_device *dev) { struct mlx5st_eqe *eqe; u32 cc = 0; while ((eqe = mlx5st_eq_get_eqe(dev, cc))) { u8 type = MLX5_GET(eqe, eqe, event_type); switch (type) { case MLX5_EVENT_TYPE_PAGE_REQUEST: { void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data); u16 func_id = MLX5_GET(pages_req_event, evdata, function_id); s32 npages = (s32)MLX5_GET(pages_req_event, evdata, num_pages); /* * The selftest doesn't use more than one func_id so a * simple counter approach is possible. */ if (dev->pages_func_id_seen) VFIO_ASSERT_EQ(func_id, dev->pages_pending_func_id); dev->pages_func_id_seen = true; if (npages > 0) mlx5st_fw_pages_give_async(dev, func_id, npages); else if (npages < 0) mlx5st_fw_pages_reclaim_async(dev, func_id, -npages); break; } case MLX5_EVENT_TYPE_CMD: { void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data); u32 vector = MLX5_GET(cmd_inter_comp_event, evdata, command_completion_vector); if (vector & (1U << dev->pages_slot)) mlx5st_pages_slot_complete(dev); break; } default: break; } cc++; } if (cc) mlx5st_eq_update_ci(dev, cc, 0); } /* * MSI EQ — dedicated EQ for CQ completion events that fires MSI-X. * Separate from the cmd/pages EQ so that only CQ completions (from * send_msi or memcpy) trigger the interrupt vector. */ static void mlx5st_msi_eq_drain(struct mlx5st_device *dev) { u32 cc = 0; u32 val; while (cc < MSI_EQ_NENT) { u32 ci = dev->msi_eq_cons_index + cc; struct mlx5st_eqe *eqe = &dev->msi_eq_buf[ci % MSI_EQ_NENT]; if (MLX5_GET_ONCE(eqe, eqe, owner) != !!(ci & MSI_EQ_NENT)) break; cc++; } /* Update consumer index and re-arm for next interrupt */ dev->msi_eq_cons_index += cc; val = (dev->msi_eq_cons_index & 0xffffff) | (dev->msi_eqn << 24); iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET); } static void mlx5st_create_msi_eq(struct mlx5st_device *dev) { struct vfio_pci_device *device = dev->device; u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {}; u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {}; struct mlx5_ifc_eqc_bits *eqc; unsigned int i; __be64 *pas; /* Initialize EQE owner bits */ for (i = 0; i < MSI_EQ_NENT; i++) { struct mlx5st_eqe *eqe = &dev->msi_eq_buf[i]; MLX5_SET_ONCE(eqe, eqe, owner, 1); } MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); /* * No event_bitmask — completion events are routed to this EQ via * the CQ's c_eqn field, not through CREATE_EQ subscription. */ eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); MLX5_SET(eqc, eqc, log_eq_size, LOG_MSI_EQ_SIZE); MLX5_SET(eqc, eqc, uar_page, dev->uar_page); MLX5_SET(eqc, eqc, intr, MSI_VECTOR); pas = MLX5_ADDR_OF(create_eq_in, in, pas); VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->msi_eq_buf, pas), 0u); MLX5_SET(eqc, eqc, log_page_size, 0); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->msi_eqn = MLX5_GET(create_eq_out, out, eq_number); dev->msi_eq_cons_index = 0; dev->have_msi_eq = true; mlx5st_msi_eq_drain(dev); dev_dbg(device, "Created MSI EQ: eqn=%u, %d entries (COMP), vector=%d\n", dev->msi_eqn, MSI_EQ_NENT, MSI_VECTOR); } static void mlx5st_destroy_msi_eq(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); MLX5_SET(destroy_eq_in, in, eq_number, dev->msi_eqn); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } /* * HCA init / teardown */ #define FW_INIT_TIMEOUT_MS 120000 #define FW_INIT_WAIT_MS 200 static void mlx5st_wait_fw_init(struct mlx5st_device *dev) { struct timespec start, now; unsigned int elapsed; clock_gettime(CLOCK_MONOTONIC, &start); while (MLX5_GET_MMIO(initial_seg, dev->init_seg, initializing)) { usleep(FW_INIT_WAIT_MS * 1000); clock_gettime(CLOCK_MONOTONIC, &now); elapsed = (now.tv_sec - start.tv_sec) * 1000 + (now.tv_nsec - start.tv_nsec) / 1000000; if (elapsed > FW_INIT_TIMEOUT_MS) VFIO_FAIL("FW init timeout after %d ms", elapsed); } } static void mlx5st_set_issi(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(set_issi_out)] = {}; u32 in[MLX5_ST_SZ_DW(set_issi_in)] = {}; MLX5_SET(set_issi_in, in, opcode, MLX5_CMD_OP_SET_ISSI); MLX5_SET(set_issi_in, in, current_issi, 1); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev_dbg(dev->device, "SET_ISSI: OK (issi=1)\n"); } static void mlx5st_set_hca_caps(struct mlx5st_device *dev) { u32 qout[MLX5_ST_SZ_DW(query_hca_cap_out)] = {}; u32 qin[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; u32 sout[MLX5_ST_SZ_DW(set_hca_cap_out)] = {}; u32 sin[MLX5_ST_SZ_DW(set_hca_cap_in)] = {}; struct mlx5_ifc_cmd_hca_cap_bits *set_hca_cap; u32 max_checksum; /* Query max caps to learn cmdif_checksum support */ MLX5_SET(query_hca_cap_in, qin, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); MLX5_SET(query_hca_cap_in, qin, op_mod, (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1)); mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout)); max_checksum = MLX5_GET( cmd_hca_cap, MLX5_ADDR_OF(query_hca_cap_out, qout, capability), cmdif_checksum); /* Query current caps as base for SET */ memset(qout, 0, sizeof(qout)); MLX5_SET(query_hca_cap_in, qin, op_mod, (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) | HCA_CAP_OPMOD_GET_CUR); mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout)); set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, sin, capability); memcpy(set_hca_cap, MLX5_ADDR_OF(query_hca_cap_out, qout, capability), MLX5_ST_SZ_BYTES(cmd_hca_cap)); MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, max_checksum); MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, 0); MLX5_SET(set_hca_cap_in, sin, opcode, MLX5_CMD_OP_SET_HCA_CAP); MLX5_SET(set_hca_cap_in, sin, op_mod, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1); mlx5st_cmd_exec(dev, sin, sizeof(sin), sout, sizeof(sout)); dev->cmd_sig_enabled = max_checksum == 0x3; dev_dbg(dev->device, "SET_HCA_CAP: OK (cmdif_checksum=%u)\n", max_checksum); } static void mlx5st_hca_init(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {}; u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {}; mlx5st_wait_fw_init(dev); dev_dbg(dev->device, "Firmware ready\n"); MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev_dbg(dev->device, "ENABLE_HCA: OK\n"); mlx5st_set_issi(dev); mlx5st_fw_pages_satisfy(dev, 1); mlx5st_set_hca_caps(dev); mlx5st_fw_pages_satisfy(dev, 0); memset(in, 0, sizeof(in)); memset(out, 0, sizeof(out)); MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev_dbg(dev->device, "INIT_HCA: OK\n"); /* * Create EQ immediately after INIT_HCA so PAGE_REQUEST events * are captured during all subsequent commands. */ mlx5st_alloc_uar(dev); mlx5st_create_eq(dev); } static void mlx5st_disable_hca(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {}; u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {}; MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } static void mlx5st_fw_pages_reclaim(struct mlx5st_device *dev, u16 func_id) { unsigned int npages = dev->fw_pages_given; unsigned int total_claimed = 0; while (npages > 0) { unsigned int batch = npages < MAX_FW_PAGES_PER_CMD ? npages : MAX_FW_PAGES_PER_CMD; unsigned int outlen = MLX5_ST_SZ_BYTES(manage_pages_out) + batch * 8; unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); unsigned int num_claimed; unsigned int i; void *in, *out; u64 *iovas; in = calloc(1, inlen); out = calloc(1, outlen); iovas = calloc(batch, sizeof(u64)); VFIO_ASSERT_NOT_NULL(in); VFIO_ASSERT_NOT_NULL(out); VFIO_ASSERT_NOT_NULL(iovas); MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); MLX5_SET(manage_pages_in, in, op_mod, MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES); MLX5_SET(manage_pages_in, in, function_id, func_id); MLX5_SET(manage_pages_in, in, input_num_entries, batch); mlx5st_cmd_exec(dev, in, inlen, out, outlen); num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); for (i = 0; i < num_claimed; i++) iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas, i); mlx5st_fw_pages_free(dev, num_claimed, iovas); total_claimed += num_claimed; npages -= num_claimed; free(iovas); free(in); free(out); if (!num_claimed && !dev->fw_pages_given) break; if (!num_claimed) VFIO_FAIL("MANAGE_PAGES RECLAIM: FW returned 0 but %d pages still given", dev->fw_pages_given); } dev_dbg(dev->device, "MANAGE_PAGES RECLAIM: %d pages (%d still given)\n", total_claimed, dev->fw_pages_given); } static void mlx5st_hca_teardown(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {}; u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {}; /* Drain async pages slot, then stop EQ processing */ while (dev->pages_slot_in_use) { if (!MLX5_GET_ONCE(cmd_queue_entry, dev->pages_cmd_lay, ownership)) mlx5st_pages_slot_complete(dev); else sched_yield(); } dev->have_eq = false; if (dev->eqn) { mlx5st_destroy_eq(dev); dev->eqn = 0; } if (dev->uar_page) { mlx5st_dealloc_uar(dev); dev->uar_page = 0; } dev_dbg(dev->device, " hca_teardown: TEARDOWN_HCA\n"); MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); MLX5_SET(teardown_hca_in, in, profile, MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (dev->fw_pages_given > 0) { dev_dbg(dev->device, " hca_teardown: reclaim %d pages\n", dev->fw_pages_given); mlx5st_fw_pages_reclaim(dev, dev->fw_func_id); } dev_dbg(dev->device, " hca_teardown: DISABLE_HCA\n"); mlx5st_disable_hca(dev); } /* * Query capabilities */ static void mlx5st_query_fl_caps(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {}; u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; bool fl_roce_en, fl_roce_dis; /* Query RoCE capabilities */ MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); MLX5_SET(query_hca_cap_in, in, op_mod, (MLX5_SET_HCA_CAP_OP_MOD_ROCE << 1) | HCA_CAP_OPMOD_GET_CUR); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); fl_roce_en = MLX5_GET(query_hca_cap_out, out, capability.roce_cap.fl_rc_qp_when_roce_enabled); fl_roce_dis = MLX5_GET(query_hca_cap_out, out, capability.roce_cap.fl_rc_qp_when_roce_disabled); /* Also check general caps */ memset(in, 0, sizeof(in)); memset(out, 0, sizeof(out)); MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); MLX5_SET(query_hca_cap_in, in, op_mod, (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) | HCA_CAP_OPMOD_GET_CUR); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); fl_roce_dis |= MLX5_GET(query_hca_cap_out, out, capability.cmd_hca_cap.fl_rc_qp_when_roce_disabled); dev->fl_supported = fl_roce_en || fl_roce_dis; dev->log_max_msg = MLX5_GET(query_hca_cap_out, out, capability.cmd_hca_cap.log_max_msg); dev_dbg(dev->device, "HCA capabilities: fl_roce_enabled=%d fl_roce_disabled=%d log_max_msg=%u\n", fl_roce_en, fl_roce_dis, dev->log_max_msg); VFIO_ASSERT_TRUE(dev->fl_supported, "Force-loopback not supported on this device"); } /* * Resource allocation */ static void mlx5st_alloc_pd(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->pdn = MLX5_GET(alloc_pd_out, out, pd); dev_dbg(dev->device, "Allocated PD pdn=%u\n", dev->pdn); } static void mlx5st_dealloc_pd(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {}; u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {}; MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD); MLX5_SET(dealloc_pd_in, in, pd, dev->pdn); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } static void mlx5st_create_mkey(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(create_mkey_out)] = {}; u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; struct mlx5_ifc_mkc_bits *mkc; MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); MLX5_SET(mkc, mkc, length64, 1); MLX5_SET(mkc, mkc, pd, dev->pdn); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, lr, 1); MLX5_SET(mkc, mkc, lw, 1); MLX5_SET(mkc, mkc, rw, 1); MLX5_SET(mkc, mkc, rr, 1); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->mkey_index = MLX5_GET(create_mkey_out, out, mkey_index); dev->global_lkey = mlx5st_idx_to_mkey(dev->mkey_index); dev->global_rkey = dev->global_lkey; dev_dbg(dev->device, "Created global PA-mode MKEY: lkey=0x%x\n", dev->global_lkey); } static void mlx5st_destroy_mkey(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); MLX5_SET(destroy_mkey_in, in, mkey_index, dev->mkey_index); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } /* * CQ create/destroy */ static void mlx5st_create_cq(struct mlx5st_device *dev) { struct vfio_pci_device *device = dev->device; u64 in[MLX5_ST_SZ_QW(create_cq_in) + 1] = {}; u32 out[MLX5_ST_SZ_DW(create_cq_out)] = {}; struct mlx5_ifc_cqc_bits *cqc; unsigned int i; __be64 *pas; /* Initialize CQEs before CREATE_CQ: opcode=0xF, owner=1 */ for (i = 0; i < CQ_CQE_CNT; i++) { struct mlx5st_cqe64 *cqe = &dev->cq_buf[i]; MLX5_SET(cqe64, cqe, opcode, 0xF); MLX5_SET_ONCE(cqe64, cqe, owner, 1); } MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ); cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); MLX5_SET(cqc, cqc, log_cq_size, LOG_CQ_SIZE); MLX5_SET(cqc, cqc, uar_page, dev->uar_page); MLX5_SET(cqc, cqc, c_eqn_or_apu_element, dev->msi_eqn); MLX5_SET(cqc, cqc, cqe_sz, 0); pas = MLX5_ADDR_OF(create_cq_in, in, pas); MLX5_SET(cqc, cqc, page_offset, mlx5st_fill_pas(device, dev->cq_buf, pas)); MLX5_SET(cqc, cqc, log_page_size, 0); MLX5_SET64(cqc, cqc, dbr_addr, to_iova(device, &dev->cq_dbrec)); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->cqn = MLX5_GET(create_cq_out, out, cqn); dev->cq_ci = 0; dev_dbg(device, "Created CQ: cqn=%u, %d entries\n", dev->cqn, CQ_CQE_CNT); } static void mlx5st_destroy_cq(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {}; MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ); MLX5_SET(destroy_cq_in, in, cqn, dev->cqn); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } /* * Arm CQ for event generation. The CQ event delivery state machine is * single-shot: after generating one EQE the CQ enters "Fired" state and * won't generate another until re-armed via ARM_NEXT. Both the CQ doorbell * record and the UAR CQ doorbell register must be written. */ static void mlx5st_arm_cq(struct mlx5st_device *dev) { u32 sn = dev->cq_arm_sn & 3; u32 ci = dev->cq_ci & 0xffffff; u64 doorbell; /* Update CQ doorbell record arm word */ WRITE_ONCE(dev->cq_dbrec.send_counter, cpu_to_be32(sn << 28 | ci)); /* Ring CQ doorbell register, iowrite has an internal dma_wmb() */ doorbell = ((u64)(sn << 28 | ci) << 32) | dev->cqn; iowrite64be(doorbell, (u8 __iomem *)dev->uar_base + MLX5_CQ_DOORBELL_OFFSET); dev->cq_arm_sn++; } /* * QP create/destroy */ static void mlx5st_create_qp(struct mlx5st_device *dev) { struct vfio_pci_device *device = dev->device; u64 in[MLX5_ST_SZ_QW(create_qp_in) + 1] = {}; u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; struct mlx5_ifc_qpc_bits *qpc; __be64 *pas; MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); MLX5_SET(qpc, qpc, st, MLX5_QPC_ST_RC); MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); MLX5_SET(qpc, qpc, pd, dev->pdn); MLX5_SET(qpc, qpc, uar_page, dev->uar_page); MLX5_SET(qpc, qpc, cqn_snd, dev->cqn); MLX5_SET(qpc, qpc, cqn_rcv, dev->cqn); MLX5_SET(qpc, qpc, log_sq_size, LOG_SQ_SIZE); MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg); MLX5_SET(qpc, qpc, rq_type, 0x3); MLX5_SET(qpc, qpc, ts_format, 1); pas = MLX5_ADDR_OF(create_qp_in, in, pas); MLX5_SET(qpc, qpc, page_offset, mlx5st_fill_pas(device, dev->sq_buf, pas)); MLX5_SET(qpc, qpc, log_page_size, 0); MLX5_SET64(qpc, qpc, dbr_addr, to_iova(device, &dev->qp_dbrec)); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev->qpn = MLX5_GET(create_qp_out, out, qpn); dev->sq_pi = 0; dev_dbg(device, "Created QP: qpn=%u, RC, sq=%d wqes\n", dev->qpn, SQ_WQE_CNT); } static void mlx5st_destroy_qp(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(destroy_qp_out)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); MLX5_SET(destroy_qp_in, in, qpn, dev->qpn); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } /* * QP state transitions */ static void mlx5st_qp_rst2init(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(rst2init_qp_out)] = {}; u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc); MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP); MLX5_SET(rst2init_qp_in, in, qpn, dev->qpn); MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); MLX5_SET(qpc, qpc, rre, 1); MLX5_SET(qpc, qpc, rwe, 1); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev_dbg(dev->device, "QP RST->INIT\n"); } static void mlx5st_qp_init2rtr(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(init2rtr_qp_out)] = {}; u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc); MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP); MLX5_SET(init2rtr_qp_in, in, qpn, dev->qpn); MLX5_SET(qpc, qpc, mtu, 3); MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg); MLX5_SET(qpc, qpc, remote_qpn, dev->qpn); MLX5_SET(qpc, qpc, min_rnr_nak, 12); MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); MLX5_SET(qpc, qpc, primary_address_path.fl, 1); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev_dbg(dev->device, "QP INIT->RTR (fl=1)\n"); } static void mlx5st_qp_rtr2rts(struct mlx5st_device *dev) { u32 out[MLX5_ST_SZ_DW(rtr2rts_qp_out)] = {}; u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc); MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP); MLX5_SET(rtr2rts_qp_in, in, qpn, dev->qpn); MLX5_SET(qpc, qpc, log_ack_req_freq, 0); MLX5_SET(qpc, qpc, retry_count, 7); MLX5_SET(qpc, qpc, rnr_retry, 7); MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 14); mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); dev_dbg(dev->device, "QP RTR->RTS\n"); } /* * Post RDMA Write WQE */ static void mlx5st_post_rdma_write(struct mlx5st_device *dev, u64 src_addr, u32 src_lkey, u64 dst_addr, u32 dst_rkey, u32 length, bool signaled) { struct mlx5st_send_wqe *wqe; unsigned int idx; idx = dev->sq_pi % SQ_WQE_CNT; wqe = &dev->sq_buf[idx]; memset(wqe, 0, sizeof(*wqe)); MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, opcode, MLX5_OPCODE_RDMA_WRITE); MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, wqe_index, dev->sq_pi); MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, qp_or_sq, dev->qpn); MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ds, MLX5_RDMA_WRITE_DS); if (signaled) MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ce, MLX5_WQE_CE_CQE_ALWAYS); MLX5_SET64(wqe_raddr_seg, &wqe->raddr, raddr, dst_addr); MLX5_SET(wqe_raddr_seg, &wqe->raddr, rkey, dst_rkey); MLX5_SET(wqe_data_seg, &wqe->data, byte_count, length); MLX5_SET(wqe_data_seg, &wqe->data, lkey, src_lkey); MLX5_SET64(wqe_data_seg, &wqe->data, addr, src_addr); dev->sq_pi++; /* Ensure WQE is visible to device before doorbell record */ dma_wmb(); WRITE_ONCE(dev->qp_dbrec.send_counter, cpu_to_be32(dev->sq_pi & 0xffff)); /* * Ring doorbell: write first 8 bytes of ctrl to UAR BF register, * iowrite has an internal dma_wmb() so the doorbell record will be * visible. */ iowrite64be(be64_to_cpu(*(__be64 *)wqe), (u8 __iomem *)dev->uar_base + dev->uar_bf_offset); dev->uar_bf_offset ^= MLX5_BF_SIZE; } /* * Poll CQ */ static int mlx5st_poll_cq_batch(struct mlx5st_device *dev, unsigned int max_cqe) { unsigned int polled = 0; while (polled < max_cqe) { unsigned int idx = dev->cq_ci % CQ_CQE_CNT; struct mlx5st_cqe64 *cqe = &dev->cq_buf[idx]; u8 owner, opcode; owner = MLX5_GET_ONCE(cqe64, cqe, owner); if (owner != ((dev->cq_ci >> LOG_CQ_SIZE) & 1)) break; dma_rmb(); opcode = MLX5_GET(cqe64, cqe, opcode); dev->cq_ci++; WRITE_ONCE(dev->cq_dbrec.recv_counter, cpu_to_be32(dev->cq_ci & 0xffffff)); if (opcode == MLX5_CQE_REQ) { dev->sq_ci = (u16)(MLX5_GET(cqe64, cqe, wqe_counter) + 1); polled++; continue; } if (opcode == MLX5_CQE_REQ_ERR || opcode == MLX5_CQE_RESP_ERR) { dev_dbg(dev->device, "CQE error: opcode=0x%x syndrome=0x%x vendor=0x%x\n", opcode, MLX5_GET(cqe64, cqe, error_syndrome.syndrome), MLX5_GET(cqe64, cqe, error_syndrome.vendor_error_syndrome)); return -1; } dev_err(dev->device, "CQE unexpected opcode=0x%x\n", opcode); return -1; } return polled; } static int mlx5st_poll_cq(struct mlx5st_device *dev, unsigned int timeout_ms) { struct timespec start, now; unsigned int elapsed; int ret; clock_gettime(CLOCK_MONOTONIC, &start); for (;;) { ret = mlx5st_poll_cq_batch(dev, 1); if (ret < 0) return -1; if (ret > 0) return 0; if (dev->have_eq) mlx5st_process_events(dev); clock_gettime(CLOCK_MONOTONIC, &now); elapsed = (now.tv_sec - start.tv_sec) * 1000 + (now.tv_nsec - start.tv_nsec) / 1000000; if (elapsed > timeout_ms) { dev_err(dev->device, "CQ poll timeout after %u ms\n", timeout_ms); return -1; } } } /* * Data path setup/teardown helpers */ static void mlx5st_setup_datapath(struct mlx5st_device *dev) { mlx5st_create_cq(dev); mlx5st_create_qp(dev); mlx5st_qp_rst2init(dev); mlx5st_qp_init2rtr(dev); mlx5st_qp_rtr2rts(dev); } static void mlx5st_teardown_datapath(struct mlx5st_device *dev) { if (dev->qpn) { mlx5st_destroy_qp(dev); dev->qpn = 0; } if (dev->cqn) { mlx5st_destroy_cq(dev); dev->cqn = 0; } dev->sq_pi = 0; dev->sq_ci = 0; dev->cq_arm_sn = 0; memset(&dev->qp_dbrec, 0, sizeof(dev->qp_dbrec)); memset(&dev->cq_dbrec, 0, sizeof(dev->cq_dbrec)); } /* * memcpy callbacks */ #define MLX5ST_MEMCPY_TIMEOUT_MS 60000 static void mlx5st_memcpy_start(struct vfio_pci_device *device, iova_t src, iova_t dst, u64 size, u64 count) { struct mlx5st_device *dev = to_mlx5st(device); u64 i; for (i = 0; i < count; i++) { bool signaled = (i == count - 1); mlx5st_post_rdma_write(dev, src, dev->global_lkey, dst, dev->global_rkey, size, signaled); } } static int mlx5st_memcpy_wait(struct vfio_pci_device *device) { struct mlx5st_device *dev = to_mlx5st(device); int ret; ret = mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS); if (ret) { /* * CQE error puts the QP in error state. Rebuild the data path * so subsequent operations can succeed. */ mlx5st_teardown_datapath(dev); mlx5st_setup_datapath(dev); } return ret; } /* * send_msi callback — trigger CQE -> EQE -> MSI-X via a small RDMA Write. * * Both the CQ and MSI EQ use single-shot arming: the CQ must be armed so the * CQE generates an EQE, and the MSI EQ must be armed so the EQE fires MSI-X. */ static void mlx5st_send_msi(struct vfio_pci_device *device) { struct mlx5st_device *dev = to_mlx5st(device); /* Drain accumulated MSI EQ events and re-arm for next interrupt */ mlx5st_msi_eq_drain(dev); /* Arm CQ so the next CQE generates an EQE on the MSI EQ */ mlx5st_arm_cq(dev); /* Post a signaled RDMA Write to trigger CQE -> EQE -> MSI-X */ mlx5st_post_rdma_write(dev, to_iova(device, &dev->send_msi_src), dev->global_lkey, to_iova(device, &dev->send_msi_dst), dev->global_rkey, sizeof(dev->send_msi_src), true); /* Consume the CQE to avoid stale completions */ VFIO_ASSERT_EQ(mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS), 0); } /* * Driver ops callbacks */ static void mlx5st_init(struct vfio_pci_device *device) { struct mlx5st_device *dev = to_mlx5st(device); iova_t iova_align = device->driver.region.iova % __alignof__(struct mlx5st_device); VFIO_ASSERT_GE(device->driver.region.size, sizeof(*dev)); VFIO_ASSERT_EQ(iova_align, 0); memset(dev, 0, sizeof(*dev)); dev->device = device; dev->bar0 = device->bars[0].vaddr; dev->init_seg = dev->bar0; vfio_pci_config_writew(device, PCI_COMMAND, vfio_pci_config_readw(device, PCI_COMMAND) | PCI_COMMAND_MASTER); mlx5st_wait_fw_init(dev); mlx5st_cmd_init(dev); mlx5st_hca_init(dev); mlx5st_query_fl_caps(dev); mlx5st_alloc_pd(dev); mlx5st_create_mkey(dev); /* MSI EQ must be created before CQ so CQ can reference its eqn */ mlx5st_create_msi_eq(dev); mlx5st_setup_datapath(dev); vfio_pci_msix_enable(device, MSI_VECTOR, 1); device->driver.msi = MSI_VECTOR; device->driver.max_memcpy_size = 1ULL << dev->log_max_msg; device->driver.max_memcpy_count = SQ_WQE_CNT - 1; dev_dbg(device, "mlx5 driver initialized\n"); } static void mlx5st_remove(struct vfio_pci_device *device) { struct mlx5st_device *dev = to_mlx5st(device); vfio_pci_msix_disable(device); mlx5st_teardown_datapath(dev); if (dev->have_msi_eq) { mlx5st_destroy_msi_eq(dev); dev->have_msi_eq = false; } dev_dbg(device, "teardown: destroy_mkey\n"); if (dev->mkey_index) { mlx5st_destroy_mkey(dev); dev->mkey_index = 0; } dev_dbg(device, "teardown: dealloc_pd\n"); if (dev->pdn) { mlx5st_dealloc_pd(dev); dev->pdn = 0; } dev_dbg(device, "teardown: hca_teardown\n"); mlx5st_hca_teardown(dev); vfio_pci_config_writew(device, PCI_COMMAND, vfio_pci_config_readw(device, PCI_COMMAND) & ~(u16)PCI_COMMAND_MASTER); dev_dbg(device, "Teardown complete\n"); } struct vfio_pci_driver_ops mlx5st_ops = { .name = "mlx5", .region_size = sizeof(struct mlx5st_device), .probe = mlx5st_probe, .init = mlx5st_init, .remove = mlx5st_remove, .memcpy_start = mlx5st_memcpy_start, .memcpy_wait = mlx5st_memcpy_wait, .send_msi = mlx5st_send_msi, };