diff options
Diffstat (limited to 'drivers/net/ethernet/meta')
26 files changed, 3851 insertions, 130 deletions
diff --git a/drivers/net/ethernet/meta/Kconfig b/drivers/net/ethernet/meta/Kconfig index ca5c7ac2a5bc..f8b5fe7e31b7 100644 --- a/drivers/net/ethernet/meta/Kconfig +++ b/drivers/net/ethernet/meta/Kconfig @@ -35,4 +35,17 @@ config FBNIC To compile this driver as a module, choose M here. The module will be called fbnic. MSI-X interrupt support is required. +config MPNIC + tristate "Meta Platforms Network Interface Controller" + depends on 64BIT || COMPILE_TEST + depends on !S390 + depends on PCI_MSI + select PAGE_POOL + help + This driver supports the Meta Platforms Network Interface + Controller. + + To compile this driver as a module, choose M here. The module + will be called mpnic. + endif # NET_VENDOR_META diff --git a/drivers/net/ethernet/meta/Makefile b/drivers/net/ethernet/meta/Makefile index 88804f3de963..633973419c21 100644 --- a/drivers/net/ethernet/meta/Makefile +++ b/drivers/net/ethernet/meta/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_FBNIC) += fbnic/ +obj-$(CONFIG_MPNIC) += mpnic/ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h index d0715695c43e..1f9fbd71447f 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic.h @@ -57,6 +57,7 @@ struct fbnic_dev { u64 dsn; u32 mps; u32 readrq; + u32 rx_cqe_nsecs; u8 relaxed_ord; /* Local copy of the devices TCAM */ @@ -160,7 +161,12 @@ void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val); static inline bool fbnic_bmc_present(struct fbnic_dev *fbd) { - return fbd->fw_cap.bmc_present; + return test_bit(FBNIC_FW_CAP_F_BMC_PRESENT, &fbd->fw_cap.state); +} + +static inline bool fbnic_bmc_all_multi(struct fbnic_dev *fbd) +{ + return test_bit(FBNIC_FW_CAP_F_BMC_ALL_MULTI, &fbd->fw_cap.state); } static inline bool fbnic_init_failure(struct fbnic_dev *fbd) @@ -250,6 +256,7 @@ int fbnic_csr_regs_len(struct fbnic_dev *fbd); void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm); void fbnic_config_rx_frames(struct fbnic_napi_vector *nv); +void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd); enum fbnic_boards { fbnic_board_asic diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index baba3471bf5a..cbb25d11638c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -109,34 +109,19 @@ enum { /* Rx Buffer Descriptor Format * - * The layout of this can vary depending on the page size of the system. + * Buffer descriptors describe 4 KiB device pages. A posted page larger than + * 4 KiB is represented by consecutive device-page descriptors. * - * If the page size is 4K then the layout will simply consist of ID for - * the 16 most significant bits, and the lower 46 are essentially the page - * address with the lowest 12 bits being reserved 0 due to the fact that - * a page will be aligned. - * - * If the page size is larger than 4K then the lower n bits of the ID and - * page address will be reserved for the fragment ID. This fragment will - * be 4K in size and will be used to index both the DMA address and the ID - * by the same amount. + * The address field stores the 4 KiB-aligned DMA address. The ID field stores + * the software page ID, with the low n bits used as the device-page ID when a + * posted page spans multiple device pages. The driver increments both the + * address and ID by one device page for each descriptor belonging to a posted + * page. */ #define FBNIC_BD_DESC_ADDR_MASK DESC_GENMASK(45, 12) #define FBNIC_BD_DESC_ID_MASK DESC_GENMASK(63, 48) -#define FBNIC_BD_FRAG_SIZE \ +#define FBNIC_BD_PAGE_SIZE \ (FBNIC_BD_DESC_ADDR_MASK & ~(FBNIC_BD_DESC_ADDR_MASK - 1)) -#define FBNIC_BD_FRAG_COUNT \ - (PAGE_SIZE / FBNIC_BD_FRAG_SIZE) -#define FBNIC_BD_FRAG_ADDR_MASK \ - (FBNIC_BD_DESC_ADDR_MASK & \ - ~(FBNIC_BD_DESC_ADDR_MASK * FBNIC_BD_FRAG_COUNT)) -#define FBNIC_BD_FRAG_ID_MASK \ - (FBNIC_BD_DESC_ID_MASK & \ - ~(FBNIC_BD_DESC_ID_MASK * FBNIC_BD_FRAG_COUNT)) -#define FBNIC_BD_PAGE_ADDR_MASK \ - (FBNIC_BD_DESC_ADDR_MASK & ~FBNIC_BD_FRAG_ADDR_MASK) -#define FBNIC_BD_PAGE_ID_MASK \ - (FBNIC_BD_DESC_ID_MASK & ~FBNIC_BD_FRAG_ID_MASK) /* Rx Completion Queue Descriptors */ #define FBNIC_RCD_TYPE_MASK DESC_GENMASK(62, 61) @@ -151,9 +136,6 @@ enum { /* Address/Length Completion Descriptors */ #define FBNIC_RCD_AL_BUFF_ID_MASK DESC_GENMASK(15, 0) -#define FBNIC_RCD_AL_BUFF_FRAG_MASK (FBNIC_BD_FRAG_COUNT - 1) -#define FBNIC_RCD_AL_BUFF_PAGE_MASK \ - (FBNIC_RCD_AL_BUFF_ID_MASK & ~FBNIC_RCD_AL_BUFF_FRAG_MASK) #define FBNIC_RCD_AL_BUFF_LEN_MASK DESC_GENMASK(28, 16) #define FBNIC_RCD_AL_BUFF_OFF_MASK DESC_GENMASK(43, 32) #define FBNIC_RCD_AL_PAGE_FIN DESC_BIT(60) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c index 6edfa0aa69f1..f9e879be254d 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -181,8 +181,8 @@ static int fbnic_dbg_tcq_desc_seq_show(struct seq_file *s, void *v) static int fbnic_dbg_bdq_desc_seq_show(struct seq_file *s, void *v) { struct fbnic_ring *ring = s->private; + unsigned int i, desc_count; char hdr[80]; - int i; /* Generate header on first entry */ fbnic_dbg_ring_show(s); @@ -197,7 +197,8 @@ static int fbnic_dbg_bdq_desc_seq_show(struct seq_file *s, void *v) return 0; } - for (i = 0; i < (ring->size_mask + 1) * FBNIC_BD_FRAG_COUNT; i++) { + desc_count = (ring->size_mask + 1) * fbnic_bd_page_count(ring); + for (i = 0; i < desc_count; i++) { u64 bd = le64_to_cpu(ring->desc[i]); seq_printf(s, "%04x %#04llx %#014llx\n", i, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c index 546e1c12d287..0b653c4c20a7 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c @@ -10,6 +10,7 @@ #include "fbnic.h" #include "fbnic_fw.h" #include "fbnic_tlv.h" +#include "fbnic_txrx.h" #define FBNIC_SN_STR_LEN 24 @@ -648,6 +649,7 @@ struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev) fbd->mps = pcie_get_mps(pdev); fbd->readrq = pcie_get_readrq(pdev); fbd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev); + fbd->rx_cqe_nsecs = FBNIC_RX_CQE_NSECS_DEFAULT; fbd->mac_addr_boundary = FBNIC_RPC_TCAM_MACDA_DEFAULT_BOUNDARY; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 76e9a545bb16..e774740fcb5d 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -246,6 +246,7 @@ static int fbnic_get_coalesce(struct net_device *netdev, ec->tx_coalesce_usecs = fbn->tx_usecs; ec->rx_coalesce_usecs = fbn->rx_usecs; ec->rx_max_coalesced_frames = fbn->rx_max_frames; + kernel_coal->rx_cqe_nsecs = fbn->fbd->rx_cqe_nsecs; return 0; } @@ -256,6 +257,7 @@ static int fbnic_set_coalesce(struct net_device *netdev, struct netlink_ext_ack *extack) { struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; /* Verify against hardware limits */ if (ec->rx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT)) { @@ -272,10 +274,20 @@ static int fbnic_set_coalesce(struct net_device *netdev, NL_SET_ERR_MSG_MOD(extack, "rx_frames is above device max"); return -EINVAL; } + if (kernel_coal->rx_cqe_nsecs < FBNIC_RX_CQE_NSECS_MIN || + kernel_coal->rx_cqe_nsecs > FBNIC_RX_CQE_NSECS_MAX) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "rx-cqe-nsecs must be between %u and %u", + FBNIC_RX_CQE_NSECS_MIN, + FBNIC_RX_CQE_NSECS_MAX); + return -EINVAL; + } fbn->tx_usecs = ec->tx_coalesce_usecs; fbn->rx_usecs = ec->rx_coalesce_usecs; fbn->rx_max_frames = ec->rx_max_coalesced_frames; + fbd->rx_cqe_nsecs = kernel_coal->rx_cqe_nsecs; + fbnic_config_rx_cqe_nsecs(fbd); if (netif_running(netdev)) { int i; @@ -2021,7 +2033,8 @@ static void fbnic_get_link_ext_stats(struct net_device *netdev, static const struct ethtool_ops fbnic_ethtool_ops = { .cap_link_lanes_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_RX_MAX_FRAMES, + ETHTOOL_COALESCE_RX_MAX_FRAMES | + ETHTOOL_COALESCE_RX_CQE_NSECS, .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT | ETHTOOL_RING_USE_HDS_THRS, .rxfh_max_num_contexts = FBNIC_RPC_RSS_TBL_COUNT, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 6d7eb8479edf..49b8813a3564 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -620,11 +620,55 @@ static int fbnic_fw_parse_bmc_addrs(u8 bmc_mac_addr[][ETH_ALEN], return 0; } +static int fbnic_fw_parse_bmc_cap(struct fbnic_dev *fbd, + struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *attr; + bool bmc_present = false; + u32 all_multi = 0; + int err; + + /* FW reports the BMC present once its NC-SI channel is enabled, before + * it has a MAC. No MAC array means nothing to program, so treat it as + * absent rather than rejecting the message. + */ + attr = results[FBNIC_FW_CAP_RESP_BMC_PRESENT] ? + results[FBNIC_FW_CAP_RESP_BMC_MAC_ARRAY] : NULL; + if (attr) { + err = fbnic_fw_parse_bmc_addrs(fbd->fw_cap.bmc_mac_addr, + attr, 4); + if (err) + return err; + + all_multi = fta_get_uint(results, + FBNIC_FW_CAP_RESP_BMC_ALL_MULTI); + bmc_present = true; + } else { + memset(fbd->fw_cap.bmc_mac_addr, 0, + sizeof(fbd->fw_cap.bmc_mac_addr)); + } + + /* all_multi is only read out of the message when a BMC is present, so + * it is never set on its own and callers can test it without pairing + * it with the presence check. + */ + assign_bit(FBNIC_FW_CAP_F_BMC_PRESENT, &fbd->fw_cap.state, bmc_present); + assign_bit(FBNIC_FW_CAP_F_BMC_ALL_MULTI, &fbd->fw_cap.state, all_multi); + + /* Always assume we need a BMC reinit. The barrier orders the BMC state + * published above ahead of the flag, pairing with the ordering implied + * by the test_and_clear_bit() in fbnic_bmc_rpc_check(). + */ + smp_mb__before_atomic(); + set_bit(FBNIC_FW_CAP_F_BMC_TCAM_REINIT, &fbd->fw_cap.state); + + return 0; +} + static int fbnic_fw_parse_cap_resp(void *opaque, struct fbnic_tlv_msg **results) { - u32 all_multi = 0, version = 0; struct fbnic_dev *fbd = opaque; - bool bmc_present; + u32 version = 0; int err; version = fta_get_uint(results, FBNIC_FW_CAP_RESP_VERSION); @@ -684,37 +728,13 @@ static int fbnic_fw_parse_cap_resp(void *opaque, struct fbnic_tlv_msg **results) fbd->fw_cap.link_fec = fta_get_uint(results, FBNIC_FW_CAP_RESP_FW_LINK_FEC); - bmc_present = !!results[FBNIC_FW_CAP_RESP_BMC_PRESENT]; - if (bmc_present) { - struct fbnic_tlv_msg *attr; - - attr = results[FBNIC_FW_CAP_RESP_BMC_MAC_ARRAY]; - if (!attr) - return -EINVAL; - - err = fbnic_fw_parse_bmc_addrs(fbd->fw_cap.bmc_mac_addr, - attr, 4); - if (err) - return err; - - all_multi = - fta_get_uint(results, FBNIC_FW_CAP_RESP_BMC_ALL_MULTI); - } else { - memset(fbd->fw_cap.bmc_mac_addr, 0, - sizeof(fbd->fw_cap.bmc_mac_addr)); - } - - fbd->fw_cap.bmc_present = bmc_present; - - if (results[FBNIC_FW_CAP_RESP_BMC_ALL_MULTI] || !bmc_present) - fbd->fw_cap.all_multi = all_multi; + err = fbnic_fw_parse_bmc_cap(fbd, results); + if (err) + return err; fbd->fw_cap.anti_rollback_version = fta_get_uint(results, FBNIC_FW_CAP_RESP_ANTI_ROLLBACK_VERSION); - /* Always assume we need a BMC reinit */ - fbd->fw_cap.need_bmc_tcam_reinit = true; - return 0; } @@ -1697,16 +1717,19 @@ static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd) if (!length) goto next_page; - /* Report descriptors with length greater than page size */ - if (length > PAGE_SIZE) { + /* Report descriptors with invalid message extents. */ + if (length < sizeof(msg->hdr) || length > PAGE_SIZE) { dev_warn(fbd->dev, "Invalid mailbox descriptor length: %lld\n", length); goto next_page; } - if (le16_to_cpu(msg->hdr.len) * sizeof(u32) > length) + if (!le16_to_cpu(msg->hdr.len) || + le16_to_cpu(msg->hdr.len) * sizeof(u32) > length) { dev_warn(fbd->dev, "Mailbox message length mismatch\n"); + goto next_page; + } /* If parsing fails dump contents of message to dmesg */ err = fbnic_tlv_msg_parse(fbd, msg, fbnic_fw_tlv_parser); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index 5f9969247e30..f47669129c9d 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -45,7 +45,22 @@ struct fbnic_fw_ver { char commit[FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE]; }; +/* Bits in fbnic_fw_cap.state. The firmware mailbox publishes the BMC state + * and the service task consumes it, so they are updated with atomic bitops. + */ +enum { + FBNIC_FW_CAP_F_BMC_PRESENT, + FBNIC_FW_CAP_F_BMC_ALL_MULTI, + FBNIC_FW_CAP_F_BMC_TCAM_REINIT, + FBNIC_FW_CAP_F_BMC_MACDA_SYNC, +}; + +#define FBNIC_FW_CAP_BMC_PENDING \ + (BIT(FBNIC_FW_CAP_F_BMC_TCAM_REINIT) | \ + BIT(FBNIC_FW_CAP_F_BMC_MACDA_SYNC)) + struct fbnic_fw_cap { + unsigned long state; struct { struct fbnic_fw_ver mgmt, bootloader; } running; @@ -54,10 +69,6 @@ struct fbnic_fw_cap { } stored; u8 active_slot; u8 bmc_mac_addr[4][ETH_ALEN]; - u8 bmc_present : 1; - u8 need_bmc_tcam_reinit : 1; - u8 need_bmc_macda_sync : 1; - u8 all_multi : 1; u8 link_speed; u8 link_fec; u32 anti_rollback_version; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c index 53b7a938b4c2..16271c717a5a 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -125,14 +125,7 @@ static void fbnic_mac_init_qm(struct fbnic_dev *fbd) FIELD_PREP(FBNIC_QM_TCQ_CTL0_COAL_WAIT, clock_freq / 12500)); - /* We will have the interrupt threshold timer tick once every - * 1 usec and coalesce writes for up to 2 usecs. - */ - wr32(fbd, FBNIC_QM_RCQ_CTL0, - FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES, - clock_freq / 1000000) | - FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT, - clock_freq / 500000)); + fbnic_config_rx_cqe_nsecs(fbd); /* Configure spacer control to 64 beats. */ wr32(fbd, FBNIC_FAB_AXI4_AR_SPACER_2_CFG, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 10bf99be3f24..8a6703afca38 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -773,6 +773,10 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT; fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT; + netdev->cfg->rings.rx_pending = fbn->rcq_size; + netdev->cfg->rings.rx_mini_pending = fbn->hpq_size; + netdev->cfg->rings.rx_jumbo_pending = fbn->ppq_size; + fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT; fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT; fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index c6698e3002a1..ad234bb0f75d 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -276,8 +276,6 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); - if (err) - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); return err; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index bc0f38b6a2b2..939d56585b26 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -117,7 +117,7 @@ void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, * BMC. */ mac_addr = &fbd->mac_addr[fbd->mac_addr_boundary - 1]; - if (fbnic_bmc_present(fbd) && fbd->fw_cap.all_multi) { + if (fbnic_bmc_all_multi(fbd)) { if (mac_addr->state != FBNIC_TCAM_S_VALID) { eth_zero_addr(mac_addr->value.addr8); eth_broadcast_addr(mac_addr->mask.addr8); @@ -126,12 +126,8 @@ void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, set_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam); mac_addr->state = FBNIC_TCAM_S_ADD; } - if (enable_host) - set_bit(FBNIC_MAC_ADDR_T_ALLMULTI, - mac_addr->act_tcam); - else - clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, - mac_addr->act_tcam); + assign_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam, + enable_host); } else { __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BMC); __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_ALLMULTI); @@ -147,9 +143,11 @@ void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, /* If we are not enabling the rule just delete it. We will fall * back to the RSS rules that support the multicast addresses. + * Delete in any live state, not just VALID, as ifdown demotes it to + * UPDATE and a miss there leaves it to be rewritten to hardware. */ - if (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi || enable_host) { - if (act_tcam->state == FBNIC_TCAM_S_VALID) + if (!fbnic_bmc_all_multi(fbd) || enable_host) { + if (act_tcam->state != FBNIC_TCAM_S_DISABLED) act_tcam->state = FBNIC_TCAM_S_DELETE; return; } @@ -238,25 +236,121 @@ void fbnic_bmc_rpc_init(struct fbnic_dev *fbd) act_tcam->state = FBNIC_TCAM_S_UPDATE; } +/** + * fbnic_bmc_rules_present - is the BMC currently programmed into the filters? + * @fbd: Pointer to fbnic device struct + * + * The BMC tag is only set while a BMC is present, so it doubles as the state. + * + * Return: true if any MACDA entry carries the BMC tag, false otherwise. + */ +static bool fbnic_bmc_rules_present(struct fbnic_dev *fbd) +{ + int idx; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + if (mac_addr->state == FBNIC_TCAM_S_DISABLED) + continue; + + if (test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) + return true; + } + + return false; +} + +/** + * fbnic_bmc_rpc_disable - remove the BMC MAC and action rules + * @fbd: Pointer to fbnic device struct + * + * Undo fbnic_bmc_rpc_init() when the BMC drops its NC-SI channel while the + * host interface stays up. Only marks shadow state; the caller's + * __fbnic_set_rx_mode() pushes it to hardware, so ordering here is moot. + */ +static void fbnic_bmc_rpc_disable(struct fbnic_dev *fbd) +{ + struct fbnic_act_tcam *act_tcam; + int idx; + + /* Drop the BMC's claim; entries the host also uses survive */ + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + if (mac_addr->state == FBNIC_TCAM_S_DISABLED) + continue; + + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BMC); + } + + /* Delete in any live state, not just VALID: init leaves it UPDATE and + * ifdown demotes VALID to UPDATE. A miss leaks the rule for good, as + * fbnic_bmc_rules_present() keys off the MACDA tag just cleared above. + */ + act_tcam = &fbd->act_tcam[FBNIC_RPC_ACT_TBL_BMC_OFFSET]; + if (act_tcam->state != FBNIC_TCAM_S_DISABLED) + act_tcam->state = FBNIC_TCAM_S_DELETE; + + /* fbnic_rss_reinit() only programs the host-unicast entries while a BMC + * is present, so the reinit after this would leave them stale and valid + * in hardware. Delete them to converge on the no-BMC layout. + */ + for (idx = 0; idx < FBNIC_RSS_EN_NUM_UNICAST; idx++) { + act_tcam = &fbd->act_tcam[FBNIC_RPC_ACT_TBL_RSS_OFFSET + idx]; + if (act_tcam->state != FBNIC_TCAM_S_DISABLED) + act_tcam->state = FBNIC_TCAM_S_DELETE; + } +} + void fbnic_bmc_rpc_check(struct fbnic_dev *fbd) { + struct fbnic_net *fbn = netdev_priv(fbd->netdev); int err; - if (fbd->fw_cap.need_bmc_tcam_reinit) { - fbnic_bmc_rpc_init(fbd); + /* Nothing to do unless the firmware raised one of the flags. Test + * before taking the lock to keep it off the common path; one raised + * after this is picked up on the next pass. + */ + if (!(READ_ONCE(fbd->fw_cap.state) & FBNIC_FW_CAP_BMC_PENDING)) + return; + + /* The rx mode work and the ethtool paths rewrite the MACDA and action + * TCAM shadows under the instance lock, and this runs from the service + * task under RTNL only, so the two do not exclude each other. + */ + netdev_lock(fbd->netdev); + + /* Consume the flag before the state it advertises. The ordering + * implied by test_and_clear_bit() pairs with the barrier in + * fbnic_fw_parse_bmc_cap(), and claiming it atomically means a set + * racing with us is kept and retried rather than overwritten. + */ + if (test_and_clear_bit(FBNIC_FW_CAP_F_BMC_TCAM_REINIT, + &fbd->fw_cap.state)) { + if (!fbnic_bmc_present(fbd) && fbnic_bmc_rules_present(fbd)) + fbnic_bmc_rpc_disable(fbd); + else + fbnic_bmc_rpc_init(fbd); + + /* Neither path touches the RSS actions, which carry the mc/bc + * copy to the BMC, so recompute them alongside. + */ + fbnic_rss_reinit(fbd, fbn); netif_addr_lock_bh(fbd->netdev); __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc); netif_addr_unlock_bh(fbd->netdev); - fbd->fw_cap.need_bmc_tcam_reinit = false; } - if (fbd->fw_cap.need_bmc_macda_sync) { + if (test_and_clear_bit(FBNIC_FW_CAP_F_BMC_MACDA_SYNC, + &fbd->fw_cap.state)) { err = fbnic_fw_xmit_rpc_macda_sync(fbd); if (err) dev_warn(fbd->dev, "Writing MACDA table to FW failed, err: %d\n", err); - fbd->fw_cap.need_bmc_macda_sync = false; } + + netdev_unlock(fbd->netdev); } #define FBNIC_ACT1_INIT(_l4, _udp, _ip, _v6) \ @@ -488,8 +582,7 @@ void fbnic_promisc_sync(struct fbnic_dev *fbd, mac_addr->act_tcam); mac_addr->state = FBNIC_TCAM_S_ADD; } - } else if (mc_promisc && - (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi)) { + } else if (mc_promisc && !fbnic_bmc_all_multi(fbd)) { /* We have to add a special handler for multicast as the * BMC may have an all-multi rule already in place. As such * adding a rule ourselves won't do any good so we will have @@ -660,12 +753,12 @@ void fbnic_write_macda(struct fbnic_dev *fbd) } /* If reinitializing the BMC TCAM we are doing an initial update */ - if (fbd->fw_cap.need_bmc_tcam_reinit) + if (test_bit(FBNIC_FW_CAP_F_BMC_TCAM_REINIT, &fbd->fw_cap.state)) updates++; /* If needed notify firmware of changes to MACDA TCAM */ if (updates != 0 && fbnic_bmc_present(fbd)) - fbd->fw_cap.need_bmc_macda_sync = true; + set_bit(FBNIC_FW_CAP_F_BMC_MACDA_SYNC, &fbd->fw_cap.state); } static void fbnic_clear_act_tcam(struct fbnic_dev *fbd, unsigned int idx) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c index c55d4f76a5fc..3ef3935c0419 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c @@ -430,6 +430,10 @@ int fbnic_tlv_attr_parse_array(struct fbnic_tlv_msg *attr, int len, u16 attr_len; int err; + attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len)); + if (!attr_len || attr_len > len) + return -EINVAL; + if (tlv_attr_id != attr_id) return -EINVAL; @@ -443,7 +447,6 @@ int fbnic_tlv_attr_parse_array(struct fbnic_tlv_msg *attr, int len, results[i++] = attr; - attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len)); len -= attr_len; attr += attr_len; } @@ -476,11 +479,16 @@ int fbnic_tlv_attr_parse(struct fbnic_tlv_msg *attr, int len, /* Work through list of attributes, parsing them as necessary */ while (len > 0) { - int err = fbnic_tlv_attr_validate(attr, tlv_index); u16 attr_id = attr->hdr.type; u16 attr_len; + int err; + + attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len)); + if (!attr_len || attr_len > len) + return -EINVAL; /* Stop parsing on full error */ + err = fbnic_tlv_attr_validate(attr, tlv_index); if (err < 0) return err; @@ -493,7 +501,6 @@ int fbnic_tlv_attr_parse(struct fbnic_tlv_msg *attr, int len, results[attr_id] = attr; } - attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len)); len -= attr_len; attr += attr_len; } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 10caacffee0f..c62d885a678a 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -6,6 +6,7 @@ #include <linux/bpf_trace.h> #include <linux/iopoll.h> #include <linux/pci.h> +#include <linux/time64.h> #include <net/netdev_queues.h> #include <net/netdev_rx_queue.h> #include <net/page_pool/helpers.h> @@ -895,19 +896,31 @@ static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, ring->head = head; } +static u16 fbnic_rcd_bd_idx(const struct fbnic_ring *bdq, u64 rcd) +{ + return FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd) >> bdq->bd_page_shift; +} + +static unsigned int fbnic_rcd_bd_page_offset(const struct fbnic_ring *bdq, + u64 rcd) +{ + u16 id = FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd); + u16 page_id = id & (fbnic_bd_page_count(bdq) - 1); + + return page_id * FBNIC_BD_PAGE_SIZE; +} + static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) { - __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; + __le64 *bdq_desc = &bdq->desc[id * fbnic_bd_page_count(bdq)]; dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); - u64 bd, i = FBNIC_BD_FRAG_COUNT; + u64 bd, i = fbnic_bd_page_count(bdq); - bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | - FIELD_PREP(FBNIC_BD_PAGE_ID_MASK, id); + bd = (FBNIC_BD_DESC_ADDR_MASK & dma) | + FIELD_PREP(FBNIC_BD_DESC_ID_MASK, (u64)id << bdq->bd_page_shift); - /* In the case that a page size is larger than 4K we will map a - * single page to multiple fragments. The fragments will be - * FBNIC_BD_FRAG_COUNT in size and the lower n bits will be use - * to indicate the individual fragment IDs. + /* Posted pages larger than 4 KiB use consecutive device-page IDs in + * the low bits of the software page ID. */ do { *bdq_desc = cpu_to_le64(bd); @@ -952,7 +965,7 @@ static void fbnic_fill_bdq(struct fbnic_ring *bdq) /* Force DMA writes to flush before writing to tail */ dma_wmb(); - writel(i * FBNIC_BD_FRAG_COUNT, bdq->doorbell); + writel(i * fbnic_bd_page_count(bdq), bdq->doorbell); } } @@ -983,26 +996,27 @@ static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, struct fbnic_pkt_buff *pkt, struct fbnic_q_triad *qt) { - unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); - struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx); unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); + unsigned int hdr_pg_idx = fbnic_rcd_bd_idx(&qt->sub0, rcd); unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; unsigned char *hdr_start; + struct page *page; /* data_hard_start should always be NULL when this is called */ WARN_ON_ONCE(pkt->buff.data_hard_start); + page = fbnic_page_pool_get_head(qt, hdr_pg_idx); + /* Short-cut the end calculation if we know page is fully consumed */ hdr_pg_end = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? - FBNIC_BD_FRAG_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); + FBNIC_BD_PAGE_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); hdr_pg_start = fbnic_hdr_pg_start(hdr_pg_off); headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; frame_sz = hdr_pg_end - hdr_pg_start; xdp_init_buff(&pkt->buff, frame_sz, &qt->xdp_rxq); - hdr_pg_start += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * - FBNIC_BD_FRAG_SIZE; + hdr_pg_start += fbnic_rcd_bd_page_offset(&qt->sub0, rcd); /* Sync DMA buffer */ dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), @@ -1023,28 +1037,34 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, struct fbnic_pkt_buff *pkt, struct fbnic_q_triad *qt) { - unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); - netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx); + unsigned int pg_idx = fbnic_rcd_bd_idx(&qt->sub1, rcd); unsigned int truesize; + netmem_ref netmem; bool added; + netmem = fbnic_page_pool_get_data(qt, pg_idx); + truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? - FBNIC_BD_FRAG_SIZE - pg_off : ALIGN(len, 128); + FBNIC_BD_PAGE_SIZE - pg_off : + ALIGN(len, FBNIC_RX_PAYLD_ALIGN); - pg_off += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * - FBNIC_BD_FRAG_SIZE; + pg_off += fbnic_rcd_bd_page_offset(&qt->sub1, rcd); /* Sync DMA buffer */ page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, pg_off, truesize); + /* Consecutive device-page completions from one PPQ page are adjacent + * ranges in the same netmem. + */ added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); if (unlikely(!added)) { pkt->add_frag_failed = true; netdev_err_once(nv->napi.dev, "Failed to add fragment to xdp_buff\n"); + page_pool_put_full_netmem(qt->sub1.page_pool, netmem, true); } } @@ -1281,12 +1301,12 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { case FBNIC_RCD_TYPE_HDR_AL: - head0 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + head0 = fbnic_rcd_bd_idx(&qt->sub0, rcd); fbnic_pkt_prepare(nv, rcd, pkt, qt); break; case FBNIC_RCD_TYPE_PAY_AL: - head1 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + head1 = fbnic_rcd_bd_idx(&qt->sub1, rcd); fbnic_add_rx_frag(nv, rcd, pkt, qt); break; @@ -1574,7 +1594,7 @@ void fbnic_free_napi_vectors(struct fbnic_net *fbn) static int fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, - unsigned int rxq_idx) + unsigned int rxq_idx, u32 rx_page_size) { struct page_pool_params pp_params = { .order = 0, @@ -1609,6 +1629,8 @@ fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, qt->sub0.page_pool = pp; if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { + pp_params.order = get_order(rx_page_size); + pp_params.max_len = rx_page_size; pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; pp_params.dma_dir = DMA_FROM_DEVICE; @@ -1627,6 +1649,16 @@ err_destroy_sub0: return PTR_ERR(pp); } +static u8 fbnic_bdq_page_shift(u32 page_size) +{ + return ilog2(page_size / FBNIC_BD_PAGE_SIZE); +} + +static void fbnic_bdq_set_page_size(struct fbnic_ring *bdq, u32 page_size) +{ + bdq->bd_page_shift = fbnic_bdq_page_shift(page_size); +} + static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, int q_idx, u8 flags) { @@ -1634,6 +1666,7 @@ static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, ring->doorbell = doorbell; ring->q_idx = q_idx; ring->flags = flags; + fbnic_bdq_set_page_size(ring, PAGE_SIZE); ring->deferred_head = -1; } @@ -1918,12 +1951,12 @@ static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn, switch (rxr->doorbell - fbnic_ring_csr_base(rxr)) { case FBNIC_QUEUE_BDQ_HPQ_TAIL: - rxq_size = fbn->hpq_size / FBNIC_BD_FRAG_COUNT; - desc_size *= FBNIC_BD_FRAG_COUNT; + rxq_size = fbn->hpq_size / fbnic_bd_page_count(rxr); + desc_size *= fbnic_bd_page_count(rxr); break; case FBNIC_QUEUE_BDQ_PPQ_TAIL: - rxq_size = fbn->ppq_size / FBNIC_BD_FRAG_COUNT; - desc_size *= FBNIC_BD_FRAG_COUNT; + rxq_size = fbn->ppq_size / fbnic_bd_page_count(rxr); + desc_size *= fbnic_bd_page_count(rxr); break; case FBNIC_QUEUE_RCQ_HEAD: rxq_size = fbn->rcq_size; @@ -2027,15 +2060,18 @@ free_sub0: static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, struct fbnic_napi_vector *nv, - struct fbnic_q_triad *qt) + struct fbnic_q_triad *qt, + u32 rx_page_size) { struct device *dev = fbn->netdev->dev.parent; int err; - err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx); + err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx, rx_page_size); if (err) return err; + fbnic_bdq_set_page_size(&qt->sub1, rx_page_size); + err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, nv->napi.napi_id); if (err) @@ -2096,7 +2132,11 @@ static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, /* Allocate Rx Resources */ for (j = 0; j < nv->rxt_count; j++, i++) { - err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]); + struct netdev_queue_config qcfg; + + netdev_queue_config(fbn->netdev, nv->qt[i].cmpl.q_idx, &qcfg); + err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i], + qcfg.rx_page_size); if (err) goto free_qt_resources; } @@ -2590,7 +2630,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) hpq->tail = 0; hpq->head = 0; - log_size = fls(hpq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); + log_size = fls(hpq->size_mask) + hpq->bd_page_shift; /* Store descriptor ring address and size */ fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); @@ -2602,7 +2642,7 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) if (!ppq->size_mask) goto write_ctl; - log_size = fls(ppq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); + log_size = fls(ppq->size_mask) + ppq->bd_page_shift; /* Add enabling of PPQ to BDQ control */ bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; @@ -2668,6 +2708,22 @@ static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 r fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, threshold); } +void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd) +{ + u32 coal_wait; + + coal_wait = DIV_ROUND_CLOSEST_ULL((u64)fbd->rx_cqe_nsecs * + FBNIC_CLOCK_FREQ, NSEC_PER_SEC); + + /* TICK_CYCLES controls the interrupt threshold timer. COAL_WAIT is + * independent and measured in core clock cycles. + */ + wr32(fbd, FBNIC_QM_RCQ_CTL0, + FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES, + FBNIC_CLOCK_FREQ / USEC_PER_SEC) | + FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT, coal_wait)); +} + void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm) { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); @@ -2875,7 +2931,8 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, struct fbnic_napi_vector *nv; if (!netif_running(dev)) - return fbnic_alloc_qt_page_pools(fbn, qt, idx); + return fbnic_alloc_qt_page_pools(fbn, qt, idx, + qcfg->rx_page_size); /* A failed PCIe recovery or resume can leave the datapath torn down * while netif_running() is still true. This ndo runs before @@ -2895,7 +2952,70 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, real->cmpl.flags); - return fbnic_alloc_rx_qt_resources(fbn, nv, qt); + return fbnic_alloc_rx_qt_resources(fbn, nv, qt, qcfg->rx_page_size); +} + +static void fbnic_default_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg) +{ + qcfg->rx_page_size = PAGE_SIZE; +} + +static int fbnic_validate_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg, + struct netlink_ext_ack *extack) +{ + u32 bd_page_count, ppq_entries, frag_count; + u32 rx_page_size = qcfg->rx_page_size; + u32 ppq_size; + + if (!qcfg->rx_jumbo_ring_size) { + NL_SET_ERR_MSG_MOD(extack, + "rx-jumbo ring size must be nonzero"); + return -EINVAL; + } + + ppq_size = roundup_pow_of_two(qcfg->rx_jumbo_ring_size); + + if (!is_power_of_2(rx_page_size)) { + NL_SET_ERR_MSG_MOD(extack, + "rx_page_size must be a power of 2"); + return -EINVAL; + } + + if (rx_page_size < FBNIC_BD_PAGE_SIZE) { + NL_SET_ERR_MSG_MOD(extack, + "rx_page_size must be at least 4K"); + return -EINVAL; + } + + /* Payload fragments occupy multiples of FBNIC_RX_PAYLD_ALIGN bytes. + * Keep at least one reference in the bias until fbnic_clean_bdq() + * observes a completion from a subsequent allocation. + */ + frag_count = rx_page_size / FBNIC_RX_PAYLD_ALIGN; + if (frag_count >= FBNIC_PAGECNT_BIAS_MAX) { + NL_SET_ERR_MSG_MOD(extack, + "rx_page_size can produce too many fragments"); + return -EINVAL; + } + + bd_page_count = rx_page_size / FBNIC_BD_PAGE_SIZE; + ppq_entries = ppq_size / bd_page_count; + /* The PPQ is sized in 4 KiB device pages. One software entry tracks + * each page-pool allocation. In addition to the unused entry for + * empty/full accounting, cleanup retains the current allocation + * until a completion identifies a subsequent allocation. A two-entry + * ring can only post one allocation and cannot make progress. + * Require at least four entries, since ring sizes are powers of two. + */ + if (ppq_entries < 4) { + NL_SET_ERR_MSG_MOD(extack, + "rx-jumbo ring size too small for rx_page_size"); + return -EINVAL; + } + + return 0; } static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) @@ -2997,4 +3117,7 @@ const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { .ndo_queue_mem_free = fbnic_queue_mem_free, .ndo_queue_start = fbnic_queue_start, .ndo_queue_stop = fbnic_queue_stop, + .ndo_default_qcfg = fbnic_default_qcfg, + .ndo_validate_qcfg = fbnic_validate_qcfg, + .supported_params = QCFG_RX_PAGE_SIZE, }; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index f5899446dcc5..6ee3cce3d942 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -4,12 +4,16 @@ #ifndef _FBNIC_TXRX_H_ #define _FBNIC_TXRX_H_ +#include <linux/bitfield.h> #include <linux/netdevice.h> #include <linux/skbuff.h> +#include <linux/time64.h> #include <linux/types.h> #include <linux/u64_stats_sync.h> #include <net/xdp.h> +#include "fbnic_csr.h" + struct fbnic_net; /* Guarantee we have space needed for storing the buffer @@ -49,12 +53,19 @@ struct fbnic_net; #define FBNIC_RX_USECS_DEFAULT 30 #define FBNIC_RX_FRAMES_DEFAULT 0 +#define FBNIC_RX_CQE_NSECS_MIN 1000 +#define FBNIC_RX_CQE_NSECS_DEFAULT 2000 +#define FBNIC_RX_CQE_NSECS_MAX \ + ((u32)(((u64)FIELD_MAX(FBNIC_QM_RCQ_CTL0_COAL_WAIT) * NSEC_PER_SEC) / \ + FBNIC_CLOCK_FREQ)) + #define FBNIC_RX_TROOM \ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) #define FBNIC_RX_HROOM_PAD 128 #define FBNIC_RX_HROOM \ (ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM) #define FBNIC_RX_PAD 0 +#define FBNIC_RX_PAYLD_ALIGN 128 #define FBNIC_RX_PAYLD_OFFSET 0 #define FBNIC_RX_PAYLD_PG_CL 0 @@ -121,6 +132,7 @@ struct fbnic_ring { u16 size_mask; /* Size of ring in descriptors - 1 */ u8 q_idx; /* Logical netdev ring index */ u8 flags; /* Ring flags (FBNIC_RING_F_*) */ + u8 bd_page_shift; /* BDQ: ilog2(page_size / 4096) */ u32 head, tail; /* Head/Tail of ring */ @@ -167,6 +179,11 @@ struct fbnic_napi_vector { extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops; +static inline u16 fbnic_bd_page_count(const struct fbnic_ring *bdq) +{ + return 1U << bdq->bd_page_shift; +} + netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); netdev_features_t fbnic_features_check(struct sk_buff *skb, struct net_device *dev, diff --git a/drivers/net/ethernet/meta/mpnic/Makefile b/drivers/net/ethernet/meta/mpnic/Makefile new file mode 100644 index 000000000000..d5bdbd5bd1c7 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# +# Makefile for the Meta(R) Platforms Network Interface Controller +# + +obj-$(CONFIG_MPNIC) += mpnic.o + +mpnic-y := \ + mpnic_init.o \ + mpnic_irq.o \ + mpnic_netdev.o \ + mpnic_pci.o \ + mpnic_txrx.o \ +# End of mpnic-y diff --git a/drivers/net/ethernet/meta/mpnic/mpnic.h b/drivers/net/ethernet/meta/mpnic/mpnic.h new file mode 100644 index 000000000000..8b08af0fadcc --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _MPNIC_H_ +#define _MPNIC_H_ + +#include <linux/interrupt.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/types.h> +#include <linux/workqueue.h> + +#include "mpnic_csr.h" + +#define MPNIC_DRV_NAME "mpnic" + +#define MPNIC_MAX_TXQS 1024u +#define MPNIC_MAX_RXQS 1024u + +/* misc IRQ entries are allocated before the completion queue IRQs */ +enum { + MPNIC_FW_MSIX_ENTRY, + MPNIC_NON_NAPI_VECTORS +}; + +struct mpnic_dev { + struct device *dev; + struct net_device *netdev; + + u32 __iomem *uc_addr0; + + struct delayed_work service_task; + + u16 num_irqs; + + u64 dsn; + u32 mps; + u32 readrq; + u8 relaxed_ord; +}; + +u64 mpnic_rd64(struct mpnic_dev *mpd, u32 reg); + +int mpnic_dev_init(struct mpnic_dev *mpd); + +int mpnic_request_irq(struct mpnic_dev *mpd, int nr, irq_handler_t handler, + unsigned long flags, const char *name, void *data); +void mpnic_free_irq(struct mpnic_dev *mpd, int nr, void *data); +void mpnic_free_irqs(struct mpnic_dev *mpd); +int mpnic_alloc_irqs(struct mpnic_dev *mpd); + +static inline void mpnic_wr64(struct mpnic_dev *mpd, u32 reg, u64 val) +{ + u32 __iomem *csr = READ_ONCE(mpd->uc_addr0); + + if (csr) + writeq(val, csr + reg); +} + +static inline void mpnic_wrfl(struct mpnic_dev *mpd) +{ + mpnic_rd64(mpd, MPNIC_BDQ_SPARE); +} + +static inline bool mpnic_present(struct mpnic_dev *mpd) +{ + return !!READ_ONCE(mpd->uc_addr0); +} + +#endif /* _MPNIC_H_ */ diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_csr.h b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h new file mode 100644 index 000000000000..d6ab9df58168 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h @@ -0,0 +1,319 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _MPNIC_CSR_H_ +#define _MPNIC_CSR_H_ + +#include <linux/bits.h> + +#define CSR_BIT(nr) BIT_ULL(nr) +#define CSR_GENMASK(h, l) GENMASK_ULL(h, l) + +#define DESC_BIT(nr) BIT_ULL(nr) +#define DESC_GENMASK(h, l) GENMASK_ULL(h, l) + +/* Transmit Work Descriptor Format */ +#define MPNIC_TWD_FLAG_REQ_COMPLETION DESC_BIT(37) +#define MPNIC_TWD_FLAG_DEST_MAC DESC_BIT(43) +#define MPNIC_TWD_TYPE DESC_GENMASK(47, 46) +enum { + MPNIC_TWD_TYPE_META = 0, + MPNIC_TWD_TYPE_AL = 2, + MPNIC_TWD_TYPE_LAST_AL = 3, +}; + +#define MPNIC_TWD_ADDR DESC_GENMASK(45, 0) +#define MPNIC_TWD_LEN DESC_GENMASK(63, 48) + +/* Tx Completion Descriptor Format */ +#define MPNIC_TCD_TYPE0_HEAD0 DESC_GENMASK(15, 0) +#define MPNIC_TCD_DONE DESC_BIT(63) + +/* Rx Buffer Descriptor Format */ +#define MPNIC_BD_DESC_ADDR DESC_GENMASK(39, 2) +#define MPNIC_BD_DESC_ID DESC_GENMASK(57, 40) +#define MPNIC_BD_DESC_BUF_SZ_LOG2 DESC_GENMASK(62, 58) + +/* Rx Completion Queue Descriptors */ +#define MPNIC_RCD_TYPE DESC_GENMASK(62, 61) +enum { + MPNIC_RCD_TYPE_HDR_AL = 0, + MPNIC_RCD_TYPE_PAY_AL = 1, + MPNIC_RCD_TYPE_META = 3, +}; + +#define MPNIC_RCD_DONE DESC_BIT(63) + +#define MPNIC_RCD_HDR_SUBTYPE DESC_GENMASK(60, 59) +enum { + MPNIC_RCD_HDR_SUBTYPE_HDR = 2, +}; + +/* Address/Length Completion Descriptors */ +#define MPNIC_RCD_AL_BUFF_OFF DESC_GENMASK(15, 0) +#define MPNIC_RCD_AL_BUFF_ID DESC_GENMASK(33, 16) +#define MPNIC_RCD_AL_BUFF_LEN DESC_GENMASK(47, 34) +#define MPNIC_RCD_AL_PAGE_FIN DESC_BIT(53) + +/* Metadata Completion Descriptors */ +#define MPNIC_RCD_META_ERR_MAC_EOP DESC_BIT(53) +#define MPNIC_RCD_META_ERR_TRUNCATED_FRAME DESC_BIT(54) +#define MPNIC_RCD_META_UNCORRECTABLE_ERR_MASK \ + (MPNIC_RCD_META_ERR_MAC_EOP | MPNIC_RCD_META_ERR_TRUNCATED_FRAME) + +/* Common fields for all DESC_CFG CSRs */ +#define MPNIC_DESC_CFG_NUM_DESCS CSR_GENMASK(2, 0) +#define MPNIC_DESC_CFG_START_ADDR CSR_GENMASK(19, 8) + +/* Register Definitions + * + * The register file is addressed as an array of le32, so the byte address of + * a register is 4 times the index below. Each register is listed with its + * name, index and byte address. + * + * Name Index Address + *****************************************************************************/ + +/* NIC_CORE_TDF */ +#define MPNIC_TWQ_CTL(i, j) (0x0 + 1024 * (i) + 2 * (j)) + /* 0x0 */ +#define MPNIC_TWQ_CTL_RESET CSR_BIT(0) +#define MPNIC_TWQ_CTL_ENABLE CSR_BIT(1) +#define MPNIC_TWQ_TAIL(i, j) (0x4 + 1024 * (i) + 2 * (j)) + /* 0x10 */ +#define MPNIC_TWQ_SIZE(i, j) (0x10 + 1024 * (i) + 2 * (j)) + /* 0x40 */ +#define MPNIC_TWQ_SIZE_SIZE CSR_GENMASK(3, 0) +#define MPNIC_TWQ_BASE_ADDR(i, j) (0x1c + 1024 * (i) + 2 * (j)) + /* 0x70 */ + +/* NIC_CORE_TCM */ +#define MPNIC_TCQ_CTL(i) (0x80 + 1024 * (i)) /* 0x200 */ +#define MPNIC_TCQ_CTL_RESET CSR_BIT(0) +#define MPNIC_TCQ_CTL_ENABLE CSR_BIT(1) +#define MPNIC_TCQ_BASE_ADDR(i) (0x86 + 1024 * (i)) /* 0x218 */ +#define MPNIC_TCQ_HEAD(i) (0x8e + 1024 * (i)) /* 0x238 */ +#define MPNIC_TCQ_SIZE(i) (0x94 + 1024 * (i)) /* 0x250 */ +#define MPNIC_TCQ_SIZE_SIZE CSR_GENMASK(4, 0) + +/* NIC_CORE_TIM */ +#define MPNIC_TIM_CTL1(i) (0xc0 + 1024 * (i)) /* 0x300 */ +#define MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT CSR_BIT(48) +#define MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT CSR_BIT(49) +#define MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT CSR_BIT(50) +#define MPNIC_TIM_CTL1_MASK CSR_BIT(51) +#define MPNIC_TIM_CTL1_MASK_EN CSR_BIT(52) +#define MPNIC_TIM_CTL1_TRIGGER CSR_BIT(53) +#define MPNIC_TIM_INTR_MASK(i) (0xc8 + 1024 * (i)) /* 0x320 */ +#define MPNIC_TIM_INTR_MASK_MASK CSR_BIT(0) + +/* NIC_CORE_RBP */ +#define MPNIC_BDQ_CTL(i) (0x200 + 1024 * (i)) /* 0x800 */ +#define MPNIC_BDQ_CTL_RESET CSR_BIT(0) +#define MPNIC_BDQ_CTL_ENABLE CSR_BIT(1) +#define MPNIC_BDQ_CTL_ENABLE_PPQ CSR_BIT(3) +#define MPNIC_HPQ_TAIL(i) (0x202 + 1024 * (i)) /* 0x808 */ +#define MPNIC_PPQ_TAIL(i) (0x204 + 1024 * (i)) /* 0x810 */ +#define MPNIC_HPQ_SIZE(i) (0x20a + 1024 * (i)) /* 0x828 */ +#define MPNIC_HPQ_SIZE_SIZE CSR_GENMASK(4, 0) +#define MPNIC_PPQ_SIZE(i) (0x20c + 1024 * (i)) /* 0x830 */ +#define MPNIC_PPQ_SIZE_SIZE CSR_GENMASK(4, 0) +#define MPNIC_HPQ_BASE_ADDR(i) (0x216 + 1024 * (i)) /* 0x858 */ +#define MPNIC_PPQ_BASE_ADDR(i) (0x218 + 1024 * (i)) /* 0x860 */ + +/* NIC_CORE_RCM */ +#define MPNIC_RCQ_CTL(i) (0x280 + 1024 * (i)) /* 0xa00 */ +#define MPNIC_RCQ_CTL_RESET CSR_BIT(0) +#define MPNIC_RCQ_CTL_ENABLE CSR_BIT(1) +#define MPNIC_RCQ_BASE_ADDR(i) (0x286 + 1024 * (i)) /* 0xa18 */ +#define MPNIC_RCQ_HEAD(i) (0x28e + 1024 * (i)) /* 0xa38 */ +#define MPNIC_RCQ_SIZE(i) (0x294 + 1024 * (i)) /* 0xa50 */ +#define MPNIC_RCQ_SIZE_SIZE CSR_GENMASK(4, 0) + +/* NIC_CORE_RIM */ +#define MPNIC_RIM_INTR_MASK(i) (0x2c8 + 1024 * (i)) /* 0xb20 */ +#define MPNIC_RIM_INTR_MASK_MASK CSR_BIT(0) + +/* NIC_CORE_TIM_PRV */ +#define MPNIC_TIM_CTL(i) (0x100100 + 1024 * (i)) /* 0x400400 */ + +/* NIC_CORE_RDE */ +#define MPNIC_RDE_CFG(i) (0x10021c + 1024 * (i)) /* 0x400870 */ +#define MPNIC_RDE_CFG_MIN_TAIL_ROOM CSR_GENMASK(9, 0) +#define MPNIC_RDE_CFG_MIN_HEAD_ROOM CSR_GENMASK(18, 10) +#define MPNIC_RDE_CFG_MAX_HEADER_BYTES CSR_GENMASK(45, 32) + +/* NIC_CORE_RIM_PRV */ +#define MPNIC_RIM_CTL(i) (0x100280 + 1024 * (i)) /* 0x400a00 */ + +/* NIC_CORE_RBP_HP_GLBL */ +#define MPNIC_HPQ_IDLE(i) (0x420000 + 2 * (i)) /* 0x1080000 */ +#define MPNIC_HPQ_IDLE_CNT 16 +#define MPNIC_PPQ_IDLE(i) (0x420060 + 2 * (i)) /* 0x1080180 */ +#define MPNIC_PPQ_IDLE_CNT 16 +#define MPNIC_BDQ_GLBL_CTL0 0x420080 /* 0x1080200 */ +#define MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE CSR_GENMASK(26, 18) +#define MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH \ + CSR_GENMASK(42, 32) +#define MPNIC_RDE_CTL 0x420082 /* 0x1080208 */ +#define MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD CSR_GENMASK(10, 0) +#define MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD CSR_GENMASK(21, 11) +#define MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD CSR_GENMASK(42, 32) +#define MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD CSR_GENMASK(53, 43) +#define MPNIC_BDQ_MEM_INIT_REQ 0x42013a /* 0x10804e8 */ +#define MPNIC_BDQ_MEM_INIT_DONE 0x42013c /* 0x10804f0 */ +#define MPNIC_BDQ_SPARE 0x42013e /* 0x10804f8 */ +#define MPNIC_HPQ_DESC_CFG(i) (0x420140 + 2 * (i)) /* 0x1080500 */ +#define MPNIC_PPQ_DESC_CFG(i) (0x420940 + 2 * (i)) /* 0x1082500 */ + +/* NIC_CORE_RDE_GLBL */ +#define MPNIC_RDE_MEM_INIT_REQ 0x4240e6 /* 0x1090398 */ +#define MPNIC_RDE_MEM_INIT_DONE 0x4240e8 /* 0x10903a0 */ + +/* NIC_CORE_RCM_GLBL */ +#define MPNIC_RCQ_IDLE(i) (0x42505e + 2 * (i)) /* 0x1094178 */ +#define MPNIC_RCQ_IDLE_CNT 16 +#define MPNIC_RCM_MEM_INIT_REQ 0x42507e /* 0x10941f8 */ +#define MPNIC_RCM_MEM_INIT_DONE 0x425080 /* 0x1094200 */ + +/* NIC_CORE_RNI_GLBL */ +#define MPNIC_RNI_RBP_CTL 0x427000 /* 0x109c000 */ +#define MPNIC_RNI_RDE_CTL 0x427002 /* 0x109c008 */ +#define MPNIC_RNI_RDE_CTL_MPS CSR_GENMASK(1, 0) +#define MPNIC_RNI_RDE_CTL_CLS CSR_GENMASK(3, 2) +#define MPNIC_RNI_RCM_CTL 0x427004 /* 0x109c010 */ + +/* NIC_CORE_TDF_GLBL */ +#define MPNIC_TWQ_IDLE(i) (0x428042 + 2 * (i)) /* 0x10a0108 */ +#define MPNIC_TWQ_IDLE_CNT 32 +#define MPNIC_TWQ_DEF_PRI_TWD 0x428082 /* 0x10a0208 */ +#define MPNIC_TDF_MEM_INIT_REQ 0x42813a /* 0x10a04e8 */ +#define MPNIC_TDF_MEM_INIT_DONE 0x42813c /* 0x10a04f0 */ +#define MPNIC_TDF_DESC_CFG(i) (0x428140 + 2 * (i)) /* 0x10a0500 */ + +/* NIC_CORE_TQS_GLBL */ +#define MPNIC_TQS_GLBL_CTL0 0x42a000 /* 0x10a8000 */ +#define MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN CSR_BIT(2) +#define MPNIC_TQS_GLBL_P0_0 0x42a002 /* 0x10a8008 */ +#define MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0 CSR_GENMASK(63, 48) +#define MPNIC_TQS_GLBL_P0_1 0x42a004 /* 0x10a8010 */ +#define MPNIC_TQS_GLBL_BMC 0x42a012 /* 0x10a8048 */ +#define MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS CSR_GENMASK(15, 0) +#define MPNIC_TQS_SLOWDOWN_CTL 0x42a026 /* 0x10a8098 */ +#define MPNIC_TQS_SLOWDOWN_CTL_ENABLE CSR_BIT(6) +#define MPNIC_TQS_MTU_CTL0 0x42a030 /* 0x10a80c0 */ +#define MPNIC_TQS_MTU_CTL1 0x42a032 /* 0x10a80c8 */ +#define MPNIC_TQS_IDLE(i) (0x42a040 + 2 * (i)) /* 0x10a8100 */ +#define MPNIC_TQS_IDLE_CNT 32 +#define MPNIC_TQS_SET_P0_MAP0(i) (0x42a082 + 2 * (i)) /* 0x10a8208 */ +#define MPNIC_TQS_SET_P0_MAP1(i) (0x42a092 + 2 * (i)) /* 0x10a8248 */ +#define MPNIC_TQS_GLBL_SHAPING 0x42a108 /* 0x10a8420 */ +#define MPNIC_TQS_GLBL_SHAPING_DISABLE CSR_BIT(0) +#define MPNIC_TQS_ARB_CTL 0x42a122 /* 0x10a8488 */ +#define MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN CSR_BIT(9) +#define MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN CSR_BIT(8) +#define MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN CSR_BIT(5) +#define MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN CSR_BIT(4) +#define MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN CSR_BIT(1) +#define MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN CSR_BIT(0) +#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0 \ + 0x42a124 /* 0x10a8490 */ +#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE CSR_GENMASK(19, 0) +#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP CSR_GENMASK(59, 32) +#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1 \ + 0x42a126 /* 0x10a8498 */ +#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET CSR_GENMASK(27, 0) +#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT CSR_GENMASK(63, 32) +#define MPNIC_TQS_SRAM_INIT_CTL 0x42a128 /* 0x10a84a0 */ +#define MPNIC_TQS_SRAM_INIT_CTL_QUANTUM CSR_GENMASK(31, 20) +#define MPNIC_TQS_SRAM_INIT_CTL_INIT CSR_BIT(32) +#define MPNIC_TQS_GROUP_INIT_CTL 0x42a12a /* 0x10a84a8 */ +#define MPNIC_TQS_GROUP_INIT_CTL_QUANTUM CSR_GENMASK(51, 32) +#define MPNIC_TQS_GROUP_INIT_CTL_INIT CSR_BIT(52) +#define MPNIC_TQS_SET_INIT_CTL 0x42a12c /* 0x10a84b0 */ +#define MPNIC_TQS_SET_INIT_CTL_QUANTUM CSR_GENMASK(51, 32) +#define MPNIC_TQS_SET_INIT_CTL_INIT CSR_BIT(52) +#define MPNIC_TQS_PORT_INIT_CTL 0x42a12e /* 0x10a84b8 */ +#define MPNIC_TQS_PORT_INIT_CTL_QUANTUM CSR_GENMASK(55, 32) +#define MPNIC_TQS_PORT_INIT_CTL_INIT CSR_BIT(56) +#define MPNIC_TQS_SRAM_STS 0x42a130 /* 0x10a84c0 */ +#define MPNIC_TQS_PORT_CTL(i) (0x42a1e4 + 2 * (i)) /* 0x10a8790 */ + +/* NIC_CORE_TDE_GLBL */ +#define MPNIC_TDE_IDLE(i) (0x42b000 + 2 * (i)) /* 0x10ac000 */ +#define MPNIC_TDE_IDLE_CNT 32 +#define MPNIC_TDE_MEM_INIT_REQ 0x42b1ee /* 0x10ac7b8 */ +#define MPNIC_TDE_MEM_INIT_DONE 0x42b1f0 /* 0x10ac7c0 */ + +/* NIC_CORE_TCM_GLBL */ +#define MPNIC_TCQ_IDLE(i) (0x42c09e + 2 * (i)) /* 0x10b0278 */ +#define MPNIC_TCQ_IDLE_CNT 16 +#define MPNIC_TCM_MEM_INIT_REQ 0x42c0be /* 0x10b02f8 */ +#define MPNIC_TCM_MEM_INIT_DONE 0x42c0c0 /* 0x10b0300 */ + +/* NIC_CORE_TNI_GLBL */ +#define MPNIC_TNI_GLBL_TDF_CTL 0x42e000 /* 0x10b8000 */ +#define MPNIC_TNI_GLBL_TDF_CTL_MRRS CSR_GENMASK(2, 0) +#define MPNIC_TNI_GLBL_TDF_CTL_CLS CSR_GENMASK(5, 3) +#define MPNIC_TNI_GLBL_TDE_CTL 0x42e002 /* 0x10b8008 */ +#define MPNIC_TNI_GLBL_TCM_CTL 0x42e004 /* 0x10b8010 */ + +/* NIC_CORE_TXB */ +#define MPNIC_TXB_PORT_CONFIG 0x600000 /* 0x1800000 */ +#define MPNIC_TXB_PORT_CONFIG_PORT_MODE CSR_GENMASK(15, 13) +#define MPNIC_TXB_BMC 0x600122 /* 0x1800488 */ +#define MPNIC_TXB_P0(i) (0x600124 + 2 * (i)) /* 0x1800490 */ +#define MPNIC_TXB_P0_CNT 17 +#define MPNIC_TXB_P0_THRESH(i) (0x60025e + 2 * (i)) /* 0x1800978 */ +#define MPNIC_TXB_P0_ARB_WEIGHTS(i) (0x600378 + 2 * (i)) /* 0x1800de0 */ + +/* NIC_CORE_RXB */ +#define MPNIC_RXB_MEM_INIT_REQ 0x620002 /* 0x1880008 */ +#define MPNIC_RXB_MEM_INIT_DONE 0x620004 /* 0x1880010 */ +#define MPNIC_RXB_PORT_CFG(i) (0x620006 + 2 * (i)) /* 0x1880018 */ +#define MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE CSR_GENMASK(22, 22) +enum { + MPNIC_FCS_MODE_KEEP = 0, + MPNIC_FCS_MODE_STRIP = 1, +}; + +#define MPNIC_RXB_PORT_CLASS_CFG(i) (0x620020 + 2 * (i)) /* 0x1880080 */ +#define MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION \ + CSR_GENMASK(0, 0) +enum { + MPNIC_L2_ACTION_DROP = 0, + MPNIC_L2_ACTION_PASS = 1, +}; + +#define MPNIC_RXB_TC_CRDTS(i) (0x620418 + 2 * (i)) /* 0x1881060 */ +#define MPNIC_RXB_POOL_COMMON_CRDTS(i) (0x620458 + 2 * (i)) /* 0x1881160 */ +#define MPNIC_RXB_COMMON_CRDT_CTRL_TC(i) \ + (0x620472 + 2 * (i)) /* 0x18811c8 */ +#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH CSR_GENMASK(15, 0) +#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN CSR_BIT(29) +#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS CSR_GENMASK(47, 32) +#define MPNIC_RXB_HOST_DROP_THRESH(i) (0x6205b0 + 2 * (i)) /* 0x18816c0 */ +#define MPNIC_RXB_BMC_CRDTS(i) (0x6205d0 + 2 * (i)) /* 0x1881740 */ + +/* NIC_CORE_RPC */ +#define MPNIC_RPC_MEM_INIT_REQ 0x780440 /* 0x1e01100 */ +#define MPNIC_RPC_MEM_INIT_DONE 0x780442 /* 0x1e01108 */ + +/* NIC_CORE_ROF */ +#define MPNIC_RSC_GLOBAL_CONF 0x7e2002 /* 0x1f88008 */ +#define MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE CSR_BIT(0) + +/* NIC_CORE_TOF */ +#define MPNIC_TOF_TCAM_DEST_REMAP 0x7e3022 /* 0x1f8c088 */ + +/* PEMO_WRAPPER */ +#define MPNIC_OB_ATTR_RO CSR_BIT(1) +#define MPNIC_OB_ATTR_TDE_H 0x9a000e /* 0x2680038 */ +#define MPNIC_OB_ATTR_TDE_P 0x9a0010 /* 0x2680040 */ +#define MPNIC_OB_ATTR_TDF 0x9a0012 /* 0x2680048 */ +#define MPNIC_OB_ATTR_RBP_HPQ 0x9a0014 /* 0x2680050 */ +#define MPNIC_OB_ATTR_RBP_PPQ 0x9a0016 /* 0x2680058 */ +#define MPNIC_OB_ATTR_RDE_H 0x9a0018 /* 0x2680060 */ +#define MPNIC_OB_ATTR_RDE_P 0x9a001a /* 0x2680068 */ + +#endif /* _MPNIC_CSR_H_ */ diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_init.c b/drivers/net/ethernet/meta/mpnic/mpnic_init.c new file mode 100644 index 000000000000..ace4124158fb --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_init.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/cache.h> +#include <linux/if_ether.h> +#include <linux/iopoll.h> +#include <linux/log2.h> +#include <linux/sizes.h> + +#include "mpnic.h" +#include "mpnic_txrx.h" + +#define MPNIC_MEM_INIT_POLL_US 500 +#define MPNIC_MEM_INIT_TO_US 5000 + +/* BDQ mem init: + * bit 1: fifo_wrptr_mem + * bit 0: fifo_rdptr_mem + */ +#define MPNIC_MEM_INIT_BDQ_VAL 0x3 + +/* RCM mem init: + * bit 3: cq_base_addr + * bit 2: cd_fifo_rptr_stats + * bit 1: cd_fifo_wptr_stats + * bit 0: cq_head_ptr_stats + */ +#define MPNIC_MEM_INIT_RCM_VAL 0xf + +/* RDE mem init, bits 0-18. Bits 0-12 are the per-queue packet, error and + * drop counters, bits 13-18 the two context memories of each of the HPQ, + * PPQ and SPQ descriptor prefetchers. + */ +#define MPNIC_MEM_INIT_RDE_VAL 0x7ffff + +/* RPC mem init, bit 0 covers the whole classifier. */ +#define MPNIC_MEM_INIT_RPC_VAL 0x1 + +/* TCM mem init: + * bit 3: cq_head_ptr_stats + * bit 2: cd_fifo_wptr_stats + * bit 1: cd_fifo_rptr_stats + * bit 0: cq_base_addr + */ +#define MPNIC_MEM_INIT_TCM_VAL 0xf + +/* TDE mem init: + * bit 1: stats mem + * bit 0: dma_head_ptr SRAM + */ +#define MPNIC_MEM_INIT_TDE_VAL 0x3 + +/* TDF mem init, bit 0 covers the descriptor fetch SRAM. */ +#define MPNIC_MEM_INIT_TDF_VAL 0x1 + +/* RXB mem init, bit 0 covers the DMAC TCAM statistics RAM. */ +#define MPNIC_MEM_INIT_RXB_VAL 0x1 + +/* TQS arbiter SRAM init done, one bit per DWRR level. */ +#define MPNIC_TQS_ARB_INIT_VAL 0xf + +/* On-chip SRAM allocated to each queue for descriptor fetch, in units of + * descriptors. Valid values are 64, 128, 256, 512 and 1024. + * + * The partition sizes are chosen for the maximum number of queues the + * device supports, so they do not have to be adjusted when the active + * queue count changes: + * + * BDQ: 1 MiB / (8 B/DESC) / (1024 HPQ + 1024 PPQ) = 64 DESC/QUEUE + * TWQ: 2 MiB / (8 B/DESC) / (1024 TXQ * 2 TWQ) = 128 DESC/QUEUE + */ +#define MPNIC_BDQ_SRAM_DESCS 64u +#define MPNIC_TDF_SRAM_DESCS 128u + +/* A total of 1 MiB worth of Tx credits is available, in units of 128 B. + * The BMC to MAC and host to BMC paths each get a guaranteed share of them + * whether or not anything is routed their way, everything else goes to + * MAC TC0. + */ +#define MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL 800 +#define MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL \ + (SZ_1M / 128 - 2 * MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL) + +/* The recommended lower bound for the TXB threshold is 80, based on a 10K + * MTU. Round up by 25% to stay on the defensive side. The same reasoning + * applies to the arbitration weight, which has to exceed the full packet + * size. + */ +#define MPNIC_TXB_INIT_BMC_THRESH 100 +#define MPNIC_TXB_INIT_P0_THRESH 100 +#define MPNIC_TXB_INIT_P0_ARB_WEIGHTS 0x64 + +/* RXB host drop threshold in units of 128 B beats. Packets targeting a + * queue are dropped when the available credits fall below it. 80 beats is + * 10 KB, which is also the largest frame the device is configured for. + */ +#define MPNIC_RXB_INIT_HOST_DROP_THRESH 0x50 + +/* A total of 8 MiB of Rx buffer is available. The recommended per-TC pool + * for a 800G configuration is 420 KB with all 8 TCs enabled. Only one TC + * is in use, so give it 8 * 420 KB (in units of 128 B) and push the rest + * to the common pool. Start drawing from the common pool as soon as the + * TC0 credits fall below one max sized frame. The BMC keeps a small + * reserve of its own whether or not the host talks to it. + */ +#define MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 (0xd20 * 8) +#define MPNIC_RXB_INIT_BMC_CRDTS 0x20 +#define MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH \ + (SZ_8M / 128 - MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 - \ + MPNIC_RXB_INIT_BMC_CRDTS) +#define MPNIC_RXB_INIT_COMMON_CRDT_THRSH 0x50 + +/* TXB port mode selects the number of active MAC ports for Tx buffer + * credit distribution and arbitration. The hardware only supports single, + * dual and quad port, encoded as b'001, b'010 and b'100. + */ +#define MPNIC_TXB_PORT_MODE_SINGLE 1 + +/* The MAC traffic classes start at index 8 in the TXB arrays, the BMC + * sits above them. + */ +#define MPNIC_TXB_TC_IDX_MAC_0 8 +#define MPNIC_TXB_TC_IDX_BMC 16 + +/* Largest frame the Tx queue scheduler will pass through. Anything above + * it gets truncated. + */ +#define MPNIC_TQS_MTU_CTL0_MAX 0x2800 + +/* The unit of the DWRR quantum is 256 B. It has to be large enough for at + * least 11 MTUs to be transmitted in one quantum; use 15 for headroom. + */ +#define MPNIC_TQS_DWRR_INIT_QUANTUM (15 * MPNIC_TQS_MTU_CTL0_MAX / 256) + +/* Lower bound on the credit available to a queue, group, set or port + * before the scheduler stops issuing requests for it. 15 MTUs, matching + * the DWRR quantum above. + */ +#define MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH (15 * MPNIC_TQS_MTU_CTL0_MAX) + +/* The TQS view of the TXB partitioning between the MAC and the BMC, in + * units of 1 KiB. It has to match the TXB private credits above. + */ +#define MPNIC_TQS_GLBL_TXB_CRDT_UNIT SZ_1K +#define MPNIC_TQS_GLBL_TXB_CRDT_BMC \ + (MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL * 128 / MPNIC_TQS_GLBL_TXB_CRDT_UNIT) +#define MPNIC_TQS_GLBL_TXB_CRDT_MAC \ + (MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL * 128 / \ + MPNIC_TQS_GLBL_TXB_CRDT_UNIT) + +struct mpnic_init_poll { + u64 exp_val; + u32 addr; +}; + +struct mpnic_poll_state { + int poll_idx; + u64 val; +}; + +static void mpnic_tdf_glbl_init(struct mpnic_dev *mpd) +{ + /* Default metadata descriptor, used for frames the driver did not + * prepend one to. + */ + mpnic_wr64(mpd, MPNIC_TWQ_DEF_PRI_TWD, MPNIC_TWD_FLAG_REQ_COMPLETION); +} + +static void mpnic_txb_init(struct mpnic_dev *mpd) +{ + int i; + + mpnic_wr64(mpd, MPNIC_TXB_BMC, MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL); + + /* Zero the private credits of every traffic class, then hand the + * unreserved ones to MAC TC0. + */ + for (i = 0; i < MPNIC_TXB_P0_CNT; i++) + mpnic_wr64(mpd, MPNIC_TXB_P0(i), 0); + mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_MAC_0), + MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL); + mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_BMC), + MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL); + + mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_BMC), + MPNIC_TXB_INIT_BMC_THRESH); + + mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_MAC_0), + MPNIC_TXB_INIT_P0_THRESH); + mpnic_wr64(mpd, MPNIC_TXB_P0_ARB_WEIGHTS(MPNIC_TXB_TC_IDX_MAC_0), + MPNIC_TXB_INIT_P0_ARB_WEIGHTS); + mpnic_wr64(mpd, MPNIC_TXB_PORT_CONFIG, + FIELD_PREP(MPNIC_TXB_PORT_CONFIG_PORT_MODE, + MPNIC_TXB_PORT_MODE_SINGLE)); +} + +static void mpnic_rxb_init(struct mpnic_dev *mpd) +{ + /* Accept all packets that miss dmac tcam, until l2 filtering is + * implemented. + */ + mpnic_wr64(mpd, MPNIC_RXB_PORT_CLASS_CFG(0), + FIELD_PREP(MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION, + MPNIC_L2_ACTION_PASS)); + mpnic_wr64(mpd, MPNIC_RXB_PORT_CFG(0), + FIELD_PREP(MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE, + MPNIC_FCS_MODE_STRIP)); + + mpnic_wr64(mpd, MPNIC_RXB_HOST_DROP_THRESH(0), + MPNIC_RXB_INIT_HOST_DROP_THRESH); + + mpnic_wr64(mpd, MPNIC_RXB_TC_CRDTS(0), + MPNIC_RXB_INIT_POOL_TC_CRDTS_P0); + mpnic_wr64(mpd, MPNIC_RXB_COMMON_CRDT_CTRL_TC(0), + FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH, + MPNIC_RXB_INIT_COMMON_CRDT_THRSH) | + FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS, + MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH) | + MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN); + + /* Only pool 0 is used, it gets all of the common credits */ + mpnic_wr64(mpd, MPNIC_RXB_POOL_COMMON_CRDTS(0), + MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH); + + mpnic_wr64(mpd, MPNIC_RXB_BMC_CRDTS(0), MPNIC_RXB_INIT_BMC_CRDTS); + + mpnic_wr64(mpd, MPNIC_RXB_MEM_INIT_REQ, MPNIC_MEM_INIT_RXB_VAL); +} + +static u64 mpnic_desc_cfg(unsigned int sram_descs, unsigned int q_idx) +{ + /* The hardware encodes the partition size as 64 * 2^n descriptors, + * and its start address in units of 64 descriptors. + */ + return FIELD_PREP(MPNIC_DESC_CFG_NUM_DESCS, __ffs(sram_descs) - 6) | + FIELD_PREP(MPNIC_DESC_CFG_START_ADDR, + q_idx * (sram_descs >> 6)); +} + +static void mpnic_desc_sram_init(struct mpnic_dev *mpd) +{ + int i; + + for (i = 0; i < MPNIC_MAX_TXQS * 2; i++) + mpnic_wr64(mpd, MPNIC_TDF_DESC_CFG(i), + mpnic_desc_cfg(MPNIC_TDF_SRAM_DESCS, i)); + + for (i = 0; i < MPNIC_MAX_RXQS; i++) { + mpnic_wr64(mpd, MPNIC_HPQ_DESC_CFG(i), + mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, i)); + mpnic_wr64(mpd, MPNIC_PPQ_DESC_CFG(i), + mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, + MPNIC_MAX_RXQS + i)); + } +} + +static void mpnic_rxglb_init(struct mpnic_dev *mpd) +{ + /* Descriptor prefetch reads are only issued once 32 descriptors + * worth of FIFO space is available, and no more than 64 descriptors + * are fetched for one queue at a time so that a single queue cannot + * monopolize the bus. Both have to be multiples of 16 to keep the + * reads 128 B aligned. + */ + mpnic_wr64(mpd, MPNIC_BDQ_GLBL_CTL0, + FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH, 32) | + FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE, 64)); + + /* Minimum number of descriptors which has to be available before + * the descriptor engine considers a queue usable, globally and in + * the per-queue prefetch FIFO. + */ + mpnic_wr64(mpd, MPNIC_RDE_CTL, + FIELD_PREP(MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD, + MPNIC_RX_HPQ_DROP_THRS - 1) | + FIELD_PREP(MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD, + MPNIC_RX_PPQ_DROP_THRS - 1) | + FIELD_PREP(MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD, + MPNIC_RX_HPQ_DROP_THRS - 1) | + FIELD_PREP(MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD, + MPNIC_RX_PPQ_DROP_THRS - 1)); + + /* Receive side coalescing is not supported yet */ + mpnic_wr64(mpd, MPNIC_RSC_GLOBAL_CONF, + MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE); + + mpnic_wr64(mpd, MPNIC_BDQ_MEM_INIT_REQ, MPNIC_MEM_INIT_BDQ_VAL); + mpnic_wr64(mpd, MPNIC_RCM_MEM_INIT_REQ, MPNIC_MEM_INIT_RCM_VAL); + mpnic_wr64(mpd, MPNIC_RDE_MEM_INIT_REQ, MPNIC_MEM_INIT_RDE_VAL); + mpnic_wr64(mpd, MPNIC_RPC_MEM_INIT_REQ, MPNIC_MEM_INIT_RPC_VAL); +} + +static void mpnic_txglb_init(struct mpnic_dev *mpd) +{ + /* Nothing is redirected to the BMC until the Tx offload TCAM gets + * programmed with its addresses. + */ + mpnic_wr64(mpd, MPNIC_TOF_TCAM_DEST_REMAP, 0); + + mpnic_wr64(mpd, MPNIC_TCM_MEM_INIT_REQ, MPNIC_MEM_INIT_TCM_VAL); + mpnic_wr64(mpd, MPNIC_TDE_MEM_INIT_REQ, MPNIC_MEM_INIT_TDE_VAL); + mpnic_wr64(mpd, MPNIC_TDF_MEM_INIT_REQ, MPNIC_MEM_INIT_TDF_VAL); +} + +/* Fill the DWRR arbiter memories. Setting the INIT bit makes the hardware + * write the given credit and quantum into every entry at the queue, group, + * set and port level, so nothing has to be programmed per queue. + */ +static void mpnic_tqs_sram_init(struct mpnic_dev *mpd) +{ + mpnic_wr64(mpd, MPNIC_TQS_GROUP_INIT_CTL, + FIELD_PREP(MPNIC_TQS_GROUP_INIT_CTL_QUANTUM, + MPNIC_TQS_DWRR_INIT_QUANTUM) | + MPNIC_TQS_GROUP_INIT_CTL_INIT); + mpnic_wr64(mpd, MPNIC_TQS_SET_INIT_CTL, + FIELD_PREP(MPNIC_TQS_SET_INIT_CTL_QUANTUM, + MPNIC_TQS_DWRR_INIT_QUANTUM) | + MPNIC_TQS_SET_INIT_CTL_INIT); + mpnic_wr64(mpd, MPNIC_TQS_PORT_INIT_CTL, + FIELD_PREP(MPNIC_TQS_PORT_INIT_CTL_QUANTUM, + MPNIC_TQS_DWRR_INIT_QUANTUM) | + MPNIC_TQS_PORT_INIT_CTL_INIT); + mpnic_wr64(mpd, MPNIC_TQS_SRAM_INIT_CTL, + FIELD_PREP(MPNIC_TQS_SRAM_INIT_CTL_QUANTUM, + MPNIC_TQS_DWRR_INIT_QUANTUM) | + MPNIC_TQS_SRAM_INIT_CTL_INIT); +} + +static void mpnic_tqs_init(struct mpnic_dev *mpd) +{ + u64 val; + + /* Initialize to the largest frame we support, the scheduler + * truncates anything above it. The BMC gets the same limit. + */ + mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL0, MPNIC_TQS_MTU_CTL0_MAX); + mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL1, MPNIC_TQS_MTU_CTL0_MAX); + + mpnic_wr64(mpd, MPNIC_TQS_GLBL_CTL0, + MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN); + + mpnic_tqs_sram_init(mpd); + + /* Only port 0 is used. A single traffic class is in use as well, so + * TC0 gets all of the Tx buffer credits not reserved for the BMC. + */ + mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_0, + FIELD_PREP(MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0, + MPNIC_TQS_GLBL_TXB_CRDT_MAC)); + mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_1, 0); + mpnic_wr64(mpd, MPNIC_TQS_GLBL_BMC, + FIELD_PREP(MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS, + MPNIC_TQS_GLBL_TXB_CRDT_BMC)); + + mpnic_wr64(mpd, MPNIC_TQS_PORT_CTL(0), 0); + + /* Map all sets to port 0 */ + mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP0(0), ~0ULL); + mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP1(0), ~0ULL); + + mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_0, + FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE, + MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) | + FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP, + MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH)); + mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_1, + FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET, + MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) | + FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT, + MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH)); + + /* The rate limiters are left uninitialized, so shaping has to stay + * off or nothing would ever get scheduled. + */ + mpnic_wr64(mpd, MPNIC_TQS_GLBL_SHAPING, MPNIC_TQS_GLBL_SHAPING_DISABLE); + + /* Enable fairness protection (phantom eligibility). Read modify + * write so that the reset default slowdown cycle is preserved. + */ + val = mpnic_rd64(mpd, MPNIC_TQS_SLOWDOWN_CTL); + val |= MPNIC_TQS_SLOWDOWN_CTL_ENABLE; + mpnic_wr64(mpd, MPNIC_TQS_SLOWDOWN_CTL, val); + + /* Use immediate credit decrement at every DWRR level so that the + * credit reflects a grant in the same cycle. + */ + val = mpnic_rd64(mpd, MPNIC_TQS_ARB_CTL); + val |= MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN | + MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN | + MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN | + MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN | + MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN | + MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN; + mpnic_wr64(mpd, MPNIC_TQS_ARB_CTL, val); +} + +/* The MPS and CLS fields sit at the same bit positions in every block, so + * one set of masks covers both the RNI and the TNI registers. + */ +static void mpnic_mps_init(struct mpnic_dev *mpd, u32 reg, unsigned int mps, + unsigned int cls) +{ + u64 val = mpnic_rd64(mpd, reg); + + val &= ~(MPNIC_RNI_RDE_CTL_MPS | MPNIC_RNI_RDE_CTL_CLS); + val |= FIELD_PREP(MPNIC_RNI_RDE_CTL_MPS, mps) | + FIELD_PREP(MPNIC_RNI_RDE_CTL_CLS, cls); + + mpnic_wr64(mpd, reg, val); +} + +/* Likewise for the MRRS and CLS fields, which have their own common + * layout. + */ +static void mpnic_mrrs_init(struct mpnic_dev *mpd, u32 reg, unsigned int mrrs, + unsigned int cls) +{ + u64 val = mpnic_rd64(mpd, reg); + + val &= ~(MPNIC_TNI_GLBL_TDF_CTL_MRRS | MPNIC_TNI_GLBL_TDF_CTL_CLS); + val |= FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_MRRS, mrrs) | + FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_CLS, cls); + + mpnic_wr64(mpd, reg, val); +} + +/** + * mpnic_axi_init - Configure AXI bus parameters from host PCIe capabilities + * @mpd: Device to configure + * + * Programs the max read request size, max payload size and cache line size + * of the DMA engines. The hardware encodes all three as a power of 2 index, + * MRRS and MPS relative to 128 B and CLS relative to 64 B. + * + * MAX_OT and MAX_OB are left at their hardware defaults. + */ +static void mpnic_axi_init(struct mpnic_dev *mpd) +{ + int mps, cls, mrrs; + + mps = clamp(ilog2(mpd->mps) - 7, 0, 3); + cls = clamp(ilog2(L1_CACHE_BYTES) - 6, 0, 3); + + mpnic_mps_init(mpd, MPNIC_RNI_RDE_CTL, mps, cls); + mpnic_mps_init(mpd, MPNIC_RNI_RCM_CTL, mps, cls); + mpnic_mps_init(mpd, MPNIC_TNI_GLBL_TCM_CTL, mps, cls); + + mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 3); + mpnic_mrrs_init(mpd, MPNIC_RNI_RBP_CTL, mrrs, cls); + mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDF_CTL, mrrs, cls); + + /* TDE supports a wider range of MRRS encodings. */ + mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 5); + mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDE_CTL, mrrs, cls); +} + +/** + * mpnic_ro_init - Set relaxed ordering on the outbound TLP attributes + * @mpd: Device to configure + * + * Completions must stay ordered so that they are not observed before the + * payload DMA they describe has landed, so RCM and TCM are left alone. + */ +static void mpnic_ro_init(struct mpnic_dev *mpd) +{ + u64 attr = mpd->relaxed_ord ? MPNIC_OB_ATTR_RO : 0; + + mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_H, attr); + mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_P, attr); + mpnic_wr64(mpd, MPNIC_OB_ATTR_TDF, attr); + mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_HPQ, attr); + mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_PPQ, attr); + mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_H, attr); + mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_P, attr); +} + +static bool mpnic_init_status_ready(struct mpnic_dev *mpd, + const struct mpnic_init_poll *polls, + struct mpnic_poll_state *state) +{ + u64 val; + int i; + + for (i = state->poll_idx; polls[i].addr; i++) { + val = mpnic_rd64(mpd, polls[i].addr); + + if ((val & polls[i].exp_val) != polls[i].exp_val) { + state->poll_idx = i; + state->val = val; + return false; + } + } + + return true; +} + +/** + * mpnic_mem_init_poll - Wait for the memory initializations to complete + * @mpd: Device to poll + * + * The blocks initialize their memories in parallel, so walk the status + * registers in order and only go back to sleep on the first one which is + * not done yet. + * + * Return: 0 on success, -ETIMEDOUT if not everything completed in time + */ +static int mpnic_mem_init_poll(struct mpnic_dev *mpd) +{ + static const struct mpnic_init_poll polls[] = { + { MPNIC_TQS_ARB_INIT_VAL, MPNIC_TQS_SRAM_STS }, + { MPNIC_MEM_INIT_BDQ_VAL, MPNIC_BDQ_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_RCM_VAL, MPNIC_RCM_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_RDE_VAL, MPNIC_RDE_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_RPC_VAL, MPNIC_RPC_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_TCM_VAL, MPNIC_TCM_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_TDE_VAL, MPNIC_TDE_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_TDF_VAL, MPNIC_TDF_MEM_INIT_DONE }, + { MPNIC_MEM_INIT_RXB_VAL, MPNIC_RXB_MEM_INIT_DONE }, + { 0 }, + }; + struct mpnic_poll_state state = {}; + bool done; + int err; + + err = read_poll_timeout(mpnic_init_status_ready, done, done, + MPNIC_MEM_INIT_POLL_US, MPNIC_MEM_INIT_TO_US, + false, mpd, polls, &state); + if (err) + dev_err(mpd->dev, "Poll timeout for reg 0x%x: 0x%llx\n", + polls[state.poll_idx].addr, state.val); + + return err; +} + +int mpnic_dev_init(struct mpnic_dev *mpd) +{ + int err; + + mpnic_tdf_glbl_init(mpd); + mpnic_txb_init(mpd); + mpnic_rxb_init(mpd); + mpnic_desc_sram_init(mpd); + mpnic_axi_init(mpd); + mpnic_ro_init(mpd); + mpnic_rxglb_init(mpd); + mpnic_txglb_init(mpd); + mpnic_tqs_init(mpd); + + err = mpnic_mem_init_poll(mpd); + if (err) { + dev_err(mpd->dev, "Device initialization failed: %d\n", err); + return err; + } + + if (!mpnic_present(mpd)) + return -EIO; + + return 0; +} diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_irq.c b/drivers/net/ethernet/meta/mpnic/mpnic_irq.c new file mode 100644 index 000000000000..bcc33655cbbe --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_irq.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/cpumask.h> +#include <linux/interrupt.h> +#include <linux/minmax.h> +#include <linux/pci.h> + +#include "mpnic.h" + +int mpnic_request_irq(struct mpnic_dev *mpd, int nr, irq_handler_t handler, + unsigned long flags, const char *name, void *data) +{ + struct pci_dev *pdev = to_pci_dev(mpd->dev); + int irq = pci_irq_vector(pdev, nr); + + if (irq < 0) + return irq; + + return request_irq(irq, handler, flags, name, data); +} + +void mpnic_free_irq(struct mpnic_dev *mpd, int nr, void *data) +{ + struct pci_dev *pdev = to_pci_dev(mpd->dev); + int irq = pci_irq_vector(pdev, nr); + + if (irq < 0) + return; + + free_irq(irq, data); +} + +void mpnic_free_irqs(struct mpnic_dev *mpd) +{ + struct pci_dev *pdev = to_pci_dev(mpd->dev); + + mpd->num_irqs = 0; + pci_free_irq_vectors(pdev); +} + +int mpnic_alloc_irqs(struct mpnic_dev *mpd) +{ + unsigned int wanted_irqs = MPNIC_NON_NAPI_VECTORS; + struct pci_dev *pdev = to_pci_dev(mpd->dev); + int num_irqs; + + wanted_irqs += min_t(unsigned int, num_online_cpus(), MPNIC_MAX_RXQS); + num_irqs = pci_alloc_irq_vectors(pdev, MPNIC_NON_NAPI_VECTORS + 1, + wanted_irqs, PCI_IRQ_MSIX); + if (num_irqs < 0) { + dev_err(mpd->dev, "Failed to allocate MSI-X entries: %d\n", + num_irqs); + return num_irqs; + } + + if (num_irqs < wanted_irqs) + dev_warn(mpd->dev, "Allocated %d IRQs, expected %u\n", + num_irqs, wanted_irqs); + + mpd->num_irqs = num_irqs; + + return 0; +} diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c new file mode 100644 index 000000000000..7fdf57f70151 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/etherdevice.h> +#include <linux/ipv6.h> +#include <linux/netdevice.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <net/netdev_lock.h> + +#include "mpnic.h" +#include "mpnic_netdev.h" +#include "mpnic_txrx.h" + +static int mpnic_open(struct net_device *netdev) +{ + struct mpnic_net *mpn = netdev_priv(netdev); + int err; + + err = mpnic_alloc_napi_vectors(mpn); + if (err) + return err; + + err = mpnic_alloc_resources(mpn); + if (err) + goto err_free_napi_vectors; + + err = mpnic_set_netif_queues(mpn); + if (err) + goto err_free_resources; + + mpnic_stats_attach_rings(mpn); + + mpnic_enable(mpn); + mpnic_fill(mpn); + mpnic_napi_enable(mpn); + + netif_tx_wake_all_queues(netdev); + netif_carrier_on(netdev); + + schedule_delayed_work(&mpn->mpd->service_task, HZ); + + return 0; + +err_free_resources: + mpnic_free_resources(mpn); +err_free_napi_vectors: + mpnic_free_napi_vectors(mpn); + return err; +} + +static int mpnic_stop(struct net_device *netdev) +{ + struct mpnic_net *mpn = netdev_priv(netdev); + + cancel_delayed_work(&mpn->mpd->service_task); + netif_carrier_off(netdev); + + mpnic_napi_disable(mpn); + netif_tx_disable(netdev); + + mpnic_disable(mpn); + mpnic_wait_all_queues_idle(mpn->mpd); + mpnic_flush(mpn); + + mpnic_stats_fold_rings(mpn); + + mpnic_reset_netif_queues(mpn); + mpnic_free_resources(mpn); + mpnic_free_napi_vectors(mpn); + + return 0; +} + +static void +mpnic_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64) +{ + struct mpnic_net *mpn = netdev_priv(dev); + u64 bytes, packets, dropped, errors; + struct mpnic_queue_stats *stats; + struct mpnic_base_stats *base; + unsigned int start, i; + + rcu_read_lock(); + + base = rcu_dereference(mpn->stats.base); + + stats64->tx_bytes = base->tx.bytes; + stats64->tx_packets = base->tx.packets; + stats64->tx_dropped = base->tx.dropped; + + stats64->rx_bytes = base->rx.bytes; + stats64->rx_packets = base->rx.packets; + stats64->rx_dropped = base->rx.dropped; + stats64->rx_errors = base->rx.errors; + + for (i = 0; i < base->num_tx_queues; i++) { + struct mpnic_ring *txr = base->txr[i]; + struct mpnic_q_triad *qt; + + qt = container_of(txr, struct mpnic_q_triad, sub0); + + stats = &qt->cmpl.stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = u64_stats_read(&stats->tcq.bytes); + packets = u64_stats_read(&stats->tcq.packets); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats = &txr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + dropped = u64_stats_read(&stats->twq.dropped); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats64->tx_bytes += bytes; + stats64->tx_packets += packets; + stats64->tx_dropped += dropped; + } + + for (i = 0; i < base->num_rx_queues; i++) { + struct mpnic_ring *rxr = base->rxr[i]; + + stats = &rxr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = u64_stats_read(&stats->rcq.bytes); + packets = u64_stats_read(&stats->rcq.packets); + dropped = u64_stats_read(&stats->rcq.dropped); + errors = u64_stats_read(&stats->rcq.errors); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats64->rx_bytes += bytes; + stats64->rx_packets += packets; + stats64->rx_dropped += dropped; + stats64->rx_errors += errors; + } + + rcu_read_unlock(); +} + +static const struct net_device_ops mpnic_netdev_ops = { + .ndo_open = mpnic_open, + .ndo_stop = mpnic_stop, + .ndo_validate_addr = eth_validate_addr, + .ndo_start_xmit = mpnic_xmit_frame, + .ndo_get_stats64 = mpnic_get_stats64, +}; + +static u64 mpnic_bdq_alloc_failed(struct mpnic_ring *bdq) +{ + struct mpnic_queue_stats *stats = &bdq->stats; + unsigned int start; + u64 alloc_failed; + + do { + start = u64_stats_fetch_begin(&stats->syncp); + alloc_failed = u64_stats_read(&stats->bdq.alloc_failed); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + return alloc_failed; +} + +static void mpnic_get_queue_stats_rx(struct net_device *dev, int idx, + struct netdev_queue_stats_rx *rx) +{ + struct mpnic_net *mpn = netdev_priv(dev); + struct mpnic_ring *rxr = mpn->rx[idx]; + u64 bytes, packets, alloc_failed; + struct mpnic_queue_stats *stats; + struct mpnic_q_triad *qt; + unsigned int start; + + if (!rxr) + return; + + stats = &rxr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = u64_stats_read(&stats->rcq.bytes); + packets = u64_stats_read(&stats->rcq.packets); + alloc_failed = u64_stats_read(&stats->rcq.alloc_failed); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + qt = container_of(rxr, struct mpnic_q_triad, cmpl); + alloc_failed += mpnic_bdq_alloc_failed(&qt->sub0); + alloc_failed += mpnic_bdq_alloc_failed(&qt->sub1); + + rx->bytes = bytes; + rx->packets = packets; + rx->alloc_fail = alloc_failed; +} + +static void mpnic_get_queue_stats_tx(struct net_device *dev, int idx, + struct netdev_queue_stats_tx *tx) +{ + struct mpnic_net *mpn = netdev_priv(dev); + struct mpnic_ring *txr = mpn->tx[idx]; + struct mpnic_queue_stats *stats; + u64 bytes, packets, stop, wake; + struct mpnic_q_triad *qt; + unsigned int start; + + if (!txr) + return; + + qt = container_of(txr, struct mpnic_q_triad, sub0); + + stats = &qt->cmpl.stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = u64_stats_read(&stats->tcq.bytes); + packets = u64_stats_read(&stats->tcq.packets); + wake = u64_stats_read(&stats->tcq.wake); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats = &txr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + stop = u64_stats_read(&stats->twq.stop); + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + tx->bytes = bytes; + tx->packets = packets; + tx->stop = stop; + tx->wake = wake; +} + +static void mpnic_get_base_stats(struct net_device *dev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx) +{ + struct mpnic_net *mpn = netdev_priv(dev); + struct mpnic_base_stats *base; + + base = netdev_lock_dereference(mpn->stats.base, dev); + + tx->bytes = base->tx.bytes; + tx->packets = base->tx.packets; + tx->stop = base->tx.stop; + tx->wake = base->tx.wake; + + rx->bytes = base->rx.bytes; + rx->packets = base->rx.packets; + rx->alloc_fail = base->rx.alloc_failed; +} + +static const struct netdev_stat_ops mpnic_stat_ops = { + .get_queue_stats_rx = mpnic_get_queue_stats_rx, + .get_queue_stats_tx = mpnic_get_queue_stats_tx, + .get_base_stats = mpnic_get_base_stats, +}; + +/** + * mpnic_netdev_free - Free the netdev associated with mpnic + * @mpd: Driver specific structure to free netdev from + **/ +void mpnic_netdev_free(struct mpnic_dev *mpd) +{ + free_netdev(mpd->netdev); + mpd->netdev = NULL; +} + +/** + * mpnic_netdev_alloc - Allocate a netdev and associate it with mpnic + * @mpd: Driver specific structure to associate the netdev with + * + * Return: NULL on failure. + **/ +struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd) +{ + struct net_device *netdev; + struct mpnic_net *mpn; + unsigned int queues; + + netdev = alloc_etherdev_mq(sizeof(*mpn), MPNIC_MAX_RXQS); + if (!netdev) + return NULL; + + SET_NETDEV_DEV(netdev, mpd->dev); + mpd->netdev = netdev; + + netdev->netdev_ops = &mpnic_netdev_ops; + netdev->stat_ops = &mpnic_stat_ops; + netdev->request_ops_lock = true; + + mpn = netdev_priv(netdev); + mpn->netdev = netdev; + mpn->mpd = mpd; + + mpn->stats.buf[0].txr = mpn->tx; + mpn->stats.buf[0].rxr = mpn->rx; + RCU_INIT_POINTER(mpn->stats.base, &mpn->stats.buf[0]); + + mpn->txq_size = MPNIC_TXQ_SIZE_DEFAULT; + mpn->hpq_size = MPNIC_HPQ_SIZE_DEFAULT; + mpn->ppq_size = MPNIC_PPQ_SIZE_DEFAULT; + mpn->rcq_size = MPNIC_RCQ_SIZE_DEFAULT; + + queues = min(netif_get_num_default_rss_queues(), + mpd->num_irqs - MPNIC_NON_NAPI_VECTORS); + mpn->num_tx_queues = queues; + mpn->num_rx_queues = queues; + mpn->num_napi = queues; + + netdev->features |= NETIF_F_SG; + netdev->hw_features |= netdev->features; + netdev->vlan_features |= netdev->features; + + netdev->min_mtu = IPV6_MIN_MTU; + netdev->max_mtu = MPNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; + + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + + return netdev; +} + +static int mpnic_dsn_to_mac_addr(u64 dsn, u8 *addr) +{ + addr[0] = (dsn >> 56) & 0xFF; + addr[1] = (dsn >> 48) & 0xFF; + addr[2] = (dsn >> 40) & 0xFF; + addr[3] = (dsn >> 16) & 0xFF; + addr[4] = (dsn >> 8) & 0xFF; + addr[5] = dsn & 0xFF; + + return is_valid_ether_addr(addr) ? 0 : -EINVAL; +} + +/** + * mpnic_netdev_register - Assign the MAC address and register the netdev + * @netdev: Netdev to register + * + * The permanent address is derived from the PCIe device serial number, the + * same way the firmware and the BMC derive it. A random address would break + * provisioning, so refuse to spawn the interface if the serial number does + * not yield a valid one. + * + * Return: non-zero on failure. + **/ +int mpnic_netdev_register(struct net_device *netdev) +{ + struct mpnic_net *mpn = netdev_priv(netdev); + struct mpnic_dev *mpd = mpn->mpd; + u8 addr[ETH_ALEN]; + int err; + + err = mpnic_dsn_to_mac_addr(mpd->dsn, addr); + if (err) { + dev_err(mpd->dev, "MAC addr %pM invalid\n", addr); + return err; + } + + ether_addr_copy(netdev->perm_addr, addr); + eth_hw_addr_set(netdev, addr); + + err = netif_set_real_num_queues(netdev, mpn->num_tx_queues, + mpn->num_rx_queues); + if (err) + return err; + + /* Abort if MMIO has failed. This has to be the last check before + * registration, the register accessors can only detach the device + * once it has been registered. + */ + if (!mpnic_present(mpd)) + return -EIO; + + return register_netdev(netdev); +} diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h new file mode 100644 index 000000000000..6711766ebc3c --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _MPNIC_NETDEV_H_ +#define _MPNIC_NETDEV_H_ + +#include <linux/netdevice.h> +#include <linux/rcupdate.h> +#include <linux/types.h> + +#include "mpnic.h" +#include "mpnic_txrx.h" + +struct mpnic_base_stats { + struct { + u64 packets; + u64 bytes; + u64 dropped; + u64 stop; + u64 wake; + } tx; + struct { + u64 packets; + u64 bytes; + u64 dropped; + u64 alloc_failed; + u64 errors; + } rx; + + unsigned int num_tx_queues; + unsigned int num_rx_queues; + struct mpnic_ring **txr; + struct mpnic_ring **rxr; +}; + +struct mpnic_net { + struct mpnic_ring *tx[MPNIC_MAX_TXQS]; + struct mpnic_ring *rx[MPNIC_MAX_RXQS]; + + struct mpnic_napi_vector *napi[MPNIC_MAX_NAPI_VECTORS]; + + struct net_device *netdev; + struct mpnic_dev *mpd; + + u32 txq_size; + u32 hpq_size; + u32 ppq_size; + u32 rcq_size; + + u16 num_napi; + u16 num_tx_queues; + u16 num_rx_queues; + + struct { + struct mpnic_base_stats __rcu *base; + struct mpnic_base_stats buf[2]; + } stats; +}; + +struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd); +void mpnic_netdev_free(struct mpnic_dev *mpd); +int mpnic_netdev_register(struct net_device *netdev); + +#endif /* _MPNIC_NETDEV_H_ */ diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_pci.c b/drivers/net/ethernet/meta/mpnic/mpnic_pci.c new file mode 100644 index 000000000000..125fb26edf93 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_pci.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/module.h> +#include <linux/netdevice.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "mpnic.h" +#include "mpnic_netdev.h" +#include "mpnic_txrx.h" + +#define PCI_DEVICE_ID_META_MPNIC 0x0014 + +static void mpnic_mmio_err(struct mpnic_dev *mpd, u32 reg) +{ + /* Hardware is giving us all 1's reads, assume it is gone */ + WRITE_ONCE(mpd->uc_addr0, NULL); + + dev_err(mpd->dev, + "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", + reg, reg << 2); + + /* Tell the stack the device has lost its PCIe link */ + if (mpd->netdev) + netif_device_detach(mpd->netdev); +} + +u64 mpnic_rd64(struct mpnic_dev *mpd, u32 reg) +{ + u32 __iomem *csr = READ_ONCE(mpd->uc_addr0); + u64 value; + + if (!csr) + return ~0ULL; + + value = readq(csr + reg); + + /* If any bits are 0 value should be valid */ + if (~value) + return value; + + /* All ones can be a valid value, so confirm against a register + * which never reads that way on a live device. + */ + if (reg != MPNIC_BDQ_SPARE && ~readq(csr + MPNIC_BDQ_SPARE)) + return value; + + mpnic_mmio_err(mpd, reg); + + return ~0ULL; +} + +static void mpnic_service_task(struct work_struct *work) +{ + struct mpnic_dev *mpd = container_of(to_delayed_work(work), + struct mpnic_dev, service_task); + struct net_device *netdev = mpd->netdev; + + netdev_lock(netdev); + + if (netif_carrier_ok(netdev)) + mpnic_napi_depletion_check(netdev_priv(netdev)); + + if (netif_running(netdev)) + schedule_delayed_work(&mpd->service_task, HZ); + + netdev_unlock(netdev); +} + +static struct mpnic_dev *mpnic_alloc(struct pci_dev *pdev) +{ + struct mpnic_dev *mpd; + + mpd = kzalloc_obj(*mpd); + if (!mpd) + return NULL; + + pci_set_drvdata(pdev, mpd); + mpd->dev = &pdev->dev; + + mpd->dsn = pci_get_dsn(pdev); + mpd->mps = pcie_get_mps(pdev); + mpd->readrq = pcie_get_readrq(pdev); + mpd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev); + + INIT_DELAYED_WORK(&mpd->service_task, mpnic_service_task); + + return mpd; +} + +/** + * mpnic_probe - Device initialization routine + * @pdev: PCI device information struct + * @ent: entry in mpnic_pci_tbl + * + * Return: 0 on success, negative on failure + **/ +static int mpnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct net_device *netdev; + void __iomem *uc_addr0; + struct mpnic_dev *mpd; + int err; + + if (pdev->error_state != pci_channel_io_normal) { + dev_err(&pdev->dev, + "PCI device still in an error state. Unable to load...\n"); + return -EIO; + } + + err = pcim_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "PCI enable device failed: %d\n", err); + return err; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); + return err; + } + + mpd = mpnic_alloc(pdev); + if (!mpd) + return -ENOMEM; + + uc_addr0 = pcim_iomap_region(pdev, 0, MPNIC_DRV_NAME); + if (IS_ERR(uc_addr0)) { + err = PTR_ERR(uc_addr0); + dev_err(&pdev->dev, "Mapping the register file failed: %d\n", + err); + goto err_free_mpd; + } + mpd->uc_addr0 = uc_addr0; + + pci_set_master(pdev); + pci_save_state(pdev); + + err = mpnic_alloc_irqs(mpd); + if (err) + goto err_free_mpd; + + err = mpnic_dev_init(mpd); + if (err) + goto err_free_irqs; + + netdev = mpnic_netdev_alloc(mpd); + if (!netdev) { + dev_err(&pdev->dev, "Netdev allocation failed\n"); + err = -ENOMEM; + goto err_free_irqs; + } + + err = mpnic_netdev_register(netdev); + if (err) { + dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); + goto err_free_netdev; + } + + return 0; + +err_free_netdev: + mpnic_netdev_free(mpd); +err_free_irqs: + mpnic_free_irqs(mpd); +err_free_mpd: + kfree(mpd); + + return err; +} + +/** + * mpnic_remove - Device removal routine + * @pdev: PCI device information struct + **/ +static void mpnic_remove(struct pci_dev *pdev) +{ + struct mpnic_dev *mpd = pci_get_drvdata(pdev); + + unregister_netdev(mpd->netdev); + cancel_delayed_work_sync(&mpd->service_task); + mpnic_netdev_free(mpd); + mpnic_free_irqs(mpd); + kfree(mpd); +} + +static const struct pci_device_id mpnic_pci_tbl[] = { + { PCI_VDEVICE(META, PCI_DEVICE_ID_META_MPNIC) }, + /* required last entry */ + {} +}; +MODULE_DEVICE_TABLE(pci, mpnic_pci_tbl); + +static struct pci_driver mpnic_driver = { + .name = MPNIC_DRV_NAME, + .id_table = mpnic_pci_tbl, + .probe = mpnic_probe, + .remove = mpnic_remove, +}; + +module_pci_driver(mpnic_driver); + +MODULE_DESCRIPTION("Meta Platforms Network Interface Controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c new file mode 100644 index 000000000000..93abdec44642 --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c @@ -0,0 +1,1573 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/dma-mapping.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <net/netdev_lock.h> +#include <net/page_pool/helpers.h> + +#include "mpnic.h" +#include "mpnic_netdev.h" +#include "mpnic_txrx.h" + +struct mpnic_xmit_cb { + u32 bytecount; + u8 desc_count; +}; + +#define MPNIC_XMIT_CB(__skb) ((struct mpnic_xmit_cb *)((__skb)->cb)) +#define MPNIC_TWD_TYPE_MASK(_type) \ + cpu_to_le64(FIELD_PREP(MPNIC_TWD_TYPE, MPNIC_TWD_TYPE_##_type)) + +/* Leave the interrupt moderation counters alone when arming or masking */ +#define MPNIC_TIM_PARAM_CFG_PRESERVE_MASK \ + (MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT | \ + MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT | \ + MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT) + +static void mpnic_nv_irq_disable(struct mpnic_napi_vector *nv) +{ + mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx), + MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | + MPNIC_TIM_CTL1_MASK_EN | MPNIC_TIM_CTL1_MASK); +} + +static void mpnic_nv_irq_rearm(struct mpnic_napi_vector *nv) +{ + /* Rearming a single queue on a given IRQ rearms all the other + * queues mapped to the same IRQ. + */ + mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx), + MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | MPNIC_TIM_CTL1_MASK_EN); +} + +static void mpnic_nv_irq_trigger(struct mpnic_napi_vector *nv) +{ + mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx), + MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | + MPNIC_TIM_CTL1_MASK_EN | MPNIC_TIM_CTL1_TRIGGER); +} + +static unsigned int mpnic_desc_unused(struct mpnic_ring *ring) +{ + return (ring->head - ring->tail - 1) & ring->size_mask; +} + +static unsigned int mpnic_desc_used(struct mpnic_ring *ring) +{ + return (READ_ONCE(ring->tail) - READ_ONCE(ring->head)) & + ring->size_mask; +} + +static struct netdev_queue *mpnic_txring_txq(const struct net_device *dev, + const struct mpnic_ring *ring) +{ + return netdev_get_tx_queue(dev, ring->q_idx); +} + +static bool +mpnic_maybe_stop_tx(const struct net_device *dev, struct mpnic_ring *ring) +{ + int res; + + res = netif_txq_maybe_stop(mpnic_txring_txq(dev, ring), + mpnic_desc_unused(ring), MPNIC_MAX_SKB_DESC, + MPNIC_TX_DESC_WAKEUP); + if (!res) { + u64_stats_update_begin(&ring->stats.syncp); + u64_stats_inc(&ring->stats.twq.stop); + u64_stats_update_end(&ring->stats.syncp); + } + + return !res; +} + +static void mpnic_tx_doorbell(struct mpnic_ring *ring, __le64 *meta) +{ + *meta |= cpu_to_le64(MPNIC_TWD_FLAG_REQ_COMPLETION); + ring->deferred_meta = -1; + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writeq(ring->tail, ring->doorbell); +} + +/* Packets handed to us with xmit_more set are left in the ring without a + * doorbell, and without a completion request, in the expectation that the + * packet ending the burst will ring for all of them. If that packet gets + * dropped instead we have to ring here, otherwise the descriptors sit in + * the ring until the next transmit, which may never come. + */ +static void mpnic_tx_flush_doorbell(struct mpnic_ring *ring) +{ + if (ring->deferred_meta >= 0) + mpnic_tx_doorbell(ring, &ring->desc[ring->deferred_meta]); +} + +static void mpnic_unmap_single_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + + dma_unmap_single(dev, FIELD_GET(MPNIC_TWD_ADDR, raw_twd), + FIELD_GET(MPNIC_TWD_LEN, raw_twd), DMA_TO_DEVICE); +} + +static void mpnic_unmap_page_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + + dma_unmap_page(dev, FIELD_GET(MPNIC_TWD_ADDR, raw_twd), + FIELD_GET(MPNIC_TWD_LEN, raw_twd), DMA_TO_DEVICE); +} + +static bool +mpnic_tx_map(struct mpnic_ring *ring, struct sk_buff *skb, __le64 *meta) +{ + struct device *dev = skb->dev->dev.parent; + unsigned int tail = ring->tail, first; + unsigned int size, data_len; + skb_frag_t *frag; + dma_addr_t dma; + __le64 *twd; + + tail++; + tail &= ring->size_mask; + first = tail; + + size = skb_headlen(skb); + data_len = skb->data_len; + + if (size > FIELD_MAX(MPNIC_TWD_LEN)) + goto err_dma; + + dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); + + for (frag = &skb_shinfo(skb)->frags[0];; frag++) { + twd = &ring->desc[tail]; + + if (dma_mapping_error(dev, dma)) + goto err_dma; + + *twd = cpu_to_le64(FIELD_PREP(MPNIC_TWD_ADDR, dma) | + FIELD_PREP(MPNIC_TWD_LEN, size) | + FIELD_PREP(MPNIC_TWD_TYPE, + MPNIC_TWD_TYPE_AL)); + + tail++; + tail &= ring->size_mask; + + if (!data_len) + break; + + size = skb_frag_size(frag); + data_len -= size; + + if (size > FIELD_MAX(MPNIC_TWD_LEN)) + goto err_dma; + + dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); + } + + *twd |= MPNIC_TWD_TYPE_MASK(LAST_AL); + + MPNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask; + + skb_tx_timestamp(skb); + + ring->tail = tail; + + /* Verify there is room for another packet */ + mpnic_maybe_stop_tx(skb->dev, ring); + + if (__netdev_tx_sent_queue(mpnic_txring_txq(skb->dev, ring), + MPNIC_XMIT_CB(skb)->bytecount, + netdev_xmit_more())) + mpnic_tx_doorbell(ring, meta); + else + ring->deferred_meta = meta - ring->desc; + + return false; +err_dma: + if (net_ratelimit()) + netdev_err(skb->dev, "TX DMA map failed\n"); + + while (tail != first) { + tail--; + tail &= ring->size_mask; + twd = &ring->desc[tail]; + if (tail == first) + mpnic_unmap_single_twd(dev, twd); + else + mpnic_unmap_page_twd(dev, twd); + } + + return true; +} + +#define MPNIC_MIN_FRAME_LEN 60 + +static netdev_tx_t mpnic_xmit_frame_ring(struct sk_buff *skb, + struct mpnic_ring *ring) +{ + __le64 *meta = &ring->desc[ring->tail]; + u32 tail = ring->tail; + + if (skb_put_padto(skb, MPNIC_MIN_FRAME_LEN)) + goto err_drop; + + if (mpnic_maybe_stop_tx(skb->dev, ring)) { + mpnic_tx_flush_doorbell(ring); + return NETDEV_TX_BUSY; + } + + ring->tx_buf[tail] = skb; + *meta = cpu_to_le64(MPNIC_TWD_FLAG_DEST_MAC); + + MPNIC_XMIT_CB(skb)->bytecount = skb->len; + MPNIC_XMIT_CB(skb)->desc_count = 0; + + if (mpnic_tx_map(ring, skb, meta)) + goto err_free; + + return NETDEV_TX_OK; + +err_free: + dev_kfree_skb_any(skb); + ring->tx_buf[tail] = NULL; + ring->tail = tail; +err_drop: + mpnic_tx_flush_doorbell(ring); + + u64_stats_update_begin(&ring->stats.syncp); + u64_stats_inc(&ring->stats.twq.dropped); + u64_stats_update_end(&ring->stats.syncp); + + return NETDEV_TX_OK; +} + +netdev_tx_t mpnic_xmit_frame(struct sk_buff *skb, struct net_device *dev) +{ + struct mpnic_net *mpn = netdev_priv(dev); + + return mpnic_xmit_frame_ring(skb, mpn->tx[skb_get_queue_mapping(skb)]); +} + +static void mpnic_clean_twq0(struct mpnic_napi_vector *nv, int napi_budget, + struct mpnic_q_triad *qt, bool discard, + unsigned int hw_head) +{ + u64 total_bytes = 0, total_packets = 0; + struct mpnic_ring *ring = &qt->sub0; + struct mpnic_ring *cmpl = &qt->cmpl; + unsigned int head = ring->head; + struct netdev_queue *txq; + unsigned int clean_desc; + + clean_desc = (hw_head - head) & ring->size_mask; + + while (clean_desc) { + struct sk_buff *skb = ring->tx_buf[head]; + unsigned int desc_cnt; + + desc_cnt = MPNIC_XMIT_CB(skb)->desc_count; + if (desc_cnt > clean_desc) + break; + + ring->tx_buf[head] = NULL; + + clean_desc -= desc_cnt; + + /* Step over the metadata descriptor */ + head++; + head &= ring->size_mask; + desc_cnt--; + + mpnic_unmap_single_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + desc_cnt--; + + while (desc_cnt--) { + mpnic_unmap_page_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + } + + total_bytes += MPNIC_XMIT_CB(skb)->bytecount; + total_packets++; + + napi_consume_skb(skb, napi_budget); + } + + if (!total_bytes) + return; + + ring->head = head; + + if (discard) { + local_bh_disable(); + u64_stats_update_begin(&ring->stats.syncp); + u64_stats_add(&ring->stats.twq.dropped, total_packets); + u64_stats_update_end(&ring->stats.syncp); + local_bh_enable(); + return; + } + + u64_stats_update_begin(&cmpl->stats.syncp); + u64_stats_add(&cmpl->stats.tcq.bytes, total_bytes); + u64_stats_add(&cmpl->stats.tcq.packets, total_packets); + u64_stats_update_end(&cmpl->stats.syncp); + + txq = mpnic_txring_txq(nv->napi.dev, ring); + if (!netif_txq_completed_wake(txq, total_packets, total_bytes, + mpnic_desc_unused(ring), + MPNIC_TX_DESC_WAKEUP)) { + u64_stats_update_begin(&cmpl->stats.syncp); + u64_stats_inc(&cmpl->stats.tcq.wake); + u64_stats_update_end(&cmpl->stats.syncp); + } +} + +static void mpnic_commit_cq_head(struct mpnic_ring *cmpl) +{ + u32 head = cmpl->head; + + /* The tail shadows the last value written to the doorbell, so a + * completion queue which has not moved costs no MMIO write. + */ + if (cmpl->tail != head) { + cmpl->tail = head; + writeq(head & cmpl->size_mask, cmpl->doorbell); + } +} + +static void mpnic_clean_tcq(struct mpnic_napi_vector *nv, + struct mpnic_q_triad *qt, int napi_budget) +{ + struct mpnic_ring *cmpl = &qt->cmpl; + __le64 *raw_tcd, done; + u32 head = cmpl->head; + s32 head0 = -1; + + done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(MPNIC_TCD_DONE); + raw_tcd = &cmpl->desc[head & cmpl->size_mask]; + + /* Walk the completion queue collecting the heads reported by NIC. + * Only the first work queue is enabled and no packet asks for a + * timestamp, so every completion is a plain head update and the + * descriptor type does not have to be decoded. + */ + while ((*raw_tcd & cpu_to_le64(MPNIC_TCD_DONE)) == done) { + u64 tcd; + + dma_rmb(); + + tcd = le64_to_cpu(*raw_tcd); + head0 = FIELD_GET(MPNIC_TCD_TYPE0_HEAD0, tcd); + + raw_tcd++; + head++; + + if (unlikely(!(head & cmpl->size_mask))) { + done ^= cpu_to_le64(MPNIC_TCD_DONE); + raw_tcd = &cmpl->desc[0]; + } + } + + cmpl->head = head; + + if (head0 >= 0) + mpnic_clean_twq0(nv, napi_budget, qt, false, head0); +} + +static void mpnic_bd_prep(struct mpnic_ring *bdq, u32 idx, struct page *page) +{ + dma_addr_t dma = page_pool_get_dma_addr(page); + + bdq->desc[idx] = cpu_to_le64(FIELD_PREP(MPNIC_BD_DESC_ADDR, dma >> 10) | + FIELD_PREP(MPNIC_BD_DESC_ID, idx) | + FIELD_PREP(MPNIC_BD_DESC_BUF_SZ_LOG2, + page_shift(page) - 10)); +} + +static unsigned int mpnic_bdq_desc_unused(struct mpnic_ring *bdq) +{ + return (ALIGN_DOWN(bdq->head - 1, MPNIC_BDQ_BATCH_SIZE) - bdq->tail) & + bdq->size_mask; +} + +static unsigned int __mpnic_fill_bdq(struct mpnic_ring *bdq) +{ + unsigned int i = bdq->tail; + unsigned int count; + + for (count = mpnic_bdq_desc_unused(bdq); count; count--) { + struct page *page; + + page = page_pool_dev_alloc_pages(bdq->page_pool); + if (!page) { + u64_stats_update_begin(&bdq->stats.syncp); + u64_stats_inc(&bdq->stats.bdq.alloc_failed); + u64_stats_update_end(&bdq->stats.syncp); + break; + } + + bdq->rx_buf[i] = page; + mpnic_bd_prep(bdq, i, page); + + i++; + i &= bdq->size_mask; + } + + return i; +} + +static void __mpnic_bdq_commit_tail(struct mpnic_ring *bdq, unsigned int tail) +{ + if (bdq->tail != tail) { + WRITE_ONCE(bdq->tail, tail); + + writeq(tail, bdq->doorbell); + } +} + +static void mpnic_fill_qt_bdqs(struct mpnic_q_triad *qt) +{ + unsigned int ppq_i = __mpnic_fill_bdq(&qt->sub1); + unsigned int hpq_i = __mpnic_fill_bdq(&qt->sub0); + + /* Force DMA writes to flush before writing to tail(s) */ + dma_wmb(); + + /* Flush out the completions we are done with */ + mpnic_commit_cq_head(&qt->cmpl); + + __mpnic_bdq_commit_tail(&qt->sub0, hpq_i); + __mpnic_bdq_commit_tail(&qt->sub1, ppq_i); +} + +/* Take one of the references batched on the page at @idx. If the device + * has moved on to a new page, first drop the unused references left on + * the previous one. + */ +static struct page * +mpnic_page_pool_get(struct mpnic_pg_ctxt *pg_ctxt, struct mpnic_ring *ring, + u32 idx) +{ + struct page *page = pg_ctxt->page; + + if (unlikely(pg_ctxt->idx != idx)) { + if (pg_ctxt->pagecnt_bias && + !page_pool_unref_page(page, pg_ctxt->pagecnt_bias)) + page_pool_put_unrefed_page(page->pp, page, -1, true); + + page = ring->rx_buf[idx]; + page_pool_fragment_page(page, MPNIC_PAGECNT_BIAS_MAX); + + pg_ctxt->page = page; + pg_ctxt->pagecnt_bias = MPNIC_PAGECNT_BIAS_MAX; + pg_ctxt->idx = idx; + } + + pg_ctxt->pagecnt_bias--; + + return page; +} + +static void mpnic_flush_pg_ctxt(struct mpnic_pg_ctxt *ctxt, bool napi) +{ + long pagecnt_bias = ctxt->pagecnt_bias; + + if (pagecnt_bias) { + struct page *page = ctxt->page; + + if (!page_pool_unref_page(page, pagecnt_bias)) + page_pool_put_unrefed_page(page->pp, page, -1, napi); + } +} + +static unsigned int mpnic_hdr_pg_start(unsigned int pg_off) +{ + /* The headroom of the first header may be larger than + * MPNIC_RX_HROOM due to alignment. So account for that by just + * making the page offset 0 if we are starting at the first header. + */ + if (ALIGN(MPNIC_RX_HROOM, 128) > MPNIC_RX_HROOM && + pg_off == ALIGN(MPNIC_RX_HROOM, 128)) + return 0; + + return pg_off - MPNIC_RX_HROOM; +} + +static unsigned int mpnic_hdr_pg_end(unsigned int pg_off, unsigned int len) +{ + /* Determine the end of the buffer by finding the start of the next + * and then subtracting the headroom from that frame. + */ + pg_off += len + MPNIC_RX_TROOM + MPNIC_RX_HROOM; + + return ALIGN(pg_off, 128) - MPNIC_RX_HROOM; +} + +static void +mpnic_pkt_prepare(u64 rcd, struct mpnic_rcq_state *state, + struct mpnic_q_triad *qt) +{ + unsigned int pg_off = FIELD_GET(MPNIC_RCD_AL_BUFF_OFF, rcd); + unsigned int pg_idx = FIELD_GET(MPNIC_RCD_AL_BUFF_ID, rcd); + unsigned int len = FIELD_GET(MPNIC_RCD_AL_BUFF_LEN, rcd); + bool fin = FIELD_GET(MPNIC_RCD_AL_PAGE_FIN, rcd); + unsigned int frame_sz, pg_start, pg_end; + struct xdp_buff *buff = &state->pkt.buff; + struct page *page; + + pg_start = mpnic_hdr_pg_start(pg_off); + + page = mpnic_page_pool_get(&state->hdr, &qt->sub0, pg_idx); + WRITE_ONCE(qt->sub0.head, (pg_idx + 1) & qt->sub0.size_mask); + + /* Short-cut the end calculation if the page is fully consumed */ + pg_end = fin ? page_size(page) : mpnic_hdr_pg_end(pg_off, len); + frame_sz = pg_end - pg_start; + + page_pool_dma_sync_for_cpu(qt->sub0.page_pool, page, pg_start, + frame_sz); + + xdp_init_buff(buff, frame_sz, &qt->xdp_rxq); + xdp_prepare_buff(buff, page_address(page) + pg_start, + pg_off - pg_start, len, true); + net_prefetch(buff->data); + + state->pkt.add_frag_failed = false; +} + +static void +mpnic_add_rx_frag(u64 rcd, struct mpnic_rcq_state *state, + struct mpnic_q_triad *qt) +{ + unsigned int pg_off = FIELD_GET(MPNIC_RCD_AL_BUFF_OFF, rcd); + unsigned int pg_idx = FIELD_GET(MPNIC_RCD_AL_BUFF_ID, rcd); + unsigned int len = FIELD_GET(MPNIC_RCD_AL_BUFF_LEN, rcd); + bool fin = FIELD_GET(MPNIC_RCD_AL_PAGE_FIN, rcd); + struct xdp_buff *buff = &state->pkt.buff; + unsigned int truesz; + struct page *page; + + page = mpnic_page_pool_get(&state->payld, &qt->sub1, pg_idx); + WRITE_ONCE(qt->sub1.head, (pg_idx + 1) & qt->sub1.size_mask); + + truesz = (fin ? page_size(page) : ALIGN(pg_off + len, 128)) - pg_off; + + page_pool_dma_sync_for_cpu(qt->sub1.page_pool, page, pg_off, truesz); + + if (!xdp_buff_add_frag(buff, page_to_netmem(page), pg_off, len, + truesz)) { + state->payld.pagecnt_bias++; + state->pkt.add_frag_failed = true; + } +} + +static void mpnic_put_pkt_buff(struct mpnic_pkt_ctxt *ctxt, bool napi) +{ + struct xdp_buff *buff = &ctxt->buff; + struct page *page; + + if (!buff->data_hard_start) + return; + + if (unlikely(xdp_buff_has_frags(buff))) { + struct skb_shared_info *shinfo; + int nr_frags; + + shinfo = xdp_get_shared_info_from_buff(buff); + nr_frags = shinfo->nr_frags; + + while (nr_frags--) { + page = skb_frag_page(&shinfo->frags[nr_frags]); + page_pool_put_full_page(page->pp, page, napi); + } + } + + page = virt_to_head_page(buff->data_hard_start); + page_pool_put_full_page(page->pp, page, napi); +} + +static int mpnic_clean_rcq(struct mpnic_napi_vector *nv, + struct mpnic_q_triad *qt, int budget) +{ + unsigned int packets = 0, bytes = 0, dropped = 0, errors = 0; + struct mpnic_ring *rcq = &qt->cmpl; + struct mpnic_rcq_state *state; + unsigned int alloc_failed = 0; + __le64 *raw_rcd, done; + u32 head = rcq->head; + + done = (head & (rcq->size_mask + 1)) ? 0 : cpu_to_le64(MPNIC_RCD_DONE); + raw_rcd = &rcq->desc[head & rcq->size_mask]; + state = rcq->state; + + while (packets < budget) { + u64 rcd; + + if ((*raw_rcd & cpu_to_le64(MPNIC_RCD_DONE)) != done) + break; + + dma_rmb(); + + rcd = le64_to_cpu(*raw_rcd); + + switch (FIELD_GET(MPNIC_RCD_TYPE, rcd)) { + case MPNIC_RCD_TYPE_HDR_AL: + if (FIELD_GET(MPNIC_RCD_HDR_SUBTYPE, rcd) == + MPNIC_RCD_HDR_SUBTYPE_HDR) + mpnic_pkt_prepare(rcd, state, qt); + break; + case MPNIC_RCD_TYPE_PAY_AL: + mpnic_add_rx_frag(rcd, state, qt); + break; + case MPNIC_RCD_TYPE_META: { + struct sk_buff *skb = NULL; + + if (likely(!(rcd & + MPNIC_RCD_META_UNCORRECTABLE_ERR_MASK) && + !state->pkt.add_frag_failed)) { + bytes += xdp_get_buff_len(&state->pkt.buff); + skb = xdp_build_skb_from_buff(&state->pkt.buff); + alloc_failed += !skb; + } + + if (likely(skb)) { + napi_gro_receive(&nv->napi, skb); + } else { + mpnic_put_pkt_buff(&state->pkt, true); + if (rcd & MPNIC_RCD_META_UNCORRECTABLE_ERR_MASK) + errors++; + else + dropped++; + } + + state->pkt.buff.data_hard_start = NULL; + packets++; + break; + } + } + + raw_rcd++; + head++; + + if (unlikely(!(head & rcq->size_mask))) { + done ^= cpu_to_le64(MPNIC_RCD_DONE); + raw_rcd = &rcq->desc[0]; + } + } + + rcq->head = head; + + u64_stats_update_begin(&rcq->stats.syncp); + u64_stats_add(&rcq->stats.rcq.packets, packets - errors); + u64_stats_add(&rcq->stats.rcq.bytes, bytes); + u64_stats_add(&rcq->stats.rcq.dropped, dropped); + u64_stats_add(&rcq->stats.rcq.alloc_failed, alloc_failed); + u64_stats_add(&rcq->stats.rcq.errors, errors); + u64_stats_update_end(&rcq->stats.syncp); + + /* Allocate buffers, force dma_wmb(), and then start writing tails */ + mpnic_fill_qt_bdqs(qt); + + return packets; +} + +static int mpnic_poll(struct napi_struct *napi, int budget) +{ + struct mpnic_napi_vector *nv = container_of(napi, + struct mpnic_napi_vector, + napi); + int i, j, work_done = 0; + + for (i = 0; i < nv->txt_count; i++) + mpnic_clean_tcq(nv, &nv->qt[i], budget); + + if (likely(budget)) + for (j = 0; j < nv->rxt_count; j++, i++) + work_done += mpnic_clean_rcq(nv, &nv->qt[i], + budget - work_done); + + for (i = 0; i < nv->txt_count; i++) + mpnic_commit_cq_head(&nv->qt[i].cmpl); + + if (work_done >= budget) + return budget; + + if (likely(napi_complete_done(napi, work_done))) + mpnic_nv_irq_rearm(nv); + + return work_done; +} + +static irqreturn_t mpnic_msix_clean_rings(int __always_unused irq, void *data) +{ + struct mpnic_napi_vector *nv = data; + + napi_schedule_irqoff(&nv->napi); + + return IRQ_HANDLED; +} + +static void mpnic_aggregate_ring_twq_counters(struct mpnic_base_stats *base, + struct mpnic_ring *twq) +{ + base->tx.dropped += u64_stats_read(&twq->stats.twq.dropped); + base->tx.stop += u64_stats_read(&twq->stats.twq.stop); +} + +static void mpnic_aggregate_ring_tcq_counters(struct mpnic_base_stats *base, + struct mpnic_ring *tcq) +{ + base->tx.packets += u64_stats_read(&tcq->stats.tcq.packets); + base->tx.bytes += u64_stats_read(&tcq->stats.tcq.bytes); + base->tx.wake += u64_stats_read(&tcq->stats.tcq.wake); +} + +static void mpnic_aggregate_ring_bdq_counters(struct mpnic_base_stats *base, + struct mpnic_ring *bdq) +{ + base->rx.alloc_failed += u64_stats_read(&bdq->stats.bdq.alloc_failed); +} + +static void mpnic_aggregate_ring_rcq_counters(struct mpnic_base_stats *base, + struct mpnic_ring *rcq) +{ + base->rx.packets += u64_stats_read(&rcq->stats.rcq.packets); + base->rx.bytes += u64_stats_read(&rcq->stats.rcq.bytes); + base->rx.dropped += u64_stats_read(&rcq->stats.rcq.dropped); + base->rx.alloc_failed += u64_stats_read(&rcq->stats.rcq.alloc_failed); + base->rx.errors += u64_stats_read(&rcq->stats.rcq.errors); +} + +static struct mpnic_base_stats *mpnic_base_stats_prepare(struct mpnic_net *mpn) +{ + struct mpnic_base_stats *old, *new; + + old = netdev_lock_dereference(mpn->stats.base, mpn->netdev); + new = &mpn->stats.buf[old == &mpn->stats.buf[0]]; + *new = *old; + + return new; +} + +void mpnic_stats_fold_rings(struct mpnic_net *mpn) +{ + struct mpnic_base_stats *new; + int i, j, t; + + new = mpnic_base_stats_prepare(mpn); + new->num_tx_queues = 0; + new->num_rx_queues = 0; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + if (!nv) + continue; + + for (t = 0; t < nv->txt_count; t++) { + struct mpnic_q_triad *qt = &nv->qt[t]; + + mpnic_aggregate_ring_twq_counters(new, &qt->sub0); + mpnic_aggregate_ring_tcq_counters(new, &qt->cmpl); + } + + for (j = 0; j < nv->rxt_count; j++, t++) { + struct mpnic_q_triad *qt = &nv->qt[t]; + + mpnic_aggregate_ring_bdq_counters(new, &qt->sub0); + mpnic_aggregate_ring_bdq_counters(new, &qt->sub1); + mpnic_aggregate_ring_rcq_counters(new, &qt->cmpl); + } + } + + rcu_assign_pointer(mpn->stats.base, new); + synchronize_net(); +} + +void mpnic_stats_attach_rings(struct mpnic_net *mpn) +{ + struct mpnic_base_stats *new; + + new = mpnic_base_stats_prepare(mpn); + new->num_tx_queues = mpn->num_tx_queues; + new->num_rx_queues = mpn->num_rx_queues; + rcu_assign_pointer(mpn->stats.base, new); + synchronize_net(); +} + +static void mpnic_free_napi_vector(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv) +{ + int i, j; + + for (i = 0; i < nv->txt_count; i++) + mpn->tx[nv->qt[i].sub0.q_idx] = NULL; + + for (j = 0; j < nv->rxt_count; j++, i++) + mpn->rx[nv->qt[i].cmpl.q_idx] = NULL; + + mpnic_free_irq(nv->mpd, nv->v_idx, nv); + netif_napi_del_locked(&nv->napi); + mpn->napi[nv->v_idx - MPNIC_NON_NAPI_VECTORS] = NULL; + kfree(nv); +} + +void mpnic_free_napi_vectors(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + if (mpn->napi[i]) + mpnic_free_napi_vector(mpn, mpn->napi[i]); +} + +static void mpnic_ring_init(struct mpnic_ring *ring, u32 __iomem *doorbell, + int q_idx) +{ + ring->doorbell = doorbell; + ring->q_idx = q_idx; + u64_stats_init(&ring->stats.syncp); +} + +static int mpnic_alloc_napi_vector(struct mpnic_dev *mpd, + struct mpnic_net *mpn, unsigned int idx) +{ + u32 __iomem *uc_addr = READ_ONCE(mpd->uc_addr0); + struct mpnic_napi_vector *nv; + int err; + + /* Doorbells are plain pointers into the register window, they have + * no way of noticing that it went away. + */ + if (!uc_addr) + return -EIO; + + nv = kzalloc_flex(*nv, qt, 2); + if (!nv) + return -ENOMEM; + + nv->txt_count = 1; + nv->rxt_count = 1; + nv->mpd = mpd; + nv->dev = mpd->dev; + nv->v_idx = idx + MPNIC_NON_NAPI_VECTORS; + + mpn->napi[idx] = nv; + netif_napi_add_config_locked(mpn->netdev, &nv->napi, mpnic_poll, idx); + netif_napi_set_irq_locked(&nv->napi, + pci_irq_vector(to_pci_dev(mpd->dev), + nv->v_idx)); + + snprintf(nv->name, sizeof(nv->name), "%s-TxRx-%u", + mpn->netdev->name, idx); + + err = mpnic_request_irq(mpd, nv->v_idx, mpnic_msix_clean_rings, 0, + nv->name, nv); + if (err) + goto err_napi_del; + + mpnic_ring_init(&nv->qt[0].sub0, &uc_addr[MPNIC_TWQ_TAIL(idx, 0)], idx); + mpnic_ring_init(&nv->qt[0].cmpl, &uc_addr[MPNIC_TCQ_HEAD(idx)], idx); + mpn->tx[idx] = &nv->qt[0].sub0; + + mpnic_ring_init(&nv->qt[1].sub0, &uc_addr[MPNIC_HPQ_TAIL(idx)], idx); + mpnic_ring_init(&nv->qt[1].sub1, &uc_addr[MPNIC_PPQ_TAIL(idx)], idx); + mpnic_ring_init(&nv->qt[1].cmpl, &uc_addr[MPNIC_RCQ_HEAD(idx)], idx); + mpn->rx[idx] = &nv->qt[1].cmpl; + + return 0; + +err_napi_del: + netif_napi_del_locked(&nv->napi); + mpn->napi[idx] = NULL; + kfree(nv); + return err; +} + +int mpnic_alloc_napi_vectors(struct mpnic_net *mpn) +{ + unsigned int i; + int err; + + for (i = 0; i < mpn->num_napi; i++) { + err = mpnic_alloc_napi_vector(mpn->mpd, mpn, i); + if (err) + goto err_free_vectors; + } + + return 0; + +err_free_vectors: + mpnic_free_napi_vectors(mpn); + + return err; +} + +static void mpnic_free_ring_resources(struct device *dev, + struct mpnic_ring *ring) +{ + kvfree(ring->buffer); + ring->buffer = NULL; + + /* If size is not set there are no descriptors present */ + if (!ring->size) + return; + + dma_free_coherent(dev, ring->size, ring->desc, ring->dma); + ring->size_mask = 0; + ring->size = 0; +} + +static int mpnic_alloc_ring_desc(struct mpnic_net *mpn, + struct mpnic_ring *ring, u32 count) +{ + struct device *dev = mpn->netdev->dev.parent; + size_t size; + + size = ALIGN(array_size(sizeof(*ring->desc), count), 4096); + + ring->desc = dma_alloc_coherent(dev, size, &ring->dma, + GFP_KERNEL | __GFP_NOWARN); + if (!ring->desc) + return -ENOMEM; + + ring->size_mask = count - 1; + ring->size = size; + + return 0; +} + +static void mpnic_free_tx_qt_resources(struct mpnic_net *mpn, + struct mpnic_q_triad *qt) +{ + struct device *dev = mpn->netdev->dev.parent; + + mpnic_free_ring_resources(dev, &qt->cmpl); + mpnic_free_ring_resources(dev, &qt->sub0); +} + +static int mpnic_alloc_tx_qt_resources(struct mpnic_net *mpn, + struct mpnic_q_triad *qt) +{ + int err; + + err = mpnic_alloc_ring_desc(mpn, &qt->sub0, mpn->txq_size); + if (err) + return err; + + qt->sub0.tx_buf = kvzalloc_objs(*qt->sub0.tx_buf, mpn->txq_size, + GFP_KERNEL | __GFP_NOWARN); + if (!qt->sub0.tx_buf) { + err = -ENOMEM; + goto err_free_qt; + } + + err = mpnic_alloc_ring_desc(mpn, &qt->cmpl, mpn->txq_size); + if (err) + goto err_free_qt; + + return 0; + +err_free_qt: + mpnic_free_tx_qt_resources(mpn, qt); + return err; +} + +static int +mpnic_alloc_qt_page_pool(struct mpnic_net *mpn, struct mpnic_napi_vector *nv, + struct mpnic_q_triad *qt) +{ + struct page_pool_params pp_params = { + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .pool_size = min(mpn->hpq_size + mpn->ppq_size, 32768u), + .nid = NUMA_NO_NODE, + .dev = nv->dev, + .dma_dir = DMA_FROM_DEVICE, + .max_len = PAGE_SIZE, + .napi = &nv->napi, + .netdev = mpn->netdev, + .queue_idx = qt->cmpl.q_idx, + }; + struct page_pool *pp; + + pp = page_pool_create(&pp_params); + if (IS_ERR(pp)) + return PTR_ERR(pp); + + qt->sub0.page_pool = pp; + page_pool_get(pp); + qt->sub1.page_pool = pp; + + return 0; +} + +static void mpnic_free_rx_qt_resources(struct mpnic_net *mpn, + struct mpnic_q_triad *qt) +{ + struct device *dev = mpn->netdev->dev.parent; + + mpnic_free_ring_resources(dev, &qt->cmpl); + mpnic_free_ring_resources(dev, &qt->sub1); + mpnic_free_ring_resources(dev, &qt->sub0); + + if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) { + xdp_rxq_info_unreg(&qt->xdp_rxq); + page_pool_destroy(qt->sub1.page_pool); + page_pool_destroy(qt->sub0.page_pool); + } +} + +static int mpnic_alloc_rx_qt_resources(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv, + struct mpnic_q_triad *qt) +{ + int err; + + err = mpnic_alloc_qt_page_pool(mpn, nv, qt); + if (err) + return err; + + err = xdp_rxq_info_reg(&qt->xdp_rxq, mpn->netdev, qt->cmpl.q_idx, + nv->napi.napi_id); + if (err) + goto err_free_page_pool; + + err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL, + qt->sub0.page_pool); + if (err) + goto err_unreg_rxq; + + err = mpnic_alloc_ring_desc(mpn, &qt->sub0, mpn->hpq_size); + if (err) + goto err_unreg_mm; + + qt->sub0.rx_buf = kvzalloc_objs(*qt->sub0.rx_buf, mpn->hpq_size, + GFP_KERNEL | __GFP_NOWARN); + if (!qt->sub0.rx_buf) { + err = -ENOMEM; + goto err_free_qt; + } + + err = mpnic_alloc_ring_desc(mpn, &qt->sub1, mpn->ppq_size); + if (err) + goto err_free_qt; + + qt->sub1.rx_buf = kvzalloc_objs(*qt->sub1.rx_buf, mpn->ppq_size, + GFP_KERNEL | __GFP_NOWARN); + if (!qt->sub1.rx_buf) { + err = -ENOMEM; + goto err_free_qt; + } + + err = mpnic_alloc_ring_desc(mpn, &qt->cmpl, mpn->rcq_size); + if (err) + goto err_free_qt; + + qt->cmpl.state = kvzalloc_obj(*qt->cmpl.state, + GFP_KERNEL | __GFP_NOWARN); + if (!qt->cmpl.state) { + err = -ENOMEM; + goto err_free_qt; + } + + return 0; + +err_free_qt: + mpnic_free_rx_qt_resources(mpn, qt); + return err; +err_unreg_mm: + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); +err_unreg_rxq: + xdp_rxq_info_unreg(&qt->xdp_rxq); +err_free_page_pool: + page_pool_destroy(qt->sub1.page_pool); + page_pool_destroy(qt->sub0.page_pool); + return err; +} + +static void mpnic_free_nv_resources(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv) +{ + int i, j; + + for (i = 0; i < nv->txt_count; i++) + mpnic_free_tx_qt_resources(mpn, &nv->qt[i]); + + for (j = 0; j < nv->rxt_count; j++, i++) + mpnic_free_rx_qt_resources(mpn, &nv->qt[i]); +} + +static int mpnic_alloc_nv_resources(struct mpnic_net *mpn, + struct mpnic_napi_vector *nv) +{ + int i, j, err; + + for (i = 0; i < nv->txt_count; i++) { + err = mpnic_alloc_tx_qt_resources(mpn, &nv->qt[i]); + if (err) + goto err_free_qt_resources; + } + + for (j = 0; j < nv->rxt_count; j++, i++) { + err = mpnic_alloc_rx_qt_resources(mpn, nv, &nv->qt[i]); + if (err) + goto err_free_qt_resources; + } + + return 0; + +err_free_qt_resources: + while (i--) { + if (i < nv->txt_count) + mpnic_free_tx_qt_resources(mpn, &nv->qt[i]); + else + mpnic_free_rx_qt_resources(mpn, &nv->qt[i]); + } + return err; +} + +void mpnic_free_resources(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + mpnic_free_nv_resources(mpn, mpn->napi[i]); +} + +int mpnic_alloc_resources(struct mpnic_net *mpn) +{ + int i, err; + + for (i = 0; i < mpn->num_napi; i++) { + err = mpnic_alloc_nv_resources(mpn, mpn->napi[i]); + if (err) + goto err_free_resources; + } + + return 0; + +err_free_resources: + while (i--) + mpnic_free_nv_resources(mpn, mpn->napi[i]); + + return err; +} + +static void mpnic_set_netif_napi(struct mpnic_napi_vector *nv, + struct napi_struct *napi) +{ + int i, j; + + for (i = 0; i < nv->txt_count; i++) + netif_queue_set_napi(nv->napi.dev, nv->qt[i].sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, napi); + + for (j = 0; j < nv->rxt_count; j++, i++) + netif_queue_set_napi(nv->napi.dev, nv->qt[i].cmpl.q_idx, + NETDEV_QUEUE_TYPE_RX, napi); +} + +int mpnic_set_netif_queues(struct mpnic_net *mpn) +{ + int i, err; + + err = netif_set_real_num_queues(mpn->netdev, mpn->num_tx_queues, + mpn->num_rx_queues); + if (err) + return err; + + for (i = 0; i < mpn->num_napi; i++) + mpnic_set_netif_napi(mpn->napi[i], &mpn->napi[i]->napi); + + return 0; +} + +void mpnic_reset_netif_queues(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + mpnic_set_netif_napi(mpn->napi[i], NULL); +} + +static void mpnic_enable_twq(struct mpnic_dev *mpd, struct mpnic_ring *twq) +{ + u32 log_size = fls(twq->size_mask); + u32 i = twq->q_idx; + + /* Reset head/tail */ + mpnic_wr64(mpd, MPNIC_TWQ_CTL(i, 0), MPNIC_TWQ_CTL_RESET); + twq->tail = 0; + twq->head = 0; + twq->deferred_meta = -1; + + /* Store descriptor ring address and size */ + mpnic_wr64(mpd, MPNIC_TWQ_BASE_ADDR(i, 0), twq->dma); + mpnic_wr64(mpd, MPNIC_TWQ_SIZE(i, 0), log_size & MPNIC_TWQ_SIZE_SIZE); + + mpnic_wr64(mpd, MPNIC_TWQ_CTL(i, 0), MPNIC_TWQ_CTL_ENABLE); +} + +static void mpnic_enable_tcq(struct mpnic_dev *mpd, + struct mpnic_napi_vector *nv, + struct mpnic_ring *tcq) +{ + u32 log_size = fls(tcq->size_mask); + u32 i = tcq->q_idx; + + /* Reset head/tail */ + mpnic_wr64(mpd, MPNIC_TCQ_CTL(i), MPNIC_TCQ_CTL_RESET); + tcq->tail = 0; + tcq->head = 0; + + /* Store descriptor ring address and size */ + mpnic_wr64(mpd, MPNIC_TCQ_BASE_ADDR(i), tcq->dma); + mpnic_wr64(mpd, MPNIC_TCQ_SIZE(i), log_size & MPNIC_TCQ_SIZE_SIZE); + + /* Store interrupt information for the completion queue */ + mpnic_wr64(mpd, MPNIC_TIM_CTL(i), nv->v_idx); + mpnic_wr64(mpd, MPNIC_TIM_INTR_MASK(i), 0); + + mpnic_wr64(mpd, MPNIC_TCQ_CTL(i), MPNIC_TCQ_CTL_ENABLE); +} + +static void mpnic_enable_bdq(struct mpnic_dev *mpd, struct mpnic_ring *hpq, + struct mpnic_ring *ppq) +{ + u32 hpq_log_size = fls(hpq->size_mask); + u32 ppq_log_size = fls(ppq->size_mask); + u32 i = hpq->q_idx; + + /* Reset head/tail */ + mpnic_wr64(mpd, MPNIC_BDQ_CTL(i), MPNIC_BDQ_CTL_RESET); + hpq->tail = 0; + hpq->head = 0; + ppq->tail = 0; + ppq->head = 0; + + /* Store descriptor ring addresses and sizes */ + mpnic_wr64(mpd, MPNIC_HPQ_BASE_ADDR(i), hpq->dma); + mpnic_wr64(mpd, MPNIC_HPQ_SIZE(i), hpq_log_size & MPNIC_HPQ_SIZE_SIZE); + mpnic_wr64(mpd, MPNIC_PPQ_BASE_ADDR(i), ppq->dma); + mpnic_wr64(mpd, MPNIC_PPQ_SIZE(i), ppq_log_size & MPNIC_PPQ_SIZE_SIZE); + + mpnic_wr64(mpd, MPNIC_BDQ_CTL(i), + MPNIC_BDQ_CTL_ENABLE | MPNIC_BDQ_CTL_ENABLE_PPQ); +} + +static void mpnic_set_rde_cfg(struct mpnic_dev *mpd, struct mpnic_ring *rcq) +{ + BUILD_BUG_ON(FIELD_MAX(MPNIC_RDE_CFG_MIN_HEAD_ROOM) < MPNIC_RX_HROOM); + BUILD_BUG_ON(FIELD_MAX(MPNIC_RDE_CFG_MIN_TAIL_ROOM) < MPNIC_RX_TROOM); + + mpnic_wr64(mpd, MPNIC_RDE_CFG(rcq->q_idx), + FIELD_PREP(MPNIC_RDE_CFG_MIN_HEAD_ROOM, MPNIC_RX_HROOM) | + FIELD_PREP(MPNIC_RDE_CFG_MIN_TAIL_ROOM, MPNIC_RX_TROOM) | + FIELD_PREP(MPNIC_RDE_CFG_MAX_HEADER_BYTES, + MPNIC_RX_MAX_HDR)); +} + +static void mpnic_enable_rcq(struct mpnic_dev *mpd, + struct mpnic_napi_vector *nv, + struct mpnic_ring *rcq) +{ + u32 log_size = fls(rcq->size_mask); + u32 i = rcq->q_idx; + + mpnic_set_rde_cfg(mpd, rcq); + + /* Reset head/tail */ + mpnic_wr64(mpd, MPNIC_RCQ_CTL(i), MPNIC_RCQ_CTL_RESET); + rcq->head = 0; + rcq->tail = 0; + + /* Store descriptor ring address and size */ + mpnic_wr64(mpd, MPNIC_RCQ_BASE_ADDR(i), rcq->dma); + mpnic_wr64(mpd, MPNIC_RCQ_SIZE(i), log_size & MPNIC_RCQ_SIZE_SIZE); + + /* Store interrupt information for the completion queue */ + mpnic_wr64(mpd, MPNIC_RIM_CTL(i), nv->v_idx); + mpnic_wr64(mpd, MPNIC_RIM_INTR_MASK(i), 0); + + mpnic_wr64(mpd, MPNIC_RCQ_CTL(i), MPNIC_RCQ_CTL_ENABLE); +} + +void mpnic_enable(struct mpnic_net *mpn) +{ + struct mpnic_dev *mpd = mpn->mpd; + int i, j, t; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (t = 0; t < nv->txt_count; t++) { + mpnic_enable_twq(mpd, &nv->qt[t].sub0); + mpnic_enable_tcq(mpd, nv, &nv->qt[t].cmpl); + } + + for (j = 0; j < nv->rxt_count; j++, t++) { + mpnic_enable_bdq(mpd, &nv->qt[t].sub0, &nv->qt[t].sub1); + mpnic_enable_rcq(mpd, nv, &nv->qt[t].cmpl); + } + } + + mpnic_wrfl(mpd); +} + +static void mpnic_disable_twq(struct mpnic_dev *mpd, struct mpnic_ring *txr) +{ + u64 twq_ctl = mpnic_rd64(mpd, MPNIC_TWQ_CTL(txr->q_idx, 0)); + + twq_ctl &= ~MPNIC_TWQ_CTL_ENABLE; + mpnic_wr64(mpd, MPNIC_TWQ_CTL(txr->q_idx, 0), twq_ctl); +} + +static void mpnic_disable_tcq(struct mpnic_dev *mpd, struct mpnic_ring *txr) +{ + mpnic_wr64(mpd, MPNIC_TCQ_CTL(txr->q_idx), 0); + mpnic_wr64(mpd, MPNIC_TIM_INTR_MASK(txr->q_idx), + MPNIC_TIM_INTR_MASK_MASK); +} + +static void mpnic_disable_bdq(struct mpnic_dev *mpd, struct mpnic_ring *hpq) +{ + u64 bdq_ctl = mpnic_rd64(mpd, MPNIC_BDQ_CTL(hpq->q_idx)); + + bdq_ctl &= ~(MPNIC_BDQ_CTL_ENABLE | MPNIC_BDQ_CTL_ENABLE_PPQ); + mpnic_wr64(mpd, MPNIC_BDQ_CTL(hpq->q_idx), bdq_ctl); +} + +static void mpnic_disable_rcq(struct mpnic_dev *mpd, struct mpnic_ring *rcq) +{ + mpnic_wr64(mpd, MPNIC_RCQ_CTL(rcq->q_idx), 0); + mpnic_wr64(mpd, MPNIC_RIM_INTR_MASK(rcq->q_idx), + MPNIC_RIM_INTR_MASK_MASK); +} + +void mpnic_disable(struct mpnic_net *mpn) +{ + struct mpnic_dev *mpd = mpn->mpd; + int i, j, t; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (t = 0; t < nv->txt_count; t++) { + mpnic_disable_twq(mpd, &nv->qt[t].sub0); + mpnic_disable_tcq(mpd, &nv->qt[t].cmpl); + } + + for (j = 0; j < nv->rxt_count; j++, t++) { + mpnic_disable_bdq(mpd, &nv->qt[t].sub0); + mpnic_disable_rcq(mpd, &nv->qt[t].cmpl); + } + } + + mpnic_wrfl(mpd); +} + +struct mpnic_idle_regs { + u32 reg_base; + u8 reg_cnt; + char name[4]; +}; + +static u32 mpnic_non_idle_queues(struct mpnic_dev *mpd, + const struct mpnic_idle_regs *regs, + unsigned int nregs) +{ + u32 non_idle_bitmap = 0; + unsigned int i, j; + + for (i = 0; i < nregs; i++) { + for (j = 0; j < regs[i].reg_cnt; j++) { + if (mpnic_rd64(mpd, regs[i].reg_base + 2 * j) != + ~0ULL) { + non_idle_bitmap |= BIT(i); + break; + } + } + } + + return non_idle_bitmap; +} + +static void mpnic_idle_dump(struct mpnic_dev *mpd, + const struct mpnic_idle_regs *regs, + unsigned int nregs, u32 non_idle_bitmap, int err) +{ + unsigned int i, j; + + dev_err(mpd->dev, "error waiting for queues idle %d\n", err); + for (i = 0; i < nregs; i++) { + if (!(non_idle_bitmap & BIT(i))) + continue; + + dev_err(mpd->dev, "%s block not idle:\n", regs[i].name); + for (j = 0; j < regs[i].reg_cnt; j++) + dev_err(mpd->dev, " 0x%04x: %016llx\n", + regs[i].reg_base + 2 * j, + mpnic_rd64(mpd, regs[i].reg_base + 2 * j)); + } +} + +void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd) +{ + static const struct mpnic_idle_regs queues[] = { + { MPNIC_TWQ_IDLE(0), MPNIC_TWQ_IDLE_CNT, "TWQ" }, + { MPNIC_TQS_IDLE(0), MPNIC_TQS_IDLE_CNT, "TQS" }, + { MPNIC_TDE_IDLE(0), MPNIC_TDE_IDLE_CNT, "TDE" }, + { MPNIC_TCQ_IDLE(0), MPNIC_TCQ_IDLE_CNT, "TCQ" }, + { MPNIC_HPQ_IDLE(0), MPNIC_HPQ_IDLE_CNT, "HPQ" }, + { MPNIC_PPQ_IDLE(0), MPNIC_PPQ_IDLE_CNT, "PPQ" }, + { MPNIC_RCQ_IDLE(0), MPNIC_RCQ_IDLE_CNT, "RCQ" }, + }; + u32 non_idle_bitmap; + int err; + + err = read_poll_timeout(mpnic_non_idle_queues, non_idle_bitmap, + !non_idle_bitmap, 20, 500000, false, mpd, + queues, ARRAY_SIZE(queues)); + if (err) + mpnic_idle_dump(mpd, queues, ARRAY_SIZE(queues), + non_idle_bitmap, err); +} + +static void mpnic_clean_bdq(struct mpnic_ring *bdq) +{ + unsigned int head = bdq->head; + + while (head != bdq->tail) { + struct page *page = bdq->rx_buf[head]; + + page_pool_put_full_page(page->pp, page, false); + + head++; + head &= bdq->size_mask; + } + + bdq->head = head; +} + +void mpnic_flush(struct mpnic_net *mpn) +{ + int i, j, t; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (t = 0; t < nv->txt_count; t++) { + struct mpnic_q_triad *qt = &nv->qt[t]; + struct netdev_queue *txq; + + /* Clean the work queue of unprocessed work */ + mpnic_clean_twq0(nv, 0, qt, true, qt->sub0.tail); + + txq = netdev_get_tx_queue(mpn->netdev, qt->sub0.q_idx); + netdev_tx_reset_queue(txq); + } + + for (j = 0; j < nv->rxt_count; j++, t++) { + struct mpnic_q_triad *qt = &nv->qt[t]; + struct mpnic_rcq_state *state = qt->cmpl.state; + + /* Release the partially assembled frame and the + * pages the queues are still handing out. + */ + mpnic_put_pkt_buff(&state->pkt, false); + mpnic_flush_pg_ctxt(&state->hdr, false); + mpnic_flush_pg_ctxt(&state->payld, false); + memset(state, 0, sizeof(*state)); + + mpnic_clean_bdq(&qt->sub0); + mpnic_clean_bdq(&qt->sub1); + } + } +} + +void mpnic_fill(struct mpnic_net *mpn) +{ + int i, j, t; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { + struct mpnic_q_triad *qt = &nv->qt[t]; + struct mpnic_rcq_state *state = qt->cmpl.state; + + /* Point the page contexts at an index the device + * cannot report, so the first buffer coming out of + * either queue is not taken for a page we hold. + */ + state->hdr.idx = UINT_MAX; + state->payld.idx = UINT_MAX; + + local_bh_disable(); + mpnic_fill_qt_bdqs(qt); + local_bh_enable(); + } + } +} + +void mpnic_napi_disable(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) { + napi_disable_locked(&mpn->napi[i]->napi); + + mpnic_nv_irq_disable(mpn->napi[i]); + } +} + +void mpnic_napi_enable(struct mpnic_net *mpn) +{ + int i; + + for (i = 0; i < mpn->num_napi; i++) + napi_enable_locked(&mpn->napi[i]->napi); + + /* Force the first interrupt on each vector to guarantee that any + * completions posted during bringup are processed. + */ + for (i = 0; i < mpn->num_napi; i++) + mpnic_nv_irq_trigger(mpn->napi[i]); + + mpnic_wrfl(mpn->mpd); +} + +void mpnic_napi_depletion_check(struct mpnic_net *mpn) +{ + int i, j, t; + + for (i = 0; i < mpn->num_napi; i++) { + struct mpnic_napi_vector *nv = mpn->napi[i]; + + for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) { + /* Check if BDs posted covers a max sized frame + * + 1 BD held by RDE as a spare + * + 1 BD of extra safety margin + */ + if (mpnic_desc_used(&nv->qt[t].sub0) < + MPNIC_RX_HPQ_DROP_THRS + 2 || + mpnic_desc_used(&nv->qt[t].sub1) < + MPNIC_RX_PPQ_DROP_THRS + 2) { + mpnic_nv_irq_trigger(nv); + break; + } + } + } + + mpnic_wrfl(mpn->mpd); +} diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h new file mode 100644 index 000000000000..52492e3f484f --- /dev/null +++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _MPNIC_TXRX_H_ +#define _MPNIC_TXRX_H_ + +#include <linux/if_ether.h> +#include <linux/netdevice.h> +#include <linux/skbuff.h> +#include <linux/types.h> +#include <linux/u64_stats_sync.h> +#include <net/netdev_queues.h> +#include <net/xdp.h> + +#include "mpnic.h" + +struct mpnic_net; + +/* Space we have to have available in a work queue to take a packet: + * 1 descriptor per page + * + 1 descriptor for the skb head + * + 1 descriptor for the metadata + * + 7 descriptors to keep the tail out of the head's cacheline + * If we cannot guarantee that we return NETDEV_TX_BUSY. + */ +#define MPNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 9) +#define MPNIC_TX_DESC_WAKEUP (MPNIC_MAX_SKB_DESC * 2) + +#define MPNIC_MAX_NAPI_VECTORS 1024u + +/* Number of buffer descriptors the driver posts before ringing the + * doorbell. The device consumes whatever the doorbell points at, this is + * purely to keep the driver from writing the CSR for every descriptor. + */ +#define MPNIC_BDQ_BATCH_SIZE 64u + +#define MPNIC_TXQ_SIZE_DEFAULT 1024 +#define MPNIC_HPQ_SIZE_DEFAULT 256 +#define MPNIC_PPQ_SIZE_DEFAULT 256 +#define MPNIC_RCQ_SIZE_DEFAULT 1024 + +/* Room the device has to leave in front of and behind every header so the + * driver can build an skb around it in place. The headroom is padded out + * so that consecutive headers in one page start 128 B aligned. + */ +#define MPNIC_RX_TROOM \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +#define MPNIC_RX_HROOM \ + (ALIGN(MPNIC_RX_TROOM + XDP_PACKET_HEADROOM, 128) - MPNIC_RX_TROOM) + +/* Headers longer than this are split off into the payload queue */ +#define MPNIC_RX_MAX_HDR 1536 + +/* A page is handed out to many packets, each of which takes one reference. + * Rather than a locked increment per packet the driver takes a batch of + * references up front and returns whatever is left when the page is done. + */ +#define MPNIC_PAGECNT_BIAS_MAX (PAGE_SIZE + 1) + +#define MPNIC_MAX_JUMBO_FRAME_SIZE 9742 + +#define MPNIC_RX_HPQ_DROP_THRS \ + DIV_ROUND_UP(MPNIC_RX_HROOM + MPNIC_RX_MAX_HDR + MPNIC_RX_TROOM, \ + PAGE_SIZE) +#define MPNIC_RX_PPQ_DROP_THRS \ + DIV_ROUND_UP(MPNIC_MAX_JUMBO_FRAME_SIZE, PAGE_SIZE) + +/* The page a buffer descriptor queue is currently handing out. Records + * how many of the references taken on it are still unused. + */ +struct mpnic_pg_ctxt { + struct page *page; + long pagecnt_bias; + u32 idx; +}; + +struct mpnic_pkt_ctxt { + struct xdp_buff buff; + bool add_frag_failed; +}; + +struct mpnic_rcq_state { + struct mpnic_pkt_ctxt pkt; + struct mpnic_pg_ctxt hdr; + struct mpnic_pg_ctxt payld; +}; + +struct mpnic_queue_stats { + union { + struct { + u64_stats_t dropped; + u64_stats_t stop; + } twq; + struct { + u64_stats_t packets; + u64_stats_t bytes; + u64_stats_t wake; + } tcq; + struct { + u64_stats_t packets; + u64_stats_t bytes; + u64_stats_t dropped; + u64_stats_t alloc_failed; + u64_stats_t errors; + } rcq; + struct { + u64_stats_t alloc_failed; + } bdq; + }; + struct u64_stats_sync syncp; +}; + +struct mpnic_ring { + union { + struct mpnic_rcq_state *state; /* RCQ */ + struct page **rx_buf; /* BDQ */ + void **tx_buf; /* TWQ */ + void *buffer; /* Generic pointer */ + }; + + u32 __iomem *doorbell; /* Pointer to CSR space for ring */ + __le64 *desc; /* Descriptor ring memory */ + u16 size_mask; /* Size of ring in descriptors - 1 */ + u16 q_idx; /* Hardware queue index */ + + u32 head, tail; /* Head/Tail of ring */ + + union { + /* BDQ only */ + struct page_pool *page_pool; + + /* TWQ only, index of the metadata descriptor of the last + * packet placed in the ring without ringing the doorbell, + * -1 if the doorbell is in sync with the tail. + */ + s32 deferred_meta; + }; + + struct mpnic_queue_stats stats; + + /* Slow path fields follow */ + dma_addr_t dma; /* Phys addr of descriptor memory */ + size_t size; /* Size of descriptor ring in memory */ +}; + +/* The device pairs two work queues with one completion queue. On the Rx + * side they are the header and the payload buffer descriptor queues; on + * the Tx side only the first one is used for now, the second one becomes + * the XDP ring. + */ +struct mpnic_q_triad { + struct xdp_rxq_info xdp_rxq; + struct mpnic_ring sub0, sub1, cmpl; +}; + +struct mpnic_napi_vector { + struct napi_struct napi; + struct device *dev; /* Device for DMA unmapping */ + struct mpnic_dev *mpd; + + u16 v_idx; + u16 txt_count; + u16 rxt_count; + + char name[IFNAMSIZ + 11]; + + struct mpnic_q_triad qt[]; +}; + +netdev_tx_t mpnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); +int mpnic_alloc_napi_vectors(struct mpnic_net *mpn); +void mpnic_free_napi_vectors(struct mpnic_net *mpn); +void mpnic_stats_attach_rings(struct mpnic_net *mpn); +void mpnic_stats_fold_rings(struct mpnic_net *mpn); +int mpnic_alloc_resources(struct mpnic_net *mpn); +void mpnic_free_resources(struct mpnic_net *mpn); +int mpnic_set_netif_queues(struct mpnic_net *mpn); +void mpnic_reset_netif_queues(struct mpnic_net *mpn); +void mpnic_napi_enable(struct mpnic_net *mpn); +void mpnic_napi_disable(struct mpnic_net *mpn); +void mpnic_napi_depletion_check(struct mpnic_net *mpn); +void mpnic_enable(struct mpnic_net *mpn); +void mpnic_disable(struct mpnic_net *mpn); +void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd); +void mpnic_flush(struct mpnic_net *mpn); +void mpnic_fill(struct mpnic_net *mpn); + +#endif /* _MPNIC_TXRX_H_ */ |