summaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/google
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-10-22 15:58:00 -1000
committerLinus Torvalds <torvalds@linux-foundation.org>2026-10-22 15:58:00 -1000
commit734bef548a96e8774d6eef5dfa2026c6edc4e80b (patch)
treece2520b2602ce3b82f9842490d21836f9838d441 /drivers/net/ethernet/google
parent864c07de0c7dcbca58b61625dfa2e461ae63b242 (diff)
parent1a2c35c49cc7c6be860038b8442b52424a8f123c (diff)
downloadlinux-next-stable.tar.gz
linux-next-stable.zip
Merge tag 'net-next-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-nextstable
Pull networking updates from Paolo Abeni: "This includes a significant number of low-priority fixes - especially in the NIC drivers area - that we deferred from the last few RCs in the previous cycle, which can't be easily outlined in the synopsis below. Core & protocols: - Continue the work on removing rtnl_lock dependencies: - per-netns ARP and ND neighbour tables, with the default settings now configurable per netns (inheritance from init_net is controlled by the new net.core.neigh_inherit_init_net sysctl) - RCU-protected configuration and rtnl_lock-less fill_info() for hsr, netkit, sit, vlan and vxlan devices - per-netns device unregistration for ip tunnel and gre devices - use the netdev instance lock instead of rtnl_lock when reading speed and duplex via sysfs - batch the flushing of uncached routes at device unregistration time, relieving rtnl_lock contention on netns dismantle. - Add hardware pacing offload support, admin-controlled via a new rtnetlink attribute, with sch_fq integration (implemented in idpf). - Re-segment BIG TCP GSO packets into GSO packets fitting the egress device limits instead of plain MSS-sized segments, improving throughput for mixed BIG TCP / non-BIG TCP paths by up to 70%. - Generate RSS keys that spread flows over all the RX queues; the previous random keys could leave some queues unused. - Optimize the bridge VLAN forwarding paths: cache the port-VLAN in fdb entries and consider only VLAN members when flooding, for ~10% more Mpps in the fdb forwarding path. - A few TCP/IP performance improvements aimed at efficient cache usage and reducing contention. - Add the TCP ROCCET congestion control, a CUBIC variant aimed at avoiding bufferbloat in cellular networks. - Continue the conversion of getsockopt callbacks to sockopt_t. - Add an ethtool command to list the front-facing ports of an interface, including SFP cages and modules, using the phy_port representation. - Enable BIG TCP support for bareudp tunnels. - Remove the IPV6_ADDRFORM socket option and disallow buggy TCP state transitions (listen() + shutdown() + connect() and connect() + connect(AF_UNSPEC) + listen()). - Remove most BUG_ON()s from core packet processing, replacing them with error paths. Wireless: - Latency improvements, including lockless station get/dump and beacon updates, plus RTNL reduction and locking cleanups. - Infrastructure for non-station TX/RX with MLO. - NAN (Wi-Fi Aware) Instant Communication support and other NAN improvements. - Control frame Integrity Protocol (CIP) and protected TWT support. - Improvements for AP regulatory checks. Netfilter: - Disable br_netfilter in user namespaces and schedule it for removal in 2027. - Restrict the raw table in nft_compat to the specific family, hook and priority. - Prepare the flowtable for mixed IPv4-over-IPv6 tunneling (and vice versa). - Use GFP_KERNEL_ACCOUNT for most netfilter allocations, for better integration with memcg. - Allow discovering conntrack zone filtering support in ctnetlink dumps, and fix nfnetlink_hook handling of large dumps. Deletions: - Move the LLC type 2 sockets out of tree. There are no known users and the code attracts a never-ending stream of LLM-driven fixes. Device drivers: - New drivers: - Meta Platforms NIC (mpnic), the second-generation NIC from Meta - initial skeleton of the Nebula-matrix NICs driver (nbl) - X-Powers AC200/AC300 Fast Ethernet PHYs - Ethernet high-speed NICs: - Intel: - i40e: XDP metadata support for RX hash and VLAN tag - ice: expose the E825 TSPLL state via the DPLL subsystem - nVidia/Mellanox: - move the data direct infrastructure from IB to core, in preparation for netdev support. - reduce the memory used per offloaded TC rule - Meta (fbnic): - support large io_uring zero-copy RX buffers - Microsoft vNIC (mana): - add BQL support - Ethernet NICs consumer, and embedded: - Airoha: - LRO offload support - Cadence (macb): - context swapping for ringparam and MTU changes, avoiding buffer loss under memory pressure - Mucse (rnpgbe): - TX/RX and link status support - NXP (enetc): - SR-IOV improvements and VF support for ENETC v4 - Qualcomm (ppe): - IPQ5424 support - Realtek PCI (r8169): - phylink support - RSS support for RTL8127 - Renesas (ravb): - gPTP support for R-Car Gen4 - Synopsys (stmmac): - XDP multi-buffer support on the TX side - tc mqprio hw queue priority offload - support for RK3568 SGMII, Allwinner H616 EMAC1, UltraRISC DP1000 and Qualcomm sa8255p - Ethernet switches: - Mediatek (mt7530): - fast age and change conduit support - Microchip (ksz): - support for KSZ8995XA/KS8995XA - Microchip (lan966x): - PCIe FDMA support, ~20x throughput increase in PCIe endpoint - Microchip (sparx5): - Time Aware Shaper (taprio) offload - Motorcomm (yt921x): - LED and SerDes PCS support - Realtek (rtl8365mb): - EEE support - Ethernet PHYs: - Broadcom: - enable jumbo frames on BCM54xx - Realtek: - support for RTL8261CE_CG and RTL8261D_VM - MDIO support for RTL838x and RTL839x - WiFi drivers: - Broadcom (brcmfmac): - FT/OKC roaming offload - Intel (iwlwifi): - UHR sniffer support - support for newer UEFI/BIOS tables - MediaTek (mt76): - mt7925 NAN improvements - offchannel/scan improvements and MLO fixes - Morse Micro (mm81x): - monitor mode support - Qualcomm (ath12k): - IPQ5332 platform infrastructure - Realtek (rtw88): - RTL8723BS support - SDIO TX performance improvements - Realtek (rtw89): - P2P_DEVICE support - BT-coex updates for RTL8922D - Bluetooth: - move H:4 reassembly into the Bluetooth core - add generic support for vendor packets - MGMT: add security level changed event - Bluetooth drivers: - btusb: support Bulk Serialization Mode - Intel: support Draco and Titan Lake, allow unsigned firmware - Mediatek: MT7928 support - Qualcomm: QCC2072 support - Broadcom: Synaptics 4384 support" * tag 'net-next-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (2126 commits) Bluetooth: btusb: disable by default Qualcomm multi-subsystem tipc: fix broadcast sender hang on last acker departure tipc: fix use-after-free of broadcast reception link at node release r8169: clear TX LPI on RTL8168h at link up after phylink conversion tun: Require CAP_NET_ADMIN to attach to an ownerless device net/sched/sch_cake: drop dead nla_nest_end() error check net/sched/sch_cake: widen buffer_used to u64 net/sched/sch_cake: reduce parent backlog when clearing a tin net/sched/sch_cake: serialize reconfiguration with the datapath tg3: Add Datto SIRIS 5 1U Server to restart AER quirk table net/mlx5e: Stop asserting for RTNL when closing channels net: shrink the payload-only part of skb dataref to one bit net: wwan: iosm: validate msg_head before indexing msg_ring net: bgmac: rate limit RX error messages selftests: net: veth: test XDP_TX to a peer with GRO enabled veth: schedule the peer's NAPI after XDP_TX when only GRO is enabled pppoe: delete session hash entries by pointer match net: lan969x: don't disable hardirqs in fdma tx clear net: sparx5/lan969x: do not clobber the PSFP cycle timer update period net: lan969x: fix DSM calendar for empty taxis ...
Diffstat (limited to 'drivers/net/ethernet/google')
-rw-r--r--drivers/net/ethernet/google/gve/gve.h87
-rw-r--r--drivers/net/ethernet/google/gve/gve_adminq.c376
-rw-r--r--drivers/net/ethernet/google/gve/gve_adminq.h30
-rw-r--r--drivers/net/ethernet/google/gve/gve_ethtool.c5
-rw-r--r--drivers/net/ethernet/google/gve/gve_flow_rule.c15
-rw-r--r--drivers/net/ethernet/google/gve/gve_main.c1059
-rw-r--r--drivers/net/ethernet/google/gve/gve_utils.c4
7 files changed, 971 insertions, 605 deletions
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index c280ff35ee77..b7ce6e8632db 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -674,6 +674,7 @@ struct gve_notify_block {
struct gve_tx_ring *tx; /* tx rings on this block */
struct gve_rx_ring *rx; /* rx rings on this block */
u32 irq;
+ bool irq_requested;
};
/* Tracks allowed and current rx queue settings */
@@ -797,6 +798,79 @@ struct gve_ptp {
struct gve_priv *priv;
};
+struct gve_device_info {
+ enum gve_queue_format queue_format;
+ u16 default_tx_queues;
+ u16 default_rx_queues;
+ u16 max_tx_queues;
+ u16 max_rx_queues;
+ u16 default_tx_ring_size;
+ u16 default_rx_ring_size;
+ u16 max_tx_ring_size;
+ u16 max_rx_ring_size;
+ u16 min_tx_ring_size;
+ u16 min_rx_ring_size;
+ u16 max_mtu;
+ u8 mac[ETH_ALEN];
+ u16 max_rx_buffer_size;
+ u16 header_buf_size;
+ u32 max_flow_rules;
+ u16 rss_key_size;
+ u16 rss_lut_size;
+ u16 tx_pages_per_qpl;
+ u16 num_event_counters;
+ u64 max_registered_pages;
+ bool default_min_ring_size;
+ bool nic_timestamp_supported;
+ bool modify_ring_size_enabled;
+ bool cache_rss_config;
+};
+
+/**
+ * struct gve_ctrl_ops - Control plane operations structure
+ * @map_db_bar: Maps the doorbell BAR for the device and store in @priv.
+ * @unmap_db_bar: Unmaps the doorbell BAR previously mapped by @map_db_bar.
+ * @set_num_queues: Sets default and max TX/RX queues into allocation
+ * structures stored in @priv to be used during initialization.
+ * @set_num_ntfy_blks: Sets no. of vectors into @priv to be used during
+ * initialization.
+ * @request_db_info: Request and store doorbell information into @priv
+ * @release_db_resources: Release device hold on DMA memory holding doorbell
+ * info (AdminQ only)
+ * @setup_mgmt_irq: Setup control plane IRQ
+ * @teardown_mgmt_irq: Teardown control plane IRQ
+ * @get_ptype_map: Learn packet type map from device and store it in @priv
+ * @configure_rss: Set up default RSS configuration
+ * @setup_stats_report: Set up DMA region for stats report (AdminQ only)
+ * @reset_flow_rules: Flush all flow rules from device
+ * @create_queues: Sends commands to the device to create TX/RX queues.
+ * @destroy_queues: Sends commands to the device to destroy TX/RX queues.
+ * @report_link_status: Set link status into @priv->link_up
+ * @report_link_speed: Set link speed into @priv->link_speed
+ */
+struct gve_ctrl_ops {
+ int (*map_db_bar)(struct gve_priv *priv);
+ void (*unmap_db_bar)(struct gve_priv *priv);
+ void (*set_num_queues)(struct gve_priv *priv);
+ int (*set_num_ntfy_blks)(struct gve_priv *priv);
+ int (*request_db_info)(struct gve_priv *priv);
+ void (*release_db_resources)(struct gve_priv *priv);
+ int (*setup_mgmt_irq)(struct gve_priv *priv);
+ void (*teardown_mgmt_irq)(struct gve_priv *priv);
+ int (*get_ptype_map)(struct gve_priv *priv);
+ int (*configure_rss)(struct gve_priv *priv,
+ struct ethtool_rxfh_param *param);
+ int (*setup_stats_report)(struct gve_priv *priv,
+ u64 stats_report_len,
+ dma_addr_t stats_report_addr,
+ u64 interval_ms); /* AQ-specific */
+ int (*reset_flow_rules)(struct gve_priv *priv);
+ int (*create_queues)(struct gve_priv *priv);
+ int (*destroy_queues)(struct gve_priv *priv);
+ int (*report_link_status)(struct gve_priv *priv);
+ int (*report_link_speed)(struct gve_priv *priv);
+};
+
struct gve_priv {
struct net_device *dev;
struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
@@ -896,8 +970,10 @@ struct gve_priv {
/* Gvnic device link speed from hypervisor. */
u64 link_speed;
+ bool link_up;
bool up_before_suspend; /* True if dev was up before suspend */
+ bool mgmt_irq_requested;
struct gve_ptype_lut *ptype_lut_dqo;
/* Must be a power of two. */
@@ -929,6 +1005,8 @@ struct gve_priv {
struct gve_nic_ts_report *nic_ts_report;
dma_addr_t nic_ts_report_bus;
u64 last_sync_nic_counter; /* Clock counter from last NIC TS report */
+ struct gve_device_info device_info;
+ const struct gve_ctrl_ops *ctrl_ops;
};
enum gve_service_task_flags_bit {
@@ -1104,6 +1182,11 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
return (priv->num_ntfy_blks / 2) + queue_idx;
}
+static inline u32 gve_ntfy_to_msix_idx(struct gve_priv *priv, u32 ntfy_blk_idx)
+{
+ return ntfy_blk_idx;
+}
+
static inline bool gve_is_qpl(struct gve_priv *priv)
{
return priv->queue_format == GVE_GQI_QPL_FORMAT ||
@@ -1207,6 +1290,8 @@ static inline bool gve_is_clock_enabled(struct gve_priv *priv)
return priv->nic_ts_report;
}
+void gve_adminq_write_version(u8 __iomem *driver_version_register);
+
/* gqi napi handler defined in gve_main.c */
int gve_napi_poll(struct napi_struct *napi, int budget);
@@ -1300,7 +1385,7 @@ struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv,
/* Reset */
void gve_schedule_reset(struct gve_priv *priv);
-int gve_reset(struct gve_priv *priv, bool attempt_teardown);
+int gve_reset(struct gve_priv *priv, bool skip_queue_setup);
void gve_get_curr_alloc_cfgs(struct gve_priv *priv,
struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
struct gve_rx_alloc_rings_cfg *rx_alloc_cfg);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index f05f4895f4c7..8b215fca9c1b 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -6,6 +6,8 @@
#include <linux/etherdevice.h>
#include <linux/pci.h>
+#include <linux/utsname.h>
+#include <linux/version.h>
#include "gve.h"
#include "gve_adminq.h"
#include "gve_register.h"
@@ -70,7 +72,7 @@ void gve_parse_device_option(struct gve_priv *priv,
dev_info(&priv->pdev->dev,
"Gqi raw addressing device option enabled.\n");
- priv->queue_format = GVE_GQI_RDA_FORMAT;
+ priv->device_info.queue_format = GVE_GQI_RDA_FORMAT;
break;
case GVE_DEV_OPT_ID_GQI_RDA:
if (option_length < sizeof(**dev_op_gqi_rda) ||
@@ -190,7 +192,7 @@ void gve_parse_device_option(struct gve_priv *priv,
/* device has not provided min ring size */
if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE)
- priv->default_min_ring_size = true;
+ priv->device_info.default_min_ring_size = true;
break;
case GVE_DEV_OPT_ID_FLOW_STEERING:
if (option_length < sizeof(**dev_op_flow_steering) ||
@@ -296,8 +298,10 @@ gve_process_device_options(struct gve_priv *priv,
return 0;
}
-int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
+static int gve_adminq_alloc(struct gve_priv *priv)
{
+ struct device *dev = &priv->pdev->dev;
+
priv->adminq_pool = dma_pool_create("adminq_pool", dev,
GVE_ADMINQ_BUFFER_SIZE, 0, 0);
if (unlikely(!priv->adminq_pool))
@@ -353,7 +357,15 @@ int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
return 0;
}
-void gve_adminq_release(struct gve_priv *priv)
+int gve_adminq_init(struct gve_priv *priv)
+{
+ struct gve_registers __iomem *reg_bar = priv->reg_bar0;
+
+ gve_adminq_write_version(&reg_bar->driver_version);
+ return gve_adminq_alloc(priv);
+}
+
+static void gve_adminq_release(struct gve_priv *priv)
{
int i = 0;
@@ -382,7 +394,6 @@ void gve_adminq_release(struct gve_priv *priv)
}
gve_clear_device_rings_ok(priv);
gve_clear_device_resources_ok(priv);
- gve_clear_admin_queue_ok(priv);
}
void gve_adminq_free(struct gve_priv *priv)
@@ -922,13 +933,40 @@ out:
static void gve_set_default_rss_sizes(struct gve_priv *priv)
{
- if (!gve_is_gqi(priv)) {
- priv->rss_key_size = GVE_RSS_KEY_SIZE;
- priv->rss_lut_size = GVE_RSS_INDIR_SIZE;
- priv->cache_rss_config = true;
+ struct gve_device_info *device_info = &priv->device_info;
+
+ if (device_info->queue_format == GVE_DQO_RDA_FORMAT ||
+ device_info->queue_format == GVE_DQO_QPL_FORMAT) {
+ device_info->rss_key_size = GVE_RSS_KEY_SIZE;
+ device_info->rss_lut_size = GVE_RSS_INDIR_SIZE;
+ device_info->cache_rss_config = true;
}
}
+int gve_adminq_destroy_queues(struct gve_priv *priv)
+{
+ int num_tx_queues = gve_num_tx_queues(priv);
+ int err;
+
+ err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to destroy tx queues\n");
+ /* This failure will trigger a reset - no need to clean up */
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
+ err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to destroy rx queues\n");
+ /* This failure will trigger a reset - no need to clean up */
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
+ return err;
+}
+
static void gve_enable_supported_features(struct gve_priv *priv,
u32 supported_features_mask,
const struct gve_device_option_jumbo_frames
@@ -946,77 +984,105 @@ static void gve_enable_supported_features(struct gve_priv *priv,
const struct gve_device_option_modify_ring
*dev_op_modify_ring)
{
+ struct gve_device_info *info = &priv->device_info;
+
/* Before control reaches this point, the page-size-capped max MTU from
* the gve_device_descriptor field has already been stored in
- * priv->dev->max_mtu. We overwrite it with the true max MTU below.
+ * device_info->max_mtu. We overwrite it with the true max MTU below.
*/
if (dev_op_jumbo_frames &&
(supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
dev_info(&priv->pdev->dev,
"JUMBO FRAMES device option enabled.\n");
- priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
+ info->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
}
if (dev_op_buffer_sizes &&
(supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) {
- priv->max_rx_buffer_size =
+ info->max_rx_buffer_size =
be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size);
- priv->header_buf_size =
+ info->header_buf_size =
be16_to_cpu(dev_op_buffer_sizes->header_buffer_size);
dev_info(&priv->pdev->dev,
"BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n",
- priv->max_rx_buffer_size, priv->header_buf_size);
- if (gve_is_dqo(priv) &&
- priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
- priv->rx_cfg.packet_buffer_size =
- priv->max_rx_buffer_size;
+ info->max_rx_buffer_size, info->header_buf_size);
}
/* Read and store ring size ranges given by device */
if (dev_op_modify_ring &&
(supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
- priv->modify_ring_size_enabled = true;
- priv->max_rx_desc_cnt =
+ info->modify_ring_size_enabled = true;
+ info->max_rx_ring_size =
be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
- priv->max_tx_desc_cnt =
+ info->max_tx_ring_size =
be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
- if (priv->default_min_ring_size) {
+ if (info->default_min_ring_size) {
/* If device hasn't provided minimums, use default minimums */
- priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
- priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
+ info->min_tx_ring_size = GVE_DEFAULT_MIN_TX_RING_SIZE;
+ info->min_rx_ring_size = GVE_DEFAULT_MIN_RX_RING_SIZE;
} else {
- priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
- priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
+ info->min_rx_ring_size =
+ be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
+ info->min_tx_ring_size =
+ be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
}
}
if (dev_op_flow_steering &&
(supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) {
if (dev_op_flow_steering->max_flow_rules) {
- priv->max_flow_rules =
+ info->max_flow_rules =
be32_to_cpu(dev_op_flow_steering->max_flow_rules);
- priv->dev->hw_features |= NETIF_F_NTUPLE;
dev_info(&priv->pdev->dev,
"FLOW STEERING device option enabled with max rule limit of %u.\n",
- priv->max_flow_rules);
+ info->max_flow_rules);
}
}
if (dev_op_rss_config &&
(supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) {
- priv->rss_key_size =
+ info->rss_key_size =
be16_to_cpu(dev_op_rss_config->hash_key_size);
- priv->rss_lut_size =
+ info->rss_lut_size =
be16_to_cpu(dev_op_rss_config->hash_lut_size);
- priv->cache_rss_config = false;
+ info->cache_rss_config = false;
dev_dbg(&priv->pdev->dev,
"RSS device option enabled with key size of %u, lut size of %u.\n",
- priv->rss_key_size, priv->rss_lut_size);
+ info->rss_key_size, info->rss_lut_size);
}
if (dev_op_nic_timestamp &&
(supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK))
- priv->nic_timestamp_supported = true;
+ info->nic_timestamp_supported = true;
+}
+
+static void gve_fill_device_info(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor)
+{
+ struct gve_device_info *device_info = &priv->device_info;
+ u16 default_num_queues;
+
+ device_info->tx_pages_per_qpl =
+ be16_to_cpu(descriptor->tx_pages_per_qpl);
+ device_info->max_registered_pages =
+ be64_to_cpu(descriptor->max_registered_pages);
+ device_info->num_event_counters = be16_to_cpu(descriptor->counters);
+ ether_addr_copy(device_info->mac, descriptor->mac);
+ device_info->max_mtu = be16_to_cpu(descriptor->mtu);
+
+ default_num_queues = be16_to_cpu(descriptor->default_num_queues);
+ device_info->default_tx_queues = default_num_queues;
+ device_info->default_rx_queues = default_num_queues;
+ device_info->default_tx_ring_size =
+ be16_to_cpu(descriptor->tx_queue_entries);
+ device_info->default_rx_ring_size =
+ be16_to_cpu(descriptor->rx_queue_entries);
+
+ /* set default ranges */
+ device_info->max_tx_ring_size = device_info->default_tx_ring_size;
+ device_info->max_rx_ring_size = device_info->default_rx_ring_size;
+ device_info->min_tx_ring_size = device_info->default_tx_ring_size;
+ device_info->min_rx_ring_size = device_info->default_rx_ring_size;
}
int gve_adminq_describe_device(struct gve_priv *priv)
@@ -1027,6 +1093,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
struct gve_device_option_rss_config *dev_op_rss_config = NULL;
+ struct gve_device_info *device_info = &priv->device_info;
struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
@@ -1070,26 +1137,26 @@ int gve_adminq_describe_device(struct gve_priv *priv)
* DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
*/
if (dev_op_dqo_rda) {
- priv->queue_format = GVE_DQO_RDA_FORMAT;
+ device_info->queue_format = GVE_DQO_RDA_FORMAT;
dev_info(&priv->pdev->dev,
"Driver is running with DQO RDA queue format.\n");
supported_features_mask =
be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
} else if (dev_op_dqo_qpl) {
- priv->queue_format = GVE_DQO_QPL_FORMAT;
+ device_info->queue_format = GVE_DQO_QPL_FORMAT;
supported_features_mask =
be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
} else if (dev_op_gqi_rda) {
- priv->queue_format = GVE_GQI_RDA_FORMAT;
+ device_info->queue_format = GVE_GQI_RDA_FORMAT;
dev_info(&priv->pdev->dev,
"Driver is running with GQI RDA queue format.\n");
supported_features_mask =
be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
- } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
+ } else if (device_info->queue_format == GVE_GQI_RDA_FORMAT) {
dev_info(&priv->pdev->dev,
"Driver is running with GQI RDA queue format.\n");
} else {
- priv->queue_format = GVE_GQI_QPL_FORMAT;
+ device_info->queue_format = GVE_GQI_QPL_FORMAT;
if (dev_op_gqi_qpl)
supported_features_mask =
be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
@@ -1097,18 +1164,9 @@ int gve_adminq_describe_device(struct gve_priv *priv)
"Driver is running with GQI QPL queue format.\n");
}
+ gve_fill_device_info(priv, descriptor);
gve_set_default_rss_sizes(priv);
- err = gve_set_mtu(priv, descriptor);
- if (err)
- goto free_device_descriptor;
-
- priv->num_event_counters = be16_to_cpu(descriptor->counters);
-
- gve_set_mac(priv, descriptor);
-
- gve_set_queue_properties(priv, descriptor);
-
gve_enable_supported_features(priv, supported_features_mask,
dev_op_jumbo_frames, dev_op_dqo_qpl,
dev_op_buffer_sizes, dev_op_flow_steering,
@@ -1120,6 +1178,27 @@ free_device_descriptor:
return err;
}
+int gve_adminq_get_device_properties(struct gve_priv *priv)
+{
+ int err;
+
+ err = gve_adminq_verify_driver_compatibility(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not verify driver compatibility: err=%d\n", err);
+ return err;
+ }
+
+ /* Get the initial information we need from the device */
+ err = gve_adminq_describe_device(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not get device information: err=%d\n", err);
+ return err;
+ }
+ return 0;
+}
+
int gve_adminq_register_page_list(struct gve_priv *priv,
struct gve_queue_page_list *qpl)
{
@@ -1182,20 +1261,53 @@ int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
return gve_adminq_execute_cmd(priv, &cmd);
}
-int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
- u64 driver_info_len,
- dma_addr_t driver_info_addr)
+int gve_adminq_verify_driver_compatibility(struct gve_priv *priv)
{
+ struct gve_driver_info *driver_info;
union gve_adminq_command cmd;
+ dma_addr_t driver_info_bus;
+ int err;
+
+ driver_info = dma_alloc_coherent(&priv->pdev->dev,
+ sizeof(struct gve_driver_info),
+ &driver_info_bus, GFP_KERNEL);
+ if (!driver_info)
+ return -ENOMEM;
+
+ *driver_info = (struct gve_driver_info) {
+ .os_type = 1, /* Linux */
+ .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR),
+ .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL),
+ .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL),
+ .driver_capability_flags = {
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1),
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2),
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3),
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4),
+ },
+ };
+ strscpy(driver_info->os_version_str1, utsname()->release,
+ sizeof(driver_info->os_version_str1));
+ strscpy(driver_info->os_version_str2, utsname()->version,
+ sizeof(driver_info->os_version_str2));
memset(&cmd, 0, sizeof(cmd));
cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
- .driver_info_len = cpu_to_be64(driver_info_len),
- .driver_info_addr = cpu_to_be64(driver_info_addr),
+ .driver_info_len = cpu_to_be64(sizeof(struct gve_driver_info)),
+ .driver_info_addr = cpu_to_be64(driver_info_bus),
};
- return gve_adminq_execute_cmd(priv, &cmd);
+ err = gve_adminq_execute_cmd(priv, &cmd);
+
+ /* It's ok if the device doesn't support this */
+ if (err == -EOPNOTSUPP)
+ err = 0;
+
+ dma_free_coherent(&priv->pdev->dev,
+ sizeof(struct gve_driver_info),
+ driver_info, driver_info_bus);
+ return err;
}
int gve_adminq_report_link_speed(struct gve_priv *priv)
@@ -1241,8 +1353,7 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
return gve_adminq_execute_cmd(priv, &cmd);
}
-int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
- struct gve_ptype_lut *ptype_lut)
+int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv)
{
struct gve_ptype_map *ptype_map;
union gve_adminq_command cmd;
@@ -1268,9 +1379,9 @@ int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
/* Populate ptype_lut. */
for (i = 0; i < GVE_NUM_PTYPES; i++) {
- ptype_lut->ptypes[i].l3_type =
+ priv->ptype_lut_dqo->ptypes[i].l3_type =
ptype_map->ptypes[i].l3_type;
- ptype_lut->ptypes[i].l4_type =
+ priv->ptype_lut_dqo->ptypes[i].l4_type =
ptype_map->ptypes[i].l4_type;
}
err:
@@ -1288,12 +1399,8 @@ gve_adminq_configure_flow_rule(struct gve_priv *priv,
sizeof(struct gve_adminq_configure_flow_rule),
flow_rule_cmd);
- if (err == -ETIME) {
- dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset");
- gve_reset(priv, true);
- } else if (!err) {
+ if (!err)
priv->flow_rules_cache.rules_cache_synced = false;
- }
return err;
}
@@ -1569,7 +1676,7 @@ out:
return err;
}
-int gve_set_num_ntfy_blks(struct gve_priv *priv)
+int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv)
{
int num_ntfy;
@@ -1593,8 +1700,10 @@ int gve_set_num_ntfy_blks(struct gve_priv *priv)
return 0;
}
-void gve_set_num_queues(struct gve_priv *priv)
+void gve_adminq_set_num_queues(struct gve_priv *priv)
{
+ struct gve_device_info *device_info = &priv->device_info;
+
priv->tx_cfg.max_queues =
min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
priv->rx_cfg.max_queues =
@@ -1602,10 +1711,139 @@ void gve_set_num_queues(struct gve_priv *priv)
priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
- if (priv->default_num_queues > 0) {
- priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
+ if (device_info->default_tx_queues > 0)
+ priv->tx_cfg.num_queues = min_t(int,
+ device_info->default_tx_queues,
priv->tx_cfg.num_queues);
- priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
+
+ if (device_info->default_rx_queues > 0)
+ priv->rx_cfg.num_queues = min_t(int,
+ device_info->default_rx_queues,
priv->rx_cfg.num_queues);
+}
+
+int gve_adminq_map_db_bar(struct gve_priv *priv)
+{
+ struct pci_dev *pdev = priv->pdev;
+ void __iomem *db_bar;
+
+ db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
+ if (!db_bar) {
+ dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
+ return -ENOMEM;
}
+ priv->db_bar2 = db_bar;
+ return 0;
+}
+
+void gve_adminq_unmap_db_bar(struct gve_priv *priv)
+{
+ struct pci_dev *pdev = priv->pdev;
+
+ pci_iounmap(pdev, priv->db_bar2);
+}
+
+int gve_adminq_request_db_info(struct gve_priv *priv)
+{
+ int err;
+ int i;
+
+ err = gve_adminq_configure_device_resources(priv,
+ priv->counter_array_bus,
+ priv->num_event_counters,
+ priv->irq_db_indices_bus,
+ priv->num_ntfy_blks);
+ if (unlikely(err)) {
+ dev_err(&priv->pdev->dev,
+ "could not setup device_resources: err=%d\n", err);
+ return -ENXIO;
+ }
+
+ for (i = 0; i < priv->num_ntfy_blks; i++)
+ priv->ntfy_blocks[i].irq_db_index =
+ &priv->irq_db_indices[i].index;
+ return 0;
+}
+
+void gve_adminq_release_db_resources(struct gve_priv *priv)
+{
+ int err;
+
+ /* Log error in deconfigure device, but don't fail. This is only ever
+ * called as a reset is about to be triggered, so it would be redundant
+ * to trigger a reset.
+ */
+ err = gve_adminq_deconfigure_device_resources(priv);
+ if (err)
+ dev_err(&priv->pdev->dev,
+ "Could not deconfigure device resources: err=%d\n",
+ err);
+}
+
+static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
+{
+ struct gve_priv *priv = arg;
+
+ queue_work(priv->gve_wq, &priv->service_task);
+ return IRQ_HANDLED;
+}
+
+int gve_adminq_setup_mgmt_irq(struct gve_priv *priv)
+{
+ int err;
+
+ snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
+ "gve-mgmnt@pci:%s", pci_name(priv->pdev));
+ err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
+ gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
+ if (err)
+ return err;
+
+ priv->mgmt_irq_requested = true;
+
+ return 0;
+}
+
+void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv)
+{
+ if (priv->mgmt_irq_requested) {
+ free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
+ priv->mgmt_irq_requested = false;
+ }
+}
+
+int gve_adminq_create_queues(struct gve_priv *priv)
+{
+ int num_tx_queues = gve_num_tx_queues(priv);
+ int err;
+
+ err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
+ num_tx_queues);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
+ num_tx_queues);
+
+ err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
+ priv->rx_cfg.num_queues);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
+ priv->rx_cfg.num_queues);
+
+ return err;
+}
+
+int gve_adminq_report_link_status(struct gve_priv *priv)
+{
+ u32 status;
+
+ status = ioread32be(&priv->reg_bar0->device_status);
+ WRITE_ONCE(priv->link_up,
+ !!(GVE_DEVICE_STATUS_LINK_STATUS_MASK & status));
+ return 0;
}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 68c63ce75505..8daac5fd685a 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -619,9 +619,8 @@ union gve_adminq_command {
static_assert(sizeof(union gve_adminq_command) == 64);
-int gve_adminq_alloc(struct device *dev, struct gve_priv *priv);
+int gve_adminq_init(struct gve_priv *priv);
void gve_adminq_free(struct gve_priv *priv);
-void gve_adminq_release(struct gve_priv *priv);
int gve_adminq_describe_device(struct gve_priv *priv);
int gve_adminq_configure_device_resources(struct gve_priv *priv,
dma_addr_t counter_array_bus_addr,
@@ -629,6 +628,8 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv,
dma_addr_t db_array_bus_addr,
u32 num_ntfy_blks);
int gve_adminq_deconfigure_device_resources(struct gve_priv *priv);
+int gve_adminq_create_queues(struct gve_priv *priv);
+int gve_adminq_destroy_queues(struct gve_priv *priv);
int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index);
@@ -640,9 +641,9 @@ int gve_adminq_register_page_list(struct gve_priv *priv,
int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id);
int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
dma_addr_t stats_report_addr, u64 interval);
-int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
- u64 driver_info_len,
- dma_addr_t driver_info_addr);
+int gve_adminq_verify_driver_compatibility(struct gve_priv *priv);
+int gve_adminq_get_device_properties(struct gve_priv *priv);
+int gve_adminq_report_link_status(struct gve_priv *priv);
int gve_adminq_report_link_speed(struct gve_priv *priv);
int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc);
int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc);
@@ -654,14 +655,13 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
dma_addr_t nic_ts_report_addr);
struct gve_ptype_lut;
-int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
- struct gve_ptype_lut *ptype_lut);
-int gve_set_num_ntfy_blks(struct gve_priv *priv);
-void gve_set_num_queues(struct gve_priv *priv);
-void gve_set_queue_properties(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor);
-int gve_set_mtu(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor);
-void gve_set_mac(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor);
+int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv);
+int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
+void gve_adminq_set_num_queues(struct gve_priv *priv);
+int gve_adminq_map_db_bar(struct gve_priv *priv);
+void gve_adminq_unmap_db_bar(struct gve_priv *priv);
+int gve_adminq_request_db_info(struct gve_priv *priv);
+void gve_adminq_release_db_resources(struct gve_priv *priv);
+int gve_adminq_setup_mgmt_irq(struct gve_priv *priv);
+void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
index 8199738ba979..a3220c651438 100644
--- a/drivers/net/ethernet/google/gve/gve_ethtool.c
+++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
@@ -651,7 +651,7 @@ static int gve_user_reset(struct net_device *netdev, u32 *flags)
if (*flags == ETH_RESET_ALL) {
*flags = 0;
- return gve_reset(priv, true);
+ return gve_reset(priv, false);
}
return -EOPNOTSUPP;
@@ -749,12 +749,11 @@ static int gve_get_link_ksettings(struct net_device *netdev,
int err = 0;
if (priv->link_speed == 0)
- err = gve_adminq_report_link_speed(priv);
+ err = priv->ctrl_ops->report_link_speed(priv);
cmd->base.speed = priv->link_speed;
cmd->base.duplex = DUPLEX_FULL;
-
return err;
}
diff --git a/drivers/net/ethernet/google/gve/gve_flow_rule.c b/drivers/net/ethernet/google/gve/gve_flow_rule.c
index 2c80cda28ef3..fae552f4ad6f 100644
--- a/drivers/net/ethernet/google/gve/gve_flow_rule.c
+++ b/drivers/net/ethernet/google/gve/gve_flow_rule.c
@@ -278,6 +278,11 @@ int gve_add_flow_rule(struct gve_priv *priv, struct ethtool_rxnfc *cmd)
goto out;
err = gve_adminq_add_flow_rule(priv, rule, fsp->location);
+ if (err == -ETIME) {
+ dev_err(&priv->pdev->dev,
+ "Timeout to add flow rule, trigger reset.");
+ gve_reset(priv, false);
+ }
out:
kvfree(rule);
@@ -290,9 +295,17 @@ out:
int gve_del_flow_rule(struct gve_priv *priv, struct ethtool_rxnfc *cmd)
{
struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
+ int err;
if (!priv->max_flow_rules)
return -EOPNOTSUPP;
- return gve_adminq_del_flow_rule(priv, fsp->location);
+ err = gve_adminq_del_flow_rule(priv, fsp->location);
+ if (err == -ETIME) {
+ dev_err(&priv->pdev->dev,
+ "Timeout to delete flow rule, trigger reset.");
+ gve_reset(priv, false);
+ }
+
+ return err;
}
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index a6c18e2431cc..2437cd8ba633 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -17,8 +17,6 @@
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
-#include <linux/utsname.h>
-#include <linux/version.h>
#include <net/netdev_queues.h>
#include <net/sch_generic.h>
#include <net/xdp_sock_drv.h>
@@ -41,49 +39,6 @@ char gve_driver_name[] = "gve";
const char gve_version_str[] = GVE_VERSION;
static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
-static int gve_verify_driver_compatibility(struct gve_priv *priv)
-{
- int err;
- struct gve_driver_info *driver_info;
- dma_addr_t driver_info_bus;
-
- driver_info = dma_alloc_coherent(&priv->pdev->dev,
- sizeof(struct gve_driver_info),
- &driver_info_bus, GFP_KERNEL);
- if (!driver_info)
- return -ENOMEM;
-
- *driver_info = (struct gve_driver_info) {
- .os_type = 1, /* Linux */
- .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR),
- .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL),
- .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL),
- .driver_capability_flags = {
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1),
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2),
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3),
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4),
- },
- };
- strscpy(driver_info->os_version_str1, utsname()->release,
- sizeof(driver_info->os_version_str1));
- strscpy(driver_info->os_version_str2, utsname()->version,
- sizeof(driver_info->os_version_str2));
-
- err = gve_adminq_verify_driver_compatibility(priv,
- sizeof(struct gve_driver_info),
- driver_info_bus);
-
- /* It's ok if the device doesn't support this */
- if (err == -EOPNOTSUPP)
- err = 0;
-
- dma_free_coherent(&priv->pdev->dev,
- sizeof(struct gve_driver_info),
- driver_info, driver_info_bus);
- return err;
-}
-
static netdev_features_t gve_features_check(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features)
@@ -248,6 +203,30 @@ static void gve_free_counter_array(struct gve_priv *priv)
priv->counter_array = NULL;
}
+static int gve_alloc_irq_db_indices(struct gve_priv *priv)
+{
+ priv->irq_db_indices =
+ dma_alloc_coherent(&priv->pdev->dev,
+ priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ &priv->irq_db_indices_bus, GFP_KERNEL);
+ if (!priv->irq_db_indices)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void gve_free_irq_db_indices(struct gve_priv *priv)
+{
+ if (!priv->irq_db_indices)
+ return;
+
+ dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ priv->irq_db_indices, priv->irq_db_indices_bus);
+ priv->irq_db_indices = NULL;
+}
+
/* NIC requests to report stats */
static void gve_stats_report_task(struct work_struct *work)
{
@@ -306,19 +285,12 @@ static void gve_free_stats_report(struct gve_priv *priv)
return;
timer_delete_sync(&priv->stats_report_timer);
+ cancel_work_sync(&priv->stats_report_task);
dma_free_coherent(&priv->pdev->dev, priv->stats_report_len,
priv->stats_report, priv->stats_report_bus);
priv->stats_report = NULL;
}
-static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
-{
- struct gve_priv *priv = arg;
-
- queue_work(priv->gve_wq, &priv->service_task);
- return IRQ_HANDLED;
-}
-
static irqreturn_t gve_intr(int irq, void *arg)
{
struct gve_notify_block *block = arg;
@@ -469,6 +441,15 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)
return work_done;
}
+static void gve_free_notify_blocks(struct gve_priv *priv)
+{
+ pci_disable_msix(priv->pdev);
+ kvfree(priv->ntfy_blocks);
+ priv->ntfy_blocks = NULL;
+ kvfree(priv->msix_vectors);
+ priv->msix_vectors = NULL;
+}
+
static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
{
if (priv->numa_node == NUMA_NO_NODE)
@@ -480,11 +461,9 @@ static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
static int gve_alloc_notify_blocks(struct gve_priv *priv)
{
int num_vecs_requested = priv->num_ntfy_blks + 1;
- const struct cpumask *node_mask;
- unsigned int cur_cpu;
int vecs_enabled;
- int i, j;
int err;
+ int i;
priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors,
num_vecs_requested);
@@ -498,7 +477,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
GVE_MIN_MSIX, vecs_enabled);
err = vecs_enabled;
- goto abort_with_msix_vectors;
+ goto abort;
}
if (vecs_enabled != num_vecs_requested) {
int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
@@ -521,38 +500,64 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
}
- /* Setup Management Vector - the last vector */
- snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s",
- pci_name(priv->pdev));
- err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
- gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
- if (err) {
- dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
- goto abort_with_msix_enabled;
- }
- priv->irq_db_indices =
- dma_alloc_coherent(&priv->pdev->dev,
- priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- &priv->irq_db_indices_bus, GFP_KERNEL);
- if (!priv->irq_db_indices) {
- err = -ENOMEM;
- goto abort_with_mgmt_vector;
- }
-
priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
sizeof(*priv->ntfy_blocks), GFP_KERNEL);
if (!priv->ntfy_blocks) {
err = -ENOMEM;
- goto abort_with_irq_db_indices;
+ goto abort;
+ }
+
+ return 0;
+abort:
+ gve_free_notify_blocks(priv);
+ return err;
+}
+
+static void gve_teardown_notify_blocks(struct gve_priv *priv)
+{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
+ int i;
+
+ ops->teardown_mgmt_irq(priv);
+
+ if (!priv->ntfy_blocks)
+ return;
+
+ for (i = 0; i < priv->num_ntfy_blks; i++) {
+ struct gve_notify_block *block = &priv->ntfy_blocks[i];
+ int msix_idx = gve_ntfy_to_msix_idx(priv, i);
+
+ if (!block->irq_requested)
+ continue;
+
+ irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
+ NULL);
+ free_irq(priv->msix_vectors[msix_idx].vector, block);
+ block->irq_requested = false;
+ }
+}
+
+static int gve_setup_notify_blocks(struct gve_priv *priv)
+{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
+ const struct cpumask *node_mask;
+ unsigned int cur_cpu;
+ int i;
+ int err;
+
+ /* Setup Management Vector */
+ err = ops->setup_mgmt_irq(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Did not receive management vector.\n");
+ return err;
}
- /* Setup the other blocks - the first n-1 vectors */
node_mask = gve_get_node_mask(priv);
cur_cpu = cpumask_first(node_mask);
for (i = 0; i < priv->num_ntfy_blks; i++) {
struct gve_notify_block *block = &priv->ntfy_blocks[i];
- int msix_idx = i;
+ int msix_idx = gve_ntfy_to_msix_idx(priv, i);
snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s",
i, pci_name(priv->pdev));
@@ -562,13 +567,13 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
IRQF_NO_AUTOEN, block->name, block);
if (err) {
dev_err(&priv->pdev->dev,
- "Failed to receive msix vector %d\n", i);
- goto abort_with_some_ntfy_blocks;
+ "Failed to receive msix vector %d\n", msix_idx);
+ goto abort;
}
block->irq = priv->msix_vectors[msix_idx].vector;
+ block->irq_requested = true;
irq_set_affinity_and_hint(block->irq,
cpumask_of(cur_cpu));
- block->irq_db_index = &priv->irq_db_indices[i].index;
cur_cpu = cpumask_next(cur_cpu, node_mask);
/* Wrap once CPUs in the node have been exhausted, or when
@@ -579,63 +584,87 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
cur_cpu = cpumask_first(node_mask);
}
return 0;
-abort_with_some_ntfy_blocks:
- for (j = 0; j < i; j++) {
- struct gve_notify_block *block = &priv->ntfy_blocks[j];
- int msix_idx = j;
- irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
- NULL);
- free_irq(priv->msix_vectors[msix_idx].vector, block);
- block->irq = 0;
- }
- kvfree(priv->ntfy_blocks);
- priv->ntfy_blocks = NULL;
-abort_with_irq_db_indices:
- dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices, priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
-abort_with_mgmt_vector:
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
-abort_with_msix_enabled:
- pci_disable_msix(priv->pdev);
-abort_with_msix_vectors:
- kvfree(priv->msix_vectors);
- priv->msix_vectors = NULL;
+abort:
+ gve_teardown_notify_blocks(priv);
return err;
}
-static void gve_free_notify_blocks(struct gve_priv *priv)
+static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv,
+ struct gve_tx_alloc_rings_cfg *cfg)
{
- int i;
+ cfg->qcfg = &priv->tx_cfg;
+ cfg->raw_addressing = !gve_is_qpl(priv);
+ cfg->ring_size = priv->tx_desc_cnt;
+ cfg->pages_per_qpl = priv->tx_pages_per_qpl;
+ cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues;
+ cfg->tx = priv->tx;
+}
- if (!priv->msix_vectors)
- return;
+static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv,
+ struct gve_rx_alloc_rings_cfg *cfg)
+{
+ cfg->qcfg_rx = &priv->rx_cfg;
+ cfg->qcfg_tx = &priv->tx_cfg;
+ cfg->raw_addressing = !gve_is_qpl(priv);
+ cfg->enable_header_split = priv->header_split_enabled;
+ cfg->ring_size = priv->rx_desc_cnt;
+ cfg->pages_per_qpl = priv->rx_pages_per_qpl;
+ cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size;
+ cfg->rx = priv->rx;
+ cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues;
+}
- /* Free the irqs */
- for (i = 0; i < priv->num_ntfy_blks; i++) {
- struct gve_notify_block *block = &priv->ntfy_blocks[i];
- int msix_idx = i;
+void gve_get_curr_alloc_cfgs(struct gve_priv *priv,
+ struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
+ struct gve_rx_alloc_rings_cfg *rx_alloc_cfg)
+{
+ gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg);
+ gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg);
+}
- irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
- NULL);
- free_irq(priv->msix_vectors[msix_idx].vector, block);
- block->irq = 0;
+static void gve_queues_mem_free(struct gve_priv *priv,
+ struct gve_tx_alloc_rings_cfg *tx_cfg,
+ struct gve_rx_alloc_rings_cfg *rx_cfg)
+{
+ if (gve_is_gqi(priv)) {
+ gve_tx_free_rings_gqi(priv, tx_cfg);
+ gve_rx_free_rings_gqi(priv, rx_cfg);
+ } else {
+ gve_tx_free_rings_dqo(priv, tx_cfg);
+ gve_rx_free_rings_dqo(priv, rx_cfg);
}
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
- kvfree(priv->ntfy_blocks);
- priv->ntfy_blocks = NULL;
- dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices, priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
- pci_disable_msix(priv->pdev);
- kvfree(priv->msix_vectors);
- priv->msix_vectors = NULL;
}
-static int gve_setup_device_resources(struct gve_priv *priv)
+static void gve_queues_mem_remove(struct gve_priv *priv)
+{
+ struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0};
+ struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0};
+
+ gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
+ gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg);
+ priv->tx = NULL;
+ priv->rx = NULL;
+}
+
+static void gve_free_control_plane_resources(struct gve_priv *priv)
+{
+ bitmap_free(priv->xsk_pools);
+ priv->xsk_pools = NULL;
+
+ kvfree(priv->ptype_lut_dqo);
+ priv->ptype_lut_dqo = NULL;
+
+ gve_teardown_clock(priv);
+ gve_free_irq_db_indices(priv);
+ gve_free_counter_array(priv);
+ gve_free_notify_blocks(priv);
+ gve_free_stats_report(priv);
+ gve_free_rss_config_cache(priv);
+ gve_free_flow_rule_caches(priv);
+}
+
+static int gve_alloc_control_plane_resources(struct gve_priv *priv)
{
int err;
@@ -644,39 +673,58 @@ static int gve_setup_device_resources(struct gve_priv *priv)
return err;
err = gve_alloc_rss_config_cache(priv);
if (err)
- goto abort_with_flow_rule_caches;
- err = gve_alloc_counter_array(priv);
- if (err)
- goto abort_with_rss_config_cache;
+ goto abort;
err = gve_alloc_notify_blocks(priv);
if (err)
- goto abort_with_counter;
+ goto abort;
err = gve_alloc_stats_report(priv);
if (err)
- goto abort_with_ntfy_blocks;
- err = gve_adminq_configure_device_resources(priv,
- priv->counter_array_bus,
- priv->num_event_counters,
- priv->irq_db_indices_bus,
- priv->num_ntfy_blks);
- if (unlikely(err)) {
- dev_err(&priv->pdev->dev,
- "could not setup device_resources: err=%d\n", err);
- err = -ENXIO;
- goto abort_with_stats_report;
- }
+ goto abort;
+ err = gve_alloc_counter_array(priv);
+ if (err)
+ goto abort;
+ err = gve_alloc_irq_db_indices(priv);
+ if (err)
+ goto abort;
if (!gve_is_gqi(priv)) {
- priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo);
+ priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo,
+ GFP_KERNEL);
if (!priv->ptype_lut_dqo) {
err = -ENOMEM;
- goto abort_with_stats_report;
+ goto abort;
}
- err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
+ }
+
+ priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
+ if (!priv->xsk_pools) {
+ err = -ENOMEM;
+ goto abort;
+ }
+
+ return 0;
+abort:
+ gve_free_control_plane_resources(priv);
+ return err;
+}
+
+static int gve_setup_control_plane_resources(struct gve_priv *priv)
+{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
+ int err;
+
+ err = ops->request_db_info(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev, "Failed to get db info");
+ return err;
+ }
+
+ if (!gve_is_gqi(priv)) {
+ err = ops->get_ptype_map(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Failed to get ptype map: err=%d\n", err);
- goto abort_with_ptype_lut;
+ goto release_db_resources;
}
}
@@ -691,79 +739,89 @@ static int gve_setup_device_resources(struct gve_priv *priv)
err = gve_init_rss_config(priv, priv->rx_cfg.num_queues);
if (err) {
dev_err(&priv->pdev->dev, "Failed to init RSS config");
- goto abort_with_clock;
+ goto release_db_resources;
}
- err = gve_adminq_report_stats(priv, priv->stats_report_len,
+ err = ops->setup_stats_report(priv, priv->stats_report_len,
priv->stats_report_bus,
GVE_STATS_REPORT_TIMER_PERIOD);
if (err)
dev_err(&priv->pdev->dev,
"Failed to report stats: err=%d\n", err);
+
gve_set_device_resources_ok(priv);
return 0;
-abort_with_clock:
- gve_teardown_clock(priv);
-abort_with_ptype_lut:
- kvfree(priv->ptype_lut_dqo);
- priv->ptype_lut_dqo = NULL;
-abort_with_stats_report:
- gve_free_stats_report(priv);
-abort_with_ntfy_blocks:
- gve_free_notify_blocks(priv);
-abort_with_counter:
- gve_free_counter_array(priv);
-abort_with_rss_config_cache:
- gve_free_rss_config_cache(priv);
-abort_with_flow_rule_caches:
- gve_free_flow_rule_caches(priv);
-
+release_db_resources:
+ ops->release_db_resources(priv);
return err;
}
-static void gve_trigger_reset(struct gve_priv *priv);
-
-static void gve_teardown_device_resources(struct gve_priv *priv)
+/**
+ * gve_teardown_control_plane_resources() - Request the device to release any
+ * shared allocated resources.
+ *
+ * @priv: Pointer to the GVE private device data structure.
+ *
+ * If any part of the teardown step fails, the failure is documented, but is
+ * otherwise ignored. It is expected that a device reset is triggered
+ * immediately after tearing down device resources, which would clear any
+ * lingering state on the device.
+ */
+static void gve_teardown_control_plane_resources(struct gve_priv *priv)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
int err;
+ if (priv->ptp)
+ ptp_cancel_worker_sync(priv->ptp->clock);
+
/* Tell device its resources are being freed */
if (gve_get_device_resources_ok(priv)) {
err = gve_flow_rules_reset(priv);
- if (err) {
+ if (err)
dev_err(&priv->pdev->dev,
"Failed to reset flow rules: err=%d\n", err);
- gve_trigger_reset(priv);
- }
/* detach the stats report */
- err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD);
- if (err) {
+ err = ops->setup_stats_report(priv, 0, 0x0,
+ GVE_STATS_REPORT_TIMER_PERIOD);
+ if (err)
dev_err(&priv->pdev->dev,
"Failed to detach stats report: err=%d\n", err);
- gve_trigger_reset(priv);
- }
- err = gve_adminq_deconfigure_device_resources(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not deconfigure device resources: err=%d\n",
- err);
- gve_trigger_reset(priv);
- }
+ ops->release_db_resources(priv);
}
- kvfree(priv->ptype_lut_dqo);
- priv->ptype_lut_dqo = NULL;
-
- gve_free_flow_rule_caches(priv);
- gve_free_rss_config_cache(priv);
- gve_free_counter_array(priv);
- gve_free_notify_blocks(priv);
- gve_free_stats_report(priv);
- gve_teardown_clock(priv);
gve_clear_device_resources_ok(priv);
}
+/**
+ * gve_reset_device() - Reset the device
+ *
+ * @priv: Pointer to the GVE private device data structure.
+ *
+ * Once this returns, the device is guaranteed not to access any memory it
+ * shares with the driver, so it is safe for the caller to recycle it. Device
+ * commands in gve_teardown_control_plane_resources() can fail, in which case
+ * the hardware reset triggered by gve_adminq_free() is the only such
+ * guarantee.
+ */
+static void gve_reset_device(struct gve_priv *priv)
+{
+ gve_teardown_control_plane_resources(priv);
+ gve_adminq_free(priv);
+}
+
+static void gve_teardown_device(struct gve_priv *priv)
+{
+ gve_reset_device(priv);
+ /* Free any resources shared with the device only after we have a
+ * guarantee that the device will not try to access such resources.
+ */
+ gve_teardown_notify_blocks(priv);
+ gve_free_control_plane_resources(priv);
+ gve_queues_mem_remove(priv);
+}
+
static int gve_unregister_qpl(struct gve_priv *priv,
struct gve_queue_page_list *qpl)
{
@@ -886,33 +944,16 @@ static int gve_unregister_qpls(struct gve_priv *priv)
static int gve_create_rings(struct gve_priv *priv)
{
- int num_tx_queues = gve_num_tx_queues(priv);
int err;
int i;
- err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
- if (err) {
- netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
- num_tx_queues);
- /* This failure will trigger a reset - no need to clean
- * up
- */
- return err;
- }
- netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
- num_tx_queues);
-
- err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
+ err = priv->ctrl_ops->create_queues(priv);
if (err) {
- netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
- priv->rx_cfg.num_queues);
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
- netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
- priv->rx_cfg.num_queues);
if (gve_is_gqi(priv)) {
/* Rx data ring has been prefilled with packet buffers at queue
@@ -961,17 +1002,6 @@ static void gve_init_sync_stats(struct gve_priv *priv)
u64_stats_init(&priv->rx[i].statss);
}
-static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv,
- struct gve_tx_alloc_rings_cfg *cfg)
-{
- cfg->qcfg = &priv->tx_cfg;
- cfg->raw_addressing = !gve_is_qpl(priv);
- cfg->ring_size = priv->tx_desc_cnt;
- cfg->pages_per_qpl = priv->tx_pages_per_qpl;
- cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues;
- cfg->tx = priv->tx;
-}
-
static void gve_tx_stop_rings(struct gve_priv *priv, int num_rings)
{
int i;
@@ -1067,39 +1097,12 @@ free_tx:
static int gve_destroy_rings(struct gve_priv *priv)
{
- int num_tx_queues = gve_num_tx_queues(priv);
int err;
- err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
- if (err) {
- netif_err(priv, drv, priv->dev,
- "failed to destroy tx queues\n");
- /* This failure will trigger a reset - no need to clean up */
- return err;
- }
- netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
- err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
- if (err) {
- netif_err(priv, drv, priv->dev,
- "failed to destroy rx queues\n");
- /* This failure will trigger a reset - no need to clean up */
- return err;
- }
- netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
- return 0;
-}
+ err = priv->ctrl_ops->destroy_queues(priv);
-static void gve_queues_mem_free(struct gve_priv *priv,
- struct gve_tx_alloc_rings_cfg *tx_cfg,
- struct gve_rx_alloc_rings_cfg *rx_cfg)
-{
- if (gve_is_gqi(priv)) {
- gve_tx_free_rings_gqi(priv, tx_cfg);
- gve_rx_free_rings_gqi(priv, rx_cfg);
- } else {
- gve_tx_free_rings_dqo(priv, tx_cfg);
- gve_rx_free_rings_dqo(priv, rx_cfg);
- }
+ /* A failure will trigger a reset - no need to clean up */
+ return err;
}
int gve_alloc_page(struct gve_priv *priv, struct device *dev,
@@ -1202,8 +1205,6 @@ void gve_schedule_reset(struct gve_priv *priv)
queue_work(priv->gve_wq, &priv->service_task);
}
-static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
-static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
static void gve_turndown(struct gve_priv *priv);
static void gve_turnup(struct gve_priv *priv);
@@ -1314,37 +1315,16 @@ err:
return err;
}
-
static void gve_drain_page_cache(struct gve_priv *priv)
{
int i;
+ if (!priv->rx)
+ return;
for (i = 0; i < priv->rx_cfg.num_queues; i++)
page_frag_cache_drain(&priv->rx[i].page_cache);
}
-static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv,
- struct gve_rx_alloc_rings_cfg *cfg)
-{
- cfg->qcfg_rx = &priv->rx_cfg;
- cfg->qcfg_tx = &priv->tx_cfg;
- cfg->raw_addressing = !gve_is_qpl(priv);
- cfg->enable_header_split = priv->header_split_enabled;
- cfg->ring_size = priv->rx_desc_cnt;
- cfg->pages_per_qpl = priv->rx_pages_per_qpl;
- cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size;
- cfg->rx = priv->rx;
- cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues;
-}
-
-void gve_get_curr_alloc_cfgs(struct gve_priv *priv,
- struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
- struct gve_rx_alloc_rings_cfg *rx_alloc_cfg)
-{
- gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg);
- gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg);
-}
-
static void gve_rx_start_ring(struct gve_priv *priv, int i)
{
if (gve_is_gqi(priv))
@@ -1380,15 +1360,16 @@ static void gve_rx_stop_rings(struct gve_priv *priv, int num_rings)
gve_rx_stop_ring(priv, i);
}
-static void gve_queues_mem_remove(struct gve_priv *priv)
+static void gve_queues_stop(struct gve_priv *priv)
{
- struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0};
- struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0};
+ gve_unreg_xdp_info(priv);
+ gve_drain_page_cache(priv);
- gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
- gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg);
- priv->tx = NULL;
- priv->rx = NULL;
+ timer_delete_sync(&priv->stats_report_timer);
+ cancel_work_sync(&priv->stats_report_task);
+
+ gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
+ gve_rx_stop_rings(priv, priv->rx_cfg.num_queues);
}
/* The passed-in queue memory is stored into priv and the queues are made live.
@@ -1413,6 +1394,8 @@ static int gve_queues_start(struct gve_priv *priv,
priv->rx_desc_cnt = rx_alloc_cfg->ring_size;
priv->tx_pages_per_qpl = tx_alloc_cfg->pages_per_qpl;
priv->rx_pages_per_qpl = rx_alloc_cfg->pages_per_qpl;
+ priv->header_split_enabled = rx_alloc_cfg->enable_header_split;
+ priv->rx_cfg.packet_buffer_size = rx_alloc_cfg->packet_buffer_size;
gve_tx_start_rings(priv, gve_num_tx_queues(priv));
gve_rx_start_rings(priv, rx_alloc_cfg->qcfg_rx->num_queues);
@@ -1420,14 +1403,14 @@ static int gve_queues_start(struct gve_priv *priv,
err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues);
if (err)
- goto stop_and_free_rings;
+ goto stop_rings;
err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues);
if (err)
- goto stop_and_free_rings;
+ goto stop_rings;
err = gve_reg_xdp_info(priv, dev);
if (err)
- goto stop_and_free_rings;
+ goto stop_rings;
if (rx_alloc_cfg->reset_rss) {
err = gve_init_rss_config(priv, priv->rx_cfg.num_queues);
@@ -1439,9 +1422,6 @@ static int gve_queues_start(struct gve_priv *priv,
if (err)
goto reset;
- priv->header_split_enabled = rx_alloc_cfg->enable_header_split;
- priv->rx_cfg.packet_buffer_size = rx_alloc_cfg->packet_buffer_size;
-
err = gve_create_rings(priv);
if (err)
goto reset;
@@ -1460,16 +1440,14 @@ static int gve_queues_start(struct gve_priv *priv,
reset:
if (gve_get_reset_in_progress(priv))
- goto stop_and_free_rings;
- gve_reset_and_teardown(priv, true);
- /* if this fails there is nothing we can do so just ignore the return */
- gve_reset_recovery(priv, false);
- /* return the original error */
- return err;
-stop_and_free_rings:
- gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
- gve_rx_stop_rings(priv, priv->rx_cfg.num_queues);
- gve_queues_mem_remove(priv);
+ goto stop_rings;
+
+ /* Attempt to reset. If reset is successful, gve_queues_start was
+ * successful with the new config.
+ */
+ return gve_reset(priv, false);
+stop_rings:
+ gve_queues_stop(priv);
return err;
}
@@ -1480,81 +1458,69 @@ static int gve_open(struct net_device *dev)
struct gve_priv *priv = netdev_priv(dev);
int err;
+ if (!gve_get_device_resources_ok(priv)) {
+ dev_err(&priv->pdev->dev,
+ "Attempting to open netdev without resources. Device must be reset.");
+ return -ENODEV;
+ }
+
gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg);
if (err)
return err;
- /* No need to free on error: ownership of resources is lost after
- * calling gve_queues_start.
- */
err = gve_queues_start(priv, &tx_alloc_cfg, &rx_alloc_cfg);
- if (err)
+ if (err) {
+ gve_queues_mem_remove(priv);
return err;
+ }
return 0;
}
-static int gve_queues_stop(struct gve_priv *priv)
+static int gve_close(struct net_device *dev)
{
+ struct gve_priv *priv = netdev_priv(dev);
int err;
- netif_carrier_off(priv->dev);
+ gve_turndown(priv);
+
+ /* Surrender to reset if the queue destroying adminq cmds fail. Reset
+ * will not re-enable the interface.
+ */
if (gve_get_device_rings_ok(priv)) {
- gve_turndown(priv);
- gve_drain_page_cache(priv);
+ gve_clear_device_rings_ok(priv);
err = gve_destroy_rings(priv);
if (err)
- goto err;
+ goto reset;
err = gve_unregister_qpls(priv);
if (err)
- goto err;
- gve_clear_device_rings_ok(priv);
+ goto reset;
}
- timer_delete_sync(&priv->stats_report_timer);
-
- gve_unreg_xdp_info(priv);
-
- gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
- gve_rx_stop_rings(priv, priv->rx_cfg.num_queues);
+ gve_queues_stop(priv);
+ gve_queues_mem_remove(priv);
priv->interface_down_cnt++;
return 0;
-err:
- /* This must have been called from a reset due to the rtnl lock
- * so just return at this point.
- */
- if (gve_get_reset_in_progress(priv))
- return err;
- /* Otherwise reset before returning */
- gve_reset_and_teardown(priv, true);
- return gve_reset_recovery(priv, false);
+reset:
+ err = gve_reset(priv, true);
+ return err;
}
-static int gve_close(struct net_device *dev)
+static void gve_handle_link_status(struct gve_priv *priv)
{
- struct gve_priv *priv = netdev_priv(dev);
- int err;
+ bool link_up;
- err = gve_queues_stop(priv);
- if (err)
- return err;
-
- gve_queues_mem_remove(priv);
- return 0;
-}
-
-static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
-{
if (!gve_get_napi_enabled(priv))
return;
- if (link_status == netif_carrier_ok(priv->dev))
+ link_up = READ_ONCE(priv->link_up);
+ if (link_up == netif_carrier_ok(priv->dev))
return;
- if (link_status) {
+ if (link_up) {
netdev_info(priv->dev, "Device link is up.\n");
netif_carrier_on(priv->dev);
} else {
@@ -1581,7 +1547,6 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
{
struct bpf_prog *old_prog;
int err = 0;
- u32 status;
old_prog = READ_ONCE(priv->xdp_prog);
if (!netif_running(priv->dev)) {
@@ -1608,8 +1573,8 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
bpf_prog_put(old_prog);
out:
- status = ioread32be(&priv->reg_bar0->device_status);
- gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ priv->ctrl_ops->report_link_status(priv);
+ gve_handle_link_status(priv);
return err;
}
@@ -1824,6 +1789,7 @@ static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp)
int gve_init_rss_config(struct gve_priv *priv, u16 num_queues)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
struct gve_rss_config *rss_config = &priv->rss_config;
struct ethtool_rxfh_param rxfh = {0};
u16 i;
@@ -1839,15 +1805,17 @@ int gve_init_rss_config(struct gve_priv *priv, u16 num_queues)
rxfh.hfunc = ETH_RSS_HASH_TOP;
- return gve_adminq_configure_rss(priv, &rxfh);
+ return ops->configure_rss(priv, &rxfh);
}
int gve_flow_rules_reset(struct gve_priv *priv)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
+
if (!priv->max_flow_rules)
return 0;
- return gve_adminq_reset_flow_rules(priv);
+ return ops->reset_flow_rules(priv);
}
int gve_adjust_config(struct gve_priv *priv,
@@ -1878,9 +1846,7 @@ int gve_adjust_config(struct gve_priv *priv,
if (err) {
netif_err(priv, drv, priv->dev,
"Adjust config failed to start new queues, !!! DISABLING ALL QUEUES !!!\n");
- /* No need to free on error: ownership of resources is lost after
- * calling gve_queues_start.
- */
+ gve_queues_mem_remove(priv);
gve_turndown(priv);
return err;
}
@@ -2039,11 +2005,11 @@ static void gve_turnup(struct gve_priv *priv)
static void gve_turnup_and_check_status(struct gve_priv *priv)
{
- u32 status;
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
gve_turnup(priv);
- status = ioread32be(&priv->reg_bar0->device_status);
- gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ ops->report_link_status(priv);
+ gve_handle_link_status(priv);
}
static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv,
@@ -2194,8 +2160,11 @@ static int gve_set_features(struct net_device *netdev,
}
if ((netdev->features & NETIF_F_NTUPLE) && !(features & NETIF_F_NTUPLE)) {
err = gve_flow_rules_reset(priv);
- if (err)
+ if (err) {
+ if (err == -ETIME)
+ gve_schedule_reset(priv);
goto revert_features;
+ }
}
return 0;
@@ -2280,7 +2249,8 @@ static void gve_handle_reset(struct gve_priv *priv)
if (gve_get_do_reset(priv)) {
rtnl_lock();
netdev_lock(priv->dev);
- gve_reset(priv, false);
+ if (gve_get_do_reset(priv))
+ gve_reset(priv, false);
netdev_unlock(priv->dev);
rtnl_unlock();
}
@@ -2367,12 +2337,14 @@ static void gve_service_task(struct work_struct *work)
{
struct gve_priv *priv = container_of(work, struct gve_priv,
service_task);
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
u32 status = ioread32be(&priv->reg_bar0->device_status);
gve_handle_status(priv, status);
gve_handle_reset(priv);
- gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ ops->report_link_status(priv);
+ gve_handle_link_status(priv);
}
static void gve_set_netdev_xdp_features(struct gve_priv *priv)
@@ -2398,96 +2370,126 @@ static const struct xdp_metadata_ops gve_xdp_metadata_ops = {
.xmo_rx_timestamp = gve_xdp_rx_timestamp,
};
-static void gve_set_default_desc_cnt(struct gve_priv *priv,
- const struct gve_device_descriptor *descriptor)
+static void gve_set_desc_cnt(struct gve_priv *priv)
{
- priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
- priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
+ struct gve_device_info *device_info = &priv->device_info;
- /* set default ranges */
- priv->max_tx_desc_cnt = priv->tx_desc_cnt;
- priv->max_rx_desc_cnt = priv->rx_desc_cnt;
- priv->min_tx_desc_cnt = priv->tx_desc_cnt;
- priv->min_rx_desc_cnt = priv->rx_desc_cnt;
+ priv->tx_desc_cnt = device_info->default_tx_ring_size;
+ priv->rx_desc_cnt = device_info->default_rx_ring_size;
+ priv->max_tx_desc_cnt = device_info->max_tx_ring_size;
+ priv->max_rx_desc_cnt = device_info->max_rx_ring_size;
+ priv->min_tx_desc_cnt = device_info->min_tx_ring_size;
+ priv->min_rx_desc_cnt = device_info->min_rx_ring_size;
}
-void gve_set_queue_properties(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor)
+static void gve_set_queue_properties(struct gve_priv *priv)
{
- /* set default descriptor counts */
- gve_set_default_desc_cnt(priv, descriptor);
+ struct gve_device_info *device_info = &priv->device_info;
- priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages);
- priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
- priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
+ gve_set_desc_cnt(priv);
+ priv->max_registered_pages = device_info->max_registered_pages;
+ priv->tx_pages_per_qpl = device_info->tx_pages_per_qpl;
}
-int gve_set_mtu(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor)
+static int gve_set_mtu(struct gve_priv *priv)
{
+ struct gve_device_info *device_info = &priv->device_info;
u16 mtu;
- mtu = be16_to_cpu(descriptor->mtu);
+ mtu = device_info->max_mtu;
if (mtu < ETH_MIN_MTU) {
dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
return -EINVAL;
}
priv->dev->max_mtu = mtu;
+ priv->dev->mtu = mtu;
return 0;
}
-void gve_set_mac(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor)
+static void gve_set_mac(struct gve_priv *priv)
{
+ struct gve_device_info *device_info = &priv->device_info;
u8 *mac;
- mac = descriptor->mac;
+ mac = device_info->mac;
eth_hw_addr_set(priv->dev, mac);
dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
}
-static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
+static void gve_set_buf_sizes(struct gve_priv *priv)
+{
+ struct gve_device_info *device_info = &priv->device_info;
+
+ if (device_info->max_rx_buffer_size)
+ priv->max_rx_buffer_size = device_info->max_rx_buffer_size;
+
+ if (gve_is_dqo(priv) &&
+ priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
+ priv->rx_cfg.packet_buffer_size = priv->max_rx_buffer_size;
+
+ if (device_info->header_buf_size)
+ priv->header_buf_size = device_info->header_buf_size;
+}
+
+static int gve_setup_device(struct gve_priv *priv)
{
int err;
- /* Set up the adminq */
- err = gve_adminq_alloc(&priv->pdev->dev, priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Failed to alloc admin queue: err=%d\n", err);
+ priv->num_registered_pages = 0;
+
+ gve_set_netdev_xdp_features(priv);
+ if (!gve_is_gqi(priv))
+ priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
+
+ err = gve_alloc_control_plane_resources(priv);
+ if (err)
return err;
- }
- err = gve_verify_driver_compatibility(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not verify driver compatibility: err=%d\n", err);
- goto err;
- }
+ err = gve_setup_control_plane_resources(priv);
+ if (err)
+ return err;
- priv->num_registered_pages = 0;
+ err = gve_setup_notify_blocks(priv);
+ if (err)
+ return err;
+ return 0;
+}
- if (skip_describe_device)
- goto setup_device;
+static const struct gve_ctrl_ops gve_adminq_ops = {
+ .map_db_bar = gve_adminq_map_db_bar,
+ .unmap_db_bar = gve_adminq_unmap_db_bar,
+ .set_num_queues = gve_adminq_set_num_queues,
+ .set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks,
+ .get_ptype_map = gve_adminq_get_ptype_map_dqo,
+ .reset_flow_rules = gve_adminq_reset_flow_rules,
+ .setup_stats_report = gve_adminq_report_stats,
+ .configure_rss = gve_adminq_configure_rss,
+ .request_db_info = gve_adminq_request_db_info,
+ .release_db_resources = gve_adminq_release_db_resources,
+ .setup_mgmt_irq = gve_adminq_setup_mgmt_irq,
+ .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
+ .create_queues = gve_adminq_create_queues,
+ .destroy_queues = gve_adminq_destroy_queues,
+ .report_link_status = gve_adminq_report_link_status,
+ .report_link_speed = gve_adminq_report_link_speed,
+};
- priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
- /* Get the initial information we need from the device */
- err = gve_adminq_describe_device(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not get device information: err=%d\n", err);
- goto err;
- }
+static int gve_init_priv(struct gve_priv *priv)
+{
+ struct gve_device_info *device_info = &priv->device_info;
+ int err;
- err = gve_set_num_ntfy_blks(priv);
+ priv->queue_format = priv->device_info.queue_format;
+
+ err = priv->ctrl_ops->set_num_ntfy_blks(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Could not setup notify blocks: err=%d\n", err);
- goto err;
+ return err;
}
- gve_set_num_queues(priv);
+ priv->ctrl_ops->set_num_queues(priv);
dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
@@ -2507,108 +2509,117 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
}
- priv->dev->mtu = priv->dev->max_mtu;
- priv->numa_node = dev_to_node(&priv->pdev->dev);
- priv->tx_cfg.num_xdp_queues = 0;
- priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
- priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
- priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
-
-setup_device:
- priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
- if (!priv->xsk_pools) {
- err = -ENOMEM;
- goto err;
- }
+ if (gve_set_mtu(priv))
+ return -EINVAL;
- gve_set_netdev_xdp_features(priv);
- if (!gve_is_gqi(priv))
- priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
+ priv->num_event_counters = device_info->num_event_counters;
- err = gve_setup_device_resources(priv);
- if (err)
- goto err_free_xsk_bitmap;
+ gve_set_mac(priv);
- return 0;
+ gve_set_queue_properties(priv);
+ priv->modify_ring_size_enabled = device_info->modify_ring_size_enabled;
-err_free_xsk_bitmap:
- bitmap_free(priv->xsk_pools);
- priv->xsk_pools = NULL;
-err:
- gve_adminq_free(priv);
- return err;
-}
+ gve_set_buf_sizes(priv);
-static void gve_teardown_priv_resources(struct gve_priv *priv)
-{
- gve_teardown_device_resources(priv);
- gve_adminq_free(priv);
- bitmap_free(priv->xsk_pools);
- priv->xsk_pools = NULL;
-}
+ priv->max_flow_rules = device_info->max_flow_rules;
+ if (priv->max_flow_rules)
+ priv->dev->hw_features |= NETIF_F_NTUPLE;
-static void gve_trigger_reset(struct gve_priv *priv)
-{
- /* Reset the device by releasing the AQ */
- gve_adminq_release(priv);
-}
+ priv->rss_key_size = device_info->rss_key_size;
+ priv->rss_lut_size = device_info->rss_lut_size;
+ priv->cache_rss_config = device_info->cache_rss_config;
-static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
-{
- gve_trigger_reset(priv);
- /* With the reset having already happened, close cannot fail */
- if (was_up)
- gve_close(priv->dev);
- gve_teardown_priv_resources(priv);
+ priv->numa_node = dev_to_node(&priv->pdev->dev);
+ priv->tx_cfg.num_xdp_queues = 0;
+ priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
+ priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
+ priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
+ priv->nic_timestamp_supported = device_info->nic_timestamp_supported;
+ return 0;
}
-static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
+static int gve_recover(struct gve_priv *priv, bool setup_queues)
{
int err;
- err = gve_init_priv(priv, true);
+ err = gve_adminq_init(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc admin queue: err=%d\n", err);
+ goto teardown_device;
+ }
+
+ err = gve_adminq_verify_driver_compatibility(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not verify driver compatibility: err=%d\n", err);
+ goto teardown_device;
+ }
+
+ err = gve_setup_device(priv);
if (err)
- goto err;
- if (was_up) {
+ goto teardown_device;
+
+ if (setup_queues) {
+ /* On failure, hold on to the control plane to give a
+ * chance for the queues to be brought up later.
+ */
err = gve_open(priv->dev);
- if (err)
- goto err;
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to start queues: err=%d, !!! DISABLING ALL QUEUES !!!\n",
+ err);
+ return err;
+ }
}
+
+ /* undo any detach from an earlier failure */
+ netif_device_attach(priv->dev);
+
return 0;
-err:
- dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
- gve_turndown(priv);
+
+teardown_device:
+ dev_err(&priv->pdev->dev, "Recover failed: err=%d, detaching device\n",
+ err);
+ netif_device_detach(priv->dev);
+ gve_teardown_device(priv);
return err;
}
-int gve_reset(struct gve_priv *priv, bool attempt_teardown)
+int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
{
bool was_up = netif_running(priv->dev);
int err;
+ if (gve_get_reset_in_progress(priv))
+ return 0;
+
dev_info(&priv->pdev->dev, "Performing reset\n");
gve_clear_do_reset(priv);
gve_set_reset_in_progress(priv);
- /* If we aren't attempting to teardown normally, just go turndown and
- * reset right away.
- */
- if (!attempt_teardown) {
+
+ if (was_up) {
gve_turndown(priv);
- gve_reset_and_teardown(priv, was_up);
- } else {
- /* Otherwise attempt to close normally */
- if (was_up) {
- err = gve_close(priv->dev);
- /* If that fails reset as we did above */
- if (err)
- gve_reset_and_teardown(priv, was_up);
+ if (gve_get_device_rings_ok(priv)) {
+ gve_clear_device_rings_ok(priv);
+ gve_destroy_rings(priv);
+ gve_unregister_qpls(priv);
}
- /* Clean up any remaining resources */
- gve_teardown_priv_resources(priv);
}
- /* Set it all back up */
- err = gve_reset_recovery(priv, was_up);
+ disable_work(&priv->service_task);
+ gve_reset_device(priv);
+ gve_queues_stop(priv);
+ gve_queues_mem_remove(priv);
+ gve_teardown_notify_blocks(priv);
+ gve_free_control_plane_resources(priv);
+
+ enable_work(&priv->service_task);
+ err = gve_recover(priv, was_up && !skip_queue_setup);
+ if (err)
+ dev_info(&priv->pdev->dev,
+ "Failed to recover in reset: %d\n", err);
+
gve_clear_reset_in_progress(priv);
priv->reset_cnt++;
priv->interface_up_cnt = 0;
@@ -2617,7 +2628,7 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown)
return err;
}
-static void gve_write_version(u8 __iomem *driver_version_register)
+void gve_adminq_write_version(u8 __iomem *driver_version_register)
{
const char *c = gve_version_prefix;
@@ -2823,7 +2834,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
int max_tx_queues, max_rx_queues;
struct net_device *dev;
- __be32 __iomem *db_bar;
struct gve_registers __iomem *reg_bar;
struct gve_priv *priv;
int err;
@@ -2851,14 +2861,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto abort_with_pci_region;
}
- db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
- if (!db_bar) {
- dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
- err = -ENOMEM;
- goto abort_with_reg_bar;
- }
-
- gve_write_version(&reg_bar->driver_version);
/* Get max queues to alloc etherdev */
max_tx_queues = ioread32be(&reg_bar->max_tx_queues);
max_rx_queues = ioread32be(&reg_bar->max_rx_queues);
@@ -2867,7 +2869,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
if (!dev) {
dev_err(&pdev->dev, "could not allocate netdev\n");
err = -ENOMEM;
- goto abort_with_db_bar;
+ goto abort_with_reg_bar;
}
SET_NETDEV_DEV(dev, &pdev->dev);
pci_set_drvdata(pdev, dev);
@@ -2899,35 +2901,60 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
priv->pdev = pdev;
priv->msg_enable = DEFAULT_MSG_LEVEL;
priv->reg_bar0 = reg_bar;
- priv->db_bar2 = db_bar;
priv->service_task_flags = 0x0;
priv->state_flags = 0x0;
priv->ethtool_flags = 0x0;
priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
+ err = gve_adminq_init(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc admin queue: err=%d\n", err);
+ goto abort_with_netdev;
+ }
+
+ priv->device_info.queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
+ err = gve_adminq_get_device_properties(priv);
+ if (err)
+ goto abort_with_adminq;
+
+ /* Set adminq ctrl ops */
+ priv->ctrl_ops = &gve_adminq_ops;
+
+ err = priv->ctrl_ops->map_db_bar(priv);
+ if (err)
+ goto abort_with_adminq;
+
gve_set_probe_in_progress(priv);
priv->gve_wq = alloc_ordered_workqueue("gve", 0);
if (!priv->gve_wq) {
dev_err(&pdev->dev, "Could not allocate workqueue");
err = -ENOMEM;
- goto abort_with_netdev;
+ goto abort_with_unmap_db_bar;
}
INIT_WORK(&priv->service_task, gve_service_task);
INIT_WORK(&priv->stats_report_task, gve_stats_report_task);
priv->tx_cfg.max_queues = max_tx_queues;
priv->rx_cfg.max_queues = max_rx_queues;
- err = gve_init_priv(priv, false);
+ err = gve_init_priv(priv);
if (err)
goto abort_with_wq;
+ err = gve_setup_device(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not setup device: err=%d\n", err);
+ goto abort_teardown_device;
+ }
+
if (!gve_is_gqi(priv) && !gve_is_qpl(priv))
dev->netmem_tx = NETMEM_TX_DMA;
err = register_netdev(dev);
if (err)
- goto abort_with_gve_init;
+ goto abort_teardown_device;
dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format);
@@ -2935,18 +2962,22 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
queue_work(priv->gve_wq, &priv->service_task);
return 0;
-abort_with_gve_init:
- gve_teardown_priv_resources(priv);
+abort_teardown_device:
+ disable_work_sync(&priv->service_task);
+ gve_teardown_device(priv);
abort_with_wq:
destroy_workqueue(priv->gve_wq);
+abort_with_unmap_db_bar:
+ priv->ctrl_ops->unmap_db_bar(priv);
+
+abort_with_adminq:
+ gve_adminq_free(priv);
+
abort_with_netdev:
free_netdev(dev);
-abort_with_db_bar:
- pci_iounmap(pdev, db_bar);
-
abort_with_reg_bar:
pci_iounmap(pdev, reg_bar);
@@ -2962,14 +2993,14 @@ static void gve_remove(struct pci_dev *pdev)
{
struct net_device *netdev = pci_get_drvdata(pdev);
struct gve_priv *priv = netdev_priv(netdev);
- __be32 __iomem *db_bar = priv->db_bar2;
void __iomem *reg_bar = priv->reg_bar0;
unregister_netdev(netdev);
- gve_teardown_priv_resources(priv);
+ disable_work_sync(&priv->service_task);
+ gve_teardown_device(priv);
destroy_workqueue(priv->gve_wq);
+ priv->ctrl_ops->unmap_db_bar(priv);
free_netdev(netdev);
- pci_iounmap(pdev, db_bar);
pci_iounmap(pdev, reg_bar);
pci_release_regions(pdev);
pci_disable_device(pdev);
@@ -2981,17 +3012,16 @@ static void gve_shutdown(struct pci_dev *pdev)
struct gve_priv *priv = netdev_priv(netdev);
bool was_up = netif_running(priv->dev);
- netif_device_detach(netdev);
+ disable_work_sync(&priv->service_task);
rtnl_lock();
netdev_lock(netdev);
- if (was_up && gve_close(priv->dev)) {
- /* If the dev was up, attempt to close, if close fails, reset */
- gve_reset_and_teardown(priv, was_up);
- } else {
- /* If the dev wasn't up or close worked, finish tearing down */
- gve_teardown_priv_resources(priv);
- }
+ if (was_up)
+ gve_close(priv->dev);
+
+ /* detach here because gve_close() might attach in recovery */
+ netif_device_detach(netdev);
+ gve_teardown_device(priv);
netdev_unlock(netdev);
rtnl_unlock();
}
@@ -3003,16 +3033,14 @@ static int gve_suspend(struct device *dev)
struct gve_priv *priv = netdev_priv(netdev);
bool was_up = netif_running(priv->dev);
+ disable_work_sync(&priv->service_task);
+
priv->suspend_cnt++;
rtnl_lock();
netdev_lock(netdev);
- if (was_up && gve_close(priv->dev)) {
- /* If the dev was up, attempt to close, if close fails, reset */
- gve_reset_and_teardown(priv, was_up);
- } else {
- /* If the dev wasn't up or close worked, finish tearing down */
- gve_teardown_priv_resources(priv);
- }
+ if (was_up)
+ gve_close(priv->dev);
+ gve_teardown_device(priv);
priv->up_before_suspend = was_up;
netdev_unlock(netdev);
rtnl_unlock();
@@ -3029,7 +3057,8 @@ static int gve_resume(struct device *dev)
priv->resume_cnt++;
rtnl_lock();
netdev_lock(netdev);
- err = gve_reset_recovery(priv, priv->up_before_suspend);
+ enable_work(&priv->service_task);
+ err = gve_recover(priv, priv->up_before_suspend);
netdev_unlock(netdev);
rtnl_unlock();
return err;
diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c
index b53b7fcdcdaf..caed38cd5f2b 100644
--- a/drivers/net/ethernet/google/gve/gve_utils.c
+++ b/drivers/net/ethernet/google/gve/gve_utils.c
@@ -119,6 +119,8 @@ void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
{
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
- disable_irq(block->irq);
+ if (block->irq_requested)
+ disable_irq(block->irq);
+
netif_napi_del_locked(&block->napi);
}