diff options
Diffstat (limited to 'drivers/net/ethernet/google')
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve.h | 87 | ||||
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve_adminq.c | 376 | ||||
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve_adminq.h | 30 | ||||
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve_ethtool.c | 5 | ||||
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve_flow_rule.c | 15 | ||||
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve_main.c | 1059 | ||||
| -rw-r--r-- | drivers/net/ethernet/google/gve/gve_utils.c | 4 |
7 files changed, 971 insertions, 605 deletions
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h index c280ff35ee77..b7ce6e8632db 100644 --- a/drivers/net/ethernet/google/gve/gve.h +++ b/drivers/net/ethernet/google/gve/gve.h @@ -674,6 +674,7 @@ struct gve_notify_block { struct gve_tx_ring *tx; /* tx rings on this block */ struct gve_rx_ring *rx; /* rx rings on this block */ u32 irq; + bool irq_requested; }; /* Tracks allowed and current rx queue settings */ @@ -797,6 +798,79 @@ struct gve_ptp { struct gve_priv *priv; }; +struct gve_device_info { + enum gve_queue_format queue_format; + u16 default_tx_queues; + u16 default_rx_queues; + u16 max_tx_queues; + u16 max_rx_queues; + u16 default_tx_ring_size; + u16 default_rx_ring_size; + u16 max_tx_ring_size; + u16 max_rx_ring_size; + u16 min_tx_ring_size; + u16 min_rx_ring_size; + u16 max_mtu; + u8 mac[ETH_ALEN]; + u16 max_rx_buffer_size; + u16 header_buf_size; + u32 max_flow_rules; + u16 rss_key_size; + u16 rss_lut_size; + u16 tx_pages_per_qpl; + u16 num_event_counters; + u64 max_registered_pages; + bool default_min_ring_size; + bool nic_timestamp_supported; + bool modify_ring_size_enabled; + bool cache_rss_config; +}; + +/** + * struct gve_ctrl_ops - Control plane operations structure + * @map_db_bar: Maps the doorbell BAR for the device and store in @priv. + * @unmap_db_bar: Unmaps the doorbell BAR previously mapped by @map_db_bar. + * @set_num_queues: Sets default and max TX/RX queues into allocation + * structures stored in @priv to be used during initialization. + * @set_num_ntfy_blks: Sets no. of vectors into @priv to be used during + * initialization. + * @request_db_info: Request and store doorbell information into @priv + * @release_db_resources: Release device hold on DMA memory holding doorbell + * info (AdminQ only) + * @setup_mgmt_irq: Setup control plane IRQ + * @teardown_mgmt_irq: Teardown control plane IRQ + * @get_ptype_map: Learn packet type map from device and store it in @priv + * @configure_rss: Set up default RSS configuration + * @setup_stats_report: Set up DMA region for stats report (AdminQ only) + * @reset_flow_rules: Flush all flow rules from device + * @create_queues: Sends commands to the device to create TX/RX queues. + * @destroy_queues: Sends commands to the device to destroy TX/RX queues. + * @report_link_status: Set link status into @priv->link_up + * @report_link_speed: Set link speed into @priv->link_speed + */ +struct gve_ctrl_ops { + int (*map_db_bar)(struct gve_priv *priv); + void (*unmap_db_bar)(struct gve_priv *priv); + void (*set_num_queues)(struct gve_priv *priv); + int (*set_num_ntfy_blks)(struct gve_priv *priv); + int (*request_db_info)(struct gve_priv *priv); + void (*release_db_resources)(struct gve_priv *priv); + int (*setup_mgmt_irq)(struct gve_priv *priv); + void (*teardown_mgmt_irq)(struct gve_priv *priv); + int (*get_ptype_map)(struct gve_priv *priv); + int (*configure_rss)(struct gve_priv *priv, + struct ethtool_rxfh_param *param); + int (*setup_stats_report)(struct gve_priv *priv, + u64 stats_report_len, + dma_addr_t stats_report_addr, + u64 interval_ms); /* AQ-specific */ + int (*reset_flow_rules)(struct gve_priv *priv); + int (*create_queues)(struct gve_priv *priv); + int (*destroy_queues)(struct gve_priv *priv); + int (*report_link_status)(struct gve_priv *priv); + int (*report_link_speed)(struct gve_priv *priv); +}; + struct gve_priv { struct net_device *dev; struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */ @@ -896,8 +970,10 @@ struct gve_priv { /* Gvnic device link speed from hypervisor. */ u64 link_speed; + bool link_up; bool up_before_suspend; /* True if dev was up before suspend */ + bool mgmt_irq_requested; struct gve_ptype_lut *ptype_lut_dqo; /* Must be a power of two. */ @@ -929,6 +1005,8 @@ struct gve_priv { struct gve_nic_ts_report *nic_ts_report; dma_addr_t nic_ts_report_bus; u64 last_sync_nic_counter; /* Clock counter from last NIC TS report */ + struct gve_device_info device_info; + const struct gve_ctrl_ops *ctrl_ops; }; enum gve_service_task_flags_bit { @@ -1104,6 +1182,11 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) return (priv->num_ntfy_blks / 2) + queue_idx; } +static inline u32 gve_ntfy_to_msix_idx(struct gve_priv *priv, u32 ntfy_blk_idx) +{ + return ntfy_blk_idx; +} + static inline bool gve_is_qpl(struct gve_priv *priv) { return priv->queue_format == GVE_GQI_QPL_FORMAT || @@ -1207,6 +1290,8 @@ static inline bool gve_is_clock_enabled(struct gve_priv *priv) return priv->nic_ts_report; } +void gve_adminq_write_version(u8 __iomem *driver_version_register); + /* gqi napi handler defined in gve_main.c */ int gve_napi_poll(struct napi_struct *napi, int budget); @@ -1300,7 +1385,7 @@ struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, /* Reset */ void gve_schedule_reset(struct gve_priv *priv); -int gve_reset(struct gve_priv *priv, bool attempt_teardown); +int gve_reset(struct gve_priv *priv, bool skip_queue_setup); void gve_get_curr_alloc_cfgs(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg); diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c index f05f4895f4c7..8b215fca9c1b 100644 --- a/drivers/net/ethernet/google/gve/gve_adminq.c +++ b/drivers/net/ethernet/google/gve/gve_adminq.c @@ -6,6 +6,8 @@ #include <linux/etherdevice.h> #include <linux/pci.h> +#include <linux/utsname.h> +#include <linux/version.h> #include "gve.h" #include "gve_adminq.h" #include "gve_register.h" @@ -70,7 +72,7 @@ void gve_parse_device_option(struct gve_priv *priv, dev_info(&priv->pdev->dev, "Gqi raw addressing device option enabled.\n"); - priv->queue_format = GVE_GQI_RDA_FORMAT; + priv->device_info.queue_format = GVE_GQI_RDA_FORMAT; break; case GVE_DEV_OPT_ID_GQI_RDA: if (option_length < sizeof(**dev_op_gqi_rda) || @@ -190,7 +192,7 @@ void gve_parse_device_option(struct gve_priv *priv, /* device has not provided min ring size */ if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE) - priv->default_min_ring_size = true; + priv->device_info.default_min_ring_size = true; break; case GVE_DEV_OPT_ID_FLOW_STEERING: if (option_length < sizeof(**dev_op_flow_steering) || @@ -296,8 +298,10 @@ gve_process_device_options(struct gve_priv *priv, return 0; } -int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) +static int gve_adminq_alloc(struct gve_priv *priv) { + struct device *dev = &priv->pdev->dev; + priv->adminq_pool = dma_pool_create("adminq_pool", dev, GVE_ADMINQ_BUFFER_SIZE, 0, 0); if (unlikely(!priv->adminq_pool)) @@ -353,7 +357,15 @@ int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) return 0; } -void gve_adminq_release(struct gve_priv *priv) +int gve_adminq_init(struct gve_priv *priv) +{ + struct gve_registers __iomem *reg_bar = priv->reg_bar0; + + gve_adminq_write_version(®_bar->driver_version); + return gve_adminq_alloc(priv); +} + +static void gve_adminq_release(struct gve_priv *priv) { int i = 0; @@ -382,7 +394,6 @@ void gve_adminq_release(struct gve_priv *priv) } gve_clear_device_rings_ok(priv); gve_clear_device_resources_ok(priv); - gve_clear_admin_queue_ok(priv); } void gve_adminq_free(struct gve_priv *priv) @@ -922,13 +933,40 @@ out: static void gve_set_default_rss_sizes(struct gve_priv *priv) { - if (!gve_is_gqi(priv)) { - priv->rss_key_size = GVE_RSS_KEY_SIZE; - priv->rss_lut_size = GVE_RSS_INDIR_SIZE; - priv->cache_rss_config = true; + struct gve_device_info *device_info = &priv->device_info; + + if (device_info->queue_format == GVE_DQO_RDA_FORMAT || + device_info->queue_format == GVE_DQO_QPL_FORMAT) { + device_info->rss_key_size = GVE_RSS_KEY_SIZE; + device_info->rss_lut_size = GVE_RSS_INDIR_SIZE; + device_info->cache_rss_config = true; } } +int gve_adminq_destroy_queues(struct gve_priv *priv) +{ + int num_tx_queues = gve_num_tx_queues(priv); + int err; + + err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues); + if (err) { + netif_err(priv, drv, priv->dev, + "failed to destroy tx queues\n"); + /* This failure will trigger a reset - no need to clean up */ + return err; + } + netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n"); + err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues); + if (err) { + netif_err(priv, drv, priv->dev, + "failed to destroy rx queues\n"); + /* This failure will trigger a reset - no need to clean up */ + return err; + } + netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n"); + return err; +} + static void gve_enable_supported_features(struct gve_priv *priv, u32 supported_features_mask, const struct gve_device_option_jumbo_frames @@ -946,77 +984,105 @@ static void gve_enable_supported_features(struct gve_priv *priv, const struct gve_device_option_modify_ring *dev_op_modify_ring) { + struct gve_device_info *info = &priv->device_info; + /* Before control reaches this point, the page-size-capped max MTU from * the gve_device_descriptor field has already been stored in - * priv->dev->max_mtu. We overwrite it with the true max MTU below. + * device_info->max_mtu. We overwrite it with the true max MTU below. */ if (dev_op_jumbo_frames && (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) { dev_info(&priv->pdev->dev, "JUMBO FRAMES device option enabled.\n"); - priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); + info->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); } if (dev_op_buffer_sizes && (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) { - priv->max_rx_buffer_size = + info->max_rx_buffer_size = be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size); - priv->header_buf_size = + info->header_buf_size = be16_to_cpu(dev_op_buffer_sizes->header_buffer_size); dev_info(&priv->pdev->dev, "BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n", - priv->max_rx_buffer_size, priv->header_buf_size); - if (gve_is_dqo(priv) && - priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE) - priv->rx_cfg.packet_buffer_size = - priv->max_rx_buffer_size; + info->max_rx_buffer_size, info->header_buf_size); } /* Read and store ring size ranges given by device */ if (dev_op_modify_ring && (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) { - priv->modify_ring_size_enabled = true; - priv->max_rx_desc_cnt = + info->modify_ring_size_enabled = true; + info->max_rx_ring_size = be16_to_cpu(dev_op_modify_ring->max_rx_ring_size); - priv->max_tx_desc_cnt = + info->max_tx_ring_size = be16_to_cpu(dev_op_modify_ring->max_tx_ring_size); - if (priv->default_min_ring_size) { + if (info->default_min_ring_size) { /* If device hasn't provided minimums, use default minimums */ - priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE; - priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE; + info->min_tx_ring_size = GVE_DEFAULT_MIN_TX_RING_SIZE; + info->min_rx_ring_size = GVE_DEFAULT_MIN_RX_RING_SIZE; } else { - priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size); - priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size); + info->min_rx_ring_size = + be16_to_cpu(dev_op_modify_ring->min_rx_ring_size); + info->min_tx_ring_size = + be16_to_cpu(dev_op_modify_ring->min_tx_ring_size); } } if (dev_op_flow_steering && (supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) { if (dev_op_flow_steering->max_flow_rules) { - priv->max_flow_rules = + info->max_flow_rules = be32_to_cpu(dev_op_flow_steering->max_flow_rules); - priv->dev->hw_features |= NETIF_F_NTUPLE; dev_info(&priv->pdev->dev, "FLOW STEERING device option enabled with max rule limit of %u.\n", - priv->max_flow_rules); + info->max_flow_rules); } } if (dev_op_rss_config && (supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) { - priv->rss_key_size = + info->rss_key_size = be16_to_cpu(dev_op_rss_config->hash_key_size); - priv->rss_lut_size = + info->rss_lut_size = be16_to_cpu(dev_op_rss_config->hash_lut_size); - priv->cache_rss_config = false; + info->cache_rss_config = false; dev_dbg(&priv->pdev->dev, "RSS device option enabled with key size of %u, lut size of %u.\n", - priv->rss_key_size, priv->rss_lut_size); + info->rss_key_size, info->rss_lut_size); } if (dev_op_nic_timestamp && (supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK)) - priv->nic_timestamp_supported = true; + info->nic_timestamp_supported = true; +} + +static void gve_fill_device_info(struct gve_priv *priv, + struct gve_device_descriptor *descriptor) +{ + struct gve_device_info *device_info = &priv->device_info; + u16 default_num_queues; + + device_info->tx_pages_per_qpl = + be16_to_cpu(descriptor->tx_pages_per_qpl); + device_info->max_registered_pages = + be64_to_cpu(descriptor->max_registered_pages); + device_info->num_event_counters = be16_to_cpu(descriptor->counters); + ether_addr_copy(device_info->mac, descriptor->mac); + device_info->max_mtu = be16_to_cpu(descriptor->mtu); + + default_num_queues = be16_to_cpu(descriptor->default_num_queues); + device_info->default_tx_queues = default_num_queues; + device_info->default_rx_queues = default_num_queues; + device_info->default_tx_ring_size = + be16_to_cpu(descriptor->tx_queue_entries); + device_info->default_rx_ring_size = + be16_to_cpu(descriptor->rx_queue_entries); + + /* set default ranges */ + device_info->max_tx_ring_size = device_info->default_tx_ring_size; + device_info->max_rx_ring_size = device_info->default_rx_ring_size; + device_info->min_tx_ring_size = device_info->default_tx_ring_size; + device_info->min_rx_ring_size = device_info->default_rx_ring_size; } int gve_adminq_describe_device(struct gve_priv *priv) @@ -1027,6 +1093,7 @@ int gve_adminq_describe_device(struct gve_priv *priv) struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL; struct gve_device_option_modify_ring *dev_op_modify_ring = NULL; struct gve_device_option_rss_config *dev_op_rss_config = NULL; + struct gve_device_info *device_info = &priv->device_info; struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; @@ -1070,26 +1137,26 @@ int gve_adminq_describe_device(struct gve_priv *priv) * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default. */ if (dev_op_dqo_rda) { - priv->queue_format = GVE_DQO_RDA_FORMAT; + device_info->queue_format = GVE_DQO_RDA_FORMAT; dev_info(&priv->pdev->dev, "Driver is running with DQO RDA queue format.\n"); supported_features_mask = be32_to_cpu(dev_op_dqo_rda->supported_features_mask); } else if (dev_op_dqo_qpl) { - priv->queue_format = GVE_DQO_QPL_FORMAT; + device_info->queue_format = GVE_DQO_QPL_FORMAT; supported_features_mask = be32_to_cpu(dev_op_dqo_qpl->supported_features_mask); } else if (dev_op_gqi_rda) { - priv->queue_format = GVE_GQI_RDA_FORMAT; + device_info->queue_format = GVE_GQI_RDA_FORMAT; dev_info(&priv->pdev->dev, "Driver is running with GQI RDA queue format.\n"); supported_features_mask = be32_to_cpu(dev_op_gqi_rda->supported_features_mask); - } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) { + } else if (device_info->queue_format == GVE_GQI_RDA_FORMAT) { dev_info(&priv->pdev->dev, "Driver is running with GQI RDA queue format.\n"); } else { - priv->queue_format = GVE_GQI_QPL_FORMAT; + device_info->queue_format = GVE_GQI_QPL_FORMAT; if (dev_op_gqi_qpl) supported_features_mask = be32_to_cpu(dev_op_gqi_qpl->supported_features_mask); @@ -1097,18 +1164,9 @@ int gve_adminq_describe_device(struct gve_priv *priv) "Driver is running with GQI QPL queue format.\n"); } + gve_fill_device_info(priv, descriptor); gve_set_default_rss_sizes(priv); - err = gve_set_mtu(priv, descriptor); - if (err) - goto free_device_descriptor; - - priv->num_event_counters = be16_to_cpu(descriptor->counters); - - gve_set_mac(priv, descriptor); - - gve_set_queue_properties(priv, descriptor); - gve_enable_supported_features(priv, supported_features_mask, dev_op_jumbo_frames, dev_op_dqo_qpl, dev_op_buffer_sizes, dev_op_flow_steering, @@ -1120,6 +1178,27 @@ free_device_descriptor: return err; } +int gve_adminq_get_device_properties(struct gve_priv *priv) +{ + int err; + + err = gve_adminq_verify_driver_compatibility(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Could not verify driver compatibility: err=%d\n", err); + return err; + } + + /* Get the initial information we need from the device */ + err = gve_adminq_describe_device(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Could not get device information: err=%d\n", err); + return err; + } + return 0; +} + int gve_adminq_register_page_list(struct gve_priv *priv, struct gve_queue_page_list *qpl) { @@ -1182,20 +1261,53 @@ int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, return gve_adminq_execute_cmd(priv, &cmd); } -int gve_adminq_verify_driver_compatibility(struct gve_priv *priv, - u64 driver_info_len, - dma_addr_t driver_info_addr) +int gve_adminq_verify_driver_compatibility(struct gve_priv *priv) { + struct gve_driver_info *driver_info; union gve_adminq_command cmd; + dma_addr_t driver_info_bus; + int err; + + driver_info = dma_alloc_coherent(&priv->pdev->dev, + sizeof(struct gve_driver_info), + &driver_info_bus, GFP_KERNEL); + if (!driver_info) + return -ENOMEM; + + *driver_info = (struct gve_driver_info) { + .os_type = 1, /* Linux */ + .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR), + .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL), + .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL), + .driver_capability_flags = { + cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1), + cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2), + cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3), + cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4), + }, + }; + strscpy(driver_info->os_version_str1, utsname()->release, + sizeof(driver_info->os_version_str1)); + strscpy(driver_info->os_version_str2, utsname()->version, + sizeof(driver_info->os_version_str2)); memset(&cmd, 0, sizeof(cmd)); cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY); cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) { - .driver_info_len = cpu_to_be64(driver_info_len), - .driver_info_addr = cpu_to_be64(driver_info_addr), + .driver_info_len = cpu_to_be64(sizeof(struct gve_driver_info)), + .driver_info_addr = cpu_to_be64(driver_info_bus), }; - return gve_adminq_execute_cmd(priv, &cmd); + err = gve_adminq_execute_cmd(priv, &cmd); + + /* It's ok if the device doesn't support this */ + if (err == -EOPNOTSUPP) + err = 0; + + dma_free_coherent(&priv->pdev->dev, + sizeof(struct gve_driver_info), + driver_info, driver_info_bus); + return err; } int gve_adminq_report_link_speed(struct gve_priv *priv) @@ -1241,8 +1353,7 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv, return gve_adminq_execute_cmd(priv, &cmd); } -int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, - struct gve_ptype_lut *ptype_lut) +int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv) { struct gve_ptype_map *ptype_map; union gve_adminq_command cmd; @@ -1268,9 +1379,9 @@ int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, /* Populate ptype_lut. */ for (i = 0; i < GVE_NUM_PTYPES; i++) { - ptype_lut->ptypes[i].l3_type = + priv->ptype_lut_dqo->ptypes[i].l3_type = ptype_map->ptypes[i].l3_type; - ptype_lut->ptypes[i].l4_type = + priv->ptype_lut_dqo->ptypes[i].l4_type = ptype_map->ptypes[i].l4_type; } err: @@ -1288,12 +1399,8 @@ gve_adminq_configure_flow_rule(struct gve_priv *priv, sizeof(struct gve_adminq_configure_flow_rule), flow_rule_cmd); - if (err == -ETIME) { - dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset"); - gve_reset(priv, true); - } else if (!err) { + if (!err) priv->flow_rules_cache.rules_cache_synced = false; - } return err; } @@ -1569,7 +1676,7 @@ out: return err; } -int gve_set_num_ntfy_blks(struct gve_priv *priv) +int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv) { int num_ntfy; @@ -1593,8 +1700,10 @@ int gve_set_num_ntfy_blks(struct gve_priv *priv) return 0; } -void gve_set_num_queues(struct gve_priv *priv) +void gve_adminq_set_num_queues(struct gve_priv *priv) { + struct gve_device_info *device_info = &priv->device_info; + priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2); priv->rx_cfg.max_queues = @@ -1602,10 +1711,139 @@ void gve_set_num_queues(struct gve_priv *priv) priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; - if (priv->default_num_queues > 0) { - priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues, + if (device_info->default_tx_queues > 0) + priv->tx_cfg.num_queues = min_t(int, + device_info->default_tx_queues, priv->tx_cfg.num_queues); - priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues, + + if (device_info->default_rx_queues > 0) + priv->rx_cfg.num_queues = min_t(int, + device_info->default_rx_queues, priv->rx_cfg.num_queues); +} + +int gve_adminq_map_db_bar(struct gve_priv *priv) +{ + struct pci_dev *pdev = priv->pdev; + void __iomem *db_bar; + + db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0); + if (!db_bar) { + dev_err(&pdev->dev, "Failed to map doorbell bar!\n"); + return -ENOMEM; } + priv->db_bar2 = db_bar; + return 0; +} + +void gve_adminq_unmap_db_bar(struct gve_priv *priv) +{ + struct pci_dev *pdev = priv->pdev; + + pci_iounmap(pdev, priv->db_bar2); +} + +int gve_adminq_request_db_info(struct gve_priv *priv) +{ + int err; + int i; + + err = gve_adminq_configure_device_resources(priv, + priv->counter_array_bus, + priv->num_event_counters, + priv->irq_db_indices_bus, + priv->num_ntfy_blks); + if (unlikely(err)) { + dev_err(&priv->pdev->dev, + "could not setup device_resources: err=%d\n", err); + return -ENXIO; + } + + for (i = 0; i < priv->num_ntfy_blks; i++) + priv->ntfy_blocks[i].irq_db_index = + &priv->irq_db_indices[i].index; + return 0; +} + +void gve_adminq_release_db_resources(struct gve_priv *priv) +{ + int err; + + /* Log error in deconfigure device, but don't fail. This is only ever + * called as a reset is about to be triggered, so it would be redundant + * to trigger a reset. + */ + err = gve_adminq_deconfigure_device_resources(priv); + if (err) + dev_err(&priv->pdev->dev, + "Could not deconfigure device resources: err=%d\n", + err); +} + +static irqreturn_t gve_mgmnt_intr(int irq, void *arg) +{ + struct gve_priv *priv = arg; + + queue_work(priv->gve_wq, &priv->service_task); + return IRQ_HANDLED; +} + +int gve_adminq_setup_mgmt_irq(struct gve_priv *priv) +{ + int err; + + snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), + "gve-mgmnt@pci:%s", pci_name(priv->pdev)); + err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, + gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv); + if (err) + return err; + + priv->mgmt_irq_requested = true; + + return 0; +} + +void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv) +{ + if (priv->mgmt_irq_requested) { + free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); + priv->mgmt_irq_requested = false; + } +} + +int gve_adminq_create_queues(struct gve_priv *priv) +{ + int num_tx_queues = gve_num_tx_queues(priv); + int err; + + err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues); + if (err) { + netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n", + num_tx_queues); + return err; + } + netif_dbg(priv, drv, priv->dev, "created %d tx queues\n", + num_tx_queues); + + err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); + if (err) { + netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n", + priv->rx_cfg.num_queues); + return err; + } + netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", + priv->rx_cfg.num_queues); + + return err; +} + +int gve_adminq_report_link_status(struct gve_priv *priv) +{ + u32 status; + + status = ioread32be(&priv->reg_bar0->device_status); + WRITE_ONCE(priv->link_up, + !!(GVE_DEVICE_STATUS_LINK_STATUS_MASK & status)); + return 0; } diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h index 68c63ce75505..8daac5fd685a 100644 --- a/drivers/net/ethernet/google/gve/gve_adminq.h +++ b/drivers/net/ethernet/google/gve/gve_adminq.h @@ -619,9 +619,8 @@ union gve_adminq_command { static_assert(sizeof(union gve_adminq_command) == 64); -int gve_adminq_alloc(struct device *dev, struct gve_priv *priv); +int gve_adminq_init(struct gve_priv *priv); void gve_adminq_free(struct gve_priv *priv); -void gve_adminq_release(struct gve_priv *priv); int gve_adminq_describe_device(struct gve_priv *priv); int gve_adminq_configure_device_resources(struct gve_priv *priv, dma_addr_t counter_array_bus_addr, @@ -629,6 +628,8 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv, dma_addr_t db_array_bus_addr, u32 num_ntfy_blks); int gve_adminq_deconfigure_device_resources(struct gve_priv *priv); +int gve_adminq_create_queues(struct gve_priv *priv); +int gve_adminq_destroy_queues(struct gve_priv *priv); int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues); int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues); int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index); @@ -640,9 +641,9 @@ int gve_adminq_register_page_list(struct gve_priv *priv, int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id); int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, dma_addr_t stats_report_addr, u64 interval); -int gve_adminq_verify_driver_compatibility(struct gve_priv *priv, - u64 driver_info_len, - dma_addr_t driver_info_addr); +int gve_adminq_verify_driver_compatibility(struct gve_priv *priv); +int gve_adminq_get_device_properties(struct gve_priv *priv); +int gve_adminq_report_link_status(struct gve_priv *priv); int gve_adminq_report_link_speed(struct gve_priv *priv); int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc); int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc); @@ -654,14 +655,13 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv, dma_addr_t nic_ts_report_addr); struct gve_ptype_lut; -int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, - struct gve_ptype_lut *ptype_lut); -int gve_set_num_ntfy_blks(struct gve_priv *priv); -void gve_set_num_queues(struct gve_priv *priv); -void gve_set_queue_properties(struct gve_priv *priv, - struct gve_device_descriptor *descriptor); -int gve_set_mtu(struct gve_priv *priv, - struct gve_device_descriptor *descriptor); -void gve_set_mac(struct gve_priv *priv, - struct gve_device_descriptor *descriptor); +int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv); +int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv); +void gve_adminq_set_num_queues(struct gve_priv *priv); +int gve_adminq_map_db_bar(struct gve_priv *priv); +void gve_adminq_unmap_db_bar(struct gve_priv *priv); +int gve_adminq_request_db_info(struct gve_priv *priv); +void gve_adminq_release_db_resources(struct gve_priv *priv); +int gve_adminq_setup_mgmt_irq(struct gve_priv *priv); +void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv); #endif /* _GVE_ADMINQ_H */ diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index 8199738ba979..a3220c651438 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -651,7 +651,7 @@ static int gve_user_reset(struct net_device *netdev, u32 *flags) if (*flags == ETH_RESET_ALL) { *flags = 0; - return gve_reset(priv, true); + return gve_reset(priv, false); } return -EOPNOTSUPP; @@ -749,12 +749,11 @@ static int gve_get_link_ksettings(struct net_device *netdev, int err = 0; if (priv->link_speed == 0) - err = gve_adminq_report_link_speed(priv); + err = priv->ctrl_ops->report_link_speed(priv); cmd->base.speed = priv->link_speed; cmd->base.duplex = DUPLEX_FULL; - return err; } diff --git a/drivers/net/ethernet/google/gve/gve_flow_rule.c b/drivers/net/ethernet/google/gve/gve_flow_rule.c index 2c80cda28ef3..fae552f4ad6f 100644 --- a/drivers/net/ethernet/google/gve/gve_flow_rule.c +++ b/drivers/net/ethernet/google/gve/gve_flow_rule.c @@ -278,6 +278,11 @@ int gve_add_flow_rule(struct gve_priv *priv, struct ethtool_rxnfc *cmd) goto out; err = gve_adminq_add_flow_rule(priv, rule, fsp->location); + if (err == -ETIME) { + dev_err(&priv->pdev->dev, + "Timeout to add flow rule, trigger reset."); + gve_reset(priv, false); + } out: kvfree(rule); @@ -290,9 +295,17 @@ out: int gve_del_flow_rule(struct gve_priv *priv, struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + int err; if (!priv->max_flow_rules) return -EOPNOTSUPP; - return gve_adminq_del_flow_rule(priv, fsp->location); + err = gve_adminq_del_flow_rule(priv, fsp->location); + if (err == -ETIME) { + dev_err(&priv->pdev->dev, + "Timeout to delete flow rule, trigger reset."); + gve_reset(priv, false); + } + + return err; } diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index a6c18e2431cc..2437cd8ba633 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -17,8 +17,6 @@ #include <linux/sched.h> #include <linux/timer.h> #include <linux/workqueue.h> -#include <linux/utsname.h> -#include <linux/version.h> #include <net/netdev_queues.h> #include <net/sch_generic.h> #include <net/xdp_sock_drv.h> @@ -41,49 +39,6 @@ char gve_driver_name[] = "gve"; const char gve_version_str[] = GVE_VERSION; static const char gve_version_prefix[] = GVE_VERSION_PREFIX; -static int gve_verify_driver_compatibility(struct gve_priv *priv) -{ - int err; - struct gve_driver_info *driver_info; - dma_addr_t driver_info_bus; - - driver_info = dma_alloc_coherent(&priv->pdev->dev, - sizeof(struct gve_driver_info), - &driver_info_bus, GFP_KERNEL); - if (!driver_info) - return -ENOMEM; - - *driver_info = (struct gve_driver_info) { - .os_type = 1, /* Linux */ - .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR), - .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL), - .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL), - .driver_capability_flags = { - cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1), - cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2), - cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3), - cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4), - }, - }; - strscpy(driver_info->os_version_str1, utsname()->release, - sizeof(driver_info->os_version_str1)); - strscpy(driver_info->os_version_str2, utsname()->version, - sizeof(driver_info->os_version_str2)); - - err = gve_adminq_verify_driver_compatibility(priv, - sizeof(struct gve_driver_info), - driver_info_bus); - - /* It's ok if the device doesn't support this */ - if (err == -EOPNOTSUPP) - err = 0; - - dma_free_coherent(&priv->pdev->dev, - sizeof(struct gve_driver_info), - driver_info, driver_info_bus); - return err; -} - static netdev_features_t gve_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) @@ -248,6 +203,30 @@ static void gve_free_counter_array(struct gve_priv *priv) priv->counter_array = NULL; } +static int gve_alloc_irq_db_indices(struct gve_priv *priv) +{ + priv->irq_db_indices = + dma_alloc_coherent(&priv->pdev->dev, + priv->num_ntfy_blks * + sizeof(*priv->irq_db_indices), + &priv->irq_db_indices_bus, GFP_KERNEL); + if (!priv->irq_db_indices) + return -ENOMEM; + + return 0; +} + +static void gve_free_irq_db_indices(struct gve_priv *priv) +{ + if (!priv->irq_db_indices) + return; + + dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * + sizeof(*priv->irq_db_indices), + priv->irq_db_indices, priv->irq_db_indices_bus); + priv->irq_db_indices = NULL; +} + /* NIC requests to report stats */ static void gve_stats_report_task(struct work_struct *work) { @@ -306,19 +285,12 @@ static void gve_free_stats_report(struct gve_priv *priv) return; timer_delete_sync(&priv->stats_report_timer); + cancel_work_sync(&priv->stats_report_task); dma_free_coherent(&priv->pdev->dev, priv->stats_report_len, priv->stats_report, priv->stats_report_bus); priv->stats_report = NULL; } -static irqreturn_t gve_mgmnt_intr(int irq, void *arg) -{ - struct gve_priv *priv = arg; - - queue_work(priv->gve_wq, &priv->service_task); - return IRQ_HANDLED; -} - static irqreturn_t gve_intr(int irq, void *arg) { struct gve_notify_block *block = arg; @@ -469,6 +441,15 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget) return work_done; } +static void gve_free_notify_blocks(struct gve_priv *priv) +{ + pci_disable_msix(priv->pdev); + kvfree(priv->ntfy_blocks); + priv->ntfy_blocks = NULL; + kvfree(priv->msix_vectors); + priv->msix_vectors = NULL; +} + static const struct cpumask *gve_get_node_mask(struct gve_priv *priv) { if (priv->numa_node == NUMA_NO_NODE) @@ -480,11 +461,9 @@ static const struct cpumask *gve_get_node_mask(struct gve_priv *priv) static int gve_alloc_notify_blocks(struct gve_priv *priv) { int num_vecs_requested = priv->num_ntfy_blks + 1; - const struct cpumask *node_mask; - unsigned int cur_cpu; int vecs_enabled; - int i, j; int err; + int i; priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors, num_vecs_requested); @@ -498,7 +477,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n", GVE_MIN_MSIX, vecs_enabled); err = vecs_enabled; - goto abort_with_msix_vectors; + goto abort; } if (vecs_enabled != num_vecs_requested) { int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1; @@ -521,38 +500,64 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; } - /* Setup Management Vector - the last vector */ - snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s", - pci_name(priv->pdev)); - err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, - gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv); - if (err) { - dev_err(&priv->pdev->dev, "Did not receive management vector.\n"); - goto abort_with_msix_enabled; - } - priv->irq_db_indices = - dma_alloc_coherent(&priv->pdev->dev, - priv->num_ntfy_blks * - sizeof(*priv->irq_db_indices), - &priv->irq_db_indices_bus, GFP_KERNEL); - if (!priv->irq_db_indices) { - err = -ENOMEM; - goto abort_with_mgmt_vector; - } - priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks), GFP_KERNEL); if (!priv->ntfy_blocks) { err = -ENOMEM; - goto abort_with_irq_db_indices; + goto abort; + } + + return 0; +abort: + gve_free_notify_blocks(priv); + return err; +} + +static void gve_teardown_notify_blocks(struct gve_priv *priv) +{ + const struct gve_ctrl_ops *ops = priv->ctrl_ops; + int i; + + ops->teardown_mgmt_irq(priv); + + if (!priv->ntfy_blocks) + return; + + for (i = 0; i < priv->num_ntfy_blks; i++) { + struct gve_notify_block *block = &priv->ntfy_blocks[i]; + int msix_idx = gve_ntfy_to_msix_idx(priv, i); + + if (!block->irq_requested) + continue; + + irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, + NULL); + free_irq(priv->msix_vectors[msix_idx].vector, block); + block->irq_requested = false; + } +} + +static int gve_setup_notify_blocks(struct gve_priv *priv) +{ + const struct gve_ctrl_ops *ops = priv->ctrl_ops; + const struct cpumask *node_mask; + unsigned int cur_cpu; + int i; + int err; + + /* Setup Management Vector */ + err = ops->setup_mgmt_irq(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Did not receive management vector.\n"); + return err; } - /* Setup the other blocks - the first n-1 vectors */ node_mask = gve_get_node_mask(priv); cur_cpu = cpumask_first(node_mask); for (i = 0; i < priv->num_ntfy_blks; i++) { struct gve_notify_block *block = &priv->ntfy_blocks[i]; - int msix_idx = i; + int msix_idx = gve_ntfy_to_msix_idx(priv, i); snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s", i, pci_name(priv->pdev)); @@ -562,13 +567,13 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) IRQF_NO_AUTOEN, block->name, block); if (err) { dev_err(&priv->pdev->dev, - "Failed to receive msix vector %d\n", i); - goto abort_with_some_ntfy_blocks; + "Failed to receive msix vector %d\n", msix_idx); + goto abort; } block->irq = priv->msix_vectors[msix_idx].vector; + block->irq_requested = true; irq_set_affinity_and_hint(block->irq, cpumask_of(cur_cpu)); - block->irq_db_index = &priv->irq_db_indices[i].index; cur_cpu = cpumask_next(cur_cpu, node_mask); /* Wrap once CPUs in the node have been exhausted, or when @@ -579,63 +584,87 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) cur_cpu = cpumask_first(node_mask); } return 0; -abort_with_some_ntfy_blocks: - for (j = 0; j < i; j++) { - struct gve_notify_block *block = &priv->ntfy_blocks[j]; - int msix_idx = j; - irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, - NULL); - free_irq(priv->msix_vectors[msix_idx].vector, block); - block->irq = 0; - } - kvfree(priv->ntfy_blocks); - priv->ntfy_blocks = NULL; -abort_with_irq_db_indices: - dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * - sizeof(*priv->irq_db_indices), - priv->irq_db_indices, priv->irq_db_indices_bus); - priv->irq_db_indices = NULL; -abort_with_mgmt_vector: - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); -abort_with_msix_enabled: - pci_disable_msix(priv->pdev); -abort_with_msix_vectors: - kvfree(priv->msix_vectors); - priv->msix_vectors = NULL; +abort: + gve_teardown_notify_blocks(priv); return err; } -static void gve_free_notify_blocks(struct gve_priv *priv) +static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv, + struct gve_tx_alloc_rings_cfg *cfg) { - int i; + cfg->qcfg = &priv->tx_cfg; + cfg->raw_addressing = !gve_is_qpl(priv); + cfg->ring_size = priv->tx_desc_cnt; + cfg->pages_per_qpl = priv->tx_pages_per_qpl; + cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues; + cfg->tx = priv->tx; +} - if (!priv->msix_vectors) - return; +static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, + struct gve_rx_alloc_rings_cfg *cfg) +{ + cfg->qcfg_rx = &priv->rx_cfg; + cfg->qcfg_tx = &priv->tx_cfg; + cfg->raw_addressing = !gve_is_qpl(priv); + cfg->enable_header_split = priv->header_split_enabled; + cfg->ring_size = priv->rx_desc_cnt; + cfg->pages_per_qpl = priv->rx_pages_per_qpl; + cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size; + cfg->rx = priv->rx; + cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues; +} - /* Free the irqs */ - for (i = 0; i < priv->num_ntfy_blks; i++) { - struct gve_notify_block *block = &priv->ntfy_blocks[i]; - int msix_idx = i; +void gve_get_curr_alloc_cfgs(struct gve_priv *priv, + struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, + struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) +{ + gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg); + gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg); +} - irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, - NULL); - free_irq(priv->msix_vectors[msix_idx].vector, block); - block->irq = 0; +static void gve_queues_mem_free(struct gve_priv *priv, + struct gve_tx_alloc_rings_cfg *tx_cfg, + struct gve_rx_alloc_rings_cfg *rx_cfg) +{ + if (gve_is_gqi(priv)) { + gve_tx_free_rings_gqi(priv, tx_cfg); + gve_rx_free_rings_gqi(priv, rx_cfg); + } else { + gve_tx_free_rings_dqo(priv, tx_cfg); + gve_rx_free_rings_dqo(priv, rx_cfg); } - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); - kvfree(priv->ntfy_blocks); - priv->ntfy_blocks = NULL; - dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * - sizeof(*priv->irq_db_indices), - priv->irq_db_indices, priv->irq_db_indices_bus); - priv->irq_db_indices = NULL; - pci_disable_msix(priv->pdev); - kvfree(priv->msix_vectors); - priv->msix_vectors = NULL; } -static int gve_setup_device_resources(struct gve_priv *priv) +static void gve_queues_mem_remove(struct gve_priv *priv) +{ + struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; + struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; + + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); + gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg); + priv->tx = NULL; + priv->rx = NULL; +} + +static void gve_free_control_plane_resources(struct gve_priv *priv) +{ + bitmap_free(priv->xsk_pools); + priv->xsk_pools = NULL; + + kvfree(priv->ptype_lut_dqo); + priv->ptype_lut_dqo = NULL; + + gve_teardown_clock(priv); + gve_free_irq_db_indices(priv); + gve_free_counter_array(priv); + gve_free_notify_blocks(priv); + gve_free_stats_report(priv); + gve_free_rss_config_cache(priv); + gve_free_flow_rule_caches(priv); +} + +static int gve_alloc_control_plane_resources(struct gve_priv *priv) { int err; @@ -644,39 +673,58 @@ static int gve_setup_device_resources(struct gve_priv *priv) return err; err = gve_alloc_rss_config_cache(priv); if (err) - goto abort_with_flow_rule_caches; - err = gve_alloc_counter_array(priv); - if (err) - goto abort_with_rss_config_cache; + goto abort; err = gve_alloc_notify_blocks(priv); if (err) - goto abort_with_counter; + goto abort; err = gve_alloc_stats_report(priv); if (err) - goto abort_with_ntfy_blocks; - err = gve_adminq_configure_device_resources(priv, - priv->counter_array_bus, - priv->num_event_counters, - priv->irq_db_indices_bus, - priv->num_ntfy_blks); - if (unlikely(err)) { - dev_err(&priv->pdev->dev, - "could not setup device_resources: err=%d\n", err); - err = -ENXIO; - goto abort_with_stats_report; - } + goto abort; + err = gve_alloc_counter_array(priv); + if (err) + goto abort; + err = gve_alloc_irq_db_indices(priv); + if (err) + goto abort; if (!gve_is_gqi(priv)) { - priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo); + priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo, + GFP_KERNEL); if (!priv->ptype_lut_dqo) { err = -ENOMEM; - goto abort_with_stats_report; + goto abort; } - err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); + } + + priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL); + if (!priv->xsk_pools) { + err = -ENOMEM; + goto abort; + } + + return 0; +abort: + gve_free_control_plane_resources(priv); + return err; +} + +static int gve_setup_control_plane_resources(struct gve_priv *priv) +{ + const struct gve_ctrl_ops *ops = priv->ctrl_ops; + int err; + + err = ops->request_db_info(priv); + if (err) { + dev_err(&priv->pdev->dev, "Failed to get db info"); + return err; + } + + if (!gve_is_gqi(priv)) { + err = ops->get_ptype_map(priv); if (err) { dev_err(&priv->pdev->dev, "Failed to get ptype map: err=%d\n", err); - goto abort_with_ptype_lut; + goto release_db_resources; } } @@ -691,79 +739,89 @@ static int gve_setup_device_resources(struct gve_priv *priv) err = gve_init_rss_config(priv, priv->rx_cfg.num_queues); if (err) { dev_err(&priv->pdev->dev, "Failed to init RSS config"); - goto abort_with_clock; + goto release_db_resources; } - err = gve_adminq_report_stats(priv, priv->stats_report_len, + err = ops->setup_stats_report(priv, priv->stats_report_len, priv->stats_report_bus, GVE_STATS_REPORT_TIMER_PERIOD); if (err) dev_err(&priv->pdev->dev, "Failed to report stats: err=%d\n", err); + gve_set_device_resources_ok(priv); return 0; -abort_with_clock: - gve_teardown_clock(priv); -abort_with_ptype_lut: - kvfree(priv->ptype_lut_dqo); - priv->ptype_lut_dqo = NULL; -abort_with_stats_report: - gve_free_stats_report(priv); -abort_with_ntfy_blocks: - gve_free_notify_blocks(priv); -abort_with_counter: - gve_free_counter_array(priv); -abort_with_rss_config_cache: - gve_free_rss_config_cache(priv); -abort_with_flow_rule_caches: - gve_free_flow_rule_caches(priv); - +release_db_resources: + ops->release_db_resources(priv); return err; } -static void gve_trigger_reset(struct gve_priv *priv); - -static void gve_teardown_device_resources(struct gve_priv *priv) +/** + * gve_teardown_control_plane_resources() - Request the device to release any + * shared allocated resources. + * + * @priv: Pointer to the GVE private device data structure. + * + * If any part of the teardown step fails, the failure is documented, but is + * otherwise ignored. It is expected that a device reset is triggered + * immediately after tearing down device resources, which would clear any + * lingering state on the device. + */ +static void gve_teardown_control_plane_resources(struct gve_priv *priv) { + const struct gve_ctrl_ops *ops = priv->ctrl_ops; int err; + if (priv->ptp) + ptp_cancel_worker_sync(priv->ptp->clock); + /* Tell device its resources are being freed */ if (gve_get_device_resources_ok(priv)) { err = gve_flow_rules_reset(priv); - if (err) { + if (err) dev_err(&priv->pdev->dev, "Failed to reset flow rules: err=%d\n", err); - gve_trigger_reset(priv); - } /* detach the stats report */ - err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD); - if (err) { + err = ops->setup_stats_report(priv, 0, 0x0, + GVE_STATS_REPORT_TIMER_PERIOD); + if (err) dev_err(&priv->pdev->dev, "Failed to detach stats report: err=%d\n", err); - gve_trigger_reset(priv); - } - err = gve_adminq_deconfigure_device_resources(priv); - if (err) { - dev_err(&priv->pdev->dev, - "Could not deconfigure device resources: err=%d\n", - err); - gve_trigger_reset(priv); - } + ops->release_db_resources(priv); } - kvfree(priv->ptype_lut_dqo); - priv->ptype_lut_dqo = NULL; - - gve_free_flow_rule_caches(priv); - gve_free_rss_config_cache(priv); - gve_free_counter_array(priv); - gve_free_notify_blocks(priv); - gve_free_stats_report(priv); - gve_teardown_clock(priv); gve_clear_device_resources_ok(priv); } +/** + * gve_reset_device() - Reset the device + * + * @priv: Pointer to the GVE private device data structure. + * + * Once this returns, the device is guaranteed not to access any memory it + * shares with the driver, so it is safe for the caller to recycle it. Device + * commands in gve_teardown_control_plane_resources() can fail, in which case + * the hardware reset triggered by gve_adminq_free() is the only such + * guarantee. + */ +static void gve_reset_device(struct gve_priv *priv) +{ + gve_teardown_control_plane_resources(priv); + gve_adminq_free(priv); +} + +static void gve_teardown_device(struct gve_priv *priv) +{ + gve_reset_device(priv); + /* Free any resources shared with the device only after we have a + * guarantee that the device will not try to access such resources. + */ + gve_teardown_notify_blocks(priv); + gve_free_control_plane_resources(priv); + gve_queues_mem_remove(priv); +} + static int gve_unregister_qpl(struct gve_priv *priv, struct gve_queue_page_list *qpl) { @@ -886,33 +944,16 @@ static int gve_unregister_qpls(struct gve_priv *priv) static int gve_create_rings(struct gve_priv *priv) { - int num_tx_queues = gve_num_tx_queues(priv); int err; int i; - err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues); - if (err) { - netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n", - num_tx_queues); - /* This failure will trigger a reset - no need to clean - * up - */ - return err; - } - netif_dbg(priv, drv, priv->dev, "created %d tx queues\n", - num_tx_queues); - - err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); + err = priv->ctrl_ops->create_queues(priv); if (err) { - netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n", - priv->rx_cfg.num_queues); /* This failure will trigger a reset - no need to clean * up */ return err; } - netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", - priv->rx_cfg.num_queues); if (gve_is_gqi(priv)) { /* Rx data ring has been prefilled with packet buffers at queue @@ -961,17 +1002,6 @@ static void gve_init_sync_stats(struct gve_priv *priv) u64_stats_init(&priv->rx[i].statss); } -static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv, - struct gve_tx_alloc_rings_cfg *cfg) -{ - cfg->qcfg = &priv->tx_cfg; - cfg->raw_addressing = !gve_is_qpl(priv); - cfg->ring_size = priv->tx_desc_cnt; - cfg->pages_per_qpl = priv->tx_pages_per_qpl; - cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues; - cfg->tx = priv->tx; -} - static void gve_tx_stop_rings(struct gve_priv *priv, int num_rings) { int i; @@ -1067,39 +1097,12 @@ free_tx: static int gve_destroy_rings(struct gve_priv *priv) { - int num_tx_queues = gve_num_tx_queues(priv); int err; - err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues); - if (err) { - netif_err(priv, drv, priv->dev, - "failed to destroy tx queues\n"); - /* This failure will trigger a reset - no need to clean up */ - return err; - } - netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n"); - err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues); - if (err) { - netif_err(priv, drv, priv->dev, - "failed to destroy rx queues\n"); - /* This failure will trigger a reset - no need to clean up */ - return err; - } - netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n"); - return 0; -} + err = priv->ctrl_ops->destroy_queues(priv); -static void gve_queues_mem_free(struct gve_priv *priv, - struct gve_tx_alloc_rings_cfg *tx_cfg, - struct gve_rx_alloc_rings_cfg *rx_cfg) -{ - if (gve_is_gqi(priv)) { - gve_tx_free_rings_gqi(priv, tx_cfg); - gve_rx_free_rings_gqi(priv, rx_cfg); - } else { - gve_tx_free_rings_dqo(priv, tx_cfg); - gve_rx_free_rings_dqo(priv, rx_cfg); - } + /* A failure will trigger a reset - no need to clean up */ + return err; } int gve_alloc_page(struct gve_priv *priv, struct device *dev, @@ -1202,8 +1205,6 @@ void gve_schedule_reset(struct gve_priv *priv) queue_work(priv->gve_wq, &priv->service_task); } -static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up); -static int gve_reset_recovery(struct gve_priv *priv, bool was_up); static void gve_turndown(struct gve_priv *priv); static void gve_turnup(struct gve_priv *priv); @@ -1314,37 +1315,16 @@ err: return err; } - static void gve_drain_page_cache(struct gve_priv *priv) { int i; + if (!priv->rx) + return; for (i = 0; i < priv->rx_cfg.num_queues; i++) page_frag_cache_drain(&priv->rx[i].page_cache); } -static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, - struct gve_rx_alloc_rings_cfg *cfg) -{ - cfg->qcfg_rx = &priv->rx_cfg; - cfg->qcfg_tx = &priv->tx_cfg; - cfg->raw_addressing = !gve_is_qpl(priv); - cfg->enable_header_split = priv->header_split_enabled; - cfg->ring_size = priv->rx_desc_cnt; - cfg->pages_per_qpl = priv->rx_pages_per_qpl; - cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size; - cfg->rx = priv->rx; - cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues; -} - -void gve_get_curr_alloc_cfgs(struct gve_priv *priv, - struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, - struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) -{ - gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg); - gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg); -} - static void gve_rx_start_ring(struct gve_priv *priv, int i) { if (gve_is_gqi(priv)) @@ -1380,15 +1360,16 @@ static void gve_rx_stop_rings(struct gve_priv *priv, int num_rings) gve_rx_stop_ring(priv, i); } -static void gve_queues_mem_remove(struct gve_priv *priv) +static void gve_queues_stop(struct gve_priv *priv) { - struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; - struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; + gve_unreg_xdp_info(priv); + gve_drain_page_cache(priv); - gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); - gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg); - priv->tx = NULL; - priv->rx = NULL; + timer_delete_sync(&priv->stats_report_timer); + cancel_work_sync(&priv->stats_report_task); + + gve_tx_stop_rings(priv, gve_num_tx_queues(priv)); + gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); } /* The passed-in queue memory is stored into priv and the queues are made live. @@ -1413,6 +1394,8 @@ static int gve_queues_start(struct gve_priv *priv, priv->rx_desc_cnt = rx_alloc_cfg->ring_size; priv->tx_pages_per_qpl = tx_alloc_cfg->pages_per_qpl; priv->rx_pages_per_qpl = rx_alloc_cfg->pages_per_qpl; + priv->header_split_enabled = rx_alloc_cfg->enable_header_split; + priv->rx_cfg.packet_buffer_size = rx_alloc_cfg->packet_buffer_size; gve_tx_start_rings(priv, gve_num_tx_queues(priv)); gve_rx_start_rings(priv, rx_alloc_cfg->qcfg_rx->num_queues); @@ -1420,14 +1403,14 @@ static int gve_queues_start(struct gve_priv *priv, err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues); if (err) - goto stop_and_free_rings; + goto stop_rings; err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues); if (err) - goto stop_and_free_rings; + goto stop_rings; err = gve_reg_xdp_info(priv, dev); if (err) - goto stop_and_free_rings; + goto stop_rings; if (rx_alloc_cfg->reset_rss) { err = gve_init_rss_config(priv, priv->rx_cfg.num_queues); @@ -1439,9 +1422,6 @@ static int gve_queues_start(struct gve_priv *priv, if (err) goto reset; - priv->header_split_enabled = rx_alloc_cfg->enable_header_split; - priv->rx_cfg.packet_buffer_size = rx_alloc_cfg->packet_buffer_size; - err = gve_create_rings(priv); if (err) goto reset; @@ -1460,16 +1440,14 @@ static int gve_queues_start(struct gve_priv *priv, reset: if (gve_get_reset_in_progress(priv)) - goto stop_and_free_rings; - gve_reset_and_teardown(priv, true); - /* if this fails there is nothing we can do so just ignore the return */ - gve_reset_recovery(priv, false); - /* return the original error */ - return err; -stop_and_free_rings: - gve_tx_stop_rings(priv, gve_num_tx_queues(priv)); - gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); - gve_queues_mem_remove(priv); + goto stop_rings; + + /* Attempt to reset. If reset is successful, gve_queues_start was + * successful with the new config. + */ + return gve_reset(priv, false); +stop_rings: + gve_queues_stop(priv); return err; } @@ -1480,81 +1458,69 @@ static int gve_open(struct net_device *dev) struct gve_priv *priv = netdev_priv(dev); int err; + if (!gve_get_device_resources_ok(priv)) { + dev_err(&priv->pdev->dev, + "Attempting to open netdev without resources. Device must be reset."); + return -ENODEV; + } + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg); if (err) return err; - /* No need to free on error: ownership of resources is lost after - * calling gve_queues_start. - */ err = gve_queues_start(priv, &tx_alloc_cfg, &rx_alloc_cfg); - if (err) + if (err) { + gve_queues_mem_remove(priv); return err; + } return 0; } -static int gve_queues_stop(struct gve_priv *priv) +static int gve_close(struct net_device *dev) { + struct gve_priv *priv = netdev_priv(dev); int err; - netif_carrier_off(priv->dev); + gve_turndown(priv); + + /* Surrender to reset if the queue destroying adminq cmds fail. Reset + * will not re-enable the interface. + */ if (gve_get_device_rings_ok(priv)) { - gve_turndown(priv); - gve_drain_page_cache(priv); + gve_clear_device_rings_ok(priv); err = gve_destroy_rings(priv); if (err) - goto err; + goto reset; err = gve_unregister_qpls(priv); if (err) - goto err; - gve_clear_device_rings_ok(priv); + goto reset; } - timer_delete_sync(&priv->stats_report_timer); - - gve_unreg_xdp_info(priv); - - gve_tx_stop_rings(priv, gve_num_tx_queues(priv)); - gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); + gve_queues_stop(priv); + gve_queues_mem_remove(priv); priv->interface_down_cnt++; return 0; -err: - /* This must have been called from a reset due to the rtnl lock - * so just return at this point. - */ - if (gve_get_reset_in_progress(priv)) - return err; - /* Otherwise reset before returning */ - gve_reset_and_teardown(priv, true); - return gve_reset_recovery(priv, false); +reset: + err = gve_reset(priv, true); + return err; } -static int gve_close(struct net_device *dev) +static void gve_handle_link_status(struct gve_priv *priv) { - struct gve_priv *priv = netdev_priv(dev); - int err; + bool link_up; - err = gve_queues_stop(priv); - if (err) - return err; - - gve_queues_mem_remove(priv); - return 0; -} - -static void gve_handle_link_status(struct gve_priv *priv, bool link_status) -{ if (!gve_get_napi_enabled(priv)) return; - if (link_status == netif_carrier_ok(priv->dev)) + link_up = READ_ONCE(priv->link_up); + if (link_up == netif_carrier_ok(priv->dev)) return; - if (link_status) { + if (link_up) { netdev_info(priv->dev, "Device link is up.\n"); netif_carrier_on(priv->dev); } else { @@ -1581,7 +1547,6 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog, { struct bpf_prog *old_prog; int err = 0; - u32 status; old_prog = READ_ONCE(priv->xdp_prog); if (!netif_running(priv->dev)) { @@ -1608,8 +1573,8 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog, bpf_prog_put(old_prog); out: - status = ioread32be(&priv->reg_bar0->device_status); - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); + priv->ctrl_ops->report_link_status(priv); + gve_handle_link_status(priv); return err; } @@ -1824,6 +1789,7 @@ static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp) int gve_init_rss_config(struct gve_priv *priv, u16 num_queues) { + const struct gve_ctrl_ops *ops = priv->ctrl_ops; struct gve_rss_config *rss_config = &priv->rss_config; struct ethtool_rxfh_param rxfh = {0}; u16 i; @@ -1839,15 +1805,17 @@ int gve_init_rss_config(struct gve_priv *priv, u16 num_queues) rxfh.hfunc = ETH_RSS_HASH_TOP; - return gve_adminq_configure_rss(priv, &rxfh); + return ops->configure_rss(priv, &rxfh); } int gve_flow_rules_reset(struct gve_priv *priv) { + const struct gve_ctrl_ops *ops = priv->ctrl_ops; + if (!priv->max_flow_rules) return 0; - return gve_adminq_reset_flow_rules(priv); + return ops->reset_flow_rules(priv); } int gve_adjust_config(struct gve_priv *priv, @@ -1878,9 +1846,7 @@ int gve_adjust_config(struct gve_priv *priv, if (err) { netif_err(priv, drv, priv->dev, "Adjust config failed to start new queues, !!! DISABLING ALL QUEUES !!!\n"); - /* No need to free on error: ownership of resources is lost after - * calling gve_queues_start. - */ + gve_queues_mem_remove(priv); gve_turndown(priv); return err; } @@ -2039,11 +2005,11 @@ static void gve_turnup(struct gve_priv *priv) static void gve_turnup_and_check_status(struct gve_priv *priv) { - u32 status; + const struct gve_ctrl_ops *ops = priv->ctrl_ops; gve_turnup(priv); - status = ioread32be(&priv->reg_bar0->device_status); - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); + ops->report_link_status(priv); + gve_handle_link_status(priv); } static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv, @@ -2194,8 +2160,11 @@ static int gve_set_features(struct net_device *netdev, } if ((netdev->features & NETIF_F_NTUPLE) && !(features & NETIF_F_NTUPLE)) { err = gve_flow_rules_reset(priv); - if (err) + if (err) { + if (err == -ETIME) + gve_schedule_reset(priv); goto revert_features; + } } return 0; @@ -2280,7 +2249,8 @@ static void gve_handle_reset(struct gve_priv *priv) if (gve_get_do_reset(priv)) { rtnl_lock(); netdev_lock(priv->dev); - gve_reset(priv, false); + if (gve_get_do_reset(priv)) + gve_reset(priv, false); netdev_unlock(priv->dev); rtnl_unlock(); } @@ -2367,12 +2337,14 @@ static void gve_service_task(struct work_struct *work) { struct gve_priv *priv = container_of(work, struct gve_priv, service_task); + const struct gve_ctrl_ops *ops = priv->ctrl_ops; u32 status = ioread32be(&priv->reg_bar0->device_status); gve_handle_status(priv, status); gve_handle_reset(priv); - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); + ops->report_link_status(priv); + gve_handle_link_status(priv); } static void gve_set_netdev_xdp_features(struct gve_priv *priv) @@ -2398,96 +2370,126 @@ static const struct xdp_metadata_ops gve_xdp_metadata_ops = { .xmo_rx_timestamp = gve_xdp_rx_timestamp, }; -static void gve_set_default_desc_cnt(struct gve_priv *priv, - const struct gve_device_descriptor *descriptor) +static void gve_set_desc_cnt(struct gve_priv *priv) { - priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); - priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); + struct gve_device_info *device_info = &priv->device_info; - /* set default ranges */ - priv->max_tx_desc_cnt = priv->tx_desc_cnt; - priv->max_rx_desc_cnt = priv->rx_desc_cnt; - priv->min_tx_desc_cnt = priv->tx_desc_cnt; - priv->min_rx_desc_cnt = priv->rx_desc_cnt; + priv->tx_desc_cnt = device_info->default_tx_ring_size; + priv->rx_desc_cnt = device_info->default_rx_ring_size; + priv->max_tx_desc_cnt = device_info->max_tx_ring_size; + priv->max_rx_desc_cnt = device_info->max_rx_ring_size; + priv->min_tx_desc_cnt = device_info->min_tx_ring_size; + priv->min_rx_desc_cnt = device_info->min_rx_ring_size; } -void gve_set_queue_properties(struct gve_priv *priv, - struct gve_device_descriptor *descriptor) +static void gve_set_queue_properties(struct gve_priv *priv) { - /* set default descriptor counts */ - gve_set_default_desc_cnt(priv, descriptor); + struct gve_device_info *device_info = &priv->device_info; - priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages); - priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); - priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); + gve_set_desc_cnt(priv); + priv->max_registered_pages = device_info->max_registered_pages; + priv->tx_pages_per_qpl = device_info->tx_pages_per_qpl; } -int gve_set_mtu(struct gve_priv *priv, - struct gve_device_descriptor *descriptor) +static int gve_set_mtu(struct gve_priv *priv) { + struct gve_device_info *device_info = &priv->device_info; u16 mtu; - mtu = be16_to_cpu(descriptor->mtu); + mtu = device_info->max_mtu; if (mtu < ETH_MIN_MTU) { dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); return -EINVAL; } priv->dev->max_mtu = mtu; + priv->dev->mtu = mtu; return 0; } -void gve_set_mac(struct gve_priv *priv, - struct gve_device_descriptor *descriptor) +static void gve_set_mac(struct gve_priv *priv) { + struct gve_device_info *device_info = &priv->device_info; u8 *mac; - mac = descriptor->mac; + mac = device_info->mac; eth_hw_addr_set(priv->dev, mac); dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); } -static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) +static void gve_set_buf_sizes(struct gve_priv *priv) +{ + struct gve_device_info *device_info = &priv->device_info; + + if (device_info->max_rx_buffer_size) + priv->max_rx_buffer_size = device_info->max_rx_buffer_size; + + if (gve_is_dqo(priv) && + priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE) + priv->rx_cfg.packet_buffer_size = priv->max_rx_buffer_size; + + if (device_info->header_buf_size) + priv->header_buf_size = device_info->header_buf_size; +} + +static int gve_setup_device(struct gve_priv *priv) { int err; - /* Set up the adminq */ - err = gve_adminq_alloc(&priv->pdev->dev, priv); - if (err) { - dev_err(&priv->pdev->dev, - "Failed to alloc admin queue: err=%d\n", err); + priv->num_registered_pages = 0; + + gve_set_netdev_xdp_features(priv); + if (!gve_is_gqi(priv)) + priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops; + + err = gve_alloc_control_plane_resources(priv); + if (err) return err; - } - err = gve_verify_driver_compatibility(priv); - if (err) { - dev_err(&priv->pdev->dev, - "Could not verify driver compatibility: err=%d\n", err); - goto err; - } + err = gve_setup_control_plane_resources(priv); + if (err) + return err; - priv->num_registered_pages = 0; + err = gve_setup_notify_blocks(priv); + if (err) + return err; + return 0; +} - if (skip_describe_device) - goto setup_device; +static const struct gve_ctrl_ops gve_adminq_ops = { + .map_db_bar = gve_adminq_map_db_bar, + .unmap_db_bar = gve_adminq_unmap_db_bar, + .set_num_queues = gve_adminq_set_num_queues, + .set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks, + .get_ptype_map = gve_adminq_get_ptype_map_dqo, + .reset_flow_rules = gve_adminq_reset_flow_rules, + .setup_stats_report = gve_adminq_report_stats, + .configure_rss = gve_adminq_configure_rss, + .request_db_info = gve_adminq_request_db_info, + .release_db_resources = gve_adminq_release_db_resources, + .setup_mgmt_irq = gve_adminq_setup_mgmt_irq, + .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq, + .create_queues = gve_adminq_create_queues, + .destroy_queues = gve_adminq_destroy_queues, + .report_link_status = gve_adminq_report_link_status, + .report_link_speed = gve_adminq_report_link_speed, +}; - priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; - /* Get the initial information we need from the device */ - err = gve_adminq_describe_device(priv); - if (err) { - dev_err(&priv->pdev->dev, - "Could not get device information: err=%d\n", err); - goto err; - } +static int gve_init_priv(struct gve_priv *priv) +{ + struct gve_device_info *device_info = &priv->device_info; + int err; - err = gve_set_num_ntfy_blks(priv); + priv->queue_format = priv->device_info.queue_format; + + err = priv->ctrl_ops->set_num_ntfy_blks(priv); if (err) { dev_err(&priv->pdev->dev, "Could not setup notify blocks: err=%d\n", err); - goto err; + return err; } - gve_set_num_queues(priv); + priv->ctrl_ops->set_num_queues(priv); dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n", priv->tx_cfg.num_queues, priv->rx_cfg.num_queues); dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n", @@ -2507,108 +2509,117 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX); } - priv->dev->mtu = priv->dev->max_mtu; - priv->numa_node = dev_to_node(&priv->pdev->dev); - priv->tx_cfg.num_xdp_queues = 0; - priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; - priv->ts_config.tx_type = HWTSTAMP_TX_OFF; - priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE; - -setup_device: - priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL); - if (!priv->xsk_pools) { - err = -ENOMEM; - goto err; - } + if (gve_set_mtu(priv)) + return -EINVAL; - gve_set_netdev_xdp_features(priv); - if (!gve_is_gqi(priv)) - priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops; + priv->num_event_counters = device_info->num_event_counters; - err = gve_setup_device_resources(priv); - if (err) - goto err_free_xsk_bitmap; + gve_set_mac(priv); - return 0; + gve_set_queue_properties(priv); + priv->modify_ring_size_enabled = device_info->modify_ring_size_enabled; -err_free_xsk_bitmap: - bitmap_free(priv->xsk_pools); - priv->xsk_pools = NULL; -err: - gve_adminq_free(priv); - return err; -} + gve_set_buf_sizes(priv); -static void gve_teardown_priv_resources(struct gve_priv *priv) -{ - gve_teardown_device_resources(priv); - gve_adminq_free(priv); - bitmap_free(priv->xsk_pools); - priv->xsk_pools = NULL; -} + priv->max_flow_rules = device_info->max_flow_rules; + if (priv->max_flow_rules) + priv->dev->hw_features |= NETIF_F_NTUPLE; -static void gve_trigger_reset(struct gve_priv *priv) -{ - /* Reset the device by releasing the AQ */ - gve_adminq_release(priv); -} + priv->rss_key_size = device_info->rss_key_size; + priv->rss_lut_size = device_info->rss_lut_size; + priv->cache_rss_config = device_info->cache_rss_config; -static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up) -{ - gve_trigger_reset(priv); - /* With the reset having already happened, close cannot fail */ - if (was_up) - gve_close(priv->dev); - gve_teardown_priv_resources(priv); + priv->numa_node = dev_to_node(&priv->pdev->dev); + priv->tx_cfg.num_xdp_queues = 0; + priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; + priv->ts_config.tx_type = HWTSTAMP_TX_OFF; + priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE; + priv->nic_timestamp_supported = device_info->nic_timestamp_supported; + return 0; } -static int gve_reset_recovery(struct gve_priv *priv, bool was_up) +static int gve_recover(struct gve_priv *priv, bool setup_queues) { int err; - err = gve_init_priv(priv, true); + err = gve_adminq_init(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Failed to alloc admin queue: err=%d\n", err); + goto teardown_device; + } + + err = gve_adminq_verify_driver_compatibility(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Could not verify driver compatibility: err=%d\n", err); + goto teardown_device; + } + + err = gve_setup_device(priv); if (err) - goto err; - if (was_up) { + goto teardown_device; + + if (setup_queues) { + /* On failure, hold on to the control plane to give a + * chance for the queues to be brought up later. + */ err = gve_open(priv->dev); - if (err) - goto err; + if (err) { + dev_err(&priv->pdev->dev, + "Failed to start queues: err=%d, !!! DISABLING ALL QUEUES !!!\n", + err); + return err; + } } + + /* undo any detach from an earlier failure */ + netif_device_attach(priv->dev); + return 0; -err: - dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n"); - gve_turndown(priv); + +teardown_device: + dev_err(&priv->pdev->dev, "Recover failed: err=%d, detaching device\n", + err); + netif_device_detach(priv->dev); + gve_teardown_device(priv); return err; } -int gve_reset(struct gve_priv *priv, bool attempt_teardown) +int gve_reset(struct gve_priv *priv, bool skip_queue_setup) { bool was_up = netif_running(priv->dev); int err; + if (gve_get_reset_in_progress(priv)) + return 0; + dev_info(&priv->pdev->dev, "Performing reset\n"); gve_clear_do_reset(priv); gve_set_reset_in_progress(priv); - /* If we aren't attempting to teardown normally, just go turndown and - * reset right away. - */ - if (!attempt_teardown) { + + if (was_up) { gve_turndown(priv); - gve_reset_and_teardown(priv, was_up); - } else { - /* Otherwise attempt to close normally */ - if (was_up) { - err = gve_close(priv->dev); - /* If that fails reset as we did above */ - if (err) - gve_reset_and_teardown(priv, was_up); + if (gve_get_device_rings_ok(priv)) { + gve_clear_device_rings_ok(priv); + gve_destroy_rings(priv); + gve_unregister_qpls(priv); } - /* Clean up any remaining resources */ - gve_teardown_priv_resources(priv); } - /* Set it all back up */ - err = gve_reset_recovery(priv, was_up); + disable_work(&priv->service_task); + gve_reset_device(priv); + gve_queues_stop(priv); + gve_queues_mem_remove(priv); + gve_teardown_notify_blocks(priv); + gve_free_control_plane_resources(priv); + + enable_work(&priv->service_task); + err = gve_recover(priv, was_up && !skip_queue_setup); + if (err) + dev_info(&priv->pdev->dev, + "Failed to recover in reset: %d\n", err); + gve_clear_reset_in_progress(priv); priv->reset_cnt++; priv->interface_up_cnt = 0; @@ -2617,7 +2628,7 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown) return err; } -static void gve_write_version(u8 __iomem *driver_version_register) +void gve_adminq_write_version(u8 __iomem *driver_version_register) { const char *c = gve_version_prefix; @@ -2823,7 +2834,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { int max_tx_queues, max_rx_queues; struct net_device *dev; - __be32 __iomem *db_bar; struct gve_registers __iomem *reg_bar; struct gve_priv *priv; int err; @@ -2851,14 +2861,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto abort_with_pci_region; } - db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0); - if (!db_bar) { - dev_err(&pdev->dev, "Failed to map doorbell bar!\n"); - err = -ENOMEM; - goto abort_with_reg_bar; - } - - gve_write_version(®_bar->driver_version); /* Get max queues to alloc etherdev */ max_tx_queues = ioread32be(®_bar->max_tx_queues); max_rx_queues = ioread32be(®_bar->max_rx_queues); @@ -2867,7 +2869,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!dev) { dev_err(&pdev->dev, "could not allocate netdev\n"); err = -ENOMEM; - goto abort_with_db_bar; + goto abort_with_reg_bar; } SET_NETDEV_DEV(dev, &pdev->dev); pci_set_drvdata(pdev, dev); @@ -2899,35 +2901,60 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) priv->pdev = pdev; priv->msg_enable = DEFAULT_MSG_LEVEL; priv->reg_bar0 = reg_bar; - priv->db_bar2 = db_bar; priv->service_task_flags = 0x0; priv->state_flags = 0x0; priv->ethtool_flags = 0x0; priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; + err = gve_adminq_init(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Failed to alloc admin queue: err=%d\n", err); + goto abort_with_netdev; + } + + priv->device_info.queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; + err = gve_adminq_get_device_properties(priv); + if (err) + goto abort_with_adminq; + + /* Set adminq ctrl ops */ + priv->ctrl_ops = &gve_adminq_ops; + + err = priv->ctrl_ops->map_db_bar(priv); + if (err) + goto abort_with_adminq; + gve_set_probe_in_progress(priv); priv->gve_wq = alloc_ordered_workqueue("gve", 0); if (!priv->gve_wq) { dev_err(&pdev->dev, "Could not allocate workqueue"); err = -ENOMEM; - goto abort_with_netdev; + goto abort_with_unmap_db_bar; } INIT_WORK(&priv->service_task, gve_service_task); INIT_WORK(&priv->stats_report_task, gve_stats_report_task); priv->tx_cfg.max_queues = max_tx_queues; priv->rx_cfg.max_queues = max_rx_queues; - err = gve_init_priv(priv, false); + err = gve_init_priv(priv); if (err) goto abort_with_wq; + err = gve_setup_device(priv); + if (err) { + dev_err(&priv->pdev->dev, + "Could not setup device: err=%d\n", err); + goto abort_teardown_device; + } + if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) dev->netmem_tx = NETMEM_TX_DMA; err = register_netdev(dev); if (err) - goto abort_with_gve_init; + goto abort_teardown_device; dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format); @@ -2935,18 +2962,22 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) queue_work(priv->gve_wq, &priv->service_task); return 0; -abort_with_gve_init: - gve_teardown_priv_resources(priv); +abort_teardown_device: + disable_work_sync(&priv->service_task); + gve_teardown_device(priv); abort_with_wq: destroy_workqueue(priv->gve_wq); +abort_with_unmap_db_bar: + priv->ctrl_ops->unmap_db_bar(priv); + +abort_with_adminq: + gve_adminq_free(priv); + abort_with_netdev: free_netdev(dev); -abort_with_db_bar: - pci_iounmap(pdev, db_bar); - abort_with_reg_bar: pci_iounmap(pdev, reg_bar); @@ -2962,14 +2993,14 @@ static void gve_remove(struct pci_dev *pdev) { struct net_device *netdev = pci_get_drvdata(pdev); struct gve_priv *priv = netdev_priv(netdev); - __be32 __iomem *db_bar = priv->db_bar2; void __iomem *reg_bar = priv->reg_bar0; unregister_netdev(netdev); - gve_teardown_priv_resources(priv); + disable_work_sync(&priv->service_task); + gve_teardown_device(priv); destroy_workqueue(priv->gve_wq); + priv->ctrl_ops->unmap_db_bar(priv); free_netdev(netdev); - pci_iounmap(pdev, db_bar); pci_iounmap(pdev, reg_bar); pci_release_regions(pdev); pci_disable_device(pdev); @@ -2981,17 +3012,16 @@ static void gve_shutdown(struct pci_dev *pdev) struct gve_priv *priv = netdev_priv(netdev); bool was_up = netif_running(priv->dev); - netif_device_detach(netdev); + disable_work_sync(&priv->service_task); rtnl_lock(); netdev_lock(netdev); - if (was_up && gve_close(priv->dev)) { - /* If the dev was up, attempt to close, if close fails, reset */ - gve_reset_and_teardown(priv, was_up); - } else { - /* If the dev wasn't up or close worked, finish tearing down */ - gve_teardown_priv_resources(priv); - } + if (was_up) + gve_close(priv->dev); + + /* detach here because gve_close() might attach in recovery */ + netif_device_detach(netdev); + gve_teardown_device(priv); netdev_unlock(netdev); rtnl_unlock(); } @@ -3003,16 +3033,14 @@ static int gve_suspend(struct device *dev) struct gve_priv *priv = netdev_priv(netdev); bool was_up = netif_running(priv->dev); + disable_work_sync(&priv->service_task); + priv->suspend_cnt++; rtnl_lock(); netdev_lock(netdev); - if (was_up && gve_close(priv->dev)) { - /* If the dev was up, attempt to close, if close fails, reset */ - gve_reset_and_teardown(priv, was_up); - } else { - /* If the dev wasn't up or close worked, finish tearing down */ - gve_teardown_priv_resources(priv); - } + if (was_up) + gve_close(priv->dev); + gve_teardown_device(priv); priv->up_before_suspend = was_up; netdev_unlock(netdev); rtnl_unlock(); @@ -3029,7 +3057,8 @@ static int gve_resume(struct device *dev) priv->resume_cnt++; rtnl_lock(); netdev_lock(netdev); - err = gve_reset_recovery(priv, priv->up_before_suspend); + enable_work(&priv->service_task); + err = gve_recover(priv, priv->up_before_suspend); netdev_unlock(netdev); rtnl_unlock(); return err; diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c index b53b7fcdcdaf..caed38cd5f2b 100644 --- a/drivers/net/ethernet/google/gve/gve_utils.c +++ b/drivers/net/ethernet/google/gve/gve_utils.c @@ -119,6 +119,8 @@ void gve_remove_napi(struct gve_priv *priv, int ntfy_idx) { struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; - disable_irq(block->irq); + if (block->irq_requested) + disable_irq(block->irq); + netif_napi_del_locked(&block->napi); } |