path: root/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) Meta Platforms, Inc. and affiliates. */

#include <linux/etherdevice.h>
#include <linux/ipv6.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/types.h>
#include <net/netdev_lock.h>

#include "mpnic.h"
#include "mpnic_netdev.h"
#include "mpnic_txrx.h"

static int mpnic_open(struct net_device *netdev)
{
	struct mpnic_net *mpn = netdev_priv(netdev);
	int err;

	err = mpnic_alloc_napi_vectors(mpn);
	if (err)
		return err;

	err = mpnic_alloc_resources(mpn);
	if (err)
		goto err_free_napi_vectors;

	err = mpnic_set_netif_queues(mpn);
	if (err)
		goto err_free_resources;

	mpnic_stats_attach_rings(mpn);

	mpnic_enable(mpn);
	mpnic_fill(mpn);
	mpnic_napi_enable(mpn);

	netif_tx_wake_all_queues(netdev);
	netif_carrier_on(netdev);

	schedule_delayed_work(&mpn->mpd->service_task, HZ);

	return 0;

err_free_resources:
	mpnic_free_resources(mpn);
err_free_napi_vectors:
	mpnic_free_napi_vectors(mpn);
	return err;
}

static int mpnic_stop(struct net_device *netdev)
{
	struct mpnic_net *mpn = netdev_priv(netdev);

	cancel_delayed_work(&mpn->mpd->service_task);
	netif_carrier_off(netdev);

	mpnic_napi_disable(mpn);
	netif_tx_disable(netdev);

	mpnic_disable(mpn);
	mpnic_wait_all_queues_idle(mpn->mpd);
	mpnic_flush(mpn);

	mpnic_stats_fold_rings(mpn);

	mpnic_reset_netif_queues(mpn);
	mpnic_free_resources(mpn);
	mpnic_free_napi_vectors(mpn);

	return 0;
}

static void
mpnic_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64)
{
	struct mpnic_net *mpn = netdev_priv(dev);
	u64 bytes, packets, dropped, errors;
	struct mpnic_queue_stats *stats;
	struct mpnic_base_stats *base;
	unsigned int start, i;

	rcu_read_lock();

	base = rcu_dereference(mpn->stats.base);

	stats64->tx_bytes = base->tx.bytes;
	stats64->tx_packets = base->tx.packets;
	stats64->tx_dropped = base->tx.dropped;

	stats64->rx_bytes = base->rx.bytes;
	stats64->rx_packets = base->rx.packets;
	stats64->rx_dropped = base->rx.dropped;
	stats64->rx_errors = base->rx.errors;

	for (i = 0; i < base->num_tx_queues; i++) {
		struct mpnic_ring *txr = base->txr[i];
		struct mpnic_q_triad *qt;

		qt = container_of(txr, struct mpnic_q_triad, sub0);

		stats = &qt->cmpl.stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			bytes = u64_stats_read(&stats->tcq.bytes);
			packets = u64_stats_read(&stats->tcq.packets);
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		stats = &txr->stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			dropped = u64_stats_read(&stats->twq.dropped);
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		stats64->tx_bytes += bytes;
		stats64->tx_packets += packets;
		stats64->tx_dropped += dropped;
	}

	for (i = 0; i < base->num_rx_queues; i++) {
		struct mpnic_ring *rxr = base->rxr[i];

		stats = &rxr->stats;
		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			bytes = u64_stats_read(&stats->rcq.bytes);
			packets = u64_stats_read(&stats->rcq.packets);
			dropped = u64_stats_read(&stats->rcq.dropped);
			errors = u64_stats_read(&stats->rcq.errors);
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		stats64->rx_bytes += bytes;
		stats64->rx_packets += packets;
		stats64->rx_dropped += dropped;
		stats64->rx_errors += errors;
	}

	rcu_read_unlock();
}

static const struct net_device_ops mpnic_netdev_ops = {
	.ndo_open		= mpnic_open,
	.ndo_stop		= mpnic_stop,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_start_xmit		= mpnic_xmit_frame,
	.ndo_get_stats64	= mpnic_get_stats64,
};

static u64 mpnic_bdq_alloc_failed(struct mpnic_ring *bdq)
{
	struct mpnic_queue_stats *stats = &bdq->stats;
	unsigned int start;
	u64 alloc_failed;

	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		alloc_failed = u64_stats_read(&stats->bdq.alloc_failed);
	} while (u64_stats_fetch_retry(&stats->syncp, start));

	return alloc_failed;
}

static void mpnic_get_queue_stats_rx(struct net_device *dev, int idx,
				     struct netdev_queue_stats_rx *rx)
{
	struct mpnic_net *mpn = netdev_priv(dev);
	struct mpnic_ring *rxr = mpn->rx[idx];
	u64 bytes, packets, alloc_failed;
	struct mpnic_queue_stats *stats;
	struct mpnic_q_triad *qt;
	unsigned int start;

	if (!rxr)
		return;

	stats = &rxr->stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		bytes = u64_stats_read(&stats->rcq.bytes);
		packets = u64_stats_read(&stats->rcq.packets);
		alloc_failed = u64_stats_read(&stats->rcq.alloc_failed);
	} while (u64_stats_fetch_retry(&stats->syncp, start));

	qt = container_of(rxr, struct mpnic_q_triad, cmpl);
	alloc_failed += mpnic_bdq_alloc_failed(&qt->sub0);
	alloc_failed += mpnic_bdq_alloc_failed(&qt->sub1);

	rx->bytes = bytes;
	rx->packets = packets;
	rx->alloc_fail = alloc_failed;
}

static void mpnic_get_queue_stats_tx(struct net_device *dev, int idx,
				     struct netdev_queue_stats_tx *tx)
{
	struct mpnic_net *mpn = netdev_priv(dev);
	struct mpnic_ring *txr = mpn->tx[idx];
	struct mpnic_queue_stats *stats;
	u64 bytes, packets, stop, wake;
	struct mpnic_q_triad *qt;
	unsigned int start;

	if (!txr)
		return;

	qt = container_of(txr, struct mpnic_q_triad, sub0);

	stats = &qt->cmpl.stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		bytes = u64_stats_read(&stats->tcq.bytes);
		packets = u64_stats_read(&stats->tcq.packets);
		wake = u64_stats_read(&stats->tcq.wake);
	} while (u64_stats_fetch_retry(&stats->syncp, start));

	stats = &txr->stats;
	do {
		start = u64_stats_fetch_begin(&stats->syncp);
		stop = u64_stats_read(&stats->twq.stop);
	} while (u64_stats_fetch_retry(&stats->syncp, start));

	tx->bytes = bytes;
	tx->packets = packets;
	tx->stop = stop;
	tx->wake = wake;
}

static void mpnic_get_base_stats(struct net_device *dev,
				 struct netdev_queue_stats_rx *rx,
				 struct netdev_queue_stats_tx *tx)
{
	struct mpnic_net *mpn = netdev_priv(dev);
	struct mpnic_base_stats *base;

	base = netdev_lock_dereference(mpn->stats.base, dev);

	tx->bytes = base->tx.bytes;
	tx->packets = base->tx.packets;
	tx->stop = base->tx.stop;
	tx->wake = base->tx.wake;

	rx->bytes = base->rx.bytes;
	rx->packets = base->rx.packets;
	rx->alloc_fail = base->rx.alloc_failed;
}

static const struct netdev_stat_ops mpnic_stat_ops = {
	.get_queue_stats_rx	= mpnic_get_queue_stats_rx,
	.get_queue_stats_tx	= mpnic_get_queue_stats_tx,
	.get_base_stats		= mpnic_get_base_stats,
};

/**
 * mpnic_netdev_free - Free the netdev associated with mpnic
 * @mpd: Driver specific structure to free netdev from
 **/
void mpnic_netdev_free(struct mpnic_dev *mpd)
{
	free_netdev(mpd->netdev);
	mpd->netdev = NULL;
}

/**
 * mpnic_netdev_alloc - Allocate a netdev and associate it with mpnic
 * @mpd: Driver specific structure to associate the netdev with
 *
 * Return: NULL on failure.
 **/
struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd)
{
	struct net_device *netdev;
	struct mpnic_net *mpn;
	unsigned int queues;

	netdev = alloc_etherdev_mq(sizeof(*mpn), MPNIC_MAX_RXQS);
	if (!netdev)
		return NULL;

	SET_NETDEV_DEV(netdev, mpd->dev);
	mpd->netdev = netdev;

	netdev->netdev_ops = &mpnic_netdev_ops;
	netdev->stat_ops = &mpnic_stat_ops;
	netdev->request_ops_lock = true;

	mpn = netdev_priv(netdev);
	mpn->netdev = netdev;
	mpn->mpd = mpd;

	mpn->stats.buf[0].txr = mpn->tx;
	mpn->stats.buf[0].rxr = mpn->rx;
	RCU_INIT_POINTER(mpn->stats.base, &mpn->stats.buf[0]);

	mpn->txq_size = MPNIC_TXQ_SIZE_DEFAULT;
	mpn->hpq_size = MPNIC_HPQ_SIZE_DEFAULT;
	mpn->ppq_size = MPNIC_PPQ_SIZE_DEFAULT;
	mpn->rcq_size = MPNIC_RCQ_SIZE_DEFAULT;

	queues = min(netif_get_num_default_rss_queues(),
		     mpd->num_irqs - MPNIC_NON_NAPI_VECTORS);
	mpn->num_tx_queues = queues;
	mpn->num_rx_queues = queues;
	mpn->num_napi = queues;

	netdev->features |= NETIF_F_SG;
	netdev->hw_features |= netdev->features;
	netdev->vlan_features |= netdev->features;

	netdev->min_mtu = IPV6_MIN_MTU;
	netdev->max_mtu = MPNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;

	netif_carrier_off(netdev);
	netif_tx_stop_all_queues(netdev);

	return netdev;
}

static int mpnic_dsn_to_mac_addr(u64 dsn, u8 *addr)
{
	addr[0] = (dsn >> 56) & 0xFF;
	addr[1] = (dsn >> 48) & 0xFF;
	addr[2] = (dsn >> 40) & 0xFF;
	addr[3] = (dsn >> 16) & 0xFF;
	addr[4] = (dsn >> 8) & 0xFF;
	addr[5] = dsn & 0xFF;

	return is_valid_ether_addr(addr) ? 0 : -EINVAL;
}

/**
 * mpnic_netdev_register - Assign the MAC address and register the netdev
 * @netdev: Netdev to register
 *
 * The permanent address is derived from the PCIe device serial number, the
 * same way the firmware and the BMC derive it. A random address would break
 * provisioning, so refuse to spawn the interface if the serial number does
 * not yield a valid one.
 *
 * Return: non-zero on failure.
 **/
int mpnic_netdev_register(struct net_device *netdev)
{
	struct mpnic_net *mpn = netdev_priv(netdev);
	struct mpnic_dev *mpd = mpn->mpd;
	u8 addr[ETH_ALEN];
	int err;

	err = mpnic_dsn_to_mac_addr(mpd->dsn, addr);
	if (err) {
		dev_err(mpd->dev, "MAC addr %pM invalid\n", addr);
		return err;
	}

	ether_addr_copy(netdev->perm_addr, addr);
	eth_hw_addr_set(netdev, addr);

	err = netif_set_real_num_queues(netdev, mpn->num_tx_queues,
					mpn->num_rx_queues);
	if (err)
		return err;

	/* Abort if MMIO has failed. This has to be the last check before
	 * registration, the register accessors can only detach the device
	 * once it has been registered.
	 */
	if (!mpnic_present(mpd))
		return -EIO;

	return register_netdev(netdev);
}