path: root/tools/testing/selftests/drivers/net/hw/tso.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0

"""A simple test for TSO."""

import fcntl
import mmap
import socket
import struct
import termios
import time

from lib.py import ksft_pr, ksft_run, ksft_exit, KsftSkipEx, KsftXfailEx
from lib.py import ksft_eq, ksft_ge, ksft_lt
from lib.py import EthtoolFamily, NetdevFamily, NetDrvEpEnv
from lib.py import bkg, cmd, defer, ethtool, ip, rand_port, wait_port_listen


MAP_HUGETLB = getattr(mmap, "MAP_HUGETLB", 0x40000)
MSG_ZEROCOPY = getattr(socket, "MSG_ZEROCOPY", 0x4000000)
SO_ZEROCOPY = getattr(socket, "SO_ZEROCOPY", 60)

GSO_LEGACY_MAX_SIZE = 65536

# Pool of the default hugepage size, the one /proc/meminfo reports on.
NR_HUGEPAGES = "/proc/sys/vm/nr_hugepages"


def default_huge_page_size():
    """Return the hugepage size in bytes"""
    try:
        with open("/proc/meminfo", encoding="utf-8") as meminfo:
            for line in meminfo:
                if line.startswith("Hugepagesize:"):
                    return int(line.split()[1]) * 1024
    except OSError:
        pass

    return 2 * 1024 * 1024


def hugepages_free():
    """Return the number of unused hugepages of the default size."""
    try:
        with open("/proc/meminfo", encoding="utf-8") as meminfo:
            for line in meminfo:
                if line.startswith("HugePages_Free:"):
                    return int(line.split()[1])
    except OSError:
        pass
    return 0


def set_nr_hugepages(count):
    with open(NR_HUGEPAGES, "w", encoding="utf-8") as sysctl:
        sysctl.write(f"{count}\n")


def tx_dropped(ifname):
    with open(f"/sys/class/net/{ifname}/statistics/tx_dropped",
              encoding="utf-8") as counter:
        return int(counter.read())


def setup_hugepage():
    """Reserve one hugepage, and put the pool back afterwards."""
    if hugepages_free() >= 1:
        return

    try:
        with open(NR_HUGEPAGES, encoding="utf-8") as sysctl:
            old_count = int(sysctl.read())
        set_nr_hugepages(old_count + 1)
    except OSError as error:
        raise KsftSkipEx(f"Unable to reserve a hugepage: {error}") from error

    defer(set_nr_hugepages, old_count)

    if hugepages_free() < 1:
        raise KsftSkipEx("Unable to reserve a hugepage")


def mmap_large_buffer():
    """Allocate a buffer backed by one huge page."""
    size = default_huge_page_size()

    setup_hugepage()

    try:
        return mmap.mmap(-1, size,
                         flags=mmap.MAP_PRIVATE |
                               mmap.MAP_ANONYMOUS |
                               MAP_HUGETLB,
                         prot=mmap.PROT_READ)
    except OSError as e:
        raise KsftSkipEx(f"Unable to allocate a {size >> 20}MB hugepage "
                         f"buffer: {e}") from e


def sock_wait_drain(sock, max_wait=1000):
    """Wait for all pending write data on the socket to get ACKed."""
    for _ in range(max_wait):
        one = b'\0' * 4
        outq = fcntl.ioctl(sock.fileno(), termios.TIOCOUTQ, one)
        outq = struct.unpack("I", outq)[0]
        if outq == 0:
            break
        time.sleep(0.01)
    ksft_eq(outq, 0)


def tcp_sock_get_retrans(sock):
    """Get the number of retransmissions for the TCP socket."""
    info = sock.getsockopt(socket.SOL_TCP, socket.TCP_INFO, 512)
    return struct.unpack("I", info[100:104])[0]


def setup_big_tcp(cfg):
    """Lift the GSO ceiling to what the device advertises for TSO."""
    if cfg.dev["tso_max_size"] <= GSO_LEGACY_MAX_SIZE:
        raise KsftSkipEx("Device does not support BIG TCP")

    ip(f"link set dev {cfg.ifname} "
       f"gso_max_size {cfg.dev['tso_max_size']} "
       f"gso_ipv4_max_size {cfg.dev['tso_max_size']}")

    defer(ip, f"link set dev {cfg.ifname} "
              f"gso_max_size {cfg.dev['gso_max_size']} "
              f"gso_ipv4_max_size {cfg.dev['gso_ipv4_max_size']}")


def sock_send_zerocopy(sock):
    """Send with MSG_ZEROCOPY, return the bytes queued."""
    try:
        sock.setsockopt(socket.SOL_SOCKET, SO_ZEROCOPY, 1)
    except OSError as e:
        raise KsftSkipEx(f"SO_ZEROCOPY not supported: {e}") from e

    with mmap_large_buffer() as tx_buf:
        sock.sendall(tx_buf, MSG_ZEROCOPY)
        return len(tx_buf)


def run_one_stream(cfg, ipver, remote_v4, remote_v6, should_lso):
    cfg.require_cmd("socat", local=False, remote=True)

    # Set recv window clamp to avoid overwhelming receiver on debug kernels
    # the 200k clamp should still let use reach > 15Gbps on real HW
    port = rand_port()
    listen_opts = f"{port},reuseport,tcp-window-clamp=200000"
    listen_cmd = f"socat -{ipver} -t 2 -u TCP-LISTEN:{listen_opts} /dev/null,ignoreeof"

    with bkg(listen_cmd, host=cfg.remote, exit_wait=True) as nc:
        wait_port_listen(port, host=cfg.remote)

        if ipver == "4":
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((remote_v4, port))
        else:
            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
            sock.connect((remote_v6, port))

        # Small send to make sure the connection is working.
        sock.send("ping".encode())
        sock_wait_drain(sock)

        # Send 4MB of data, record the LSO packet count.
        qstat_old = cfg.netnl.qstats_get({"ifindex": cfg.ifindex}, dump=True)[0]
        buf = b"0" * 1024 * 1024 * 4
        sock.send(buf)
        sock_wait_drain(sock)
        qstat_new = cfg.netnl.qstats_get({"ifindex": cfg.ifindex}, dump=True)[0]

        # Check that at least 90% of the data was sent as LSO packets.
        # System noise may cause false negatives. Also header overheads
        # will add up to 5% of extra packes... The check is best effort.
        total_lso_wire  = len(buf) * 0.90 // cfg.dev["mtu"]
        total_lso_super = len(buf) * 0.90 // cfg.dev["tso_max_size"]

        # Make sure we have order of magnitude more LSO packets than
        # retransmits, in case TCP retransmitted all the LSO packets.
        ksft_lt(tcp_sock_get_retrans(sock), total_lso_wire / 16)
        sock.close()

        if should_lso:
            if cfg.have_stat_super_count:
                ksft_ge(qstat_new['tx-hw-gso-packets'] -
                        qstat_old['tx-hw-gso-packets'],
                        total_lso_super,
                        comment="Number of LSO super-packets with LSO enabled")
            if cfg.have_stat_wire_count:
                ksft_ge(qstat_new['tx-hw-gso-wire-packets'] -
                        qstat_old['tx-hw-gso-wire-packets'],
                        total_lso_wire,
                        comment="Number of LSO wire-packets with LSO enabled")
        else:
            if cfg.have_stat_super_count:
                ksft_lt(qstat_new['tx-hw-gso-packets'] -
                        qstat_old['tx-hw-gso-packets'],
                        15, comment="Number of LSO super-packets with LSO disabled")
            if cfg.have_stat_wire_count:
                ksft_lt(qstat_new['tx-hw-gso-wire-packets'] -
                        qstat_old['tx-hw-gso-wire-packets'],
                        500, comment="Number of LSO wire-packets with LSO disabled")


def run_big_tcp_stream(cfg, ipver, remote_v4, remote_v6):
    """Send with MSG_ZEROCOPY out of a huge page, so the frags exceed 64kB."""
    cfg.require_cmd("socat", local=False, remote=True)

    # No clamping, as it would keep the frags under 64kB
    port = rand_port()
    listen_opts = f"{port},reuseport"
    listen_cmd = f"socat -{ipver} -t 2 -u TCP-LISTEN:{listen_opts} /dev/null,ignoreeof"

    with bkg(listen_cmd, host=cfg.remote, exit_wait=True):
        wait_port_listen(port, host=cfg.remote)

        if ipver == "4":
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((remote_v4, port))
        else:
            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
            sock.connect((remote_v6, port))

        # Small send to make sure the connection is working.
        sock.send("ping".encode())
        sock_wait_drain(sock)

        retrans_old = tcp_sock_get_retrans(sock)
        drops_old = tx_dropped(cfg.ifname)

        sent = sock_send_zerocopy(sock)
        sock_wait_drain(sock)

        drops = tx_dropped(cfg.ifname) - drops_old
        retrans = tcp_sock_get_retrans(sock) - retrans_old
        sock.close()

        ksft_eq(drops, 0, comment="Driver TX drops during BIG TCP send")

        # Same best effort bound as the plain stream.
        total_lso_wire = sent * 0.90 // cfg.dev["mtu"]
        ksft_lt(retrans, total_lso_wire / 16)


def build_tunnel(cfg, outer_ipver, tun_info):
    local_v4  = NetDrvEpEnv.nsim_v4_pfx + "1"
    local_v6  = NetDrvEpEnv.nsim_v6_pfx + "1"
    remote_v4 = NetDrvEpEnv.nsim_v4_pfx + "2"
    remote_v6 = NetDrvEpEnv.nsim_v6_pfx + "2"

    local_addr  = cfg.addr_v[outer_ipver]
    remote_addr = cfg.remote_addr_v[outer_ipver]

    tun_type = tun_info[0]
    tun_arg  = tun_info[1]
    ip(f"link add {tun_type}-ksft type {tun_type} {tun_arg} local {local_addr} remote {remote_addr} dev {cfg.ifname}")
    defer(ip, f"link del {tun_type}-ksft")
    ip(f"link set dev {tun_type}-ksft up")
    ip(f"addr add {local_v4}/24 dev {tun_type}-ksft")
    ip(f"addr add {local_v6}/64 dev {tun_type}-ksft")

    ip(f"link add {tun_type}-ksft type {tun_type} {tun_arg} local {remote_addr} remote {local_addr} dev {cfg.remote_ifname}",
        host=cfg.remote)
    defer(ip, f"link del {tun_type}-ksft", host=cfg.remote)
    ip(f"link set dev {tun_type}-ksft up", host=cfg.remote)
    ip(f"addr add {remote_v4}/24 dev {tun_type}-ksft", host=cfg.remote)
    ip(f"addr add {remote_v6}/64 dev {tun_type}-ksft", host=cfg.remote)

    return remote_v4, remote_v6


def restore_wanted_features(cfg):
    features_cmd = ""
    for feature in cfg.hw_features:
        setting = "on" if feature in cfg.wanted_features else "off"
        features_cmd += f" {feature} {setting}"
    try:
        ethtool(f"-K {cfg.ifname} {features_cmd}")
    except Exception as e:
        ksft_pr(f"WARNING: failure restoring wanted features: {e}")


def test_builder(name, cfg, outer_ipver, feature, tun=None, inner_ipver=None):
    """Construct specific tests from the common template."""
    def f(cfg):
        cfg.require_ipver(outer_ipver)
        defer(restore_wanted_features, cfg)

        if not cfg.have_stat_super_count and \
           not cfg.have_stat_wire_count:
            raise KsftSkipEx(f"Device does not support LSO queue stats")

        if feature not in cfg.hw_features:
            raise KsftSkipEx(f"Device does not support {feature}")

        # Run non-tunnel test cases under the BIG TCP limits too.
        big_tcp = "big_tcp" in name
        if big_tcp:
            setup_big_tcp(cfg)

        ipver = outer_ipver
        if tun:
            remote_v4, remote_v6 = build_tunnel(cfg, ipver, tun)
            ipver = inner_ipver
        else:
            remote_v4 = cfg.remote_addr_v["4"]
            remote_v6 = cfg.remote_addr_v["6"]

        # First test without the feature enabled.
        ethtool(f"-K {cfg.ifname} {feature} off")
        run_one_stream(cfg, ipver, remote_v4, remote_v6, should_lso=False)

        if big_tcp:
            run_big_tcp_stream(cfg, ipver, remote_v4, remote_v6)

        ethtool(f"-K {cfg.ifname} tx-gso-partial off")
        ethtool(f"-K {cfg.ifname} tx-tcp-mangleid-segmentation off")
        if feature in cfg.partial_features:
            ethtool(f"-K {cfg.ifname} tx-gso-partial on")
            if ipver == "4":
                ksft_pr("Testing with mangleid enabled")
                ethtool(f"-K {cfg.ifname} tx-tcp-mangleid-segmentation on")

        # Full feature enabled.
        ethtool(f"-K {cfg.ifname} {feature} on")
        run_one_stream(cfg, ipver, remote_v4, remote_v6, should_lso=True)

        if big_tcp:
            run_big_tcp_stream(cfg, ipver, remote_v4, remote_v6)

    f.__name__ = name + ((outer_ipver + "_") if tun else "") + "ipv" + inner_ipver
    return f


def query_nic_features(cfg) -> None:
    """Query and cache the NIC features."""
    cfg.have_stat_super_count = False
    cfg.have_stat_wire_count = False

    features = cfg.ethnl.features_get({"header": {"dev-index": cfg.ifindex}})

    cfg.wanted_features = set()
    for f in features["wanted"]["bits"]["bit"]:
        cfg.wanted_features.add(f["name"])

    cfg.hw_features = set()
    for f in features["hw"]["bits"]["bit"]:
        if f.get("value", False):
            cfg.hw_features.add(f["name"])

    # Check which features are supported via GSO partial
    cfg.partial_features = set()
    if 'tx-gso-partial' in cfg.hw_features:
        seg_features = {f for f in cfg.hw_features if "segmentation" in f}
        ethtool(f"-K {cfg.ifname} " +
                " ".join(f"{f} on" for f in seg_features))

        ethtool(f"-K {cfg.ifname} tx-gso-partial off")

        no_partial = set()
        features = cfg.ethnl.features_get({"header": {"dev-index": cfg.ifindex}})
        for f in features["active"]["bits"]["bit"]:
            no_partial.add(f["name"])
        cfg.partial_features = seg_features - no_partial
        ethtool(f"-K {cfg.ifname} tx-gso-partial on")

    restore_wanted_features(cfg)

    stats = cfg.netnl.qstats_get({"ifindex": cfg.ifindex}, dump=True)
    if stats:
        if 'tx-hw-gso-packets' in stats[0]:
            ksft_pr("Detected qstat for LSO super-packets")
            cfg.have_stat_super_count = True
        if 'tx-hw-gso-wire-packets' in stats[0]:
            ksft_pr("Detected qstat for LSO wire-packets")
            cfg.have_stat_wire_count = True


def main() -> None:
    with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
        cfg.ethnl = EthtoolFamily()
        cfg.netnl = NetdevFamily()

        query_nic_features(cfg)

        test_info = (
            # name,       v4/v6  ethtool_feature               tun:(type, args, inner ip versions)
            ("",           "4", "tx-tcp-segmentation",         None),
            ("",           "6", "tx-tcp6-segmentation",        None),
            ("big_tcp_",   "4", "tx-tcp-segmentation",         None),
            ("big_tcp_",   "6", "tx-tcp6-segmentation",        None),
            ("vxlan",      "4", "tx-udp_tnl-segmentation",     ("vxlan", "id 100 dstport 4789 noudpcsum", ("4", "6"))),
            ("vxlan",      "6", "tx-udp_tnl-segmentation",     ("vxlan", "id 100 dstport 4789 udp6zerocsumtx udp6zerocsumrx", ("4", "6"))),
            ("vxlan_csum", "", "tx-udp_tnl-csum-segmentation", ("vxlan", "id 100 dstport 4789 udpcsum", ("4", "6"))),
            ("gre",        "4", "tx-gre-segmentation",         ("gre",   "", ("4", "6"))),
            ("gre",        "6", "tx-gre-segmentation",         ("ip6gre","", ("4", "6"))),
            ("ip",         "6", "tx-ipxip6-segmentation",      ("ip6tnl","mode any", ("4", "6"))),
            ("ip",         "4", "tx-ipxip4-segmentation",      ("sit","", ("6", ))),
            ("ip",         "4", "tx-ipxip4-segmentation",      ("ipip","", ("4", ))),
        )

        cases = []
        for outer_ipver in ["4", "6"]:
            for info in test_info:
                # Skip if test which only works for a specific IP version
                if info[1] and outer_ipver != info[1]:
                    continue

                if info[3]:
                    cases += [
                        test_builder(info[0], cfg, outer_ipver, info[2], info[3], inner_ipver)
                        for inner_ipver in info[3][2]
                    ]
                else:
                    cases.append(test_builder(info[0], cfg, outer_ipver, info[2], None, outer_ipver))

        ksft_run(cases=cases, args=(cfg, ))
    ksft_exit()


if __name__ == "__main__":
    main()