path: root/tools/testing/selftests/bpf/prog_tests/arena_race.c

// SPDX-License-Identifier: GPL-2.0

#define _GNU_SOURCE
#include <sched.h>
#include <setjmp.h>
#include <signal.h>
#include <sys/syscall.h>
#include <test_progs.h>

#include "arena_race.skel.h"

struct free_thread_ctx {
	struct arena_race *skel;
	int err;
	__u32 retval;
};

struct fault_thread_ctx {
	__u64 *addr;
	int stop;
};

static int run_prog(struct bpf_program *prog, const char *name)
{
	LIBBPF_OPTS(bpf_test_run_opts, opts);
	int err;

	err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
	return ASSERT_OK(err, name) && ASSERT_OK(opts.retval, name) ? 0 : -1;
}

/* Trigger the sleepable free page path. */
static void *run_free_thread(void *arg)
{
	LIBBPF_OPTS(bpf_test_run_opts, opts);
	struct free_thread_ctx *ctx = arg;

	ctx->skel->bss->target_tid = sys_gettid();
	ctx->err = bpf_prog_test_run_opts(
		bpf_program__fd(ctx->skel->progs.free_page), &opts);
	ctx->retval = opts.retval;
	return NULL;
}

static __thread sigjmp_buf reader_env;
static __thread bool reader_armed;

/* Only the reader expects SIGBUS; anywhere else it keeps its default action. */
static void reader_sigbus(int sig)
{
	if (!reader_armed) {
		signal(sig, SIG_DFL);
		raise(sig);
		return;
	}
	siglongjmp(reader_env, 1);
}

/* Continuously fault in the address. */
static void *fault_reader_thread(void *arg)
{
	struct fault_thread_ctx *ctx = arg;
	__u64 *addr;

	/* Touching a page while BPF frees it raises SIGBUS; skip that read. */
	sigsetjmp(reader_env, 1);
	reader_armed = true;
	while (!READ_ONCE(ctx->stop)) {
		addr = __atomic_load_n(&ctx->addr, __ATOMIC_ACQUIRE);
		(void)READ_ONCE(*addr);
	}
	reader_armed = false;
	return NULL;
}

static int wait_for(int *p)
{
	__u64 deadline = get_time_ns() + 5ULL * 1000 * 1000 * 1000;

	while (!READ_ONCE(*p)) {
		if (get_time_ns() > deadline)
			return -ETIMEDOUT;
	}
	return 0;
}

/* Skip the subtest, saying why, if @func cannot be traced with fentry. */
static bool require_fentry_target(const char *test, const char *func)
{
	if (libbpf_find_vmlinux_btf_id(func, BPF_TRACE_FENTRY) > 0)
		return true;
	printf("%s:SKIP: %s is not an fentry target\n", test, func);
	test__skip();
	return false;
}

static struct arena_race *setup_arena(__u64 **addr, bool trace_zap)
{
	struct arena_race *skel;
	size_t arena_sz;
	char *base;
	int err;

	skel = arena_race__open();
	if (!ASSERT_OK_PTR(skel, "open"))
		return NULL;
	err = bpf_program__set_autoload(skel->progs.trace_zap, trace_zap);
	if (!ASSERT_OK(err, "trace_zap_autoload"))
		goto err_out;

	err = arena_race__load(skel);
	if (!ASSERT_OK(err, "load"))
		goto err_out;
	err = arena_race__attach(skel);
	if (!ASSERT_OK(err, "attach"))
		goto err_out;

	if (run_prog(skel->progs.alloc_old, "alloc_old"))
		goto err_out;
	if (skel->bss->skip) {
		printf("%s:SKIP: compiler doesn't support arena_cast\n", __func__);
		test__skip();
		goto err_out;
	}

	base = bpf_map__initial_value(skel->maps.arena, &arena_sz);
	if (!ASSERT_OK_PTR(base, "arena_base"))
		goto err_out;
	*addr = (__u64 *)(base + getpagesize());
	if (!ASSERT_EQ((unsigned long)skel->bss->ptr, (unsigned long)*addr,
		       "arena_ptr"))
		goto err_out;
	return skel;

err_out:
	arena_race__destroy(skel);
	return NULL;
}

static void test_realloc_during_free(bool deferred)
{
	struct free_thread_ctx ctx = {};
	struct arena_race *skel;
	pthread_t thread;
	__u64 *addr;
	bool thread_created = false, zap_seen = false, completed = false;
	int err;

	if (!require_fentry_target(__func__, "zap_vma_range"))
		return;

	skel = setup_arena(&addr, true);
	if (!skel)
		return;

	/* Map the page into user space, so that the free has to zap it. */
	if (!ASSERT_EQ(READ_ONCE(*addr), skel->rodata->old_marker, "old_marker"))
		goto out;

	/* Pause the free after its TLB flush, before it releases the range. */
	skel->bss->pause_on_zap = 1;

	if (deferred) {
		/*
		 * Test the nonsleepable free path that gets
		 * deferred to a worker in the kernel. We do
		 * so by triggering the arena operation from
		 * a nonsleepable tracepoint context.
		 */
		skel->bss->trigger_pid_tgid =
			((__u64)getpid() << 32) | (__u32)sys_gettid();
		skel->bss->trigger_syscall = SYS_getpgid;
		skel->bss->deferred_free = 1;
		if (!ASSERT_GE(syscall(SYS_getpgid, 0), 0, "deferred_free"))
			goto release;
	} else {
		/*
		 * Test the sleepable arena free path through a
		 * syscall test prog.
		 */
		ctx.skel = skel;
		err = pthread_create(&thread, NULL, run_free_thread, &ctx);
		if (!ASSERT_OK(err, "pthread_create")) {
			skel->bss->release = 1;
			goto out;
		}
		thread_created = true;
	}

	/* Wait until the free reaches the user zap. */
	err = wait_for(&skel->bss->zap_entered);
	if (!ASSERT_OK(err, "zap_entered"))
		goto release;

	zap_seen = true;

	/* The range must stay allocated until the free completes. */
	run_prog(skel->progs.try_realloc, "realloc_during_free");
	ASSERT_NULL(skel->bss->realloc_ptr, "realloc_during_free");

release:
	skel->bss->release = 1;
	if (thread_created) {
		ASSERT_OK(pthread_join(thread, NULL), "pthread_join");
		thread_created = false;
		completed = ASSERT_OK(ctx.err, "free_run") &&
			    ASSERT_OK(ctx.retval, "free_retval");
	} else if (skel->bss->target_tid) {
		err = wait_for(&skel->bss->worker_exited);
		completed = ASSERT_OK(err, "worker_exited");
	}
	ASSERT_FALSE(skel->bss->timed_out, "zap_timed_out");

	if (zap_seen && completed &&
	    !run_prog(skel->progs.try_realloc, "realloc_after_free")) {
		ASSERT_EQ((unsigned long)skel->bss->realloc_ptr,
			  (unsigned long)addr, "realloc_after_free");
		ASSERT_EQ(*addr, skel->rodata->new_marker, "new_marker");
	}
out:
	arena_race__destroy(skel);
}

/*
 * Force a race between a faulting thread in userspace and a
 * free operation on the arena.
 */
static void test_fault_free_realloc(void)
{
	struct sigaction sigbus = { .sa_handler = reader_sigbus }, old_sigbus;
	struct fault_thread_ctx fault = {};
	struct arena_race *skel;
	pthread_t fault_thread;
	bool fault_created = false;
	__u64 *addr;
	__u64 expected, value;
	int err, i;

	skel = setup_arena(&addr, false);
	if (!skel)
		return;

	skel->bss->realloc_after_free = 1;

	if (!ASSERT_OK(sigaction(SIGBUS, &sigbus, &old_sigbus), "sigaction_sigbus"))
		goto out_destroy;
	fault.addr = addr;
	err = pthread_create(&fault_thread, NULL, fault_reader_thread, &fault);
	if (!ASSERT_OK(err, "pthread_create_fault"))
		goto out;
	fault_created = true;

	for (i = 0; i < 1000; i++) {
		LIBBPF_OPTS(bpf_test_run_opts, opts);

		/* Keep deferred frees from earlier iterations on separate slots. */
		if (i) {
			skel->bss->page_idx = i;
			if (run_prog(skel->progs.alloc_old, "alloc_next"))
				goto out;
			addr = (__u64 *)(unsigned long)skel->bss->ptr;
			__atomic_store_n(&fault.addr, addr, __ATOMIC_RELEASE);
		}

		err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.free_page),
					     &opts);
		if (err) {
			ASSERT_OK(err, "free_realloc");
			break;
		}
		if (opts.retval) {
			ASSERT_OK(opts.retval, "free_realloc");
			break;
		}
		/* The reader may populate the slot first, or free may defer. */
		if (!skel->bss->realloc_ptr)
			continue;
		value = *addr;
		expected = skel->bss->current_marker;
		if (value != expected) {
			ASSERT_EQ(value, expected, "marker_after_realloc");
			break;
		}
	}

	WRITE_ONCE(fault.stop, 1);
	if (fault_created) {
		ASSERT_OK(pthread_join(fault_thread, NULL), "pthread_join_fault");
		fault_created = false;
	}
out:
	WRITE_ONCE(fault.stop, 1);
	if (fault_created)
		pthread_join(fault_thread, NULL);
	sigaction(SIGBUS, &old_sigbus, NULL);
out_destroy:
	arena_race__destroy(skel);
}

void serial_test_arena_race(void)
{
	cpu_set_t cpuset;
	int err;

	err = sched_getaffinity(0, sizeof(cpuset), &cpuset);
	if (!ASSERT_OK(err, "sched_getaffinity"))
		return;
	if (CPU_COUNT(&cpuset) < 2) {
		printf("%s:SKIP: at least two runnable CPUs are required\n", __func__);
		test__skip();
		return;
	}

	if (test__start_subtest("realloc_during_free"))
		test_realloc_during_free(false);
	if (test__start_subtest("realloc_during_deferred_free"))
		test_realloc_during_free(true);
	if (test__start_subtest("fault_free_realloc"))
		test_fault_free_realloc();
}