path: root/tools/testing/selftests/bpf/progs/task_work_busy.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026 KylinSoft Corporation. */
#include <vmlinux.h>
#include <stdbool.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>

char _license[] SEC("license") = "GPL";

extern bool CONFIG_PREEMPT_RT __kconfig __weak;

/*
 * Bounds for the hardirq busy-wait that holds the scheduling handler
 * inside the real window: task_work_add() has published the callback,
 * but the handler's tail state transition has not run yet.
 * irq_work_single() clears BUSY only after the handler returns, so
 * while this wait runs, the context's scheduling irq_work is BUSY,
 * even though the first callback may already have completed and
 * returned the context to STANDBY.
 *
 * The wait is bounded both by a bpf_ktime_get_ns() check and by an
 * iteration budget.  The time bound is checked against a monotonic
 * clock: host vCPU preemption can stretch the wall-clock duration of
 * the window, which only makes the window longer, never shorter.
 */
#define BUSY_WAIT_NS		4000000ULL
#define BUSY_WAIT_ITERS		1000000ULL

int trigger_tid;
int target_tid;

/*
 * High 32 bits of the result slots hold the sequence, low 32 bits the
 * signed error.  BUSY_WINDOW low bits: 1 open, 2 the in-window attempt
 * was seen before the handler returned, 3 the wait bound was reached
 * without it.  Must match the test.
 */
enum task_work_busy_status {
	BUSY_ARM_SEQ,
	BUSY_WINDOW,
	BUSY_ACK,
	BUSY_GEN1_RESULT,
	BUSY_GEN2_RESULT,
	BUSY_CB1_SEQ,
	BUSY_CB2_COUNT,
	BUSY_CB2_SEQ,
	BUSY_RECOVER_SEQ,
	BUSY_LAST_WAIT_NS,
	BUSY_STATUS_MAX,
};

struct {
	__uint(type, BPF_MAP_TYPE_ARRAY);
	__uint(max_entries, BUSY_STATUS_MAX);
	__type(key, int);
	__type(value, __s64);
} status SEC(".maps");

struct task_work_busy_value {
	__u32 seq;
	char data[60];
	struct bpf_task_work tw;
};

/*
 * max_entries must cover TASK_WORK_BUSY_MAX_ATTEMPTS in the test: each
 * attempt schedules into its own fresh value.
 */
struct {
	__uint(type, BPF_MAP_TYPE_HASH);
	__uint(map_flags, BPF_F_NO_PREALLOC);
	__uint(max_entries, 128);
	__type(key, int);
	__type(value, struct task_work_busy_value);
} hmap SEC(".maps");

static __always_inline void set_status(int key, __s64 value)
{
	__s64 *slot;

	slot = bpf_map_lookup_elem(&status, &key);
	if (slot)
		*slot = value;
}

static __always_inline __s64 get_status(int key)
{
	__s64 *slot = bpf_map_lookup_elem(&status, &key);

	return slot ? *slot : 0;
}

struct busy_wait_ctx {
	__u64 seq;
	__u64 start;
	bool acked;
};

/* bpf_loop() callback: return 0 continues the loop, return 1 stops it. */
static long busy_wait_cb(__u32 i, void *pctx)
{
	struct busy_wait_ctx *w = pctx;

	if (((__u64)get_status(BUSY_ACK) >> 32) == w->seq) {
		w->acked = true;
		return 1;
	}
	if (bpf_ktime_get_ns() - w->start > BUSY_WAIT_NS)
		return 1;
	return 0;
}

static int busy_cb_gen1(struct bpf_map *map, void *key, void *value)
{
	struct task_work_busy_value *work = value;
	__u64 we = (__u64)get_status(BUSY_WINDOW) >> 32;

	/*
	 * Backstop for deliveries from abandoned attempts: accept only
	 * while no window for a different sequence is published.  The
	 * test waits for the old workers to disappear from PID lookup,
	 * after their exit task_work, before resetting the shared state.
	 * This callback may run before fexit publishes the window state.
	 */
	if (we && we != (__u64)work->seq)
		return 0;
	set_status(BUSY_CB1_SEQ, work->seq);
	return 0;
}

static int busy_cb_gen2(struct bpf_map *map, void *key, void *value)
{
	struct task_work_busy_value *work = value;
	__s64 *count;
	int key2 = BUSY_CB2_COUNT;

	/*
	 * Count only deliveries belonging to the sequence whose window is
	 * (or was) published: a late delivery from an abandoned attempt
	 * must not satisfy a later exactly-once check.
	 */
	if ((__u64)work->seq != ((__u64)get_status(BUSY_WINDOW) >> 32))
		return 0;
	set_status(BUSY_CB2_SEQ, work->seq);
	count = bpf_map_lookup_elem(&status, &key2);
	if (count)
		__sync_fetch_and_add(count, 1);
	return 0;
}

SEC("fexit/task_work_add")
int BPF_PROG(busy_fexit_task_work_add, struct task_struct *task,
	     struct callback_head *work, int notify, int retval)
{
	struct busy_wait_ctx w = {};
	__u64 seq, elapsed;

	seq = (__u64)get_status(BUSY_ARM_SEQ);
	if (!seq)
		return 0;

	/*
	 * One-shot arm.  Only this test's own first-round add may consume
	 * it and hold the window: a successful TWA_SIGNAL add of the
	 * bpf_task_work callback, issued from the trigger thread's CPU by
	 * the self-IPI'd scheduling handler and targeting the target
	 * thread.  Everything else leaves the arm untouched.
	 */
	if (retval != 0 || notify != TWA_SIGNAL ||
	    BPF_CORE_READ(task, pid) != target_tid)
		return 0;
	if ((__u32)bpf_get_current_pid_tgid() != (__u32)trigger_tid)
		return 0;

	set_status(BUSY_ARM_SEQ, 0);
	set_status(BUSY_WINDOW, (seq << 32) | 1);

	w.seq = seq;
	w.start = bpf_ktime_get_ns();
	bpf_loop(BUSY_WAIT_ITERS, busy_wait_cb, &w, 0);
	elapsed = bpf_ktime_get_ns() - w.start;

	/*
	 * Publish elapsed time for logging after worker cleanup.  Window
	 * validity depends on seeing ACK here, not on this diagnostic.
	 */
	set_status(BUSY_LAST_WAIT_NS, (__s64)elapsed);
	if (w.acked)
		set_status(BUSY_WINDOW, (seq << 32) | 2);
	else
		set_status(BUSY_WINDOW, (seq << 32) | 3);
	return 0;
}

/*
 * Attempt the second scheduling round on the same context, targeting the
 * trigger thread.  The attempt uses the value keyed by its own sequence,
 * so it can never touch another attempt's context.  Returns true if a
 * terminal result was published into res_key; a transient -EBUSY is left
 * unpublished for the caller's retry policy, mirroring the race test's
 * convention.
 */
static __always_inline bool busy_try_gen2(__u64 seq, int res_key)
{
	struct task_work_busy_value *work;
	struct task_struct *task;
	int key = (int)seq, err;

	task = bpf_task_from_pid(trigger_tid);
	if (!task) {
		set_status(res_key, ((__s64)seq << 32) | (__u32)-ESRCH);
		return true;
	}
	work = bpf_map_lookup_elem(&hmap, &key);
	if (!work) {
		bpf_task_release(task);
		set_status(res_key, ((__s64)seq << 32) | (__u32)-ENOENT);
		return true;
	}
	work->seq = seq;
	err = bpf_task_work_schedule_signal(task, &work->tw, &hmap,
					    busy_cb_gen2);
	bpf_task_release(task);
	if (err == -EBUSY)
		return false;
	set_status(res_key, ((__s64)seq << 32) | (__u32)err);
	return true;
}

SEC("tracepoint/syscalls/sys_enter_getppid")
int busy_sched_work(void *ctx)
{
	__u32 tid = (__u32)bpf_get_current_pid_tgid();
	struct task_work_busy_value *work;
	struct task_struct *task;
	__u64 window, seq;
	int key, err;

	/* Also keep the kconfig extern in object BTF for the userspace skip. */
	if (CONFIG_PREEMPT_RT)
		return 0;

	if (tid == (__u32)trigger_tid) {
		/*
		 * Trigger thread: schedule the first round once per armed
		 * sequence, targeting the target thread.  The kfunc queues
		 * the scheduling irq_work and returns; the handler then
		 * parks inside the fexit window above.  A fresh value has no
		 * expected BUSY source, so publish every scheduling result.
		 */
		seq = (__u64)get_status(BUSY_ARM_SEQ);
		if (!seq ||
		    ((__u64)get_status(BUSY_GEN1_RESULT) >> 32) == seq)
			return 0;
		key = (int)seq;
		task = bpf_task_from_pid(target_tid);
		if (!task) {
			set_status(BUSY_GEN1_RESULT,
				   ((__s64)seq << 32) | (__u32)-ESRCH);
			set_status(BUSY_ARM_SEQ, 0);
			return 0;
		}
		work = bpf_map_lookup_elem(&hmap, &key);
		if (!work) {
			bpf_task_release(task);
			set_status(BUSY_GEN1_RESULT,
				   ((__s64)seq << 32) | (__u32)-ENOENT);
			set_status(BUSY_ARM_SEQ, 0);
			return 0;
		}
		work->seq = seq;
		err = bpf_task_work_schedule_signal(task, &work->tw, &hmap,
						    busy_cb_gen1);
		bpf_task_release(task);
		set_status(BUSY_GEN1_RESULT, ((__s64)seq << 32) | (__u32)err);
		return 0;
	}

	if (tid != (__u32)target_tid)
		return 0;

	window = (__u64)get_status(BUSY_WINDOW);
	seq = window >> 32;
	if (!seq || (__u32)window == 0)
		return 0;
	/*
	 * The first callback must have completed for this sequence.  Being
	 * inside the target thread's own subsequent syscall proves the
	 * callback wrapper has returned and the context is STANDBY, which
	 * is stronger than observing the completion marker from another
	 * thread.
	 */
	if ((__u64)get_status(BUSY_CB1_SEQ) != seq)
		return 0;

	if ((__u64)get_status(BUSY_ACK) >> 32 != seq) {
		/*
		 * The in-window attempt, exactly once per sequence.  On a
		 * fixed kernel it must be rejected with -EBUSY: the context
		 * is STANDBY, the map and program references are alive and
		 * no other scheduler exists, so only the BUSY gate can
		 * reject here.
		 */
		if (!busy_try_gen2(seq, BUSY_ACK))
			set_status(BUSY_ACK,
				   ((__s64)seq << 32) | (__u32)-EBUSY);
		return 0;
	}

	/*
	 * Recovery, gated by the test: the fexit program releasing the
	 * window does not mean the whole old handler has returned and
	 * BUSY has cleared yet, so retry until the schedule succeeds.
	 */
	if ((__u64)get_status(BUSY_RECOVER_SEQ) != seq)
		return 0;
	if ((__s32)get_status(BUSY_ACK) == 0)
		return 0;	/* scheduled in-window; nothing to recover */
	if (((__u64)get_status(BUSY_GEN2_RESULT) >> 32) == seq)
		return 0;	/* terminal recovery result published */
	busy_try_gen2(seq, BUSY_GEN2_RESULT);
	return 0;
}