summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/audit_tree.c24
-rw-r--r--kernel/bpf/core.c10
-rw-r--r--kernel/bpf/hashtab.c19
-rw-r--r--kernel/bpf/memalloc.c57
-rw-r--r--kernel/bpf/syscall.c19
-rw-r--r--kernel/bpf/trampoline.c85
-rw-r--r--kernel/bpf/verifier.c10
-rw-r--r--kernel/events/core.c315
-rw-r--r--kernel/events/ring_buffer.c7
-rw-r--r--kernel/futex/core.c10
-rw-r--r--kernel/irq/Kconfig12
-rw-r--r--kernel/irq/Makefile2
-rw-r--r--kernel/kprobes.c8
-rw-r--r--kernel/sysctl.c2
-rw-r--r--kernel/time/jiffies.c2
-rw-r--r--kernel/trace/fprobe.c26
16 files changed, 368 insertions, 240 deletions
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 1ed19b775912..f2e81be8265e 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -545,22 +545,38 @@ static void kill_rules(struct audit_context *context, struct audit_tree *tree)
{
struct audit_krule *rule, *next;
struct audit_entry *entry;
+ bool need_sync = false;
list_for_each_entry_safe(rule, next, &tree->rules, rlist) {
entry = container_of(rule, struct audit_entry, rule);
- list_del_init(&rule->rlist);
if (rule->tree) {
/* not a half-baked one */
audit_tree_log_remove_rule(context, rule);
- if (entry->rule.exe)
- audit_remove_mark(entry->rule.exe);
rule->tree = NULL;
list_del_rcu(&entry->list);
list_del(&entry->rule.list);
- call_rcu(&entry->rcu, audit_free_rule_rcu);
+ if (entry->rule.exe)
+ need_sync = true;
+ } else {
+ list_del_init(&rule->rlist);
}
}
+
+ if (list_empty(&tree->rules))
+ return;
+
+ if (need_sync)
+ synchronize_rcu();
+
+ list_for_each_entry_safe(rule, next, &tree->rules, rlist) {
+ entry = container_of(rule, struct audit_entry, rule);
+
+ list_del_init(&rule->rlist);
+ if (entry->rule.exe)
+ audit_remove_mark(entry->rule.exe);
+ call_rcu(&entry->rcu, audit_free_rule_rcu);
+ }
}
/*
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 2e3bf8113ae9..7f11555a5070 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1362,7 +1362,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
{
struct bpf_insn *to = to_buff;
u32 imm_rnd = get_random_u32();
- s16 off;
+ int off;
BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG);
BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG);
@@ -1438,6 +1438,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
off = from->off;
if (off < 0)
off -= 2;
+ if (off < S16_MIN)
+ return -ERANGE;
*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off);
@@ -1458,6 +1460,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
off = from->off;
if (off < 0)
off -= 2;
+ if (off < S16_MIN)
+ return -ERANGE;
*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
*to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
*to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX,
@@ -1606,7 +1610,9 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
if (!rewritten)
continue;
- if (env)
+ if (rewritten < 0)
+ tmp = ERR_PTR(rewritten);
+ else if (env)
tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
else
tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index f9464e566f10..13a2356c84cf 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -128,6 +128,7 @@ struct htab_elem {
struct htab_btf_record {
struct btf_record *record;
+ struct btf *btf;
u32 key_size;
};
@@ -497,8 +498,13 @@ static void htab_dtor_ctx_free(void *ctx)
{
struct htab_btf_record *hrec = ctx;
+ /*
+ * The duplicated record still points into the map BTF, so free it
+ * before dropping the reference that keeps that BTF alive.
+ */
btf_record_free(hrec->record);
- kfree(ctx);
+ btf_put(hrec->btf);
+ kfree(hrec);
}
static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
@@ -521,6 +527,15 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
kfree(hrec);
return err;
}
+ /*
+ * btf_record_dup() only acquires kernel and module BTF. Fields whose
+ * types live in the map BTF keep pointing into it: kptrs to local
+ * types refer to map->btf, and graph roots carry a value record owned
+ * by its struct meta table. The context can outlive the map when the
+ * allocator defers its teardown, so hold a reference of our own.
+ */
+ hrec->btf = map->btf;
+ btf_get(hrec->btf);
bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec);
return 0;
}
@@ -3359,8 +3374,10 @@ static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map,
}
if (do_delete) {
+ migrate_disable();
for (i = 0; i < total; i++)
rhtab_delete_elem(rhtab, del_elems[i], NULL, 0);
+ migrate_enable();
}
rcu_read_unlock();
diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index 8a8f088e83e6..15684d0fc883 100644
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -118,6 +118,11 @@ struct bpf_mem_cache {
struct llist_head free_by_rcu_ttrace;
struct llist_head waiting_for_gp_ttrace;
struct rcu_head rcu_ttrace;
+ /*
+ * 0 - idle
+ * 1 - __free_rcu() is queued
+ * 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since
+ */
atomic_t call_rcu_ttrace_in_progress;
raw_spinlock_t lock;
};
@@ -276,6 +281,8 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per
return cnt;
}
+static void __do_call_rcu_ttrace(struct bpf_mem_cache *c);
+
static void __free_rcu(struct rcu_head *head)
{
struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace);
@@ -285,7 +292,19 @@ static void __free_rcu(struct rcu_head *head)
llnode = llist_del_all(&c->waiting_for_gp_ttrace);
free_all(c, llnode, !!c->percpu_size);
- atomic_set(&c->call_rcu_ttrace_in_progress, 0);
+
+ /*
+ * do_call_rcu_ttrace() that ran while GP was in flight left its objects
+ * in free_by_rcu_ttrace. This cache may never free or alloc in bulk
+ * again, so start the next GP from here.
+ * 'c' can be freed as soon as call_rcu_ttrace_in_progress is zero.
+ */
+ if (atomic_cmpxchg(&c->call_rcu_ttrace_in_progress, 1, 0) == 1)
+ return;
+
+ /* Pairs with synchronize_rcu() in free_mem_alloc() */
+ guard(rcu)();
+ __do_call_rcu_ttrace(c);
}
static void enque_to_free(struct bpf_mem_cache *c, void *obj)
@@ -298,18 +317,15 @@ static void enque_to_free(struct bpf_mem_cache *c, void *obj)
llist_add(llnode, &c->free_by_rcu_ttrace);
}
-static void do_call_rcu_ttrace(struct bpf_mem_cache *c)
+static void __do_call_rcu_ttrace(struct bpf_mem_cache *c)
{
struct llist_node *llnode, *t;
- if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) {
- if (unlikely(READ_ONCE(c->draining))) {
- scoped_guard(raw_spinlock_irqsave, &c->lock)
- llnode = llist_del_all(&c->free_by_rcu_ttrace);
- free_all(c, llnode, !!c->percpu_size);
- }
- return;
- }
+ /*
+ * Must be done before llist_del_all(). Objects that it misses were
+ * added by do_call_rcu_ttrace() that will set 2 after this store.
+ */
+ atomic_set(&c->call_rcu_ttrace_in_progress, 1);
WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace));
llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace))
@@ -328,6 +344,22 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c)
call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu);
}
+static void do_call_rcu_ttrace(struct bpf_mem_cache *c)
+{
+ struct llist_node *llnode;
+
+ if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 2)) {
+ if (unlikely(READ_ONCE(c->draining))) {
+ scoped_guard(raw_spinlock_irqsave, &c->lock)
+ llnode = llist_del_all(&c->free_by_rcu_ttrace);
+ free_all(c, llnode, !!c->percpu_size);
+ }
+ return;
+ }
+
+ __do_call_rcu_ttrace(c);
+}
+
static void free_bulk(struct bpf_mem_cache *c)
{
struct bpf_mem_cache *tgt = c->tgt;
@@ -700,7 +732,12 @@ static void free_mem_alloc(struct bpf_mem_alloc *ma)
* to wait for the pending __free_by_rcu(), and __free_rcu(). RCU Tasks
* Trace grace period implies RCU grace period, so all __free_rcu don't
* need extra call_rcu() (and thus extra rcu_barrier() here).
+ *
+ * __free_rcu() queues itself again unless it sees 'draining'. After
+ * synchronize_rcu() it either did that already or will not do it, so
+ * rcu_barrier_tasks_trace() cannot miss it.
*/
+ synchronize_rcu();
rcu_barrier(); /* wait for __free_by_rcu */
rcu_barrier_tasks_trace(); /* wait for __free_rcu */
free_mem_alloc_no_barrier(ma);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 244a939b9d2d..96217b99399d 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2448,6 +2448,21 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu)
bpf_prog_free(aux->prog);
}
+/*
+ * Progs called from a trampoline can also be reached by a task that was
+ * preempted in the trampoline before the prog's enter helper took its RCU
+ * read lock, wait for those first.
+ */
+static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu)
+{
+ struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog;
+
+ if (prog->sleepable)
+ call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu);
+ else
+ call_rcu(rcu, __bpf_prog_put_rcu);
+}
+
static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
{
bpf_prog_kallsyms_del_all(prog);
@@ -2461,7 +2476,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
btf_put(prog->aux->attach_btf);
if (deferred) {
- if (prog->sleepable)
+ if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked)
+ call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks);
+ else if (prog->sleepable)
call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
else
call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 90b70ea0d370..bf4cb3dd444d 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)];
hlist_add_head(&tr->hlist_ip, head);
refcount_set(&tr->refcnt, 1);
+ INIT_LIST_HEAD(&tr->images);
for (i = 0; i < BPF_TRAMP_MAX; i++)
INIT_HLIST_HEAD(&tr->progs_hlist[i]);
out:
@@ -565,15 +566,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im)
arch_free_bpf_trampoline(im->image, im->size);
bpf_jit_uncharge_modmem(im->size);
percpu_ref_exit(&im->pcref);
+ kfree(im->skips);
kfree_rcu(im, rcu);
}
static void __bpf_tramp_image_put_deferred(struct work_struct *work)
{
struct bpf_tramp_image *im;
+ struct bpf_trampoline *tr;
im = container_of(work, struct bpf_tramp_image, work);
+ tr = im->tr;
+ trampoline_lock(tr);
+ list_del(&im->list);
+ trampoline_unlock(tr);
bpf_tramp_image_free(im);
+ bpf_trampoline_put(tr);
}
/* callback, fexit step 3 or fentry step 2 */
@@ -601,7 +609,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu)
struct bpf_tramp_image *im;
im = container_of(rcu, struct bpf_tramp_image, rcu);
- if (im->ip_after_call)
+ if (im->call_orig)
/* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */
percpu_ref_kill(&im->pcref);
else
@@ -621,9 +629,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
*
* The trampoline is unreachable before bpf_tramp_image_put().
*
- * First, patch the trampoline to avoid calling into fexit progs.
- * The progs will be freed even if the original function is still
- * executing or sleeping.
+ * Progs are patched out of the image when they are detached, see
+ * bpf_trampoline_skip_prog(), so they can be freed even if a task is
+ * still in the image.
* In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on
* first few asm instructions to execute and call into
* __bpf_tramp_enter->percpu_ref_get.
@@ -637,11 +645,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
* percpu_ref_kill will be waiting for. Hence the first
* call_rcu_tasks() is not necessary.
*/
- if (im->ip_after_call) {
- int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP,
- BPF_MOD_JUMP, NULL,
- im->ip_epilogue);
- WARN_ON(err);
+ if (im->call_orig) {
if (IS_ENABLED(CONFIG_TASKS_RCU))
call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
else
@@ -658,7 +662,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
}
-static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
+static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs)
{
struct bpf_tramp_image *im;
struct bpf_ksym *ksym;
@@ -669,6 +673,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
if (!im)
goto out;
+ im->skips = kzalloc_objs(*im->skips, nr_progs);
+ if (!im->skips)
+ goto out_free_im;
+
err = bpf_jit_charge_modmem(size);
if (err)
goto out_free_im;
@@ -695,6 +703,7 @@ out_free_image:
out_uncharge:
bpf_jit_uncharge_modmem(size);
out_free_im:
+ kfree(im->skips);
kfree(im);
out:
return ERR_PTR(err);
@@ -771,11 +780,12 @@ again:
goto out;
}
- im = bpf_tramp_image_alloc(tr->key, size);
+ im = bpf_tramp_image_alloc(tr->key, size, total);
if (IS_ERR(im)) {
err = PTR_ERR(im);
goto out;
}
+ im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG;
err = arch_prepare_bpf_trampoline(im, im->image, im->image + size,
&tr->func.model, tr->flags, tnodes,
@@ -806,8 +816,14 @@ again:
#endif
out_free:
- if (err)
+ if (err) {
bpf_tramp_image_free(im);
+ } else {
+ /* track the image until it is freed, for bpf_trampoline_skip_prog() */
+ refcount_inc(&tr->refcnt);
+ im->tr = tr;
+ list_add(&im->list, &tr->images);
+ }
out:
/* If any error happens, restore previous flags */
if (err)
@@ -907,6 +923,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr,
}
hlist_add_head(&node->tramp_hlist, prog_list);
+ node->link->prog->aux->tramp_linked = true;
if (kind == BPF_TRAMP_FSESSION) {
tr->progs_cnt[BPF_TRAMP_FENTRY]++;
fexit = fsession_exit(node);
@@ -920,6 +937,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr,
return 0;
}
+/*
+ * Patch the nop in front of a prog call to a jump over it. A task can be
+ * preempted anywhere in the image, so archs that need several instructions for
+ * a jump of any range patch a single near branch here instead.
+ */
+int __weak arch_bpf_trampoline_skip(void *nop, void *target)
+{
+ return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target);
+}
+
+/*
+ * prog was detached and can be freed, but tasks may still be running in images
+ * that call it, sleeping in an earlier prog for example. They can be in any
+ * image that is not freed yet, not only in cur_image, so patch all of them to
+ * jump over prog.
+ */
+static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog)
+{
+ struct bpf_tramp_image *im;
+ int i, err;
+
+ list_for_each_entry(im, &tr->images, list) {
+ for (i = 0; i < im->nr_skips; i++) {
+ struct bpf_tramp_skip *skip = &im->skips[i];
+
+ if (skip->prog != prog)
+ continue;
+ err = arch_bpf_trampoline_skip(skip->nop, skip->target);
+ WARN_ON_ONCE(err);
+ /* not a nop anymore, and prog's address can be reused */
+ skip->prog = NULL;
+ }
+ }
+}
+
static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr,
struct bpf_tramp_node *node)
{
@@ -937,6 +989,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr,
}
hlist_del_init(&node->tramp_hlist);
tr->progs_cnt[kind]--;
+ bpf_trampoline_skip_prog(tr, node->link->prog);
}
static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node,
@@ -1245,11 +1298,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
goto out;
- /* This code will be executed even when the last bpf_tramp_image
- * is alive. All progs are detached from the trampoline and the
- * trampoline image is patched with jmp into epilogue to skip
- * fexit progs. The fentry-only trampoline will be freed via
- * multiple rcu callbacks.
+ /*
+ * All progs are detached and the last image has been freed, images
+ * hold a reference on the trampoline until then.
*/
hlist_del(&tr->hlist_key);
hlist_del(&tr->hlist_ip);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 41b49c56e123..5f874979b8d7 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -14826,7 +14826,15 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
"Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range.");
return -EINVAL;
}
- reg_bounds_sync(dst_reg);
+ /*
+ * A packet pointer that keeps its id or range is checked against a
+ * range set from the checked pointer's umax, so var_off must not tighten
+ * its umax. r32 must still match var_off for reg_bounds_sanity_check().
+ */
+ if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0))
+ __update_reg32_bounds(dst_reg);
+ else
+ reg_bounds_sync(dst_reg);
bounds_ret = sanitize_check_bounds(env, insn, dst_reg);
if (bounds_ret == -EACCES)
return bounds_ret;
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 634d2ccbab82..7846d70be57f 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7626,9 +7626,11 @@ static void perf_sigtrap(struct perf_event *event)
{
/*
* Both perf_pending_task() and perf_pending_irq() can race with the
- * task exiting.
+ * task exiting or exec-ing. We can determine if such a race has
+ * occurred by checking if perf_event_exit_task(), which will set
+ * ctx->task to TASK_TOMBSTONE, has already been called.
*/
- if (current->flags & PF_EXITING)
+ if (event->ctx->task == TASK_TOMBSTONE)
return;
/*
@@ -8129,10 +8131,15 @@ static void __perf_event_header__init_id(struct perf_sample_data *data,
}
}
-void perf_event_header__init_id(struct perf_event_header *header,
- struct perf_sample_data *data,
- struct perf_event *event)
+void perf_event_header__init(struct perf_event_header *header,
+ struct perf_sample_data *data,
+ u32 type, u16 misc, u16 size,
+ struct perf_event *event)
{
+ header->type = type;
+ header->misc = misc;
+ header->size = size;
+
if (event->attr.sample_id_all) {
header->size += event->id_header_size;
__perf_event_header__init_id(data, event, event->attr.sample_type);
@@ -8969,17 +8976,16 @@ perf_event_read_event(struct perf_event *event,
struct perf_output_handle handle;
struct perf_sample_data sample;
struct perf_read_event read_event = {
- .header = {
- .type = PERF_RECORD_READ,
- .misc = 0,
- .size = sizeof(read_event) + event->read_size,
- },
.pid = perf_event_pid(event, task),
.tid = perf_event_tid(event, task),
};
int ret;
- perf_event_header__init_id(&read_event.header, &sample, event);
+ perf_event_header__init(&read_event.header, &sample,
+ PERF_RECORD_READ,
+ /* misc= */ 0,
+ sizeof(read_event) + event->read_size,
+ event);
ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
if (ret)
return;
@@ -9220,6 +9226,7 @@ struct perf_task_event {
u32 ptid;
u64 time;
} event_id;
+ int new;
};
static int perf_event_task_match(struct perf_event *event)
@@ -9236,17 +9243,21 @@ static void perf_event_task_output(struct perf_event *event,
struct perf_output_handle handle;
struct perf_sample_data sample;
struct task_struct *task = task_event->task;
- int ret, size = task_event->event_id.header.size;
+ int ret;
if (!perf_event_task_match(event))
return;
- perf_event_header__init_id(&task_event->event_id.header, &sample, event);
+ perf_event_header__init(&task_event->event_id.header, &sample,
+ task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
+ /* misc= */ 0,
+ sizeof(task_event->event_id),
+ event);
ret = perf_output_begin(&handle, &sample, event,
task_event->event_id.header.size);
if (ret)
- goto out;
+ return;
task_event->event_id.pid = perf_event_pid(event, task);
task_event->event_id.tid = perf_event_tid(event, task);
@@ -9268,8 +9279,6 @@ static void perf_event_task_output(struct perf_event *event,
perf_event__output_id_sample(event, &handle, &sample);
perf_output_end(&handle);
-out:
- task_event->event_id.header.size = size;
}
static void perf_event_task(struct task_struct *task,
@@ -9286,18 +9295,7 @@ static void perf_event_task(struct task_struct *task,
task_event = (struct perf_task_event){
.task = task,
.task_ctx = task_ctx,
- .event_id = {
- .header = {
- .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
- .misc = 0,
- .size = sizeof(task_event.event_id),
- },
- /* .pid */
- /* .ppid */
- /* .tid */
- /* .ptid */
- /* .time */
- },
+ .new = new,
};
perf_iterate_sb(perf_event_task_output,
@@ -9374,6 +9372,7 @@ struct perf_comm_event {
u32 pid;
u32 tid;
} event_id;
+ bool exec;
};
static int perf_event_comm_match(struct perf_event *event)
@@ -9387,18 +9386,21 @@ static void perf_event_comm_output(struct perf_event *event,
struct perf_comm_event *comm_event = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
- int size = comm_event->event_id.header.size;
int ret;
if (!perf_event_comm_match(event))
return;
- perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
+ perf_event_header__init(&comm_event->event_id.header, &sample,
+ PERF_RECORD_COMM,
+ comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
+ sizeof(comm_event->event_id) + comm_event->comm_size,
+ event);
ret = perf_output_begin(&handle, &sample, event,
comm_event->event_id.header.size);
if (ret)
- goto out;
+ return;
comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
@@ -9410,8 +9412,6 @@ static void perf_event_comm_output(struct perf_event *event,
perf_event__output_id_sample(event, &handle, &sample);
perf_output_end(&handle);
-out:
- comm_event->event_id.header.size = size;
}
static void perf_event_comm_event(struct perf_comm_event *comm_event)
@@ -9426,8 +9426,6 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
comm_event->comm = comm;
comm_event->comm_size = size;
- comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
-
perf_iterate_sb(perf_event_comm_output,
comm_event,
NULL);
@@ -9444,15 +9442,8 @@ void perf_event_comm(struct task_struct *task, bool exec)
.task = task,
/* .comm */
/* .comm_size */
- .event_id = {
- .header = {
- .type = PERF_RECORD_COMM,
- .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
- /* .size */
- },
- /* .pid */
- /* .tid */
- },
+ /* .event_id */
+ .exec = exec,
};
perf_event_comm_event(&comm_event);
@@ -9486,18 +9477,20 @@ static void perf_event_namespaces_output(struct perf_event *event,
struct perf_namespaces_event *namespaces_event = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
- u16 header_size = namespaces_event->event_id.header.size;
int ret;
if (!perf_event_namespaces_match(event))
return;
- perf_event_header__init_id(&namespaces_event->event_id.header,
- &sample, event);
+ perf_event_header__init(&namespaces_event->event_id.header, &sample,
+ PERF_RECORD_NAMESPACES,
+ /* misc= */ 0,
+ sizeof(namespaces_event->event_id),
+ event);
ret = perf_output_begin(&handle, &sample, event,
namespaces_event->event_id.header.size);
if (ret)
- goto out;
+ return;
namespaces_event->event_id.pid = perf_event_pid(event,
namespaces_event->task);
@@ -9509,8 +9502,6 @@ static void perf_event_namespaces_output(struct perf_event *event,
perf_event__output_id_sample(event, &handle, &sample);
perf_output_end(&handle);
-out:
- namespaces_event->event_id.header.size = header_size;
}
static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
@@ -9541,11 +9532,7 @@ void perf_event_namespaces(struct task_struct *task)
namespaces_event = (struct perf_namespaces_event){
.task = task,
.event_id = {
- .header = {
- .type = PERF_RECORD_NAMESPACES,
- .misc = 0,
- .size = sizeof(namespaces_event.event_id),
- },
+ /* .header */
/* .pid */
/* .tid */
.nr_namespaces = NR_NAMESPACES,
@@ -9613,18 +9600,19 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data)
struct perf_cgroup_event *cgroup_event = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
- u16 header_size = cgroup_event->event_id.header.size;
int ret;
+ u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size;
if (!perf_event_cgroup_match(event))
return;
- perf_event_header__init_id(&cgroup_event->event_id.header,
- &sample, event);
+ perf_event_header__init(&cgroup_event->event_id.header, &sample,
+ PERF_RECORD_CGROUP, /* misc= */ 0, size,
+ event);
ret = perf_output_begin(&handle, &sample, event,
cgroup_event->event_id.header.size);
if (ret)
- goto out;
+ return;
perf_output_put(&handle, cgroup_event->event_id);
__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
@@ -9632,8 +9620,6 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data)
perf_event__output_id_sample(event, &handle, &sample);
perf_output_end(&handle);
-out:
- cgroup_event->event_id.header.size = header_size;
}
static void perf_event_cgroup(struct cgroup *cgrp)
@@ -9648,11 +9634,6 @@ static void perf_event_cgroup(struct cgroup *cgrp)
cgroup_event = (struct perf_cgroup_event){
.event_id = {
- .header = {
- .type = PERF_RECORD_CGROUP,
- .misc = 0,
- .size = sizeof(cgroup_event.event_id),
- },
.id = cgroup_id(cgrp),
},
};
@@ -9675,7 +9656,6 @@ static void perf_event_cgroup(struct cgroup *cgrp)
while (!IS_ALIGNED(size, sizeof(u64)))
cgroup_event.path[size++] = '\0';
- cgroup_event.event_id.header.size += size;
cgroup_event.path_size = size;
perf_iterate_sb(perf_event_cgroup_output,
@@ -9731,38 +9711,40 @@ static void perf_event_mmap_output(struct perf_event *event,
struct perf_mmap_event *mmap_event = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
- int size = mmap_event->event_id.header.size;
- u32 type = mmap_event->event_id.header.type;
- bool use_build_id;
+ int size = sizeof(mmap_event->event_id) + mmap_event->file_size;
+ u32 type = PERF_RECORD_MMAP;
+ u16 misc = PERF_RECORD_MISC_USER;
+ bool use_build_id = false;
int ret;
if (!perf_event_mmap_match(event, data))
return;
if (event->attr.mmap2) {
- mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
- mmap_event->event_id.header.size += sizeof(mmap_event->maj);
- mmap_event->event_id.header.size += sizeof(mmap_event->min);
- mmap_event->event_id.header.size += sizeof(mmap_event->ino);
- mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
- mmap_event->event_id.header.size += sizeof(mmap_event->prot);
- mmap_event->event_id.header.size += sizeof(mmap_event->flags);
- }
-
- perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
+ type = PERF_RECORD_MMAP2;
+ size += sizeof(mmap_event->maj);
+ size += sizeof(mmap_event->min);
+ size += sizeof(mmap_event->ino);
+ size += sizeof(mmap_event->ino_generation);
+ size += sizeof(mmap_event->prot);
+ size += sizeof(mmap_event->flags);
+ use_build_id = event->attr.build_id && mmap_event->build_id_size;
+ if (use_build_id)
+ misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
+ }
+ if (!(mmap_event->vma->vm_flags & VM_EXEC))
+ misc |= PERF_RECORD_MISC_MMAP_DATA;
+
+ perf_event_header__init(&mmap_event->event_id.header, &sample,
+ type, misc, size, event);
ret = perf_output_begin(&handle, &sample, event,
mmap_event->event_id.header.size);
if (ret)
- goto out;
+ return;
mmap_event->event_id.pid = perf_event_pid(event, current);
mmap_event->event_id.tid = perf_event_tid(event, current);
- use_build_id = event->attr.build_id && mmap_event->build_id_size;
-
- if (event->attr.mmap2 && use_build_id)
- mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
-
perf_output_put(&handle, mmap_event->event_id);
if (event->attr.mmap2) {
@@ -9787,9 +9769,6 @@ static void perf_event_mmap_output(struct perf_event *event,
perf_event__output_id_sample(event, &handle, &sample);
perf_output_end(&handle);
-out:
- mmap_event->event_id.header.size = size;
- mmap_event->event_id.header.type = type;
}
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
@@ -9885,11 +9864,6 @@ got_name:
mmap_event->prot = prot;
mmap_event->flags = flags;
- if (!(vma->vm_flags & VM_EXEC))
- mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
-
- mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
-
if (atomic_read(&nr_build_id_events))
build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);
@@ -10009,11 +9983,7 @@ void perf_event_mmap(struct vm_area_struct *vma)
/* .file_name */
/* .file_size */
.event_id = {
- .header = {
- .type = PERF_RECORD_MMAP,
- .misc = PERF_RECORD_MISC_USER,
- /* .size */
- },
+ /* .header */
/* .pid */
/* .tid */
.start = vma->vm_start,
@@ -10043,18 +10013,15 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head,
u64 size;
u64 flags;
} rec = {
- .header = {
- .type = PERF_RECORD_AUX,
- .misc = 0,
- .size = sizeof(rec),
- },
.offset = head,
.size = size,
.flags = flags,
};
int ret;
- perf_event_header__init_id(&rec.header, &sample, event);
+ perf_event_header__init(&rec.header, &sample,
+ PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec),
+ event);
ret = perf_output_begin(&handle, &sample, event, rec.header.size);
if (ret)
@@ -10079,15 +10046,14 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost)
struct perf_event_header header;
u64 lost;
} lost_samples_event = {
- .header = {
- .type = PERF_RECORD_LOST_SAMPLES,
- .misc = 0,
- .size = sizeof(lost_samples_event),
- },
.lost = lost,
};
- perf_event_header__init_id(&lost_samples_event.header, &sample, event);
+ perf_event_header__init(&lost_samples_event.header, &sample,
+ PERF_RECORD_LOST_SAMPLES,
+ /* misc= */ 0,
+ sizeof(lost_samples_event),
+ event);
ret = perf_output_begin(&handle, &sample, event,
lost_samples_event.header.size);
@@ -10112,6 +10078,8 @@ struct perf_switch_event {
u32 next_prev_pid;
u32 next_prev_tid;
} event_id;
+ bool sched_in;
+ bool preempt;
};
static int perf_event_switch_match(struct perf_event *event)
@@ -10124,6 +10092,9 @@ static void perf_event_switch_output(struct perf_event *event, void *data)
struct perf_switch_event *se = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
+ u32 type;
+ u16 misc;
+ u16 size;
int ret;
if (!perf_event_switch_match(event))
@@ -10131,18 +10102,22 @@ static void perf_event_switch_output(struct perf_event *event, void *data)
/* Only CPU-wide events are allowed to see next/prev pid/tid */
if (event->ctx->task) {
- se->event_id.header.type = PERF_RECORD_SWITCH;
- se->event_id.header.size = sizeof(se->event_id.header);
+ type = PERF_RECORD_SWITCH;
+ size = sizeof(se->event_id.header);
} else {
- se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
- se->event_id.header.size = sizeof(se->event_id);
+ type = PERF_RECORD_SWITCH_CPU_WIDE;
+ size = sizeof(se->event_id);
se->event_id.next_prev_pid =
perf_event_pid(event, se->next_prev);
se->event_id.next_prev_tid =
perf_event_tid(event, se->next_prev);
}
+ misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT;
+ if (se->preempt)
+ misc |= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
- perf_event_header__init_id(&se->event_id.header, &sample, event);
+ perf_event_header__init(&se->event_id.header, &sample,
+ type, misc, size, event);
ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
if (ret)
@@ -10168,22 +10143,11 @@ static void perf_event_switch(struct task_struct *task,
switch_event = (struct perf_switch_event){
.task = task,
.next_prev = next_prev,
- .event_id = {
- .header = {
- /* .type */
- .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
- /* .size */
- },
- /* .next_prev_pid */
- /* .next_prev_tid */
- },
+ /* .event_id */
+ .sched_in = sched_in,
+ .preempt = !sched_in && task_is_runnable(task),
};
- if (!sched_in && task_is_runnable(task)) {
- switch_event.event_id.header.misc |=
- PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
- }
-
perf_iterate_sb(perf_event_switch_output, &switch_event, NULL);
}
@@ -10203,20 +10167,17 @@ static void perf_log_throttle(struct perf_event *event, int enable)
u64 id;
u64 stream_id;
} throttle_event = {
- .header = {
- .type = PERF_RECORD_THROTTLE,
- .misc = 0,
- .size = sizeof(throttle_event),
- },
.time = perf_event_clock(event),
.id = primary_event_id(event),
.stream_id = event->id,
};
- if (enable)
- throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
-
- perf_event_header__init_id(&throttle_event.header, &sample, event);
+ perf_event_header__init(&throttle_event.header, &sample,
+ enable ? PERF_RECORD_UNTHROTTLE
+ : PERF_RECORD_THROTTLE,
+ /* misc= */ 0,
+ sizeof(throttle_event),
+ event);
ret = perf_output_begin(&handle, &sample, event,
throttle_event.header.size);
@@ -10255,12 +10216,14 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data)
struct perf_output_handle handle;
struct perf_sample_data sample;
int ret;
+ u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len;
if (!perf_event_ksymbol_match(event))
return;
- perf_event_header__init_id(&ksymbol_event->event_id.header,
- &sample, event);
+ perf_event_header__init(&ksymbol_event->event_id.header, &sample,
+ PERF_RECORD_KSYMBOL, /* misc= */ 0, size,
+ event);
ret = perf_output_begin(&handle, &sample, event,
ksymbol_event->event_id.header.size);
if (ret)
@@ -10301,11 +10264,6 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
.name = name,
.name_len = name_len,
.event_id = {
- .header = {
- .type = PERF_RECORD_KSYMBOL,
- .size = sizeof(ksymbol_event.event_id) +
- name_len,
- },
.addr = addr,
.len = len,
.ksym_type = ksym_type,
@@ -10349,8 +10307,11 @@ static void perf_event_bpf_output(struct perf_event *event, void *data)
if (!perf_event_bpf_match(event))
return;
- perf_event_header__init_id(&bpf_event->event_id.header,
- &sample, event);
+ perf_event_header__init(&bpf_event->event_id.header, &sample,
+ PERF_RECORD_BPF_EVENT,
+ /* misc= */ 0,
+ sizeof(bpf_event->event_id),
+ event);
ret = perf_output_begin(&handle, &sample, event,
bpf_event->event_id.header.size);
if (ret)
@@ -10406,10 +10367,6 @@ void perf_event_bpf_event(struct bpf_prog *prog,
bpf_event = (struct perf_bpf_event){
.prog = prog,
.event_id = {
- .header = {
- .type = PERF_RECORD_BPF_EVENT,
- .size = sizeof(bpf_event.event_id),
- },
.type = type,
.flags = flags,
.id = prog->aux->id,
@@ -10437,18 +10394,23 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data)
struct perf_callchain_deferred_event *deferred_event = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
- int ret, size = deferred_event->event.header.size;
+ int ret;
+ u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64));
if (!event->attr.defer_output)
return;
/* XXX do we really need sample_id_all for this ??? */
- perf_event_header__init_id(&deferred_event->event.header, &sample, event);
+ perf_event_header__init(&deferred_event->event.header, &sample,
+ PERF_RECORD_CALLCHAIN_DEFERRED,
+ PERF_RECORD_MISC_USER,
+ size,
+ event);
ret = perf_output_begin(&handle, &sample, event,
deferred_event->event.header.size);
if (ret)
- goto out;
+ return;
perf_output_put(&handle, deferred_event->event);
for (int i = 0; i < deferred_event->trace->nr; i++) {
@@ -10458,8 +10420,6 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data)
perf_event__output_id_sample(event, &handle, &sample);
perf_output_end(&handle);
-out:
- deferred_event->event.header.size = size;
}
static void perf_unwind_deferred_callback(struct unwind_work *work,
@@ -10468,12 +10428,6 @@ static void perf_unwind_deferred_callback(struct unwind_work *work,
struct perf_callchain_deferred_event deferred_event = {
.trace = trace,
.event = {
- .header = {
- .type = PERF_RECORD_CALLCHAIN_DEFERRED,
- .misc = PERF_RECORD_MISC_USER,
- .size = sizeof(deferred_event.event) +
- (trace->nr * sizeof(u64)),
- },
.cookie = cookie,
.nr = trace->nr,
},
@@ -10485,7 +10439,8 @@ static void perf_unwind_deferred_callback(struct unwind_work *work,
struct perf_text_poke_event {
const void *old_bytes;
const void *new_bytes;
- size_t pad;
+ u16 tot;
+ u16 pad;
u16 old_len;
u16 new_len;
@@ -10506,13 +10461,18 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data)
struct perf_text_poke_event *text_poke_event = data;
struct perf_output_handle handle;
struct perf_sample_data sample;
+ u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad;
u64 padding = 0;
int ret;
if (!perf_event_text_poke_match(event))
return;
- perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
+ perf_event_header__init(&text_poke_event->event_id.header, &sample,
+ PERF_RECORD_TEXT_POKE,
+ PERF_RECORD_MISC_KERNEL,
+ size,
+ event);
ret = perf_output_begin(&handle, &sample, event,
text_poke_event->event_id.header.size);
@@ -10550,15 +10510,11 @@ void perf_event_text_poke(const void *addr, const void *old_bytes,
text_poke_event = (struct perf_text_poke_event){
.old_bytes = old_bytes,
.new_bytes = new_bytes,
+ .tot = tot,
.pad = pad,
.old_len = old_len,
.new_len = new_len,
.event_id = {
- .header = {
- .type = PERF_RECORD_TEXT_POKE,
- .misc = PERF_RECORD_MISC_KERNEL,
- .size = sizeof(text_poke_event.event_id) + tot + pad,
- },
.addr = (unsigned long)addr,
},
};
@@ -10589,13 +10545,12 @@ static void perf_log_itrace_start(struct perf_event *event)
event->attach_state & PERF_ATTACH_ITRACE)
return;
- rec.header.type = PERF_RECORD_ITRACE_START;
- rec.header.misc = 0;
- rec.header.size = sizeof(rec);
rec.pid = perf_event_pid(event, current);
rec.tid = perf_event_tid(event, current);
- perf_event_header__init_id(&rec.header, &sample, event);
+ perf_event_header__init(&rec.header, &sample,
+ PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec),
+ event);
ret = perf_output_begin(&handle, &sample, event, rec.header.size);
if (ret)
@@ -10620,12 +10575,10 @@ void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
if (event->parent)
event = event->parent;
- rec.header.type = PERF_RECORD_AUX_OUTPUT_HW_ID;
- rec.header.misc = 0;
- rec.header.size = sizeof(rec);
- rec.hw_id = hw_id;
-
- perf_event_header__init_id(&rec.header, &sample, event);
+ rec.hw_id = hw_id;
+ perf_event_header__init(&rec.header, &sample,
+ PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0,
+ sizeof(rec), event);
ret = perf_output_begin(&handle, &sample, event, rec.header.size);
if (ret)
@@ -13953,7 +13906,7 @@ SYSCALL_DEFINE5(perf_event_open,
if (err)
return err;
- if (!attr.exclude_kernel ||
+ if (!attr.exclude_kernel || attr.text_poke ||
((attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
!attr.exclude_callchain_kernel)) {
err = perf_allow_kernel();
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 1b1ffe0533e5..13bd42e5493d 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -246,14 +246,13 @@ __perf_output_begin(struct perf_output_handle *handle,
handle->size = (1UL << page_shift) - offset;
if (unlikely(have_lost)) {
- lost_event.header.size = sizeof(lost_event);
- lost_event.header.type = PERF_RECORD_LOST;
- lost_event.header.misc = 0;
lost_event.id = event->id;
lost_event.lost = local_xchg(&rb->lost, 0);
/* XXX mostly redundant; @data is already fully initializes */
- perf_event_header__init_id(&lost_event.header, data, event);
+ perf_event_header__init(&lost_event.header, data,
+ PERF_RECORD_LOST, /* misc= */ 0,
+ sizeof(lost_event), event);
perf_output_put(handle, lost_event);
perf_event__output_id_sample(event, handle, data);
}
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index a061f54b606d..095f9fe440e3 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -213,10 +213,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash *
futex_rehash_private(fph, new);
}
new->state = FR_PERCPU;
- scoped_guard(rcu) {
- mmph->batches = get_state_synchronize_rcu();
- rcu_assign_pointer(mmph->hash, new);
- }
+ rcu_assign_pointer(mmph->hash, new);
+ /*
+ * mmph->batches must reference a grace period which started after
+ * mmph->hash was assigned. See futex_ref_drop().
+ */
+ mmph->batches = get_state_synchronize_rcu();
kvfree_rcu(fph, rcu);
return true;
}
diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig
index 05cba4e16dad..6923f37eaab4 100644
--- a/kernel/irq/Kconfig
+++ b/kernel/irq/Kconfig
@@ -150,6 +150,18 @@ config IRQ_KUNIT_TEST
If unsure, say N.
+config REFCOUNT_INTERRUPT_KUNIT_TEST
+ tristate "Test refcounted interrupt enable/disable" if !KUNIT_ALL_TESTS
+ depends on KUNIT
+ default KUNIT_ALL_TESTS
+ help
+ This builds the kunit tests for the refcounted interrupt
+ infrastructure. It verifies the correctness of single, nested,
+ and multiple interrupt enable/disable state changes and ensures
+ that the underlying reference counting mechanisms work as expected.
+
+ If unsure, say N.
+
endmenu
config GENERIC_IRQ_MULTI_HANDLER
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 44c4d6fc502a..0e5df962a149 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -16,4 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o
obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o
obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o
obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o
-obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o
+obj-$(CONFIG_REFCOUNT_INTERRUPT_KUNIT_TEST) += refcount_interrupt_test.o
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 4edd8ca5c657..e787e4948c8b 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -496,14 +496,16 @@ static bool kprobe_queued(struct kprobe *p)
static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
{
int i;
- struct kprobe *p = NULL;
+ struct kprobe *p;
struct optimized_kprobe *op;
/* Don't check i == 0, since that is a breakpoint case. */
- for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
+ for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) {
p = get_kprobe(addr - i);
+ /* A disabled probe can have prepared, but inactive, optinsns. */
+ if (!p || !kprobe_optready(p) || kprobe_disarmed(p))
+ continue;
- if (p && kprobe_optready(p)) {
op = container_of(p, struct optimized_kprobe, kp);
if (arch_within_optimized_kprobe(op, addr))
return p;
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index f7b75985d542..38597f26b34c 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -483,7 +483,7 @@ int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp,
if (val < 0) {
*negp = true;
- *u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val;
+ *u_ptr = k_ptr_op ? k_ptr_op(-(ulong)val) : -(ulong)val;
} else {
*negp = false;
*u_ptr = k_ptr_op ? k_ptr_op((ulong)val) : (ulong) val;
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 80c354811538..9b3487d40cd6 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -101,6 +101,8 @@ void __init register_refined_jiffies(long cycles_per_second)
#ifdef CONFIG_SYSCTL
static ulong mult_hz(const ulong val)
{
+ if (val >= ULONG_MAX / HZ)
+ return ULONG_MAX;
return val * HZ;
}
diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index 9f2d98181779..da286619c5d8 100644
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -47,6 +47,10 @@ static struct rhltable fprobe_ip_table;
static DEFINE_MUTEX(fprobe_mutex);
static struct fgraph_ops fprobe_graph_ops;
+DEFINE_LOCK_GUARD_0(rcu_sched_notrace,
+ rcu_read_lock_sched_notrace(),
+ rcu_read_unlock_sched_notrace())
+
static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed)
{
return hash_ptr(*(unsigned long **)data, 32);
@@ -329,16 +333,14 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip,
struct fprobe *fp;
int bit;
+ if (!rcu_is_watching())
+ return;
+
bit = ftrace_test_recursion_trylock(ip, parent_ip);
if (bit < 0)
return;
- /*
- * ftrace_test_recursion_trylock() disables preemption, but
- * rhltable_lookup() checks whether rcu_read_lcok is held.
- * So we take rcu_read_lock() here.
- */
- rcu_read_lock();
+ guard(rcu_sched_notrace)();
head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params);
rhl_for_each_entry_rcu(node, pos, head, hlist) {
@@ -353,7 +355,6 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip,
else
__fprobe_handler(ip, parent_ip, fp, fregs, NULL);
}
- rcu_read_unlock();
ftrace_test_recursion_unlock(bit);
}
NOKPROBE_SYMBOL(fprobe_ftrace_entry);
@@ -567,10 +568,13 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops
struct fprobe *fp;
int used, ret;
+ if (!rcu_is_watching())
+ return 0;
+
if (WARN_ON_ONCE(!fregs))
return 0;
- guard(rcu)();
+ guard(rcu_sched_notrace)();
head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params);
reserved_words = 0;
rhl_for_each_entry_rcu(node, pos, head, hlist) {
@@ -665,13 +669,16 @@ static void fprobe_return(struct ftrace_graph_ret *trace,
int size, curr;
int size_words;
+ if (!rcu_is_watching())
+ return;
+
fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size);
if (WARN_ON_ONCE(!fgraph_data))
return;
size_words = SIZE_IN_LONG(size);
ret_ip = ftrace_regs_get_instruction_pointer(fregs);
- preempt_disable_notrace();
+ guard(rcu_sched_notrace)();
curr = 0;
while (size_words > curr) {
@@ -687,7 +694,6 @@ static void fprobe_return(struct ftrace_graph_ret *trace,
}
curr += size;
}
- preempt_enable_notrace();
}
NOKPROBE_SYMBOL(fprobe_return);