diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/hashtab.c | 22 |
1 files changed, 16 insertions, 6 deletions
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9d4791ee1741..66ebb5a32807 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1279,11 +1279,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, if (prealloc) { if (old_elem) { /* if we're updating the existing element, - * use per-cpu extra elems to avoid freelist_pop/push + * use per-cpu extra elems to avoid freelist_pop/push. + * The caller puts old_elem into the slot once it is + * unlinked. Until then the slot is empty for an NMI + * or a kprobe that updates this map on this cpu. */ pl_new = this_cpu_ptr(htab->extra_elems); - l_new = *pl_new; - *pl_new = old_elem; + l_new = READ_ONCE(*pl_new); + do { + if (!l_new) + return ERR_PTR(-EBUSY); + } while (!try_cmpxchg_local(pl_new, &l_new, NULL)); } else { struct pcpu_freelist_node *l; @@ -1459,11 +1465,15 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, cancel - * its reusable special fields before it is available for reuse. + /* + * l_old replaces the extra element that alloc_htab_elem() took. + * Cancel its reusable special fields first: once it is in the + * slot, the next update on this cpu can reuse it. */ - if (htab_is_prealloc(htab)) + if (htab_is_prealloc(htab)) { check_and_cancel_fields(htab, l_old); + WRITE_ONCE(*this_cpu_ptr(htab->extra_elems), l_old); + } } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) |