summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig.debug28
-rw-r--r--lib/Makefile1
-rw-r--r--lib/alloc_tag.c1029
-rw-r--r--lib/codetag.c18
-rw-r--r--lib/maple_tree.c361
-rw-r--r--lib/percpu-refcount.c2
-rw-r--r--lib/radix-tree.c7
-rw-r--r--lib/test_hmm.c109
-rw-r--r--lib/test_hmm_uapi.h1
9 files changed, 380 insertions, 1176 deletions
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 1244dcac2294..b82515cde538 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1048,34 +1048,6 @@ config CODE_TAGGING
bool
select KALLSYMS
-config MEM_ALLOC_PROFILING
- bool "Enable memory allocation profiling"
- default n
- depends on MMU
- depends on PROC_FS
- depends on !DEBUG_FORCE_WEAK_PER_CPU
- select CODE_TAGGING
- select PAGE_EXTENSION
- select SLAB_OBJ_EXT
- help
- Track allocation source code and record total allocation size
- initiated at that code location. The mechanism can be used to track
- memory leaks with a low performance and memory impact.
-
-config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
- bool "Enable memory allocation profiling by default"
- default y
- depends on MEM_ALLOC_PROFILING
-
-config MEM_ALLOC_PROFILING_DEBUG
- bool "Memory allocation profiler debugging"
- default n
- depends on MEM_ALLOC_PROFILING
- select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
- help
- Adds warnings with helpful error messages for memory allocation
- profiling.
-
source "lib/Kconfig.kasan"
source "lib/Kconfig.kfence"
source "lib/Kconfig.kmsan"
diff --git a/lib/Makefile b/lib/Makefile
index 7f75cc6edf94..531a0be88062 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -196,7 +196,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
obj-$(CONFIG_CODE_TAGGING) += codetag.o
-obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o
lib-$(CONFIG_GENERIC_BUG) += bug.o
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
deleted file mode 100644
index e5b218176c5a..000000000000
--- a/lib/alloc_tag.c
+++ /dev/null
@@ -1,1029 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <linux/alloc_tag.h>
-#include <linux/execmem.h>
-#include <linux/fs.h>
-#include <linux/gfp.h>
-#include <linux/kallsyms.h>
-#include <linux/module.h>
-#include <linux/page_ext.h>
-#include <linux/pgalloc_tag.h>
-#include <linux/proc_fs.h>
-#include <linux/rcupdate.h>
-#include <linux/seq_buf.h>
-#include <linux/seq_file.h>
-#include <linux/string_choices.h>
-#include <linux/vmalloc.h>
-#include <linux/kmemleak.h>
-
-#define ALLOCINFO_FILE_NAME "allocinfo"
-#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
-#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
-#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
-static bool mem_profiling_support = true;
-#else
-static bool mem_profiling_support;
-#endif
-
-/*
- * Memory allocation profiling is permanently disabled and cannot be enabled.
- * Must be called after setup_early_mem_profiling().
- */
-bool mem_alloc_profiling_permanently_disabled(void)
-{
- return !mem_profiling_support;
-}
-
-static struct codetag_type *alloc_tag_cttype;
-
-#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
-DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
-EXPORT_SYMBOL(_shared_alloc_tag);
-#endif
-
-DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
- mem_alloc_profiling_key);
-EXPORT_SYMBOL(mem_alloc_profiling_key);
-
-DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
-
-struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
-unsigned long alloc_tag_ref_mask;
-int alloc_tag_ref_offs;
-
-struct allocinfo_private {
- struct codetag_iterator iter;
- struct codetag_iterator reported_iter;
- bool print_header;
-};
-
-static void *allocinfo_start(struct seq_file *m, loff_t *pos)
-{
- struct allocinfo_private *priv;
- loff_t node = *pos;
-
- priv = (struct allocinfo_private *)m->private;
- codetag_lock_module_list(alloc_tag_cttype);
- if (node == 0) {
- priv->print_header = true;
- priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
- } else {
- priv->iter = priv->reported_iter;
- }
- codetag_next_ct(&priv->iter);
- return priv->iter.ct ? priv : NULL;
-}
-
-static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
-{
- struct allocinfo_private *priv = (struct allocinfo_private *)arg;
- struct codetag *ct;
-
- priv->reported_iter = priv->iter;
- ct = codetag_next_ct(&priv->iter);
- (*pos)++;
- if (!ct)
- return NULL;
-
- return priv;
-}
-
-static void allocinfo_stop(struct seq_file *m, void *arg)
-{
- codetag_unlock_module_list(alloc_tag_cttype);
-}
-
-static void print_allocinfo_header(struct seq_buf *buf)
-{
- /* Output format version, so we can change it. */
- seq_buf_printf(buf, "allocinfo - version: 2.0\n");
- seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
-}
-
-static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
-{
- struct alloc_tag *tag = ct_to_alloc_tag(ct);
- struct alloc_tag_counters counter = alloc_tag_read(tag);
- s64 bytes = counter.bytes;
-
- seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
- codetag_to_text(out, ct);
- if (unlikely(alloc_tag_is_inaccurate(tag)))
- seq_buf_printf(out, " accurate:no");
- seq_buf_putc(out, ' ');
- seq_buf_putc(out, '\n');
-}
-
-static int allocinfo_show(struct seq_file *m, void *arg)
-{
- struct allocinfo_private *priv = (struct allocinfo_private *)arg;
- char *bufp;
- size_t n = seq_get_buf(m, &bufp);
- struct seq_buf buf;
-
- seq_buf_init(&buf, bufp, n);
- if (priv->print_header) {
- print_allocinfo_header(&buf);
- priv->print_header = false;
- }
- alloc_tag_to_text(&buf, priv->iter.ct);
- seq_commit(m, seq_buf_used(&buf));
- return 0;
-}
-
-static const struct seq_operations allocinfo_seq_op = {
- .start = allocinfo_start,
- .next = allocinfo_next,
- .stop = allocinfo_stop,
- .show = allocinfo_show,
-};
-
-size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
-{
- struct codetag_iterator iter;
- struct codetag *ct;
- struct codetag_bytes n;
- unsigned int i, nr = 0;
-
- if (IS_ERR_OR_NULL(alloc_tag_cttype))
- return 0;
-
- if (can_sleep)
- codetag_lock_module_list(alloc_tag_cttype);
- else if (!codetag_trylock_module_list(alloc_tag_cttype))
- return 0;
-
- iter = codetag_get_ct_iter(alloc_tag_cttype);
- while ((ct = codetag_next_ct(&iter))) {
- struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
-
- n.ct = ct;
- n.bytes = counter.bytes;
-
- for (i = 0; i < nr; i++)
- if (n.bytes > tags[i].bytes)
- break;
-
- if (i < count) {
- nr -= nr == count;
- memmove(&tags[i + 1],
- &tags[i],
- sizeof(tags[0]) * (nr - i));
- nr++;
- tags[i] = n;
- }
- }
-
- codetag_unlock_module_list(alloc_tag_cttype);
-
- return nr;
-}
-
-void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
-{
- int i;
- struct alloc_tag *tag;
- unsigned int nr_pages = 1 << new_order;
-
- if (!mem_alloc_profiling_enabled())
- return;
-
- tag = __pgalloc_tag_get(&folio->page);
- if (!tag)
- return;
-
- for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
- union pgtag_ref_handle handle;
- union codetag_ref ref;
-
- if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
- /* Set new reference to point to the original tag */
- alloc_tag_ref_set(&ref, tag);
- update_page_tag_ref(handle, &ref);
- put_page_tag_ref(handle);
- }
- }
-}
-
-void pgalloc_tag_swap(struct folio *new, struct folio *old)
-{
- union pgtag_ref_handle handle_old, handle_new;
- union codetag_ref ref_old, ref_new;
- struct alloc_tag *tag_old, *tag_new;
-
- if (!mem_alloc_profiling_enabled())
- return;
-
- tag_old = __pgalloc_tag_get(&old->page);
- if (!tag_old)
- return;
- tag_new = __pgalloc_tag_get(&new->page);
- if (!tag_new)
- return;
-
- if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
- return;
- if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
- put_page_tag_ref(handle_old);
- return;
- }
-
- /*
- * Clear tag references to avoid debug warning when using
- * __alloc_tag_ref_set() with non-empty reference.
- */
- set_codetag_empty(&ref_old);
- set_codetag_empty(&ref_new);
-
- /* swap tags */
- __alloc_tag_ref_set(&ref_old, tag_new);
- update_page_tag_ref(handle_old, &ref_old);
- __alloc_tag_ref_set(&ref_new, tag_old);
- update_page_tag_ref(handle_new, &ref_new);
-
- put_page_tag_ref(handle_old);
- put_page_tag_ref(handle_new);
-}
-
-static void shutdown_mem_profiling(bool remove_file)
-{
- if (mem_alloc_profiling_enabled())
- static_branch_disable(&mem_alloc_profiling_key);
-
- if (!mem_profiling_support)
- return;
-
- if (remove_file)
- remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
- mem_profiling_support = false;
-}
-
-void __init alloc_tag_sec_init(void)
-{
- struct alloc_tag *last_codetag;
-
- if (!mem_profiling_support)
- return;
-
- if (!static_key_enabled(&mem_profiling_compressed))
- return;
-
- kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
- SECTION_START(ALLOC_TAG_SECTION_NAME));
- last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
- SECTION_STOP(ALLOC_TAG_SECTION_NAME));
- kernel_tags.count = last_codetag - kernel_tags.first_tag;
-
- /* Check if kernel tags fit into page flags */
- if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
- shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
- pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
- kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
- return;
- }
-
- alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
- alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
- pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
- NR_UNUSED_PAGEFLAG_BITS);
-}
-
-#ifdef CONFIG_MODULES
-
-static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
-static struct vm_struct *vm_module_tags;
-/* A dummy object used to indicate an unloaded module */
-static struct module unloaded_mod;
-/* A dummy object used to indicate a module prepended area */
-static struct module prepend_mod;
-
-struct alloc_tag_module_section module_tags;
-
-static inline unsigned long alloc_tag_align(unsigned long val)
-{
- if (!static_key_enabled(&mem_profiling_compressed)) {
- /* No alignment requirements when we are not indexing the tags */
- return val;
- }
-
- if (val % sizeof(struct alloc_tag) == 0)
- return val;
- return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
-}
-
-static bool ensure_alignment(unsigned long align, unsigned int *prepend)
-{
- if (!static_key_enabled(&mem_profiling_compressed)) {
- /* No alignment requirements when we are not indexing the tags */
- return true;
- }
-
- /*
- * If alloc_tag size is not a multiple of required alignment, tag
- * indexing does not work.
- */
- if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
- return false;
-
- /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
- if (*prepend)
- *prepend = alloc_tag_align(*prepend);
-
- return true;
-}
-
-static inline bool tags_addressable(void)
-{
- unsigned long tag_idx_count;
-
- if (!static_key_enabled(&mem_profiling_compressed))
- return true; /* with page_ext tags are always addressable */
-
- tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
- module_tags.size / sizeof(struct alloc_tag);
-
- return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
-}
-
-static bool needs_section_mem(struct module *mod, unsigned long size)
-{
- if (!mem_profiling_support)
- return false;
-
- return size >= sizeof(struct alloc_tag);
-}
-
-static bool clean_unused_counters(struct alloc_tag *start_tag,
- struct alloc_tag *end_tag)
-{
- struct alloc_tag *tag;
- bool ret = true;
-
- for (tag = start_tag; tag <= end_tag; tag++) {
- struct alloc_tag_counters counter;
-
- if (!tag->counters)
- continue;
-
- counter = alloc_tag_read(tag);
- if (!counter.bytes) {
- free_percpu(tag->counters);
- tag->counters = NULL;
- } else {
- ret = false;
- }
- }
-
- return ret;
-}
-
-/* Called with mod_area_mt locked */
-static void clean_unused_module_areas_locked(void)
-{
- MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
- struct module *val;
-
- mas_for_each(&mas, val, module_tags.size) {
- struct alloc_tag *start_tag;
- struct alloc_tag *end_tag;
-
- if (val != &unloaded_mod)
- continue;
-
- /* Release area if all tags are unused */
- start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
- end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
- if (clean_unused_counters(start_tag, end_tag))
- mas_erase(&mas);
- }
-}
-
-/* Called with mod_area_mt locked */
-static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
- unsigned long size, unsigned int prepend, unsigned long align)
-{
- bool cleanup_done = false;
-
-repeat:
- /* Try finding exact size and hope the start is aligned */
- if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
- if (IS_ALIGNED(mas->index + prepend, align))
- return true;
-
- /* Try finding larger area to align later */
- mas_reset(mas);
- if (!mas_empty_area(mas, 0, section_size - 1,
- size + prepend + align - 1))
- return true;
- }
-
- /* No free area, try cleanup stale data and repeat the search once */
- if (!cleanup_done) {
- clean_unused_module_areas_locked();
- cleanup_done = true;
- mas_reset(mas);
- goto repeat;
- }
-
- return false;
-}
-
-static int vm_module_tags_populate(void)
-{
- unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
- (vm_module_tags->nr_pages << PAGE_SHIFT);
- unsigned long new_end = module_tags.start_addr + module_tags.size;
-
- if (phys_end < new_end) {
- struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
- unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
- unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
- unsigned long more_pages;
- unsigned long nr = 0;
-
- more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
- while (nr < more_pages) {
- unsigned long allocated;
-
- allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
- NUMA_NO_NODE, more_pages - nr, next_page + nr);
-
- if (!allocated)
- break;
- nr += allocated;
- }
-
- if (nr < more_pages ||
- vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
- next_page, PAGE_SHIFT) < 0) {
- release_pages_arg arg = { .pages = next_page };
-
- /* Clean up and error out */
- release_pages(arg, nr);
- return -ENOMEM;
- }
-
- vm_module_tags->nr_pages += nr;
-
- /*
- * Kasan allocates 1 byte of shadow for every 8 bytes of data.
- * When kasan_alloc_module_shadow allocates shadow memory,
- * its unit of allocation is a page.
- * Therefore, here we need to align to MODULE_ALIGN.
- */
- if (old_shadow_end < new_shadow_end)
- kasan_alloc_module_shadow((void *)old_shadow_end,
- new_shadow_end - old_shadow_end,
- GFP_KERNEL);
- }
-
- /*
- * Mark the pages as accessible, now that they are mapped.
- * With hardware tag-based KASAN, marking is skipped for
- * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
- */
- kasan_unpoison_vmalloc((void *)module_tags.start_addr,
- new_end - module_tags.start_addr,
- KASAN_VMALLOC_PROT_NORMAL);
-
- return 0;
-}
-
-static void *reserve_module_tags(struct module *mod, unsigned long size,
- unsigned int prepend, unsigned long align)
-{
- unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
- MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
- unsigned long offset;
- void *ret = NULL;
-
- /* If no tags return error */
- if (size < sizeof(struct alloc_tag))
- return ERR_PTR(-EINVAL);
-
- /*
- * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
- * align 0 or 1 means no alignment, to simplify set to 1.
- */
- if (!align)
- align = 1;
-
- if (!ensure_alignment(align, &prepend)) {
- shutdown_mem_profiling(true);
- pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
- mod->name, align);
- return ERR_PTR(-EINVAL);
- }
-
- mas_lock(&mas);
- if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
- ret = ERR_PTR(-ENOMEM);
- goto unlock;
- }
-
- /* Mark found area as reserved */
- offset = mas.index;
- offset += prepend;
- offset = ALIGN(offset, align);
- if (offset != mas.index) {
- unsigned long pad_start = mas.index;
-
- mas.last = offset - 1;
- mas_store(&mas, &prepend_mod);
- if (mas_is_err(&mas)) {
- ret = ERR_PTR(xa_err(mas.node));
- goto unlock;
- }
- mas.index = offset;
- mas.last = offset + size - 1;
- mas_store(&mas, mod);
- if (mas_is_err(&mas)) {
- mas.index = pad_start;
- mas_erase(&mas);
- ret = ERR_PTR(xa_err(mas.node));
- }
- } else {
- mas.last = offset + size - 1;
- mas_store(&mas, mod);
- if (mas_is_err(&mas))
- ret = ERR_PTR(xa_err(mas.node));
- }
-unlock:
- mas_unlock(&mas);
-
- if (IS_ERR(ret))
- return ret;
-
- if (module_tags.size < offset + size) {
- int grow_res;
-
- module_tags.size = offset + size;
- if (mem_alloc_profiling_enabled() && !tags_addressable()) {
- shutdown_mem_profiling(true);
- pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
- mod->name, NR_UNUSED_PAGEFLAG_BITS);
- }
-
- grow_res = vm_module_tags_populate();
- if (grow_res) {
- shutdown_mem_profiling(true);
- pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
- mod->name);
- return ERR_PTR(grow_res);
- }
- }
-
- return (struct alloc_tag *)(module_tags.start_addr + offset);
-}
-
-static void release_module_tags(struct module *mod, bool used)
-{
- MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
- struct alloc_tag *start_tag;
- struct alloc_tag *end_tag;
- struct module *val;
-
- mas_lock(&mas);
- mas_for_each_rev(&mas, val, 0)
- if (val == mod)
- break;
-
- if (!val) /* module not found */
- goto out;
-
- if (!used)
- goto release_area;
-
- start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
- end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
- if (!clean_unused_counters(start_tag, end_tag)) {
- struct alloc_tag *tag;
-
- for (tag = start_tag; tag <= end_tag; tag++) {
- struct alloc_tag_counters counter;
-
- if (!tag->counters)
- continue;
-
- counter = alloc_tag_read(tag);
- pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
- tag->ct.filename, tag->ct.lineno, tag->ct.modname,
- tag->ct.function, counter.bytes);
- }
- } else {
- used = false;
- }
-release_area:
- mas_store(&mas, used ? &unloaded_mod : NULL);
- val = mas_prev_range(&mas, 0);
- if (val == &prepend_mod)
- mas_store(&mas, NULL);
-out:
- mas_unlock(&mas);
-}
-
-static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
-{
- /* Allocate module alloc_tag percpu counters */
- struct alloc_tag *start_tag;
- struct alloc_tag *stop_tag;
- struct alloc_tag *tag;
-
- /* percpu counters for core allocations are already statically allocated */
- if (!mod)
- return 0;
-
- start_tag = ct_to_alloc_tag(start);
- stop_tag = ct_to_alloc_tag(stop);
- for (tag = start_tag; tag < stop_tag; tag++) {
- WARN_ON(tag->counters);
- tag->counters = alloc_percpu(struct alloc_tag_counters);
- if (!tag->counters) {
- while (--tag >= start_tag) {
- free_percpu(tag->counters);
- tag->counters = NULL;
- }
- pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
- mod->name);
- return -ENOMEM;
- }
-
- /*
- * Avoid a kmemleak false positive. The pointer to the counters is stored
- * in the alloc_tag section of the module and cannot be directly accessed.
- */
- kmemleak_ignore_percpu(tag->counters);
- }
- return 0;
-}
-
-static void replace_module(struct module *mod, struct module *new_mod)
-{
- MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
- struct module *val;
-
- mas_lock(&mas);
- mas_for_each(&mas, val, module_tags.size) {
- if (val != mod)
- continue;
-
- mas_store_gfp(&mas, new_mod, GFP_KERNEL);
- break;
- }
- mas_unlock(&mas);
-}
-
-static int __init alloc_mod_tags_mem(void)
-{
- /* Map space to copy allocation tags */
- vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
- if (!vm_module_tags) {
- pr_err("Failed to map %lu bytes for module allocation tags\n",
- MODULE_ALLOC_TAG_VMAP_SIZE);
- module_tags.start_addr = 0;
- return -ENOMEM;
- }
-
- vm_module_tags->pages = kmalloc_objs(struct page *,
- get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
- GFP_KERNEL | __GFP_ZERO);
- if (!vm_module_tags->pages) {
- free_vm_area(vm_module_tags);
- return -ENOMEM;
- }
-
- module_tags.start_addr = (unsigned long)vm_module_tags->addr;
- module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
- /* Ensure the base is alloc_tag aligned when required for indexing */
- module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
-
- return 0;
-}
-
-static void __init free_mod_tags_mem(void)
-{
- release_pages_arg arg = { .pages = vm_module_tags->pages };
-
- module_tags.start_addr = 0;
- release_pages(arg, vm_module_tags->nr_pages);
- kfree(vm_module_tags->pages);
- free_vm_area(vm_module_tags);
-}
-
-#else /* CONFIG_MODULES */
-
-static inline int alloc_mod_tags_mem(void) { return 0; }
-static inline void free_mod_tags_mem(void) {}
-
-#endif /* CONFIG_MODULES */
-
-/* See: Documentation/mm/allocation-profiling.rst */
-static int __init setup_early_mem_profiling(char *str)
-{
- bool compressed = false;
- bool enable;
-
- if (!str || !str[0])
- return -EINVAL;
-
- if (!strncmp(str, "never", 5)) {
- enable = false;
- mem_profiling_support = false;
- pr_info("Memory allocation profiling is disabled!\n");
- } else {
- char *token = strsep(&str, ",");
-
- if (kstrtobool(token, &enable))
- return -EINVAL;
-
- if (str) {
-
- if (strcmp(str, "compressed"))
- return -EINVAL;
-
- compressed = true;
- }
- mem_profiling_support = true;
- pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
- compressed ? "with" : "without", str_on_off(enable));
- }
-
- if (enable != mem_alloc_profiling_enabled()) {
- if (enable)
- static_branch_enable(&mem_alloc_profiling_key);
- else
- static_branch_disable(&mem_alloc_profiling_key);
- }
- if (compressed != static_key_enabled(&mem_profiling_compressed)) {
- if (compressed)
- static_branch_enable(&mem_profiling_compressed);
- else
- static_branch_disable(&mem_profiling_compressed);
- }
-
- return 0;
-}
-early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
-
-static __init bool need_page_alloc_tagging(void)
-{
- if (static_key_enabled(&mem_profiling_compressed))
- return false;
-
- return mem_profiling_support;
-}
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-/*
- * Track page allocations before page_ext is initialized.
- * Some pages are allocated before page_ext becomes available, leaving
- * their codetag uninitialized. Track these early PFNs so we can clear
- * their codetag refs later to avoid warnings when they are freed.
- *
- * Each page is cast to a pfn_pool: the first few bytes hold metadata
- * (next pointer and slot count), the remainder stores PFNs.
- */
-struct pfn_pool {
- struct pfn_pool *next;
- atomic_t count;
- unsigned long pfns[];
-};
-
-#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
- sizeof(unsigned long))
-
-/*
- * Skip early PFN recording for a page allocation. Reuses the
- * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid
- * recursion when allocating pages for the early PFN tracking list
- * itself.
- *
- * Codetags of the pages allocated with __GFP_NO_CODETAG should be
- * cleared (via clear_page_tag_ref()) before freeing the pages to prevent
- * alloc_tag_sub_check() from triggering a warning.
- */
-#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT
-
-static struct pfn_pool *current_pfn_pool __initdata;
-
-static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
-{
- struct pfn_pool *pool;
- int idx;
-
- do {
- pool = READ_ONCE(current_pfn_pool);
- if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
- struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG);
- struct pfn_pool *new;
-
- if (!new_page) {
- pr_warn_once("early PFN tracking page allocation failed\n");
- return;
- }
- new = page_address(new_page);
- new->next = pool;
- atomic_set(&new->count, 0);
- if (cmpxchg(&current_pfn_pool, pool, new) != pool) {
- clear_page_tag_ref(new_page);
- __free_page(new_page);
- continue;
- }
- pool = new;
- }
- idx = atomic_read(&pool->count);
- if (idx >= PFN_POOL_SIZE)
- continue;
- if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
- break;
- } while (1);
-
- pool->pfns[idx] = pfn;
-}
-
-typedef void alloc_tag_add_func(unsigned long pfn);
-static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
- RCU_INITIALIZER(__alloc_tag_add_early_pfn);
-
-void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
-{
- alloc_tag_add_func *alloc_tag_add;
-
- if (static_key_enabled(&mem_profiling_compressed))
- return;
-
- /* Skip allocations for the tracking list itself to avoid recursion. */
- if (gfp_flags & __GFP_NO_CODETAG)
- return;
-
- rcu_read_lock();
- alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
- if (alloc_tag_add)
- alloc_tag_add(pfn);
- rcu_read_unlock();
-}
-
-static void __init clear_early_alloc_pfn_tag_refs(void)
-{
- struct pfn_pool *pool, *next;
- struct page *page;
- int i;
-
- if (static_key_enabled(&mem_profiling_compressed))
- return;
-
- rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL);
- /* Make sure we are not racing with __alloc_tag_add_early_pfn() */
- synchronize_rcu();
-
- for (pool = current_pfn_pool; pool; pool = next) {
- int nr_pfns = atomic_read(&pool->count);
-
- for (i = 0; i < nr_pfns; i++) {
- unsigned long pfn = pool->pfns[i];
-
- if (pfn_valid(pfn)) {
- union pgtag_ref_handle handle;
- union codetag_ref ref;
-
- if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
- /*
- * An early-allocated page could be freed and reallocated
- * after its page_ext is initialized but before we clear it.
- * In that case, it already has a valid tag set.
- * We should not overwrite that valid tag
- * with CODETAG_EMPTY.
- *
- * Note: there is still a small race window between checking
- * ref.ct and calling set_codetag_empty(). We accept this
- * race as it's unlikely and the extra complexity of atomic
- * cmpxchg is not worth it for this debug-only code path.
- */
- if (ref.ct) {
- put_page_tag_ref(handle);
- continue;
- }
-
- set_codetag_empty(&ref);
- update_page_tag_ref(handle, &ref);
- put_page_tag_ref(handle);
- }
- }
- }
-
- next = pool->next;
- page = virt_to_page(pool);
- clear_page_tag_ref(page);
- __free_page(page);
- }
-}
-#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-static inline void __init clear_early_alloc_pfn_tag_refs(void) {}
-#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-
-static __init void init_page_alloc_tagging(void)
-{
- clear_early_alloc_pfn_tag_refs();
-}
-
-struct page_ext_operations page_alloc_tagging_ops = {
- .size = sizeof(union codetag_ref),
- .need = need_page_alloc_tagging,
- .init = init_page_alloc_tagging,
-};
-EXPORT_SYMBOL(page_alloc_tagging_ops);
-
-#ifdef CONFIG_SYSCTL
-/*
- * Not using proc_do_static_key() directly to prevent enabling profiling
- * after it was shut down.
- */
-static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
- void *buffer, size_t *lenp, loff_t *ppos)
-{
- if (write) {
- /*
- * Call from do_sysctl_args() which is a no-op since the same
- * value was already set by setup_early_mem_profiling.
- * Return success to avoid warnings from do_sysctl_args().
- */
- if (!current->mm)
- return 0;
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
- /* User can't toggle profiling while debugging */
- return -EACCES;
-#endif
- if (!mem_profiling_support)
- return -EINVAL;
- }
-
- return proc_do_static_key(table, write, buffer, lenp, ppos);
-}
-
-
-static const struct ctl_table memory_allocation_profiling_sysctls[] = {
- {
- .procname = "mem_profiling",
- .data = &mem_alloc_profiling_key,
- .mode = 0644,
- .proc_handler = proc_mem_profiling_handler,
- },
-};
-
-static void __init sysctl_init(void)
-{
- register_sysctl_init("vm", memory_allocation_profiling_sysctls);
-}
-#else /* CONFIG_SYSCTL */
-static inline void sysctl_init(void) {}
-#endif /* CONFIG_SYSCTL */
-
-static int __init alloc_tag_init(void)
-{
- const struct codetag_type_desc desc = {
- .section = ALLOC_TAG_SECTION_NAME,
- .tag_size = sizeof(struct alloc_tag),
-#ifdef CONFIG_MODULES
- .needs_section_mem = needs_section_mem,
- .alloc_section_mem = reserve_module_tags,
- .free_section_mem = release_module_tags,
- .module_load = load_module,
- .module_replaced = replace_module,
-#endif
- };
- int res;
-
- sysctl_init();
-
- if (!mem_profiling_support) {
- pr_info("Memory allocation profiling is not supported!\n");
- return 0;
- }
-
- if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op,
- sizeof(struct allocinfo_private), NULL)) {
- pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
- shutdown_mem_profiling(false);
- return -ENOMEM;
- }
-
- res = alloc_mod_tags_mem();
- if (res) {
- pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
- shutdown_mem_profiling(true);
- return res;
- }
-
- alloc_tag_cttype = codetag_register_type(&desc);
- if (IS_ERR(alloc_tag_cttype)) {
- pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
- free_mod_tags_mem();
- shutdown_mem_profiling(true);
- return PTR_ERR(alloc_tag_cttype);
- }
-
- return 0;
-}
-module_init(alloc_tag_init);
diff --git a/lib/codetag.c b/lib/codetag.c
index 4001a7ea6675..a9cda4c962a3 100644
--- a/lib/codetag.c
+++ b/lib/codetag.c
@@ -19,6 +19,8 @@ struct codetag_type {
struct codetag_type_desc desc;
/* generates unique sequence number for module load */
unsigned long next_mod_seq;
+ /* bumped on every module load and unload */
+ unsigned long content_id;
};
struct codetag_range {
@@ -50,6 +52,20 @@ void codetag_unlock_module_list(struct codetag_type *cttype)
up_read(&cttype->mod_lock);
}
+unsigned long codetag_get_content_id(struct codetag_type *cttype)
+{
+ lockdep_assert_held(&cttype->mod_lock);
+
+ return cttype->content_id;
+}
+
+unsigned int codetag_get_count(struct codetag_type *cttype)
+{
+ lockdep_assert_held(&cttype->mod_lock);
+
+ return cttype->count;
+}
+
struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype)
{
struct codetag_iterator iter = {
@@ -204,6 +220,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod)
down_write(&cttype->mod_lock);
cmod->mod_seq = ++cttype->next_mod_seq;
+ ++cttype->content_id;
mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL);
if (mod_id >= 0) {
if (cttype->desc.module_load) {
@@ -368,6 +385,7 @@ void codetag_unload_module(struct module *mod)
cttype->count -= range_size(cttype, &cmod->range);
idr_remove(&cttype->mod_idr, mod_id);
kfree(cmod);
+ ++cttype->content_id;
}
up_write(&cttype->mod_lock);
if (found && cttype->desc.free_section_mem)
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index e52876435b77..e8eb2e1219db 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas)
return mas->status == ma_underflow;
}
+static inline void mas_make_walkable(struct ma_state *mas)
+{
+ if (!mas_is_active(mas) && !mas_is_start(mas))
+ mas->status = ma_start;
+}
+
static __always_inline struct maple_node *mte_to_node(
const struct maple_enode *entry)
{
@@ -456,46 +462,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode)
}
/*
- * mas_set_parent() - Set the parent node and encode the slot
- * @mas: The maple state
- * @enode: The encoded maple node.
- * @parent: The encoded maple node that is the parent of @enode.
- * @slot: The slot that @enode resides in @parent.
- *
- * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the
- * parent type.
- */
-static inline
-void mas_set_parent(struct ma_state *mas, struct maple_enode *enode,
- const struct maple_enode *parent, unsigned char slot)
-{
- unsigned long val = (unsigned long)parent;
- unsigned long shift;
- unsigned long type;
- enum maple_type p_type = mte_node_type(parent);
-
- MAS_BUG_ON(mas, p_type == maple_dense);
- MAS_BUG_ON(mas, p_type == maple_leaf_64);
-
- switch (p_type) {
- case maple_range_64:
- case maple_arange_64:
- shift = MAPLE_PARENT_SLOT_SHIFT;
- type = MAPLE_PARENT_RANGE64;
- break;
- default:
- case maple_dense:
- case maple_leaf_64:
- shift = type = 0;
- break;
- }
-
- val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */
- val |= (slot << shift) | type;
- mte_to_node(enode)->parent = ma_parent_ptr(val);
-}
-
-/*
* mte_parent_slot() - get the parent slot of @enode.
* @enode: The encoded maple node.
*
@@ -877,6 +843,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt,
}
/*
+ * mas_set_parent_slots() - Bulk operation to set many slot parent pointers
+ * @mas: The maple state
+ * @parent: The encoded maple node that is the parent of @enode.
+ * @slot: The slot that of the @enode.
+ * @start_slot: The offset into @slot
+ * @count: The number of slots to set (eg: exclusive)
+ */
+static inline
+void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent,
+ void __rcu **slots, unsigned char start_slot, unsigned char count)
+{
+ unsigned long val;
+ unsigned long shift;
+ unsigned long type;
+ enum maple_type p_type = mte_node_type(parent);
+ unsigned char i;
+
+ MAS_BUG_ON(mas, p_type != maple_range_64 &&
+ p_type != maple_arange_64);
+
+ shift = MAPLE_PARENT_SLOT_SHIFT;
+ type = MAPLE_PARENT_RANGE64;
+
+ val = (unsigned long)parent;
+ val &= ~MAPLE_NODE_MASK;
+
+ for (i = 0; i < count; i++) {
+ unsigned long pval = val | ((start_slot + i) << shift) | type;
+ struct maple_enode *child;
+
+ child = mt_slot_locked(mas->tree, slots, i);
+ mte_to_node(child)->parent = ma_parent_ptr(pval);
+ }
+}
+
+/*
* mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.
* @mat: the ma_topiary, a linked list of dead nodes.
* @dead_enode: the node to be marked as dead and added to the tail of the list
@@ -1153,6 +1155,73 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used)
ma_free_rcu(mte_to_node(used));
}
+
+#ifdef CONFIG_LOCKDEP
+static struct lockdep_map *mas_lockdep_map(struct ma_state *mas)
+{
+ struct maple_tree *mt = mas->tree;
+
+ if (mt_external_lock(mt))
+ return mt->ma_external_lock;
+
+ return &(mt->ma_lock).dep_map;
+}
+
+#endif
+
+static void mas_lock_check(struct ma_state *mas)
+{
+#ifdef CONFIG_LOCKDEP
+ struct lockdep_map *map;
+ if (!mas_is_active(mas))
+ return;
+
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+ if (!mt_locked(mas->tree)) {
+ if (mt_in_rcu(mas->tree))
+ WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp));
+ }
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+
+ map = mas_lockdep_map(mas);
+ if (map && lock_is_held(map))
+ WARN_ON_ONCE(mas->ld_seq != lock_sequence(map));
+#endif /* CONFIG_LOCKDEP */
+
+}
+
+static void mas_init_lock_check(struct ma_state *mas)
+{
+#ifdef CONFIG_LOCKDEP
+ struct lockdep_map *map;
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+ if (!mt_locked(mas->tree)) {
+ if (mt_in_rcu(mas->tree))
+ mas->rcu_gp = get_state_synchronize_rcu();
+ return;
+ }
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+
+ map = mas_lockdep_map(mas);
+ if (map && lock_is_held(map))
+ mas->ld_seq = lock_sequence(mas_lockdep_map(mas));
+#endif /* CONFIG_LOCKDEP */
+
+}
+
+static void mas_may_init_lock_check(struct ma_state *mas)
+{
+#ifdef CONFIG_LOCKDEP
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+ if (mas_is_start(mas) || mas_is_paused(mas)) {
+ mas_init_lock_check(mas);
+ return;
+ }
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+ mas_lock_check(mas);
+#endif /* CONFIG_LOCKDEP */
+}
+
/*
* mas_start() - Sets up maple state for operations.
* @mas: The maple state.
@@ -1171,6 +1240,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas)
if (likely(mas_is_start(mas))) {
struct maple_enode *root;
+ mas_init_lock_check(mas);
mas->min = 0;
mas->max = ULONG_MAX;
@@ -1501,14 +1571,26 @@ ascend:
goto ascend;
}
+static __always_inline void mas_update_gap_known(struct ma_state *mas,
+ unsigned long gap)
+{
+ unsigned char pslot;
+ unsigned long p_gap;
+
+ pslot = mte_parent_slot(mas->node);
+ p_gap = ma_gaps(mte_parent(mas->node),
+ mas_parent_type(mas, mas->node))[pslot];
+
+ if (p_gap != gap)
+ mas_parent_gap(mas, pslot, gap);
+}
+
/*
* mas_update_gap() - Update a nodes gaps and propagate up if necessary.
* @mas: the maple state.
*/
static inline void mas_update_gap(struct ma_state *mas)
{
- unsigned char pslot;
- unsigned long p_gap;
unsigned long max_gap;
if (!mt_is_alloc(mas->tree))
@@ -1518,13 +1600,7 @@ static inline void mas_update_gap(struct ma_state *mas)
return;
max_gap = mas_max_gap(mas);
-
- pslot = mte_parent_slot(mas->node);
- p_gap = ma_gaps(mte_parent(mas->node),
- mas_parent_type(mas, mas->node))[pslot];
-
- if (p_gap != max_gap)
- mas_parent_gap(mas, pslot, max_gap);
+ mas_update_gap_known(mas, max_gap);
}
/*
@@ -1540,14 +1616,10 @@ static inline void mas_adopt_children(struct ma_state *mas,
struct maple_node *node = mte_to_node(parent);
void __rcu **slots = ma_slots(node, type);
unsigned long *pivots = ma_pivots(node, type);
- struct maple_enode *child;
- unsigned char offset;
+ unsigned char end;
- offset = ma_data_end(node, type, pivots, mas->max);
- do {
- child = mas_slot_locked(mas, slots, offset);
- mas_set_parent(mas, child, parent, offset);
- } while (offset--);
+ end = ma_data_end(node, type, pivots, mas->max);
+ mas_set_parent_slots(mas, parent, slots, 0, end + 1);
}
/*
@@ -1929,15 +2001,10 @@ unsigned long node_copy(struct ma_state *mas, struct maple_node *src,
s_slots = ma_slots(src, s_mt) + start;
s_pivots = ma_pivots(src, s_mt) + start;
memcpy(d_slots, s_slots, size * sizeof(void __rcu *));
- if (!ma_is_leaf(d_mt) && s_mt == maple_copy) {
- struct maple_enode *edst = mt_mk_node(dst, d_mt);
-
- for (int i = 0; i < size; i++)
- mas_set_parent(mas,
- mt_slot_locked(mas->tree, d_slots, i),
- edst, d_start + i);
- }
+ if (!ma_is_leaf(d_mt) && s_mt == maple_copy)
+ mas_set_parent_slots(mas, mt_mk_node(dst, d_mt),
+ d_slots, d_start, size);
d_gaps = ma_gaps(dst, d_mt);
if (d_gaps) {
@@ -2081,8 +2148,8 @@ static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp)
mas->node = mt_slot_locked(mas->tree, cp->slot, 0);
/* Insert the new data in the tree */
mas_topiary_replace(mas, old_enode, cp->height);
- if (!mte_is_leaf(mas->node))
- mas_update_gap(mas);
+ if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node))
+ mas_update_gap_known(mas, cp->gap[0]);
mtree_range_walk(mas);
}
@@ -2928,13 +2995,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
if (r_mas->last < r_wr_mas->r_max)
r_mas->last = r_wr_mas->r_max;
r_mas->offset++;
- } else if ((r_mas->last == r_wr_mas->r_max) &&
- (r_mas->last < r_mas->max) &&
- !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
- r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots,
- r_wr_mas->type, r_mas->offset + 1);
- r_mas->offset++;
- r_wr_mas->r_max = r_mas->last;
}
}
@@ -3125,7 +3185,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
{
unsigned char dst_offset, offset_end;
- unsigned char copy_size, node_pivots;
+ unsigned char copy_size, node_pivots, node_slots;
struct maple_node reuse, *newnode;
unsigned long *dst_pivots;
void __rcu **dst_slots;
@@ -3138,6 +3198,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
in_rcu = mt_in_rcu(mas->tree);
offset_end = wr_mas->offset_end;
node_pivots = mt_pivots[wr_mas->type];
+ node_slots = mt_slots[wr_mas->type];
/* Assume last adds an entry */
new_end = mas->end + 1 - offset_end + mas->offset;
if (mas->last == wr_mas->end_piv) {
@@ -3149,7 +3210,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
if (in_rcu) {
newnode = mas_pop_node(mas);
} else {
- memset(&reuse, 0, sizeof(struct maple_node));
newnode = &reuse;
}
@@ -3193,7 +3253,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
dst_pivots[new_end] = mas->max;
done:
- mas_leaf_set_meta(newnode, maple_leaf_64, new_end);
+ if (!in_rcu && new_end + 2 < node_slots) {
+ unsigned char clear_from = new_end + 1;
+
+ /*
+ * Note that the last slot is never cleared, since the metadata
+ * will be stored there or it has a value.
+ */
+ memset(dst_slots + clear_from, 0,
+ sizeof(void __rcu *) * (node_slots - clear_from));
+ if (clear_from < node_pivots)
+ memset(dst_pivots + clear_from, 0,
+ sizeof(unsigned long) * (node_pivots - clear_from));
+ }
+
+ mas_leaf_set_meta(newnode, wr_mas->type, new_end);
if (in_rcu) {
struct maple_enode *old_enode = mas->node;
@@ -3218,7 +3292,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas)
void __rcu **slots = wr_mas->slots;
bool gap = false;
- gap |= !mt_slot_locked(mas->tree, slots, offset);
+ gap |= !wr_mas->content;
gap |= !mt_slot_locked(mas->tree, slots, offset + 1);
if (wr_mas->offset_end - offset == 1) {
@@ -3661,6 +3735,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
unsigned char new_end;
+ bool appending;
+ bool one_slot;
+ bool in_rcu;
if (unlikely(mas_is_none(mas) || mas_is_ptr(mas)))
return wr_store_root;
@@ -3680,21 +3757,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas)
return wr_new_root;
new_end = mas_wr_new_end(wr_mas);
+ in_rcu = mt_in_rcu(mas->tree);
+ appending = mas->offset == mas->end;
+ one_slot = wr_mas->offset_end - mas->offset == 1;
+
/* Potential spanning rebalance collapsing a node */
if (new_end < mt_min_slots[wr_mas->type]) {
if (!mte_is_root(mas->node))
return wr_rebalance;
+ if (!in_rcu) {
+ if (appending)
+ return wr_append;
+ else if (mas->end == new_end && one_slot)
+ return wr_slot_store;
+ }
return wr_node_store;
}
if (new_end >= mt_slots[wr_mas->type])
return wr_split_store;
- if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end))
+ if (!in_rcu && appending)
return wr_append;
- if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) ||
- (wr_mas->offset_end - mas->offset == 1)))
+ if (new_end == mas->end && (!in_rcu || one_slot))
return wr_slot_store;
return wr_node_store;
@@ -3793,35 +3879,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp,
void *entry, unsigned long range_lo, unsigned long range_hi,
unsigned long *next, gfp_t gfp)
{
- unsigned long min = range_lo;
- int ret = 0;
-
- range_lo = max(min, *next);
- ret = mas_empty_area(mas, range_lo, range_hi, 1);
- if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) {
- mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
- ret = 1;
- }
- if (ret < 0 && range_lo > min) {
- mas_reset(mas);
- ret = mas_empty_area(mas, min, range_hi, 1);
- if (ret == 0)
- ret = 1;
- }
- if (ret < 0)
- return ret;
+ int ret;
+ unsigned long min;
+ min = range_lo;
do {
+ range_lo = max(min, *next);
+ ret = mas_empty_area(mas, range_lo, range_hi, 1);
+ if (ret < 0 && range_lo > min) {
+ mas_reset(mas);
+ ret = mas_empty_area(mas, min, range_hi, 1);
+ if (ret == 0)
+ ret = 1;
+ }
+ if (ret < 0)
+ goto out;
+
mas_insert(mas, entry);
} while (mas_nomem(mas, gfp));
- if (mas_is_err(mas))
- return xa_err(mas->node);
+ if (mas_is_err(mas)) {
+ ret = xa_err(mas->node);
+ goto out;
+ }
+
+ if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) {
+ mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
+ ret = 1;
+ }
*startp = mas->index;
*next = *startp + 1;
if (*next == 0)
mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED;
+out:
mas_destroy(mas);
return ret;
}
@@ -4360,8 +4451,8 @@ void *mas_walk(struct ma_state *mas)
{
void *entry;
- if (!mas_is_active(mas) && !mas_is_start(mas))
- mas->status = ma_start;
+ mas_may_init_lock_check(mas);
+ mas_make_walkable(mas);
retry:
entry = mas_state_walk(mas);
if (mas_is_start(mas)) {
@@ -4831,6 +4922,7 @@ void *mas_store(struct ma_state *mas, void *entry)
{
MA_WR_STATE(wr_mas, mas, entry);
+ mas_may_init_lock_check(mas);
trace_ma_write(TP_FCT, mas, 0, entry);
#ifdef CONFIG_DEBUG_MAPLE_TREE
if (MAS_WARN_ON(mas, mas->index > mas->last))
@@ -4889,6 +4981,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp)
MA_WR_STATE(wr_mas, mas, entry);
int ret = 0;
+ mas_may_init_lock_check(mas);
retry:
mas_wr_preallocate(&wr_mas, entry);
if (unlikely(mas_nomem(mas, gfp))) {
@@ -4919,6 +5012,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry)
{
MA_WR_STATE(wr_mas, mas, entry);
+ mas_lock_check(mas);
if (mas->store_type == wr_store_root) {
mas_wr_prealloc_setup(&wr_mas);
goto store;
@@ -4951,6 +5045,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp)
{
MA_WR_STATE(wr_mas, mas, entry);
+ mas_may_init_lock_check(mas);
mas_wr_prealloc_setup(&wr_mas);
mas->store_type = mas_wr_store_type(&wr_mas);
mas_prealloc_calc(&wr_mas, entry);
@@ -4997,6 +5092,7 @@ static void mas_may_activate(struct ma_state *mas)
mas->status = ma_start;
} else {
mas->status = ma_active;
+ mas_lock_check(mas);
}
}
@@ -5074,6 +5170,7 @@ void *mas_next(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_next_setup(mas, max, &entry))
return entry;
@@ -5097,6 +5194,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_next_setup(mas, max, &entry))
return entry;
@@ -5205,6 +5303,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_prev_setup(mas, min, &entry))
return entry;
@@ -5228,6 +5327,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_prev_setup(mas, min, &entry))
return entry;
@@ -5274,6 +5374,7 @@ EXPORT_SYMBOL_GPL(mt_prev);
*/
void mas_pause(struct ma_state *mas)
{
+ mas_lock_check(mas);
mas->status = ma_pause;
mas->node = NULL;
}
@@ -5382,6 +5483,7 @@ void *mas_find(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_find_setup(mas, max, &entry))
return entry;
@@ -5409,6 +5511,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_find_setup(mas, max, &entry))
return entry;
@@ -5428,7 +5531,6 @@ EXPORT_SYMBOL_GPL(mas_find_range);
static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
void **entry)
{
-
switch (mas->status) {
case ma_active:
goto active;
@@ -5521,6 +5623,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_find_rev_setup(mas, min, &entry))
return entry;
@@ -5547,6 +5650,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
+ mas_may_init_lock_check(mas);
if (mas_find_rev_setup(mas, min, &entry))
return entry;
@@ -5564,6 +5668,10 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev);
* Searches for @mas->index, sets @mas->index and @mas->last to the range and
* erases that range.
*
+ * Note that erase requires allocations and will use GFP_KERNEL to do so if
+ * necessary. If the allocation fails, the internal lock will be dropped to
+ * retry.
+ *
* Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated.
*/
void *mas_erase(struct ma_state *mas)
@@ -5572,13 +5680,19 @@ void *mas_erase(struct ma_state *mas)
unsigned long index = mas->index;
MA_WR_STATE(wr_mas, mas, NULL);
- if (!mas_is_active(mas) || !mas_is_start(mas))
- mas->status = ma_start;
+ /*
+ * In low memory situations, the allocation is retried with the gfp flag
+ * GFP_KERNEL. The internal spinlock is dropped in mas_nomem(), however
+ * the external lock is not dropped.
+ */
+ if (mt_external_lock(mas->tree))
+ might_alloc(GFP_KERNEL);
+ mas_make_walkable(mas);
write_retry:
entry = mas_state_walk(mas);
if (!entry)
- return NULL;
+ goto out;
/* Must reset to ensure spanning writes of last slot are detected */
mas_reset(mas);
@@ -5589,8 +5703,10 @@ write_retry:
goto write_retry;
}
- if (mas_is_err(mas))
+ if (mas_is_err(mas)) {
+ entry = NULL;
goto out;
+ }
mas_wr_store_entry(&wr_mas);
out:
@@ -5601,10 +5717,11 @@ EXPORT_SYMBOL_GPL(mas_erase);
/**
* mas_nomem() - Check if there was an error allocating and do the allocation
- * if necessary If there are allocations, then free them.
+ * if necessary.
+ *
* @mas: The maple state
* @gfp: The GFP_FLAGS to use for allocations
- * Return: true on allocation, false otherwise.
+ * Return: False on no memory. True otherwise (partial success as well)
*/
bool mas_nomem(struct ma_state *mas, gfp_t gfp)
__must_hold(mas->tree->ma_lock)
@@ -5612,6 +5729,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp)
if (likely(mas->node != MA_ERROR(-ENOMEM)))
return false;
+ /* Allocations can fail, don't do this. */
+ WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) &&
+ mt_external_lock(mas->tree));
+
if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) {
mtree_unlock(mas->tree);
mas_alloc_nodes(mas, gfp);
@@ -5620,10 +5741,17 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp)
mas_alloc_nodes(mas, gfp);
}
- if (!mas->sheaf && !mas->alloc)
+ /*
+ * Return false on zero forward progress. Partial allocations are kept
+ * so the retry path will attempt to get the rest. The failure should
+ * not happen as we try our best to reclaim. The user would need an
+ * external lock with a non-blocking gfp in a low memory situation -
+ * which would have triggered the first warning in this function.
+ */
+ if (WARN_ON_ONCE(!mas->sheaf && !mas->alloc))
return false;
- mas->status = ma_start;
+ mas_reset(mas);
return true;
}
@@ -5913,6 +6041,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange);
* Erasing is the same as a walk to an entry then a store of a NULL to that
* ENTIRE range. In fact, it is implemented as such using the advanced API.
*
+ * Note that erase requires allocations and will use GFP_KERNEL to do so if
+ * necessary. If the allocation fails, the internal lock will be dropped to
+ * retry.
+ *
* Return: The entry stored at the @index or %NULL
*/
void *mtree_erase(struct maple_tree *mt, unsigned long index)
@@ -5922,6 +6054,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index)
MA_STATE(mas, mt, index, index);
trace_ma_op(TP_FCT, &mas);
+ might_alloc(GFP_KERNEL);
mtree_lock(mt);
entry = mas_erase(&mas);
mtree_unlock(mt);
@@ -6447,7 +6580,7 @@ static void mt_dump_range(unsigned long min, unsigned long max,
{
static const char spaces[] = " ";
- switch(format) {
+ switch (format) {
case mt_dump_hex:
if (min == max)
pr_info("%.*s%lx: ", depth * 2, spaces, min);
@@ -6489,7 +6622,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry,
pr_cont(" contents: ");
for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) {
- switch(format) {
+ switch (format) {
case mt_dump_hex:
pr_cont(PTR_FMT " %lX ", node->slot[i], node->pivot[i]);
break;
@@ -6517,7 +6650,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry,
if (last == max)
break;
if (last > max) {
- switch(format) {
+ switch (format) {
case mt_dump_hex:
pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n",
node, last, max, i);
@@ -6576,7 +6709,7 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry,
if (last == max)
break;
if (last > max) {
- switch(format) {
+ switch (format) {
case mt_dump_hex:
pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n",
node, last, max, i);
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 97772e42b9b2..f8d90689af9f 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref,
* @confirm_switch: optional confirmation callback
*
* There's no reason to use this function for the usual reference counting.
- * Use percpu_ref_kill[_and_confirm]().
+ * Use percpu_ref_kill() or percpu_ref_kill_and_confirm().
*
* Schedule switching of @ref to atomic mode. All its percpu counts will
* be collected to the main atomic counter. On completion, when all CPUs
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 976b9bd02a1b..cb5ca920fddb 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -455,6 +455,8 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp,
node->slots[0] = (void __rcu *)entry;
entry = node_to_entry(node);
rcu_assign_pointer(root->xa_head, entry);
+ /* new head may be missed by an in-progress kmemleak scan */
+ kmemleak_transient_leak(node);
shift += RADIX_TREE_MAP_SHIFT;
} while (shift <= maxshift);
out:
@@ -495,8 +497,11 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root)
if (!node->shift && is_idr(root))
break;
- if (radix_tree_is_internal_node(child))
+ if (radix_tree_is_internal_node(child)) {
entry_to_node(child)->parent = NULL;
+ /* new head may be missed by an in-progress kmemleak scan */
+ kmemleak_transient_leak(entry_to_node(child));
+ }
/*
* We don't need rcu_assign_pointer(), since we are simply
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index c4adbf98fac7..6205fb313bd0 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -389,6 +389,67 @@ out:
return ret;
}
+static int dmirror_range_fault_unlocked(struct dmirror *dmirror,
+ struct hmm_range *range,
+ unsigned long timeout)
+{
+ int ret;
+
+ while (true) {
+ ret = hmm_range_fault_unlocked_timeout(range, timeout);
+ if (ret)
+ goto out;
+
+ mutex_lock(&dmirror->mutex);
+ if (mmu_interval_read_retry(range->notifier,
+ range->notifier_seq)) {
+ mutex_unlock(&dmirror->mutex);
+ continue;
+ }
+ break;
+ }
+
+ ret = dmirror_do_fault(dmirror, range);
+
+ mutex_unlock(&dmirror->mutex);
+out:
+ return ret;
+}
+
+static int dmirror_fault_unlocked(struct dmirror *dmirror,
+ unsigned long start,
+ unsigned long end, bool write,
+ unsigned long timeout)
+{
+ struct mm_struct *mm = dmirror->notifier.mm;
+ unsigned long addr;
+ unsigned long pfns[32];
+ struct hmm_range range = {
+ .notifier = &dmirror->notifier,
+ .hmm_pfns = pfns,
+ .pfn_flags_mask = 0,
+ .default_flags =
+ HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
+ .dev_private_owner = dmirror->mdevice,
+ };
+ int ret = 0;
+
+ if (!mmget_not_zero(mm))
+ return -EFAULT;
+
+ for (addr = start; addr < end; addr = range.end) {
+ range.start = addr;
+ range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
+
+ ret = dmirror_range_fault_unlocked(dmirror, &range, timeout);
+ if (ret)
+ break;
+ }
+
+ mmput(mm);
+ return ret;
+}
+
static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
unsigned long end, bool write)
{
@@ -407,7 +468,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
/* Since the mm is for the mirrored process, get a reference first. */
if (!mmget_not_zero(mm))
- return 0;
+ return -EFAULT;
for (addr = start; addr < end; addr = range.end) {
range.start = addr;
@@ -488,6 +549,48 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
return ret;
}
+static int dmirror_read_unlocked(struct dmirror *dmirror,
+ struct hmm_dmirror_cmd *cmd,
+ unsigned long timeout)
+{
+ struct dmirror_bounce bounce;
+ unsigned long start, end;
+ unsigned long size = cmd->npages << PAGE_SHIFT;
+ int ret;
+
+ start = cmd->addr;
+ end = start + size;
+ if (end < start)
+ return -EINVAL;
+
+ ret = dmirror_bounce_init(&bounce, start, size);
+ if (ret)
+ return ret;
+
+ while (1) {
+ mutex_lock(&dmirror->mutex);
+ ret = dmirror_do_read(dmirror, start, end, &bounce);
+ mutex_unlock(&dmirror->mutex);
+ if (ret != -ENOENT)
+ break;
+
+ start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
+ ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout);
+ if (ret)
+ break;
+ cmd->faults++;
+ }
+
+ if (ret == 0) {
+ if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
+ bounce.size))
+ ret = -EFAULT;
+ }
+ cmd->cpages = bounce.cpages;
+ dmirror_bounce_fini(&bounce);
+ return ret;
+}
+
static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
unsigned long end, struct dmirror_bounce *bounce)
{
@@ -1572,7 +1675,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp,
dmirror->flags = cmd.npages;
ret = 0;
break;
-
+ case HMM_DMIRROR_READ_UNLOCKED:
+ ret = dmirror_read_unlocked(dmirror, &cmd, 0);
+ break;
default:
return -EINVAL;
}
diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h
index f94c6d457338..ea9b0ec404fb 100644
--- a/lib/test_hmm_uapi.h
+++ b/lib/test_hmm_uapi.h
@@ -38,6 +38,7 @@ struct hmm_dmirror_cmd {
#define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd)
+#define HMM_DMIRROR_READ_UNLOCKED _IOWR('H', 0x09, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0)