diff options
Diffstat (limited to 'mm')
108 files changed, 8697 insertions, 4323 deletions
diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca4824905..c52ab6afcb15 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -125,8 +125,6 @@ config ZSWAP_COMPRESSOR_DEFAULT config ZSMALLOC tristate -if ZSMALLOC - menu "Zsmalloc allocator options" depends on ZSMALLOC @@ -161,8 +159,6 @@ config ZSMALLOC_CHAIN_SIZE endmenu -endif - menu "Slab allocator options" config SLUB @@ -509,13 +505,6 @@ config EXCLUSIVE_SYSTEM_RAM def_bool y depends on !DEVMEM || STRICT_DEVMEM -# -# Only be set on architectures that have completely implemented memory hotplug -# feature. If you are not sure, don't touch it. -# -config HAVE_BOOTMEM_INFO_NODE - def_bool n - config ARCH_ENABLE_MEMORY_HOTPLUG bool @@ -590,13 +579,10 @@ endchoice config MEMORY_HOTREMOVE bool "Allow for memory hot remove" - select HAVE_BOOTMEM_INFO_NODE if X86_64 - depends on MEMORY_HOTPLUG select MIGRATION config MHP_MEMMAP_ON_MEMORY def_bool y - depends on MEMORY_HOTPLUG && SPARSEMEM_VMEMMAP depends on ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE endif # MEMORY_HOTPLUG @@ -703,7 +689,7 @@ config DEVICE_MIGRATION config ARCH_ENABLE_HUGEPAGE_MIGRATION bool -config ARCH_ENABLE_THP_MIGRATION +config ARCH_HAS_PMD_SOFTLEAVES bool config HUGETLB_PAGE_SIZE_VARIABLE @@ -1235,9 +1221,7 @@ config ZONE_DMA32 config ZONE_DEVICE bool "Device memory (pmem, HMM, etc...) hotplug support" - depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE - depends on SPARSEMEM_VMEMMAP select XARRAY_MULTI help @@ -1386,6 +1370,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR help Arch has userfaultfd minor fault support +config USERFAULTFD_RWP + def_bool y + depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP + help + Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a + userfaultfd notification on every access -- read or write -- to a + protected range, letting userspace observe the working set of a + process. + menuconfig USERFAULTFD bool "Enable userfaultfd() system call" depends on MMU @@ -1393,17 +1386,15 @@ menuconfig USERFAULTFD Enable the userfaultfd() system call that allows to intercept and handle page faults in userland. -if USERFAULTFD config PTE_MARKER_UFFD_WP bool "Userfaultfd write protection support for shmem/hugetlbfs" default y - depends on HAVE_ARCH_USERFAULTFD_WP + depends on USERFAULTFD && HAVE_ARCH_USERFAULTFD_WP help Allows to create marker PTEs for userfaultfd write protection purposes. It is required to enable userfaultfd write protection on file-backed memory types like shmem and hugetlbfs. -endif # USERFAULTFD # multi-gen LRU { config LRU_GEN diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 91b3e027b753..5737a504efbb 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -320,3 +320,31 @@ config PER_VMA_LOCK_STATS overhead in the page fault path. If in doubt, say N. + +config MEM_ALLOC_PROFILING + bool "Enable memory allocation profiling" + default n + depends on MMU + depends on PROC_FS + depends on !DEBUG_FORCE_WEAK_PER_CPU + select CODE_TAGGING + select PAGE_EXTENSION + select SLAB_OBJ_EXT + help + Track allocation source code and record total allocation size + initiated at that code location. The mechanism can be used to track + memory leaks with a low performance and memory impact. + +config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + bool "Enable memory allocation profiling by default" + default y + depends on MEM_ALLOC_PROFILING + +config MEM_ALLOC_PROFILING_DEBUG + bool "Memory allocation profiler debugging" + default n + depends on MEM_ALLOC_PROFILING + select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + help + Adds warnings with helpful error messages for memory allocation + profiling. diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061..ab37ef428d98 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -50,7 +50,7 @@ endif obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ maccess.o page-writeback.o folio-compat.o \ - readahead.o swap.o truncate.o vmscan.o shrinker.o \ + readahead.o folio.o truncate.o vmscan.o shrinker.o \ shmem.o util.o mmzone.o vmstat.o backing-dev.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ @@ -141,9 +141,9 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o obj-$(CONFIG_PTDUMP) += ptdump.o obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o -obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c new file mode 100644 index 000000000000..52aece27b00e --- /dev/null +++ b/mm/alloc_tag.c @@ -0,0 +1,1361 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/alloc_tag.h> +#include <linux/execmem.h> +#include <linux/fs.h> +#include <linux/gfp.h> +#include <linux/kallsyms.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/compat.h> +#include <linux/page_ext.h> +#include <linux/pgalloc_tag.h> +#include <linux/proc_fs.h> +#include <linux/rcupdate.h> +#include <linux/seq_buf.h> +#include <linux/seq_file.h> +#include <linux/string_choices.h> +#include <linux/vmalloc.h> +#include <linux/kmemleak.h> +#include <uapi/linux/alloc_tag.h> + +#include "internal.h" +#include "page_alloc.h" + +#define ALLOCINFO_FILE_NAME "allocinfo" +#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) +#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) +#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME) + +#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT +static bool mem_profiling_support = true; +#else +static bool mem_profiling_support; +#endif + +/* + * Memory allocation profiling is permanently disabled and cannot be enabled. + * Must be called after setup_early_mem_profiling(). + */ +bool mem_alloc_profiling_permanently_disabled(void) +{ + return !mem_profiling_support; +} + +static struct codetag_type *alloc_tag_cttype; + +#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU +DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); +EXPORT_SYMBOL(_shared_alloc_tag); +#endif + +DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + mem_alloc_profiling_key); +EXPORT_SYMBOL(mem_alloc_profiling_key); + +DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed); + +struct alloc_tag_kernel_section kernel_tags = { NULL, 0 }; +unsigned long alloc_tag_ref_mask; +int alloc_tag_ref_offs; + +struct allocinfo_private { + struct codetag_iterator iter; + struct codetag_iterator reported_iter; + bool print_header; + struct allocinfo_filter filter; + /* ioctl uses a separate iterator not to interfere with reads */ + struct codetag_iterator ioctl_iter; + bool positioned; /* seq_open_private() sets to 0 */ + struct mutex ioctl_lock; +}; + +static void *allocinfo_start(struct seq_file *m, loff_t *pos) +{ + struct allocinfo_private *priv; + loff_t node = *pos; + + priv = (struct allocinfo_private *)m->private; + codetag_lock_module_list(alloc_tag_cttype); + if (node == 0) { + priv->print_header = true; + priv->iter = codetag_get_ct_iter(alloc_tag_cttype); + } else { + priv->iter = priv->reported_iter; + } + codetag_next_ct(&priv->iter); + return priv->iter.ct ? priv : NULL; +} + +static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) +{ + struct allocinfo_private *priv = (struct allocinfo_private *)arg; + struct codetag *ct; + + priv->reported_iter = priv->iter; + ct = codetag_next_ct(&priv->iter); + (*pos)++; + if (!ct) + return NULL; + + return priv; +} + +static void allocinfo_stop(struct seq_file *m, void *arg) +{ + codetag_unlock_module_list(alloc_tag_cttype); +} + +static void print_allocinfo_header(struct seq_buf *buf) +{ + /* Output format version, so we can change it. */ + seq_buf_printf(buf, "allocinfo - version: 2.0\n"); + seq_buf_printf(buf, "# <size> <calls> <tag info>\n"); +} + +static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) +{ + struct alloc_tag *tag = ct_to_alloc_tag(ct); + struct alloc_tag_counters counter = alloc_tag_read(tag); + s64 bytes = counter.bytes; + + seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); + codetag_to_text(out, ct); + if (unlikely(alloc_tag_is_inaccurate(tag))) + seq_buf_printf(out, " accurate:no"); + seq_buf_putc(out, ' '); + seq_buf_putc(out, '\n'); +} + +static int allocinfo_show(struct seq_file *m, void *arg) +{ + struct allocinfo_private *priv = (struct allocinfo_private *)arg; + char *bufp; + size_t n = seq_get_buf(m, &bufp); + struct seq_buf buf; + + seq_buf_init(&buf, bufp, n); + if (priv->print_header) { + print_allocinfo_header(&buf); + priv->print_header = false; + } + alloc_tag_to_text(&buf, priv->iter.ct); + seq_commit(m, seq_buf_used(&buf)); + return 0; +} + +static const struct seq_operations allocinfo_seq_op = { + .start = allocinfo_start, + .next = allocinfo_next, + .stop = allocinfo_stop, + .show = allocinfo_show, +}; + +/* + * Initializes seq_file operations and allocates private state when opening + * the /proc/allocinfo procfs entry. + */ +static int allocinfo_open(struct inode *inode, struct file *file) +{ + int ret; + + ret = seq_open_private(file, &allocinfo_seq_op, + sizeof(struct allocinfo_private)); + if (!ret) { + struct seq_file *m = file->private_data; + struct allocinfo_private *priv = m->private; + + mutex_init(&priv->ioctl_lock); + } + return ret; +} + +/* + * Cleans up the seq_file state and frees up the private state allocated in + * allocinfo_open() when closing the /proc/allocinfo file descriptor. + */ +static int allocinfo_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + struct allocinfo_private *priv = m->private; + + mutex_destroy(&priv->ioctl_lock); + return seq_release_private(inode, file); +} + +/* + * Returns a pointer to the suffix of a string so that its length fits within + * ALLOCINFO_STR_SIZE, preserving the trailing characters. + * Function, file and module names often have the same prefixes, therefore + * when filtering by these criteria, we compare the last 64 characters to + * minimize the chances of name collisions + */ +static const char *allocinfo_str(const char *str) +{ + size_t len = strlen(str); + + /* Keep an extra space for the trailing NULL. */ + if (len >= ALLOCINFO_STR_SIZE) + str += (len - ALLOCINFO_STR_SIZE) + 1; + return str; +} + +/* Copy a string and trim from the beginning if it's too long */ +static void allocinfo_copy_str(char *dest, const char *src) +{ + strscpy_pad(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE); +} + +/* Compare two strings and only consider the trimmed suffix if s1 is too long */ +static int allocinfo_cmp_str(const char *str, const char *template) +{ + return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE); +} + +/* Fetch the per-CPU counters */ +static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct) +{ + return alloc_tag_read(ct_to_alloc_tag(ct)); +} + +/* + * Populates the UAPI allocinfo_tag_data structure with active runtime + * profiling counters extracted from the given kernel codetag. + */ +static void allocinfo_to_params(struct codetag *ct, + struct allocinfo_tag_data *data, + struct alloc_tag_counters *counters) +{ + if (ct->modname) + allocinfo_copy_str(data->tag.modname, ct->modname); + else + data->tag.modname[0] = '\0'; + allocinfo_copy_str(data->tag.function, ct->function); + allocinfo_copy_str(data->tag.filename, ct->filename); + data->tag.lineno = ct->lineno; + data->counter.bytes = counters->bytes; + data->counter.calls = counters->calls; + data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct)); +} + +/* + * Retrieves the unique content ID representing the current allocation tag module + * layout, allowing userspace to detect if modules were loaded / unloaded. + */ +static int allocinfo_ioctl_get_content_id(struct seq_file *m, void __user *arg) +{ + struct allocinfo_content_id params; + + codetag_lock_module_list(alloc_tag_cttype); + params.id = codetag_get_content_id(alloc_tag_cttype); + codetag_unlock_module_list(alloc_tag_cttype); + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +/* + * Verifies whether a given codetag satisfies the active filtering criteria by + * matching its characteristics against the specified filter. + */ +static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter, + struct alloc_tag_counters *counters, + bool *fetched_counters) +{ + bool inaccurate; + + if (!filter || !filter->mask) + return true; + + if (filter->mask & ALLOCINFO_FILTER_MASK_MODNAME) { + /* user wants to filter by modname but ct->modname is NULL */ + if (!ct->modname) { + /* validate if user was attempting to filter for built-in allocations */ + if (filter->fields.modname[0] != '\0') + return false; + } else if (allocinfo_cmp_str(ct->modname, filter->fields.modname)) + return false; + } + + if ((filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) && + ct->function && allocinfo_cmp_str(ct->function, filter->fields.function)) + return false; + + if ((filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) && + ct->filename && allocinfo_cmp_str(ct->filename, filter->fields.filename)) + return false; + + if ((filter->mask & ALLOCINFO_FILTER_MASK_LINENO) && + ct->lineno != filter->fields.lineno) + return false; + + if (filter->mask & ALLOCINFO_FILTER_MASK_INACCURATE) { + inaccurate = !!(ct->flags & CODETAG_FLAG_INACCURATE); + if (inaccurate != !!(filter->fields.inaccurate)) + return false; + } + + if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE)) { + if (!*fetched_counters) { + *counters = allocinfo_prefetch_counters(ct); + *fetched_counters = true; + } + if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) && + counters->bytes < filter->min_size) + return false; + if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) && + counters->bytes > filter->max_size) + return false; + } + + return true; +} + +/* + * Seeks the ioctl iterator to the specified 0-indexed tag position, reads its + * profiling data and returns it to userspace. + */ +static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg) +{ + struct allocinfo_private *priv; + struct codetag *ct; + struct allocinfo_get_at params = {0}; + __u64 skip_count; + struct alloc_tag_counters counters; + bool fetched_counters; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS) + return -EINVAL; + + if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) && + (params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) && + params.filter.min_size > params.filter.max_size) + return -EINVAL; + + priv = m->private; + + mutex_lock(&priv->ioctl_lock); + codetag_lock_module_list(alloc_tag_cttype); + + if (params.pos >= codetag_get_count(alloc_tag_cttype)) { + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + return -ENOENT; + } + + skip_count = params.pos; + + if (params.filter.mask) + priv->filter = params.filter; + else + priv->filter.mask = 0; + + /* Find the codetag */ + priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype); + ct = codetag_next_ct(&priv->ioctl_iter); + + while (ct) { + fetched_counters = false; + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) { + if (skip_count == 0) + break; + skip_count--; + } + ct = codetag_next_ct(&priv->ioctl_iter); + } + + if (ct) { + if (!fetched_counters) + counters = allocinfo_prefetch_counters(ct); + allocinfo_to_params(ct, ¶ms.data, &counters); + priv->positioned = true; + } + + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + + if (!ct) + return -ENOENT; + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +/* + * Advances the ioctl iterator to the next allocation tag in the sequence and + * returns its profiling data to userspace. + */ +static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg) +{ + struct allocinfo_private *priv; + struct codetag *ct; + struct allocinfo_tag_data params; + int ret = 0; + struct alloc_tag_counters counters; + bool fetched_counters; + + memset(¶ms, 0, sizeof(params)); + priv = m->private; + + mutex_lock(&priv->ioctl_lock); + codetag_lock_module_list(alloc_tag_cttype); + + if (!priv->positioned) { + priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype); + priv->positioned = true; + } + + ct = codetag_next_ct(&priv->ioctl_iter); + while (ct) { + fetched_counters = false; + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) + break; + ct = codetag_next_ct(&priv->ioctl_iter); + } + + if (ct) { + if (!fetched_counters) + counters = allocinfo_prefetch_counters(ct); + allocinfo_to_params(ct, ¶ms, &counters); + } + if (!ct) { + priv->positioned = false; + ret = -ENOENT; + } + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + + if (ret == 0) { + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + } + return ret; +} + +/* + * Entry point ioctl function for /proc/allocinfo routing requests to fetch the + * layout content ID, seek to a specific tag, or read sequential tags. + */ +static long allocinfo_ioctl(struct file *file, unsigned int cmd, + unsigned long __arg) +{ + void __user *arg = (void __user *)__arg; + int ret; + + switch (cmd) { + case ALLOCINFO_IOC_CONTENT_ID: + ret = allocinfo_ioctl_get_content_id(file->private_data, arg); + break; + case ALLOCINFO_IOC_GET_AT: + ret = allocinfo_ioctl_get_at(file->private_data, arg); + break; + case ALLOCINFO_IOC_GET_NEXT: + ret = allocinfo_ioctl_get_next(file->private_data, arg); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + return ret; +} + +#ifdef CONFIG_COMPAT +static long allocinfo_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return allocinfo_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static const struct proc_ops allocinfo_proc_ops = { + .proc_open = allocinfo_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = allocinfo_release, + .proc_ioctl = allocinfo_ioctl, +#ifdef CONFIG_COMPAT + .proc_compat_ioctl = allocinfo_compat_ioctl, +#endif +}; + +size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) +{ + struct codetag_iterator iter; + struct codetag *ct; + struct codetag_bytes n; + unsigned int i, nr = 0; + + if (IS_ERR_OR_NULL(alloc_tag_cttype)) + return 0; + + if (can_sleep) + codetag_lock_module_list(alloc_tag_cttype); + else if (!codetag_trylock_module_list(alloc_tag_cttype)) + return 0; + + iter = codetag_get_ct_iter(alloc_tag_cttype); + while ((ct = codetag_next_ct(&iter))) { + struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); + + n.ct = ct; + n.bytes = counter.bytes; + + for (i = 0; i < nr; i++) + if (n.bytes > tags[i].bytes) + break; + + if (i < count) { + nr -= nr == count; + memmove(&tags[i + 1], + &tags[i], + sizeof(tags[0]) * (nr - i)); + nr++; + tags[i] = n; + } + } + + codetag_unlock_module_list(alloc_tag_cttype); + + return nr; +} + +void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) +{ + int i; + struct alloc_tag *tag; + unsigned int nr_pages = 1 << new_order; + + if (!mem_alloc_profiling_enabled()) + return; + + tag = __pgalloc_tag_get(&folio->page); + if (!tag) + return; + + for (i = nr_pages; i < (1 << old_order); i += nr_pages) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { + /* Set new reference to point to the original tag */ + alloc_tag_ref_set(&ref, tag); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } +} + +void pgalloc_tag_swap(struct folio *new, struct folio *old) +{ + union pgtag_ref_handle handle_old, handle_new; + union codetag_ref ref_old, ref_new; + struct alloc_tag *tag_old, *tag_new; + + if (!mem_alloc_profiling_enabled()) + return; + + tag_old = __pgalloc_tag_get(&old->page); + if (!tag_old) + return; + tag_new = __pgalloc_tag_get(&new->page); + if (!tag_new) + return; + + if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) + return; + if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { + put_page_tag_ref(handle_old); + return; + } + + /* + * Clear tag references to avoid debug warning when using + * __alloc_tag_ref_set() with non-empty reference. + */ + set_codetag_empty(&ref_old); + set_codetag_empty(&ref_new); + + /* swap tags */ + __alloc_tag_ref_set(&ref_old, tag_new); + update_page_tag_ref(handle_old, &ref_old); + __alloc_tag_ref_set(&ref_new, tag_old); + update_page_tag_ref(handle_new, &ref_new); + + put_page_tag_ref(handle_old); + put_page_tag_ref(handle_new); +} + +static void shutdown_mem_profiling(bool remove_file) +{ + if (mem_alloc_profiling_enabled()) + static_branch_disable(&mem_alloc_profiling_key); + + if (!mem_profiling_support) + return; + + if (remove_file) + remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); + mem_profiling_support = false; +} + +void __init alloc_tag_sec_init(void) +{ + struct alloc_tag *last_codetag; + + if (!mem_profiling_support) + return; + + if (!static_key_enabled(&mem_profiling_compressed)) + return; + + kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( + SECTION_START(ALLOC_TAG_SECTION_NAME)); + last_codetag = (struct alloc_tag *)kallsyms_lookup_name( + SECTION_STOP(ALLOC_TAG_SECTION_NAME)); + kernel_tags.count = last_codetag - kernel_tags.first_tag; + + /* Check if kernel tags fit into page flags */ + if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { + shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ + pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", + kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); + return; + } + + alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); + alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); + pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", + NR_UNUSED_PAGEFLAG_BITS); +} + +#ifdef CONFIG_MODULES + +static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE); +static struct vm_struct *vm_module_tags; +/* A dummy object used to indicate an unloaded module */ +static struct module unloaded_mod; +/* A dummy object used to indicate a module prepended area */ +static struct module prepend_mod; + +struct alloc_tag_module_section module_tags; + +static inline unsigned long alloc_tag_align(unsigned long val) +{ + if (!static_key_enabled(&mem_profiling_compressed)) { + /* No alignment requirements when we are not indexing the tags */ + return val; + } + + if (val % sizeof(struct alloc_tag) == 0) + return val; + return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag); +} + +static bool ensure_alignment(unsigned long align, unsigned int *prepend) +{ + if (!static_key_enabled(&mem_profiling_compressed)) { + /* No alignment requirements when we are not indexing the tags */ + return true; + } + + /* + * If alloc_tag size is not a multiple of required alignment, tag + * indexing does not work. + */ + if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) + return false; + + /* Ensure prepend consumes multiple of alloc_tag-sized blocks */ + if (*prepend) + *prepend = alloc_tag_align(*prepend); + + return true; +} + +static inline bool tags_addressable(void) +{ + unsigned long tag_idx_count; + + if (!static_key_enabled(&mem_profiling_compressed)) + return true; /* with page_ext tags are always addressable */ + + tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + + module_tags.size / sizeof(struct alloc_tag); + + return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS); +} + +static bool needs_section_mem(struct module *mod, unsigned long size) +{ + if (!mem_profiling_support) + return false; + + return size >= sizeof(struct alloc_tag); +} + +static bool clean_unused_counters(struct alloc_tag *start_tag, + struct alloc_tag *end_tag) +{ + struct alloc_tag *tag; + bool ret = true; + + for (tag = start_tag; tag <= end_tag; tag++) { + struct alloc_tag_counters counter; + + if (!tag->counters) + continue; + + counter = alloc_tag_read(tag); + if (!counter.bytes) { + free_percpu(tag->counters); + tag->counters = NULL; + } else { + ret = false; + } + } + + return ret; +} + +/* Called with mod_area_mt locked */ +static void clean_unused_module_areas_locked(void) +{ + MA_STATE(mas, &mod_area_mt, 0, module_tags.size); + struct module *val; + + mas_for_each(&mas, val, module_tags.size) { + struct alloc_tag *start_tag; + struct alloc_tag *end_tag; + + if (val != &unloaded_mod) + continue; + + /* Release area if all tags are unused */ + start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); + end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); + if (clean_unused_counters(start_tag, end_tag)) + mas_erase(&mas); + } +} + +/* Called with mod_area_mt locked */ +static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, + unsigned long size, unsigned int prepend, unsigned long align) +{ + bool cleanup_done = false; + +repeat: + /* Try finding exact size and hope the start is aligned */ + if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { + if (IS_ALIGNED(mas->index + prepend, align)) + return true; + + /* Try finding larger area to align later */ + mas_reset(mas); + if (!mas_empty_area(mas, 0, section_size - 1, + size + prepend + align - 1)) + return true; + } + + /* No free area, try cleanup stale data and repeat the search once */ + if (!cleanup_done) { + clean_unused_module_areas_locked(); + cleanup_done = true; + mas_reset(mas); + goto repeat; + } + + return false; +} + +static int vm_module_tags_populate(void) +{ + unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + + (vm_module_tags->nr_pages << PAGE_SHIFT); + unsigned long new_end = module_tags.start_addr + module_tags.size; + + if (phys_end < new_end) { + struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; + unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); + unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); + unsigned long more_pages; + unsigned long nr = 0; + + more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; + while (nr < more_pages) { + unsigned long allocated; + + allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN, + NUMA_NO_NODE, more_pages - nr, next_page + nr); + + if (!allocated) + break; + nr += allocated; + } + + if (nr < more_pages || + vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, + next_page, PAGE_SHIFT) < 0) { + release_pages_arg arg = { .pages = next_page }; + + /* Clean up and error out */ + release_pages(arg, nr); + return -ENOMEM; + } + + vm_module_tags->nr_pages += nr; + + /* + * Kasan allocates 1 byte of shadow for every 8 bytes of data. + * When kasan_alloc_module_shadow allocates shadow memory, + * its unit of allocation is a page. + * Therefore, here we need to align to MODULE_ALIGN. + */ + if (old_shadow_end < new_shadow_end) + kasan_alloc_module_shadow((void *)old_shadow_end, + new_shadow_end - old_shadow_end, + GFP_KERNEL); + } + + /* + * Mark the pages as accessible, now that they are mapped. + * With hardware tag-based KASAN, marking is skipped for + * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). + */ + kasan_unpoison_vmalloc((void *)module_tags.start_addr, + new_end - module_tags.start_addr, + KASAN_VMALLOC_PROT_NORMAL); + + return 0; +} + +static void *reserve_module_tags(struct module *mod, unsigned long size, + unsigned int prepend, unsigned long align) +{ + unsigned long section_size = module_tags.end_addr - module_tags.start_addr; + MA_STATE(mas, &mod_area_mt, 0, section_size - 1); + unsigned long offset; + void *ret = NULL; + + /* If no tags return error */ + if (size < sizeof(struct alloc_tag)) + return ERR_PTR(-EINVAL); + + /* + * align is always power of 2, so we can use IS_ALIGNED and ALIGN. + * align 0 or 1 means no alignment, to simplify set to 1. + */ + if (!align) + align = 1; + + if (!ensure_alignment(align, &prepend)) { + shutdown_mem_profiling(true); + pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", + mod->name, align); + return ERR_PTR(-EINVAL); + } + + mas_lock(&mas); + if (!find_aligned_area(&mas, section_size, size, prepend, align)) { + ret = ERR_PTR(-ENOMEM); + goto unlock; + } + + /* Mark found area as reserved */ + offset = mas.index; + offset += prepend; + offset = ALIGN(offset, align); + if (offset != mas.index) { + unsigned long pad_start = mas.index; + + mas.last = offset - 1; + mas_store(&mas, &prepend_mod); + if (mas_is_err(&mas)) { + ret = ERR_PTR(xa_err(mas.node)); + goto unlock; + } + mas.index = offset; + mas.last = offset + size - 1; + mas_store(&mas, mod); + if (mas_is_err(&mas)) { + mas.index = pad_start; + mas_erase(&mas); + ret = ERR_PTR(xa_err(mas.node)); + } + } else { + mas.last = offset + size - 1; + mas_store(&mas, mod); + if (mas_is_err(&mas)) + ret = ERR_PTR(xa_err(mas.node)); + } +unlock: + mas_unlock(&mas); + + if (IS_ERR(ret)) + return ret; + + if (module_tags.size < offset + size) { + int grow_res; + + module_tags.size = offset + size; + if (mem_alloc_profiling_enabled() && !tags_addressable()) { + shutdown_mem_profiling(true); + pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", + mod->name, NR_UNUSED_PAGEFLAG_BITS); + } + + grow_res = vm_module_tags_populate(); + if (grow_res) { + shutdown_mem_profiling(true); + pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", + mod->name); + return ERR_PTR(grow_res); + } + } + + return (struct alloc_tag *)(module_tags.start_addr + offset); +} + +static void release_module_tags(struct module *mod, bool used) +{ + MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); + struct alloc_tag *start_tag; + struct alloc_tag *end_tag; + struct module *val; + + mas_lock(&mas); + mas_for_each_rev(&mas, val, 0) + if (val == mod) + break; + + if (!val) /* module not found */ + goto out; + + if (!used) + goto release_area; + + start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); + end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); + if (!clean_unused_counters(start_tag, end_tag)) { + struct alloc_tag *tag; + + for (tag = start_tag; tag <= end_tag; tag++) { + struct alloc_tag_counters counter; + + if (!tag->counters) + continue; + + counter = alloc_tag_read(tag); + pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", + tag->ct.filename, tag->ct.lineno, tag->ct.modname, + tag->ct.function, counter.bytes); + } + } else { + used = false; + } +release_area: + mas_store(&mas, used ? &unloaded_mod : NULL); + val = mas_prev_range(&mas, 0); + if (val == &prepend_mod) + mas_store(&mas, NULL); +out: + mas_unlock(&mas); +} + +static int load_module(struct module *mod, struct codetag *start, struct codetag *stop) +{ + /* Allocate module alloc_tag percpu counters */ + struct alloc_tag *start_tag; + struct alloc_tag *stop_tag; + struct alloc_tag *tag; + + /* percpu counters for core allocations are already statically allocated */ + if (!mod) + return 0; + + start_tag = ct_to_alloc_tag(start); + stop_tag = ct_to_alloc_tag(stop); + for (tag = start_tag; tag < stop_tag; tag++) { + WARN_ON(tag->counters); + tag->counters = alloc_percpu(struct alloc_tag_counters); + if (!tag->counters) { + while (--tag >= start_tag) { + free_percpu(tag->counters); + tag->counters = NULL; + } + pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n", + mod->name); + return -ENOMEM; + } + + /* + * Avoid a kmemleak false positive. The pointer to the counters is stored + * in the alloc_tag section of the module and cannot be directly accessed. + */ + kmemleak_ignore_percpu(tag->counters); + } + return 0; +} + +static void replace_module(struct module *mod, struct module *new_mod) +{ + MA_STATE(mas, &mod_area_mt, 0, module_tags.size); + struct module *val; + + mas_lock(&mas); + mas_for_each(&mas, val, module_tags.size) { + if (val != mod) + continue; + + mas_store_gfp(&mas, new_mod, GFP_KERNEL); + break; + } + mas_unlock(&mas); +} + +static int __init alloc_mod_tags_mem(void) +{ + /* Map space to copy allocation tags */ + vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); + if (!vm_module_tags) { + pr_err("Failed to map %lu bytes for module allocation tags\n", + MODULE_ALLOC_TAG_VMAP_SIZE); + module_tags.start_addr = 0; + return -ENOMEM; + } + + vm_module_tags->pages = kmalloc_objs(struct page *, + get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, + GFP_KERNEL | __GFP_ZERO); + if (!vm_module_tags->pages) { + free_vm_area(vm_module_tags); + return -ENOMEM; + } + + module_tags.start_addr = (unsigned long)vm_module_tags->addr; + module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; + /* Ensure the base is alloc_tag aligned when required for indexing */ + module_tags.start_addr = alloc_tag_align(module_tags.start_addr); + + return 0; +} + +static void __init free_mod_tags_mem(void) +{ + release_pages_arg arg = { .pages = vm_module_tags->pages }; + + module_tags.start_addr = 0; + release_pages(arg, vm_module_tags->nr_pages); + kfree(vm_module_tags->pages); + free_vm_area(vm_module_tags); +} + +#else /* CONFIG_MODULES */ + +static inline int alloc_mod_tags_mem(void) { return 0; } +static inline void free_mod_tags_mem(void) {} + +#endif /* CONFIG_MODULES */ + +/* See: Documentation/mm/allocation-profiling.rst */ +static int __init setup_early_mem_profiling(char *str) +{ + bool compressed = false; + bool enable; + + if (!str || !str[0]) + return -EINVAL; + + if (!strncmp(str, "never", 5)) { + enable = false; + mem_profiling_support = false; + pr_info("Memory allocation profiling is disabled!\n"); + } else { + char *token = strsep(&str, ","); + + if (kstrtobool(token, &enable)) + return -EINVAL; + + if (str) { + + if (strcmp(str, "compressed")) + return -EINVAL; + + compressed = true; + } + mem_profiling_support = true; + pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", + compressed ? "with" : "without", str_on_off(enable)); + } + + if (enable != mem_alloc_profiling_enabled()) { + if (enable) + static_branch_enable(&mem_alloc_profiling_key); + else + static_branch_disable(&mem_alloc_profiling_key); + } + if (compressed != static_key_enabled(&mem_profiling_compressed)) { + if (compressed) + static_branch_enable(&mem_profiling_compressed); + else + static_branch_disable(&mem_profiling_compressed); + } + + return 0; +} +early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); + +static __init bool need_page_alloc_tagging(void) +{ + if (static_key_enabled(&mem_profiling_compressed)) + return false; + + return mem_profiling_support; +} + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* + * Track page allocations before page_ext is initialized. + * Some pages are allocated before page_ext becomes available, leaving + * their codetag uninitialized. Track these early PFNs so we can clear + * their codetag refs later to avoid warnings when they are freed. + * + * Each page is cast to a pfn_pool: the first few bytes hold metadata + * (next pointer and slot count), the remainder stores PFNs. + */ +struct pfn_pool { + struct pfn_pool *next; + atomic_t count; + unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) +static struct pfn_pool *current_pfn_pool __initdata; + +static void __init __alloc_tag_add_early_pfn(unsigned long pfn) +{ + struct pfn_pool *pool; + int idx; + + do { + pool = READ_ONCE(current_pfn_pool); + if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { + struct page *new_page = __alloc_pages(__GFP_HIGH, 0, numa_mem_id(), + NULL, ALLOC_NO_CODETAG); + struct pfn_pool *new; + + if (!new_page) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new = page_address(new_page); + new->next = pool; + atomic_set(&new->count, 0); + if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { + clear_page_tag_ref(new_page); + __free_page(new_page); + continue; + } + pool = new; + } + idx = atomic_read(&pool->count); + if (idx >= PFN_POOL_SIZE) + continue; + if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) + break; + } while (1); + + pool->pfns[idx] = pfn; +} + +typedef void alloc_tag_add_func(unsigned long pfn); +static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = + RCU_INITIALIZER(__alloc_tag_add_early_pfn); + +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) +{ + alloc_tag_add_func *alloc_tag_add; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + /* Skip allocations for the tracking list itself to avoid recursion. */ + if (alloc_flags & ALLOC_NO_CODETAG) + return; + + rcu_read_lock(); + alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); + if (alloc_tag_add) + alloc_tag_add(pfn); + rcu_read_unlock(); +} + +static void __init clear_early_alloc_pfn_tag_refs(void) +{ + struct pfn_pool *pool, *next; + struct page *page; + int i; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL); + /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ + synchronize_rcu(); + + for (pool = current_pfn_pool; pool; pool = next) { + int nr_pfns = atomic_read(&pool->count); + + for (i = 0; i < nr_pfns; i++) { + unsigned long pfn = pool->pfns[i]; + + if (pfn_valid(pfn)) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { + /* + * An early-allocated page could be freed and reallocated + * after its page_ext is initialized but before we clear it. + * In that case, it already has a valid tag set. + * We should not overwrite that valid tag + * with CODETAG_EMPTY. + * + * Note: there is still a small race window between checking + * ref.ct and calling set_codetag_empty(). We accept this + * race as it's unlikely and the extra complexity of atomic + * cmpxchg is not worth it for this debug-only code path. + */ + if (ref.ct) { + put_page_tag_ref(handle); + continue; + } + + set_codetag_empty(&ref); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } + } + + next = pool->next; + page = virt_to_page(pool); + clear_page_tag_ref(page); + __free_page(page); + } +} +#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ +static inline void __init clear_early_alloc_pfn_tag_refs(void) {} +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static __init void init_page_alloc_tagging(void) +{ + clear_early_alloc_pfn_tag_refs(); +} + +struct page_ext_operations page_alloc_tagging_ops = { + .size = sizeof(union codetag_ref), + .need = need_page_alloc_tagging, + .init = init_page_alloc_tagging, +}; +EXPORT_SYMBOL(page_alloc_tagging_ops); + +#ifdef CONFIG_SYSCTL +/* + * Not using proc_do_static_key() directly to prevent enabling profiling + * after it was shut down. + */ +static int proc_mem_profiling_handler(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + if (write) { + /* + * Call from do_sysctl_args() which is a no-op since the same + * value was already set by setup_early_mem_profiling. + * Return success to avoid warnings from do_sysctl_args(). + */ + if (!current->mm) + return 0; + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + /* User can't toggle profiling while debugging */ + return -EACCES; +#endif + if (!mem_profiling_support) + return -EINVAL; + } + + return proc_do_static_key(table, write, buffer, lenp, ppos); +} + + +static const struct ctl_table memory_allocation_profiling_sysctls[] = { + { + .procname = "mem_profiling", + .data = &mem_alloc_profiling_key, + .mode = 0644, + .proc_handler = proc_mem_profiling_handler, + }, +}; + +static void __init sysctl_init(void) +{ + register_sysctl_init("vm", memory_allocation_profiling_sysctls); +} +#else /* CONFIG_SYSCTL */ +static inline void sysctl_init(void) {} +#endif /* CONFIG_SYSCTL */ + +static int __init alloc_tag_init(void) +{ + const struct codetag_type_desc desc = { + .section = ALLOC_TAG_SECTION_NAME, + .tag_size = sizeof(struct alloc_tag), +#ifdef CONFIG_MODULES + .needs_section_mem = needs_section_mem, + .alloc_section_mem = reserve_module_tags, + .free_section_mem = release_module_tags, + .module_load = load_module, + .module_replaced = replace_module, +#endif + }; + int res; + + sysctl_init(); + + if (!mem_profiling_support) { + pr_info("Memory allocation profiling is not supported!\n"); + return 0; + } + + if (!proc_create(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_proc_ops)) { + pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); + shutdown_mem_profiling(false); + return -ENOMEM; + } + + res = alloc_mod_tags_mem(); + if (res) { + pr_err("Failed to reserve address space for module tags, errno = %d\n", res); + shutdown_mem_profiling(true); + return res; + } + + alloc_tag_cttype = codetag_register_type(&desc); + if (IS_ERR(alloc_tag_cttype)) { + pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype); + free_mod_tags_mem(); + shutdown_mem_profiling(true); + return PTR_ERR(alloc_tag_cttype); + } + + return 0; +} +module_init(alloc_tag_init); diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c deleted file mode 100644 index 0fa78db7fbc0..000000000000 --- a/mm/bootmem_info.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Bootmem core functions. - * - * Copyright (c) 2020, Bytedance. - * - * Author: Muchun Song <songmuchun@bytedance.com> - * - */ -#include <linux/mm.h> -#include <linux/compiler.h> -#include <linux/memblock.h> -#include <linux/bootmem_info.h> -#include <linux/memory_hotplug.h> -#include <linux/kmemleak.h> - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ - BUG_ON(type > 0xf); - BUG_ON(info > (ULONG_MAX >> 4)); - set_page_private(page, info << 4 | type); - page_ref_inc(page); -} - -void put_page_bootmem(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || - type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); - - if (page_ref_dec_return(page) == 1) { - set_page_private(page, 0); - free_reserved_page(page); - } -} - -static void __init register_page_bootmem_info_section(unsigned long start_pfn) -{ - unsigned long section_nr; - struct mem_section *ms; - - start_pfn = SECTION_ALIGN_DOWN(start_pfn); - section_nr = pfn_to_section_nr(start_pfn); - ms = __nr_to_section(section_nr); - - if (!preinited_vmemmap_section(ms)) - register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn), - PAGES_PER_SECTION); -} - -void __init register_page_bootmem_info_node(struct pglist_data *pgdat) -{ - unsigned long pfn, end_pfn; - int node = pgdat->node_id; - - pfn = pgdat->node_start_pfn; - end_pfn = pgdat_end_pfn(pgdat); - - /* register section info */ - for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { - /* - * Some platforms can assign the same pfn to multiple nodes - on - * node0 as well as nodeN. To avoid registering a pfn against - * multiple nodes we check that this pfn does not already - * reside in some other nodes. - */ - if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) - register_page_bootmem_info_section(pfn); - } -} @@ -33,6 +33,7 @@ #include "internal.h" #include "cma.h" +#include "mm_init.h" struct cma cma_areas[MAX_CMA_AREAS]; unsigned int cma_area_count; @@ -126,7 +127,6 @@ bool cma_validate_zones(struct cma *cma) * to be in the same zone. Simplify by forcing the entire * CMA resv range to be in the same zone. */ - WARN_ON_ONCE(!pfn_valid(base_pfn)); if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) { set_bit(CMA_ZONES_INVALID, &cma->flags); return false; @@ -165,6 +165,8 @@ static void __init cma_activate_area(struct cma *cma) bitmap_set(cmr->bitmap, 0, bitmap_count); } + WARN_ON_ONCE(!pfn_valid(cmr->base_pfn)); + for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count; pfn += pageblock_nr_pages) init_cma_reserved_pageblock(pfn_to_page(pfn)); diff --git a/mm/compaction.c b/mm/compaction.c index f08765ade014..0568623d9384 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -24,6 +24,7 @@ #include <linux/page_owner.h> #include <linux/psi.h> #include <linux/cpuset.h> +#include "page_alloc.h" #include "internal.h" #ifdef CONFIG_COMPACTION @@ -82,7 +83,7 @@ static inline bool is_via_compact_memory(int order) { return false; } static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags) { - post_alloc_hook(page, order, __GFP_MOVABLE); + post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT); set_page_refcounted(page); return page; } @@ -644,7 +645,6 @@ static unsigned long isolate_freepages_block(struct compact_control *cc, isolated = __isolate_free_page(page, order); if (!isolated) break; - set_page_private(page, order); nr_scanned += isolated - 1; total_isolated += isolated; @@ -1381,12 +1381,44 @@ static bool suitable_migration_source(struct compact_control *cc, if (pageblock_skip_persistent(page)) return false; - if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) + /* + * Background compaction produces blocks for the zone at + * large, with no particular allocation context. Allow all + * block types, including CMA. + */ + if (!cc->direct_compaction) return true; block_mt = get_pageblock_migratetype(page); - if (cc->migratetype == MIGRATE_MOVABLE) + /* + * CMA pages can only be taken by ALLOC_CMA requests. For anybody + * else, vacating a CMA block consumes free pages the caller + * could have used, and produces free pages it cannot. + */ + if (is_migrate_cma(block_mt) && !(cc->alloc_flags & ALLOC_CMA)) + return false; + + /* + * Per default, scans are restricted to blocks compatible with + * the request, to prevent cross-contamination. Once + * compaction priority escalates to synchronous scans, though, + * scan all blocks to try to make forward progress. For + * movable request, this likely helps little: there shouldn't + * be many migratable pages inside non-movable blocks besides + * allocator fallbacks. For non-movable requests, this helps a + * lot, as they can finally scan movable blocks. + */ + if (cc->mode != MIGRATE_ASYNC) + return true; + + /* + * Prevent <pageblock_order unmovable/reclaimable requests from + * polluting movable blocks through fallbacks. Whole-block production + * (directly requested, or defrag_mode) is exempt as the allocator + * claims and converts these. + */ + if (cc->migratetype == MIGRATE_MOVABLE || cc->order >= pageblock_order) return is_migrate_movable(block_mt); else return block_mt == cc->migratetype; @@ -1617,7 +1649,6 @@ static void fast_isolate_freepages(struct compact_control *cc) /* Isolate the page if available */ if (page) { if (__isolate_free_page(page, order)) { - set_page_private(page, order); nr_isolated = 1 << order; nr_scanned += nr_isolated - 1; total_isolated += nr_isolated; @@ -1846,11 +1877,10 @@ again: size >>= 1; list_add(&freepage[size].lru, &cc->freepages[start_order]); - set_page_private(&freepage[size], start_order); } dst = (struct folio *)freepage; - post_alloc_hook(&dst->page, order, __GFP_MOVABLE); + post_alloc_hook(&dst->page, order, __GFP_MOVABLE, ALLOC_DEFAULT); set_page_refcounted(&dst->page); if (order) prep_compound_page(&dst->page, order); @@ -1974,12 +2004,12 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) return pfn; /* - * Only allow kcompactd and direct requests for movable pages to - * quickly clear out a MOVABLE pageblock for allocation. This - * reduces the risk that a large movable pageblock is freed for - * an unmovable/reclaimable small allocation. + * Prevent <pageblock_order unmovable/reclaimable requests from + * polluting movable blocks through fallbacks. Whole-block production + * is exempt as the allocator claims and converts these. */ - if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE) + if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE && + cc->order < pageblock_order) return pfn; /* @@ -2770,9 +2800,8 @@ out: static enum compact_result compact_zone_order(struct zone *zone, int order, gfp_t gfp_mask, enum compact_priority prio, unsigned int alloc_flags, int highest_zoneidx, - struct page **capture) + struct capture_control *capc) { - enum compact_result ret; struct compact_control cc = { .order = order, .search_order = order, @@ -2787,38 +2816,8 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) }; - struct capture_control capc = { - .cc = &cc, - .page = NULL, - }; - /* - * Make sure the structs are really initialized before we expose the - * capture control, in case we are interrupted and the interrupt handler - * frees a page. - */ - barrier(); - WRITE_ONCE(current->capture_control, &capc); - - ret = compact_zone(&cc, &capc); - - /* - * Make sure we hide capture control first before we read the captured - * page pointer, otherwise an interrupt could free and capture a page - * and we would leak it. - */ - WRITE_ONCE(current->capture_control, NULL); - *capture = READ_ONCE(capc.page); - /* - * Technically, it is also possible that compaction is skipped but - * the page is still captured out of luck(IRQ came and freed the page). - * Returning COMPACT_SUCCESS in such cases helps in properly accounting - * the COMPACT[STALL|FAIL] when compaction is skipped. - */ - if (*capture) - ret = COMPACT_SUCCESS; - - return ret; + return compact_zone(&cc, capc); } /** @@ -2828,13 +2827,13 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, * @alloc_flags: The allocation flags of the current allocation * @ac: The context of current allocation * @prio: Determines how hard direct compaction should try to succeed - * @capture: Pointer to free page created by compaction will be stored here + * @capc: Free page capture bypassing the freelist * * This is the main entry point for direct page compaction. */ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, - enum compact_priority prio, struct page **capture) + enum compact_priority prio, struct capture_control *capc) { struct zoneref *z; struct zone *zone; @@ -2861,8 +2860,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, continue; } + WRITE_ONCE(capc->zone, zone); + status = compact_zone_order(zone, order, gfp_mask, prio, - alloc_flags, ac->highest_zoneidx, capture); + alloc_flags, ac->highest_zoneidx, capc); + + WRITE_ONCE(capc->zone, NULL); + + /* Stop if a page has been captured */ + if (READ_ONCE(capc->page)) + status = COMPACT_SUCCESS; + rc = max(status, rc); /* The allocation should succeed, stop compacting */ diff --git a/mm/damon/core.c b/mm/damon/core.c index cff932b3317d..644daf5a1656 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -166,6 +166,7 @@ struct damon_probe *damon_new_probe(void) p = kmalloc_obj(*p); if (!p) return NULL; + p->weight = 0; INIT_LIST_HEAD(&p->filters); INIT_LIST_HEAD(&p->list); return p; @@ -208,6 +209,106 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx) return NULL; } +static bool damon_has_probe_weights(struct damon_ctx *c) +{ + struct damon_probe *p; + + damon_for_each_probe(p, c) { + if (p->weight) + return true; + } + return false; +} + +/* + * damon_mvsum() - Returns pseudo moving sum value for a time window. + * @current_nr: The value of the current aggregation window. + * @last_nr: The value of the last aggregation window. + * @left_window_bp: Left time of the current aggregation window. + * + * This function calculates a pseudo moving sum value of a counter that is + * aggregated for each time window. @current_nr is the value of the counter + * that aggregated so far (maybe not yet complete), from the beginning of the + * current aggregation time window. @last_nr is the value of the counter that + * has completely aggregated in the last aggregation time window. + * @left_window_bp represents how much time is left for the current aggregation + * time window in bp (1/10,000). For example, the aggregation time window is + * for every 10 seconds and 7 seconds has passed since the beginning of the + * current window, this parameter will be 3000 ((10 - 7) / 10 * 10000). + * + * The logic assumes the aggregation in the last phase was made in a single + * speed. Based on the assumption, the value from the last window that needs + * to be added to the current value is calculated as a portion of the last + * value based on the remaining time window. + */ +static unsigned long damon_mvsum(unsigned long current_nr, + unsigned long last_nr, unsigned long left_window_bp) +{ + return current_nr + mult_frac(last_nr, left_window_bp, 10000); +} + +/** + * damon_nr_accesses_mvsum() - Returns moving sum access frequency score. + * @r: Region to get the access frequency of. + * @ctx: DAMON context of @r. + * + * This function returns for how many sampling iterations in the last + * aggregation interval (&damon_attrs->aggr_interval) the region was found to + * be accessed. Hence the value can be interpreted as the relative access + * frequency score of the region (@r). The value is calculated as a pseudo + * moving sum, and hence it is not an exact value but just a best-effort + * reasonable estimation. + * + * Return: the pseudo moving sum access frequency score. + */ +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + if (left_window_bp == 10000) + return r->last_nr_accesses; + + return damon_mvsum(r->nr_accesses, r->last_nr_accesses, + left_window_bp); +} + +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + if (left_window_bp == 10000) + return r->last_probe_hits[probe_idx]; + + return damon_mvsum(r->probe_hits[probe_idx], + r->last_probe_hits[probe_idx], left_window_bp); +} + #ifdef CONFIG_DAMON_DEBUG_SANITY static void damon_verify_new_region(unsigned long start, unsigned long end) { @@ -237,9 +338,10 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) region->ar.start = start; region->ar.end = end; region->nr_accesses = 0; - region->nr_accesses_bp = 0; - for (i = 0; i < DAMON_MAX_PROBES; i++) + for (i = 0; i < DAMON_MAX_PROBES; i++) { region->probe_hits[i] = 0; + region->last_probe_hits[i] = 0; + } INIT_LIST_HEAD(®ion->list); region->age = 0; @@ -302,6 +404,30 @@ static bool damon_is_last_region(struct damon_region *r, return list_is_last(&r->list, &t->regions_list); } +/** + * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region. + * @r: region to get the weighted sum of. + * @last: if the request is for last-window aggregated probe hits. + * @ctx: context of &r. + * + * Return: the weighted sum of probe hits of the region. + */ +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx) +{ + struct damon_probe *probe; + unsigned int sum = 0; + int i = 0; + + damon_for_each_probe(probe, ctx) { + if (last) + sum += r->last_probe_hits[i++] * probe->weight; + else + sum += r->probe_hits[i++] * probe->weight; + } + return sum; +} + /* * Check whether a region is intersecting an address range * @@ -579,6 +705,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->ops_filters); scheme->stat = (struct damos_stat){}; scheme->max_nr_snapshots = 0; + scheme->last_applied = NULL; INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); @@ -789,55 +916,66 @@ static unsigned int damon_age_for_new_attrs(unsigned int age, return age * old_attrs->aggr_interval / new_attrs->aggr_interval; } -/* convert access ratio in bp (per 10,000) to nr_accesses */ -static unsigned int damon_accesses_bp_to_nr_accesses( - unsigned int accesses_bp, struct damon_attrs *attrs) +/* convert sample ratio in bp (per 10,000) to count */ +static unsigned int damon_sample_bp_to_count( + unsigned int bp, struct damon_attrs *attrs) { - return accesses_bp * damon_max_nr_accesses(attrs) / 10000; + return bp * damon_nr_samples_per_aggr(attrs) / 10000; } -/* - * Convert nr_accesses to access ratio in bp (per 10,000). - * - * Callers should ensure attrs.aggr_interval is not zero, like - * damon_update_monitoring_results() does . Otherwise, divide-by-zero would - * happen. - */ -static unsigned int damon_nr_accesses_to_accesses_bp( - unsigned int nr_accesses, struct damon_attrs *attrs) +/* convert sample count to ratio in bp (per 10,000) */ +static unsigned int damon_sample_count_to_bp( + unsigned int count, struct damon_attrs *attrs) { - return mult_frac(nr_accesses, 10000, damon_max_nr_accesses(attrs)); + return mult_frac(count, 10000, damon_nr_samples_per_aggr(attrs)); } -static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, +static unsigned int damon_nr_samples_for_new_attrs(unsigned int nr, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) { - return damon_accesses_bp_to_nr_accesses( - damon_nr_accesses_to_accesses_bp( - nr_accesses, old_attrs), - new_attrs); + return damon_sample_bp_to_count( + damon_sample_count_to_bp(nr, old_attrs), new_attrs); +} + +static void damon_update_probe_hits(struct damon_region *r, + struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, + bool aggregating, struct damon_ctx *ctx) +{ + struct damon_probe *p; + int i = 0; + + damon_for_each_probe(p, ctx) { + r->last_probe_hits[i] = damon_nr_samples_for_new_attrs( + r->last_probe_hits[i], old_attrs, new_attrs); + if (!aggregating) + r->probe_hits[i] = damon_nr_samples_for_new_attrs( + r->probe_hits[i], old_attrs, + new_attrs); + else + r->probe_hits[i] = 0; + i++; + } } static void damon_update_monitoring_result(struct damon_region *r, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, - bool aggregating) + bool aggregating, struct damon_ctx *ctx) { - if (!aggregating) { - r->nr_accesses = damon_nr_accesses_for_new_attrs( + damon_update_probe_hits(r, old_attrs, new_attrs, aggregating, ctx); + + r->last_nr_accesses = damon_nr_samples_for_new_attrs( + r->last_nr_accesses, old_attrs, new_attrs); + if (!aggregating) + r->nr_accesses = damon_nr_samples_for_new_attrs( r->nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->nr_accesses * 10000; - } else { + else /* * if this is called in the middle of the aggregation, reset * the aggregations we made so far for this aggregation * interval. In other words, make the status like * kdamond_reset_aggregated() is called. */ - r->last_nr_accesses = damon_nr_accesses_for_new_attrs( - r->last_nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->last_nr_accesses * 10000; r->nr_accesses = 0; - } r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs); } @@ -864,8 +1002,8 @@ static void damon_update_monitoring_results(struct damon_ctx *ctx, damon_for_each_target(t, ctx) damon_for_each_region(r, t) - damon_update_monitoring_result( - r, old_attrs, new_attrs, aggregating); + damon_update_monitoring_result(r, old_attrs, new_attrs, + aggregating, ctx); } /* @@ -1201,6 +1339,34 @@ static void damos_set_filters_default_reject(struct damos *s) damos_filters_default_reject(&s->ops_filters); } +static bool damon_valid_probe_params(struct damon_ctx *ctx) +{ + unsigned long sample_interval; + unsigned char max_probe_hits; + struct damon_probe *probe; + unsigned int wsum, wsum_to_add; + + if (!damon_has_probe_weights(ctx)) + return true; + + sample_interval = ctx->attrs.sample_interval ? : 1; + if (ctx->attrs.aggr_interval / sample_interval > U8_MAX) + return false; + + /* invalid if probe hits weighted sum can overflow */ + max_probe_hits = damon_nr_samples_per_aggr(&ctx->attrs); + wsum = 0; + damon_for_each_probe(probe, ctx) { + if (probe->weight > UINT_MAX / max_probe_hits) + return false; + wsum_to_add = probe->weight * max_probe_hits; + if (UINT_MAX - wsum < wsum_to_add) + return false; + wsum += wsum_to_add; + } + return true; +} + /* * damos_commit_dests() - Copy migration destinations from @src to @dst. * @dst: Destination structure to update. @@ -1360,23 +1526,35 @@ static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx) static int damon_commit_target_regions(struct damon_target *dst, struct damon_target *src, unsigned long src_min_region_sz) { - struct damon_region *src_region; + struct damon_region *src_region, *prev = NULL; struct damon_addr_range *ranges; int i = 0, err; - damon_for_each_region(src_region, src) - i++; + damon_for_each_region(src_region, src) { + if (!prev || prev->ar.end != src_region->ar.start) + i++; + prev = src_region; + } if (!i) return 0; - ranges = kmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); + ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); if (!ranges) return -ENOMEM; + prev = NULL; i = 0; - damon_for_each_region(src_region, src) - ranges[i++] = src_region->ar; + damon_for_each_region(src_region, src) { + if (!prev) { + ranges[i].start = src_region->ar.start; + } else if (prev->ar.end != src_region->ar.start) { + ranges[i++].end = prev->ar.end; + ranges[i].start = src_region->ar.start; + } + prev = src_region; + } + ranges[i++].end = damon_last_region(src)->ar.end; err = damon_set_regions(dst, ranges, i, src_min_region_sz); - kfree(ranges); + kvfree(ranges); return err; } @@ -1545,6 +1723,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) damon_for_each_probe_safe(dst_probe, next, dst) { src_probe = damon_nth_probe(i++, src); if (src_probe) { + dst_probe->weight = src_probe->weight; err = damon_commit_filters(dst_probe, src_probe); if (err) return err; @@ -1561,6 +1740,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) if (!new_probe) return -ENOMEM; damon_add_probe(dst, new_probe); + new_probe->weight = src_probe->weight; err = damon_commit_filters(new_probe, src_probe); if (err) return err; @@ -1568,20 +1748,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) return 0; } -/** - * damon_commit_ctx() - Commit parameters of a DAMON context to another. - * @dst: The commit destination DAMON context. - * @src: The commit source DAMON context. - * - * This function copies user-specified parameters from @src to @dst and update - * the internal status and results accordingly. Users should use this function - * for context-level parameters update of running context, instead of manual - * in-place updates. - * - * This function should be called from parameters-update safe context, like - * damon_call(). - */ -int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) +static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) { int err; struct damos *scheme; @@ -1604,6 +1771,9 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) } } + if (!damon_valid_probe_params(src)) + return -EINVAL; + err = damon_commit_schemes(dst, src); if (err) return err; @@ -1636,6 +1806,52 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) return 0; } +static struct damon_ctx *damon_new_test_ctx(struct damon_ctx *dst) +{ + struct damon_ctx *test_ctx; + int err; + + test_ctx = damon_new_ctx(); + if (!test_ctx) + return NULL; + err = __damon_commit_ctx(test_ctx, dst); + if (err) { + damon_destroy_ctx(test_ctx); + return NULL; + } + return test_ctx; +} + +/** + * damon_commit_ctx() - Commit parameters of a DAMON context to another. + * @dst: The commit destination DAMON context. + * @src: The commit source DAMON context. + * + * This function copies user-specified parameters from @src to @dst and update + * the internal status and results accordingly. Users should use this function + * for context-level parameters update of running context, instead of manual + * in-place updates. + * + * This function should be called from parameters-update safe context, like + * damon_call(). + */ +int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) +{ + struct damon_ctx *test_ctx; + int err; + + test_ctx = damon_new_test_ctx(dst); + if (!test_ctx) + return -ENOMEM; + err = __damon_commit_ctx(test_ctx, src); + if (err) + goto out; + err = __damon_commit_ctx(dst, src); +out: + damon_destroy_ctx(test_ctx); + return err; +} + /** * damon_nr_running_ctxs() - Return number of currently running contexts. */ @@ -1670,7 +1886,7 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) return sz; } -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r); /* @@ -1696,11 +1912,13 @@ static unsigned long damon_apply_min_nr_regions(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) { while (damon_sz_region(r) > max_region_sz) { - damon_split_region_at(t, r, max_region_sz); + if (damon_split_region_at(t, r, max_region_sz)) + goto out; r = damon_next_region(r); } } } +out: return max_region_sz; } @@ -1736,6 +1954,8 @@ static int __damon_start(struct damon_ctx *ctx) return err; } +static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src); + /** * damon_start() - Starts the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to start monitoring @@ -1757,8 +1977,16 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) int err = 0; for (i = 0; i < nr_ctxs; i++) { - if (!is_power_of_2(ctxs[i]->min_region_sz)) - return -EINVAL; + struct damon_ctx *test_ctx; + + test_ctx = damon_new_ctx(); + if (!test_ctx) + return -ENOMEM; + + err = __damon_commit_ctx(test_ctx, ctxs[i]); + damon_destroy_ctx(test_ctx); + if (err) + return err; } mutex_lock(&damon_lock); @@ -1778,16 +2006,17 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) running_exclusive_ctxs = true; mutex_unlock(&damon_lock); + if (i != nr_ctxs) + damon_stop(ctxs, i); + return err; } /* * __damon_stop() - Stops monitoring of a given context. * @ctx: monitoring context - * - * Return: 0 on success, negative error code otherwise. */ -static int __damon_stop(struct damon_ctx *ctx) +static void __damon_stop(struct damon_ctx *ctx) { struct task_struct *tsk; @@ -1797,31 +2026,23 @@ static int __damon_stop(struct damon_ctx *ctx) get_task_struct(tsk); mutex_unlock(&ctx->kdamond_lock); kthread_stop_put(tsk); - return 0; + return; } mutex_unlock(&ctx->kdamond_lock); - - return -EPERM; } /** * damon_stop() - Stops the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to stop monitoring * @nr_ctxs: size of @ctxs - * - * Return: 0 on success, negative error code otherwise. */ -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs) { - int i, err = 0; + int i; - for (i = 0; i < nr_ctxs; i++) { + for (i = 0; i < nr_ctxs; i++) /* nr_running_ctxs is decremented in kdamond_fn */ - err = __damon_stop(ctxs[i]); - if (err) - break; - } - return err; + __damon_stop(ctxs[i]); } /** @@ -1878,6 +2099,8 @@ int damon_kdamond_pid(struct damon_ctx *ctx) * @ctx has succeeded. Otherwise, this function could fall into an indefinite * wait. * + * When this function is failed, the @ctx is guaranteed to be stopped. + * * Return: 0 on success, negative error code otherwise. */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) @@ -1890,7 +2113,7 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) mutex_lock(&ctx->call_controls_lock); if (ctx->call_controls_obsolete) { mutex_unlock(&ctx->call_controls_lock); - return -ECANCELED; + goto canceled; } list_add_tail(&control->list, &ctx->call_controls); mutex_unlock(&ctx->call_controls_lock); @@ -1898,8 +2121,14 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) return 0; wait_for_completion(&control->completion); if (control->canceled) - return -ECANCELED; + goto canceled; return 0; + +canceled: + while (damon_is_running(ctx)) + schedule_timeout_idle(msecs_to_jiffies(100)); + return -ECANCELED; + } /** @@ -1949,36 +2178,6 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) } /* - * Warn and fix corrupted ->nr_accesses[_bp] for investigations and preventing - * the problem being propagated. - */ -static void damon_warn_fix_nr_accesses_corruption(struct damon_region *r) -{ - if (r->nr_accesses_bp == r->nr_accesses * 10000) - return; - WARN_ONCE(true, "invalid nr_accesses_bp at reset: %u %u\n", - r->nr_accesses_bp, r->nr_accesses); - r->nr_accesses_bp = r->nr_accesses * 10000; -} - -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ - WARN_ONCE(r->nr_accesses_bp != r->last_nr_accesses * 10000, - "nr_accesses_bp %u last_nr_accesses %u sis %lu %lu\n", - r->nr_accesses_bp, r->last_nr_accesses, - c->passed_sample_intervals, c->next_aggregation_sis); -} -#else -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ -} -#endif - - -/* * Reset the aggregated monitoring results ('nr_accesses' of each region). */ static void kdamond_reset_aggregated(struct damon_ctx *c) @@ -2002,12 +2201,12 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) trace_damon_aggregated(ti, r, damon_nr_regions(t)); trace_damon_region_aggregated(ti, r, damon_nr_regions(t), nr_probes); - damon_warn_fix_nr_accesses_corruption(r); r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; - for (i = 0; i < DAMON_MAX_PROBES; i++) + for (i = 0; i < DAMON_MAX_PROBES; i++) { + r->last_probe_hits[i] = r->probe_hits[i]; r->probe_hits[i] = 0; - damon_verify_reset_aggregated(r, c); + } } ti++; } @@ -2052,7 +2251,7 @@ static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c) return adaptation_bp; } -static void kdamond_tune_intervals(struct damon_ctx *c) +static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c) { unsigned long adaptation_bp; struct damon_attrs new_attrs; @@ -2074,10 +2273,11 @@ static void kdamond_tune_intervals(struct damon_ctx *c) damon_set_attrs(c, &new_attrs); } -static bool __damos_valid_target(struct damon_region *r, struct damos *s) +static bool __damos_valid_target(struct damon_region *r, struct damos *s, + struct damon_ctx *c) { unsigned long sz; - unsigned int nr_accesses = r->nr_accesses_bp / 10000; + unsigned int nr_accesses = damon_nr_accesses_mvsum(r, c); sz = damon_sz_region(r); return s->pattern.min_sz_region <= sz && @@ -2103,7 +2303,7 @@ static bool damos_quota_is_set(struct damos_quota *quota) static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r, struct damos *s) { - bool ret = __damos_valid_target(r, s); + bool ret = __damos_valid_target(r, s, c); if (!ret || !damos_quota_is_set(&s->quota) || !c->ops.get_scheme_score) return ret; @@ -2387,7 +2587,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, struct damos *siter; /* schemes iterator */ unsigned int sidx = 0; struct damon_target *titer; /* targets iterator */ - unsigned int tidx = 0; + unsigned int tidx = 0, nr_accesses = 0; bool do_trace = false; /* get indices for trace_damos_before_apply() */ @@ -2402,6 +2602,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, break; tidx++; } + nr_accesses = damon_nr_accesses_mvsum(r, c); do_trace = true; } @@ -2417,7 +2618,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (damos_core_filter_out(c, t, r, s)) return; ktime_get_coarse_ts64(&begin); - trace_damos_before_apply(cidx, sidx, tidx, r, + trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses, damon_nr_regions(t), do_trace); sz_applied = c->ops.apply_scheme(c, t, r, s, &sz_ops_filter_passed); @@ -2590,7 +2791,7 @@ static u64 damos_get_some_mem_psi_total(void) static inline u64 damos_get_some_mem_psi_total(void) { return 0; -}; +} #endif /* CONFIG_PSI */ @@ -2689,7 +2890,7 @@ static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c, damon_for_each_region(r, t) { phys_addr_t addr, end_addr; - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; /* Convert from core address units to physical bytes */ @@ -2978,7 +3179,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) (DAMOS_MAX_SCORE + 1)); damon_for_each_target(t, c) { damon_for_each_region(r, t) { - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; if (damos_core_filter_out(c, t, r, s)) continue; @@ -3082,7 +3283,6 @@ static void damon_merge_two_regions(struct damon_target *t, l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / (sz_l + sz_r); - l->nr_accesses_bp = l->nr_accesses * 10000; l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); l->ar.end = r->ar.end; /* todo: do this for only installed probes */ @@ -3093,19 +3293,15 @@ static void damon_merge_two_regions(struct damon_target *t, damon_destroy_region(r, t); } -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_merge_regions_of(struct damon_region *r) -{ - WARN_ONCE(r->nr_accesses != r->nr_accesses_bp / 10000, - "nr_accesses (%u) != nr_accesses_bp (%u)\n", - r->nr_accesses, r->nr_accesses_bp); -} -#else -static void damon_verify_merge_regions_of(struct damon_region *r) +static unsigned int damon_merge_score(struct damon_region *r, bool last, + struct damon_ctx *ctx, bool use_probe_hits) { + if (use_probe_hits) + return damon_probe_hits_wsum(r, last, ctx); + if (last) + return r->last_nr_accesses; + return r->nr_accesses; } -#endif - /* * Merge adjacent regions having similar access frequencies @@ -3115,25 +3311,40 @@ static void damon_verify_merge_regions_of(struct damon_region *r) * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit) + unsigned long sz_limit, struct damon_ctx *ctx, bool count_age) { struct damon_region *r, *prev = NULL, *next; + bool use_probe_hits = damon_has_probe_weights(ctx); damon_for_each_region_safe(r, next, t) { - damon_verify_merge_regions_of(r); - if (abs(r->nr_accesses - r->last_nr_accesses) > thres) - r->age = 0; - else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) - r->age = 0; - else - r->age++; + unsigned int score, last_score, diff; - if (prev && prev->ar.end == r->ar.start && - abs(prev->nr_accesses - r->nr_accesses) <= thres && - damon_sz_region(prev) + damon_sz_region(r) <= sz_limit) - damon_merge_two_regions(t, prev, r); - else - prev = r; + score = damon_merge_score(r, false, ctx, use_probe_hits); + last_score = damon_merge_score(r, true, ctx, use_probe_hits); + + if (count_age) { + if (abs_diff(score, last_score) > thres) + r->age = 0; + else if ((score == 0) != (last_score == 0)) + r->age = 0; + else + r->age++; + } + + if (!prev) + goto set_prev_continue; + if (prev->ar.end != r->ar.start) + goto set_prev_continue; + diff = abs_diff(score, damon_merge_score(prev, false, ctx, + use_probe_hits)); + if (diff > thres) + goto set_prev_continue; + if (damon_sz_region(prev) + damon_sz_region(r) > sz_limit) + goto set_prev_continue; + damon_merge_two_regions(t, prev, r); + continue; +set_prev_continue: + prev = r; } } @@ -3160,18 +3371,26 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, struct damon_target *t; unsigned int nr_regions; unsigned int max_thres; + bool count_age = true; max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); - do { + while (true) { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(t, threshold, sz_limit, c, + count_age); nr_regions += damon_nr_regions(t); } - threshold = max(1, threshold * 2); - } while (nr_regions > c->attrs.max_nr_regions && - threshold / 2 < max_thres); + count_age = false; + if (nr_regions <= c->attrs.max_nr_regions || + max_thres <= threshold) + break; + if (threshold < max_thres / 2) + threshold = max(1, threshold * 2); + else + threshold = max_thres; + } } #ifdef CONFIG_DAMON_DEBUG_SANITY @@ -3194,8 +3413,10 @@ static void damon_verify_split_region_at(struct damon_region *r, * * r the region to be split * sz_r size of the first sub-region that will be made + * + * Return: 0 on success, negative error code otherwise. */ -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r) { struct damon_region *new; @@ -3203,18 +3424,20 @@ static void damon_split_region_at(struct damon_target *t, damon_verify_split_region_at(r, sz_r); new = damon_new_region(r->ar.start + sz_r, r->ar.end); if (!new) - return; + return -ENOMEM; r->ar.end = new->ar.start; new->age = r->age; new->last_nr_accesses = r->last_nr_accesses; - new->nr_accesses_bp = r->nr_accesses_bp; new->nr_accesses = r->nr_accesses; /* todo: do this for only installed probes */ memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); + memcpy(new->last_probe_hits, r->last_probe_hits, + sizeof(r->last_probe_hits)); damon_insert_region(new, r, damon_next_region(r), t); + return 0; } /* Split every region in the given target into 'nr_subs' regions */ @@ -3247,6 +3470,37 @@ static void damon_split_regions_of(struct damon_ctx *ctx, } } +/* Split one in every @split_step regions into two, from a rotating offset */ +static void damon_split_some_regions(struct damon_ctx *ctx, + unsigned long split_step) +{ + static unsigned long rotation; + struct damon_target *t; + struct damon_region *r, *next; + unsigned long offset = rotation++ % split_step; + unsigned long idx = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region_safe(r, next, t) { + unsigned long sz_region, sz_sub; + + if (idx++ % split_step != offset) + continue; + sz_region = damon_sz_region(r); + if (sz_region < 2 * ctx->min_region_sz) + continue; + + sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) * + sz_region / 10, ctx->min_region_sz); + /* Do not allow blank region */ + if (sz_sub == 0 || sz_sub >= sz_region) + continue; + + damon_split_region_at(t, r, sz_sub); + } + } +} + /* * Split every target region into randomly-sized small regions * @@ -3260,25 +3514,33 @@ static void damon_split_regions_of(struct damon_ctx *ctx, static void kdamond_split_regions(struct damon_ctx *ctx) { struct damon_target *t; - unsigned int nr_regions = 0; - static unsigned int last_nr_regions; + unsigned long nr_regions = 0; + unsigned long max_nr_regions = ctx->attrs.max_nr_regions; + static unsigned long last_nr_regions; int nr_subregions = 2; damon_for_each_target(t, ctx) nr_regions += damon_nr_regions(t); - if (nr_regions > ctx->attrs.max_nr_regions / 2) - return; + if (nr_regions >= max_nr_regions) + goto done; + + if (nr_regions > max_nr_regions / 2) { + damon_split_some_regions(ctx, + max_nr_regions / (max_nr_regions - nr_regions)); + goto done; + } /* Maybe the middle of the region has different access frequency */ if (last_nr_regions == nr_regions && - nr_regions < ctx->attrs.max_nr_regions / 3) + nr_regions < max_nr_regions / 3) nr_subregions = 3; damon_for_each_target(t, ctx) damon_split_regions_of(ctx, t, nr_subregions, ctx->min_region_sz); +done: last_nr_regions = nr_regions; } @@ -3490,7 +3752,6 @@ static void kdamond_init_ctx(struct damon_ctx *ctx) static int kdamond_fn(void *data) { struct damon_ctx *ctx = data; - unsigned int max_nr_accesses = 0; unsigned long sz_limit = 0; pr_debug("kdamond (%d) starts\n", current->pid); @@ -3523,25 +3784,38 @@ static int kdamond_fn(void *data) unsigned long next_aggregation_sis = ctx->next_aggregation_sis; unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; + bool access_check_disabled = damon_has_probe_weights(ctx); + unsigned int max_merge_score = 0, max_wsum; + bool get_max_wsum; if (kdamond_wait_activation(ctx)) break; - if (ctx->ops.prepare_access_checks) + if (!access_check_disabled && ctx->ops.prepare_access_checks) ctx->ops.prepare_access_checks(ctx); kdamond_usleep(sample_interval); ctx->passed_sample_intervals++; - if (ctx->ops.check_accesses) - max_nr_accesses = ctx->ops.check_accesses(ctx); - if (ctx->ops.apply_probes) - ctx->ops.apply_probes(ctx); + if (!access_check_disabled && ctx->ops.check_accesses) + max_merge_score = ctx->ops.check_accesses(ctx); + if (ctx->ops.apply_probes) { + if (time_after_eq(ctx->passed_sample_intervals, + next_aggregation_sis) && + access_check_disabled) + get_max_wsum = true; + else + get_max_wsum = false; + max_wsum = ctx->ops.apply_probes(ctx, + access_check_disabled, get_max_wsum); + if (get_max_wsum) + max_merge_score = max_wsum; + } if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { kdamond_merge_regions(ctx, - max_nr_accesses / 10, + max_merge_score / 10, sz_limit); /* online updates might be made */ sz_limit = damon_apply_min_nr_regions(ctx); @@ -3584,8 +3858,7 @@ static int kdamond_fn(void *data) * aggregation, and make aggregation * information reset for all regions. Then, * following kdamond_reset_aggregated() call - * will make the region information invalid, - * particularly for ->nr_accesses_bp. + * will make the region information invalid. * * Reset ->next_aggregation_sis to avoid that. * It will anyway correctly updated after this @@ -3712,9 +3985,6 @@ int damon_set_region_system_rams_default(struct damon_target *t, { struct damon_addr_range addr_range; - if (*start > *end) - return -EINVAL; - if (!*start && !*end && !damon_find_system_rams_range(start, end, addr_unit)) return -EINVAL; @@ -3724,72 +3994,18 @@ int damon_set_region_system_rams_default(struct damon_target *t, return damon_set_regions(t, &addr_range, 1, min_region_sz); } -/* - * damon_moving_sum() - Calculate an inferred moving sum value. - * @mvsum: Inferred sum of the last @len_window values. - * @nomvsum: Non-moving sum of the last discrete @len_window window values. - * @len_window: The number of last values to take care of. - * @new_value: New value that will be added to the pseudo moving sum. - * - * Moving sum (moving average * window size) is good for handling noise, but - * the cost of keeping past values can be high for arbitrary window size. This - * function implements a lightweight pseudo moving sum function that doesn't - * keep the past window values. - * - * It simply assumes there was no noise in the past, and get the no-noise - * assumed past value to drop from @nomvsum and @len_window. @nomvsum is a - * non-moving sum of the last window. For example, if @len_window is 10 and we - * have 25 values, @nomvsum is the sum of the 11th to 20th values of the 25 - * values. Hence, this function simply drops @nomvsum / @len_window from - * given @mvsum and add @new_value. - * - * For example, if @len_window is 10 and @nomvsum is 50, the last 10 values for - * the last window could be vary, e.g., 0, 10, 0, 10, 0, 10, 0, 0, 0, 20. For - * calculating next moving sum with a new value, we should drop 0 from 50 and - * add the new value. However, this function assumes it got value 5 for each - * of the last ten times. Based on the assumption, when the next value is - * measured, it drops the assumed past value, 5 from the current sum, and add - * the new value to get the updated pseduo-moving average. - * - * This means the value could have errors, but the errors will be disappeared - * for every @len_window aligned calls. For example, if @len_window is 10, the - * pseudo moving sum with 11th value to 19th value would have an error. But - * the sum with 20th value will not have the error. - * - * Return: Pseudo-moving average after getting the @new_value. - */ -static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum, - unsigned int len_window, unsigned int new_value) -{ - return mvsum - nomvsum / len_window + new_value; -} - /** * damon_update_region_access_rate() - Update the access rate of a region. * @r: The DAMON region to update for its access check result. * @accessed: Whether the region has accessed during last sampling interval. - * @attrs: The damon_attrs of the DAMON context. * * Update the access rate of a region with the region's last sampling interval * access check result. * * Usually this will be called by &damon_operations->check_accesses callback. */ -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs) +void damon_update_region_access_rate(struct damon_region *r, bool accessed) { - unsigned int len_window = 1; - - /* - * sample_interval can be zero, but cannot be larger than - * aggr_interval, owing to validation of damon_set_attrs(). - */ - if (attrs->sample_interval) - len_window = damon_max_nr_accesses(attrs); - r->nr_accesses_bp = damon_moving_sum(r->nr_accesses_bp, - r->last_nr_accesses * 10000, len_window, - accessed ? 10000 : 0); - if (accessed) r->nr_accesses++; } diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 32f41491b726..7e077084cb03 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -284,11 +284,6 @@ static int damon_lru_sort_apply_parameters(void) param_ctx->addr_unit = addr_unit; param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); - if (!is_power_of_2(param_ctx->min_region_sz)) { - err = -EINVAL; - goto out; - } - if (!damon_lru_sort_mon_attrs.sample_interval) { err = -EINVAL; goto out; @@ -308,7 +303,7 @@ static int damon_lru_sort_apply_parameters(void) goto out; err = -ENOMEM; - hot_thres = damon_max_nr_accesses(&attrs) * + hot_thres = damon_nr_samples_per_aggr(&attrs) * hot_thres_access_freq / 1000; hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); if (!hot_scheme) @@ -414,8 +409,10 @@ static int damon_lru_sort_turn(bool on) { int err; - if (!on) - return damon_stop(&ctx, 1); + if (!on) { + damon_stop(&ctx, 1); + return 0; + } err = damon_lru_sort_apply_parameters(); if (err) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 6bdd1cfd3863..e59f77eca83b 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -111,8 +111,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_max_nr_accesses(&c->attrs); + freq_subscore = mult_frac(damon_nr_accesses_mvsum(r, c), + DAMON_MAX_SUBSCORE, + damon_nr_samples_per_aggr(&c->attrs)); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; if (age_in_sec) @@ -311,7 +312,7 @@ static unsigned int __damon_migrate_folio_list( * instead of migrated. */ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | - __GFP_NOMEMALLOC | GFP_NOWAIT, + __GFP_NOMEMALLOC | GFP_NOWAIT | __GFP_THISNODE, .nid = target_nid, }; @@ -339,8 +340,6 @@ static unsigned int damon_migrate_folio_list(struct list_head *folio_list, LIST_HEAD(migrate_folios); while (!list_empty(folio_list)) { - struct folio *folio; - cond_resched(); folio = lru_to_folio(folio_list); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5c2da45f988c..5a6a78054784 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -65,7 +65,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -74,31 +74,19 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) return false; accessed = damon_folio_young(folio); - *folio_sz = folio_size(folio); folio_put(folio); return accessed; } static void __damon_pa_check_access(struct damon_region *r, - struct damon_attrs *attrs, unsigned long addr_unit) + unsigned long addr_unit) { - static phys_addr_t last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; phys_addr_t sampling_addr = damon_pa_phys_addr( r->sampling_addr, addr_unit); - /* If the region is in the last checked page, reuse the result */ - if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = sampling_addr; + accessed = damon_pa_young(sampling_addr); + damon_update_region_access_rate(r, accessed); } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) @@ -109,8 +97,7 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { - __damon_pa_check_access( - r, &ctx->attrs, ctx->addr_unit); + __damon_pa_check_access(r, ctx->addr_unit); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } } @@ -167,11 +154,13 @@ static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, return pass; } -static void damon_pa_apply_probes(struct damon_ctx *ctx) +static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, + bool set_samples, bool return_max_wsum) { struct damon_target *t; struct damon_region *r; struct damon_probe *p; + unsigned int max_wsum = 0; damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { @@ -179,6 +168,9 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx) phys_addr_t pa; struct folio *folio; + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); pa = damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit); folio = damon_get_folio(PHYS_PFN(pa)); @@ -189,8 +181,12 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx) } if (folio) folio_put(folio); + if (return_max_wsum) + max_wsum = max(damon_probe_hits_wsum(r, false, + ctx), max_wsum); } } + return max_wsum; } /* @@ -451,6 +447,6 @@ static int __init damon_pa_initcall(void) }; return damon_register_ops(&ops); -}; +} subsys_initcall(damon_pa_initcall); diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 11b70d0a9a6f..09e941d75f67 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -208,11 +208,6 @@ static int damon_reclaim_apply_parameters(void) param_ctx->addr_unit = addr_unit; param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); - if (!is_power_of_2(param_ctx->min_region_sz)) { - err = -EINVAL; - goto out; - } - if (!damon_reclaim_mon_attrs.aggr_interval) { err = -EINVAL; goto out; @@ -337,8 +332,10 @@ static int damon_reclaim_turn(bool on) { int err; - if (!on) - return damon_stop(&ctx, 1); + if (!on) { + damon_stop(&ctx, 1); + return 0; + } err = damon_reclaim_apply_parameters(); if (err) diff --git a/mm/damon/stat.c b/mm/damon/stat.c index 0e14f5bb8f75..b05b68f73e10 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -138,7 +138,7 @@ static int damon_stat_damon_call_fn(void *data) /* avoid unnecessarily frequent stat update */ if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies + - msecs_to_jiffies(5 * MSEC_PER_SEC))) + secs_to_jiffies(5))) return 0; damon_stat_last_refresh_jiffies = jiffies; diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3cbeccd436e4..32f495a96b17 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -110,7 +110,8 @@ static int damos_sysfs_probes_add_dirs(struct damos_sysfs_probes *probes, struct damos_sysfs_probe *sys_probe; int err; - sys_probe = damos_sysfs_probe_alloc(region->probe_hits[i]); + sys_probe = damos_sysfs_probe_alloc( + damon_probe_hits_mvsum(i, region, ctx)); if (!sys_probe) { damos_sysfs_probes_rm_dirs(probes); return -ENOMEM; @@ -157,7 +158,7 @@ struct damon_sysfs_scheme_region { }; static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( - struct damon_region *region) + struct damon_region *region, struct damon_ctx *ctx) { struct damon_sysfs_scheme_region *sysfs_region = kmalloc_obj(*sysfs_region); @@ -165,7 +166,7 @@ static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( return NULL; sysfs_region->kobj = (struct kobject){}; sysfs_region->ar = region->ar; - sysfs_region->nr_accesses = region->nr_accesses_bp / 10000; + sysfs_region->nr_accesses = damon_nr_accesses_mvsum(region, ctx); sysfs_region->age = region->age; sysfs_region->probes = NULL; INIT_LIST_HEAD(&sysfs_region->list); @@ -332,6 +333,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { damos_sysfs_region_rm_dirs(r); list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } @@ -911,8 +913,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1460,8 +1464,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; @@ -2138,8 +2144,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; @@ -2681,6 +2689,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2722,13 +2731,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); @@ -3112,7 +3123,7 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, if (total_bytes_only) return; - region = damon_sysfs_scheme_region_alloc(r); + region = damon_sysfs_scheme_region_alloc(r, ctx); if (!region) return; region->sz_filter_passed = sz_filter_passed; @@ -3122,12 +3133,14 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, sysfs_regions->nr_regions)) goto out; if (damos_sysfs_region_add_dirs(region, ctx, r)) - goto out; + goto del_out; list_add_tail(®ion->list, &sysfs_regions->regions_list); sysfs_regions->nr_regions++; return; +del_out: + kobject_del(®ion->kobj); out: kobject_put(®ion->kobj); } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index a9e187158067..e3858ffab4b2 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -105,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -331,6 +333,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -369,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); @@ -955,8 +960,10 @@ static void damon_sysfs_filters_rm_dirs(struct damon_sysfs_filters *filters) struct damon_sysfs_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1058,6 +1065,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = { struct damon_sysfs_probe { struct kobject kobj; + unsigned int weight; struct damon_sysfs_filters *filters; }; @@ -1066,7 +1074,7 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void) return kzalloc_obj(struct damon_sysfs_probe); } -static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) +static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe) { struct damon_sysfs_filters *filters; int err; @@ -1074,31 +1082,54 @@ static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) filters = damon_sysfs_filters_alloc(); if (!filters) return -ENOMEM; - attr->filters = filters; + probe->filters = filters; err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype, - &attr->kobj, "filters"); + &probe->kobj, "filters"); if (err) { kobject_put(&filters->kobj); - attr->filters = NULL; + probe->filters = NULL; } return err; } -static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *attr) +static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { - if (attr->filters) { - damon_sysfs_filters_rm_dirs(attr->filters); - kobject_put(&attr->filters->kobj); + if (probe->filters) { + damon_sysfs_filters_rm_dirs(probe->filters); + kobject_put(&probe->filters->kobj); } } +static ssize_t weight_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct damon_sysfs_probe *probe = container_of(kobj, + struct damon_sysfs_probe, kobj); + + return sysfs_emit(buf, "%u\n", probe->weight); +} + +static ssize_t weight_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_probe *probe = container_of(kobj, + struct damon_sysfs_probe, kobj); + int err = kstrtouint(buf, 0, &probe->weight); + + return err ? err : count; +} + static void damon_sysfs_probe_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_probe, kobj)); } +static struct kobj_attribute damon_sysfs_probe_weight_attr = + __ATTR_RW_MODE(weight, 0600); + static struct attribute *damon_sysfs_probe_attrs[] = { + &damon_sysfs_probe_weight_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_probe); @@ -1132,6 +1163,7 @@ static void damon_sysfs_probes_rm_dirs( for (i = 0; i < probes->nr; i++) { damon_sysfs_probe_rm_dirs(probes_arr[i]); + kobject_del(&probes_arr[i]->kobj); kobject_put(&probes_arr[i]->kobj); } probes->nr = 0; @@ -1173,6 +1205,7 @@ static int damon_sysfs_probes_add_dirs( err = damon_sysfs_probe_add_dirs(probe); if (err) { + kobject_del(&probe->kobj); kobject_put(&probe->kobj); damon_sysfs_probes_rm_dirs(probes); return err; @@ -1640,6 +1673,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1678,13 +1712,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -1897,47 +1933,66 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } -static int damon_sysfs_set_probes(struct damon_ctx *ctx, - struct damon_sysfs_probes *sys_probes) +static int damon_sysfs_set_filters(struct damon_probe *probe, + struct damon_sysfs_filters *sys_filters) { int i; - for (i = 0; i < sys_probes->nr; i++) { - struct damon_sysfs_filters *sys_filters = - sys_probes->probes_arr[i]->filters; - struct damon_probe *c; - int j; + for (i = 0; i < sys_filters->nr; i++) { + struct damon_sysfs_filter *sys_filter = + sys_filters->filters_arr[i]; + struct damon_filter *filter; - if (!sys_filters) - continue; - c = damon_new_probe(); - if (!c) + filter = damon_new_filter(sys_filter->type, + sys_filter->matching, + sys_filter->allow); + if (!filter) return -ENOMEM; - damon_add_probe(ctx, c); - - for (j = 0; j < sys_filters->nr; j++) { - struct damon_sysfs_filter *sys_filter = - sys_filters->filters_arr[j]; - struct damon_filter *filter; - - filter = damon_new_filter(sys_filter->type, - sys_filter->matching, - sys_filter->allow); - if (!filter) - return -ENOMEM; - if (filter->type == DAMON_FILTER_TYPE_MEMCG) { - int err; - - err = damon_sysfs_memcg_path_to_id( - sys_filter->path, - &filter->memcg_id); - if (err) { - damon_destroy_filter(filter); - return err; - } + if (filter->type == DAMON_FILTER_TYPE_MEMCG) { + int err; + + err = damon_sysfs_memcg_path_to_id( + sys_filter->path, + &filter->memcg_id); + if (err) { + damon_destroy_filter(filter); + return err; } - damon_add_filter(c, filter); } + damon_add_filter(probe, filter); + } + return 0; +} + +static int damon_sysfs_set_probe(struct damon_probe *probe, + struct damon_sysfs_probe *sys_probe) +{ + struct damon_sysfs_filters *sys_filters; + + sys_filters = sys_probe->filters; + if (!sys_filters) + return 0; + return damon_sysfs_set_filters(probe, sys_filters); +} + +static int damon_sysfs_set_probes(struct damon_ctx *ctx, + struct damon_sysfs_probes *sys_probes) +{ + int i, err; + + for (i = 0; i < sys_probes->nr; i++) { + struct damon_sysfs_probe *sys_probe; + struct damon_probe *p; + + p = damon_new_probe(); + if (!p) + return -ENOMEM; + damon_add_probe(ctx, p); + sys_probe = sys_probes->probes_arr[i]; + p->weight = sys_probe->weight; + err = damon_sysfs_set_probe(p, sys_probe); + if (err) + return err; } return 0; } @@ -1957,9 +2012,6 @@ static int damon_sysfs_set_regions(struct damon_target *t, struct damon_sysfs_region *sys_region = sysfs_regions->regions_arr[i]; - if (sys_region->ar.start > sys_region->ar.end) - goto out; - ranges[i].start = sys_region->ar.start; ranges[i].end = sys_region->ar.end; if (i == 0) @@ -1999,7 +2051,7 @@ static int damon_sysfs_add_targets(struct damon_ctx *ctx, int i, err; /* Multiple physical address space monitoring targets makes no sense */ - if (ctx->ops.id == DAMON_OPS_PADDR && sysfs_targets->nr > 1) + if (!damon_target_has_pid(ctx) && sysfs_targets->nr > 1) return -EINVAL; for (i = 0; i < sysfs_targets->nr; i++) { @@ -2042,16 +2094,18 @@ static inline bool damon_sysfs_kdamond_running( static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, struct damon_sysfs_context *sys_ctx) { + enum damon_ops_id ops_id; int err; - err = damon_select_ops(ctx, sys_ctx->ops_id); + ops_id = READ_ONCE(sys_ctx->ops_id); + err = damon_select_ops(ctx, ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ - if (sys_ctx->ops_id == DAMON_OPS_PADDR) + if (ops_id == DAMON_OPS_PADDR) ctx->min_region_sz = max( - DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1); + DAMON_MIN_REGION_SZ / ctx->addr_unit, 1); ctx->pause = sys_ctx->pause; err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) @@ -2069,26 +2123,6 @@ static struct damon_ctx *damon_sysfs_build_ctx( struct damon_sysfs_context *sys_ctx); /* - * Return a new damon_ctx for testing new parameters to commit. - */ -static struct damon_ctx *damon_sysfs_new_test_ctx( - struct damon_ctx *running_ctx) -{ - struct damon_ctx *test_ctx; - int err; - - test_ctx = damon_new_ctx(); - if (!test_ctx) - return NULL; - err = damon_commit_ctx(test_ctx, running_ctx); - if (err) { - damon_destroy_ctx(test_ctx); - return NULL; - } - return test_ctx; -} - -/* * damon_sysfs_commit_input() - Commit user inputs to a running kdamond. * @kdamond: The kobject wrapper for the associated kdamond. * @@ -2097,7 +2131,7 @@ static struct damon_ctx *damon_sysfs_new_test_ctx( static int damon_sysfs_commit_input(void *data) { struct damon_sysfs_kdamond *kdamond = data; - struct damon_ctx *param_ctx, *test_ctx; + struct damon_ctx *param_ctx; int err; if (!damon_sysfs_kdamond_running(kdamond)) @@ -2109,17 +2143,7 @@ static int damon_sysfs_commit_input(void *data) param_ctx = damon_sysfs_build_ctx(kdamond->contexts->contexts_arr[0]); if (IS_ERR(param_ctx)) return PTR_ERR(param_ctx); - test_ctx = damon_sysfs_new_test_ctx(kdamond->damon_ctx); - if (!test_ctx) { - damon_destroy_ctx(param_ctx); - return -ENOMEM; - } - err = damon_commit_ctx(test_ctx, param_ctx); - if (err) - goto out; err = damon_commit_ctx(kdamond->damon_ctx, param_ctx); -out: - damon_destroy_ctx(test_ctx); damon_destroy_ctx(param_ctx); return err; } @@ -2265,12 +2289,13 @@ static int damon_sysfs_turn_damon_off(struct damon_sysfs_kdamond *kdamond) { if (!kdamond->damon_ctx) return -EINVAL; - return damon_stop(&kdamond->damon_ctx, 1); + damon_stop(&kdamond->damon_ctx, 1); /* * To allow users show final monitoring results of already turned-off * DAMON, we free kdamond->damon_ctx in next * damon_sysfs_turn_damon_on(), or kdamonds_nr_store() */ + return 0; } static int damon_sysfs_damon_call(int (*fn)(void *data), @@ -2499,6 +2524,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -2553,13 +2579,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fcf7c7fadb5f..4a536d41cdb2 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -114,7 +114,6 @@ static void damon_test_aggregate(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = accesses[it][ir]; - r->nr_accesses_bp = accesses[it][ir] * 10000; damon_add_region(r, t); } it++; @@ -151,12 +150,15 @@ static void damon_test_split_at(struct kunit *test) damon_free_target(t); kunit_skip(test, "region alloc fail"); } - r->nr_accesses_bp = 420000; r->nr_accesses = 42; r->last_nr_accesses = 15; r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -164,11 +166,11 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r_new->ar.start, 25ul); KUNIT_EXPECT_EQ(test, r_new->ar.end, 100ul); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, r_new->nr_accesses_bp); KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); +out: damon_free_target(t); } @@ -187,7 +189,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = 10; - r->nr_accesses_bp = 100000; r->age = 9; damon_add_region(r, t); r2 = damon_new_region(100, 300); @@ -196,7 +197,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "second region alloc fail"); } r2->nr_accesses = 20; - r2->nr_accesses_bp = 200000; r2->age = 21; damon_add_region(r2, t); @@ -204,7 +204,6 @@ static void damon_test_merge_two(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, 160000u); KUNIT_EXPECT_EQ(test, r->age, 17u); i = 0; @@ -232,6 +231,7 @@ static struct damon_region *__nth_region_of(struct damon_target *t, int idx) static void damon_test_merge_regions_of(struct kunit *test) { + struct damon_ctx *ctx; struct damon_target *t; struct damon_region *r; unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184, 230}; @@ -242,29 +242,39 @@ static void damon_test_merge_regions_of(struct kunit *test) unsigned long eaddrs[] = {112, 130, 156, 170, 230, 10170}; int i; + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + t = damon_new_target(); - if (!t) + if (!t) { + damon_destroy_ctx(ctx); kunit_skip(test, "target alloc fail"); + } for (i = 0; i < ARRAY_SIZE(sa); i++) { r = damon_new_region(sa[i], ea[i]); if (!r) { damon_free_target(t); + damon_destroy_ctx(ctx); kunit_skip(test, "region alloc fail"); } r->nr_accesses = nrs[i]; - r->nr_accesses_bp = nrs[i] * 10000; damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999); + damon_merge_regions_of(t, 9, 9999, ctx, true); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); + if (damon_nr_regions(t) != 6) + goto out; for (i = 0; i < 6; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } +out: damon_free_target(t); + damon_destroy_ctx(ctx); } static void damon_test_split_regions_of(struct kunit *test) @@ -335,6 +345,69 @@ static void damon_test_split_regions_of(struct kunit *test) damon_destroy_ctx(c); } +/* + * When the total region count is already above max_nr_regions / 2, + * kdamond_split_regions() must keep refining the resolution by splitting a + * fraction of the regions (making progress), without exceeding + * max_nr_regions. + */ +static void damon_test_split_above_half_progresses(struct kunit *test) +{ + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + unsigned long start; + unsigned int nr_before, nr_after, i; + const unsigned int nr_init = 760; + const unsigned long region_sz = 100; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + /* Keep the split arithmetic independent of the page size */ + c->min_region_sz = 1; + c->attrs.min_nr_regions = 10; + c->attrs.max_nr_regions = 1500; + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + + for (i = 0; i < nr_init; i++) { + start = i * region_sz; + r = damon_new_region(start, start + region_sz); + if (!r) { + damon_free_target(t); + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + r->nr_accesses = (i & 1) ? 0 : 100; + r->age = 5; + damon_add_region(r, t); + } + + damon_add_target(c, t); + + nr_before = damon_nr_regions(t); + /* Above max_nr_regions / 2, so the blanket-split path is skipped */ + KUNIT_EXPECT_GT(test, (unsigned long)nr_before, + c->attrs.max_nr_regions / 2); + + kdamond_split_regions(c); + + nr_after = damon_nr_regions(t); + /* Still made progress ... */ + KUNIT_EXPECT_GT(test, nr_after, nr_before); + /* ... but did not overshoot the configured maximum */ + KUNIT_EXPECT_LE(test, (unsigned long)nr_after, + c->attrs.max_nr_regions); + + damon_destroy_ctx(c); +} + static void damon_test_ops_registration(struct kunit *test) { struct damon_ctx *c = damon_new_ctx(); @@ -519,61 +592,45 @@ static void damon_test_set_regions(struct kunit *test) }, 3); } -static void damon_test_nr_accesses_to_accesses_bp(struct kunit *test) -{ - struct damon_attrs attrs = { - .sample_interval = 10, - .aggr_interval = ((unsigned long)UINT_MAX + 1) * 10 - }; - - /* - * In some cases such as 32bit architectures where UINT_MAX is - * ULONG_MAX, attrs.aggr_interval becomes zero. Calling - * damon_nr_accesses_to_accesses_bp() in the case will cause - * divide-by-zero. Such case is prohibited in normal execution since - * the caution is documented on the comment for the function, and - * damon_update_monitoring_results() does the check. Skip the test in - * the case. - */ - if (!attrs.aggr_interval) - kunit_skip(test, "aggr_interval is zero."); - - KUNIT_EXPECT_EQ(test, damon_nr_accesses_to_accesses_bp(123, &attrs), 0); -} - static void damon_test_update_monitoring_result(struct kunit *test) { struct damon_attrs old_attrs = { .sample_interval = 10, .aggr_interval = 1000,}; struct damon_attrs new_attrs; struct damon_region *r = damon_new_region(3, 7); + struct damon_ctx *ctx; if (!r) kunit_skip(test, "region alloc fail"); + ctx = damon_new_ctx(); + if (!ctx) { + damon_free_region(r); + kunit_skip(test, "ctx alloc fail"); + } r->nr_accesses = 15; - r->nr_accesses_bp = 150000; r->age = 20; new_attrs = (struct damon_attrs){ .sample_interval = 100, .aggr_interval = 10000,}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 15); KUNIT_EXPECT_EQ(test, r->age, 2); new_attrs = (struct damon_attrs){ .sample_interval = 1, .aggr_interval = 1000}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 150); KUNIT_EXPECT_EQ(test, r->age, 2); new_attrs = (struct damon_attrs){ .sample_interval = 1, .aggr_interval = 100}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 150); KUNIT_EXPECT_EQ(test, r->age, 20); damon_free_region(r); + damon_destroy_ctx(ctx); } static void damon_test_set_attrs(struct kunit *test) @@ -604,19 +661,76 @@ static void damon_test_set_attrs(struct kunit *test) damon_destroy_ctx(c); } -static void damon_test_moving_sum(struct kunit *test) +static void damon_test_mvsum(struct kunit *test) +{ + unsigned long input_expects[] = { + /* current value, last value, remaining window (bp) */ + 0, 49, 10000, 49, /* 0 + 49 * 1 */ + 3, 10, 7000, 10, /* 3 + 10 * 0.7 */ + 3, 10, 5000, 8, /* 3 + 10 * 0.5 */ + 32, 100, 1000, 42, /* 32 + 100 * 0.1 */ + 42, 49, 0, 42, /* 42 + 49 * 0 */ + }; + + int i; + + for (i = 0; i < ARRAY_SIZE(input_expects); i += 4) { + unsigned long current_nr = input_expects[i]; + unsigned long last_nr = input_expects[i + 1]; + unsigned long left_window_bp = input_expects[i + 2]; + unsigned long expect = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, damon_mvsum(current_nr, last_nr, + left_window_bp), expect); + } +} + +/* + * Test damon_nr_accesses_mvsum(), which wraps damon_mvsum() with the + * monitoring intervals of the context. With a sample interval of 1 and an + * aggregation interval of 10, an aggregation window is 10 sample intervals + * long. Each row below specifies the passed sample intervals, the next + * aggregation time in sample intervals, the current and last nr_accesses of a + * region, and the expected return value. + */ +static void damon_test_nr_accesses_mvsum(struct kunit *test) { - unsigned int mvsum = 50000, nomvsum = 50000, len_window = 10; - unsigned int new_values[] = {10000, 0, 10000, 0, 0, 0, 10000, 0, 0, 0}; - unsigned int expects[] = {55000, 50000, 55000, 50000, 45000, 40000, - 45000, 40000, 35000, 30000}; + unsigned long input_expects[] = { + /* passed, next_aggr, nr_accesses, last_nr_accesses, expect */ + 0, 10, 5, 3, 3, /* full window left, unreset */ + 0, 10, 0, 7, 7, /* full window left, reset */ + 5, 10, 3, 10, 8, /* half window left */ + 8, 10, 3, 10, 5, /* 20% window left */ + 10, 10, 42, 49, 42, /* no window left */ + }; + struct damon_ctx *c = damon_new_ctx(); + struct damon_region *r; int i; - for (i = 0; i < ARRAY_SIZE(new_values); i++) { - mvsum = damon_moving_sum(mvsum, nomvsum, len_window, - new_values[i]); - KUNIT_EXPECT_EQ(test, mvsum, expects[i]); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + r = damon_new_region(0, 4096); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + + c->attrs.sample_interval = 1; + c->attrs.aggr_interval = 10; + + for (i = 0; i < ARRAY_SIZE(input_expects); i += 5) { + c->passed_sample_intervals = input_expects[i]; + c->next_aggregation_sis = input_expects[i + 1]; + r->nr_accesses = input_expects[i + 2]; + r->last_nr_accesses = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, (unsigned int)input_expects[i + 4], + damon_nr_accesses_mvsum(r, c)); } + + damon_free_region(r); + damon_destroy_ctx(c); } static void damos_test_new_filter(struct kunit *test) @@ -725,6 +839,7 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, struct damos_quota_goal *goal, *next; bool skip = true; int i; + int nr_dst = 0, nr_src = 0; INIT_LIST_HEAD(&dst.goals); INIT_LIST_HEAD(&src.goals); @@ -747,6 +862,14 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, damos_commit_quota_goals(&dst, &src); + damos_for_each_quota_goal(goal, &dst) + nr_dst++; + damos_for_each_quota_goal(goal, &src) + nr_src++; + KUNIT_EXPECT_EQ(test, nr_dst, nr_src); + if (nr_dst != nr_src) + goto out; + i = 0; damos_for_each_quota_goal(goal, (&dst)) { KUNIT_EXPECT_EQ(test, goal->metric, src_goals[i].metric); @@ -894,6 +1017,8 @@ static void damos_test_commit_dests_for(struct kunit *test, skip = false; KUNIT_EXPECT_EQ(test, dst.nr_dests, src_nr_dests); + if (dst.nr_dests != src_nr_dests) + goto out; for (i = 0; i < dst.nr_dests; i++) { KUNIT_EXPECT_EQ(test, dst.node_id_arr[i], src_node_id_arr[i]); KUNIT_EXPECT_EQ(test, dst.weight_arr[i], src_weight_arr[i]); @@ -1152,14 +1277,19 @@ static void damon_test_commit_target_regions_for(struct kunit *test, kunit_skip(test, "src target setup fail"); } damon_commit_target_regions(dst_target, src_target, 1); + + KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); + if (damon_nr_regions(dst_target) != nr_expect_regions) + goto out; + i = 0; damon_for_each_region(r, dst_target) { KUNIT_EXPECT_EQ(test, r->ar.start, expect_start_end[i][0]); KUNIT_EXPECT_EQ(test, r->ar.end, expect_start_end[i][1]); i++; } - KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); - KUNIT_EXPECT_EQ(test, i, nr_expect_regions); + +out: damon_free_target(dst_target); damon_free_target(src_target); } @@ -1251,6 +1381,8 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 1); KUNIT_EXPECT_EQ(test, r->ar.end, 2); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 2); KUNIT_EXPECT_EQ(test, r2->ar.end, 4); @@ -1265,11 +1397,14 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 2); KUNIT_EXPECT_EQ(test, r->ar.end, 6); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 6); KUNIT_EXPECT_EQ(test, r2->ar.end, 8); damon_destroy_region(r2, t); +out: damon_free_target(t); damos_free_filter(f); } @@ -1456,6 +1591,47 @@ static void damon_test_is_last_region(struct kunit *test) damon_free_target(t); } +/* + * Verify that damos_walk() rejects new requests when + * walk_control_obsolete is set. + * + * This tests the invariant introduced by: + * commit 33c3f6c2b48c ("mm/damon/core: fix damos_walk() vs kdamond_fn() exit race") + */ +static void damon_test_walk_control_obsolete(struct kunit *test) +{ + struct damon_ctx *ctx; + struct damos_walk_control control = {}; + int ret; + + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + + /* Simulate shutdown phase */ + ctx->walk_control_obsolete = true; + + ret = damos_walk(ctx, &control); + + KUNIT_EXPECT_EQ(test, ret, -ECANCELED); + + damon_destroy_ctx(ctx); +} + +static void damon_test_rand(struct kunit *test) +{ + struct damon_ctx ctx; + int i; + + prandom_seed_state(&ctx.rnd_state, get_random_u64()); + for (i = 0; i < 10000; i++) { + unsigned long rnd = damon_rand(&ctx, 0, 10); + + KUNIT_EXPECT_GE(test, rnd, 0); + KUNIT_EXPECT_LE(test, rnd, 9); + } +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_target), KUNIT_CASE(damon_test_regions), @@ -1464,12 +1640,13 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_merge_two), KUNIT_CASE(damon_test_merge_regions_of), KUNIT_CASE(damon_test_split_regions_of), + KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), - KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), - KUNIT_CASE(damon_test_moving_sum), + KUNIT_CASE(damon_test_mvsum), + KUNIT_CASE(damon_test_nr_accesses_mvsum), KUNIT_CASE(damos_test_new_filter), KUNIT_CASE(damos_test_commit_quota_goal), KUNIT_CASE(damos_test_commit_quota_goals), @@ -1485,6 +1662,8 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_set_filters_default_reject), KUNIT_CASE(damon_test_apply_min_nr_regions), KUNIT_CASE(damon_test_is_last_region), + KUNIT_CASE(damon_test_walk_control_obsolete), + KUNIT_CASE(damon_test_rand), {}, }; diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 61f844336ffb..6a95441d193a 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -158,12 +158,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test, kunit_skip(test, "second damon_set_regions() fail"); } + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2); + if (damon_nr_regions(t) != nr_expected / 2) + goto out; + for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } +out: damon_destroy_target(t, NULL); } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index e73ec1ce016e..4976775fe276 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -10,6 +10,7 @@ #include <linux/mman.h> #include <linux/mmu_notifier.h> #include <linux/page_idle.h> +#include <linux/pagemap.h> #include <linux/pagewalk.h> #include <linux/sched/mm.h> @@ -382,8 +383,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) } struct damon_young_walk_private { - /* size of the folio for the access checked virtual memory address */ - unsigned long *folio_sz; bool young; }; @@ -410,7 +409,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = HPAGE_PMD_SIZE; huge_out: spin_unlock(ptl); return 0; @@ -429,7 +427,6 @@ huge_out: if (pte_young(ptent) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = folio_size(folio); out: pte_unmap_unlock(pte, ptl); return 0; @@ -457,7 +454,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, if (pte_young(entry) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = huge_page_size(h); folio_put(folio); @@ -469,11 +465,9 @@ out: #define damon_young_hugetlb_entry NULL #endif /* CONFIG_HUGETLB_PAGE */ -static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *folio_sz) +static bool damon_va_young(struct mm_struct *mm, unsigned long addr) { struct damon_young_walk_private arg = { - .folio_sz = folio_sz, .young = false, }; @@ -493,29 +487,17 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, - struct damon_attrs *attrs) + struct damon_region *r) { - static unsigned long last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; if (!mm) { - damon_update_region_access_rate(r, false, attrs); + damon_update_region_access_rate(r, false); return; } - /* If the region is in the last checked page, reuse the result */ - if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = r->sampling_addr; + accessed = damon_va_young(mm, r->sampling_addr); + damon_update_region_access_rate(r, accessed); } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) @@ -524,16 +506,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; - bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); - same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); - same_target = true; } if (mm) mmput(mm); @@ -625,8 +603,8 @@ static void damos_va_migrate_dests_add(struct folio *folio, } order = folio_order(folio); - ilx = vma->vm_pgoff >> order; - ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + ilx = vma_start_pgoff(vma) >> order; + ilx += linear_page_delta(vma, addr) >> order; for (i = 0; i < dests->nr_dests; i++) weight_total += dests->weight_arr[i]; @@ -981,7 +959,7 @@ static int __init damon_va_initcall(void) if (err) return err; return damon_register_ops(&ops_fvaddr); -}; +} subsys_initcall(damon_va_initcall); diff --git a/mm/debug.c b/mm/debug.c index 77fa8fe1d641..9a0297b3988d 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -163,7 +163,7 @@ void dump_vma(const struct vm_area_struct *vma) "flags: %#lx(%pGv)\n", vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, (unsigned long)pgprot_val(vma->vm_page_prot), - vma->anon_vma, vma->vm_ops, vma->vm_pgoff, + vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma), vma->vm_file, vma->vm_private_data, #ifdef CONFIG_PER_VMA_LOCK refcount_read(&vma->vm_refcnt), @@ -197,7 +197,7 @@ void dump_mm(const struct mm_struct *mm) "numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n" #endif "tlb_flush_pending %d\n" - "def_flags: %#lx(%pGv)\n", + "def_flags: %*pb(%pGv)\n", mm, mm->task_size, mm->mmap_base, mm->mmap_legacy_base, @@ -226,7 +226,8 @@ void dump_mm(const struct mm_struct *mm) mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq, #endif atomic_read(&mm->tlb_flush_pending), - mm->def_flags, &mm->def_flags + NUM_VMA_FLAG_BITS, mm->def_vma_flags.__vma_flags, + &mm->def_vma_flags ); } EXPORT_SYMBOL(dump_mm); diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 23dc3ee09561..2875fd22d7bb 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -672,7 +672,7 @@ static void __init pte_protnone_tests(struct pgtable_debug_args *args) { pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; pr_debug("Validating PTE protnone\n"); @@ -685,7 +685,7 @@ static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { pmd_t pmd; - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; if (!has_transparent_hugepage()) @@ -751,14 +751,14 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) pmd_t pmd; if (!pgtable_supports_soft_dirty() || - !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) + !IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES)) return; if (!has_transparent_hugepage()) return; pr_debug("Validating PMD swap soft dirty\n"); - pmd = swp_entry_to_pmd(args->leaf_entry); + pmd = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd)); WARN_ON(!pmd_is_valid_softleaf(pmd)); @@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args) WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { swp_entry_t arch_entry; @@ -829,7 +829,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap\n"); - pmd1 = swp_entry_to_pmd(args->leaf_entry); + pmd1 = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd1)); WARN_ON(!pmd_is_valid_softleaf(pmd1)); @@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) pmd2 = __swp_entry_to_pmd(arch_entry); WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1))); } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init swap_migration_tests(struct pgtable_debug_args *args) { diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c index 96c29b9dc85d..6215b90dfef3 100644 --- a/mm/early_ioremap.c +++ b/mm/early_ioremap.c @@ -47,15 +47,19 @@ pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr, return prot; } +/* + * Only architectures whose early_ioremap() must stop using __early_set_fixmap() + * after paging_init() need to call this. + */ void __init early_ioremap_reset(void) { after_paging_init = 1; } /* - * Generally, ioremap() is available after paging_init() has been called. - * Architectures wanting to allow early_ioremap after paging_init() can - * define __late_set_fixmap and __late_clear_fixmap to do the right thing. + * Only architectures that call early_ioremap_reset() need to define + * __late_set_fixmap() and __late_clear_fixmap(), which early_ioremap() uses + * instead of __early_set_fixmap() after the reset. */ #ifndef __late_set_fixmap static inline void __init __late_set_fixmap(enum fixed_addresses idx, diff --git a/mm/execmem.c b/mm/execmem.c index 084a207e4278..74a178a87e75 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -20,6 +20,7 @@ #include <asm/tlbflush.h> #include "internal.h" +#include "vmalloc.h" static struct execmem_info *execmem_info __ro_after_init; static struct execmem_info default_execmem_info __ro_after_init; diff --git a/mm/filemap.c b/mm/filemap.c index 58eb9d240643..1dbb4c6f824e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2270,10 +2270,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, unsigned long nr; struct folio *folio; - rcu_read_lock(); + if (*start > end) + return 0; - for (folio = xas_load(&xas); folio && xas.xa_index <= end; - folio = xas_next(&xas)) { + rcu_read_lock(); + for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; /* @@ -2281,11 +2282,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, * No current caller is looking for DAX entries. */ if (xa_is_value(folio)) - goto update_start; + break; /* If we landed in the middle of a THP, continue at its end. */ if (xa_is_sibling(folio)) - goto update_start; + break; if (!folio_try_get(folio)) goto retry; @@ -2293,29 +2294,27 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, if (unlikely(folio != xas_reload(&xas))) goto put_folio; - if (!folio_batch_add(fbatch, folio)) { - *start = folio_next_index(folio); - goto out; - } + if (!folio_batch_add(fbatch, folio)) + break; + xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= end) + break; continue; + put_folio: folio_put(folio); - retry: xas_reset(&xas); } + rcu_read_unlock(); -update_start: nr = folio_batch_count(fbatch); - if (nr) { folio = fbatch->folios[nr - 1]; *start = folio_next_index(folio); } -out: - rcu_read_unlock(); - return folio_batch_count(fbatch); + return nr; } EXPORT_SYMBOL(filemap_get_folios_contig); @@ -2467,11 +2466,14 @@ static void filemap_get_read_batch(struct address_space *mapping, XA_STATE(xas, &mapping->i_pages, index); struct folio *folio; + if (index > max) + return; + rcu_read_lock(); for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; - if (xas.xa_index > max || xa_is_value(folio)) + if (xa_is_value(folio)) break; if (xa_is_sibling(folio)) break; @@ -2488,6 +2490,8 @@ static void filemap_get_read_batch(struct address_space *mapping, if (folio_test_readahead(folio)) break; xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= max) + break; continue; put_folio: folio_put(folio); @@ -3225,6 +3229,7 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; size_t seek_size; + u64 next; if (start < pos) { if (!seek_data) @@ -3233,7 +3238,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, } seek_size = seek_folio_size(&xas, folio); - pos = round_up((u64)pos + 1, seek_size); + next = round_up((u64)pos + 1, seek_size); + if (next > (u64)end) + pos = end; + else + pos = next; start = folio_seek_hole_data(&xas, mapping, folio, start, pos, seek_data); if (start < pos) @@ -3402,8 +3411,8 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) * of memory. */ struct vm_area_struct *vma = vmf->vma; - unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + const unsigned long start = vma_start_pgoff(vma); + const unsigned long end = vma_end_pgoff(vma); unsigned long ra_end; ra->order = exec_folio_order(); @@ -3921,7 +3930,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, goto out; } - addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + + ((start_pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); if (!vmf->pte) { folio_unlock(folio); diff --git a/mm/swap.c b/mm/folio.c index 588f50d8f1a8..d2937600cf72 100644 --- a/mm/swap.c +++ b/mm/folio.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/mm/swap.c + * linux/mm/folio.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */ /* - * This file contains the default values for the operation of the - * Linux VM subsystem. Fine-tuning documentation can be found in - * Documentation/admin-guide/sysctl/vm.rst. - * Started 18.12.91 - * Swap aging added 23.2.95, Stephen Tweedie. - * Buffermem limits added 12.3.98, Rik van Riel. + * Folio LRU helpers: add/remove folios from LRU lists, batching, + * activation/deactivation, and page cache release paths. */ #include <linux/mm.h> @@ -39,14 +35,11 @@ #include <linux/buffer_head.h> #include "internal.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include <trace/events/pagemap.h> -/* How many pages do we try to swap or page in/out together? As a power of 2 */ -int page_cluster; -static const int page_cluster_max = 31; - struct cpu_fbatches { /* * The following folio batches are grouped together because they are protected @@ -694,9 +687,12 @@ void lru_add_drain_cpu(int cpu) { struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); struct folio_batch *fbatch = &fbatches->lru_add; + unsigned int nr_folios = folio_batch_count(fbatch); - if (folio_batch_count(fbatch)) + if (nr_folios) { folio_batch_move_lru(fbatch, lru_add); + trace_mm_lru_add_drain_tp(cpu, nr_folios); + } fbatch = &fbatches->lru_move_tail; /* Disabling interrupts below acts as a compiler barrier. */ @@ -828,13 +824,13 @@ static bool cpu_needs_drain(unsigned int cpu) struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); /* Check these in order of likelihood that they're not zero */ - return folio_batch_count(&fbatches->lru_add) || - folio_batch_count(&fbatches->lru_move_tail) || - folio_batch_count(&fbatches->lru_deactivate_file) || - folio_batch_count(&fbatches->lru_deactivate) || - folio_batch_count(&fbatches->lru_lazyfree) || - folio_batch_count(&fbatches->lru_activate) || - need_mlock_drain(cpu) || + return data_race(folio_batch_count(&fbatches->lru_add) || + folio_batch_count(&fbatches->lru_move_tail) || + folio_batch_count(&fbatches->lru_deactivate_file) || + folio_batch_count(&fbatches->lru_deactivate) || + folio_batch_count(&fbatches->lru_lazyfree) || + folio_batch_count(&fbatches->lru_activate) || + need_mlock_drain(cpu)) || has_bh_in_lru(cpu, NULL); } @@ -869,6 +865,8 @@ static inline void __lru_add_drain_all(bool force_all_cpus) if (WARN_ON(!mm_percpu_wq)) return; + trace_mm_lru_add_drain_all_tp(force_all_cpus); + /* * Guarantee folio_batch counter stores visible by this CPU * are visible to other CPUs before loading the current drain @@ -1171,35 +1169,3 @@ void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid); } #endif - -static const struct ctl_table swap_sysctl_table[] = { - { - .procname = "page-cluster", - .data = &page_cluster, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&page_cluster_max, - } -}; - -/* - * Perform any setup for the swap system - */ -void __init swap_setup(void) -{ - unsigned long megs = PAGES_TO_MB(totalram_pages()); - - /* Use a smaller cluster for small-memory machines */ - if (megs < 16) - page_cluster = 2; - else - page_cluster = 3; - /* - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ - - register_sysctl_init("vm", swap_sysctl_table); -} @@ -2784,12 +2784,17 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) mapping = READ_ONCE(folio->mapping); /* - * The mapping may have been truncated, in any case we cannot determine - * if this mapping is safe - fall back to slow path to determine how to - * proceed. + * If the mapping is NULL (truncated, or never set), we cannot + * determine whether the folio is file-backed, so a long-term writable + * pin must fall back to the slow path. + * + * Otherwise, a NULL mapping proves this is not a secretmem folio + * (secretmem folios always have a valid mapping to the secretmem + * inode's address_space), so in that case, we can continue with the + * fast path. */ if (!mapping) - return false; + return !reject_file_backed; /* Anonymous folios pose no problem. */ mapping_flags = (unsigned long)mapping & FOLIO_MAPPING_FLAGS; @@ -17,6 +17,7 @@ #include <linux/slab.h> #include <linux/sched.h> #include <linux/mmzone.h> +#include <linux/oom.h> #include <linux/pagemap.h> #include <linux/leafops.h> #include <linux/hugetlb.h> @@ -32,9 +33,27 @@ struct hmm_vma_walk { struct hmm_range *range; + bool *locked; unsigned long last; + unsigned long end; + unsigned int required_fault; }; +/* + * Internal sentinel returned by walk callbacks when they need a page fault. + * The callback stores end/required_fault in hmm_vma_walk; the outer loop + * consumes the sentinel and never propagates it to the caller. + */ +#define HMM_FAULT_PENDING -EAGAIN + +/* + * Internal sentinel returned by hmm_do_fault() when handle_mm_fault() + * completes a page fault with the mmap lock dropped. hmm_do_fault() sets + * *locked = false; the outer loop consumes the sentinel and never propagates + * it to the caller. + */ +#define HMM_FAULT_UNLOCKED -ENOLCK + enum { HMM_NEED_FAULT = 1 << 0, HMM_NEED_WRITE_FAULT = 1 << 1, @@ -60,37 +79,25 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end, } /* - * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s) - * @addr: range virtual start address (inclusive) - * @end: range virtual end address (exclusive) - * @required_fault: HMM_NEED_* flags - * @walk: mm_walk structure - * Return: -EBUSY after page fault, or page fault error + * hmm_record_fault() - record a range that needs to be faulted in * - * This function will be called whenever pmd_none() or pte_none() returns true, - * or whenever there is no page directory covering the virtual address range. + * Called by the walk callbacks when they discover that part of the range + * needs a page fault. The callback records what to fault and returns + * HMM_FAULT_PENDING; the outer loop in hmm_range_fault_locked() drops + * back out of walk_page_range() and invokes handle_mm_fault() from a context + * where no page-table or hugetlb_vma_lock is held. */ -static int hmm_vma_fault(unsigned long addr, unsigned long end, - unsigned int required_fault, struct mm_walk *walk) +static int hmm_record_fault(unsigned long addr, unsigned long end, + unsigned int required_fault, + struct mm_walk *walk) { struct hmm_vma_walk *hmm_vma_walk = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned int fault_flags = FAULT_FLAG_REMOTE; WARN_ON_ONCE(!required_fault); hmm_vma_walk->last = addr; - - if (required_fault & HMM_NEED_WRITE_FAULT) { - if (!(vma->vm_flags & VM_WRITE)) - return -EPERM; - fault_flags |= FAULT_FLAG_WRITE; - } - - for (; addr < end; addr += PAGE_SIZE) - if (handle_mm_fault(vma, addr, fault_flags, NULL) & - VM_FAULT_ERROR) - return -EFAULT; - return -EBUSY; + hmm_vma_walk->end = end; + hmm_vma_walk->required_fault = required_fault; + return HMM_FAULT_PENDING; } static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, @@ -174,7 +181,7 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR); } if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); return hmm_pfns_fill(addr, end, range, 0); } @@ -209,7 +216,7 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags); if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { @@ -328,10 +335,10 @@ out: fault: pte_unmap(ptep); /* Fault any virtual address we were asked to fault */ - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -371,7 +378,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, npages, 0); if (required_fault) { if (softleaf_is_device_private(entry)) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); else return -EFAULT; } @@ -391,7 +398,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, @@ -517,7 +524,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, npages, cpu_flags); if (required_fault) { spin_unlock(ptl); - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); @@ -564,21 +571,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, required_fault = hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags); if (required_fault) { - int ret; - spin_unlock(ptl); - hugetlb_vma_unlock_read(vma); - /* - * Avoid deadlock: drop the vma lock before calling - * hmm_vma_fault(), which will itself potentially take and - * drop the vma lock. This is also correct from a - * protection point of view, because there is no further - * use here of either pte or ptl after dropping the vma - * lock. - */ - ret = hmm_vma_fault(addr, end, required_fault, walk); - hugetlb_vma_lock_read(vma); - return ret; + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); @@ -637,29 +631,67 @@ static const struct mm_walk_ops hmm_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; -/** - * hmm_range_fault - try to fault some address in a virtual address range - * @range: argument structure +/* + * hmm_do_fault - fault in a range recorded by a walk callback * - * Returns 0 on success or one of the following error codes: + * Called from the outer loop in hmm_range_fault_locked() after a callback + * returned HMM_FAULT_PENDING. At this point we hold only mmap_lock; + * the page-table spinlock and any hugetlb_vma_lock acquired by the walk + * framework have already been released by the unwind. * - * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma - * (e.g., device file vma). - * -ENOMEM: Out of memory. - * -EPERM: Invalid permission (e.g., asking for write and range is read - * only). - * -EBUSY: The range has been invalidated and the caller needs to wait for - * the invalidation to finish. - * -EFAULT: A page was requested to be valid and could not be made valid - * ie it has no backing VMA or it is illegal to access - * - * This is similar to get_user_pages(), except that it can read the page tables - * without mutating them (ie causing faults). + * Returns -EBUSY on success (all pages faulted, caller should re-walk). + * Returns a negative errno on failure. */ -int hmm_range_fault(struct hmm_range *range) +static int hmm_do_fault(struct mm_struct *mm, + struct hmm_vma_walk *hmm_vma_walk) +{ + unsigned long addr = hmm_vma_walk->last; + unsigned long end = hmm_vma_walk->end; + unsigned int required_fault = hmm_vma_walk->required_fault; + unsigned int fault_flags = FAULT_FLAG_REMOTE; + struct vm_area_struct *vma; + + if (hmm_vma_walk->locked) + fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + + vma = vma_lookup(mm, addr); + if (!vma) + return -EFAULT; + + if (required_fault & HMM_NEED_WRITE_FAULT) { + if (!(vma->vm_flags & VM_WRITE)) + return -EPERM; + fault_flags |= FAULT_FLAG_WRITE; + } + + for (; addr < end; addr += PAGE_SIZE) { + vm_fault_t ret; + + ret = handle_mm_fault(vma, addr, fault_flags, NULL); + + if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) { + *hmm_vma_walk->locked = false; + return HMM_FAULT_UNLOCKED; + } + + if (ret & VM_FAULT_ERROR) { + int err = vm_fault_to_errno(ret, 0); + + if (WARN_ON(!err)) + err = -EINVAL; + + return err; + } + } + + return -EBUSY; +} + +static int hmm_range_fault_locked(struct hmm_range *range, bool *locked) { struct hmm_vma_walk hmm_vma_walk = { .range = range, + .locked = locked, .last = range->start, }; struct mm_struct *mm = range->notifier->mm; @@ -675,6 +707,22 @@ int hmm_range_fault(struct hmm_range *range) ret = walk_page_range(mm, hmm_vma_walk.last, range->end, &hmm_walk_ops, &hmm_vma_walk); /* + * When HMM_FAULT_PENDING is returned a walk callback + * recorded a range that needs handle_mm_fault(); + * hmm_do_fault() runs the fault outside walk_page_range() + * (so no page-table or hugetlb_vma_lock is held) and + * returns -EBUSY so the loop re-walks and picks up the + * now-present entries. + */ + if (ret == HMM_FAULT_PENDING) { + ret = hmm_do_fault(mm, &hmm_vma_walk); + if (ret == HMM_FAULT_UNLOCKED) { + if (fatal_signal_pending(current)) + return -EINTR; + return -EBUSY; + } + } + /* * When -EBUSY is returned the loop restarts with * hmm_vma_walk.last set to an address that has not been stored * in pfns. All entries < last in the pfn array are set to their @@ -683,9 +731,103 @@ int hmm_range_fault(struct hmm_range *range) } while (ret == -EBUSY); return ret; } + +/** + * hmm_range_fault - try to fault some address in a virtual address range + * @range: argument structure + * + * Returns 0 on success or one of the following error codes: + * + * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma + * (e.g., device file vma). + * -ENOMEM: Out of memory. + * -EPERM: Invalid permission (e.g., asking for write and range is read + * only). + * -EBUSY: The range has been invalidated and the caller needs to wait for + * the invalidation to finish. + * -EFAULT: A page was requested to be valid and could not be made valid + * ie it has no backing VMA or it is illegal to access + * + * This is similar to get_user_pages(), except that it can read the page tables + * without mutating them (ie causing faults). + * + * The mmap lock must be held by the caller and will remain held on return. + * New users should prefer hmm_range_fault_unlocked_timeout() unless they + * specifically need to keep the mmap lock held across the call. This helper + * cannot support VMAs whose fault handlers need to drop the mmap lock. + */ +int hmm_range_fault(struct hmm_range *range) +{ + return hmm_range_fault_locked(range, NULL); +} EXPORT_SYMBOL(hmm_range_fault); /** + * hmm_range_fault_unlocked_timeout - fault in a range with a retry timeout + * @range: argument structure + * @timeout: timeout in jiffies for internal -EBUSY retries, or 0 to retry + * indefinitely + * + * The caller must not hold the mmap lock. The function takes the mmap read + * lock internally and allows handle_mm_fault() to drop it during faults. If + * the mmap lock is dropped or the range is invalidated, the function refreshes + * range->notifier_seq and restarts the walk internally. + * + * Passing 0 for @timeout retries indefinitely. A non-zero @timeout is a caller + * policy limit for repeated mmu-notifier invalidation retries. HMM does not + * interrupt page fault handling when the timeout expires, but returns -EBUSY + * if the retry budget is exhausted before a stable range is obtained. + * + * Returns 0 on success or one of the error codes documented for + * hmm_range_fault(). -EINTR is returned if mmap_lock acquisition is + * interrupted or a fatal signal is pending during retry handling. + */ +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout) +{ + struct mm_struct *mm = range->notifier->mm; + unsigned long deadline = 0; + bool locked = false; + int ret; + + do { + /* + * If the previous fault dropped mmap_lock, then the fault + * handler made progress. Restart the retry timeout in that + * case, but keep the existing deadline for ordinary -EBUSY + * retries. + */ + if (timeout && !locked) + deadline = jiffies + timeout; + + range->notifier_seq = + mmu_interval_read_begin(range->notifier); + + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + + if (check_stable_address_space(mm)) { + mmap_read_unlock(mm); + return -EFAULT; + } + + if (timeout && time_after(jiffies, deadline)) { + mmap_read_unlock(mm); + return -EBUSY; + } + + locked = true; + ret = hmm_range_fault_locked(range, &locked); + if (locked) + mmap_read_unlock(mm); + } while (ret == -EBUSY); + + return ret; +} +EXPORT_SYMBOL(hmm_range_fault_unlocked_timeout); + +/** * hmm_dma_map_alloc - Allocate HMM map structure * @dev: device to allocate structure for * @map: HMM map to allocate diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8c1f35df0e91..73522f296cee 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -180,7 +180,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ if (!in_pf && shmem_file(vma->vm_file)) return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), - vma, vma->vm_pgoff, 0, + vma, vma_start_pgoff(vma), 0, forced_collapse); if (!vma_is_anonymous(vma)) { @@ -818,10 +818,8 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent) ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, "hugepages-%lukB", size); - if (ret) { - kfree(thpsize); - goto err; - } + if (ret) + goto err_put; ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); @@ -1196,7 +1194,7 @@ static inline bool is_transparent_hugepage(const struct folio *folio) static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, loff_t off, unsigned long flags, unsigned long size, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { loff_t off_end = off + len; loff_t off_align = round_up(off, size); @@ -1212,8 +1210,9 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, if (len_pad < len || (off + len_pad) < off) return 0; - ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad, - off >> PAGE_SHIFT, flags, vm_flags); + ret = mm_get_unmapped_area_vmaflags(filp, addr, len_pad, + off >> PAGE_SHIFT, flags, + vma_flags); /* * The failure might be due to length padding. The caller will retry @@ -1238,25 +1237,27 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, return ret; } -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { unsigned long ret; loff_t off = (loff_t)pgoff << PAGE_SHIFT; - ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); + ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, + vma_flags); if (ret) return ret; - return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, - vm_flags); + return mm_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + vma_flags); } unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); + return thp_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL_GPL(thp_get_unmapped_area); @@ -1819,11 +1820,11 @@ static void copy_huge_non_present_pmd( if (softleaf_is_migration_write(entry) || softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } else if (softleaf_is_device_private(entry)) { /* @@ -1832,12 +1833,12 @@ static void copy_huge_non_present_pmd( */ if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } @@ -1856,8 +1857,8 @@ static void copy_huge_non_present_pmd( add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); - if (!userfaultfd_wp(dst_vma)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (!userfaultfd_protected(dst_vma)) + pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); } @@ -1951,9 +1952,15 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, out_zero_page: mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); + + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd)) + pmd = pmd_modify(pmd, dst_vma->vm_page_prot); + pmd = pmd_clear_uffd(pmd); + } + pmdp_set_wrprotect(src_mm, addr, src_pmd); - if (!userfaultfd_wp(dst_vma)) - pmd = pmd_clear_uffd_wp(pmd); pmd = pmd_wrprotect(pmd); set_pmd: pmd = pmd_mkold(pmd); @@ -2196,6 +2203,34 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, return pmd_dirty(pmd); } +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + pmd_t pmd; + + if (!userfaultfd_rwp_async(vma)) + return handle_userfault(vmf, VM_UFFD_RWP); + + vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { + spin_unlock(vmf->ptl); + return 0; + } + pmd = pmd_modify(vmf->orig_pmd, vma->vm_page_prot); + /* pmd_modify() preserves _PAGE_UFFD; drop it on resolution */ + pmd = pmd_clear_uffd(pmd); + pmd = pmd_mkyoung(pmd); + if (!pmd_write(pmd) && + vma_wants_manual_pte_write_upgrade(vma) && + can_change_pmd_writable(vma, vmf->address, pmd)) + pmd = pmd_mkwrite(pmd, vma); + set_pmd_at(vma->vm_mm, vmf->address & HPAGE_PMD_MASK, + vmf->pmd, pmd); + update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); + spin_unlock(vmf->ptl); + return 0; +} + /* NUMA hinting page fault entry point for trans huge pmds */ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { @@ -2297,8 +2332,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto out; } @@ -2494,9 +2529,9 @@ static pmd_t clear_uffd_wp_pmd(pmd_t pmd) if (pmd_none(pmd)) return pmd; if (pmd_present(pmd)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); else - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); return pmd; } @@ -2539,8 +2574,19 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pgtable_trans_huge_deposit(mm, new_pmd, pgtable); } pmd = move_soft_dirty_pmd(pmd); - if (vma_has_uffd_without_event_remap(vma)) + if (vma_has_uffd_without_event_remap(vma)) { + /* + * See __copy_present_ptes(): normalise the RWP marker + * so the destination starts accessible instead of + * taking a numa-hinting fault on first access. Only the + * marker (protnone + uffd) needs it; leave other present + * PMDs in the VMA untouched. + */ + if (pmd_present(pmd) && userfaultfd_rwp(vma) && + pmd_uffd(pmd)) + pmd = pmd_modify(pmd, vma->vm_page_prot); pmd = clear_uffd_wp_pmd(pmd); + } set_pmd_at(mm, new_addr, new_pmd, pmd); if (force_flush) flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); @@ -2553,15 +2599,16 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, } static void change_non_present_huge_pmd(struct mm_struct *mm, - unsigned long addr, pmd_t *pmd, bool uffd_wp, - bool uffd_wp_resolve) + unsigned long addr, pmd_t *pmd, bool uffd_prot, + bool uffd_prot_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); - const struct folio *folio = softleaf_to_folio(entry); pmd_t newpmd; VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); if (softleaf_is_migration_write(entry)) { + const struct folio *folio = softleaf_to_folio(entry); + /* * A protection check is difficult so * just be safe and disable write @@ -2570,22 +2617,22 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, entry = make_readable_exclusive_migration_entry(swp_offset(entry)); else entry = make_readable_migration_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pmd)) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); - if (pmd_swp_uffd_wp(*pmd)) - newpmd = pmd_swp_mkuffd_wp(newpmd); + newpmd = softleaf_to_pmd(entry); + if (pmd_swp_uffd(*pmd)) + newpmd = pmd_swp_mkuffd(newpmd); } else { newpmd = *pmd; } - if (uffd_wp) - newpmd = pmd_swp_mkuffd_wp(newpmd); - else if (uffd_wp_resolve) - newpmd = pmd_swp_clear_uffd_wp(newpmd); + if (uffd_prot) + newpmd = pmd_swp_mkuffd(newpmd); + else if (uffd_prot_resolve) + newpmd = pmd_swp_clear_uffd(newpmd); if (!pmd_same(*pmd, newpmd)) set_pmd_at(mm, addr, pmd, newpmd); } @@ -2605,8 +2652,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, spinlock_t *ptl; pmd_t oldpmd, entry; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; - bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); int ret = 1; tlb_change_page_size(tlb, HPAGE_PMD_SIZE); @@ -2619,11 +2667,17 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, return 0; if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) { - change_non_present_huge_pmd(mm, addr, pmd, uffd_wp, - uffd_wp_resolve); + change_non_present_huge_pmd(mm, addr, pmd, uffd_prot, + uffd_prot_resolve); goto unlock; } + /* Already in the desired state */ + if (prot_numa && pmd_protnone(*pmd)) + goto unlock; + if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd)) + goto unlock; + if (prot_numa) { /* @@ -2634,9 +2688,6 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (is_huge_zero_pmd(*pmd)) goto unlock; - if (pmd_protnone(*pmd)) - goto unlock; - if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, vma_is_single_threaded_private(vma))) goto unlock; @@ -2665,15 +2716,19 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, oldpmd = pmdp_invalidate_ad(vma, addr, pmd); entry = pmd_modify(oldpmd, newprot); - if (uffd_wp) - entry = pmd_mkuffd_wp(entry); - else if (uffd_wp_resolve) + if (uffd_prot) + entry = pmd_mkuffd(entry); + else if (uffd_prot_resolve) /* * Leave the write bit to be handled by PF interrupt * handler, then things like COW could be properly * handled. */ - entry = pmd_clear_uffd_wp(entry); + entry = pmd_clear_uffd(entry); + + /* See change_pte_range(): preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && pmd_uffd(entry)) + entry = pmd_modify(entry, PAGE_NONE); /* See change_pte_range(). */ if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && @@ -2713,10 +2768,10 @@ int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, return 1; /* - * Huge entries on userfault-wp only works with anonymous, while we - * don't have anonymous PUDs yet. + * Huge entries on userfault-wp or userfault-rwp only work with + * anonymous, while we don't have anonymous PUDs yet. */ - if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) + if (WARN_ON_ONCE(cp_flags & (MM_CP_UFFD_WP_ALL | MM_CP_UFFD_RWP_ALL))) return 1; ptl = __pud_trans_huge_lock(pudp, vma); @@ -2832,7 +2887,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_folio_page_index(src_folio, dst_vma, + dst_addr); _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ @@ -2842,6 +2898,13 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm _dst_pmd = move_soft_dirty_pmd(src_pmdval); _dst_pmd = clear_uffd_wp_pmd(_dst_pmd); } + + /* Re-arm RWP on the moved PMD if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + _dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE); + _dst_pmd = pmd_mkuffd(_dst_pmd); + } + set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); @@ -3016,8 +3079,13 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, entry = pfn_pte(zero_pfn(addr), vma->vm_page_prot); entry = pte_mkspecial(entry); - if (pmd_uffd_wp(old_pmd)) - entry = pte_mkuffd_wp(entry); + if (pmd_uffd(old_pmd)) + entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && pmd_uffd(old_pmd)) + entry = pte_modify(entry, PAGE_NONE); + VM_BUG_ON(!pte_none(ptep_get(pte))); set_pte_at(mm, addr, pte, entry); pte++; @@ -3103,7 +3171,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_migration_write(entry); if (PageAnon(page)) @@ -3119,7 +3187,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_device_private_write(entry); anon_exclusive = PageAnonExclusive(page); @@ -3176,7 +3244,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); - uffd_wp = pmd_uffd_wp(old_pmd); + uffd_wp = pmd_uffd(old_pmd); VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); @@ -3247,7 +3315,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3274,7 +3342,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3292,7 +3360,11 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && uffd_wp) + entry = pte_modify(entry, PAGE_NONE); for (i = 0; i < HPAGE_PMD_NR; i++) VM_WARN_ON(!pte_none(ptep_get(pte + i))); @@ -3590,6 +3662,13 @@ static void __split_folio_to_order(struct folio *folio, int old_order, new_folio->mapping = folio->mapping; new_folio->index = folio->index + i; + /* + * page->private should not be set in tail pages. Warn once + * if private is unexpectedly set. Do it before swap.val assignment + * since private overlaps with swap.val. + */ + VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head); + if (folio_test_swapcache(folio)) new_folio->swap.val = folio->swap.val + i; @@ -4881,7 +4960,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -4905,7 +4984,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = pmd_write(pmdval); softdirty = pmd_soft_dirty(pmdval); - uffd_wp = pmd_uffd_wp(pmdval); + uffd_wp = pmd_uffd(pmdval); } else { softleaf_t old_entry; @@ -4914,7 +4993,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = softleaf_is_device_private_write(old_entry); softdirty = pmd_swp_soft_dirty(pmdval); - uffd_wp = pmd_swp_uffd_wp(pmdval); + uffd_wp = pmd_swp_uffd(pmdval); } /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ @@ -4941,11 +5020,11 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } /* Set PMD. */ - pmdswp = swp_entry_to_pmd(entry); + pmdswp = softleaf_to_pmd(entry); if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) - pmdswp = pmd_swp_mkuffd_wp(pmdswp); + pmdswp = pmd_swp_mkuffd(pmdswp); set_pmd_at(mm, address, pvmw->pmd, pmdswp); /* Migration entry installed: cleanup rmap, folio. */ @@ -4977,8 +5056,13 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) pmde = pmd_mksoft_dirty(pmde); if (softleaf_is_migration_write(entry)) pmde = pmd_mkwrite(pmde, vma); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_mkuffd(pmde); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pmd_swp_uffd(*pvmw->pmd) && userfaultfd_rwp(vma)) + pmde = pmd_modify(pmde, PAGE_NONE); + if (!softleaf_is_migration_young(entry)) pmde = pmd_mkold(pmde); /* NOTE: this may contain setting soft-dirty on some archs */ @@ -4994,12 +5078,12 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) else entry = make_readable_device_private_entry( page_to_pfn(new)); - pmde = swp_entry_to_pmd(entry); + pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_swp_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_swp_mkuffd(pmde); } if (folio_test_anon(folio)) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 940b52ac17c4..566ce53c154f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -47,9 +47,11 @@ #include <linux/node.h> #include <linux/page_owner.h> #include "internal.h" +#include "page_alloc.h" #include "hugetlb_vmemmap.h" #include "hugetlb_cma.h" #include "hugetlb_internal.h" +#include "mm_init.h" #include <linux/page-isolation.h> int hugetlb_max_hstate __read_mostly; @@ -58,7 +60,6 @@ struct hstate hstates[HUGE_MAX_HSTATE]; __initdata nodemask_t hugetlb_bootmem_nodes; __initdata struct list_head huge_boot_pages[MAX_NUMNODES]; -static unsigned long hstate_boot_nrinvalid[HUGE_MAX_HSTATE] __initdata; /* * Due to ordering constraints across the init code for various @@ -140,12 +141,14 @@ static inline bool subpool_is_free(struct hugepage_subpool *spool) static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, unsigned long irq_flags) { - spin_unlock_irqrestore(&spool->lock, irq_flags); + bool free_subpool = subpool_is_free(spool); /* If no pages are used, and no other handles to the subpool * remain, give up any reservations based on minimum size and * free the subpool */ - if (subpool_is_free(spool)) { + spin_unlock_irqrestore(&spool->lock, irq_flags); + + if (free_subpool) { if (spool->min_hpages != -1) hugetlb_acct_memory(spool->hstate, -spool->min_hpages); @@ -181,6 +184,9 @@ void hugepage_put_subpool(struct hugepage_subpool *spool) { unsigned long flags; + if (!spool) + return; + spin_lock_irqsave(&spool->lock, flags); BUG_ON(!spool->count); spool->count--; @@ -1011,8 +1017,7 @@ static long region_count(struct resv_map *resv, long f, long t) static pgoff_t vma_hugecache_offset(struct hstate *h, struct vm_area_struct *vma, unsigned long address) { - return ((address - vma->vm_start) >> huge_page_shift(h)) + - (vma->vm_pgoff >> huge_page_order(h)); + return linear_page_index(vma, address) >> huge_page_order(h); } /* @@ -1317,43 +1322,27 @@ static unsigned long available_huge_pages(struct hstate *h) return h->free_huge_pages - h->resv_huge_pages; } -static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, - struct vm_area_struct *vma, - unsigned long address, long gbl_chg) +static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, + struct mempolicy_interpreted *mpoli) { + nodemask_t *nodemask = mpoli->nodemask; struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask; - nodemask_t *nodemask; - int nid; - - /* - * gbl_chg==1 means the allocation requires a new page that was not - * reserved before. Making sure there's at least one free page. - */ - if (gbl_chg && !available_huge_pages(h)) - goto err; - - gfp_mask = htlb_alloc_mask(h); - nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); + mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) + if (!folio) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); - - mpol_cond_put(mpol); + mpoli->nid, + nodemask); + } return folio; - -err: - return NULL; } #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC) @@ -1806,7 +1795,8 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask, if (alloc_try_hard) gfp_mask |= __GFP_RETRY_MAYFAIL; - folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask); + folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, + ALLOC_DEFAULT); /* * If we did not specify __GFP_RETRY_MAYFAIL, but still got a @@ -2160,32 +2150,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas return folio; } -/* - * Use the VMA's mpolicy to allocate a huge page from the buddy. - */ static -struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, - struct vm_area_struct *vma, unsigned long addr) +struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, + gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) { struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask = htlb_alloc_mask(h); - int nid; - nodemask_t *nodemask; + nodemask_t *nodemask = mpoli->nodemask; - nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); + folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) - folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); - mpol_cond_put(mpol); + if (!folio) { + folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid, + nodemask); + } + return folio; } @@ -2838,6 +2824,104 @@ void wait_for_freed_hugetlb_folios(void) flush_work(&free_hpage_work); } +/** + * hugetlb_alloc_folio - Allocate a hugetlb folio. + * @h: Hugetlb state control block. + * @mpoli: Interpreted memory policy to use for allocation. + * @alloc_flags: Flags controlling the allocation behavior. + * + * Allocates a hugetlb folio and handles cgroup charging and global hstate + * reservations. + * + * Return: A pointer to the allocated folio, or an ERR_PTR on failure. + * -ENOSPC if cgroup charging fails or no folio is available. + * -ENOMEM if mem cgroup charging fails. + */ +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags) +{ + bool charge_hugetlb_cgroup_rsvd = alloc_flags & + HUGETLB_ALLOC_CHARG_CGROUP_RSVD; + bool use_global_reservation = alloc_flags & + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; + size_t nr_pages = pages_per_huge_page(h); + struct hugetlb_cgroup *h_cg_rsvd = NULL; + struct hugetlb_cgroup *h_cg = NULL; + gfp_t gfp = htlb_alloc_mask(h); + int idx = hstate_index(h); + struct folio *folio; + int ret; + + if (charge_hugetlb_cgroup_rsvd && + hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd)) + return ERR_PTR(-ENOSPC); + + if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup_rsvd; + } + + spin_lock_irq(&hugetlb_lock); + + folio = NULL; + if (use_global_reservation || available_huge_pages(h)) + folio = dequeue_hugetlb_folio(h, gfp, mpoli); + + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli); + if (!folio) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup; + } + spin_lock_irq(&hugetlb_lock); + list_add(&folio->lru, &h->hugepage_activelist); + folio_ref_unfreeze(folio, 1); + } + + if (use_global_reservation) { + folio_set_hugetlb_restore_reserve(folio); + h->resv_huge_pages--; + } + + hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio); + + if (charge_hugetlb_cgroup_rsvd) { + hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd, + folio); + } + + spin_unlock_irq(&hugetlb_lock); + + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + /* + * Unconditionally increment NR_HUGETLB here because if + * mem_cgroup_charge_hugetlb failed, freeing the page will + * decrement NR_HUGETLB. + */ + lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages); + + if (ret == -ENOMEM) { + free_huge_folio(folio); + /* + * Skip uncharging hugetlb_cgroup since the charges + * were committed to the folio and freeing the folio + * would have cleared those up. + */ + return ERR_PTR(ret); + } + + return folio; + + err_uncharge_hugetlb_cgroup: + hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg); + err_uncharge_hugetlb_cgroup_rsvd: + if (charge_hugetlb_cgroup_rsvd) + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd); + + return ERR_PTR(ret); +} + typedef enum { /* * For either 0/1: we checked the per-vma resv map, and one resv @@ -2872,12 +2956,13 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *folio; long retval, gbl_chg, gbl_reserve; map_chg_state map_chg; - int ret, idx; - struct hugetlb_cgroup *h_cg = NULL; - struct hugetlb_cgroup *h_cg_rsvd = NULL; - gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; - - idx = hstate_index(h); + struct mempolicy_interpreted mpoli; + gfp_t gfp = htlb_alloc_mask(h); + struct mempolicy *mpol; + nodemask_t *nodemask; + u8 alloc_flags = 0; + int nid; + int ret; /* Whether we need a separate per-vma reservation? */ if (cow_from_owner) { @@ -2909,8 +2994,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ if (map_chg) { gbl_chg = hugepage_subpool_get_pages(spool, 1); - if (gbl_chg < 0) + if (gbl_chg < 0) { + ret = -ENOSPC; goto out_end_reservation; + } } else { /* * If we have the vma reservation ready, no need for extra @@ -2920,57 +3007,39 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } /* - * If this allocation is not consuming a per-vma reservation, - * charge the hugetlb cgroup now. + * If allocation doesn't reuse a reservation in the resv_map, + * charge for the reservation. */ - if (map_chg) { - ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg_rsvd); - if (ret) - goto out_subpool_put; - } - - ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_uncharge_cgroup_reservation; + if (map_chg != MAP_CHG_REUSE) + alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD; - spin_lock_irq(&hugetlb_lock); /* - * glb_chg is passed to indicate whether or not a page must be taken - * from the global free pool (global change). gbl_chg == 0 indicates - * a reservation exists for the allocation. + * gbl_chg == 0 indicates a reservation exists for this + * allocation, so try to use it. */ - folio = dequeue_hugetlb_folio_vma(h, vma, addr, gbl_chg); - if (!folio) { - spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); - if (!folio) - goto out_uncharge_cgroup; - spin_lock_irq(&hugetlb_lock); - list_add(&folio->lru, &h->hugepage_activelist); - folio_ref_unfreeze(folio, 1); - /* Fall through */ - } + if (gbl_chg == 0) + alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; - /* - * Either dequeued or buddy-allocated folio needs to add special - * mark to the folio when it consumes a global reservation. - */ - if (!gbl_chg) { - folio_set_hugetlb_restore_reserve(folio); - h->resv_huge_pages--; - } + /* Takes reference on mpol. */ + nid = huge_node(vma, addr, gfp, &mpol, &nodemask); + mpoli = (struct mempolicy_interpreted){ + .nid = nid, +#ifdef CONFIG_NUMA + .mode = mpol ? mpol->mode : MPOL_DEFAULT, +#else + .mode = MPOL_DEFAULT, +#endif + .nodemask = nodemask, + }; - hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); - /* If allocation is not consuming a reservation, also store the - * hugetlb_cgroup pointer on the page. - */ - if (map_chg) { - hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd, folio); - } + folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags); - spin_unlock_irq(&hugetlb_lock); + mpol_cond_put(mpol); + + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_subpool_put; + } hugetlb_set_folio_subpool(folio, spool); @@ -2999,27 +3068,8 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } } - ret = mem_cgroup_charge_hugetlb(folio, gfp); - /* - * Unconditionally increment NR_HUGETLB here. If it turns out that - * mem_cgroup_charge_hugetlb failed, then immediately free the page and - * decrement NR_HUGETLB. - */ - lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h)); - - if (ret == -ENOMEM) { - free_huge_folio(folio); - return ERR_PTR(-ENOMEM); - } - return folio; -out_uncharge_cgroup: - hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); -out_uncharge_cgroup_reservation: - if (map_chg) - hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd); out_subpool_put: /* * put page to subpool iff the quota of subpool's rsv_hpages is used @@ -3030,100 +3080,94 @@ out_subpool_put: hugetlb_acct_memory(h, -gbl_reserve); } - out_end_reservation: if (map_chg != MAP_CHG_ENFORCED) vma_end_reservation(h, vma, addr); - return ERR_PTR(-ENOSPC); + return ERR_PTR(ret); } static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) { - struct huge_bootmem_page *m; - int listnode = nid; - if (hugetlb_early_cma(h)) - m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); - else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), + return hugetlb_cma_alloc_bootmem(h, nid, node_exact); + + if (node_exact) + return memblock_alloc_exact_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), + + return memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - /* - * For pre-HVO to work correctly, pages need to be on - * the list for the node they were actually allocated - * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, - * extract the actual node first. - */ - if (m) - listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); - } - - if (m) { - m->flags = 0; - m->cma = NULL; - } - } - - if (m) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - * - * Put them into a private list first because mem_map - * is not up yet. - */ - INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[listnode]); - m->hstate = h; - } - - return m; } -int alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node = nid; /* do node specific alloc */ - if (nid != NUMA_NO_NODE) { - m = alloc_bootmem(h, node, true); - if (!m) - return 0; - goto found; - } + if (nid != NUMA_NO_NODE) + return alloc_bootmem(h, node, true); /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, - &hugetlb_bootmem_nodes) { - m = alloc_bootmem(h, node, false); - if (!m) - return 0; - goto found; - } + &hugetlb_bootmem_nodes) + return alloc_bootmem(h, node, false); -found: + return NULL; +} +static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) +{ + unsigned long pfn; + unsigned int nid_request = nid; + struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); + + if (!m) + return false; + + pfn = PHYS_PFN(__pa(m)); + nid = early_pfn_to_nid(pfn); /* - * Only initialize the head struct page in memmap_init_reserved_pages, - * rest of the struct pages will be initialized by the HugeTLB - * subsystem itself. - * The head struct page is used to get folio information by the HugeTLB - * subsystem like zone id and node id. + * Use the beginning of the huge page to store the huge_bootmem_page + * struct (until gather_bootmem puts them into the mem_map). + * + * Put them into a private list first because mem_map is not up yet. */ - memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), - huge_page_size(h) - PAGE_SIZE); + INIT_LIST_HEAD(&m->list); + m->hstate = h; + m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0; - return 1; + /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ + if (!hugetlb_early_cma(h) && + pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) { + /* + * If the allocated page is on a different node than requested + * (e.g., on PowerPC LPARs), put it on the requested node's list, + * because hugetlb_free_cross_zone_pages() only frees cross-zone + * pages belonging to the requested node. + */ + if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request)) + list_add(&m->list, &huge_boot_pages[nid_request]); + else + list_add(&m->list, &huge_boot_pages[nid]); + } else { + list_add_tail(&m->list, &huge_boot_pages[nid]); + m->flags |= HUGE_BOOTMEM_ZONES_VALID; + /* + * Only initialize the head struct page in memmap_init_reserved_pages, + * rest of the struct pages will be initialized by the HugeTLB + * subsystem itself. + * The head struct page is used to get folio information by the HugeTLB + * subsystem like zone id and node id. + */ + memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), + huge_page_size(h) - PAGE_SIZE); + } + + return true; } /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */ @@ -3235,57 +3279,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, } } -bool __init hugetlb_bootmem_page_zones_valid(int nid, - struct huge_bootmem_page *m) -{ - unsigned long start_pfn; - bool valid; - - if (m->flags & HUGE_BOOTMEM_ZONES_VALID) { - /* - * Already validated, skip check. - */ - return true; - } - - if (hugetlb_bootmem_page_earlycma(m)) { - valid = cma_validate_zones(m->cma); - goto out; - } - - start_pfn = virt_to_phys(m) >> PAGE_SHIFT; - - valid = !pfn_range_intersects_zones(nid, start_pfn, - pages_per_huge_page(m->hstate)); -out: - if (!valid) - hstate_boot_nrinvalid[hstate_index(m->hstate)]++; - - return valid; -} - -/* - * Free a bootmem page that was found to be invalid (intersecting with - * multiple zones). - * - * Since it intersects with multiple zones, we can't just do a free - * operation on all pages at once, but instead have to walk all - * pages, freeing them one by one. - */ -static void __init hugetlb_bootmem_free_invalid_page(int nid, struct page *page, - struct hstate *h) -{ - unsigned long npages = pages_per_huge_page(h); - unsigned long pfn; - - while (npages--) { - pfn = page_to_pfn(page); - __init_page_from_nid(pfn, nid); - free_reserved_page(page); - page++; - } -} - /* * Put bootmem huge pages into the standard lists after mem_map is up. * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages. @@ -3301,17 +3294,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) struct folio *folio = (void *)page; h = m->hstate; - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Can't use this page. Initialize the - * page structures if that hasn't already - * been done, and give them to the page - * allocator. - */ - hugetlb_bootmem_free_invalid_page(nid, page, h); - continue; - } - /* * It is possible to have multiple huge page sizes (hstates) * in this list. If so, process each size separately. @@ -3365,7 +3347,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start, gather_bootmem_prealloc_node(nid); } -static void __init gather_bootmem_prealloc(void) +void __init hugetlb_bootmem_struct_page_init(void) { struct padata_mt_job job = { .thread_fn = gather_bootmem_prealloc_parallel, @@ -3377,10 +3359,63 @@ static void __init gather_bootmem_prealloc(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + struct zone *zone; + + for_each_zone(zone) { + for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { + struct page *tail, *p; + unsigned int order; + + tail = zone->vmemmap_tails[i]; + if (!tail) + continue; + + order = i + VMEMMAP_TAIL_MIN_ORDER; + p = page_to_virt(tail); + /* + * prep_and_add_bootmem_folios() can access pageblock + * flags on bootmem HugeTLB pages, so initialize the + * shared tail struct pages here before bootmem folios + * start using them. + */ + for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) + init_compound_tail(p + j, NULL, order, zone); + } + } +#endif padata_do_multithreaded(&job); } +static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid) +{ + unsigned long freed = 0; + struct huge_bootmem_page *m, *tmp; + + if (!hstate_is_gigantic(h)) + return freed; + + list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) { + if (m->flags & HUGE_BOOTMEM_ZONES_VALID) + break; + + list_del(&m->list); + memblock_free(m, huge_page_size(h)); + freed++; + } + + if (freed) { + char buf[32]; + + string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf)); + pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n", + freed, buf, nid); + } + + return freed; +} + static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) { unsigned long i; @@ -3411,6 +3446,8 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) cond_resched(); } + i -= hugetlb_free_cross_zone_pages(h, nid); + if (!list_empty(&folio_list)) prep_and_add_allocated_folios(h, &folio_list); @@ -3484,6 +3521,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) { + int nid; unsigned long i; for (i = 0; i < h->max_huge_pages; ++i) { @@ -3492,6 +3530,9 @@ static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) cond_resched(); } + for_each_node(nid) + i -= hugetlb_free_cross_zone_pages(h, nid); + return i; } @@ -3569,7 +3610,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) * - For gigantic pages, this is called early in the boot process and * pages are allocated from memblock allocated or something similar. * Gigantic pages are actually added to pools later with the routine - * gather_bootmem_prealloc. + * hugetlb_bootmem_struct_page_init. * - For non-gigantic pages, this is called later in the boot process after * all of mm is up and functional. Pages are allocated from buddy and * then added to hugetlb pools. @@ -3647,20 +3688,13 @@ static void __init hugetlb_init_hstates(void) static void __init report_hugepages(void) { struct hstate *h; - unsigned long nrinvalid; for_each_hstate(h) { char buf[32]; - nrinvalid = hstate_boot_nrinvalid[hstate_index(h)]; - h->max_huge_pages -= nrinvalid; - string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n", buf, h->nr_huge_pages); - if (nrinvalid) - pr_info("HugeTLB: %s page size: %lu invalid page%s discarded\n", - buf, nrinvalid, str_plural(nrinvalid)); pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n", hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf); } @@ -4139,7 +4173,6 @@ static int __init hugetlb_init(void) } hugetlb_init_hstates(); - gather_bootmem_prealloc(); report_hugepages(); hugetlb_sysfs_init(); @@ -4855,8 +4888,16 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add __folio_mark_uptodate(new_folio); hugetlb_add_new_anon_rmap(new_folio, vma, addr); - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old)) - newpte = huge_pte_mkuffd_wp(newpte); + if (userfaultfd_protected(vma) && huge_pte_uffd(old)) { + newpte = huge_pte_mkuffd(newpte); + /* Restore PAGE_NONE so the RWP marker keeps trapping. */ + if (userfaultfd_rwp(vma)) { + unsigned int shift = huge_page_shift(hstate_vma(vma)); + + newpte = huge_pte_modify(newpte, PAGE_NONE); + newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags); + } + } set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz); hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); folio_set_hugetlb_migratable(new_folio); @@ -4937,7 +4978,7 @@ again: */ set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(softleaf_is_migration(softleaf))) { - bool uffd_wp = pte_swp_uffd_wp(entry); + bool uffd = pte_swp_uffd(entry); if (!softleaf_is_migration_read(softleaf) && cow) { /* @@ -4947,12 +4988,12 @@ again: softleaf = make_readable_migration_entry( swp_offset(softleaf)); entry = swp_entry_to_pte(softleaf); - if (userfaultfd_wp(src_vma) && uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + if (userfaultfd_protected(src_vma) && uffd) + entry = pte_swp_mkuffd(entry); set_huge_pte_at(src, addr, src_pte, entry, sz); } - if (!userfaultfd_wp(dst_vma)) - entry = pte_swp_clear_uffd_wp(entry); + if (!userfaultfd_protected(dst_vma)) + entry = pte_swp_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(pte_is_marker(entry))) { const pte_marker marker = copy_pte_marker(softleaf, dst_vma); @@ -5016,6 +5057,16 @@ again: goto next; } + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) { + entry = huge_pte_modify(entry, dst_vma->vm_page_prot); + entry = arch_make_huge_pte(entry, huge_page_shift(h), + dst_vma->vm_flags); + } + entry = huge_pte_clear_uffd(entry); + } + if (cow) { /* * No need to notify as we are downgrading page @@ -5028,9 +5079,6 @@ again: entry = huge_pte_wrprotect(entry); } - if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); - set_huge_pte_at(dst, addr, dst_pte, entry, sz); hugetlb_count_add(npages, dst); } @@ -5076,10 +5124,23 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, huge_pte_clear(mm, new_addr, dst_pte, sz); } else { if (need_clear_uffd_wp) { - if (pte_present(pte)) - pte = huge_pte_clear_uffd_wp(pte); - else - pte = pte_swp_clear_uffd_wp(pte); + if (pte_present(pte)) { + /* + * See __copy_present_ptes(): normalise the RWP + * marker so the destination starts accessible + * instead of taking a numa-hinting fault on + * first access. Only the marker (protnone + uffd) + * needs it; leave other present PTEs untouched. + */ + if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) { + pte = huge_pte_modify(pte, vma->vm_page_prot); + pte = arch_make_huge_pte(pte, huge_page_shift(h), + vma->vm_flags); + } + pte = huge_pte_clear_uffd(pte); + } else { + pte = pte_swp_clear_uffd(pte); + } } set_huge_pte_at(mm, new_addr, dst_pte, pte, sz); } @@ -5213,7 +5274,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, * drop the uffd-wp bit in this zap, then replace the * pte with a marker. */ - if (pte_swp_uffd_wp_any(pte) && + if (pte_swp_uffd_any(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5249,7 +5310,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, if (huge_pte_dirty(pte)) folio_mark_dirty(folio); /* Leave a uffd-wp pte marker if needed */ - if (huge_pte_uffd_wp(pte) && + if (huge_pte_uffd(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5388,8 +5449,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * from page cache lookup which is in HPAGE_SIZE units. */ address = address & huge_page_mask(h); - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; + pgoff = linear_page_index(vma, address); mapping = vma->vm_file->f_mapping; /* @@ -5398,7 +5458,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * __unmap_hugepage_range() is called as the lock is already held */ i_mmap_lock_write(mapping); - vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) { /* Do not unmap the current VMA */ if (iter_vma == vma) continue; @@ -5453,7 +5513,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf) * can trigger this, because hugetlb_fault() will always resolve * uffd-wp bit first. */ - if (!unshare && huge_pte_uffd_wp(pte)) + if (!unshare && huge_pte_uffd(pte)) return 0; /* Let's take out MAP_SHARED mappings first. */ @@ -5597,8 +5657,8 @@ retry_avoidcopy: huge_ptep_clear_flush(vma, vmf->address, vmf->pte); hugetlb_remove_rmap(old_folio); hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address); - if (huge_pte_uffd_wp(pte)) - newpte = huge_pte_mkuffd_wp(newpte); + if (huge_pte_uffd(pte)) + newpte = huge_pte_mkuffd(newpte); set_huge_pte_at(mm, vmf->address, vmf->pte, newpte, huge_page_size(h)); folio_set_hugetlb_migratable(new_folio); @@ -5876,7 +5936,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, * if populated. */ if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte))) - new_pte = huge_pte_mkuffd_wp(new_pte); + new_pte = huge_pte_mkuffd(new_pte); set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h)); hugetlb_count_add(pages_per_huge_page(h), mm); @@ -6052,6 +6112,47 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, } /* + * Protnone hugetlb PTEs with the uffd bit are used by + * userfaultfd RWP for access tracking. Plain PROT_NONE (without the + * marker) is not an RWP fault and is not expected on hugetlb (no + * NUMA hinting), so let normal hugetlb fault handling proceed. + */ + if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) && + userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) { + spinlock_t *ptl; + pte_t pte; + + /* Sync: drop hugetlb locks before blocking in handle_userfault() */ + if (!userfaultfd_rwp_async(vma)) + return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP); + + ptl = huge_pte_lock(h, mm, vmf.pte); + pte = huge_ptep_get(mm, vmf.address, vmf.pte); + if (pte_protnone(pte) && huge_pte_uffd(pte)) { + unsigned int shift = huge_page_shift(h); + + pte = huge_pte_modify(pte, vma->vm_page_prot); + pte = arch_make_huge_pte(pte, shift, vma->vm_flags); + /* huge_pte_modify() preserves _PAGE_UFFD; drop it on resolution */ + pte = huge_pte_clear_uffd(pte); + pte = pte_mkyoung(pte); + /* + * Unlike do_uffd_rwp(), do not upgrade to writable + * here. Hugetlb lacks a can_change_huge_pte_writable() + * equivalent, so a write access will take a separate + * COW fault — acceptable for the rare private hugetlb + * case. + */ + set_huge_pte_at(mm, vmf.address, vmf.pte, pte, + huge_page_size(h)); + update_mmu_cache(vma, vmf.address, vmf.pte); + } + spin_unlock(ptl); + ret = 0; + goto out_mutex; + } + + /* * If we are going to COW/unshare the mapping later, we examine the * pending reservations for this page now. This will ensure that any * allocations necessary to record that reservation occur outside the @@ -6074,7 +6175,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, goto out_ptl; /* Handle userfault-wp first, before trying to lock more pages */ - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(mm, vmf.address, vmf.pte)) && + if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) && (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) { if (!userfaultfd_wp_async(vma)) { spin_unlock(vmf.ptl); @@ -6083,7 +6184,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, return handle_userfault(&vmf, VM_UFFD_WP); } - vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte); + vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte); set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte, huge_page_size(hstate_vma(vma))); /* Fallthrough to CoW */ @@ -6368,7 +6469,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, _dst_pte = pte_mkyoung(_dst_pte); if (wp_enabled) - _dst_pte = huge_pte_mkuffd_wp(_dst_pte); + _dst_pte = huge_pte_mkuffd(_dst_pte); set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); @@ -6411,6 +6512,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long last_addr_mask; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP; + bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE; struct mmu_gather tlb; /* @@ -6436,6 +6539,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma, ptep = hugetlb_walk(vma, address, psize); if (!ptep) { + /* + * uffd_wp installs a pte marker on the unpopulated + * entry; uffd_rwp does not install markers so the + * allocation is unnecessary for it. + */ if (!uffd_wp) { address |= last_addr_mask; continue; @@ -6457,7 +6565,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, * shouldn't happen at all. Warn about it if it * happened due to some reason. */ - WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); + WARN_ON_ONCE(uffd_wp || uffd_wp_resolve || + uffd_rwp || uffd_rwp_resolve); pages++; spin_unlock(ptl); address |= last_addr_mask; @@ -6491,10 +6600,10 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pages++; } - if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); - else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + if (uffd_wp || uffd_rwp) + newpte = pte_swp_mkuffd(newpte); + else if (uffd_wp_resolve || uffd_rwp_resolve) + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(pte, newpte)) set_huge_pte_at(mm, address, ptep, newpte, psize); } else if (unlikely(pte_is_marker(pte))) { @@ -6504,20 +6613,32 @@ long hugetlb_change_protection(struct vm_area_struct *vma, * pte_marker_uffd_wp()==true implies !poison * because they're mutual exclusive. */ - if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve) + if (pte_is_uffd_wp_marker(pte) && + (uffd_wp_resolve || uffd_rwp_resolve)) /* Safe to modify directly (non-present->none). */ huge_pte_clear(mm, address, ptep, psize); } else { pte_t old_pte; unsigned int shift = huge_page_shift(hstate_vma(vma)); + /* Already protnone with uffd bit set? Nothing to do. */ + if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte)) + goto next; + old_pte = huge_ptep_modify_prot_start(vma, address, ptep); pte = huge_pte_modify(old_pte, newprot); pte = arch_make_huge_pte(pte, shift, vma->vm_flags); - if (uffd_wp) - pte = huge_pte_mkuffd_wp(pte); - else if (uffd_wp_resolve) - pte = huge_pte_clear_uffd_wp(pte); + if (uffd_wp || uffd_rwp) + pte = huge_pte_mkuffd(pte); + else if (uffd_wp_resolve || uffd_rwp_resolve) + pte = huge_pte_clear_uffd(pte); + + /* Preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) { + pte = huge_pte_modify(pte, PAGE_NONE); + pte = arch_make_huge_pte(pte, shift, vma->vm_flags); + } + huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); @@ -6787,7 +6908,7 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma, struct vm_area_struct *vma, unsigned long addr, pgoff_t idx) { - unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + + unsigned long saddr = ((idx - vma_start_pgoff(svma)) << PAGE_SHIFT) + svma->vm_start; unsigned long sbase = saddr & PUD_MASK; unsigned long s_end = sbase + PUD_SIZE; @@ -6872,15 +6993,14 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pud_t *pud) { struct address_space *mapping = vma->vm_file->f_mapping; - pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; + const pgoff_t idx = linear_page_index(vma, addr); struct vm_area_struct *svma; unsigned long saddr; pte_t *spte = NULL; pte_t *pte; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { + mapping_rmap_tree_foreach(svma, mapping, idx, idx) { if (svma == vma) continue; @@ -7198,7 +7318,8 @@ void folio_putback_hugetlb(struct folio *folio) folio_put(folio); } -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason) +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason) { struct hstate *h = folio_hstate(old_folio); @@ -7227,14 +7348,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re * There is no need to transfer the per-node surplus state * when we do not cross the node. */ - if (new_nid == old_nid) - return; - spin_lock_irq(&hugetlb_lock); - if (h->surplus_huge_pages_node[old_nid]) { - h->surplus_huge_pages_node[old_nid]--; - h->surplus_huge_pages_node[new_nid]++; + if (new_nid != old_nid) { + spin_lock_irq(&hugetlb_lock); + if (h->surplus_huge_pages_node[old_nid]) { + h->surplus_huge_pages_node[old_nid]--; + h->surplus_huge_pages_node[new_nid]++; + } + spin_unlock_irq(&hugetlb_lock); } - spin_unlock_irq(&hugetlb_lock); } /* diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 79dbd0baafa3..07faf625675b 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -9,6 +9,9 @@ #include <asm/setup.h> #include <linux/hugetlb.h> +#include <linux/memblock.h> +#include <linux/math.h> +#include <linux/math64.h> #include "internal.h" #include "hugetlb_cma.h" @@ -18,6 +21,28 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; static bool hugetlb_cma_only __ro_after_init; static unsigned long hugetlb_cma_size __ro_after_init; +static unsigned int hugetlb_cma_percent __initdata; +static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata; + +#ifdef CONFIG_NUMA +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + struct memblock_region *reg; + phys_addr_t size = 0; + + for_each_mem_region(reg) { + if (reg->nid == nid) + size += reg->size; + } + return size; +} +#else +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + return memblock_phys_mem_size(); +} +#endif + void hugetlb_cma_free_frozen_folio(struct folio *folio) { WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], @@ -56,37 +81,27 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, return folio; } -struct huge_bootmem_page * __init -hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) +void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; - struct huge_bootmem_page *m; - int node = *nid; + void *m; + int node; - cma = hugetlb_cma[*nid]; + cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); - if (!m) { - if (node_exact) - return NULL; - - for_each_node_mask(node, hugetlb_bootmem_nodes) { - cma = hugetlb_cma[node]; - if (!cma || node == *nid) - continue; - m = cma_reserve_early(cma, huge_page_size(h)); - if (m) { - *nid = node; - break; - } - } - } + if (m || node_exact) + return m; - if (m) { - m->flags = HUGE_BOOTMEM_CMA; - m->cma = cma; + for_each_node_mask(node, hugetlb_bootmem_nodes) { + cma = hugetlb_cma[node]; + if (!cma || node == nid) + continue; + m = cma_reserve_early(cma, huge_page_size(h)); + if (m) + return m; } - return m; + return NULL; } static int __init cmdline_parse_hugetlb_cma(char *p) @@ -100,14 +115,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) break; if (s[count] == ':') { + char *next; + if (tmp >= MAX_NUMNODES) break; nid = array_index_nospec(tmp, MAX_NUMNODES); s += count + 1; - tmp = memparse(s, &s); - hugetlb_cma_size_in_node[nid] = tmp; - hugetlb_cma_size += tmp; + tmp = memparse(s, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n", + tmp, nid); + break; + } + hugetlb_cma_percent_in_node[nid] = tmp; + hugetlb_cma_size_in_node[nid] = 0; + s = next + 1; + } else { + hugetlb_cma_size_in_node[nid] = tmp; + hugetlb_cma_percent_in_node[nid] = 0; + s = next; + } /* * Skip the separator if have one, otherwise @@ -118,7 +147,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) else break; } else { - hugetlb_cma_size = memparse(p, &p); + char *next; + + tmp = memparse(p, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp); + } else { + hugetlb_cma_percent = tmp; + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } + } else { + hugetlb_cma_size = tmp; + hugetlb_cma_percent = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } break; } } @@ -144,8 +194,36 @@ void __init hugetlb_cma_reserve(void) { unsigned long size, reserved, per_node, order, gigantic_page_size; bool node_specific_cma_alloc = false; + bool has_node_specific_param = false; int nid; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_size_in_node[nid] || hugetlb_cma_percent_in_node[nid]) { + has_node_specific_param = true; + break; + } + } + + if (has_node_specific_param) { + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_percent_in_node[nid]) { + phys_addr_t node_gfp_mem = memblock_node_memory_size(nid); + u64 s; + + s = mul_u64_u32_div((u64)node_gfp_mem, + hugetlb_cma_percent_in_node[nid], + 100); + + hugetlb_cma_size_in_node[nid] = s; + } + hugetlb_cma_size += hugetlb_cma_size_in_node[nid]; + } + } else if (hugetlb_cma_percent) { + hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(), + hugetlb_cma_percent, 100); + } + if (!hugetlb_cma_size) return; @@ -231,9 +309,11 @@ void __init hugetlb_cma_reserve(void) res = cma_declare_contiguous_multi(size, gigantic_page_size, HUGETLB_PAGE_ORDER, name, &hugetlb_cma[nid], nid); - if (res) { - pr_warn("hugetlb_cma: reservation failed: err %d, node %d", + if (res || !cma_validate_zones(hugetlb_cma[nid])) { + pr_warn("hugetlb_cma: %s: err %d, node %d\n", + res ? "reservation failed" : "reserved area spans zones", res, nid); + hugetlb_cma[nid] = NULL; continue; } diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index c619c394b1ae..3aa483573d17 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -6,8 +6,7 @@ void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int nid, nodemask_t *nodemask); -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact); +void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact); bool hugetlb_cma_exclusive_alloc(void); unsigned long hugetlb_cma_total_size(void); void hugetlb_cma_validate_params(void); @@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, return NULL; } -static inline -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact) +static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, + bool node_exact) { return NULL; } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 133b46dfb09f..917db0984143 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -12,7 +12,6 @@ #include <linux/pgtable.h> #include <linux/moduleparam.h> -#include <linux/bootmem_info.h> #include <linux/mmdebug.h> #include <linux/pagewalk.h> #include <linux/pgalloc.h> @@ -177,13 +176,13 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end, * Free a vmemmap page. A vmemmap page can be allocated from the memblock * allocator or buddy allocator. If the PG_reserved flag is set, it means * that it allocated from the memblock allocator, just free it via the - * free_bootmem_page(). Otherwise, use __free_page(). + * free_reserved_page(). Otherwise, use __free_page(). */ static inline void free_vmemmap_page(struct page *page) { if (PageReserved(page)) { memmap_boot_pages_add(-1); - free_bootmem_page(page); + free_reserved_page(page); } else { memmap_pages_add(-1); __free_page(page); @@ -624,6 +623,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, LIST_HEAD(vmemmap_pages); unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; + if (list_empty(folio_list)) + return; + nr_to_optimize = 0; list_for_each_entry(folio, folio_list, lru) { int ret; @@ -635,12 +637,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, * mirrored tail page structs RO. */ spfn = (unsigned long)&folio->page; - epfn = spfn + pages_per_huge_page(h); + epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(spfn), - &folio->page, - HUGETLB_VMEMMAP_RESERVE_SIZE); continue; } @@ -745,6 +744,20 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) return true; } +static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) +{ + struct zone *zone; + enum zone_type zone_type; + + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + zone = &NODE_DATA(nid)->node_zones[zone_type]; + if (zone_spans_pfn(zone, pfn)) + return zone; + } + + return NULL; +} + /* * Initialize memmap section for a gigantic page, HVO-style. */ @@ -752,6 +765,7 @@ void __init hugetlb_vmemmap_init_early(int nid) { unsigned long psize, paddr, section_size; unsigned long ns, i, pnum, pfn, nr_pages; + unsigned long start, end; struct huge_bootmem_page *m = NULL; void *map; @@ -761,6 +775,8 @@ void __init hugetlb_vmemmap_init_early(int nid) section_size = (1UL << PA_SECTION_SHIFT); list_for_each_entry(m, &huge_boot_pages[nid], list) { + struct zone *zone; + if (!vmemmap_should_optimize_bootmem_page(m)) continue; @@ -769,6 +785,14 @@ void __init hugetlb_vmemmap_init_early(int nid) paddr = virt_to_phys(m); pfn = PHYS_PFN(paddr); map = pfn_to_page(pfn); + start = (unsigned long)map; + end = start + hugetlb_vmemmap_size(m->hstate); + zone = pfn_to_zone(nid, pfn); + + if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), + zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) + panic("Failed to allocate memmap for HugeTLB page\n"); + memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE)); pnum = pfn_to_section_nr(pfn); ns = psize / section_size; @@ -783,78 +807,6 @@ void __init hugetlb_vmemmap_init_early(int nid) m->flags |= HUGE_BOOTMEM_HVO; } } - -static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) -{ - struct zone *zone; - enum zone_type zone_type; - - for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { - zone = &NODE_DATA(nid)->node_zones[zone_type]; - if (zone_spans_pfn(zone, pfn)) - return zone; - } - - return NULL; -} - -void __init hugetlb_vmemmap_init_late(int nid) -{ - struct huge_bootmem_page *m, *tm; - unsigned long phys, nr_pages, start, end; - unsigned long pfn, nr_mmap; - struct zone *zone = NULL; - struct hstate *h; - void *map; - - if (!READ_ONCE(vmemmap_optimize_enabled)) - return; - - list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) { - if (!(m->flags & HUGE_BOOTMEM_HVO)) - continue; - - phys = virt_to_phys(m); - h = m->hstate; - pfn = PHYS_PFN(phys); - nr_pages = pages_per_huge_page(h); - map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + nr_pages * sizeof(struct page); - - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Oops, the hugetlb page spans multiple zones. - * Remove it from the list, and populate it normally. - */ - list_del(&m->list); - - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - - memblock_phys_free(phys, huge_page_size(h)); - continue; - } - - if (!zone || !zone_spans_pfn(zone, pfn)) - zone = pfn_to_zone(nid, pfn); - if (WARN_ON_ONCE(!zone)) - continue; - - if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone, - HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) { - /* Fallback if HVO population fails */ - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - } else { - m->flags |= HUGE_BOOTMEM_ZONES_VALID; - nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE; - } - - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - } -} #endif static const struct ctl_table hugetlb_vmemmap_sysctls[] = { @@ -870,27 +822,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = { static int __init hugetlb_vmemmap_init(void) { const struct hstate *h; - struct zone *zone; /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */ BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES); - for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_TAIL_MIN_ORDER; - p = page_to_virt(tail); - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } - for_each_hstate(h) { if (hugetlb_vmemmap_optimizable(h)) { register_sysctl_init("vm", hugetlb_vmemmap_sysctls); diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 18b490825215..7ac49c52457d 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -29,7 +29,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); #ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT void hugetlb_vmemmap_init_early(int nid); -void hugetlb_vmemmap_init_late(int nid); #endif @@ -81,10 +80,6 @@ static inline void hugetlb_vmemmap_init_early(int nid) { } -static inline void hugetlb_vmemmap_init_late(int nid) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; diff --git a/mm/internal.h b/mm/internal.h index 181e79f1d6a2..5758dcaf4392 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,80 @@ #include "vma.h" struct folio_batch; +struct hstate; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; +}; + +/* mm/workingset.c */ +bool workingset_test_recent(void *shadow, bool file, bool *workingset, + bool flush); +void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); +void *workingset_eviction(struct folio *folio, + struct mem_cgroup *target_memcg); +void workingset_refault(struct folio *folio, void *shadow); +void workingset_activation(struct folio *folio); + +/* mm/folio.c */ +void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, + unsigned int nr_io, unsigned int nr_rotated); +void lru_note_cost_refault(struct folio *folio); +void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); + +static inline bool folio_may_be_lru_cached(struct folio *folio) +{ + /* + * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. + * Holding small numbers of low-order mTHP folios in per-CPU LRU cache + * will be sensible, but nobody has implemented and tested that yet. + */ + return !folio_test_large(folio); +} + +static inline void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} + +void lru_cache_disable(void); +void lru_add_drain(void); +void lru_add_drain_cpu(int cpu); +void lru_add_drain_cpu_zone(struct zone *zone); +void folio_deactivate(struct folio *folio); +void folio_mark_lazyfree(struct folio *folio); + +/* mm/vmscan.c */ +unsigned long zone_reclaimable_pages(struct zone *zone); +unsigned long try_to_free_pages(struct zonelist *zonelist, int order, + gfp_t gfp_mask, const nodemask_t *mask); +unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx); + +#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) +#define MEMCG_RECLAIM_PROACTIVE (1 << 2) +#define MIN_SWAPPINESS 0 +#define MAX_SWAPPINESS 200 + +/* Just reclaim from anon folios in proactive memory reclaim */ +#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + +unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness); +unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, + gfp_t gfp_mask, bool noswap, + pg_data_t *pgdat, + unsigned long *nr_scanned); + +#ifdef CONFIG_NUMA +extern int sysctl_min_unmapped_ratio; +extern int sysctl_min_slab_ratio; +#endif /* * Maintains state across a page table move. The operation assumes both source @@ -166,6 +240,10 @@ static inline int mmap_file(struct file *file, struct vm_area_struct *vma) { int err = vfs_mmap(file, vma); + /* Hooks cannot mark themselves anonymous. */ + if (WARN_ON_ONCE(vma_is_anonymous(vma))) + err = -EINVAL; + if (likely(!err)) return 0; @@ -202,6 +280,10 @@ void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap); #ifdef CONFIG_MMU +bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, + unsigned long nr_ptes); + static inline void get_anon_vma(struct anon_vma *anon_vma) { atomic_inc(&anon_vma->refcount); @@ -412,8 +494,8 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) new = pte_swp_mksoft_dirty(new); if (pte_swp_exclusive(pte)) new = pte_swp_mkexclusive(new); - if (pte_swp_uffd_wp(pte)) - new = pte_swp_mkuffd_wp(new); + if (pte_swp_uffd(pte)) + new = pte_swp_mkuffd(new); return new; } @@ -642,190 +724,16 @@ void set_recommended_min_free_kbytes(void); extern char * const zone_names[MAX_NR_ZONES]; -/* perform sanity checks on struct pages being allocated or freed */ -DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); - extern int min_free_kbytes; extern int defrag_mode; void setup_per_zone_wmarks(void); void calculate_min_free_kbytes(void); int __meminit init_per_zone_wmark_min(void); -void page_alloc_sysctl_init(void); - -/* - * Structure for holding the mostly immutable allocation parameters passed - * between functions involved in allocations, including the alloc_pages* - * family of functions. - * - * nodemask, migratetype and highest_zoneidx are initialized only once in - * __alloc_pages() and then never change. - * - * zonelist, preferred_zone and highest_zoneidx are set first in - * __alloc_pages() for the fast path, and might be later changed - * in __alloc_pages_slowpath(). All other functions pass the whole structure - * by a const pointer. - */ -struct alloc_context { - struct zonelist *zonelist; - nodemask_t *nodemask; - struct zoneref *preferred_zoneref; - int migratetype; - - /* - * highest_zoneidx represents highest usable zone index of - * the allocation request. Due to the nature of the zone, - * memory on lower zone than the highest_zoneidx will be - * protected by lowmem_reserve[highest_zoneidx]. - * - * highest_zoneidx is also used by reclaim/compaction to limit - * the target zone since higher zone than this index cannot be - * usable for this allocation request. - */ - enum zone_type highest_zoneidx; - bool spread_dirty_pages; -}; - -/* - * This function returns the order of a free page in the buddy system. In - * general, page_zone(page)->lock must be held by the caller to prevent the - * page from being allocated in parallel and returning garbage as the order. - * If a caller does not hold page_zone(page)->lock, it must guarantee that the - * page cannot be allocated or merged in parallel. Alternatively, it must - * handle invalid values gracefully, and use buddy_order_unsafe() below. - */ -static inline unsigned int buddy_order(struct page *page) -{ - /* PageBuddy() must be checked by the caller */ - return page_private(page); -} - -/* - * Like buddy_order(), but for callers who cannot afford to hold the zone lock. - * PageBuddy() should be checked first by the caller to minimize race window, - * and invalid values must be handled gracefully. - * - * READ_ONCE is used so that if the caller assigns the result into a local - * variable and e.g. tests it for valid range before using, the compiler cannot - * decide to remove the variable and inline the page_private(page) multiple - * times, potentially observing different values in the tests and the actual - * use of the result. - */ -#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) - -/* - * This function checks whether a page is free && is the buddy - * we can coalesce a page and its buddy if - * (a) the buddy is not in a hole (check before calling!) && - * (b) the buddy is in the buddy system && - * (c) a page and its buddy have the same order && - * (d) a page and its buddy are in the same zone. - * - * For recording whether a page is in the buddy system, we set PageBuddy. - * Setting, clearing, and testing PageBuddy is serialized by zone->lock. - * - * For recording page's order, we use page_private(page). - */ -static inline bool page_is_buddy(struct page *page, struct page *buddy, - unsigned int order) -{ - if (!page_is_guard(buddy) && !PageBuddy(buddy)) - return false; - - if (buddy_order(buddy) != order) - return false; - - /* - * zone check is done late to avoid uselessly calculating - * zone/node ids for pages that could never merge. - */ - if (page_zone_id(page) != page_zone_id(buddy)) - return false; - - VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); - - return true; -} - -/* - * Locate the struct page for both the matching buddy in our - * pair (buddy1) and the combined O(n+1) page they form (page). - * - * 1) Any buddy B1 will have an order O twin B2 which satisfies - * the following equation: - * B2 = B1 ^ (1 << O) - * For example, if the starting buddy (buddy2) is #8 its order - * 1 buddy is #10: - * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 - * - * 2) Any buddy B will have an order O+1 parent P which - * satisfies the following equation: - * P = B & ~(1 << O) - * - * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER - */ -static inline unsigned long -__find_buddy_pfn(unsigned long page_pfn, unsigned int order) -{ - return page_pfn ^ (1 << order); -} - -/* - * Find the buddy of @page and validate it. - * @page: The input page - * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the - * function is used in the performance-critical __free_one_page(). - * @order: The order of the page - * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to - * page_to_pfn(). - * - * The found buddy can be a non PageBuddy, out of @page's zone, or its order is - * not the same as @page. The validation is necessary before use it. - * - * Return: the found buddy page or NULL if not found. - */ -static inline struct page *find_buddy_page_pfn(struct page *page, - unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) -{ - unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); - struct page *buddy; - - buddy = page + (__buddy_pfn - pfn); - if (buddy_pfn) - *buddy_pfn = __buddy_pfn; - - if (page_is_buddy(page, buddy, order)) - return buddy; - return NULL; -} - -extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone); - -static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone) -{ - if (zone->contiguous) - return pfn_to_page(start_pfn); - - return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); -} - -void set_zone_contiguous(struct zone *zone); -bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, - unsigned long nr_pages); - -static inline void clear_zone_contiguous(struct zone *zone) -{ - zone->contiguous = false; -} extern int __isolate_free_page(struct page *page, unsigned int order); extern void __putback_isolated_page(struct page *page, unsigned int order, int mt); -extern void memblock_free_pages(unsigned long pfn, unsigned int order); -extern void __free_pages_core(struct page *page, unsigned int order, - enum meminit_context context); /* * This will have no effect, other than possibly generating a warning, if the @@ -895,7 +803,7 @@ static inline void prep_compound_tail(struct page *tail, { tail->mapping = TAIL_MAPPING; set_compound_head(tail, head, order); - set_page_private(tail, 0); + VM_WARN_ON_ONCE(tail->private); } static inline void init_compound_tail(struct page *tail, @@ -907,101 +815,6 @@ static inline void init_compound_tail(struct page *tail, prep_compound_tail(tail, head, order); } -void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); -extern bool free_pages_prepare(struct page *page, unsigned int order); - -extern int user_min_free_kbytes; - -struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid, - nodemask_t *); -#define __alloc_frozen_pages(...) \ - alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) -void free_frozen_pages(struct page *page, unsigned int order); -void free_unref_folios(struct folio_batch *fbatch); - -#ifdef CONFIG_NUMA -struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); -#else -static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) -{ - return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); -} -#endif - -#define alloc_frozen_pages(...) \ - alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) - -struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); -#define alloc_frozen_pages_nolock(...) \ - alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) -void free_frozen_pages_nolock(struct page *page, unsigned int order); - -extern void zone_pcp_reset(struct zone *zone); -extern void zone_pcp_disable(struct zone *zone); -extern void zone_pcp_enable(struct zone *zone); -extern void zone_pcp_init(struct zone *zone); - -extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, - int nid, bool exact_nid); - -void memmap_init_range(unsigned long, int, unsigned long, unsigned long, - unsigned long, enum meminit_context, struct vmem_altmap *, int, - bool); - -/* - * mm/sparse.c - */ -#ifdef CONFIG_SPARSEMEM -void sparse_init(void); -int sparse_index_init(unsigned long section_nr, int nid); - -static inline void sparse_init_one_section(struct mem_section *ms, - unsigned long pnum, struct page *mem_map, - struct mem_section_usage *usage, unsigned long flags) -{ - unsigned long coded_mem_map; - - BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); - - /* - * We encode the start PFN of the section into the mem_map such that - * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it - * from the page pointer to obtain the PFN. - */ - coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); - VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); - - ms->section_mem_map &= ~SECTION_MAP_MASK; - ms->section_mem_map |= coded_mem_map; - ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; - ms->usage = usage; -} - -static inline void __section_mark_present(struct mem_section *ms, - unsigned long section_nr) -{ - if (section_nr > __highest_present_section_nr) - __highest_present_section_nr = section_nr; - - ms->section_mem_map |= SECTION_MARKED_PRESENT; -} -#else -static inline void sparse_init(void) {} -#endif /* CONFIG_SPARSEMEM */ - -/* - * mm/sparse-vmemmap.c - */ -#ifdef CONFIG_SPARSEMEM_VMEMMAP -void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages); -#else -static inline void sparse_init_subsection_map(unsigned long pfn, - unsigned long nr_pages) -{ -} -#endif /* CONFIG_SPARSEMEM_VMEMMAP */ - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* @@ -1059,7 +872,15 @@ struct compact_control { * immediately when one is created during the free path. */ struct capture_control { - struct compact_control *cc; + struct zone *zone; + int migratetype; + /* + * Allocation request order. May differ from the compaction + * order: defrag_mode promotes sub-block allocations to + * pageblock-order compaction; capture still matches at the + * original allocation order so prep_new_page() is consistent. + */ + int order; struct page *page; }; @@ -1070,9 +891,6 @@ int isolate_migratepages_range(struct compact_control *cc, unsigned long low_pfn, unsigned long end_pfn); -/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ -void init_cma_reserved_pageblock(struct page *page); - #endif /* CONFIG_COMPACTION || CONFIG_CMA */ struct cma; @@ -1080,7 +898,6 @@ struct cma; #ifdef CONFIG_CMA bool cma_validate_zones(struct cma *cma); void *cma_reserve_early(struct cma *cma, unsigned long size); -void init_cma_pageblock(struct page *page); #else static inline bool cma_validate_zones(struct cma *cma) { @@ -1090,28 +907,8 @@ static inline void *cma_reserve_early(struct cma *cma, unsigned long size) { return NULL; } -static inline void init_cma_pageblock(struct page *page) -{ -} #endif -enum fallback_result { - /* Found suitable migratetype, *mt_out is valid. */ - FALLBACK_FOUND, - /* No fallback found in requested order. */ - FALLBACK_EMPTY, - /* Passed @claimable, but claiming whole block is a bad idea. */ - FALLBACK_NOCLAIM, -}; -enum fallback_result -find_suitable_fallback(struct free_area *area, unsigned int order, - int migratetype, bool claimable, int *mt_out); - -static inline bool free_area_empty(struct free_area *area, int migratetype) -{ - return list_empty(&area->free_list[migratetype]); -} - /* mm/util.c */ struct anon_vma *folio_anon_vma(const struct folio *folio); @@ -1143,26 +940,29 @@ static inline bool folio_within_range(struct folio *folio, struct vm_area_struct *vma, unsigned long start, unsigned long end) { - pgoff_t pgoff, addr; - unsigned long vma_pglen = vma_pages(vma); + const unsigned long vma_pglen = vma_pages(vma); + pgoff_t pgoff_folio, pgoff_vma_start; + unsigned long addr; VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); if (start > end) return false; + pgoff_folio = folio_pgoff(folio); + pgoff_vma_start = folio_test_anon(folio) ? + vma_start_virt_pgoff(vma) : vma_start_pgoff(vma); + if (start < vma->vm_start) start = vma->vm_start; if (end > vma->vm_end) end = vma->vm_end; - pgoff = folio_pgoff(folio); - /* if folio start address is not in vma range */ - if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) + if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen)) return false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT); return !(addr < start || end - addr < folio_size(folio)); } @@ -1222,27 +1022,18 @@ void mlock_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); -/** - * vma_address - Find the virtual address a page range is mapped at - * @vma: The vma which maps this object. - * @pgoff: The page offset within its object. - * @nr_pages: The number of pages to consider. - * - * If any page in this range is mapped by this VMA, return the first address - * where any of these pages appear. Otherwise, return -EFAULT. - */ -static inline unsigned long vma_address(const struct vm_area_struct *vma, - pgoff_t pgoff, unsigned long nr_pages) +static inline unsigned long __vma_address(const struct vm_area_struct *vma, + pgoff_t pgoff, pgoff_t pgoff_start, unsigned long nr_pages) { unsigned long address; - if (pgoff >= vma->vm_pgoff) { + if (pgoff >= pgoff_start) { address = vma->vm_start + - ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + ((pgoff - pgoff_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; - } else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) { + } else if (pgoff + nr_pages - 1 >= pgoff_start) { /* Test above avoids possibility of wrap to 0 on 32-bit */ address = vma->vm_start; } else { @@ -1251,22 +1042,68 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, return address; } +/** + * vma_filebacked_address - Find the virtual address a file-backed page range is + * mapped at. + * @vma: The vma which maps this object. + * @pgoff: The page offset within its object. + * @nr_pages: The number of pages to consider. + * + * Returns: If any page in this range is mapped by this VMA, return the first + * address where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma, + pgoff_t pgoff, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(vma_is_anonymous(vma)); + + return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages); +} + +/** + * vma_anon_address - Find the virtual address an anonymous page range is mapped + * at. + * @vma: The vma which maps this object. + * @pgoff_virt: The virtual page index belonging to the folio. + * @nr_pages: The number of pages to consider. + * + * This is only valid for anonymous or MAP_PRIVATE-mapped file-backed VMAs. + * + * Returns: If any page in this range is mapped by this VMA, return the first address + * where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_anon_address(const struct vm_area_struct *vma, + pgoff_t pgoff_virt, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(!vma_is_anonymous(vma) && vma_test(vma, VMA_SHARED_BIT)); + + return __vma_address(vma, pgoff_virt, vma_start_virt_pgoff(vma), nr_pages); +} + /* - * Then at what user virtual address will none of the range be found in vma? + * At what user virtual address will none of the range be found in vma? * Assumes that vma_address() already returned a good starting address. */ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) { - struct vm_area_struct *vma = pvmw->vma; - pgoff_t pgoff; + const struct vm_area_struct *vma = pvmw->vma; + const pgoff_t pgoff = pvmw->pgoff; + pgoff_t pgoff_vma_start; unsigned long address; + pgoff_t pgoff_end; /* Common case, plus ->pgoff is invalid for KSM */ if (pvmw->nr_pages == 1) return pvmw->address + PAGE_SIZE; - pgoff = pvmw->pgoff + pvmw->nr_pages; - address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pvmw->is_anon_walk) + pgoff_vma_start = vma_start_virt_pgoff(vma); + else + pgoff_vma_start = vma_start_pgoff(vma); + + pgoff_end = pgoff + pvmw->nr_pages; + address = vma->vm_start + + ((pgoff_end - pgoff_vma_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; @@ -1311,85 +1148,21 @@ static inline void mlock_new_folio(struct folio *folio) { } static inline bool need_mlock_drain(int cpu) { return false; } static inline void mlock_drain_local(void) { } static inline void mlock_drain_remote(int cpu) { } -static inline void vunmap_range_noflush(unsigned long start, unsigned long end) -{ -} #endif /* !CONFIG_MMU */ -/* Memory initialisation debug and verification */ -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -DECLARE_STATIC_KEY_TRUE(deferred_pages); - -static inline bool deferred_pages_enabled(void) -{ - return static_branch_unlikely(&deferred_pages); -} - -bool __init deferred_grow_zone(struct zone *zone, unsigned int order); -#else -static inline bool deferred_pages_enabled(void) -{ - return false; -} -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - -void init_deferred_page(unsigned long pfn, int nid); - -enum mminit_level { - MMINIT_WARNING, - MMINIT_VERIFY, - MMINIT_TRACE -}; - -#ifdef CONFIG_DEBUG_MEMORY_INIT - -extern int mminit_loglevel; - -#define mminit_dprintk(level, prefix, fmt, arg...) \ -do { \ - if (level < mminit_loglevel) { \ - if (level <= MMINIT_WARNING) \ - pr_warn("mminit::" prefix " " fmt, ##arg); \ - else \ - printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ - } \ -} while (0) - -extern void mminit_verify_pageflags_layout(void); -extern void mminit_verify_zonelist(void); -#else - -static inline void mminit_dprintk(enum mminit_level level, - const char *prefix, const char *fmt, ...) -{ -} - -static inline void mminit_verify_pageflags_layout(void) -{ -} - -static inline void mminit_verify_zonelist(void) -{ -} -#endif /* CONFIG_DEBUG_MEMORY_INIT */ - -#define NODE_RECLAIM_NOSCAN -2 -#define NODE_RECLAIM_FULL -1 -#define NODE_RECLAIM_SOME 0 -#define NODE_RECLAIM_SUCCESS 1 - #ifdef CONFIG_NUMA extern int node_reclaim_mode; -extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int); +extern unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t gfp_mask, unsigned int order); extern int find_next_best_node(int node, nodemask_t *used_node_mask); #else #define node_reclaim_mode 0 -static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask, - unsigned int order) +static inline unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t mask, unsigned int order) { - return NODE_RECLAIM_NOSCAN; + return 0; } static inline int find_next_best_node(int node, nodemask_t *used_node_mask) { @@ -1436,50 +1209,9 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -extern void set_pageblock_order(void); unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); -/* The ALLOC_WMARK bits are used as an index to zone->watermark */ -#define ALLOC_WMARK_MIN WMARK_MIN -#define ALLOC_WMARK_LOW WMARK_LOW -#define ALLOC_WMARK_HIGH WMARK_HIGH -#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ - -/* Mask to get the watermark bits */ -#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) - -/* - * Only MMU archs have async oom victim reclaim - aka oom_reaper so we - * cannot assume a reduced access to memory reserves is sufficient for - * !MMU - */ -#ifdef CONFIG_MMU -#define ALLOC_OOM 0x08 -#else -#define ALLOC_OOM ALLOC_NO_WATERMARKS -#endif - -#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access - * to 25% of the min watermark or - * 62.5% if __GFP_HIGH is set. - */ -#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% - * of the min watermark. - */ -#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ -#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ -#ifdef CONFIG_ZONE_DMA32 -#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ -#else -#define ALLOC_NOFRAGMENT 0x0 -#endif -#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ -#define ALLOC_TRYLOCK 0x400 /* Only use spin_trylock in allocation path */ -#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ - -/* Flags that allow allocations below the min watermark. */ -#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) enum ttu_flags; struct tlbflush_unmap_batch; @@ -1526,37 +1258,6 @@ struct migration_target_control { size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, struct folio *folio, loff_t fpos, size_t size); -/* - * mm/vmalloc.c - */ -#ifdef CONFIG_MMU -void __init vmalloc_init(void); -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask); -unsigned int get_vm_area_page_order(struct vm_struct *vm); -#else -static inline void vmalloc_init(void) -{ -} - -static inline -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask) -{ - return -EINVAL; -} -#endif - -void clear_vm_uninitialized_flag(struct vm_struct *vm); - -int __must_check __vmap_pages_range_noflush(unsigned long addr, - unsigned long end, pgprot_t prot, - struct page **pages, unsigned int page_shift); - -void vunmap_range_noflush(unsigned long start, unsigned long end); - -void __vunmap_range_noflush(unsigned long start, unsigned long end); - static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) { if (vma->vm_flags & VM_SHARED) @@ -1584,12 +1285,6 @@ int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, void free_zone_device_folio(struct folio *folio); int migrate_device_coherent_folio(struct folio *folio); -struct vm_struct *__get_vm_area_node(unsigned long size, - unsigned long align, unsigned long shift, - unsigned long vm_flags, unsigned long start, - unsigned long end, int node, gfp_t gfp_mask, - const void *caller); - /* * mm/gup.c */ @@ -1712,18 +1407,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, return !PageAnonExclusive(page); } -extern bool mirrored_kernelcore; -bool memblock_has_mirror(void); -void memblock_free_all(void); - -static __always_inline void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) { @@ -1753,10 +1436,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte); } -void __meminit __init_single_page(struct page *page, unsigned long pfn, - unsigned long zone, int nid); -void __meminit __init_page_from_nid(unsigned long pfn, int nid); - /* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, int priority); @@ -1951,4 +1630,37 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) +{ + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } +} + +static inline bool can_spin_trylock(void) +{ + /* + * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is + * unsafe in NMI. If spin_trylock() is called from hard IRQ the current + * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will + * mark the task as the owner of another rt_spin_lock which will + * confuse PI logic, so return immediately if called from hard IRQ or + * NMI. + * + * Note, irqs_disabled() case is ok. spin_trylock() can be called + * from raw_spin_lock_irqsave region. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) + return false; + + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return false; + + return true; +} + #endif /* __MM_INTERNAL_H */ diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 32bcfbfcf15f..26b8437e3b1b 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * mm/interval_tree.c - interval tree for mapping->i_mmap + * mm/interval_tree.c - interval tree for address_space->i_mmap and + * anon_vma->rb_root * * Copyright (C) 2012, Michel Lespinasse <walken@google.com> */ @@ -10,25 +11,29 @@ #include <linux/rmap.h> #include <linux/interval_tree_generic.h> -static inline unsigned long vma_start_pgoff(struct vm_area_struct *v) -{ - return v->vm_pgoff; -} +/* File-backed interval tree (address_space->i_mmap) */ INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, - unsigned long, shared.rb_subtree_last, - vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree) + pgoff_t, shared.rb_subtree_last, + vma_start_pgoff, vma_last_pgoff, static, + __mapping_rmap_tree) + +void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __mapping_rmap_tree_insert(vma, &mapping->i_mmap); +} -/* Insert node immediately after prev in the interval tree */ -void vma_interval_tree_insert_after(struct vm_area_struct *node, +/* Insert vma immediately after prev in the interval tree */ +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root) + struct address_space *mapping) { struct rb_node **link; struct vm_area_struct *parent; - unsigned long last = vma_last_pgoff(node); + const pgoff_t pgoff_last = vma_last_pgoff(vma); - VM_BUG_ON_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node); + VM_WARN_ON_ONCE_VMA(vma_start_pgoff(vma) != vma_start_pgoff(prev), vma); if (!prev->shared.rb.rb_right) { parent = prev; @@ -36,71 +41,95 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node, } else { parent = rb_entry(prev->shared.rb.rb_right, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; while (parent->shared.rb.rb_left) { parent = rb_entry(parent->shared.rb.rb_left, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; } link = &parent->shared.rb.rb_left; } - node->shared.rb_subtree_last = last; - rb_link_node(&node->shared.rb, &parent->shared.rb, link); - rb_insert_augmented(&node->shared.rb, &root->rb_root, - &vma_interval_tree_augment); + vma->shared.rb_subtree_last = pgoff_last; + rb_link_node(&vma->shared.rb, &parent->shared.rb, link); + rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root, + &__mapping_rmap_tree_augment); +} + +void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __mapping_rmap_tree_remove(vma, &mapping->i_mmap); } -static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc) +struct vm_area_struct * +mapping_rmap_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __mapping_rmap_tree_iter_first(&mapping->i_mmap, + pgoff_start, pgoff_last); +} + +struct vm_area_struct * +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __mapping_rmap_tree_iter_next(vma, pgoff_start, pgoff_last); +} + +/* Anonymous interval tree (anon_vma->rb_root) */ + +static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc) { - return vma_start_pgoff(avc->vma); + return vma_start_virt_pgoff(avc->vma); } -static inline unsigned long avc_last_pgoff(struct anon_vma_chain *avc) +static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc) { - return vma_last_pgoff(avc->vma); + return vma_last_virt_pgoff(avc->vma); } -INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last, +INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last, avc_start_pgoff, avc_last_pgoff, - static inline, __anon_vma_interval_tree) + static, __anon_rmap_tree) -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root) +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { #ifdef CONFIG_DEBUG_VM_RB - node->cached_vma_start = avc_start_pgoff(node); - node->cached_vma_last = avc_last_pgoff(node); + avc->cached_vma_start = avc_start_pgoff(avc); + avc->cached_vma_last = avc_last_pgoff(avc); #endif - __anon_vma_interval_tree_insert(node, root); + __anon_rmap_tree_insert(avc, &anon_vma->rb_root); } -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root) +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { - __anon_vma_interval_tree_remove(node, root); + __anon_rmap_tree_remove(avc, &anon_vma->rb_root); } struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long first, unsigned long last) +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_first(root, first, last); + return __anon_rmap_tree_iter_first(&anon_vma->rb_root, + pgoff_start, pgoff_last); } struct anon_vma_chain * -anon_vma_interval_tree_iter_next(struct anon_vma_chain *node, - unsigned long first, unsigned long last) +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_next(node, first, last); + return __anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last); } #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node) +void anon_rmap_tree_verify(struct anon_vma_chain *avc) { - WARN_ON_ONCE(node->cached_vma_start != avc_start_pgoff(node)); - WARN_ON_ONCE(node->cached_vma_last != avc_last_pgoff(node)); + WARN_ON_ONCE(avc->cached_vma_start != avc_start_pgoff(avc)); + WARN_ON_ONCE(avc->cached_vma_last != avc_last_pgoff(avc)); } #endif diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index cbef5e450954..a848eb2f9910 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -61,7 +61,7 @@ DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc); EXPORT_SYMBOL_GPL(kasan_flag_vmalloc); /* Whether to check write accesses only. */ -static bool kasan_flag_write_only = false; +static bool kasan_flag_write_only; #define PAGE_ALLOC_SAMPLE_DEFAULT 1 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3 diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 617bca76db49..27e8f3077e80 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -26,6 +26,7 @@ #include <asm/tlb.h> #include "internal.h" +#include "page_alloc.h" #include "mm_slot.h" enum scan_result { @@ -37,7 +38,7 @@ enum scan_result { SCAN_EXCEED_SWAP_PTE, SCAN_EXCEED_SHARED_PTE, SCAN_PTE_NON_PRESENT, - SCAN_PTE_UFFD_WP, + SCAN_PTE_UFFD, SCAN_PTE_MAPPED_HUGEPAGE, SCAN_LACK_REFERENCED_PAGE, SCAN_PAGE_NULL, @@ -605,8 +606,7 @@ void __khugepaged_exit(struct mm_struct *mm) spin_lock(&khugepaged_mm_lock); slot = mm_slot_lookup(mm_slots_hash, mm); if (slot && khugepaged_scan.mm_slot != slot) { - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); free = 1; } spin_unlock(&khugepaged_mm_lock); @@ -695,8 +695,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, result = SCAN_PTE_NON_PRESENT; goto out; } - if (pte_uffd_wp(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_uffd(pteval)) { + result = SCAN_PTE_UFFD; goto out; } page = vm_normal_page(vma, addr, pteval); @@ -1543,7 +1543,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, case SCAN_PAGE_NULL: case SCAN_DEL_PAGE_LRU: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_PAGE_LAZYFREE: last_result = ret; goto next_order; @@ -1664,15 +1664,15 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, /* * Always be strict with uffd-wp * enabled swap entries. Please see - * comment below for pte_uffd_wp(). + * comment below for pte_uffd(). */ - if (pte_swp_uffd_wp_any(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_swp_uffd_any(pteval)) { + result = SCAN_PTE_UFFD; goto out_unmap; } continue; } - if (pte_uffd_wp(pteval)) { + if (pte_uffd(pteval)) { /* * Don't collapse the page if any of the small * PTEs are armed with uffd write protection. @@ -1682,7 +1682,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, * userfault messages that falls outside of * the registered range. So, just be simple. */ - result = SCAN_PTE_UFFD_WP; + result = SCAN_PTE_UFFD; goto out_unmap; } @@ -1801,8 +1801,7 @@ static void collect_mm_slot(struct mm_slot *slot) if (collapse_test_exit(mm)) { /* free mm_slot */ - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); /* * Not strictly needed because the mm exited already. @@ -1892,9 +1891,12 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER)) return SCAN_VMA_CHECK; - /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ - if (userfaultfd_wp(vma)) - return SCAN_PTE_UFFD_WP; + /* + * Keep pmd pgtable while the uffd bit is in use; see comment in + * retract_page_tables(). + */ + if (userfaultfd_protected(vma)) + return SCAN_PTE_UFFD; folio = filemap_lock_folio(vma->vm_file->f_mapping, linear_page_index(vma, haddr)); @@ -2106,13 +2108,14 @@ static bool file_backed_vma_is_retractable(struct vm_area_struct *vma) return false; /* - * When a vma is registered with uffd-wp, we cannot recycle + * When a vma is registered with uffd-wp or RWP, we cannot recycle * the page table because there may be pte markers installed. - * Other vmas can still have the same file mapped hugely, but - * skip this one: it will always be mapped in small page size - * for uffd-wp registered ranges. + * VM_UFFD_RWP ranges similarly rely on per-PTE uffd state + * and cannot be recycled to a shared PMD. Other vmas can still + * have the same file mapped hugely, but skip this one: it will + * always be mapped in small page size for these registrations. */ - if (userfaultfd_wp(vma)) + if (userfaultfd_protected(vma)) return false; /* @@ -2136,7 +2139,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct vm_area_struct *vma; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; @@ -2145,7 +2148,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) spinlock_t *ptl; bool success = false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + + ((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); if (addr & ~HPAGE_PMD_MASK || vma->vm_end < addr + HPAGE_PMD_SIZE) continue; @@ -2568,7 +2572,7 @@ xa_unlocked: * not be able to observe any missing pages due to the * previously inserted retry entries. */ - vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { + mapping_rmap_tree_foreach(vma, mapping, start, end) { if (userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; @@ -3241,7 +3245,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, /* Whitelisted set of results where continuing OK */ case SCAN_NO_PTE_TABLE: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_LACK_REFERENCED_PAGE: case SCAN_PAGE_NULL: case SCAN_PAGE_COUNT: diff --git a/mm/kmemleak.c b/mm/kmemleak.c index e196f53f9b46..f63dfacee7ca 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -151,6 +151,8 @@ struct kmemleak_object { int min_count; /* the total number of pointers found pointing to this object */ int count; + /* consecutive scans the object has been seen unreferenced */ + unsigned int unref_scans; /* checksum for detecting modified objects */ u32 checksum; depot_stack_handle_t trace_handle; @@ -175,6 +177,8 @@ struct kmemleak_object { #define OBJECT_PHYS (1 << 4) /* flag set for per-CPU pointers */ #define OBJECT_PERCPU (1 << 5) +/* flag set on an object left unreferenced by the full scan, pending confirmation */ +#define OBJECT_SUSPECT (1 << 6) /* set when __remove_object() called */ #define DELSTATE_REMOVED (1 << 0) @@ -232,9 +236,14 @@ static unsigned long max_percpu_addr; static struct task_struct *scan_thread; /* used to avoid reporting of recently allocated objects */ static unsigned long jiffies_min_age; +/* consecutive scans an object must stay unreferenced before reporting */ +static unsigned int min_unref_scans = 1; +module_param(min_unref_scans, uint, 0644); static unsigned long jiffies_last_scan; /* delay between automatic memory scannings */ static unsigned long jiffies_scan_wait; +/* number of objects flagged OBJECT_SUSPECT during the current scan */ +static int nr_suspects; /* enables or disables the task stacks scanning */ static int kmemleak_stack_scan = 1; /* protects the memory scanning, parameters and debug/kmemleak file access */ @@ -688,6 +697,7 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp) object->excess_ref = 0; object->count = 0; /* white color initially */ object->checksum = ~0; + object->unref_scans = 0; object->del_state = 0; /* task information */ @@ -1440,6 +1450,11 @@ static void update_refs(struct kmemleak_object *object) */ object->count++; if (color_gray(object)) { + /* referenced after all, no longer a suspect */ + if (object->flags & OBJECT_SUSPECT) { + object->flags &= ~OBJECT_SUSPECT; + nr_suspects--; + } /* put_object() called when removing from gray_list */ WARN_ON(!get_object(object)); list_add_tail(&object->gray_list, &gray_list); @@ -1525,22 +1540,25 @@ static int scan_should_stop(void) /* * Scan a memory block (exclusive range) for valid pointers and add those - * found to the gray list. + * found to the gray list. Return non-zero if the scan was interrupted. */ -static void scan_block(void *_start, void *_end, - struct kmemleak_object *scanned) +static int scan_block(void *_start, void *_end, + struct kmemleak_object *scanned) { unsigned long *ptr; unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER); unsigned long *end = _end - (BYTES_PER_POINTER - 1); unsigned long flags; + int stop = 0; raw_spin_lock_irqsave(&kmemleak_lock, flags); for (ptr = start; ptr < end; ptr++) { unsigned long pointer; - if (scan_should_stop()) + if (scan_should_stop()) { + stop = 1; break; + } kasan_disable_current(); pointer = *(unsigned long *)kasan_reset_tag((void *)ptr); @@ -1550,22 +1568,28 @@ static void scan_block(void *_start, void *_end, pointer_update_refs(scanned, pointer, OBJECT_PERCPU); } raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + + return stop; } /* * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency. + * Return non-zero if the scan was interrupted. */ #ifdef CONFIG_SMP -static void scan_large_block(void *start, void *end) +static int scan_large_block(void *start, void *end) { void *next; while (start < end) { next = min(start + MAX_SCAN_SIZE, end); - scan_block(start, next, NULL); + if (scan_block(start, next, NULL)) + return 1; start = next; - cond_resched(); + cond_resched_tasks_rcu_qs(); } + + return 0; } #endif @@ -1599,7 +1623,7 @@ static void scan_object(struct kmemleak_object *object) scan_block(start, end, object); raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); if (!(object->flags & OBJECT_ALLOCATED)) break; @@ -1621,7 +1645,7 @@ static void scan_object(struct kmemleak_object *object) break; raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); } while (object->flags & OBJECT_ALLOCATED); } else { @@ -1649,7 +1673,7 @@ static void scan_gray_list(void) */ object = list_entry(gray_list.next, typeof(*object), gray_list); while (&object->gray_list != &gray_list) { - cond_resched(); + cond_resched_tasks_rcu_qs(); /* may add new objects to the list */ if (!scan_should_stop()) @@ -1684,7 +1708,7 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) raw_spin_unlock_irq(&kmemleak_lock); rcu_read_unlock(); - cond_resched(); + cond_resched_tasks_rcu_qs(); rcu_read_lock(); raw_spin_lock_irq(&kmemleak_lock); @@ -1697,6 +1721,43 @@ unlock_put: } /* + * Scan all task kernel stacks, rescheduling between tasks. Each task is looked + * up and pinned within its own RCU read-side section, so no lock is held across + * the scan and the walk cannot trip the soft lockup watchdog. + */ +static void kmemleak_scan_task_stacks(void) +{ + struct pid *pid; + int nr = 1; + int stop = 0; + + do { + struct task_struct *p = NULL; + + rcu_read_lock(); + pid = find_ge_pid(nr, &init_pid_ns); + if (pid) { + nr = pid_nr(pid) + 1; + p = pid_task(pid, PIDTYPE_PID); + if (p) + get_task_struct(p); + } + rcu_read_unlock(); + + if (p) { + void *stack = try_get_task_stack(p); + + if (stack) { + stop = scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } + put_task_struct(p); + } + cond_resched_tasks_rcu_qs(); + } while (pid && !stop); +} + +/* * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it * does not touch user memory that was freed concurrently; the rest of the * report (backtrace, comm, pid) is always emitted since the kmemleak_object @@ -1798,15 +1859,16 @@ static void dedup_flush(struct xarray *dedup) * kernel's standard allocators. This function must be called with the * scan_mutex held. */ -static void kmemleak_scan(void) +static int __kmemleak_scan(bool full) { struct kmemleak_object *object; struct zone *zone; int __maybe_unused i; - struct xarray dedup; - int new_leaks = 0; + int stop = 0; jiffies_last_scan = jiffies; + if (full) + nr_suspects = 0; /* prepare the kmemleak_object's */ rcu_read_lock(); @@ -1834,8 +1896,13 @@ static void kmemleak_scan(void) __paint_it(object, KMEMLEAK_BLACK); } + /* referenced last scan: restart the unreferenced run */ + if (!color_white(object)) + object->unref_scans = 0; /* reset the reference count (whiten the object) */ object->count = 0; + if (full) + object->flags &= ~OBJECT_SUSPECT; if (color_gray(object) && get_object(object)) list_add_tail(&object->gray_list, &gray_list); @@ -1848,9 +1915,11 @@ static void kmemleak_scan(void) #ifdef CONFIG_SMP /* per-cpu sections scanning */ - for_each_possible_cpu(i) - scan_large_block(__per_cpu_start + per_cpu_offset(i), - __per_cpu_end + per_cpu_offset(i)); + for_each_possible_cpu(i) { + if (scan_large_block(__per_cpu_start + per_cpu_offset(i), + __per_cpu_end + per_cpu_offset(i))) + goto scan_gray; + } #endif /* @@ -1866,7 +1935,7 @@ static void kmemleak_scan(void) struct page *page = pfn_to_online_page(pfn); if (!(pfn & 63)) - cond_resched(); + cond_resched_tasks_rcu_qs(); if (!page) continue; @@ -1877,34 +1946,34 @@ static void kmemleak_scan(void) /* only scan if page is in use */ if (page_count(page) == 0) continue; - scan_block(page, page + 1, NULL); + stop = scan_block(page, page + 1, NULL); + if (stop) + break; } + if (stop) + break; } put_online_mems(); + if (stop) + goto scan_gray; /* * Scanning the task stacks (may introduce false negatives). */ - if (kmemleak_stack_scan) { - struct task_struct *p, *g; - - rcu_read_lock(); - for_each_process_thread(g, p) { - void *stack = try_get_task_stack(p); - if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); - put_task_stack(p); - } - } - rcu_read_unlock(); - } + if (kmemleak_stack_scan) + kmemleak_scan_task_stacks(); /* * Scan the objects already referenced from the sections scanned * above. */ +scan_gray: scan_gray_list(); + /* a confirmation scan does not look for modified objects */ + if (!full) + return nr_suspects; + /* * Check for new or unreferenced objects modified since the previous * scan and color them gray until the next scan. @@ -1927,6 +1996,11 @@ static void kmemleak_scan(void) /* color it gray temporarily */ object->count = object->min_count; list_add_tail(&object->gray_list, &gray_list); + } else if (unreferenced_object(object) && + !(object->flags & OBJECT_REPORTED)) { + /* flag the objects left unreferenced by this scan */ + object->flags |= OBJECT_SUSPECT; + nr_suspects++; } raw_spin_unlock_irq(&object->lock); } @@ -1937,6 +2011,46 @@ static void kmemleak_scan(void) */ scan_gray_list(); + return nr_suspects; +} + +/* + * Promote a suspected object to a reported leak once it has stayed + * unreferenced for min_unref_scans consecutive scans. Called with + * object->lock held; returns true when the object is newly reported. + */ +static bool confirm_leak(struct kmemleak_object *object) +{ + if (!unreferenced_object(object) || + !(object->flags & OBJECT_SUSPECT) || + (object->flags & OBJECT_REPORTED)) + return false; + + object->unref_scans += 1; + if (object->unref_scans < min_unref_scans) + return false; + + object->flags |= OBJECT_REPORTED; + return true; +} + +/* + * Scan the memory and report the unreferenced objects as leaks. Must be + * called with the scan_mutex held. + */ +static void kmemleak_scan(void) +{ + struct kmemleak_object *object; + struct xarray dedup; + int new_leaks = 0; + + /* + * Full scan. Objects left unreferenced are flagged OBJECT_SUSPECT and + * counted in the return value; nothing to confirm or report otherwise. + */ + if (!__kmemleak_scan(true)) + return; + /* * If scanning was stopped do not report any new unreferenced objects. */ @@ -1944,6 +2058,16 @@ static void kmemleak_scan(void) return; /* + * A live object whose only reference is moved by, for example, a + * concurrent RCU update can be missed for one scan and reported as a + * transient false positive. Scan again and only report the objects + * left unreferenced (still flagged OBJECT_SUSPECT) by both scans. + */ + __kmemleak_scan(false); + if (scan_should_stop()) + return; + + /* * Scanning result reporting. When verbose printing is enabled, dedupe * by stackdepot trace_handle so each unique backtrace is logged once * per scan, annotated with the number of objects that share it. The @@ -1969,9 +2093,8 @@ static void kmemleak_scan(void) raw_spin_lock_irq(&object->lock); trace_handle = 0; dedup_print = false; - if (unreferenced_object(object) && - !(object->flags & OBJECT_REPORTED)) { - object->flags |= OBJECT_REPORTED; + + if (confirm_leak(object)) { if (kmemleak_verbose) { trace_handle = object->trace_handle; dedup_print = true; diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c index 8f22d1f22981..5f1b8053f9fa 100644 --- a/mm/kmsan/hooks.c +++ b/mm/kmsan/hooks.c @@ -21,6 +21,7 @@ #include <linux/usb.h> #include "../internal.h" +#include "../vmalloc.h" #include "../slab.h" #include "kmsan.h" diff --git a/mm/kmsan/init.c b/mm/kmsan/init.c index b14ce3417e65..4983b6e9f7c9 100644 --- a/mm/kmsan/init.c +++ b/mm/kmsan/init.c @@ -13,7 +13,7 @@ #include <linux/mm.h> #include <linux/memblock.h> -#include "../internal.h" +#include "../page_alloc.h" #define NUM_FUTURE_RANGES 128 struct start_end_pair { diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c index 8fde939784a7..0c88d89bf0d6 100644 --- a/mm/kmsan/shadow.c +++ b/mm/kmsan/shadow.c @@ -17,6 +17,7 @@ #include <linux/stddef.h> #include "../internal.h" +#include "../vmalloc.h" #include "kmsan.h" #define shadow_page_for(page) ((page)->kmsan_shadow) @@ -195,22 +195,28 @@ struct ksm_stable_node { * @node: rb node of this rmap_item in the unstable tree * @head: pointer to stable_node heading this list in the stable tree * @hlist: link into hlist of rmap_items hanging off that stable_node - * @age: number of scan iterations since creation - * @remaining_skips: how many scans to skip + * @age: number of scan iterations since creation (unstable node) + * @remaining_skips: how many scans to skip (unstable node) + * @linear_page_index: the original page's index before merged by KSM (stable node) */ struct ksm_rmap_item { struct ksm_rmap_item *rmap_list; union { - struct anon_vma *anon_vma; /* when stable */ + struct anon_vma *anon_vma; /* for reverse mapping, when stable */ #ifdef CONFIG_NUMA int nid; /* when node of unstable tree */ #endif }; struct mm_struct *mm; unsigned long address; /* + low bits used for flags below */ - unsigned int oldchecksum; /* when unstable */ - rmap_age_t age; - rmap_age_t remaining_skips; + union { + struct { + unsigned int oldchecksum; + rmap_age_t age; + rmap_age_t remaining_skips; + }; /* when unstable */ + unsigned long linear_page_index; /* for reverse mapping, when stable */ + }; union { struct rb_node node; /* when node of unstable tree */ struct { /* when listed from stable tree */ @@ -776,6 +782,11 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, return vma; } +/* + * break_cow: actively break COW, replacing the KSM page by a fresh anonymous + * page. This is called when rmap_item has not yet become stable, but page + * has been merged. + */ static void break_cow(struct ksm_rmap_item *rmap_item) { struct mm_struct *mm = rmap_item->mm; @@ -787,6 +798,11 @@ static void break_cow(struct ksm_rmap_item *rmap_item) * to undo, we also need to drop a reference to the anon_vma. */ put_anon_vma(rmap_item->anon_vma); + /* + * Reset linear_page_index that might overlay age-related + * information. (it's still unstable node) + */ + rmap_item->linear_page_index = 0; mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); @@ -899,6 +915,8 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node) VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->address &= PAGE_MASK; cond_resched(); } @@ -1052,6 +1070,8 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->head = NULL; rmap_item->address &= PAGE_MASK; @@ -1237,8 +1257,7 @@ mm_exiting: struct mm_slot, mm_node); ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); if (ksm_test_exit(mm)) { - hash_del(&mm_slot->slot.hash); - list_del(&mm_slot->slot.mm_node); + mm_slot_remove(&mm_slot->slot); spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); @@ -1598,8 +1617,16 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, /* Unstable nid is in union with stable anon_vma: remove first */ remove_rmap_item_from_tree(rmap_item); - /* Must get reference to anon_vma while still holding mmap_lock */ + /* + * We can consider the VMA only while still holding the mmap lock, + * so lock, so reference the anon_vma and calculate the linear + * page index early, before stable_tree_append(). If anything goes + * wrong that prevents the rmap_item from being added to the + * stable_tree, break_cow() will clean it up. + */ rmap_item->anon_vma = vma->anon_vma; + /* The VMA is always anon/MAP_PRIVATE-file backed so use anon index. */ + rmap_item->linear_page_index = linear_virt_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: mmap_read_unlock(mm); @@ -2327,23 +2354,24 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite tree_rmap_item = unstable_tree_search_insert(rmap_item, page, &tree_page); if (tree_rmap_item) { + struct folio *tree_folio; bool split; kfolio = try_to_merge_two_pages(rmap_item, page, tree_rmap_item, tree_page); + tree_folio = page_folio(tree_page); /* - * If both pages we tried to merge belong to the same compound - * page, then we actually ended up increasing the reference - * count of the same compound page twice, and split_huge_page - * failed. + * If both pages we tried to merge belong to the same (large) + * folio, then we actually ended up increasing the reference + * count of the same folio twice, and split_huge_page failed. + * * Here we set a flag if that happened, and we use it later to - * try split_huge_page again. Since we call put_page right + * try split_huge_page again. Since we call folio_put() right * afterwards, the reference count will be correct and * split_huge_page should succeed. */ - split = PageTransCompound(page) - && compound_head(page) == compound_head(tree_page); - put_page(tree_page); + split = folio == tree_folio; + folio_put(tree_folio); if (kfolio) { /* * The pages were successfully merged: insert new @@ -2458,6 +2486,13 @@ static bool should_skip_rmap_item(struct folio *folio, if (folio_test_ksm(folio)) return false; + /* + * There is no age information in stable-tree nodes. We might end up + * here without a KSM page for example after COW. + */ + if (rmap_item->address & STABLE_FLAG) + return false; + age = rmap_item->age; if (age != U8_MAX) rmap_item->age++; @@ -2737,8 +2772,7 @@ no_vmas: * or when all VM_MERGEABLE areas have been unmapped (and * mmap_lock then protects against race with MADV_MERGEABLE). */ - hash_del(&mm_slot->slot.hash); - list_del(&mm_slot->slot.mm_node); + mm_slot_remove(&mm_slot->slot); spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); @@ -3081,8 +3115,7 @@ void __ksm_exit(struct mm_struct *mm) if (ksm_scan.mm_slot == mm_slot) goto unlock; if (!mm_slot->rmap_list) { - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); easy_to_free = 1; } else { list_move(&slot->mm_node, @@ -3120,7 +3153,7 @@ struct folio *ksm_might_need_to_copy(struct folio *folio, return folio; /* no need to copy it */ } else if (!anon_vma) { return folio; /* no need to copy it */ - } else if (folio->index == linear_page_index(vma, addr) && + } else if (folio->index == linear_virt_page_index(vma, addr) && anon_vma->root == vma->anon_vma->root) { return folio; /* still no need to copy it */ } @@ -3173,6 +3206,7 @@ again: hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { /* Ignore the stable/unstable/sqnr flags */ const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct anon_vma *anon_vma = rmap_item->anon_vma; struct anon_vma_chain *vmac; struct vm_area_struct *vma; @@ -3186,8 +3220,17 @@ again: anon_vma_lock_read(anon_vma); } - anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root, - 0, ULONG_MAX) { + /* + * Currently, KSM folios are always small folios, so it's + * sufficient to search for a single page. We can simply use + * the linear_virt_page_index of the original de-duplicate + * anonymous page that we remembered in the rmap_item while + * de-duplicating. Note that mremap() always de-duplicates KSM + * folios: so if there was mremap() in our parent or our child, + * we wouldn't have the KSM folio mapped in these processes + * anymore. + */ + anon_rmap_tree_foreach(vmac, anon_vma, index, index) { cond_resched(); vma = vmac->vma; @@ -3243,17 +3286,16 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, rcu_read_lock(); for_each_process(tsk) { struct anon_vma_chain *vmac; - unsigned long addr; + const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct task_struct *t = task_early_kill(tsk, force_early); if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0, - ULONG_MAX) + anon_rmap_tree_foreach(vmac, av, index, index) { vma = vmac->vma; if (vma->vm_mm == t->mm) { - addr = rmap_item->address & PAGE_MASK; add_to_kill_ksm(t, page, vma, to_kill, addr); } diff --git a/mm/madvise.c b/mm/madvise.c index 77552b03d318..07a21ca31bad 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *walk) { struct vm_area_struct *vma = walk->private; - struct swap_iocb *splug = NULL; + struct swap_io_ctx ctx = {}; pte_t *ptep = NULL; spinlock_t *ptl; unsigned long addr; @@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, pte_unmap_unlock(ptep, ptl); ptep = NULL; - folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE, - vma, addr, &splug); + folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE, + vma, addr); if (folio) folio_put(folio); } if (ptep) pte_unmap_unlock(ptep, ptl); - swap_read_unplug(splug); + swap_read_submit(&ctx); cond_resched(); return 0; @@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); pgoff_t end_index = linear_page_index(vma, end) - 1; struct folio *folio; - struct swap_iocb *splug = NULL; + struct swap_io_ctx ctx = {}; rcu_read_lock(); xas_for_each(&xas, folio, end_index) { @@ -253,19 +253,19 @@ static void shmem_swapin_range(struct vm_area_struct *vma, continue; addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + ((xas.xa_index - vma_start_pgoff(vma)) << PAGE_SHIFT); xas_pause(&xas); rcu_read_unlock(); - folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping), - vma, addr, &splug); + folio = read_swap_cache_async(&ctx, entry, + mapping_gfp_mask(mapping), vma, addr); if (folio) folio_put(folio); rcu_read_lock(); } rcu_read_unlock(); - swap_read_unplug(splug); + swap_read_submit(&ctx); } #endif /* CONFIG_SWAP */ @@ -318,7 +318,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -388,8 +388,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, goto huge_unlock; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto huge_unlock; } @@ -694,10 +694,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, nr = swap_pte_batch(pte, max_nr, ptent); nr_swap -= nr; swap_put_entries_direct(entry, nr); - clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(mm, addr, pte, nr); } else if (softleaf_is_hwpoison(entry) || softleaf_is_poison_marker(entry)) { - pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); + pte_clear(mm, addr, pte); } continue; } @@ -1022,7 +1022,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1233,7 +1233,7 @@ static int guard_remove_pte_entry(pte_t *pte, unsigned long addr, if (is_guard_pte_marker(ptent)) { /* Simply clear the PTE marker. */ - pte_clear_not_present_full(walk->mm, addr, pte, false); + pte_clear(walk->mm, addr, pte); update_mmu_cache(walk->vma, addr, pte); } diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c index 737c407f4081..e0efa36e0a07 100644 --- a/mm/mapping_dirty_helpers.c +++ b/mm/mapping_dirty_helpers.c @@ -95,7 +95,7 @@ static int clean_record_pte(pte_t *pte, unsigned long addr, if (pte_dirty(ptent)) { pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) + - walk->vma->vm_pgoff - cwalk->bitmap_pgoff; + vma_start_pgoff(walk->vma) - cwalk->bitmap_pgoff; pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte); ptent = pte_mkclean(old_pte); diff --git a/mm/memblock.c b/mm/memblock.c index 6349c48154f4..777c69f05400 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -29,6 +29,7 @@ #include <linux/io.h> #include "internal.h" +#include "mm_init.h" #define INIT_MEMBLOCK_REGIONS 128 #define INIT_PHYSMEM_REGIONS 4 diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 765069211567..2dc599484d00 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -6,6 +6,7 @@ #include <linux/pagewalk.h> #include <linux/backing-dev.h> #include <linux/eventfd.h> +#include <linux/log2.h> #include <linux/poll.h> #include <linux/sort.h> #include <linux/file.h> @@ -751,7 +752,7 @@ static int compare_thresholds(const void *a, const void *b) return 0; } -static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) +static void mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) { struct mem_cgroup_eventfd_list *ev; @@ -761,7 +762,6 @@ static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) eventfd_signal(ev->eventfd); spin_unlock(&memcg_oom_lock); - return 0; } static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) @@ -1181,13 +1181,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, event->unregister_event = mem_cgroup_usage_unregister_event; } else if (!strcmp(name, "memory.oom_control")) { pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org" + "Please report your usecase to linux-mm@kvack.org" " if you depend on this functionality.\n"); event->register_event = mem_cgroup_oom_register_event; event->unregister_event = mem_cgroup_oom_unregister_event; } else if (!strcmp(name, "memory.pressure_level")) { pr_warn_once("pressure_level is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org " + "Please report your usecase to linux-mm@kvack.org " "if you depend on this functionality.\n"); event->register_event = vmpressure_register_event; event->unregister_event = vmpressure_unregister_event; @@ -1476,6 +1476,297 @@ void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked) mem_cgroup_oom_unlock(memcg); } +/* + * cgroup v1 userspace vmpressure interface (memory.pressure_level / + * cgroup.event_control). Kept here so v2-only kernels (CONFIG_MEMCG_V1=n) + * drop the whole eventfd accumulator, its work item, and the per-memcg + * state it requires. + * + * When there are too little pages left to scan, vmpressure() may miss the + * critical pressure as number of pages will be less than "window size". + * However, in that case the vmscan priority will raise fast as the + * reclaimer will try to scan LRUs more deeply. + * + * The vmscan logic considers these special priorities: + * + * prio == DEF_PRIORITY (12): reclaimer starts with that value + * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed + * prio == 0 : close to OOM, kernel scans every page in an lru + * + * Any value in this range is acceptable for this tunable (i.e. from 12 to + * 0). Current value for the vmpressure_level_critical_prio is chosen + * empirically, but the number, in essence, means that we consider + * critical level when scanning depth is ~10% of the lru size (vmscan + * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one + * eights). + */ +static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); + +enum vmpressure_modes { + VMPRESSURE_NO_PASSTHROUGH = 0, + VMPRESSURE_HIERARCHY, + VMPRESSURE_LOCAL, + VMPRESSURE_NUM_MODES, +}; + +static const char * const vmpressure_str_levels[] = { + [VMPRESSURE_LOW] = "low", + [VMPRESSURE_MEDIUM] = "medium", + [VMPRESSURE_CRITICAL] = "critical", +}; + +static const char * const vmpressure_str_modes[] = { + [VMPRESSURE_NO_PASSTHROUGH] = "default", + [VMPRESSURE_HIERARCHY] = "hierarchy", + [VMPRESSURE_LOCAL] = "local", +}; + +struct vmpressure_event { + struct eventfd_ctx *efd; + enum vmpressure_levels level; + enum vmpressure_modes mode; + struct list_head node; +}; + +static struct vmpressure *work_to_vmpressure(struct work_struct *work) +{ + return container_of(work, struct vmpressure, work); +} + +static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) +{ + struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); + + memcg = parent_mem_cgroup(memcg); + if (!memcg) + return NULL; + return memcg_to_vmpressure(memcg); +} + +static bool vmpressure_event(struct vmpressure *vmpr, + const enum vmpressure_levels level, + bool ancestor, bool signalled) +{ + struct vmpressure_event *ev; + bool ret = false; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ancestor && ev->mode == VMPRESSURE_LOCAL) + continue; + if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) + continue; + if (level < ev->level) + continue; + eventfd_signal(ev->efd); + ret = true; + } + mutex_unlock(&vmpr->events_lock); + + return ret; +} + +static void vmpressure_work_fn(struct work_struct *work) +{ + struct vmpressure *vmpr = work_to_vmpressure(work); + unsigned long scanned; + unsigned long reclaimed; + enum vmpressure_levels level; + bool ancestor = false; + bool signalled = false; + + spin_lock(&vmpr->sr_lock); + /* + * Several contexts might be calling vmpressure(), so it is + * possible that the work was rescheduled again before the old + * work context cleared the counters. In that case we will run + * just after the old work returns, but then scanned might be zero + * here. No need for any locks here since we don't care if + * vmpr->reclaimed is in sync. + */ + scanned = vmpr->tree_scanned; + if (!scanned) { + spin_unlock(&vmpr->sr_lock); + return; + } + + reclaimed = vmpr->tree_reclaimed; + vmpr->tree_scanned = 0; + vmpr->tree_reclaimed = 0; + spin_unlock(&vmpr->sr_lock); + + level = vmpressure_calc_level(scanned, reclaimed); + + do { + if (vmpressure_event(vmpr, level, ancestor, signalled)) + signalled = true; + ancestor = true; + } while ((vmpr = vmpressure_parent(vmpr))); +} + +/* + * Tree-mode accumulator: accumulate per-memcg scanned/reclaimed and + * schedule the work that walks the parent chain and signals registered + * eventfd listeners once we cross the window threshold. + */ +void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) +{ + spin_lock(&vmpr->sr_lock); + scanned = vmpr->tree_scanned += scanned; + vmpr->tree_reclaimed += reclaimed; + spin_unlock(&vmpr->sr_lock); + + if (scanned < vmpressure_win) + return; + schedule_work(&vmpr->work); +} + +void vmpressure_v1_init(struct vmpressure *vmpr) +{ + mutex_init(&vmpr->events_lock); + INIT_LIST_HEAD(&vmpr->events); + INIT_WORK(&vmpr->work, vmpressure_work_fn); +} + +void vmpressure_v1_cleanup(struct vmpressure *vmpr) +{ + /* + * Make sure there is no pending work before eventfd infrastructure + * goes away. + */ + flush_work(&vmpr->work); +} + +/** + * vmpressure_prio() - Account memory pressure through reclaimer priority level + * @gfp: reclaimer's gfp mask + * @memcg: cgroup memory controller handle + * @prio: reclaimer's priority + * + * This function should be called from the reclaim path every time when + * the vmscan's reclaiming priority (scanning depth) changes. + * + * This function does not return any value. + */ +void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) +{ + /* + * We only use prio for accounting critical level. For more info + * see comment for vmpressure_level_critical_prio variable above. + */ + if (prio > vmpressure_level_critical_prio) + return; + + /* + * OK, the prio is below the threshold, updating vmpressure + * information before shrinker dives into long shrinking of long + * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 + * to the vmpressure() basically means that we signal 'critical' + * level. + */ + vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); +} + +#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) + +/** + * vmpressure_register_event() - Bind vmpressure notifications to an eventfd + * @memcg: memcg that is interested in vmpressure notifications + * @eventfd: eventfd context to link notifications with + * @args: event arguments (pressure level threshold, optional mode) + * + * This function associates eventfd context with the vmpressure + * infrastructure, so that the notifications will be delivered to the + * @eventfd. The @args parameter is a comma-delimited string that denotes a + * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", + * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. + * "hierarchy" or "local"). + * + * To be used as memcg event method. + * + * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could + * not be parsed. + */ +int vmpressure_register_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd, const char *args) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; + enum vmpressure_levels level; + char *spec, *spec_orig; + char *token; + int ret = 0; + + spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); + if (!spec) + return -ENOMEM; + + /* Find required level */ + token = strsep(&spec, ","); + ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); + if (ret < 0) + goto out; + level = ret; + + /* Find optional mode */ + token = strsep(&spec, ","); + if (token) { + ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); + if (ret < 0) + goto out; + mode = ret; + } + + ev = kzalloc_obj(*ev, GFP_KERNEL_ACCOUNT); + if (!ev) { + ret = -ENOMEM; + goto out; + } + + ev->efd = eventfd; + ev->level = level; + ev->mode = mode; + + mutex_lock(&vmpr->events_lock); + list_add(&ev->node, &vmpr->events); + mutex_unlock(&vmpr->events_lock); + ret = 0; +out: + kfree(spec_orig); + return ret; +} + +/** + * vmpressure_unregister_event() - Unbind eventfd from vmpressure + * @memcg: memcg handle + * @eventfd: eventfd context that was used to link vmpressure with the @cg + * + * This function does internal manipulations to detach the @eventfd from + * the vmpressure notifications, and then frees internal resources + * associated with the @eventfd (but the @eventfd itself is not freed). + * + * To be used as memcg event method. + */ +void vmpressure_unregister_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ev->efd != eventfd) + continue; + list_del(&ev->node); + kfree(ev); + break; + } + mutex_unlock(&vmpr->events_lock); +} + static DEFINE_MUTEX(memcg_max_mutex); static int mem_cgroup_resize_max(struct mem_cgroup *memcg, @@ -1513,6 +1804,10 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg, if (!ret) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -1551,6 +1846,10 @@ static int mem_cgroup_force_empty(struct mem_cgroup *memcg) if (signal_pending(current)) return -EINTR; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL)) nr_retries--; @@ -2040,7 +2339,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, struct mem_cgroup *memcg = mem_cgroup_from_css(css); pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org if you " + "Please report your usecase to linux-mm@kvack.org if you " "depend on this functionality.\n"); /* cannot set to root cgroup and only 0 and 1 are allowed */ diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 4fa6e2bc8413..0f703f239c80 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -17,14 +17,8 @@ iter != NULL; \ iter = mem_cgroup_iter(root, iter, NULL)) -#define for_each_mem_cgroup(iter) \ - for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ - iter != NULL; \ - iter = mem_cgroup_iter(NULL, iter, NULL)) - void drain_all_stock(struct mem_cgroup *root_memcg); -unsigned long memcg_events(struct mem_cgroup *memcg, int event); int memory_stat_show(struct seq_file *m, void *v); struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6dc4888a90f3..8319ad8c5c23 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -137,6 +137,14 @@ bool mem_cgroup_kmem_disabled(void) static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); +static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages) +{ + mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); + memcg1_account_kmem(memcg, -nr_pages); + if (!mem_cgroup_is_root(memcg)) + memcg_uncharge(memcg, nr_pages); +} + static void obj_cgroup_release(struct percpu_ref *ref) { struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); @@ -172,10 +180,7 @@ static void obj_cgroup_release(struct percpu_ref *ref) struct mem_cgroup *memcg; memcg = get_mem_cgroup_from_objcg(objcg); - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); - memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - memcg_uncharge(memcg, nr_pages); + memcg_uncharge_kmem(memcg, nr_pages); mem_cgroup_put(memcg); } @@ -2039,7 +2044,7 @@ struct obj_stock_pcp { /* * On rare archs with 256KiB base page size (hexagon and powerpc 44x) * keep nr_bytes to unsigned int as uint16_t cannot represent the full -e patches/memcg-uint16_t-for-nr_bytes-in-obj_stock_pcp.patch * sub-page remainder. Such archs are not cacheline optimization target. + * sub-page remainder. Such archs are not cacheline optimization targets. */ unsigned int nr_bytes[NR_OBJ_STOCK]; #else @@ -3329,10 +3334,7 @@ static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i) memcg = get_mem_cgroup_from_objcg(old); - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); - memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - memcg_uncharge(memcg, nr_pages); + memcg_uncharge_kmem(memcg, nr_pages); css_put(&memcg->css); } @@ -4217,7 +4219,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) /* * A memcg must be visible for expand_shrinker_info() * by the time the maps are allocated. So, we allocate maps - * here, when for_each_mem_cgroup() can't skip it. + * here, when mem_cgroup_iter() can't skip it. */ if (alloc_shrinker_info(memcg)) goto offline_kmem; @@ -4362,6 +4364,11 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); + WRITE_ONCE(memcg->oom_group, false); +#ifdef CONFIG_ZSWAP + WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX); + WRITE_ONCE(memcg->zswap_writeback, true); +#endif #ifdef CONFIG_MEMCG_V1 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); @@ -4428,8 +4435,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) } #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC -static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, - int cpu) +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) { int nid; @@ -4438,6 +4444,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; + memcg->vmstats->state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4455,9 +4462,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4466,17 +4475,18 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } } } #else -static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, - int cpu) +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) {} #endif @@ -4488,7 +4498,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) struct aggregate_control ac; int nid; - flush_nmi_stats(memcg, parent, cpu); + flush_nmi_stats(memcg, parent); statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); @@ -4794,6 +4804,10 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -4845,6 +4859,10 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; diff --git a/mm/memfd.c b/mm/memfd.c index abe13b291ddc..c708d92533f4 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -19,6 +19,7 @@ #include <linux/memfd.h> #include <linux/pid_namespace.h> #include <uapi/linux/memfd.h> +#include "internal.h" #include "swap.h" /* @@ -369,39 +370,36 @@ static inline bool is_write_sealed(unsigned int seals) return seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE); } -static int check_write_seal(vm_flags_t *vm_flags_ptr) +static int check_write_seal(vma_flags_t *vma_flags_ptr) { - vm_flags_t vm_flags = *vm_flags_ptr; - vm_flags_t mask = vm_flags & (VM_SHARED | VM_WRITE); - /* If a private mapping then writability is irrelevant. */ - if (!(mask & VM_SHARED)) + if (!vma_flags_test(vma_flags_ptr, VMA_SHARED_BIT)) return 0; /* * New PROT_WRITE and MAP_SHARED mmaps are not allowed when * write seals are active. */ - if (mask & VM_WRITE) + if (vma_flags_test(vma_flags_ptr, VMA_WRITE_BIT)) return -EPERM; /* * This is a read-only mapping, disallow mprotect() from making a * write-sealed mapping writable in future. */ - *vm_flags_ptr &= ~VM_MAYWRITE; + vma_flags_clear(vma_flags_ptr, VMA_MAYWRITE_BIT); return 0; } -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr) +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr) { int err = 0; unsigned int *seals_ptr = memfd_file_seals_ptr(file); unsigned int seals = seals_ptr ? *seals_ptr : 0; if (is_write_sealed(seals)) - err = check_write_seal(vm_flags_ptr); + err = check_write_seal(vma_flags_ptr); return err; } diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51508a55c405..a8b03e2920ba 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -66,6 +66,7 @@ #include <trace/events/memory-failure.h> #include "swap.h" +#include "page_alloc.h" #include "internal.h" static int sysctl_memory_failure_early_kill __read_mostly; @@ -74,9 +75,11 @@ static int sysctl_memory_failure_recovery __read_mostly = 1; static int sysctl_enable_soft_offline __read_mostly = 1; +static int sysctl_panic_on_unrecoverable_mf __read_mostly; + atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); -static bool hw_memory_failure __read_mostly = false; +static bool hw_memory_failure __read_mostly; static DEFINE_MUTEX(mf_mutex); @@ -155,6 +158,15 @@ static const struct ctl_table memory_failure_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, + }, + { + .procname = "panic_on_unrecoverable_memory_failure", + .data = &sysctl_panic_on_unrecoverable_mf, + .maxlen = sizeof(sysctl_panic_on_unrecoverable_mf), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, } }; @@ -552,8 +564,7 @@ static void collect_procs_anon(const struct folio *folio, if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, - pgoff, pgoff) { + anon_rmap_tree_foreach(vmac, av, pgoff, pgoff) { vma = vmac->vma; if (vma->vm_mm != t->mm) continue; @@ -586,8 +597,7 @@ static void collect_procs_file(const struct folio *folio, if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, - pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { /* * Send early kill signal to tasks where a vma covers * the page but the corrupted page is not necessarily @@ -610,7 +620,7 @@ static void add_to_kill_fsdax(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, pgoff_t pgoff) { - unsigned long addr = vma_address(vma, pgoff, 1); + unsigned long addr = vma_filebacked_address(vma, pgoff, 1); __add_to_kill(tsk, p, vma, to_kill, addr); } @@ -638,7 +648,7 @@ static void collect_procs_fsdax(const struct page *page, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (vma->vm_mm == t->mm) add_to_kill_fsdax(t, page, vma, to_kill, pgoff); } @@ -981,17 +991,6 @@ static bool has_extra_refcount(struct page_state *ps, struct page *p, } /* - * Error hit kernel page. - * Do nothing, try to be lucky and not touch this instead. For a few cases we - * could be more sophisticated. - */ -static int me_kernel(struct page_state *ps, struct page *p) -{ - unlock_page(p); - return MF_IGNORED; -} - -/* * Page in unknown state. Do nothing. * This is a catch-all in case we fail to make sense of the page state. */ @@ -1199,10 +1198,8 @@ static int me_huge_page(struct page_state *ps, struct page *p) #define mlock (1UL << PG_mlocked) #define lru (1UL << PG_lru) #define head (1UL << PG_head) -#define reserved (1UL << PG_reserved) static struct page_state error_states[] = { - { reserved, reserved, MF_MSG_KERNEL, me_kernel }, /* * free pages are specially detected outside this table: * PG_buddy pages only make a small fraction of all free pages. @@ -1234,7 +1231,6 @@ static struct page_state error_states[] = { #undef mlock #undef lru #undef head -#undef reserved static void update_per_node_mf_stats(unsigned long pfn, enum mf_result result) @@ -1269,6 +1265,15 @@ static void update_per_node_mf_stats(unsigned long pfn, ++mf_stats->total; } +static bool panic_on_unrecoverable_mf(enum mf_action_page_type type, + enum mf_result result) +{ + if (!sysctl_panic_on_unrecoverable_mf) + return false; + + return type == MF_MSG_KERNEL && result == MF_IGNORED; +} + /* * "Dirty/Clean" indication is not 100% accurate due to the possibility of * setting PG_dirty outside page lock. See also comment above set_page_dirty(). @@ -1286,6 +1291,9 @@ static int action_result(unsigned long pfn, enum mf_action_page_type type, pr_err("%#lx: recovery action for %s: %s\n", pfn, action_page_types[type], action_name[result]); + if (panic_on_unrecoverable_mf(type, result)) + panic("Memory failure: %#lx: unrecoverable page", pfn); + return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; } @@ -1339,6 +1347,38 @@ static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) return PageLRU(page) || is_free_buddy_page(page); } +/* + * Positive identification of pages the hwpoison handler cannot recover: + * pages owned by kernel internals with no userspace mapping to unmap, no + * file mapping to invalidate, and no migration target. + */ +static inline bool is_kernel_owned_page(struct page *page) +{ + struct page *head; + bool kernel_owned; + + /* PG_reserved is a per-page flag, never set on a compound page. */ + if (PageReserved(page)) + return true; + + /* + * Page-type bits live only on the head page, so resolve any tail + * first. The check takes no refcount; recheck the head afterwards + * so a concurrent split or compound free cannot leave us trusting + * a stale view. A residual free->alloc->free cannot be closed here + * (frozen slab and large-kmalloc pages cannot be pinned), but is + * harmless: where a wrong verdict could panic, memory_failure() has + * already set PageHWPoison, which bars the page from the allocator. + */ +retry: + head = compound_head(page); + kernel_owned = PageSlab(head) || PageTable(head) || + PageLargeKmalloc(head); + if (head != compound_head(page)) + goto retry; + return kernel_owned; +} + static int __get_hwpoison_page(struct page *page, unsigned long flags) { struct folio *folio = page_folio(page); @@ -1385,6 +1425,19 @@ static int get_any_page(struct page *p, unsigned long flags) if (flags & MF_COUNT_INCREASED) count_increased = true; + /* + * Page types we know are kernel-owned and cannot be recovered. + * Short-circuit before the shake_page() / retry loop, which + * cannot turn any of these into something HWPoisonHandlable(). + * Drop the caller's reference if MF_COUNT_INCREASED took one. + */ + if (is_kernel_owned_page(p)) { + if (count_increased) + put_page(p); + ret = -ENOTRECOVERABLE; + goto out; + } + try_again: if (!count_increased) { ret = __get_hwpoison_page(p, flags); @@ -1432,7 +1485,7 @@ try_again: ret = -EIO; } out: - if (ret == -EIO) + if (ret == -EIO || ret == -ENOTRECOVERABLE) pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); return ret; @@ -1489,7 +1542,10 @@ static int __get_unpoison_page(struct page *page) * -EIO for pages on which we can not handle memory errors, * -EBUSY when get_hwpoison_page() has raced with page lifecycle * operations like allocation and free, - * -EHWPOISON when the page is hwpoisoned and taken off from buddy. + * -EHWPOISON when the page is hwpoisoned and taken off from buddy, + * -ENOTRECOVERABLE for kernel-owned pages identified by + * is_kernel_owned_page() (PG_reserved, slab, + * page-table, large-kmalloc) that the handler cannot recover. */ static int get_hwpoison_page(struct page *p, unsigned long flags) { @@ -2209,7 +2265,7 @@ static void add_to_kill_pgoff(struct task_struct *tsk, } /* Check for pgoff not backed by struct page */ - tk->addr = vma_address(vma, pgoff, 1); + tk->addr = vma_filebacked_address(vma, pgoff, 1); tk->size_shift = PAGE_SHIFT; if (tk->addr == -EFAULT) @@ -2239,7 +2295,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, mapping, 0, ULONG_MAX) { pgoff_t pgoff; if (vma->vm_mm == t->mm && @@ -2402,7 +2458,8 @@ try_again: * that may make page_ref_freeze()/page_ref_unfreeze() mismatch. */ res = get_hwpoison_page(p, flags); - if (!res) { + switch (res) { + case 0: if (is_free_buddy_page(p)) { if (take_page_off_buddy(p)) { page_ref_inc(p); @@ -2421,7 +2478,19 @@ try_again: res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); } goto unlock_mutex; - } else if (res < 0) { + case 1: + /* Got a refcount on a handlable page. */ + break; + case -ENOTRECOVERABLE: + /* + * Stable unhandlable kernel-owned page (PG_reserved, + * slab, page tables, large-kmalloc). + * No recovery possible. + */ + res = action_result(pfn, MF_MSG_KERNEL, MF_IGNORED); + goto unlock_mutex; + default: + /* Transient lifecycle race with the page allocator. */ res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); goto unlock_mutex; } diff --git a/mm/memory.c b/mm/memory.c index ff338c2abe92..a620d425ec95 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -519,9 +519,52 @@ static bool is_bad_page_map_ratelimited(void) return false; } +static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) +{ + if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) { + snprintf(buf, buf_size, "overflow"); + return; + } + + switch (entry_size) { + case sizeof(u32): + snprintf(buf, buf_size, "%08x", *(const u32 *)entry); + break; + case sizeof(u64): + snprintf(buf, buf_size, "%016llx", *(const u64 *)entry); + break; +#if defined(__SIZEOF_INT128__) + case sizeof(u128): + snprintf(buf, buf_size, "%016llx%016llx", + (unsigned long long)(*(const u128 *)entry >> 64), + (unsigned long long)*(const u128 *)entry); + break; +#endif + default: + snprintf(buf, buf_size, "unsupported"); + break; + } +} + +#define ptval_to_str(buf, val) \ + do { \ + auto __val = (val); \ + \ + ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ + } while (0) + +#if defined(__SIZEOF_INT128__) +#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ +#else +#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ +#endif + static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr) { - unsigned long long pgdv, p4dv, pudv, pmdv; + char pgd_str[PTVAL_STR_MAX]; + char p4d_str[PTVAL_STR_MAX]; + char pud_str[PTVAL_STR_MAX]; + char pmd_str[PTVAL_STR_MAX]; p4d_t p4d, *p4dp; pud_t pud, *pudp; pmd_t pmd, *pmdp; @@ -532,34 +575,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * see locking requirements for print_bad_page_map(). */ pgdp = pgd_offset(mm, addr); - pgdv = pgd_val(*pgdp); + ptval_to_str(pgd_str, pgd_val(*pgdp)); if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) { - pr_alert("pgd:%08llx\n", pgdv); + pr_alert("pgd:%s\n", pgd_str); return; } p4dp = p4d_offset(pgdp, addr); p4d = p4dp_get(p4dp); - p4dv = p4d_val(p4d); + ptval_to_str(p4d_str, p4d_val(p4d)); if (!p4d_present(p4d) || p4d_leaf(p4d)) { - pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); + pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str); return; } pudp = pud_offset(p4dp, addr); pud = pudp_get(pudp); - pudv = pud_val(pud); + ptval_to_str(pud_str, pud_val(pud)); if (!pud_present(pud) || pud_leaf(pud)) { - pr_alert("pgd:%08llx p4d:%08llx pud:%08llx\n", pgdv, p4dv, pudv); + pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str); return; } pmdp = pmd_offset(pudp, addr); pmd = pmdp_get(pmdp); - pmdv = pmd_val(pmd); + ptval_to_str(pmd_str, pmd_val(pmd)); /* * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE, @@ -567,8 +610,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * doing another map would be bad. print_bad_page_map() should * already take care of printing the PTE. */ - pr_alert("pgd:%08llx p4d:%08llx pud:%08llx pmd:%08llx\n", pgdv, - p4dv, pudv, pmdv); + pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str); } /* @@ -584,25 +626,29 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * page table lock. */ static void print_bad_page_map(struct vm_area_struct *vma, - unsigned long addr, unsigned long long entry, struct page *page, - enum pgtable_level level) + unsigned long addr, const void *entry, size_t entry_size, + struct page *page, enum pgtable_level level) { struct address_space *mapping; - pgoff_t index; + char entry_str[PTVAL_STR_MAX]; + pgoff_t index, virt_index; if (is_bad_page_map_ratelimited()) return; mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; index = linear_page_index(vma, addr); + virt_index = __linear_virt_page_index(vma, addr); - pr_alert("BUG: Bad page map in process %s %s:%08llx", current->comm, - pgtable_level_to_str(level), entry); + ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size); + pr_alert("BUG: Bad page map in process %s %s:%s", current->comm, + pgtable_level_to_str(level), entry_str); __print_bad_page_map_pgtable(vma->vm_mm, addr); if (page) dump_page(page, "bad page map"); - pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n", - (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); + pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx virt_index:%lx\n", + (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index, + virt_index); pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n", vma->vm_file, vma->vm_ops ? vma->vm_ops->fault : NULL, @@ -627,8 +673,13 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) } } -#define print_bad_pte(vma, addr, pte, page) \ - print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE) +static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, + pte_t pte, struct page *page) +{ + auto entry = pte_val(pte); + + print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE); +} /** * __vm_normal_page() - Get the "struct page" associated with a page table entry. @@ -636,8 +687,9 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) * @addr: The address where the page table entry is mapped. * @pfn: The PFN stored in the page table entry. * @special: Whether the page table entry is marked "special". - * @level: The page table level for error reporting purposes only. * @entry: The page table entry value for error reporting purposes only. + * @entry_size: The size of @entry. + * @level: The page table level for error reporting purposes only. * * "Special" mappings do not wish to be associated with a "struct page" (either * it doesn't exist, or it exists but they don't want to touch it). In this @@ -697,7 +749,7 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) */ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, bool special, - unsigned long long entry, enum pgtable_level level) + const void *entry, size_t entry_size, enum pgtable_level level) { if (pgtable_level_has_pxx_special(level)) { if (unlikely(special)) { @@ -710,7 +762,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return NULL; - print_bad_page_map(vma, addr, entry, NULL, level); + print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* @@ -725,10 +777,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (!pfn_valid(pfn)) return NULL; } else { - unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT; + const pgoff_t index = linear_page_index(vma, addr); /* Only CoW'ed anon folios are "normal". */ - if (pfn == vma->vm_pgoff + off) + if (pfn == index) return NULL; if (!is_cow_mapping(vma->vm_flags)) return NULL; @@ -741,7 +793,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (unlikely(pfn > highest_memmap_pfn)) { /* Corrupted page table entry. */ - print_bad_page_map(vma, addr, entry, NULL, level); + print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* @@ -767,8 +819,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { + auto entry = pte_val(pte); + return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte), - pte_val(pte), PGTABLE_LEVEL_PTE); + &entry, sizeof(entry), PGTABLE_LEVEL_PTE); } /** @@ -809,8 +863,10 @@ struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd) { + auto entry = pmd_val(pmd); + return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd), - pmd_val(pmd), PGTABLE_LEVEL_PMD); + &entry, sizeof(entry), PGTABLE_LEVEL_PMD); } /** @@ -850,8 +906,10 @@ struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud) { + auto entry = pud_val(pud); + return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud), - pud_val(pud), PGTABLE_LEVEL_PUD); + &entry, sizeof(entry), PGTABLE_LEVEL_PUD); } #endif @@ -893,8 +951,12 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, if (pte_swp_soft_dirty(orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_mkuffd(pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma)) + pte = pte_modify(pte, PAGE_NONE); if ((vma->vm_flags & VM_WRITE) && can_change_pte_writable(vma, address, pte)) { @@ -953,14 +1015,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (swap_dup_entry_direct(entry) < 0) return -EIO; - /* make sure dst_mm is on swapoff's mmlist. */ - if (unlikely(list_empty(&dst_mm->mmlist))) { - spin_lock(&mmlist_lock); - if (list_empty(&dst_mm->mmlist)) - list_add(&dst_mm->mmlist, - &src_mm->mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(dst_mm); /* Mark the swap entry as shared. */ if (pte_swp_exclusive(orig_pte)) { pte = pte_swp_clear_exclusive(orig_pte); @@ -984,8 +1039,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_private(entry)) { @@ -1018,8 +1073,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_device_private_entry( swp_offset(entry)); pte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_exclusive(entry)) { @@ -1041,8 +1096,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, make_pte_marker(marker)); return 0; } - if (!userfaultfd_wp(dst_vma)) - pte = pte_swp_clear_uffd_wp(pte); + if (!userfaultfd_protected(dst_vma)) + pte = pte_swp_clear_uffd(pte); set_pte_at(dst_mm, addr, dst_pte, pte); return 0; } @@ -1088,9 +1143,13 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma /* All done, just insert the new page copy in the child */ pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); - if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte))) - /* Uffd-wp needs to be delivered to dest pte as well */ - pte = pte_mkuffd_wp(pte); + if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) { + /* The uffd bit needs to be delivered to the dest pte as well */ + pte = pte_mkuffd(pte); + /* Restore PAGE_NONE so the RWP marker keeps trapping */ + if (userfaultfd_rwp(dst_vma)) + pte = pte_modify(pte, PAGE_NONE); + } set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); return 0; } @@ -1100,9 +1159,31 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte_t pte, unsigned long addr, int nr) { struct mm_struct *src_mm = src_vma->vm_mm; + bool writable; + + /* + * Snapshot writability before the RWP-disarm rewrite below: when the + * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can + * silently drop _PAGE_RW from a resolved (no-marker) writable PTE, + * so a later pte_write(pte) check would skip the COW wrprotect and + * leave the parent writable over a folio shared with the child. + */ + writable = pte_write(pte); + + /* + * Child is not RWP-armed: restore accessible protection so the + * inherited PAGE_NONE does not cost a fault on first read. Gate on + * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing) + * are not normalised away. + */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pte_uffd(pte)) + pte = pte_modify(pte, dst_vma->vm_page_prot); + pte = pte_clear_uffd(pte); + } /* If it's a COW mapping, write protect it both processes. */ - if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) { + if (is_cow_mapping(src_vma->vm_flags) && writable) { wrprotect_ptes(src_mm, addr, src_pte, nr); pte = pte_wrprotect(pte); } @@ -1112,9 +1193,6 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte = pte_mkclean(pte); pte = pte_mkold(pte); - if (!userfaultfd_wp(dst_vma)) - pte = pte_clear_uffd_wp(pte); - set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); } @@ -1599,40 +1677,89 @@ static inline bool zap_drop_markers(struct zap_details *details) return details->zap_flags & ZAP_FLAG_DROP_MARKER; } -/* - * This function makes sure that we'll replace the none pte with an uffd-wp - * swap special pte marker when necessary. Must be with the pgtable lock held. +/** + * cond_install_uffd_wp_ptes - install uffd-wp markers after clearing PTEs + * @vma: The VMA the pages are mapped into. + * @addr: Address the first page of this batch is mapped at. + * @ptep: Page table pointer for the first entry of this batch. + * @pte: Old value of the entry pointed to by @ptep. + * @nr_ptes: Number of entries to install. + * + * If the PTEs were write-protected by uffd-wp in any form, arm special PTEs + * to replace none PTEs. NOTE! This should only be called when the PTEs are + * already cleared so we will never accidentally replace something valuable. + * Meanwhile none PTEs also mean we are not demoting the PTEs so a TLB flush is + * not needed. E.g., when the PTEs were cleared, the caller should have taken + * care of the TLB flush. + * + * Must be called with the page table lock held so that no thread will see the + * none PTEs, and if they see them, they'll fault and serialize at the page table + * lock. * - * Returns true if uffd-wp ptes was installed, false otherwise. + * Returns true if uffd-wp PTEs were installed, false otherwise. */ -static inline bool -zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, - unsigned long addr, pte_t *pte, int nr, - struct zap_details *details, pte_t pteval) +bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, + unsigned long nr_ptes) { - bool was_installed = false; + bool arm_uffd_pte = false; if (!uffd_supports_wp_marker()) return false; - /* Zap on anonymous always means dropping everything */ - if (vma_is_anonymous(vma)) + /* The current status of the pte should be "cleared" before calling */ + WARN_ON_ONCE(!pte_none(ptep_get(ptep))); + + /* + * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole + * thing, because when zapping either it means it's dropping the + * page, or in TTU where the present pte will be quickly replaced + * with a swap pte. There's no way of leaking the bit. + */ + if (vma_is_anonymous(vma) || !userfaultfd_wp(vma)) return false; - if (zap_drop_markers(details)) + /* A uffd-wp wr-protected normal pte */ + if (unlikely(pte_present(pte) && pte_uffd(pte))) + arm_uffd_pte = true; + + /* + * A uffd-wp wr-protected swap pte. Note: this should even cover an + * existing pte marker with uffd-wp bit set. + */ + if (unlikely(pte_swp_uffd_any(pte))) + arm_uffd_pte = true; + + if (likely(!arm_uffd_pte)) return false; for (;;) { - /* the PFN in the PTE is irrelevant. */ - if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval)) - was_installed = true; - if (--nr == 0) + set_pte_at(vma->vm_mm, addr, ptep, + make_pte_marker(PTE_MARKER_UFFD_WP)); + if (--nr_ptes == 0) break; - pte++; + ptep++; addr += PAGE_SIZE; } - return was_installed; + return true; +} + +/* + * This function makes sure that we'll replace the none pte with an uffd-wp + * swap special pte marker when necessary. Must be with the pgtable lock held. + * + * Returns true if uffd-wp ptes was installed, false otherwise. + */ +static inline bool +zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, + unsigned long addr, pte_t *pte, int nr, + struct zap_details *details, pte_t pteval) +{ + if (zap_drop_markers(details)) + return false; + + return cond_install_uffd_wp_ptes(vma, addr, pte, pteval, nr); } static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, @@ -1797,7 +1924,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } - clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr); *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); return nr; @@ -2643,7 +2770,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma_start_pgoff(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -3298,7 +3425,8 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn; int err; - err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + err = __simple_ioremap_prep(vm_len, vma_start_pgoff(vma), start, len, + &pfn); if (err) return err; @@ -3925,8 +4053,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (unlikely(unshare)) { if (pte_soft_dirty(vmf->orig_pte)) entry = pte_mksoft_dirty(entry); - if (pte_uffd_wp(vmf->orig_pte)) - entry = pte_mkuffd_wp(entry); + if (pte_uffd(vmf->orig_pte)) + entry = pte_mkuffd(entry); } else { entry = maybe_mkwrite(pte_mkdirty(entry), vma); } @@ -4181,6 +4309,9 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { + const bool maybe_in_lru_cache = !folio_test_lru(folio); + const bool in_swapcache = folio_test_swapcache(folio); + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); @@ -4191,15 +4322,16 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, * * KSM doesn't necessarily raise the folio refcount. */ - if (folio_test_ksm(folio) || folio_ref_count(folio) > 3) + if (folio_test_ksm(folio) || + folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache) return false; - if (!folio_test_lru(folio)) + if (maybe_in_lru_cache) /* * We cannot easily detect+handle references from * remote LRU caches or references to LRU folios. */ lru_add_drain(); - if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio)) + if (folio_ref_count(folio) > 1 + in_swapcache) return false; if (!folio_trylock(folio)) return false; @@ -4261,7 +4393,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) * etc.) because we're only removing the uffd-wp bit, * which is completely invisible to the user. */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + pte = pte_clear_uffd(ptep_get(vmf->pte)); set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); /* @@ -4336,21 +4468,21 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_copy(vmf); } -static inline void unmap_mapping_range_tree(struct rb_root_cached *root, +static inline void unmap_mapping_range_tree(struct address_space *mapping, pgoff_t first_index, pgoff_t last_index, struct zap_details *details) { struct vm_area_struct *vma; - unsigned long start, size; struct mmu_gather tlb; - vma_interval_tree_foreach(vma, root, first_index, last_index) { - const pgoff_t start_idx = max(first_index, vma->vm_pgoff); + mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) { + const pgoff_t start_idx = max(first_index, vma_start_pgoff(vma)); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; - - start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); - size = (end_idx - start_idx) << PAGE_SHIFT; + const pgoff_t offset = start_idx - vma_start_pgoff(vma); + const unsigned long offset_bytes = offset << PAGE_SHIFT; + const unsigned long start = vma->vm_start + offset_bytes; + const unsigned long size = (end_idx - start_idx) << PAGE_SHIFT; tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); @@ -4387,7 +4519,7 @@ void unmap_mapping_folio(struct folio *folio) i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } @@ -4417,7 +4549,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } @@ -4512,7 +4644,7 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf) static inline bool should_try_to_free_swap(struct swap_info_struct *si, struct folio *folio, struct vm_area_struct *vma, - unsigned int extra_refs, + bool exclusive, unsigned int fault_flags) { if (!folio_test_swapcache(folio)) @@ -4528,14 +4660,12 @@ static inline bool should_try_to_free_swap(struct swap_info_struct *si, if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) || folio_test_mlocked(folio)) return true; + /* - * If we want to map a page that's in the swapcache writable, we - * have to detect via the refcount if we're really the exclusive - * user. Try freeing the swapcache to get rid of the swapcache - * reference only in case it's likely that we'll be the exclusive user. + * Free the swapcache only if we are the exclusive user and + * this is a write fault. */ - return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) && - folio_ref_count(folio) == (extra_refs + folio_nr_pages(folio)); + return (fault_flags & FAULT_FLAG_WRITE) && exclusive; } static vm_fault_t pte_marker_clear(struct vm_fault *vmf) @@ -4752,6 +4882,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) struct swap_info_struct *si = NULL; rmap_t rmap_flags = RMAP_NONE; bool exclusive = false; + bool rwp_restore = false; softleaf_t entry; pte_t pte; vm_fault_t ret = 0; @@ -4897,16 +5028,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) } else if (folio != swapcache) page = folio_page(folio, 0); - /* - * If we want to map a page that's in the swapcache writable, we - * have to detect via the refcount if we're really the exclusive - * owner. Try removing the extra reference from the local LRU - * caches if required. - */ - if ((vmf->flags & FAULT_FLAG_WRITE) && - !folio_test_ksm(folio) && !folio_test_lru(folio)) - lru_add_drain(); - folio_throttle_swaprate(folio, GFP_KERNEL); /* @@ -5038,24 +5159,34 @@ check_folio: pte = mk_pte(page, vma->vm_page_prot); if (pte_swp_soft_dirty(vmf->orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(vmf->orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(vmf->orig_pte)) + pte = pte_mkuffd(pte); + + /* + * A page reclaimed while RWP-protected carries the uffd bit on + * its swap entry. Re-apply PAGE_NONE on swap-in so the first access + * still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD. + */ + if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) { + pte = pte_modify(pte, PAGE_NONE); + rwp_restore = true; + } /* - * Same logic as in do_wp_page(); however, optimize for pages that are - * certainly not shared either because we just allocated them without - * exposing them to the swapcache or because the swap entry indicates - * exclusivity. + * Similar logic as in do_wp_page(); however, optimize for pages that + * are certainly exclusive. + * + * Skip the write upgrade for an RWP-restored pte: it must stay + * PROT_NONE so the access retries through the RWP fault path + * (do_uffd_rwp()) rather than being made writable here. */ - if (!folio_test_ksm(folio) && - (exclusive || folio_ref_count(folio) == 1)) { - if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + if (exclusive) { + if (!rwp_restore && + (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && !pte_needs_soft_dirty_wp(vma, pte)) { pte = pte_mkwrite(pte, vma); - if (vmf->flags & FAULT_FLAG_WRITE) { + if (vmf->flags & FAULT_FLAG_WRITE) pte = pte_mkdirty(pte); - vmf->flags &= ~FAULT_FLAG_WRITE; - } } rmap_flags |= RMAP_EXCLUSIVE; } @@ -5095,7 +5226,7 @@ check_folio: * Do it after mapping, so raced page faults will likely see the folio * in swap cache and wait on the folio lock. */ - if (should_try_to_free_swap(si, folio, vma, nr_pages, vmf->flags)) + if (should_try_to_free_swap(si, folio, vma, exclusive, vmf->flags)) folio_free_swap(folio); folio_unlock(folio); @@ -5112,7 +5243,12 @@ check_folio: folio_put(swapcache); } - if (vmf->flags & FAULT_FLAG_WRITE) { + /* + * For an RWP-restored pte, leave it PROT_NONE and let the write + * retry through the RWP fault path; do not COW it here, which would + * drop the marker for a non-exclusive page. + */ + if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) { ret |= do_wp_page(vmf); if (ret & VM_FAULT_ERROR) ret &= VM_FAULT_ERROR; @@ -5259,7 +5395,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); @@ -5326,7 +5462,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) return handle_userfault(vmf, VM_UFFD_MISSING); } if (vmf_orig_pte_uffd_wp(vmf)) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); set_pte_at(vma->vm_mm, addr, vmf->pte, entry); /* No need to invalidate - it was non-present before */ @@ -5576,7 +5712,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, else if (pte_write(entry) && folio_test_dirty(folio)) entry = pte_mkdirty(entry); if (unlikely(vmf_orig_pte_uffd_wp(vmf))) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { VM_BUG_ON_FOLIO(nr != 1, folio); @@ -5684,7 +5820,7 @@ fallback: } else if (nr_pages > 1) { pgoff_t idx = folio_page_idx(folio, page); /* The page offset of vmf->address within the VMA. */ - pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; + pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); /* The index of the entry in the pagetable for fault page. */ pgoff_t pte_off = pte_index(vmf->address); @@ -5796,7 +5932,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) pgoff_t nr_pages = READ_ONCE(fault_around_pages); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ - pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; + pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); pgoff_t from_pte, to_pte; vm_fault_t ret; @@ -6091,6 +6227,16 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru if (!pte_present(ptent) || !pte_protnone(ptent)) continue; + /* + * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a + * marker. Leave them alone -- rewriting to vm_page_prot would + * stop the RWP trap. Gate on userfaultfd_rwp(vma) too: + * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs + * when applying PROT_NONE, and those still need rebuilding. + */ + if (userfaultfd_rwp(vma) && pte_uffd(ptent)) + continue; + if (pfn_folio(pte_pfn(ptent)) != folio) continue; @@ -6106,6 +6252,35 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru } } +static vm_fault_t do_uffd_rwp(struct vm_fault *vmf) +{ + pte_t pte; + + if (!userfaultfd_rwp_async(vmf->vma)) { + /* Sync mode: unmap PTE and deliver to userfaultfd handler */ + pte_unmap(vmf->pte); + return handle_userfault(vmf, VM_UFFD_RWP); + } + + spin_lock(vmf->ptl); + if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return 0; + } + pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot); + /* pte_modify() preserves _PAGE_UFFD; drop it on resolution */ + pte = pte_clear_uffd(pte); + pte = pte_mkyoung(pte); + if (!pte_write(pte) && + vma_wants_manual_pte_write_upgrade(vmf->vma) && + can_change_pte_writable(vmf->vma, vmf->address, pte)) + pte = pte_mkwrite(pte, vmf->vma); + set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte); + update_mmu_cache(vmf->vma, vmf->address, vmf->pte); + pte_unmap_unlock(vmf->pte, vmf->ptl); + return 0; +} + static vm_fault_t do_numa_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -6381,8 +6556,16 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) if (!pte_present(vmf->orig_pte)) return do_swap_page(vmf); - if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) + if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) { + /* + * RWP-protected PTEs are protnone plus the uffd bit. On a + * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is + * NUMA hinting and must still fall through to do_numa_page(). + */ + if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) + return do_uffd_rwp(vmf); return do_numa_page(vmf); + } spin_lock(vmf->ptl); entry = vmf->orig_pte; @@ -6496,8 +6679,11 @@ retry_pud: return 0; } if (pmd_trans_huge(vmf.orig_pmd)) { - if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) + if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) { + if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd)) + return do_huge_pmd_uffd_rwp(&vmf); return do_huge_pmd_numa_page(&vmf); + } if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && !pmd_write(vmf.orig_pmd)) { @@ -7274,7 +7460,7 @@ void print_vma_addr(char *prefix, unsigned long ip) if (vma && vma->vm_file) { struct file *f = vma->vm_file; ip -= vma->vm_start; - ip += vma->vm_pgoff << PAGE_SHIFT; + ip += vma_start_pgoff(vma) << PAGE_SHIFT; printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip, vma->vm_start, vma->vm_end - vma->vm_start); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 8b137328dcf0..226ab9cb078a 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -40,6 +40,8 @@ #include <asm/tlbflush.h> #include "internal.h" +#include "mm_init.h" +#include "page_alloc.h" #include "shuffle.h" enum { @@ -239,6 +241,7 @@ enum mmop mhp_get_default_online_type(void) return mhp_default_online_type; } +EXPORT_SYMBOL_GPL(mhp_get_default_online_type); void mhp_set_default_online_type(enum mmop online_type) { @@ -1338,7 +1341,9 @@ static int check_hotplug_memory_range(u64 start, u64 size) static int online_memory_block(struct memory_block *mem, void *arg) { - mem->online_type = mhp_get_default_online_type(); + enum mmop *online_type = arg; + + mem->online_type = *online_type; return device_online(&mem->dev); } @@ -1492,7 +1497,8 @@ out: * * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ -int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +static int __add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags, + enum mmop online_type) { struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) }; enum memblock_flags memblock_flags = MEMBLOCK_NONE; @@ -1582,8 +1588,9 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) merge_system_ram_resource(res); /* online pages if requested */ - if (mhp_get_default_online_type() != MMOP_OFFLINE) - walk_memory_blocks(start, size, NULL, online_memory_block); + if (online_type != MMOP_OFFLINE) + walk_memory_blocks(start, size, &online_type, + online_memory_block); return ret; error: @@ -1599,7 +1606,13 @@ error_mem_hotplug_end: return ret; } -/* requires device_hotplug_lock, see add_memory_resource() */ +int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +{ + return __add_memory_resource(nid, res, mhp_flags, + mhp_get_default_online_type()); +} + +/* requires device_hotplug_lock, see __add_memory_resource() */ int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) { struct resource *res; @@ -1627,7 +1640,15 @@ int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) } EXPORT_SYMBOL_GPL(add_memory); -/* +/** + * __add_memory_driver_managed - add driver-managed memory with explicit online_type + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * @online_type: Auto-Online behavior (offline, online, kernel, movable) + * * Add special, driver-managed memory to the system as system RAM. Such * memory is not exposed via the raw firmware-provided memmap as system * RAM, instead, it is detected and added by a driver - during cold boot, @@ -1635,6 +1656,7 @@ EXPORT_SYMBOL_GPL(add_memory); * * Reasons why this memory should not be used for the initial memmap of a * kexec kernel or for placing kexec images: + * * - The booting kernel is in charge of determining how this memory will be * used (e.g., use persistent memory as system RAM) * - Coordination with a hypervisor is required before this memory @@ -1647,9 +1669,12 @@ EXPORT_SYMBOL_GPL(add_memory); * * The resource_name (visible via /proc/iomem) has to have the format * "System RAM ($DRIVER)". + * + * Return: 0 on success, negative error code on failure. */ -int add_memory_driver_managed(int nid, u64 start, u64 size, - const char *resource_name, mhp_t mhp_flags) +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type) { struct resource *res; int rc; @@ -1659,6 +1684,9 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, resource_name[strlen(resource_name) - 1] != ')') return -EINVAL; + if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE) + return -EINVAL; + lock_device_hotplug(); res = register_memory_resource(start, size, resource_name); @@ -1667,7 +1695,7 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, goto out_unlock; } - rc = add_memory_resource(nid, res, mhp_flags); + rc = __add_memory_resource(nid, res, mhp_flags, online_type); if (rc < 0) release_memory_resource(res); @@ -1675,6 +1703,30 @@ out_unlock: unlock_device_hotplug(); return rc; } +EXPORT_SYMBOL_FOR_MODULES(__add_memory_driver_managed, "kmem"); + +/** + * add_memory_driver_managed - add driver-managed memory + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * + * Add driver-managed memory with the system default online type set by + * build config or kernel boot parameter. + * + * See __add_memory_driver_managed for more details. + * + * Return: 0 on success, negative error code on failure. + */ +int add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags) +{ + return __add_memory_driver_managed(nid, start, size, resource_name, + mhp_flags, + mhp_get_default_online_type()); +} EXPORT_SYMBOL_GPL(add_memory_driver_managed); /* @@ -2380,58 +2432,98 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg) */ int offline_and_remove_memory(u64 start, u64 size) { - const unsigned long mb_count = size / memory_block_size_bytes(); + struct range range = { + .start = start, + .end = start + size - 1, + }; + + return offline_and_remove_memory_ranges(&range, 1); +} +EXPORT_SYMBOL_GPL(offline_and_remove_memory); + +/** + * offline_and_remove_memory_ranges - offline and remove multiple memory ranges + * @ranges: array of physical address ranges to offline and remove + * @nr_ranges: number of entries in @ranges + * + * Offline and remove several memory ranges as one operation, serialized + * against other hotplug operations by a single lock_device_hotplug(). + * + * This offlines all ranges before removing any of them. If offlining any + * range fails, the entire process is reverted and nothing is removed. + * This provides a fully atomic semantic for unplugging an entire device. + * + * Each range must be memory-block aligned in start and size. + * + * Return: 0 on success, negative errno on failure (never positive). On + * failure no range has been removed. + */ +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + unsigned long mb_count = 0; uint8_t *online_types, *tmp; - int rc; + unsigned int i; + int rc = 0; - if (!IS_ALIGNED(start, memory_block_size_bytes()) || - !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + if (!ranges || !nr_ranges) return -EINVAL; + for (i = 0; i < nr_ranges; i++) { + const u64 start = ranges[i].start; + const u64 size = range_len(&ranges[i]); + + if (!IS_ALIGNED(start, memory_block_size_bytes()) || + !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + return -EINVAL; + mb_count += size / memory_block_size_bytes(); + } + /* - * We'll remember the old online type of each memory block, so we can - * try to revert whatever we did when offlining one memory block fails - * after offlining some others succeeded. + * Remember the old online type of every memory block across all ranges, + * so we can revert if offlining a later block fails. All entries start + * as MMOP_OFFLINE so blocks we never touched are skipped on rollback. */ online_types = kmalloc_array(mb_count, sizeof(*online_types), GFP_KERNEL); if (!online_types) return -ENOMEM; - /* - * Initialize all states to MMOP_OFFLINE, so when we abort processing in - * try_offline_memory_block(), we'll skip all unprocessed blocks in - * try_reonline_memory_block(). - */ memset(online_types, MMOP_OFFLINE, mb_count); lock_device_hotplug(); - tmp = online_types; - rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block); - /* - * In case we succeeded to offline all memory, remove it. - * This cannot fail as it cannot get onlined in the meantime. + * Phase 1: offline every block in every range. An already-offline + * block folds to success, so out-of-band offlining never blocks unplug. */ - if (!rc) { - rc = try_remove_memory(start, size); + tmp = online_types; + for (i = 0; i < nr_ranges; i++) { + rc = walk_memory_blocks(ranges[i].start, range_len(&ranges[i]), + &tmp, try_offline_memory_block); if (rc) - pr_err("%s: Failed to remove memory: %d", __func__, rc); + break; } - /* - * Rollback what we did. While memory onlining might theoretically fail - * (nacked by a notifier), it barely ever happens. - */ + /* If any failure occurred at all, rollback any changes and bail */ if (rc) { tmp = online_types; - walk_memory_blocks(start, size, &tmp, - try_reonline_memory_block); + for (i = 0; i < nr_ranges; i++) + walk_memory_blocks(ranges[i].start, + range_len(&ranges[i]), &tmp, + try_reonline_memory_block); + goto out_unlock; } + + /* Phase 2: Remove. This should never fail holding the hotplug lock */ + for (i = 0; i < nr_ranges; i++) + WARN_ON_ONCE(try_remove_memory(ranges[i].start, + range_len(&ranges[i]))); + +out_unlock: unlock_device_hotplug(); kfree(online_types); return rc; } -EXPORT_SYMBOL_GPL(offline_and_remove_memory); +EXPORT_SYMBOL_GPL(offline_and_remove_memory_ranges); #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 36699fabd3c2..5720f7f54d94 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -119,6 +119,7 @@ #include <linux/memory.h> #include "internal.h" +#include "page_alloc.h" /* Internal flags */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ @@ -653,12 +654,14 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) { struct folio *folio; struct queue_pages *qp = walk->private; + pmd_t pmdval = pmdp_get(pmd); - if (unlikely(pmd_is_migration_entry(*pmd))) { - qp->nr_failed++; + if (unlikely(!pmd_present(pmdval))) { + if (pmd_is_migration_entry(pmdval)) + qp->nr_failed++; return; } - folio = pmd_folio(*pmd); + folio = pmd_folio(pmdval); if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; @@ -2048,8 +2051,8 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, pol = get_task_policy(current); if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { - *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += vma_start_pgoff(vma) >> order; + *ilx += linear_page_delta(vma, addr) >> order; } return pol; } @@ -2057,24 +2060,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma) { struct mempolicy *pol; + pgoff_t ilx; + bool mof; - if (vma->vm_ops && vma->vm_ops->get_policy) { - bool ret = false; - pgoff_t ilx; /* ignored here */ - - pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx); - if (pol && (pol->flags & MPOL_F_MOF)) - ret = true; - mpol_cond_put(pol); - - return ret; - } - - pol = vma->vm_policy; + pol = __get_vma_policy(vma, vma->vm_start, &ilx); if (!pol) pol = get_task_policy(current); - - return pol->flags & MPOL_F_MOF; + mof = pol->flags & MPOL_F_MOF; + mpol_cond_put(pol); + return mof; } bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) @@ -2425,9 +2419,11 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, */ preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask); + page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, + ALLOC_DEFAULT); if (!page) - page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL); + page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, + ALLOC_DEFAULT); return page; } @@ -2475,7 +2471,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, */ page = __alloc_frozen_pages_noprof( gfp | __GFP_THISNODE | __GFP_NORETRY, order, - nid, NULL); + nid, NULL, ALLOC_DEFAULT); if (page || !(gfp & __GFP_DIRECT_RECLAIM)) return page; /* @@ -2487,7 +2483,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } - page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask); + page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, ALLOC_DEFAULT); if (unlikely(pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) { @@ -3250,16 +3246,17 @@ EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_init, "kvm"); int mpol_set_shared_policy(struct shared_policy *sp, struct vm_area_struct *vma, struct mempolicy *pol) { - int err; + const pgoff_t pgoff = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); struct sp_node *new = NULL; - unsigned long sz = vma_pages(vma); + int err; if (pol) { - new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); + new = sp_alloc(pgoff, pgoff_end, pol); if (!new) return -ENOMEM; } - err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new); + err = shared_policy_replace(sp, pgoff, pgoff_end, new); if (err && new) sp_free(new); return err; diff --git a/mm/migrate.c b/mm/migrate.c index dd15a84b2a52..ab15a4dddd04 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -49,6 +49,7 @@ #include <trace/events/migrate.h> #include "internal.h" +#include "page_alloc.h" #include "swap.h" static const struct movable_operations *offline_movable_ops; @@ -326,8 +327,12 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, if (pte_swp_soft_dirty(old_pte)) newpte = pte_mksoft_dirty(newpte); - if (pte_swp_uffd_wp(old_pte)) - newpte = pte_mkuffd_wp(newpte); + if (pte_swp_uffd(old_pte)) + newpte = pte_mkuffd(newpte); + + /* See remove_migration_pte(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma)) + newpte = pte_modify(newpte, PAGE_NONE); set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); @@ -358,11 +363,13 @@ static bool remove_migration_pte(struct folio *folio, unsigned long idx = 0; /* pgoff is invalid for ksm pages, but they are never large */ - if (folio_test_large(folio) && !folio_test_hugetlb(folio)) - idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; + if (folio_test_large(folio) && !folio_test_hugetlb(folio)) { + idx += linear_folio_page_index(folio, vma, pvmw.address); + idx -= pvmw.pgoff; + } new = folio_page(folio, idx); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || @@ -371,7 +378,11 @@ static bool remove_migration_pte(struct folio *folio, continue; } #endif - old_pte = ptep_get(pvmw.pte); + if (folio_test_hugetlb(folio)) + old_pte = huge_ptep_get(vma->vm_mm, pvmw.address, + pvmw.pte); + else + old_pte = ptep_get(pvmw.pte); if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; @@ -391,8 +402,12 @@ static bool remove_migration_pte(struct folio *folio, if (softleaf_is_migration_write(entry)) pte = pte_mkwrite(pte, vma); - else if (pte_swp_uffd_wp(old_pte)) - pte = pte_mkuffd_wp(pte); + else if (pte_swp_uffd(old_pte)) + pte = pte_mkuffd(pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma)) + pte = pte_modify(pte, PAGE_NONE); if (folio_test_anon(folio) && !softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; @@ -407,8 +422,8 @@ static bool remove_migration_pte(struct folio *folio, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(old_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(old_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(old_pte)) + pte = pte_swp_mkuffd(pte); } #ifdef CONFIG_HUGETLB_PAGE @@ -545,7 +560,7 @@ fail: } #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; @@ -885,7 +900,7 @@ static int __migrate_folio(struct address_space *mapping, struct folio *dst, * @mapping: The address_space containing the folio. * @dst: The folio to migrate the data to. * @src: The folio containing the current data. - * @mode: How to migrate the page. + * @mode: How to migrate the folio. * * Common logic to directly migrate a single LRU folio suitable for * folios that do not have private data. @@ -1131,7 +1146,7 @@ static int move_to_new_folio(struct folio *dst, struct folio *src, } /* - * To record some information during migration, we use unused private + * To record some information during migration, we use the migrate_info * field of struct folio of the newly allocated destination folio. * This is safe because nobody is using it except us. */ @@ -1144,27 +1159,24 @@ enum { static void __migrate_folio_record(struct folio *dst, int old_folio_state, struct anon_vma *anon_vma) { - dst->private = (void *)anon_vma + old_folio_state; + dst->migrate_info = (unsigned long)anon_vma | old_folio_state; } static void __migrate_folio_extract(struct folio *dst, int *old_folio_state, struct anon_vma **anon_vmap) { - unsigned long private = (unsigned long)dst->private; + unsigned long info = dst->migrate_info; - *anon_vmap = (struct anon_vma *)(private & ~FOLIO_OLD_STATES); - *old_folio_state = private & FOLIO_OLD_STATES; - dst->private = NULL; + *anon_vmap = (struct anon_vma *)(info & ~FOLIO_OLD_STATES); + *old_folio_state = info & FOLIO_OLD_STATES; + dst->migrate_info = 0; } /* Restore the source folio to the original state upon failure */ -static void migrate_folio_undo_src(struct folio *src, - int page_was_mapped, - struct anon_vma *anon_vma, - bool locked, - struct list_head *ret) +static void migrate_folio_undo_src(struct folio *src, int was_mapped, + struct anon_vma *anon_vma, bool locked, struct list_head *ret) { - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, src, 0); /* Drop an anon_vma reference if we took one */ if (anon_vma) @@ -1218,7 +1230,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return -ENOMEM; *dstp = dst; - dst->private = NULL; + dst->migrate_info = 0; if (!folio_trylock(src)) { if (mode == MIGRATE_ASYNC) @@ -1450,7 +1462,8 @@ out: } /* - * Counterpart of unmap_and_move_page() for hugepage migration. + * Counterpart of migrate_folio_unmap() and migrate_folio_move() for hugetlb + * folio migration. * * This function doesn't wait the completion of hugepage I/O * because there is no race between I/O and migration for hugepage. @@ -1467,20 +1480,20 @@ out: * because then pte is replaced with migration swap entry and direct I/O code * will wait in the page fault for migration to complete. */ -static int unmap_and_move_huge_page(new_folio_t get_new_folio, +static int unmap_and_move_hugetlb_folio(new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, struct folio *src, int force, enum migrate_mode mode, - int reason, struct list_head *ret) + enum migrate_reason reason, struct list_head *ret) { struct folio *dst; int rc = -EAGAIN; - int page_was_mapped = 0; + int was_mapped = 0; struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; enum ttu_flags ttu = 0; if (folio_ref_count(src) == 1) { - /* page was freed from under us. So we are done. */ + /* folio was freed from under us. So we are done. */ folio_putback_hugetlb(src); return 0; } @@ -1502,8 +1515,8 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } /* - * Check for pages which are in the process of being freed. Without - * folio_mapping() set, hugetlbfs specific move page routine will not + * Check for folios which are in the process of being freed. Without + * folio_mapping() set, hugetlbfs specific move folio routine will not * be called and we could leak usage counts for subpools. */ if (hugetlb_folio_subpool(src) && !folio_mapping(src)) { @@ -1533,13 +1546,13 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } try_to_migrate(src, ttu); - page_was_mapped = 1; + was_mapped = 1; } if (!folio_mapped(src)) rc = move_to_new_folio(dst, src, mode); - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, !rc ? dst : src, ttu); if (ttu & TTU_RMAP_LOCKED) @@ -1627,7 +1640,7 @@ struct migrate_pages_stats { */ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct migrate_pages_stats *stats, struct list_head *ret_folios) { @@ -1664,10 +1677,10 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, continue; } - rc = unmap_and_move_huge_page(get_new_folio, - put_new_folio, private, - folio, pass > 2, mode, - reason, ret_folios); + rc = unmap_and_move_hugetlb_folio(get_new_folio, + put_new_folio, private, + folio, pass > 2, mode, + reason, ret_folios); /* * The rules are: * 0: hugetlb folio will be put back @@ -1717,7 +1730,7 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, static void migrate_folios_move(struct list_head *src_folios, struct list_head *dst_folios, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct migrate_pages_stats *stats, int *retry, int *thp_retry, int *nr_failed, @@ -1742,7 +1755,7 @@ static void migrate_folios_move(struct list_head *src_folios, /* * The rules are: * 0: folio will be freed - * -EAGAIN: stay on the unmap_folios list + * -EAGAIN: stay on the src_folios list * Other errno: put on ret_folios list */ switch (rc) { @@ -1800,7 +1813,7 @@ static void migrate_folios_undo(struct list_head *src_folios, */ static int migrate_pages_batch(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, - unsigned long private, enum migrate_mode mode, int reason, + unsigned long private, enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats, int nr_pass) { @@ -2012,7 +2025,7 @@ out: static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats) { @@ -2089,7 +2102,7 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, */ int migrate_pages(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, unsigned int *ret_succeeded) { int rc, rc_gather; int nr_pages; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 908d2d4ec43a..18d097c38853 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -77,6 +77,9 @@ static int migrate_vma_collect_hole(unsigned long start, * @folio: the folio to split * @fault_page: struct page associated with the fault if any * + * If @folio is not the folio containing @fault_page, the caller must hold a + * reference on @folio. The helper consumes that reference. + * * Returns 0 on success */ static int migrate_vma_split_folio(struct folio *folio, @@ -86,10 +89,8 @@ static int migrate_vma_split_folio(struct folio *folio, struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; struct folio *new_fault_folio = NULL; - if (folio != fault_folio) { - folio_get(folio); + if (folio != fault_folio) folio_lock(folio); - } ret = split_folio(folio); if (ret) { @@ -166,11 +167,14 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); - folio = softleaf_to_folio(entry); - if (!softleaf_is_device_private(entry) || - !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || - (folio->pgmap->owner != migrate->pgmap_owner)) { + !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { + spin_unlock(ptl); + return migrate_vma_collect_skip(start, end, walk); + } + + folio = softleaf_to_folio(entry); + if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } @@ -307,6 +311,9 @@ again: if (folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -350,6 +357,9 @@ again: if (folio && folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -446,13 +456,13 @@ again: if (pte_present(pte)) { if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, addr, ptep, swp_pte); @@ -514,7 +524,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate) migrate->pgmap_owner); mmu_notifier_invalidate_range_start(&range); - walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end, + walk_page_range_vma(migrate->vma, migrate->start, migrate->end, &migrate_vma_walk_ops, migrate); mmu_notifier_invalidate_range_end(&range); @@ -769,7 +779,7 @@ int migrate_vma_setup(struct migrate_vma *args) } EXPORT_SYMBOL(migrate_vma_setup); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm * at @addr. folio is already allocated as a part of the migration process with @@ -836,7 +846,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, else swp_entry = make_readable_device_private_entry( page_to_pfn(page)); - entry = swp_entry_to_pmd(swp_entry); + entry = softleaf_to_pmd(swp_entry); } else { if (folio_is_zone_device(folio) && !folio_is_device_coherent(folio)) { @@ -924,7 +934,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; return ret; } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, unsigned long addr, struct page *page, @@ -945,7 +955,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, static unsigned long migrate_vma_nr_pages(unsigned long *src) { unsigned long nr = 1; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES if (*src & MIGRATE_PFN_COMPOUND) nr = HPAGE_PMD_NR; #else diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf..ff4ac8281768 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -12,6 +12,7 @@ #include <linux/gfp.h> #include <linux/pagewalk.h> #include <linux/mman.h> +#include <linux/slab.h> #include <linux/syscalls.h> #include <linux/swap.h> #include <linux/leafops.h> @@ -27,30 +28,16 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) { #ifdef CONFIG_HUGETLB_PAGE - unsigned char present; - unsigned char *vec = walk->private; + const unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned char resident; spinlock_t *ptl; + pte_t ptep; ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); - - /* - * Hugepages under user process are always in RAM and never - * swapped out, but theoretically it needs to be checked. - */ - if (!pte) { - present = 0; - } else { - const pte_t ptep = huge_ptep_get(walk->mm, addr, pte); - - if (huge_pte_none(ptep) || pte_is_marker(ptep)) - present = 0; - else - present = 1; - } - - for (; addr != end; vec++, addr += PAGE_SIZE) - *vec = present; - walk->private = vec; + ptep = huge_ptep_get(walk->mm, addr, pte); + resident = !huge_pte_none(ptep) && !pte_is_marker(ptep); + memset(walk->private, resident, nr); + walk->private += nr; spin_unlock(ptl); #else BUG(); @@ -90,8 +77,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) folio = swap_cache_get_folio(entry); if (shmem) put_swap_device(si); - /* The swap cache space contains either folio, shadow or NULL */ - if (folio && !xa_is_value(folio)) { + if (folio) { present = folio_test_uptodate(folio); folio_put(folio); } @@ -107,7 +93,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) */ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) { - unsigned char present = 0; + unsigned char present; struct folio *folio; /* @@ -117,17 +103,16 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) * tmpfs's .fault). So swapped out tmpfs mappings are tested here. */ folio = filemap_get_entry(mapping, index); - if (folio) { - if (xa_is_value(folio)) { - if (shmem_mapping(mapping)) - return mincore_swap(radix_to_swp_entry(folio), - true); - else - return 0; - } - present = folio_test_uptodate(folio); - folio_put(folio); + if (!folio) + return 0; + + if (xa_is_value(folio)) { + if (!shmem_mapping(mapping)) + return 0; + return mincore_swap(radix_to_swp_entry(folio), true); } + present = folio_test_uptodate(folio); + folio_put(folio); return present; } @@ -160,6 +145,20 @@ static int mincore_unmapped_range(unsigned long addr, unsigned long end, return 0; } +static int mincore_pud_entry(pud_t *pudp, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + if (pud_is_huge(pudp_get(pudp))) { + const unsigned long nr = (end - addr) >> PAGE_SHIFT; + + memset(walk->private, 1, nr); + walk->private += nr; + walk->action = ACTION_CONTINUE; + } + + return 0; +} + static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -232,6 +231,7 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) } static const struct mm_walk_ops mincore_walk_ops = { + .pud_entry = mincore_pud_entry, .pmd_entry = mincore_pte_range, .pte_hole = mincore_unmapped_range, .hugetlb_entry = mincore_hugetlb, @@ -258,7 +258,8 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v memset(vec, 1, pages); return pages; } - err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); + + err = walk_page_range_vma(vma, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; return (end - addr) >> PAGE_SHIFT; @@ -312,7 +313,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, if (!access_ok(vec, pages)) return -EFAULT; - tmp = (void *) __get_free_page(GFP_USER); + tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!tmp) return -EAGAIN; @@ -337,6 +338,6 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start += retval << PAGE_SHIFT; retval = 0; } - free_page((unsigned long) tmp); + kfree(tmp); return retval; } diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df..efa6716e4dfb 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -329,7 +329,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, * be split. And the pages are not in VM_LOCKed VMA * can be reclaimed. */ - if (!(vma->vm_flags & VM_LOCKED)) + if (!vma_test(vma, VMA_LOCKED_BIT)) return true; /* folio_within_range() cannot take KSM, but any small folio is OK */ @@ -368,7 +368,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, folio = pmd_folio(*pmd); if (folio_is_zone_device(folio)) goto out; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mlock_folio(folio); else munlock_folio(folio); @@ -393,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mlock_folio(folio); else munlock_folio(folio); @@ -417,8 +417,8 @@ out: * @end - end of range in @vma * @new_vma_flags - the new set of flags for @vma. * - * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED; - * called for munlock() and munlockall(), to clear VM_LOCKED from @vma. + * Called for mlock(), mlock2() and mlockall(), to set @vma VMA_LOCKED_BIT; + * called for munlock() and munlockall(), to clear VMA_LOCKED_BIT from @vma. */ static void mlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, @@ -431,14 +431,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, /* * There is a slight chance that concurrent page migration, - * or page reclaim finding a page of this now-VM_LOCKED vma, + * or page reclaim finding a page of this now-VMA_LOCKED_BIT vma, * will call mlock_vma_folio() and raise page's mlock_count: * double counting, leaving the page unevictable indefinitely. - * Communicate this danger to mlock_vma_folio() with VM_IO, - * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas. + * Communicate this danger to mlock_vma_folio() with VMA_IO_BIT, + * which is a VMA_SPECIAL_FLAGS flag not allowed on VMA_LOCKED_BIT vmas. * mmap_lock is held in write mode here, so this weird * combination should not be visible to other mmap_lock users; - * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED. + * but WRITE_ONCE so rmap walkers must see VMA_IO_BIT if VMA_LOCKED_BIT. */ if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT)) vma_flags_set(new_vma_flags, VMA_IO_BIT); @@ -446,7 +446,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, vma_flags_reset_once(vma, new_vma_flags); lru_add_drain(); - walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL); + walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL); lru_add_drain(); if (vma_flags_test(new_vma_flags, VMA_IO_BIT)) { @@ -458,7 +458,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, /* * mlock_fixup - handle mlock[all]/munlock[all] requests. * - * Filters out "special" vmas -- VM_LOCKED never gets set for these, and + * Filters out "special" vmas -- VMA_LOCKED_BIT never gets set for these, and * munlock is a no-op. However, for some special vmas, we go ahead and * populate the ptes. * @@ -466,24 +466,23 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, */ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, - unsigned long end, vm_flags_t newflags) + unsigned long end, vma_flags_t *new_vma_flags) { - vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; int nr_pages; int ret = 0; - if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || + if (vma_flags_same_pair(&old_vma_flags, new_vma_flags) || vma_is_secretmem(vma) || !vma_supports_mlock(vma)) { /* - * Don't set VM_LOCKED or VM_LOCKONFAULT and don't count. - * For secretmem, don't allow the memory to be unlocked. + * Don't set VMA_LOCKED_BIT or VMA_LOCKONFAULT_BIT and don't + * count. For secretmem, don't allow the memory to be unlocked. */ goto out; } - vma = vma_modify_flags(vmi, *prev, vma, start, end, &new_vma_flags); + vma = vma_modify_flags(vmi, *prev, vma, start, end, new_vma_flags); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto out; @@ -493,7 +492,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, * Keep track of amount of locked VM. */ nr_pages = (end - start) >> PAGE_SHIFT; - if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) + if (!vma_flags_test(new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) nr_pages = 0; @@ -502,15 +501,15 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * vm_flags is protected by the mmap_lock held in write mode. * It's okay if try_to_unmap_one unmaps a page just after we - * set VM_LOCKED, populate_vma_page_range will bring it back. + * set VMA_LOCKED_BIT, populate_vma_page_range will bring it back. */ - if (vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT) && + if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT) && vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) { /* No work to do, and mlocking twice would be wrong */ vma_start_write(vma); - vma->flags = new_vma_flags; + vma->flags = *new_vma_flags; } else { - mlock_vma_pages_range(vma, start, end, &new_vma_flags); + mlock_vma_pages_range(vma, start, end, new_vma_flags); } out: *prev = vma; @@ -518,7 +517,7 @@ out: } static int apply_vma_lock_flags(unsigned long start, size_t len, - vm_flags_t flags) + const vma_flags_t *flags) { unsigned long nstart, end, tmp; struct vm_area_struct *vma, *prev; @@ -543,18 +542,20 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, tmp = vma->vm_start; for_each_vma_range(vmi, vma, end) { int error; - vm_flags_t newflags; + vma_flags_t newflags; if (vma->vm_start != tmp) return -ENOMEM; - newflags = vma->vm_flags & ~VM_LOCKED_MASK; - newflags |= flags; + newflags = vma->flags; + vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK); + vma_flags_set_mask(&newflags, *flags); + /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ tmp = vma->vm_end; if (tmp > end) tmp = end; - error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags); + error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, &newflags); if (error) return error; tmp = vma_iter_end(&vmi); @@ -589,7 +590,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, end = start + len; for_each_vma_range(vmi, vma, end) { - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { if (start > vma->vm_start) count -= (start - vma->vm_start); if (end < vma->vm_end) { @@ -615,7 +616,8 @@ static int __mlock_posix_error_return(long retval) return retval; } -static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags) +static __must_check int do_mlock(unsigned long start, size_t len, + vma_flags_t *flags) { unsigned long locked; unsigned long lock_limit; @@ -664,24 +666,27 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len) { - return do_mlock(start, len, VM_LOCKED); + vma_flags_t flags = mk_vma_flags(VMA_LOCKED_BIT); + + return do_mlock(start, len, &flags); } SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags) { - vm_flags_t vm_flags = VM_LOCKED; + vma_flags_t vma_flags = mk_vma_flags(VMA_LOCKED_BIT); if (flags & ~MLOCK_ONFAULT) return -EINVAL; if (flags & MLOCK_ONFAULT) - vm_flags |= VM_LOCKONFAULT; + vma_flags_set(&vma_flags, VMA_LOCKONFAULT_BIT); - return do_mlock(start, len, vm_flags); + return do_mlock(start, len, &vma_flags); } SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) { + vma_flags_t flags = EMPTY_VMA_FLAGS; int ret; start = untagged_addr(start); @@ -691,7 +696,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) if (mmap_write_lock_killable(current->mm)) return -EINTR; - ret = apply_vma_lock_flags(start, len, 0); + ret = apply_vma_lock_flags(start, len, &flags); mmap_write_unlock(current->mm); return ret; @@ -699,46 +704,48 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) /* * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall) - * and translate into the appropriate modifications to mm->def_flags and/or the - * flags for all current VMAs. + * and translate into the appropriate modifications to mm->def_vma_flags and/or + * the flags for all current VMAs. * * There are a couple of subtleties with this. If mlockall() is called multiple * times with different flags, the values do not necessarily stack. If mlockall * is called once including the MCL_FUTURE flag and then a second time without - * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags. + * it, VMA_LOCKED_BIT and VMA_LOCKONFAULT_BIT will be cleared from + * mm->def_vma_flags. */ static int apply_mlockall_flags(int flags) { VMA_ITERATOR(vmi, current->mm, 0); + struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev = NULL; - vm_flags_t to_add = 0; + vma_flags_t to_add = EMPTY_VMA_FLAGS; - current->mm->def_flags &= ~VM_LOCKED_MASK; + vma_flags_clear_mask(&mm->def_vma_flags, VMA_LOCKED_MASK); if (flags & MCL_FUTURE) { - current->mm->def_flags |= VM_LOCKED; + vma_flags_set(&mm->def_vma_flags, VMA_LOCKED_BIT); if (flags & MCL_ONFAULT) - current->mm->def_flags |= VM_LOCKONFAULT; + vma_flags_set(&mm->def_vma_flags, VMA_LOCKONFAULT_BIT); if (!(flags & MCL_CURRENT)) goto out; } if (flags & MCL_CURRENT) { - to_add |= VM_LOCKED; + vma_flags_set(&to_add, VMA_LOCKED_BIT); if (flags & MCL_ONFAULT) - to_add |= VM_LOCKONFAULT; + vma_flags_set(&to_add, VMA_LOCKONFAULT_BIT); } for_each_vma(vmi, vma) { int error; - vm_flags_t newflags; + vma_flags_t newflags = vma->flags; - newflags = vma->vm_flags & ~VM_LOCKED_MASK; - newflags |= to_add; + vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK); + vma_flags_set_mask(&newflags, to_add); error = mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end, - newflags); + &newflags); /* Ignore errors, but prev needs fixing up. */ if (error) prev = vma; diff --git a/mm/mm_init.c b/mm/mm_init.c index fbd62670650c..73992a01c452 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -33,8 +33,12 @@ #include <linux/kexec_handover.h> #include <linux/hugetlb.h> #include "internal.h" +#include "mm_init.h" +#include "page_alloc.h" +#include "sparse.h" #include "slab.h" #include "shuffle.h" +#include "vmalloc.h" #include <asm/setup.h> @@ -688,31 +692,6 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn, } #endif -/* - * Initialize a reserved page unconditionally, finding its zone first. - */ -void __meminit __init_page_from_nid(unsigned long pfn, int nid) -{ - pg_data_t *pgdat; - int zid; - - pgdat = NODE_DATA(nid); - - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - - if (pageblock_aligned(pfn)) { - enum migratetype mt = - kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); - init_pageblock_migratetype(pfn_to_page(pfn), mt, false); - } -} - #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) { @@ -771,10 +750,25 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) static void __meminit __init_deferred_page(unsigned long pfn, int nid) { + pg_data_t *pgdat = NODE_DATA(nid); + int zid; + if (early_page_initialised(pfn, nid)) return; - __init_page_from_nid(pfn, nid); + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + struct zone *zone = &pgdat->node_zones[zid]; + + if (zone_spans_pfn(zone, pfn)) + break; + } + __init_single_page(pfn_to_page(pfn), pfn, zid, nid); + + if (pageblock_aligned(pfn)) { + enum migratetype mt = + kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); + init_pageblock_migratetype(pfn_to_page(pfn), mt, false); + } } #else static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} @@ -1495,7 +1489,7 @@ static inline void setup_usemap(struct zone *zone) {} #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -void __init set_pageblock_order(void) +static void __init set_pageblock_order(void) { unsigned int order = PAGE_BLOCK_MAX_ORDER; @@ -1521,7 +1515,7 @@ void __init set_pageblock_order(void) * include/linux/pageblock-flags.h for the values of pageblock_order based on * the kernel config */ -void __init set_pageblock_order(void) +static inline void __init set_pageblock_order(void) { } @@ -1828,7 +1822,6 @@ static void __init free_area_init(void) bool descending; arch_zone_limits_init(max_zone_pfn); - sparse_init(); start_pfn = PHYS_PFN(memblock_start_of_DRAM()); descending = arch_has_descending_max_zone_pfns(); @@ -1878,18 +1871,12 @@ static void __init free_area_init(void) (u64)zone_movable_pfn[i] << PAGE_SHIFT); } - /* - * Print out the early node map, and initialize the - * subsection-map relative to active online memory ranges to - * enable future "sub-section" extensions of the memory map. - */ + /* Print out the early node map. */ pr_info("Early memory node ranges\n"); - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); - sparse_init_subsection_map(start_pfn, end_pfn - start_pfn); - } /* Initialise every node */ mminit_verify_pageflags_layout(); @@ -1923,11 +1910,7 @@ static void __init free_area_init(void) } } - for_each_node_state(nid, N_MEMORY) - sparse_vmemmap_init_nid_late(nid); - calc_nr_kernel_pages(); - memmap_init(); /* disable hash distribution for systems with a single node */ fixup_hashdist(); @@ -2337,6 +2320,7 @@ void __init page_alloc_init_late(void) /* Reinit limits that are based on free pages after the kernel is up */ files_maxfiles_init(); #endif + hugetlb_bootmem_struct_page_init(); /* Accounting of total+free memory is stable at this point. */ mem_init_print_info(); @@ -2701,10 +2685,13 @@ void __init __weak mem_init(void) void __init mm_core_init_early(void) { + free_area_init(); + hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); - free_area_init(); + sparse_init(); + memmap_init(); } /* diff --git a/mm/mm_init.h b/mm/mm_init.h new file mode 100644 index 000000000000..39f75df9be1c --- /dev/null +++ b/mm/mm_init.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm_init.h: + * + * mm/ internal mm_init and memblock declarations + */ + +#ifndef __MM_MM_INIT_H +#define __MM_MM_INIT_H + +#include <linux/types.h> +#include <linux/init.h> +#include <linux/mmzone.h> +#include <linux/jump_label.h> +#include <linux/printk.h> + +struct page; +struct vmem_altmap; + +/* perform sanity checks on struct pages being allocated or freed */ +DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); + +void set_zone_contiguous(struct zone *zone); +bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, + unsigned long nr_pages); + +static inline void clear_zone_contiguous(struct zone *zone) +{ + zone->contiguous = false; +} + +void memblock_free_pages(unsigned long pfn, unsigned int order); + +void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, + int nid, bool exact_nid); + +void memmap_init_range(unsigned long size, int nid, unsigned long zone, + unsigned long start_pfn, unsigned long zone_end_pfn, + enum meminit_context context, + struct vmem_altmap *altmap, int migratetype, + bool isolate_pageblock); + +#if defined CONFIG_COMPACTION || defined CONFIG_CMA +/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ +void init_cma_reserved_pageblock(struct page *page); +#endif + +#ifdef CONFIG_CMA +void init_cma_pageblock(struct page *page); +#else +static inline void init_cma_pageblock(struct page *page) +{ +} +#endif + +/* Memory initialisation debug and verification */ +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +DECLARE_STATIC_KEY_TRUE(deferred_pages); + +static inline bool deferred_pages_enabled(void) +{ + return static_branch_unlikely(&deferred_pages); +} + +bool __init deferred_grow_zone(struct zone *zone, unsigned int order); +#else +static inline bool deferred_pages_enabled(void) +{ + return false; +} +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + +void init_deferred_page(unsigned long pfn, int nid); + +enum mminit_level { + MMINIT_WARNING, + MMINIT_VERIFY, + MMINIT_TRACE +}; + +#ifdef CONFIG_DEBUG_MEMORY_INIT + +extern int mminit_loglevel; + +#define mminit_dprintk(level, prefix, fmt, arg...) \ +do { \ + if (level < mminit_loglevel) { \ + if (level <= MMINIT_WARNING) \ + pr_warn("mminit::" prefix " " fmt, ##arg); \ + else \ + printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ + } \ +} while (0) + +void mminit_verify_pageflags_layout(void); +void mminit_verify_zonelist(void); +#else + +static inline void mminit_dprintk(enum mminit_level level, + const char *prefix, const char *fmt, ...) +{ +} + +static inline void mminit_verify_pageflags_layout(void) +{ +} + +static inline void mminit_verify_zonelist(void) +{ +} +#endif /* CONFIG_DEBUG_MEMORY_INIT */ + +extern bool mirrored_kernelcore; +bool memblock_has_mirror(void); +void memblock_free_all(void); + +void __meminit __init_single_page(struct page *page, unsigned long pfn, + unsigned long zone, int nid); + +#endif /* __MM_MM_INIT_H */ diff --git a/mm/mm_slot.h b/mm/mm_slot.h index 83f18ed1c4bd..9b09b68e5742 100644 --- a/mm/mm_slot.h +++ b/mm/mm_slot.h @@ -33,6 +33,12 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp) kmem_cache_free(cache, objp); } +/* + * Note: mm_slot_lookup and mm_slot_insert cannot be converted to static inline + * functions because the hash helpers (hash_for_each_possible and hash_add) rely + * on the actual array argument 'hashtable' for sizeof() instead of pointers. + */ + #define mm_slot_lookup(_hashtable, _mm) \ ({ \ struct mm_slot *tmp_slot, *mm_slot = NULL; \ @@ -52,4 +58,9 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp) hash_add(_hashtable, &_mm_slot->hash, (unsigned long)_mm); \ }) +static inline void mm_slot_remove(struct mm_slot *slot) +{ + hash_del(&slot->hash); + list_del(&slot->mm_node); +} #endif /* _LINUX_MM_SLOT_H */ diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff4..4bf26b0f1e6e 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -80,13 +80,13 @@ core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); /* Update vma->vm_page_prot to reflect vma->vm_flags. */ void vma_set_page_prot(struct vm_area_struct *vma) { - vm_flags_t vm_flags = vma->vm_flags; + vma_flags_t vma_flags = vma->flags; pgprot_t vm_page_prot; - vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); + vm_page_prot = vma_pgprot_modify(vma->vm_page_prot, vma_flags); if (vma_wants_writenotify(vma, vm_page_prot)) { - vm_flags &= ~VM_SHARED; - vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); + vma_flags_clear(&vma_flags, VMA_SHARED_BIT); + vm_page_prot = vma_pgprot_modify(vm_page_prot, vma_flags); } /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ WRITE_ONCE(vma->vm_page_prot, vm_page_prot); @@ -102,15 +102,16 @@ void vma_set_page_prot(struct vm_area_struct *vma) */ static int check_brk_limits(unsigned long addr, unsigned long len) { + const struct mm_struct *mm = current->mm; + const bool is_def_locked = + vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT); unsigned long mapped_addr; mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); if (IS_ERR_VALUE(mapped_addr)) return mapped_addr; - return mlock_future_ok(current->mm, - current->mm->def_flags & VM_LOCKED, len) - ? 0 : -EAGAIN; + return mlock_future_ok(mm, is_def_locked, len) ? 0 : -EAGAIN; } SYSCALL_DEFINE1(brk, unsigned long, brk) @@ -197,7 +198,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) goto out; mm->brk = brk; - if (mm->def_flags & VM_LOCKED) + if (vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT)) populate = true; success: @@ -280,7 +281,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * do_mmap() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to - * apply @vm_flags. If this is a file-backed mapping then the file is specified + * apply @vma_flags. If this is a file-backed mapping then the file is specified * in @file and page offset into the file via @pgoff. * * This function does not perform security checks on the file and assumes, if @@ -320,7 +321,8 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * (2) for details. * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. - * @vm_flags: VMA flags which should be set by default, or 0 otherwise. + * @vma_flags: VMA flags which should be set by default, or EMPTY_VMA_FLAGS + * otherwise. * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be @@ -335,7 +337,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, */ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flags, vm_flags_t vm_flags, + unsigned long flags, vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { @@ -399,13 +401,18 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * to. we assume access permissions have been handled by the open * of the memory object, so we don't do any here. */ - vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) | - mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_prot_bits(prot, pkey))); + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_flag_bits(file, flags))); + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, + VMA_MAYEXEC_BIT); /* Obtain the address to map to. we verify (or select) it and ensure * that it represents a valid section of the address space. */ - addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags); + addr = __get_unmapped_area(file, addr, len, pgoff, flags, vma_flags); if (IS_ERR_VALUE(addr)) return addr; @@ -418,7 +425,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (!can_do_mlock()) return -EPERM; - if (!mlock_future_ok(mm, vm_flags & VM_LOCKED, len)) + if (!mlock_future_ok(mm, vma_flags_test(&vma_flags, VMA_LOCKED_BIT), len)) return -EAGAIN; if (file) { @@ -461,22 +468,23 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) return -EACCES; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); if (!(file->f_mode & FMODE_WRITE)) - vm_flags &= ~(VM_MAYWRITE | VM_SHARED); + vma_flags_clear(&vma_flags, VMA_MAYWRITE_BIT, + VMA_SHARED_BIT); fallthrough; case MAP_PRIVATE: if (!(file->f_mode & FMODE_READ)) return -EACCES; if (path_noexec(&file->f_path)) { - if (vm_flags & VM_EXEC) + if (vma_flags_test(&vma_flags, VMA_EXEC_BIT)) return -EPERM; - vm_flags &= ~VM_MAYEXEC; + vma_flags_clear(&vma_flags, VMA_MAYEXEC_BIT); } if (!can_mmap_file(file)) return -ENODEV; - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; break; @@ -488,23 +496,27 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Check to see if we are violating any seals and update VMA * flags if necessary to avoid future seal violations. */ - err = memfd_check_seals_mmap(file, &vm_flags); + err = memfd_check_seals_mmap(file, &vma_flags); if (err) return (unsigned long)err; } else { switch (flags & MAP_TYPE) { case MAP_SHARED: - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; /* * Ignore pgoff. */ pgoff = 0; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); break; - case MAP_DROPPABLE: - if (VM_DROPPABLE == VM_NONE) + case MAP_DROPPABLE: { + vma_flags_t droppable = VMA_DROPPABLE; + + if (vma_flags_empty(&droppable)) return -EOPNOTSUPP; + vma_flags_set_mask(&vma_flags, droppable); + /* * A locked or stack area makes no sense to be droppable. * @@ -515,23 +527,24 @@ unsigned long do_mmap(struct file *file, unsigned long addr, */ if (flags & (MAP_LOCKED | MAP_HUGETLB)) return -EINVAL; - if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; - vm_flags |= VM_DROPPABLE; - /* * If the pages can be dropped, then it doesn't make * sense to reserve them. */ - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* * Likewise, they're volatile enough that they * shouldn't survive forks or coredumps. */ - vm_flags |= VM_WIPEONFORK | VM_DONTDUMP; + vma_flags_set(&vma_flags, VMA_WIPEONFORK_BIT, + VMA_DONTDUMP_BIT); + fallthrough; + } case MAP_PRIVATE: /* * Set pgoff according to addr for anon_vma. @@ -544,22 +557,22 @@ unsigned long do_mmap(struct file *file, unsigned long addr, } /* - * Set 'VM_NORESERVE' if we should not account for the - * memory use of this mapping. + * Set VMA_NORESERVE_BIT if we should not account for the memory use + * of this mapping. */ if (flags & MAP_NORESERVE) { /* We honor MAP_NORESERVE if allowed to overcommit */ if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* hugetlb applies strict overcommit unless MAP_NORESERVE */ if (file && is_file_hugepages(file)) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vma_flags, pgoff, uf); if (!IS_ERR_VALUE(addr) && - ((vm_flags & VM_LOCKED) || + (vma_flags_test(&vma_flags, VMA_LOCKED_BIT) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) *populate = len; return addr; @@ -645,9 +658,9 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) * Determine if the allocation needs to ensure that there is no * existing mapping within it's guard gaps, for use as start_gap. */ -static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) +static inline unsigned long stack_guard_placement(vma_flags_t vma_flags) { - if (vm_flags & VM_SHADOW_STACK) + if (vma_flags_test_single_mask(&vma_flags, VMA_SHADOW_STACK)) return PAGE_SIZE; return 0; @@ -689,7 +702,7 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev; @@ -714,7 +727,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = mmap_end; - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); return vm_unmapped_area(&info); @@ -727,7 +740,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif @@ -738,7 +751,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct vm_area_struct *vma, *prev; struct mm_struct *mm = current->mm; @@ -766,7 +779,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); addr = vm_unmapped_area(&info); @@ -795,23 +808,24 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags) { if (mm_flags_test(MMF_TOPDOWN, current->mm)) return arch_get_unmapped_area_topdown(filp, addr, len, pgoff, - flags, vm_flags); - return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); + flags, vma_flags_to_legacy(vma_flags)); + return arch_get_unmapped_area(filp, addr, len, pgoff, flags, + vma_flags_to_legacy(vma_flags)); } unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) + unsigned long pgoff, unsigned long flags, vma_flags_t vma_flags) { unsigned long (*get_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long) @@ -846,11 +860,11 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, && !addr /* no hint */ && IS_ALIGNED(len, PMD_SIZE)) { /* Ensures that larger anonymous mappings are THP aligned. */ - addr = thp_get_unmapped_area_vmflags(file, addr, len, - pgoff, flags, vm_flags); + addr = thp_get_unmapped_area_vmaflags(file, addr, len, + pgoff, flags, vma_flags); } else { - addr = mm_get_unmapped_area_vmflags(file, addr, len, - pgoff, flags, vm_flags); + addr = mm_get_unmapped_area_vmaflags(file, addr, len, + pgoff, flags, vma_flags); } if (IS_ERR_VALUE(addr)) return addr; @@ -868,7 +882,8 @@ unsigned long mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return mm_get_unmapped_area_vmflags(file, addr, len, pgoff, flags, 0); + return mm_get_unmapped_area_vmaflags(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL(mm_get_unmapped_area); @@ -970,7 +985,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon return NULL; if (expand_stack_locked(prev, addr)) return NULL; - if (prev->vm_flags & VM_LOCKED) + if (vma_test(prev, VMA_LOCKED_BIT)) populate_vma_page_range(prev, addr, prev->vm_end, NULL); return prev; } @@ -994,7 +1009,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon start = vma->vm_start; if (expand_stack_locked(vma, addr)) return NULL; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) populate_vma_page_range(vma, addr, start, NULL); return vma; } @@ -1119,18 +1134,18 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, */ vma = vma_lookup(mm, start); - if (!vma || !(vma->vm_flags & VM_SHARED)) { + if (!vma || !vma_test(vma, VMA_SHARED_BIT)) { mmap_read_unlock(mm); return -EINVAL; } - prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; - prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; - prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; + prot |= vma_test(vma, VMA_READ_BIT) ? PROT_READ : 0; + prot |= vma_test(vma, VMA_WRITE_BIT) ? PROT_WRITE : 0; + prot |= vma_test(vma, VMA_EXEC_BIT) ? PROT_EXEC : 0; flags &= MAP_NONBLOCK; flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) flags |= MAP_LOCKED; /* Save vm_flags used to calculate prot and flags, and recheck later. */ @@ -1191,7 +1206,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } ret = do_mmap(vma->vm_file, start, size, - prot, flags, 0, pgoff, &populate, NULL); + prot, flags, EMPTY_VMA_FLAGS, pgoff, &populate, NULL); out: mmap_write_unlock(mm); fput(file); @@ -1233,7 +1248,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) vma = vma_prev(&vmi); ret = do_brk_flags(&vmi, vma, addr, len, vma_flags); - populate = ((mm->def_flags & VM_LOCKED) != 0); + populate = vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate && !ret) @@ -1256,7 +1271,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, mmap_assert_write_locked(mm); vma_iter_set(vmi, vma->vm_end); do { - if (vma->vm_flags & VM_ACCOUNT) + if (vma_test(vma, VMA_ACCOUNT_BIT)) nr_accounted += vma_pages(vma); vma_mark_detached(vma); remove_vma(vma); @@ -1405,7 +1420,7 @@ static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) { /* * Forbid splitting special mappings - kernel has expectations over - * the number of pages in mapping. Together with VM_DONTEXPAND + * the number of pages in mapping. Together with VMA_DONTEXPAND_BIT * the size of vma should stay the same over the special mapping's * lifetime. */ @@ -1447,44 +1462,6 @@ static vm_fault_t special_mapping_fault(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -static struct vm_area_struct *__install_special_mapping( - struct mm_struct *mm, - unsigned long addr, unsigned long len, - vm_flags_t vm_flags, void *priv, - const struct vm_operations_struct *ops) -{ - int ret; - struct vm_area_struct *vma; - - vma = vm_area_alloc(mm); - if (unlikely(vma == NULL)) - return ERR_PTR(-ENOMEM); - - vma_set_range(vma, addr, addr + len, 0); - vm_flags |= mm->def_flags | VM_DONTEXPAND; - if (pgtable_supports_soft_dirty()) - vm_flags |= VM_SOFTDIRTY; - vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); - - vma->vm_ops = ops; - vma->vm_private_data = priv; - - ret = insert_vm_struct(mm, vma); - if (ret) - goto out; - - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); - - perf_event_mmap(vma); - - return vma; - -out: - vm_area_free(vma); - return ERR_PTR(ret); -} - bool vma_is_special_mapping(const struct vm_area_struct *vma, const struct vm_special_mapping *sm) { @@ -1715,7 +1692,7 @@ bool mmap_read_lock_maybe_expand(struct mm_struct *mm, return true; } - if (!(new_vma->vm_flags & VM_GROWSDOWN)) + if (!vma_test(new_vma, VMA_GROWSDOWN_BIT)) return false; mmap_write_lock(mm); @@ -1765,7 +1742,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) retval = vma_start_write_killable(mpnt); if (retval < 0) goto loop_out; - if (mpnt->vm_flags & VM_DONTCOPY) { + if (vma_test(mpnt, VMA_DONTCOPY_BIT)) { retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, mpnt->vm_end, GFP_KERNEL); if (retval) @@ -1775,7 +1752,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) continue; } charge = 0; - if (mpnt->vm_flags & VM_ACCOUNT) { + if (vma_test(mpnt, VMA_ACCOUNT_BIT)) { unsigned long len = vma_pages(mpnt); if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ @@ -1793,16 +1770,19 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) retval = dup_userfaultfd(tmp, &uf); if (retval) goto fail_nomem_anon_vma_fork; - if (tmp->vm_flags & VM_WIPEONFORK) { + + if (vma_test(tmp, VMA_WIPEONFORK_BIT)) { /* - * VM_WIPEONFORK gets a clean slate in the child. + * VMA_WIPEONFORK_BIT gets a clean slate in the child. * Don't prepare anon_vma until fault since we don't * copy page for current vma. */ tmp->anon_vma = NULL; } else if (anon_vma_fork(tmp, mpnt)) goto fail_nomem_anon_vma_fork; - vm_flags_clear(tmp, VM_LOCKED_MASK); + + vma_start_write(tmp); + vma_clear_flags_mask(tmp, VMA_LOCKED_MASK); /* * Copy/update hugetlb private vma information. */ @@ -1830,13 +1810,12 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, - &mapping->i_mmap); + mapping_rmap_tree_insert_after(tmp, mpnt, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } - if (!(tmp->vm_flags & VM_WIPEONFORK)) + if (!vma_test(tmp, VMA_WIPEONFORK_BIT)) retval = copy_page_range(tmp, mpnt); if (retval) { diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 245b74f39f91..df69ba6e797f 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -19,7 +19,7 @@ #include <linux/sched/mm.h> #include <linux/slab.h> -#include "vma.h" +#include "internal.h" /* global SRCU for all MMs */ DEFINE_STATIC_SRCU(srcu); diff --git a/mm/mmzone.c b/mm/mmzone.c index 0c8f181d9d50..59dc3f2076a6 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -43,7 +43,8 @@ struct zone *next_zone(struct zone *zone) return zone; } -static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) +static inline int zref_in_nodemask(struct zoneref *zref, + const nodemask_t *nodes) { #ifdef CONFIG_NUMA return node_isset(zonelist_node_idx(zref), *nodes); @@ -55,7 +56,7 @@ static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) /* Returns the next zone at or below highest_zoneidx in a zonelist */ struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { /* * Find the next suitable zone to use for the allocation. diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028c..2888ee638d87 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -40,7 +40,7 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) { - if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) + if (WARN_ON_ONCE(!vma_test(vma, VMA_WRITE_BIT))) return false; /* Don't touch entries that are not even readable. */ @@ -97,7 +97,7 @@ static bool can_change_shared_pte_writable(struct vm_area_struct *vma, bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { - if (!(vma->vm_flags & VM_SHARED)) + if (!vma_test(vma, VMA_SHARED_BIT)) return can_change_private_pte_writable(vma, addr, pte); return can_change_shared_pte_writable(vma, pte); @@ -143,7 +143,7 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce * that the ptes point to consecutive pages of the same anon large folio. */ -static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len, +static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len, struct page *first_page, bool expected_anon_exclusive) { int idx; @@ -174,16 +174,16 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb) { bool expected_anon_exclusive; - int sub_batch_idx = 0; + int batch_idx = 0; int len; while (nr_ptes) { - expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); - len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, + expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx); + len = page_anon_exclusive_batch(batch_idx, nr_ptes, first_page, expected_anon_exclusive); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, - sub_batch_idx, expected_anon_exclusive, tlb); - sub_batch_idx += len; + batch_idx, expected_anon_exclusive, tlb); + batch_idx += len; nr_ptes -= len; } } @@ -194,7 +194,7 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru { bool set_write; - if (vma->vm_flags & VM_SHARED) { + if (vma_test(vma, VMA_SHARED_BIT)) { set_write = can_change_shared_pte_writable(vma, ptent); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes, /* idx = */ 0, set_write, tlb); @@ -214,8 +214,9 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru static long change_softleaf_pte(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags) { - const bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + const bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); softleaf_t entry = softleaf_from_pte(oldpte); pte_t newpte; @@ -240,8 +241,8 @@ static long change_softleaf_pte(struct vm_area_struct *vma, */ entry = make_readable_device_private_entry(swp_offset(entry)); newpte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(oldpte)) - newpte = pte_swp_mkuffd_wp(newpte); + if (pte_swp_uffd(oldpte)) + newpte = pte_swp_mkuffd(newpte); } else if (softleaf_is_marker(entry)) { /* * Ignore error swap entries unconditionally, @@ -256,7 +257,7 @@ static long change_softleaf_pte(struct vm_area_struct *vma, * to unprotect it, drop it; the next page * fault will trigger without uffd trapping. */ - if (uffd_wp_resolve) { + if (uffd_prot_resolve) { pte_clear(vma->vm_mm, addr, pte); return 1; } @@ -265,10 +266,10 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } - if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); - else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + if (uffd_prot) + newpte = pte_swp_mkuffd(newpte); + else if (uffd_prot_resolve) + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(oldpte, newpte)) { set_pte_at(vma->vm_mm, addr, pte, newpte); @@ -282,17 +283,28 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb, int nr_ptes, unsigned long end, pgprot_t newprot, struct folio *folio, struct page *page, unsigned long cp_flags) { - const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; - const bool uffd_wp = cp_flags & MM_CP_UFFD_WP; + const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + const bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); pte_t ptent, oldpte; oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes); ptent = pte_modify(oldpte, newprot); - if (uffd_wp) - ptent = pte_mkuffd_wp(ptent); - else if (uffd_wp_resolve) - ptent = pte_clear_uffd_wp(ptent); + if (uffd_prot) + ptent = pte_mkuffd(ptent); + else if (uffd_prot_resolve) + ptent = pte_clear_uffd(ptent); + + /* + * The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE + * semantics. If mprotect() or NUMA hinting changed the + * base protection, restore PAGE_NONE so the PTE still + * traps on any access. pte_modify() preserves + * _PAGE_UFFD. + */ + if (userfaultfd_rwp(vma) && pte_uffd(ptent)) + ptent = pte_modify(ptent, PAGE_NONE); /* * In some writable, shared mappings, we might want @@ -325,6 +337,7 @@ static long change_pte_range(struct mmu_gather *tlb, long pages = 0; bool is_private_single_threaded; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; + bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; @@ -350,6 +363,14 @@ static long change_pte_range(struct mmu_gather *tlb, /* Already in the desired state. */ if (prot_numa && pte_protnone(oldpte)) continue; + /* + * RWP-protected PTEs carry _PAGE_UFFD as a marker on + * top of PROT_NONE. Skip only entries already in that + * exact state; plain PROT_NONE from mprotect() still needs + * to be promoted so future faults can be distinguished. + */ + if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte)) + continue; page = vm_normal_page(vma, addr, oldpte); if (page) @@ -358,6 +379,8 @@ static long change_pte_range(struct mmu_gather *tlb, /* * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. + * Skip this filter for uffd RWP which + * must set protnone regardless of NUMA placement. */ if (prot_numa && !folio_can_map_prot_numa(folio, vma, @@ -428,7 +451,7 @@ pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags) * (e.g. 2M shmem) because file thp is handled differently when * split by erasing the pmd so far. */ - return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma); + return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma); } /* @@ -667,7 +690,16 @@ long change_protection(struct mmu_gather *tlb, pgprot_t newprot = vma->vm_page_prot; long pages; - BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); + /* + * MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one + * change, and WP and RWP cannot mix. Miswired callers get a warn and + * a no-op; userspace cannot reach this state. + */ + if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL || + (cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL || + ((cp_flags & MM_CP_UFFD_WP_ALL) && + (cp_flags & MM_CP_UFFD_RWP_ALL)))) + return 0; #ifdef CONFIG_NUMA_BALANCING /* @@ -681,6 +713,10 @@ long change_protection(struct mmu_gather *tlb, WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA); #endif + if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) && + (cp_flags & MM_CP_UFFD_RWP)) + newprot = PAGE_NONE; + if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot, cp_flags); @@ -699,25 +735,24 @@ static int prot_none_pte_entry(pte_t *pte, unsigned long addr, 0 : -EACCES; } +#ifdef CONFIG_HUGETLB_PAGE static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long next, struct mm_walk *walk) { - return pfn_modify_allowed(pte_pfn(ptep_get(pte)), - *(pgprot_t *)(walk->private)) ? - 0 : -EACCES; -} + const pte_t entry = huge_ptep_get(walk->mm, addr, pte); -static int prot_none_test(unsigned long addr, unsigned long next, - struct mm_walk *walk) -{ - return 0; + if (pfn_modify_allowed(pte_pfn(entry), *(pgprot_t *)(walk->private))) + return 0; + return -EACCES; } +#else +#define prot_none_hugetlb_entry NULL +#endif static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, .hugetlb_entry = prot_none_hugetlb_entry, - .test_walk = prot_none_test, .walk_lock = PGWALK_WRLOCK, }; @@ -753,7 +788,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) { pgprot_t new_pgprot = vm_get_page_prot(newflags); - error = walk_page_range(current->mm, start, end, + error = walk_page_range_vma(vma, start, end, &prot_none_walk_ops, &new_pgprot); if (error) return error; @@ -811,8 +846,8 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, vm_unacct_memory(nrpages); /* - * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major - * fault on access. + * Private VMA_LOCKED_BIT VMA becoming writable: trigger COW to avoid + * major fault on access. */ if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT) && vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT) && @@ -886,7 +921,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, goto out; start = vma->vm_start; error = -EINVAL; - if (!(vma->vm_flags & VM_GROWSDOWN)) + if (!vma_test(vma, VMA_GROWSDOWN_BIT)) goto out; } else { if (vma->vm_start > start) @@ -894,7 +929,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (unlikely(grows & PROT_GROWSUP)) { end = vma->vm_end; error = -EINVAL; - if (!(vma->vm_flags & VM_GROWSUP)) + if (!vma_test_single_mask(vma, VMA_GROWSUP)) goto out; } } @@ -918,7 +953,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, } /* Does the application expect PROT_READ to imply PROT_EXEC */ - if (rier && (vma->vm_flags & VM_MAYEXEC)) + if (rier && vma_test(vma, VMA_MAYEXEC_BIT)) prot |= PROT_EXEC; /* diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832..f07fc4e3ef2e 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -68,7 +68,7 @@ struct vma_remap_struct { bool populate_expand; /* mlock()'d expanded, must populate. */ enum mremap_type remap_type; /* expand, shrink, etc. */ bool mmap_locked; /* Is mm currently write-locked? */ - unsigned long charged; /* If VM_ACCOUNT, # pages to account. */ + unsigned long charged; /* If VMA_ACCOUNT_BIT, # pgs to account */ bool vmi_needs_invalidate; /* Is the VMA iterator invalidated? */ }; @@ -296,10 +296,19 @@ static int move_ptes(struct pagetable_move_control *pmc, pte_clear(mm, new_addr, new_ptep); else { if (need_clear_uffd_wp) { - if (pte_present(pte)) - pte = pte_clear_uffd_wp(pte); - else - pte = pte_swp_clear_uffd_wp(pte); + if (pte_present(pte)) { + /* + * See __copy_present_ptes(): normalise + * RWP PTEs so the destination starts + * accessible instead of taking a + * numa-hinting fault on first access. + */ + if (userfaultfd_rwp(vma) && pte_uffd(pte)) + pte = pte_modify(pte, vma->vm_page_prot); + pte = pte_clear_uffd(pte); + } else { + pte = pte_swp_clear_uffd(pte); + } } set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); } @@ -948,14 +957,13 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + const pgoff_t pgoff = linear_page_index(vma, vrm->addr); unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; if (vrm->flags & MREMAP_FIXED) map_flags |= MAP_FIXED; - if (vma->vm_flags & VM_MAYSHARE) + if (vma_test(vma, VMA_MAYSHARE_BIT)) map_flags |= MAP_SHARED; res = get_unmapped_area(vma->vm_file, new_addr, vrm->new_len, pgoff, @@ -977,7 +985,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) { unsigned long charged; - if (!(vrm->vma->vm_flags & VM_ACCOUNT)) + if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT)) return true; /* @@ -1004,7 +1012,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) */ static void vrm_uncharge(struct vma_remap_struct *vrm) { - if (!(vrm->vma->vm_flags & VM_ACCOUNT)) + if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT)) return; vm_unacct_memory(vrm->charged); @@ -1024,7 +1032,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm, struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += pages; } @@ -1168,7 +1176,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * arose, in which case we _do_ wish to unmap the _new_ VMA, which means * we actually _do_ want it be unaccounted. */ - bool accountable_move = (vma->vm_flags & VM_ACCOUNT) && + bool accountable_move = vma_test(vma, VMA_ACCOUNT_BIT) && !(vrm->flags & MREMAP_DONTUNMAP); /* @@ -1187,7 +1195,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * portions of the original VMA that remain. */ if (accountable_move) { - vm_flags_clear(vma, VM_ACCOUNT); + vma_clear_flags(vma, VMA_ACCOUNT_BIT); /* We are about to split vma, so store the start/end. */ vm_start = vma->vm_start; vm_end = vma->vm_end; @@ -1212,8 +1220,8 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * | | * |-------------| * - * Having cleared VM_ACCOUNT from the whole VMA, after we unmap above - * we'll end up with: + * Having cleared VMA_ACCOUNT_BIT from the whole VMA, after we unmap + * above we'll end up with: * * addr end * | | @@ -1233,13 +1241,15 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) if (vm_start < addr) { struct vm_area_struct *prev = vma_prev(&vmi); - vm_flags_set(prev, VM_ACCOUNT); /* Acquires VMA lock. */ + vma_start_write(prev); + vma_set_flags(prev, VMA_ACCOUNT_BIT); } if (vm_end > end) { struct vm_area_struct *next = vma_next(&vmi); - vm_flags_set(next, VM_ACCOUNT); /* Acquires VMA lock. */ + vma_start_write(next); + vma_set_flags(next, VMA_ACCOUNT_BIT); } } } @@ -1255,17 +1265,17 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; - unsigned long moved_len; + const pgoff_t new_pgoff = linear_page_index(vrm->vma, vrm->addr); + const pgoff_t new_virt_pgoff = + __linear_virt_page_index(vrm->vma, vrm->addr); struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; + unsigned long moved_len; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, - &pmc.need_rmap_locks); + new_virt_pgoff, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); *new_vma_ptr = NULL; @@ -1324,8 +1334,8 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, unsigned long old_start = vrm->vma->vm_start; unsigned long old_end = vrm->vma->vm_end; - /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ - vm_flags_clear(vrm->vma, VM_LOCKED_MASK); + /* We always clear VMA_LOCKED[ONFAULT]_BIT on the old VMA. */ + vma_clear_flags_mask(vrm->vma, VMA_LOCKED_MASK); /* * anon_vma links of the old vma is no longer needed after its page @@ -1761,14 +1771,14 @@ static int check_prep_vma(struct vma_remap_struct *vrm) * based on the original. There are no known use cases for this * behavior. As a result, fail such attempts. */ - if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) { + if (!old_len && !vma_test_any(vma, VMA_SHARED_BIT, VMA_MAYSHARE_BIT)) { pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid); return -EINVAL; } if ((vrm->flags & MREMAP_DONTUNMAP) && - (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))) + vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT)) return -EINVAL; /* @@ -1798,19 +1808,18 @@ static int check_prep_vma(struct vma_remap_struct *vrm) return 0; /* We are expanding and the VMA is mlock()'d so we need to populate. */ - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; + pgoff = linear_page_index(vma, addr); if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) return -EINVAL; - if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) + if (vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT)) return -EFAULT; - if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) + if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), vrm->delta)) return -EAGAIN; if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) diff --git a/mm/mseal.c b/mm/mseal.c index 9781647483d1..7a8ac66dc215 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -16,32 +16,11 @@ #include <linux/sched.h> #include "internal.h" -/* - * mseal() disallows an input range which contain unmapped ranges (VMA holes). - * - * It disallows unmapped regions from start to end whether they exist at the - * start, in the middle, or at the end of the range, or any combination thereof. - * - * This is because after sealing a range, there's nothing to stop memory mapping - * of ranges in the remaining gaps later, meaning that the user might then - * wrongly consider the entirety of the mseal()'d range to be sealed when it - * in fact isn't. - */ - -/* - * Does the [start, end) range contain any unmapped memory? - * - * We ensure that: - * - start is part of a valid VMA. - * - end is part of a valid VMA. - * - no gap (unallocated memory) exists between start and end. - */ -static bool range_contains_unmapped(struct mm_struct *mm, - unsigned long start, unsigned long end) +static bool range_contains_unmapped(unsigned long start, unsigned long end) { - struct vm_area_struct *vma; - unsigned long prev_end = start; VMA_ITERATOR(vmi, current->mm, start); + unsigned long prev_end = start; + struct vm_area_struct *vma; for_each_vma_range(vmi, vma, end) { if (vma->vm_start > prev_end) @@ -53,11 +32,10 @@ static bool range_contains_unmapped(struct mm_struct *mm, return prev_end < end; } -static int mseal_apply(struct mm_struct *mm, - unsigned long start, unsigned long end) +static int __mseal_range(unsigned long start, unsigned long end) { + VMA_ITERATOR(vmi, current->mm, start); struct vm_area_struct *vma, *prev; - VMA_ITERATOR(vmi, mm, start); /* We know there are no gaps so this will be non-NULL. */ vma = vma_iter_load(&vmi); @@ -88,64 +66,58 @@ static int mseal_apply(struct mm_struct *mm, return 0; } +static int mseal_range(unsigned long start, unsigned long end) +{ + int err; + + err = mmap_write_lock_killable(current->mm); + if (err) + return err; + if (range_contains_unmapped(start, end)) + err = -ENOMEM; + else + err = __mseal_range(start, end); + mmap_write_unlock(current->mm); + return err; +} + +/** + * mseal_mmap_page_zero() - If the MMAP_PAGE_ZERO personality is set, mseal() + * the page mapped at address zero. + */ +void mseal_mmap_page_zero(void) +{ + int err; + + if (WARN_ON_ONCE(!(current->personality & MMAP_PAGE_ZERO))) + return; + + err = mseal_range(0, PAGE_SIZE); + if (err) + pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", + task_pid_nr(current), err); +} + /* - * mseal(2) seals the VM's meta data from - * selected syscalls. - * - * addr/len: VM address range. - * - * The address range by addr/len must meet: - * start (addr) must be in a valid VMA. - * end (addr + len) must be in a valid VMA. - * no gap (unallocated memory) between start and end. - * start (addr) must be page aligned. + * Seal VMAs in the specified input range to prevent an attacker replacing what + * is mapped in the range with something else. * - * len: len will be page aligned implicitly. + * Disallows: + * - VMA unmapping, remapping or shrinking. + * - Overwriting the VMA with another one via mmap(), mremap() or similar. + * - Alteration of properties via mprotect()/pkey_mprotect(). + * - Destructive madvise() behaviours (like MADV_DONTNEED) on anonymous read-only + * ranges. * - * Below VMA operations are blocked after sealing. - * 1> Unmapping, moving to another location, and shrinking - * the size, via munmap() and mremap(), can leave an empty - * space, therefore can be replaced with a VMA with a new - * set of attributes. - * 2> Moving or expanding a different vma into the current location, - * via mremap(). - * 3> Modifying a VMA via mmap(MAP_FIXED). - * 4> Size expansion, via mremap(), does not appear to pose any - * specific risks to sealed VMAs. It is included anyway because - * the use case is unclear. In any case, users can rely on - * merging to expand a sealed VMA. - * 5> mprotect and pkey_mprotect. - * 6> Some destructive madvice() behavior (e.g. MADV_DONTNEED) - * for anonymous memory, when users don't have write permission to the - * memory. Those behaviors can alter region contents by discarding pages, - * effectively a memset(0) for anonymous memory. + * Since unmapped ranges can be mapped at any time, the input range must span + * mapped ranges only. * - * flags: reserved. - * - * return values: - * zero: success. - * -EINVAL: - * invalid input flags. - * start address is not page aligned. - * Address range (start + len) overflow. - * -ENOMEM: - * addr is not a valid address (not allocated). - * end (start + len) is not a valid address. - * a gap (unallocated memory) between start and end. - * -EPERM: - * - In 32 bit architecture, sealing is not supported. - * Note: - * user can call mseal(2) multiple times, adding a seal on an - * already sealed memory is a no-action (no error). - * - * unseal() is not supported. + * The flags parameter is currently reserved. */ -int do_mseal(unsigned long start, size_t len_in, unsigned long flags) +SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, flags) { - size_t len; - int ret = 0; + size_t len_aligned; unsigned long end; - struct mm_struct *mm = current->mm; /* Verify flags not set. */ if (flags) @@ -155,41 +127,17 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) if (!PAGE_ALIGNED(start)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len_aligned = PAGE_ALIGN(len); /* Check to see whether len was rounded up from small -ve to zero. */ - if (len_in && !len) + if (len && !len_aligned) return -EINVAL; - end = start + len; + end = start + len_aligned; if (end < start) return -EINVAL; if (end == start) return 0; - if (mmap_write_lock_killable(mm)) - return -EINTR; - - if (range_contains_unmapped(mm, start, end)) { - ret = -ENOMEM; - goto out; - } - - /* - * Second pass, this should success, unless there are errors - * from vma_modify_flags, e.g. merge/split error, or process - * reaching the max supported VMAs, however, those cases shall - * be rare. - */ - ret = mseal_apply(mm, start, end); - -out: - mmap_write_unlock(mm); - return ret; -} - -SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, - flags) -{ - return do_mseal(start, len, flags); + return mseal_range(start, end); } diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7..90b491a27a14 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -12,6 +12,7 @@ #include <linux/mm.h> #include <linux/mman.h> #include <linux/file.h> +#include <linux/pagemap.h> #include <linux/syscalls.h> #include <linux/sched.h> @@ -85,8 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) goto out_unlock; } file = vma->vm_file; - fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + fstart = (loff_t)linear_page_index(vma, start) << PAGE_SHIFT; fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/nommu.c b/mm/nommu.c index ed3934bc2de4..498e01ee40b0 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -975,7 +975,7 @@ static int do_mmap_private(struct vm_area_struct *vma, /* read the contents of a file into the copy */ loff_t fpos; - fpos = vma->vm_pgoff; + fpos = vma_start_pgoff(vma); fpos <<= PAGE_SHIFT; ret = kernel_read(vma->vm_file, base, len, &fpos); @@ -1014,11 +1014,12 @@ unsigned long do_mmap(struct file *file, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { + vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); struct vm_area_struct *vma; struct vm_region *region; struct rb_node *rb; @@ -1035,6 +1036,9 @@ unsigned long do_mmap(struct file *file, if (ret < 0) return ret; + if (current->mm->map_count >= get_sysctl_max_map_count()) + return -ENOMEM; + /* we ignore the address hint */ addr = 0; len = PAGE_ALIGN(len); @@ -1058,7 +1062,7 @@ unsigned long do_mmap(struct file *file, region->vm_pgoff = pgoff; vm_flags_init(vma, vm_flags); - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); if (file) { region->vm_file = get_file(file); @@ -1178,7 +1182,6 @@ unsigned long do_mmap(struct file *file, ret = do_mmap_private(vma, region, len, capabilities); if (ret < 0) goto error_just_free; - add_nommu_region(region); /* clear anonymous mappings that don't ask for uninitialized data */ if (!vma->vm_file && @@ -1196,7 +1199,9 @@ share: BUG_ON(!vma->vm_region); vma_iter_config(&vmi, vma->vm_start, vma->vm_end); if (vma_iter_prealloc(&vmi, vma)) - goto error_just_free; + goto error_vma_iter_prealloc; + + add_nommu_region(region); setup_vma_to_mm(vma, current->mm); current->mm->map_count++; @@ -1215,22 +1220,41 @@ share: return result; error_just_free: + vma_close(vma); + /* if the error was from shared mapping/existing region, don't free the region. + * this has to be before releasing semaphore. + */ + if (region->vm_usage == 1) { + if (region->vm_file) + fput(region->vm_file); + kmem_cache_free(vm_region_jar, region); + + } else + region->vm_usage--; + up_write(&nommu_region_sem); -error: vma_iter_free(&vmi); - if (region->vm_file) - fput(region->vm_file); - kmem_cache_free(vm_region_jar, region); + if (vma->vm_file) fput(vma->vm_file); vm_area_free(vma); return ret; sharing_violation: - up_write(&nommu_region_sem); pr_warn("Attempt to share mismatched mappings\n"); ret = -EINVAL; - goto error; + goto error_just_free; + +error_vma_iter_prealloc: + pr_warn("Allocation of vma iterator for process %d failed\n", current->pid); + show_mem(); + ret = -ENOMEM; + + /* in case that the region is allocated via do_mmap_private() */ + if ((region->vm_usage == 1) && (region->vm_flags & VM_MAPPED_COPY)) + free_page_series(region->vm_start, region->vm_top); + + goto error_just_free; error_getting_vma: kmem_cache_free(vm_region_jar, region); @@ -1332,13 +1356,14 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, *region = *vma->vm_region; new->vm_region = region; - npages = (addr - vma->vm_start) >> PAGE_SHIFT; + npages = linear_page_delta(vma, addr); if (new_below) { region->vm_top = region->vm_end = new->vm_end = addr; } else { region->vm_start = new->vm_start = addr; - region->vm_pgoff = new->vm_pgoff += npages; + vma_add_pgoff(new, npages); + region->vm_pgoff = vma_start_pgoff(new); } vma_iter_config(vmi, new->vm_start, new->vm_end); @@ -1355,7 +1380,8 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, delete_nommu_region(vma->vm_region); if (new_below) { vma->vm_region->vm_start = vma->vm_start = addr; - vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; + vma_add_pgoff(vma, npages); + vma->vm_region->vm_pgoff = vma_start_pgoff(vma); } else { vma->vm_region->vm_end = vma->vm_end = addr; vma->vm_region->vm_top = addr; @@ -1367,6 +1393,10 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, setup_vma_to_mm(vma, mm); setup_vma_to_mm(new, mm); vma_iter_store_new(vmi, new); + + /* vmi should point lower address */ + if (new_below) + vma_next(vmi); mm->map_count++; return 0; @@ -1603,7 +1633,7 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn = start >> PAGE_SHIFT; unsigned long vm_len = vma->vm_end - vma->vm_start; - pfn += vma->vm_pgoff; + pfn += vma_start_pgoff(vma); return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); } EXPORT_SYMBOL(vm_iomap_memory); @@ -1816,7 +1846,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, i_mmap_lock_read(inode->i_mapping); /* search for VMAs that fall within the dead zone */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, low, high) { /* found one - only interested if it's shared out of the page * cache */ if (vma->vm_flags & VM_SHARED) { @@ -1832,7 +1862,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, * we don't check for any regions that start beyond the EOF as there * shouldn't be any */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { if (!(vma->vm_flags & VM_SHARED)) continue; diff --git a/mm/page-writeback.c b/mm/page-writeback.c index e98748112d1e..47495be68598 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -584,16 +584,6 @@ static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr) wb->bdi->max_prop_frac, nr); } -void wb_writeout_inc(struct bdi_writeback *wb) -{ - unsigned long flags; - - local_irq_save(flags); - __wb_writeout_add(wb, 1); - local_irq_restore(flags); -} -EXPORT_SYMBOL_GPL(wb_writeout_inc); - /* * On idle system, we can be called long after we scheduled because we use * deferred timers so count with missed periods. diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee902a468c2f..083cbcb5bdde 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -56,6 +56,8 @@ #include <linux/pgalloc_tag.h> #include <asm/div64.h> #include "internal.h" +#include "mm_init.h" +#include "page_alloc.h" #include "shuffle.h" #include "page_reporting.h" @@ -88,7 +90,7 @@ typedef int __bitwise fpi_t; #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* Free the page without taking locks. Rely on trylock only. */ -#define FPI_TRYLOCK ((__force fpi_t)BIT(2)) +#define FPI_NOLOCK ((__force fpi_t)BIT(2)) /* free_pages_prepare() has already been called for page(s) being freed. */ #define FPI_PREPARED ((__force fpi_t)BIT(3)) @@ -165,7 +167,9 @@ DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); #endif +#ifdef CONFIG_NUMA DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); +#endif #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* @@ -721,14 +725,14 @@ static inline struct capture_control *task_capc(struct zone *zone) return unlikely(capc) && !(current->flags & PF_KTHREAD) && !capc->page && - capc->cc->zone == zone ? capc : NULL; + capc->zone == zone ? capc : NULL; } static inline bool compaction_capture(struct capture_control *capc, struct page *page, int order, int migratetype) { - if (!capc || order != capc->cc->order) + if (!capc || order != capc->order) return false; /* Do not accidentally pollute CMA or isolated regions*/ @@ -744,12 +748,12 @@ compaction_capture(struct capture_control *capc, struct page *page, * have trouble finding a high-order free page. */ if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && - capc->cc->migratetype != MIGRATE_MOVABLE) + capc->migratetype != MIGRATE_MOVABLE) return false; - if (migratetype != capc->cc->migratetype) - trace_mm_page_alloc_extfrag(page, capc->cc->order, order, - capc->cc->migratetype, migratetype); + if (migratetype != capc->migratetype) + trace_mm_page_alloc_extfrag(page, capc->order, order, + capc->migratetype, migratetype); capc->page = page; return true; @@ -1246,7 +1250,7 @@ void __clear_page_tag_ref(struct page *page) /* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) + unsigned int nr, unsigned int alloc_flags) { union pgtag_ref_handle handle; union codetag_ref ref; @@ -1260,17 +1264,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task, * page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ - alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags); + alloc_tag_add_early_pfn(page_to_pfn(page), alloc_flags); if (task->alloc_tag) alloc_tag_set_inaccurate(task->alloc_tag); } } static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) + unsigned int nr, unsigned int alloc_flags) { if (mem_alloc_profiling_enabled()) - __pgalloc_tag_add(page, task, nr, gfp_flags); + __pgalloc_tag_add(page, task, nr, alloc_flags); } /* Should be called only if mem_alloc_profiling_enabled() */ @@ -1303,7 +1307,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) {} + unsigned int nr, unsigned int alloc_flags) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} @@ -1374,15 +1378,23 @@ static __always_inline bool __free_pages_prepare(struct page *page, #endif } for (i = 1; i < (1 << order); i++) { + struct page *tail_page = page + i; + if (compound) - bad += free_tail_page_prepare(page, page + i); + bad += free_tail_page_prepare(page, tail_page); if (is_check_pages_enabled()) { - if (free_page_is_bad(page + i)) { + if (free_page_is_bad(tail_page)) { + bad++; + continue; + } + + if (tail_page->private) { + bad_page(tail_page, "nonzero private"); bad++; continue; } } - (page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; + tail_page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; } } if (folio_test_anon(folio)) { @@ -1407,7 +1419,7 @@ static __always_inline bool __free_pages_prepare(struct page *page, page_table_check_free(page, order); pgalloc_tag_sub(page, 1 << order); - if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { + if (!PageHighMem(page) && !(fpi_flags & FPI_NOLOCK)) { debug_check_no_locks_freed(page_address(page), PAGE_SIZE << order); debug_check_no_obj_freed(page_address(page), @@ -1546,8 +1558,8 @@ static void free_one_page(struct zone *zone, struct page *page, struct llist_head *llhead; unsigned long flags; - if (unlikely(fpi_flags & FPI_TRYLOCK)) { - if (!spin_trylock_irqsave(&zone->lock, flags)) { + if (unlikely(fpi_flags & FPI_NOLOCK)) { + if (!can_spin_trylock() || !spin_trylock_irqsave(&zone->lock, flags)) { add_page_to_zone_llist(zone, page, order); return; } @@ -1557,7 +1569,7 @@ static void free_one_page(struct zone *zone, struct page *page, /* The lock succeeded. Process deferred pages. */ llhead = &zone->trylock_free_pages; - if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_TRYLOCK))) { + if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_NOLOCK))) { struct llist_node *llnode; struct page *p, *tmp; @@ -1807,7 +1819,7 @@ static inline bool should_skip_init(gfp_t flags) } inline void post_alloc_hook(struct page *page, unsigned int order, - gfp_t gfp_flags) + gfp_t gfp_flags, unsigned int alloc_flags) { const bool zero_tags = gfp_flags & __GFP_ZEROTAGS; bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && @@ -1858,13 +1870,13 @@ inline void post_alloc_hook(struct page *page, unsigned int order, set_page_owner(page, order, gfp_flags); page_table_check_alloc(page, order); - pgalloc_tag_add(page, current, 1 << order, gfp_flags); + pgalloc_tag_add(page, current, 1 << order, alloc_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, unsigned int alloc_flags) { - post_alloc_hook(page, order, gfp_flags); + post_alloc_hook(page, order, gfp_flags, alloc_flags); if (order && (gfp_flags & __GFP_COMP)) prep_compound_page(page, order); @@ -2528,7 +2540,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, unsigned long flags; int i; - if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { + if (unlikely(alloc_flags & ALLOC_NOLOCK)) { if (!spin_trylock_irqsave(&zone->lock, flags)) return 0; } else { @@ -2870,7 +2882,7 @@ static bool free_frozen_page_commit(struct zone *zone, if (pcp->free_count < (batch << CONFIG_PCP_BATCH_SCALE_MAX)) pcp->free_count += (1 << order); - if (unlikely(fpi_flags & FPI_TRYLOCK)) { + if (unlikely(fpi_flags & FPI_NOLOCK)) { /* * Do not attempt to take a zone lock. Let pcp->count get * over high mark temporarily. @@ -2967,8 +2979,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, migratetype = MIGRATE_MOVABLE; } - if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT) - && (in_nmi() || in_hardirq()))) { + if (unlikely((fpi_flags & FPI_NOLOCK) && !can_spin_trylock())) { add_page_to_zone_llist(zone, page, order); return; } @@ -2990,7 +3001,7 @@ void free_frozen_pages(struct page *page, unsigned int order) void free_frozen_pages_nolock(struct page *page, unsigned int order) { - __free_frozen_pages(page, order, FPI_TRYLOCK); + __free_frozen_pages(page, order, FPI_NOLOCK); } /* @@ -3216,7 +3227,7 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, do { page = NULL; - if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { + if (unlikely(alloc_flags & ALLOC_NOLOCK)) { if (!spin_trylock_irqsave(&zone->lock, flags)) return NULL; } else { @@ -3247,10 +3258,11 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, } while (check_new_pages(page, order)); /* - * If this is a high-order atomic allocation then check - * if the pageblock should be reserved for the future + * Slowpath (precarious) high-atomic allocations may reserve + * a pageblock for future use. */ - if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) + if (unlikely((alloc_flags & ALLOC_HIGHATOMIC) && + ((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN))) reserve_highatomic_pageblock(page, order, zone); __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); @@ -3739,13 +3751,10 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) static inline unsigned int alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) { - unsigned int alloc_flags; + unsigned int alloc_flags = 0; - /* - * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save a branch. - */ - alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; if (defrag_mode) { alloc_flags |= ALLOC_NOFRAGMENT; @@ -3774,14 +3783,13 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) } /* Must be called after current_gfp_context() which can change gfp_mask */ -static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, - unsigned int alloc_flags) +static inline unsigned int alloc_flags_cma(gfp_t gfp_mask) { #ifdef CONFIG_CMA if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) - alloc_flags |= ALLOC_CMA; + return ALLOC_CMA; #endif - return alloc_flags; + return ALLOC_DEFAULT; } /* @@ -3899,8 +3907,6 @@ check_alloc_wmark: if (!zone_watermark_fast(zone, order, mark, ac->highest_zoneidx, alloc_flags, gfp_mask)) { - int ret; - if (cond_accept_memory(zone, order, alloc_flags)) goto try_this_zone; @@ -3921,22 +3927,13 @@ check_alloc_wmark: !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone)) continue; - ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); - switch (ret) { - case NODE_RECLAIM_NOSCAN: - /* did not scan */ - continue; - case NODE_RECLAIM_FULL: - /* scanned but unreclaimable */ + if (!node_reclaim(zone->zone_pgdat, gfp_mask, order)) continue; - default: - /* did we reclaim enough */ - if (zone_watermark_ok(zone, order, mark, - ac->highest_zoneidx, alloc_flags)) - goto try_this_zone; + /* did we reclaim enough */ + if (!zone_watermark_ok(zone, order, mark, + ac->highest_zoneidx, alloc_flags)) continue; - } } try_this_zone: @@ -3979,7 +3976,7 @@ try_this_zone: return NULL; } -static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) +static void warn_alloc_show_mem(gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned int filter = SHOW_MEM_FILTER_NODES; @@ -3999,7 +3996,7 @@ static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) mem_cgroup_show_protected_memory(NULL); } -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...) { struct va_format vaf; va_list args; @@ -4077,7 +4074,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, */ page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & ~__GFP_DIRECT_RECLAIM, order, - ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); + ac->alloc_flags|ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); if (page) goto out; @@ -4123,7 +4120,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, */ if (gfp_mask & __GFP_NOFAIL) page = __alloc_pages_cpuset_fallback(gfp_mask, order, - ALLOC_NO_WATERMARKS, ac); + ac->alloc_flags|ALLOC_NO_WATERMARKS, ac); } out: mutex_unlock(&oom_lock); @@ -4146,18 +4143,67 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; unsigned int noreclaim_flag; + struct capture_control capc = { + .zone = NULL, + .migratetype = ac->migratetype, + .order = order, + .page = NULL, + }; + int compact_order = order; - if (!order) + /* + * If fallbacks are not permitted (defrag_mode), we either + * need to reclaim space in a block of matching type, or clear + * out an entire block to allow __rmqueue_claim() to convert. + * + * Reclaim by itself is primarily freeing space in movable + * blocks, since that's where the LRU pages live. So this + * works for movable requests, but not for others. + * + * For those, promote the order to help make blocks, instead + * of spinning in reclaim alone unproductively. + */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + compact_order = max(order, pageblock_order); + + if (!compact_order) return NULL; psi_memstall_enter(&pflags); delayacct_compact_start(); + fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); - *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, - prio, &page); + /* + * Make sure the structs are really initialized before we expose the + * capture control, in case we are interrupted and the interrupt handler + * frees a page. + */ + barrier(); + WRITE_ONCE(current->capture_control, &capc); + + *compact_result = try_to_compact_pages(gfp_mask, compact_order, + alloc_flags, ac, prio, &capc); + + /* + * Make sure we hide capture control first before we read the captured + * page pointer, otherwise an interrupt could free and capture a page + * and we would leak it. + */ + WRITE_ONCE(current->capture_control, NULL); + page = READ_ONCE(capc.page); + + /* + * Technically, it is also possible that compaction is skipped but + * the page is still captured out of luck(IRQ came and freed the page). + * Returning COMPACT_SUCCESS in such cases helps in properly accounting + * the COMPACT[STALL|FAIL] when compaction is skipped. + */ + if (page) + *compact_result = COMPACT_SUCCESS; memalloc_noreclaim_restore(noreclaim_flag); + fs_reclaim_release(gfp_mask); psi_memstall_leave(&pflags); delayacct_compact_end(); @@ -4182,7 +4228,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct zone *zone = page_zone(page); zone->compact_blockskip_flush = false; - compaction_defer_reset(zone, order, true); + compaction_defer_reset(zone, compact_order, true); count_vm_event(COMPACTSUCCESS); return page; } @@ -4422,9 +4468,14 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; bool drained = false; + int reclaim_order = order; + + /* Match the slowpath compaction promotion in __alloc_pages_direct_compact */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + reclaim_order = max(order, pageblock_order); psi_memstall_enter(&pflags); - *did_some_progress = __perform_reclaim(gfp_mask, order, ac); + *did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac); if (unlikely(!(*did_some_progress))) goto out; @@ -4474,17 +4525,32 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, } static inline unsigned int -gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) +alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) { - unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; + unsigned int alloc_flags = 0; + + if (gfp_mask & __GFP_DIRECT_RECLAIM) + return 0; /* - * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE - * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save two branches. + * Not worth trying to allocate harder for __GFP_NOMEMALLOC even + * if it can't schedule. */ - BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE); - BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); + if (gfp_mask & __GFP_NOMEMALLOC) + return 0; + + alloc_flags |= ALLOC_NON_BLOCK; + + if (order > 0 && (gfp_mask & __GFP_HIGH)) + alloc_flags |= ALLOC_HIGHATOMIC; + + return alloc_flags; +} + +static inline unsigned int +alloc_flags_slowpath(gfp_t gfp_mask, unsigned int order) +{ + unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; /* * The caller may dip into page reserves a bit more if the caller @@ -4492,21 +4558,14 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH). */ - alloc_flags |= (__force int) - (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); + if (gfp_mask & __GFP_HIGH) + alloc_flags |= ALLOC_MIN_RESERVE; + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; - if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { - /* - * Not worth trying to allocate harder for __GFP_NOMEMALLOC even - * if it can't schedule. - */ - if (!(gfp_mask & __GFP_NOMEMALLOC)) { - alloc_flags |= ALLOC_NON_BLOCK; - - if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE)) - alloc_flags |= ALLOC_HIGHATOMIC; - } + alloc_flags |= alloc_flags_nonblocking(gfp_mask, order); + if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { /* * Ignore cpuset mems for non-blocking __GFP_HIGH (probably * GFP_ATOMIC) rather than fail, see the comment for @@ -4517,7 +4576,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) } else if (unlikely(rt_or_dl_task(current)) && in_task()) alloc_flags |= ALLOC_MIN_RESERVE; - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); + alloc_flags |= alloc_flags_cma(gfp_mask); if (defrag_mode) alloc_flags |= ALLOC_NOFRAGMENT; @@ -4687,7 +4746,7 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) return false; } -static void check_alloc_stall_warn(gfp_t gfp_mask, nodemask_t *nodemask, +static void check_alloc_stall_warn(gfp_t gfp_mask, const nodemask_t *nodemask, unsigned int order, unsigned long alloc_start_time) { static DEFINE_SPINLOCK(alloc_stall_lock); @@ -4782,8 +4841,12 @@ restart: * The fast path uses conservative alloc_flags to succeed only until * kswapd needs to be woken up, and to avoid the cost of setting up * alloc_flags precisely. So we do that now. + * + * Can't just or alloc_flags if it contains WMARK bits, but those flags + * shouldn't be set in ac->alloc_flags. */ - alloc_flags = gfp_to_alloc_flags(gfp_mask, order); + VM_WARN_ON(ac->alloc_flags & ALLOC_WMARK_MASK); + alloc_flags = ac->alloc_flags | alloc_flags_slowpath(gfp_mask, order); /* * We need to recalculate the starting point for the zonelist iterator @@ -4824,8 +4887,8 @@ retry: reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); if (reserve_flags) - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | - (alloc_flags & ALLOC_KSWAPD); + alloc_flags = alloc_flags_cma(gfp_mask) | reserve_flags | + ac->alloc_flags | (alloc_flags & ALLOC_KSWAPD); /* * Reset the nodemask and zonelist iterators if memory policies can be @@ -4994,6 +5057,8 @@ nopage: * we always retry */ if (unlikely(nofail)) { + unsigned int alloc_flags = ac->alloc_flags | ALLOC_MIN_RESERVE; + /* * Lacking direct_reclaim we can't do anything to reclaim memory, * we disregard these unreasonable nofail requests and still @@ -5009,7 +5074,7 @@ nopage: * could deplete whole memory reserves which would just make * the situation worse. */ - page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac); + page = __alloc_pages_cpuset_fallback(gfp_mask, order, alloc_flags, ac); if (page) goto got_pg; @@ -5051,11 +5116,11 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, * Don't invoke should_fail logic, since it may call * get_random_u32() and printk() which need to spin_lock. */ - if (!(*alloc_flags & ALLOC_TRYLOCK) && + if (!(*alloc_flags & ALLOC_NOLOCK) && should_fail_alloc_page(gfp_mask, order)) return false; - *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); + *alloc_flags |= alloc_flags_cma(gfp_mask); /* Dirty zone balancing only done in the fast path */ ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); @@ -5213,7 +5278,7 @@ retry_this_zone: } nr_account++; - prep_new_page(page, 0, gfp, 0); + prep_new_page(page, 0, gfp, ALLOC_DEFAULT); set_page_refcounted(page); page_array[nr_populated++] = page; } @@ -5227,7 +5292,7 @@ out: return nr_populated; failed: - page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask); + page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask, ALLOC_DEFAULT); if (page) page_array[nr_populated++] = page; goto out; @@ -5262,24 +5327,87 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages) } } +static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order, + unsigned int alloc_flags) +{ + if (alloc_flags & ALLOC_NOLOCK) + return pcp_allowed_order(order); + + /* + * There are several places where we assume that the order value is sane + * so bail out early if the request is out of bound. + */ + return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)); +} + +static inline bool alloc_nolock_allowed(void) +{ + if (!can_spin_trylock()) + return false; + + /* Bailout, since _deferred_grow_zone() needs to take a lock */ + if (deferred_pages_enabled()) + return false; + + return true; +} + +/* + * GFP flags to set for ALLOC_NOLOCK i.e. alloc_pages_nolock(). + * + * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. + * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd + * is not safe in arbitrary context. + * + * These two are the conditions for gfpflags_allow_spinning() being true. + * + * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason + * to warn. Also warn would trigger printk() which is unsafe from + * various contexts. We cannot use printk_deferred_enter() to mitigate, + * since the running context is unknown. + * + * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below + * is safe in any context. Also zeroing the page is mandatory for + * BPF use cases. + * + * Though __GFP_NOMEMALLOC is not checked in the code path below, + * specify it here to highlight that alloc_pages_nolock() + * doesn't want to deplete reserves. + */ +static const gfp_t gfp_nolock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | + __GFP_COMP; + /* * This is the 'heart' of the zoned buddy allocator. */ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, - int preferred_nid, nodemask_t *nodemask) + int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags) { struct page *page; - unsigned int alloc_flags = ALLOC_WMARK_LOW; gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ - struct alloc_context ac = { }; + struct alloc_context ac = { + .alloc_flags = alloc_flags, + }; + unsigned int fastpath_alloc_flags = alloc_flags; - /* - * There are several places where we assume that the order value is sane - * so bail out early if the request is out of bound. - */ - if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)) + /* Other flags could be supported later if needed. */ + if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG))) return NULL; + if (!alloc_order_allowed(gfp, order, alloc_flags)) + return NULL; + + if (alloc_flags & ALLOC_NOLOCK) { + /* Certain other flags could be supported later if needed. */ + VM_WARN_ON_ONCE(gfp & ~(__GFP_ACCOUNT | gfp_nolock)); + if (!alloc_nolock_allowed()) + return NULL; + gfp |= gfp_nolock; + fastpath_alloc_flags |= ALLOC_WMARK_MIN; + } else { + fastpath_alloc_flags |= ALLOC_WMARK_LOW; + } + gfp &= gfp_allowed_mask; /* * Apply scoped allocation constraints. This is mainly about GFP_NOFS @@ -5291,18 +5419,22 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, gfp = current_gfp_context(gfp); alloc_gfp = gfp; if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, - &alloc_gfp, &alloc_flags)) + &alloc_gfp, &fastpath_alloc_flags)) return NULL; - /* - * Forbid the first pass from falling back to types that fragment - * memory until all local zones are considered. - */ - alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + if (!(alloc_flags & ALLOC_NOLOCK)) { + /* + * Forbid the first pass from falling back to types that + * fragment memory until all local zones are considered. + */ + fastpath_alloc_flags |= alloc_flags_nofragment( + zonelist_zone(ac.preferred_zoneref), gfp); + } + fastpath_alloc_flags |= alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; - /* First allocation attempt */ - page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); - if (likely(page)) + /* First allocation attempt (or, for nolock, only attempt) */ + page = get_page_from_freelist(alloc_gfp, order, fastpath_alloc_flags, &ac); + if (likely(page) || (alloc_flags & ALLOC_NOLOCK)) goto out; alloc_gfp = gfp; @@ -5319,7 +5451,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, out: if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { - free_frozen_pages(page, order); + __free_frozen_pages(page, order, + alloc_flags & ALLOC_NOLOCK ? FPI_NOLOCK : 0); page = NULL; } @@ -5331,22 +5464,33 @@ out: EXPORT_SYMBOL(__alloc_frozen_pages_noprof); struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, - int preferred_nid, nodemask_t *nodemask) + int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags) { struct page *page; - page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask); + page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, + alloc_flags); if (page) set_page_refcounted(page); return page; } -EXPORT_SYMBOL(__alloc_pages_noprof); + +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) +{ + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + warn_if_node_offline(nid, gfp_mask); + + return __alloc_pages_noprof(gfp_mask, order, nid, NULL, ALLOC_DEFAULT); +} +EXPORT_SYMBOL(alloc_pages_node_noprof); struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask) { struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order, - preferred_nid, nodemask); + preferred_nid, nodemask, ALLOC_DEFAULT); return page_rmappable_folio(page); } EXPORT_SYMBOL(__folio_alloc_noprof); @@ -5430,7 +5574,7 @@ EXPORT_SYMBOL(__free_pages); */ void free_pages_nolock(struct page *page, unsigned int order) { - ___free_pages(page, order, FPI_TRYLOCK); + ___free_pages(page, order, FPI_NOLOCK); } /** @@ -6287,15 +6431,34 @@ void adjust_managed_page_count(struct page *page, long count) } EXPORT_SYMBOL(adjust_managed_page_count); -void free_reserved_page(struct page *page) +/** + * free_reserved_pages - free reserved pages + * @page: First page to free. + * @order: The page order to free. + * + * Free pages allocated through memblock during boot, letting the buddy + * manage them from now on. + * + * @page must be naturally aligned to the order and the order must not + * exceed MAX_PAGE_ORDER. All pages must be reserved. + */ +void free_reserved_pages(struct page *page, unsigned int order) { - clear_page_tag_ref(page); - ClearPageReserved(page); - init_page_count(page); - __free_page(page); - adjust_managed_page_count(page, 1); + const unsigned long nr_pages = 1UL << order; + int i; + + VM_WARN_ON_ONCE(!IS_ALIGNED(page_to_pfn(page), nr_pages)); + VM_WARN_ON_ONCE(order > MAX_PAGE_ORDER); + + for (i = 0; i < nr_pages; i++) { + clear_page_tag_ref(page + i); + set_page_count(page + i, 0); + ClearPageReserved(page + i); + } + adjust_managed_page_count(page, nr_pages); + __free_frozen_pages(page, order, FPI_NONE); } -EXPORT_SYMBOL(free_reserved_page); +EXPORT_SYMBOL(free_reserved_pages); static int page_alloc_cpu_dead(unsigned int cpu) { @@ -6904,15 +7067,15 @@ static void __free_contig_range_common(unsigned long pfn, unsigned long nr_pages continue; } - if (start && memdesc_section(page->flags) != start_sec) { + if (start && memdesc_section(&page->flags) != start_sec) { free_prepared_contig_range(start, i - nr_start); start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } else if (!start) { start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } } @@ -7028,7 +7191,7 @@ static void split_free_frozen_pages(struct list_head *list, gfp_t gfp_mask) list_for_each_entry_safe(page, next, &list[order], lru) { int i; - post_alloc_hook(page, order, gfp_mask); + post_alloc_hook(page, order, gfp_mask, ALLOC_DEFAULT); if (!order) continue; @@ -7233,7 +7396,7 @@ int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end, struct page *head = pfn_to_page(start); check_new_pages(head, order); - prep_new_page(head, order, gfp_mask, 0); + prep_new_page(head, order, gfp_mask, ALLOC_DEFAULT); } else { ret = -EINVAL; WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n", @@ -7795,7 +7958,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, return false; /* Bailout, since try_to_accept_memory_one() needs to take a lock */ - if (alloc_flags & ALLOC_TRYLOCK) + if (alloc_flags & ALLOC_NOLOCK) return false; wmark = promo_wmark_pages(zone); @@ -7865,84 +8028,15 @@ static bool __free_unaccepted(struct page *page) struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order) { - /* - * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. - * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd - * is not safe in arbitrary context. - * - * These two are the conditions for gfpflags_allow_spinning() being true. - * - * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason - * to warn. Also warn would trigger printk() which is unsafe from - * various contexts. We cannot use printk_deferred_enter() to mitigate, - * since the running context is unknown. - * - * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below - * is safe in any context. Also zeroing the page is mandatory for - * BPF use cases. - * - * Though __GFP_NOMEMALLOC is not checked in the code path below, - * specify it here to highlight that alloc_pages_nolock() - * doesn't want to deplete reserves. - */ - gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP - | gfp_flags; - unsigned int alloc_flags = ALLOC_TRYLOCK; - struct alloc_context ac = { }; - struct page *page; - - VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT); - /* - * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is - * unsafe in NMI. If spin_trylock() is called from hard IRQ the current - * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will - * mark the task as the owner of another rt_spin_lock which will - * confuse PI logic, so return immediately if called from hard IRQ or - * NMI. - * - * Note, irqs_disabled() case is ok. This function can be called - * from raw_spin_lock_irqsave region. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return NULL; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) - return NULL; - - if (!pcp_allowed_order(order)) - return NULL; - - /* Bailout, since _deferred_grow_zone() needs to take a lock */ - if (deferred_pages_enabled()) - return NULL; - if (nid == NUMA_NO_NODE) nid = numa_node_id(); - prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac, - &alloc_gfp, &alloc_flags); - - /* - * Best effort allocation from percpu free list. - * If it's empty attempt to spin_trylock zone->lock. - */ - page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); - - /* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */ - - if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) && - unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) { - __free_frozen_pages(page, order, FPI_TRYLOCK); - page = NULL; - } - trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); - kmsan_alloc_page(page, order, alloc_gfp); - return page; + return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_NOLOCK); } /** * alloc_pages_nolock - opportunistic reentrant allocation from any context - * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed. + * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, plus some flags that get set + * internally regardless (see %gfp_nolock) are allowed. * @nid: node to allocate from * @order: allocation order size * diff --git a/mm/page_alloc.h b/mm/page_alloc.h new file mode 100644 index 000000000000..b9259deddb59 --- /dev/null +++ b/mm/page_alloc.h @@ -0,0 +1,312 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal API for the page (buddy) allocator. Public API lives in + * include/linux/gfp.h. + */ +#ifndef __MM_PAGE_ALLOC_H +#define __MM_PAGE_ALLOC_H + +#include <linux/mm.h> +#include <linux/mmzone.h> +#include <linux/nodemask.h> +#include <linux/types.h> + +#define ALLOC_DEFAULT 0 +/* The ALLOC_WMARK bits are used as an index to zone->watermark */ +#define ALLOC_WMARK_MIN WMARK_MIN +#define ALLOC_WMARK_LOW WMARK_LOW +#define ALLOC_WMARK_HIGH WMARK_HIGH +#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ + +/* Mask to get the watermark bits */ +#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) + +/* + * Only MMU archs have async oom victim reclaim - aka oom_reaper so we + * cannot assume a reduced access to memory reserves is sufficient for + * !MMU + */ +#ifdef CONFIG_MMU +#define ALLOC_OOM 0x08 +#else +#define ALLOC_OOM ALLOC_NO_WATERMARKS +#endif + +#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access + * to 25% of the min watermark or + * 62.5% if __GFP_HIGH is set. + */ +#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% + * of the min watermark. + */ +#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ +#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ +#ifdef CONFIG_ZONE_DMA32 +#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ +#else +#define ALLOC_NOFRAGMENT 0x0 +#endif +#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ +#define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ +#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ +/* + * Avoid alloc_tag recursion for internal allocations. + * + * Callers must clear_page_tag_ref() before freeing to avoid warnings from + * alloc_tag_sub_check(). + */ +#define ALLOC_NO_CODETAG 0x1000 + +/* Flags that allow allocations below the min watermark. */ +#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) + +/* + * Structure for holding the mostly immutable allocation parameters passed + * between functions involved in allocations, including the alloc_pages* + * family of functions. + * + * nodemask, migratetype and highest_zoneidx are initialized only once in + * __alloc_pages() and then never change. + * + * zonelist, preferred_zone and highest_zoneidx are set first in + * __alloc_pages() for the fast path, and might be later changed + * in __alloc_pages_slowpath(). All other functions pass the whole structure + * by a const pointer. + */ +struct alloc_context { + struct zonelist *zonelist; + const nodemask_t *nodemask; + struct zoneref *preferred_zoneref; + int migratetype; + + /* + * highest_zoneidx represents highest usable zone index of + * the allocation request. Due to the nature of the zone, + * memory on lower zone than the highest_zoneidx will be + * protected by lowmem_reserve[highest_zoneidx]. + * + * highest_zoneidx is also used by reclaim/compaction to limit + * the target zone since higher zone than this index cannot be + * usable for this allocation request. + */ + enum zone_type highest_zoneidx; + bool spread_dirty_pages; + /* Only flags that are global to the whole allocation go here. */ + unsigned int alloc_flags; +}; + +/* + * This function returns the order of a free page in the buddy system. In + * general, page_zone(page)->lock must be held by the caller to prevent the + * page from being allocated in parallel and returning garbage as the order. + * If a caller does not hold page_zone(page)->lock, it must guarantee that the + * page cannot be allocated or merged in parallel. Alternatively, it must + * handle invalid values gracefully, and use buddy_order_unsafe() below. + */ +static inline unsigned int buddy_order(struct page *page) +{ + /* PageBuddy() must be checked by the caller */ + return page_private(page); +} + +/* + * Like buddy_order(), but for callers who cannot afford to hold the zone lock. + * PageBuddy() should be checked first by the caller to minimize race window, + * and invalid values must be handled gracefully. + * + * READ_ONCE is used so that if the caller assigns the result into a local + * variable and e.g. tests it for valid range before using, the compiler cannot + * decide to remove the variable and inline the page_private(page) multiple + * times, potentially observing different values in the tests and the actual + * use of the result. + */ +#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) + +/* + * This function checks whether a page is free && is the buddy + * we can coalesce a page and its buddy if + * (a) the buddy is not in a hole (check before calling!) && + * (b) the buddy is in the buddy system && + * (c) a page and its buddy have the same order && + * (d) a page and its buddy are in the same zone. + * + * For recording whether a page is in the buddy system, we set PageBuddy. + * Setting, clearing, and testing PageBuddy is serialized by zone->lock. + * + * For recording page's order, we use page_private(page). + */ +static inline bool page_is_buddy(struct page *page, struct page *buddy, + unsigned int order) +{ + if (!page_is_guard(buddy) && !PageBuddy(buddy)) + return false; + + if (buddy_order(buddy) != order) + return false; + + /* + * zone check is done late to avoid uselessly calculating + * zone/node ids for pages that could never merge. + */ + if (page_zone_id(page) != page_zone_id(buddy)) + return false; + + VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); + + return true; +} + +/* + * Locate the struct page for both the matching buddy in our + * pair (buddy1) and the combined O(n+1) page they form (page). + * + * 1) Any buddy B1 will have an order O twin B2 which satisfies + * the following equation: + * B2 = B1 ^ (1 << O) + * For example, if the starting buddy (buddy2) is #8 its order + * 1 buddy is #10: + * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 + * + * 2) Any buddy B will have an order O+1 parent P which + * satisfies the following equation: + * P = B & ~(1 << O) + * + * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER + */ +static inline unsigned long +__find_buddy_pfn(unsigned long page_pfn, unsigned int order) +{ + return page_pfn ^ (1 << order); +} + +/* + * Find the buddy of @page and validate it. + * @page: The input page + * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the + * function is used in the performance-critical __free_one_page(). + * @order: The order of the page + * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to + * page_to_pfn(). + * + * The found buddy can be a non PageBuddy, out of @page's zone, or its order is + * not the same as @page. The validation is necessary before use it. + * + * Return: the found buddy page or NULL if not found. + */ +static inline struct page *find_buddy_page_pfn(struct page *page, + unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) +{ + unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); + struct page *buddy; + + buddy = page + (__buddy_pfn - pfn); + if (buddy_pfn) + *buddy_pfn = __buddy_pfn; + + if (page_is_buddy(page, buddy, order)) + return buddy; + return NULL; +} + +extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, + unsigned long end_pfn, struct zone *zone); + +static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, + unsigned long end_pfn, struct zone *zone) +{ + if (zone->contiguous) + return pfn_to_page(start_pfn); + + return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); +} + +extern void __free_pages_core(struct page *page, unsigned int order, + enum meminit_context context); + +void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags, + unsigned int alloc_flags); +extern bool free_pages_prepare(struct page *page, unsigned int order); + +extern int user_min_free_kbytes; + +struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, + nodemask_t *nodemask, unsigned int alloc_flags); +#define __alloc_frozen_pages(...) \ + alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) +void free_frozen_pages(struct page *page, unsigned int order); +void free_unref_folios(struct folio_batch *fbatch); + +#ifdef CONFIG_NUMA +struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); +#else +static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) +{ + return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, + ALLOC_DEFAULT); +} +#endif + +#define alloc_frozen_pages(...) \ + alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) + +struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); +#define alloc_frozen_pages_nolock(...) \ + alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) +void free_frozen_pages_nolock(struct page *page, unsigned int order); + +struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, + nodemask_t *nodemask, unsigned int alloc_flags); +#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) + +extern void zone_pcp_reset(struct zone *zone); +extern void zone_pcp_disable(struct zone *zone); +extern void zone_pcp_enable(struct zone *zone); +extern void zone_pcp_init(struct zone *zone); + +enum fallback_result { + /* Found suitable migratetype, *mt_out is valid. */ + FALLBACK_FOUND, + /* No fallback found in requested order. */ + FALLBACK_EMPTY, + /* Passed @claimable, but claiming whole block is a bad idea. */ + FALLBACK_NOCLAIM, +}; +enum fallback_result +find_suitable_fallback(struct free_area *area, unsigned int order, + int migratetype, bool claimable, int *mt_out); + +static inline bool free_area_empty(struct free_area *area, int migratetype) +{ + return list_empty(&area->free_list[migratetype]); +} + +/* Convert GFP flags to their corresponding migrate type */ +#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) +#define GFP_MOVABLE_SHIFT 3 + +static inline int gfp_migratetype(const gfp_t gfp_flags) +{ + VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); + BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); + BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); + BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); + BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> + GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); + + if (unlikely(page_group_by_mobility_disabled)) + return MIGRATE_UNMOVABLE; + + /* Group based on mobility */ + return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; +} +#undef GFP_MOVABLE_MASK +#undef GFP_MOVABLE_SHIFT + +bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); +void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); +void drain_all_pages(struct zone *zone); + +void page_alloc_init_cpuhp(void); +void page_alloc_sysctl_init(void); + +#endif /* __MM_PAGE_ALLOC_H */ diff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c index d2423f30577e..e63efe78b7d4 100644 --- a/mm/page_frag_cache.c +++ b/mm/page_frag_cache.c @@ -18,7 +18,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/page_frag_cache.h> -#include "internal.h" +#include "page_alloc.h" static unsigned long encoded_page_create(struct page *page, unsigned int order, bool pfmemalloc) @@ -57,10 +57,10 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER, - numa_mem_id(), NULL); + numa_mem_id(), NULL, ALLOC_DEFAULT); #endif if (unlikely(!page)) { - page = __alloc_pages(gfp, 0, numa_mem_id(), NULL); + page = __alloc_pages(gfp, 0, numa_mem_id(), NULL, ALLOC_DEFAULT); order = 0; } diff --git a/mm/page_io.c b/mm/page_io.c index b23f494fcc83..e4fa7ffffe8b 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -28,54 +28,6 @@ #include "swap.h" #include "swap_table.h" -static void __end_swap_bio_write(struct bio *bio) -{ - struct folio *folio = bio_first_folio_all(bio); - - if (bio->bi_status) { - /* - * We failed to write the page out to swap-space. - * Re-dirty the page in order to avoid it being reclaimed. - * Also print a dire warning that things will go BAD (tm) - * very quickly. - * - * Also clear PG_reclaim to avoid folio_rotate_reclaimable() - */ - folio_mark_dirty(folio); - pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); - folio_clear_reclaim(folio); - } - folio_end_writeback(folio); -} - -static void end_swap_bio_write(struct bio *bio) -{ - __end_swap_bio_write(bio); - bio_put(bio); -} - -static void __end_swap_bio_read(struct bio *bio) -{ - struct folio *folio = bio_first_folio_all(bio); - - if (bio->bi_status) { - pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); - } else { - folio_mark_uptodate(folio); - } - folio_unlock(folio); -} - -static void end_swap_bio_read(struct bio *bio) -{ - __end_swap_bio_read(bio); - bio_put(bio); -} - int generic_swapfile_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *span) @@ -248,7 +200,7 @@ static void swap_zeromap_folio_clear(struct folio *folio) * We may have stale swap cache pages in memory: notice * them here and get rid of the unnecessary final write. */ -int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) +int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { int ret = 0; @@ -295,39 +247,44 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) } rcu_read_unlock(); - __swap_writepage(folio, swap_plug); + __swap_writepage(ctx, folio); return 0; out_unlock: folio_unlock(folio); return ret; } -static inline void count_swpout_vm_event(struct folio *folio) +#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) +static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (unlikely(folio_test_pmd_mappable(folio))) { - count_memcg_folio_events(folio, THP_SWPOUT, 1); - count_vm_event(THP_SWPOUT); + return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys); +} + +static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +{ + bool can_merge = true; + + if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio)) + return false; + if (folio_memcg_charged(folio)) { + rcu_read_lock(); + if (folio_memcg_blkg_css(folio) != + folio_memcg_blkg_css(prev_folio)) + can_merge = false; + rcu_read_unlock(); } -#endif - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); - count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio)); - count_vm_events(PSWPOUT, folio_nr_pages(folio)); + return can_merge; } -#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) { struct cgroup_subsys_state *css; - struct mem_cgroup *memcg; if (!folio_memcg_charged(folio)) return; - rcu_read_lock(); - memcg = folio_memcg(folio); - css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); - if (!css || !css_tryget(css)) + css = folio_memcg_blkg_css(folio); + if (css && !css_tryget(css)) css = NULL; rcu_read_unlock(); @@ -336,11 +293,18 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) css_put(css); } #else +static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +{ + return true; +} #define bio_associate_blkg_from_page(bio, folio) do { } while (0) #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ struct swap_iocb { - struct kiocb iocb; + union { + struct kiocb iocb; + struct bio bio; + }; struct bio_vec bvecs[SWAP_CLUSTER_MAX]; int nr_bvecs; int len; @@ -360,168 +324,65 @@ int sio_pool_init(void) return 0; } -static void sio_write_complete(struct kiocb *iocb, long ret) +static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio, + int rw) { - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); - struct page *page = sio->bvecs[0].bv_page; - int p; - - if (ret != sio->len) { - /* - * In the case of swap-over-nfs, this can be a - * temporary failure if the system has limited - * memory for allocating transmit buffers. - * Mark the page dirty and avoid - * folio_rotate_reclaimable but rate-limit the - * messages. - */ - pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", - ret, swap_dev_pos(page_swap_entry(page))); - for (p = 0; p < sio->nr_bvecs; p++) { - page = sio->bvecs[p].bv_page; - set_page_dirty(page); - ClearPageReclaim(page); - } - } - - for (p = 0; p < sio->nr_bvecs; p++) - end_page_writeback(sio->bvecs[p].bv_page); + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1]; + struct folio *prev_folio = bvec_folio(last_bv); + size_t prev_folio_size = folio_size(prev_folio); - mempool_free(sio, sio_pool); + if (ctx->sis != sis) + return false; + return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw); } -static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) +static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw) { - struct swap_iocb *sio = swap_plug ? *swap_plug : NULL; struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct file *swap_file = sis->swap_file; - loff_t pos = swap_dev_pos(folio->swap); - - count_swpout_vm_event(folio); - folio_start_writeback(folio); - folio_unlock(folio); - if (sio) { - if (sio->iocb.ki_filp != swap_file || - sio->iocb.ki_pos + sio->len != pos) { - swap_write_unplug(sio); - sio = NULL; - } + struct swap_iocb *sio = ctx->sio; + + if (sio && !swap_can_merge(ctx, folio, rw)) { + if (rw == WRITE) + swap_write_submit(ctx); + else + swap_read_submit(ctx); + sio = ctx->sio; } + if (!sio) { - sio = mempool_alloc(sio_pool, GFP_NOIO); - init_sync_kiocb(&sio->iocb, swap_file); - sio->iocb.ki_complete = sio_write_complete; - sio->iocb.ki_pos = pos; + ctx->sis = sis; + ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO); sio->nr_bvecs = 0; sio->len = 0; } bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0); sio->len += folio_size(folio); - sio->nr_bvecs += 1; - if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !swap_plug) { - swap_write_unplug(sio); - sio = NULL; + if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs)) { + if (rw == WRITE) + swap_write_submit(ctx); + else + swap_read_submit(ctx); } - if (swap_plug) - *swap_plug = sio; -} - -static void swap_writepage_bdev_sync(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio_vec bv; - struct bio bio; - - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_WRITE | REQ_SWAP); - bio.bi_iter.bi_sector = swap_folio_sector(folio); - bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); - - bio_associate_blkg_from_page(&bio, folio); - count_swpout_vm_event(folio); - - folio_start_writeback(folio); - folio_unlock(folio); - - submit_bio_wait(&bio); - __end_swap_bio_write(&bio); } -static void swap_writepage_bdev_async(struct folio *folio, - struct swap_info_struct *sis) +void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) { - struct bio *bio; + VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); - bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE | REQ_SWAP, GFP_NOIO); - bio->bi_iter.bi_sector = swap_folio_sector(folio); - bio->bi_end_io = end_swap_bio_write; - bio_add_folio_nofail(bio, folio, folio_size(folio), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (unlikely(folio_test_pmd_mappable(folio))) { + count_memcg_folio_events(folio, THP_SWPOUT, 1); + count_vm_event(THP_SWPOUT); + } +#endif + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); + count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio)); + count_vm_events(PSWPOUT, folio_nr_pages(folio)); - bio_associate_blkg_from_page(bio, folio); - count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); -} - -void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) -{ - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - - VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); - /* - * ->flags can be updated non-atomically, - * but that will never affect SWP_FS_OPS, so the data_race - * is safe. - */ - if (data_race(sis->flags & SWP_FS_OPS)) - swap_writepage_fs(folio, swap_plug); - /* - * ->flags can be updated non-atomically, - * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race - * is safe. - */ - else if (data_race(sis->flags & SWP_SYNCHRONOUS_IO)) - swap_writepage_bdev_sync(folio, sis); - else - swap_writepage_bdev_async(folio, sis); -} - -void swap_write_unplug(struct swap_iocb *sio) -{ - struct iov_iter from; - struct address_space *mapping = sio->iocb.ki_filp->f_mapping; - int ret; - - iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw(&sio->iocb, &from); - if (ret != -EIOCBQUEUED) - sio_write_complete(&sio->iocb, ret); -} - -static void sio_read_complete(struct kiocb *iocb, long ret) -{ - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); - int p; - - if (ret == sio->len) { - for (p = 0; p < sio->nr_bvecs; p++) { - struct folio *folio = bvec_folio(&sio->bvecs[p]); - - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - folio_mark_uptodate(folio); - folio_unlock(folio); - } - count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); - } else { - for (p = 0; p < sio->nr_bvecs; p++) { - struct folio *folio = bvec_folio(&sio->bvecs[p]); - - folio_unlock(folio); - } - pr_alert_ratelimited("Read-error on swap-device\n"); - } - mempool_free(sio, sio_pool); + swap_add_folio(ctx, folio, WRITE); } /* @@ -587,78 +448,7 @@ static bool swap_read_folio_zeromap(struct folio *folio) return true; } -static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug) -{ - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_iocb *sio = NULL; - loff_t pos = swap_dev_pos(folio->swap); - - if (plug) - sio = *plug; - if (sio) { - if (sio->iocb.ki_filp != sis->swap_file || - sio->iocb.ki_pos + sio->len != pos) { - swap_read_unplug(sio); - sio = NULL; - } - } - if (!sio) { - sio = mempool_alloc(sio_pool, GFP_KERNEL); - init_sync_kiocb(&sio->iocb, sis->swap_file); - sio->iocb.ki_pos = pos; - sio->iocb.ki_complete = sio_read_complete; - sio->nr_bvecs = 0; - sio->len = 0; - } - bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0); - sio->len += folio_size(folio); - sio->nr_bvecs += 1; - if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !plug) { - swap_read_unplug(sio); - sio = NULL; - } - if (plug) - *plug = sio; -} - -static void swap_read_folio_bdev_sync(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio_vec bv; - struct bio bio; - - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ); - bio.bi_iter.bi_sector = swap_folio_sector(folio); - bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); - /* - * Keep this task valid during swap readpage because the oom killer may - * attempt to access it in the page fault retry time check. - */ - get_task_struct(current); - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); - __end_swap_bio_read(&bio); - put_task_struct(current); -} - -static void swap_read_folio_bdev_async(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio *bio; - - bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL); - bio->bi_iter.bi_sector = swap_folio_sector(folio); - bio->bi_end_io = end_swap_bio_read; - bio_add_folio_nofail(bio, folio, folio_size(folio), 0); - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); -} - -void swap_read_folio(struct folio *folio, struct swap_iocb **plug) +void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) { struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO; @@ -691,14 +481,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug) /* We have to read from slower devices. Increase zswap protection. */ zswap_folio_swapin(folio); - - if (data_race(sis->flags & SWP_FS_OPS)) { - swap_read_folio_fs(folio, plug); - } else if (synchronous) { - swap_read_folio_bdev_sync(folio, sis); - } else { - swap_read_folio_bdev_async(folio, sis); - } + swap_add_folio(ctx, folio, READ); finish: if (workingset) { @@ -708,14 +491,231 @@ finish: delayacct_swapin_end(); } -void __swap_read_unplug(struct swap_iocb *sio) +static void swap_write_end(struct swap_iocb *sio, bool failed) +{ + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + struct page *page = sio->bvecs[p].bv_page; + + if (failed) { + set_page_dirty(page); + ClearPageReclaim(page); + } + end_page_writeback(page); + } + mempool_free(sio, sio_pool); +} + +static void swap_fs_write_complete(struct kiocb *iocb, long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); + bool failed = ret != sio->len; + + if (failed) { + struct page *page = sio->bvecs[0].bv_page; + + /* + * In the case of swap-over-nfs, this can be a temporary failure + * if the system has limited memory for allocating transmit + * buffers. Mark the page dirty and avoid + * folio_rotate_reclaimable but rate-limit the messages. + */ + pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", + ret, swap_dev_pos(page_swap_entry(page))); + } + + swap_write_end(sio, failed); +} + +static void end_swap_bio_write(struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); + bool failed = !!bio->bi_status; + + if (failed) + pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); + bio_uninit(bio); + swap_write_end(sio, failed); +} + +static void swap_read_end(struct swap_iocb *sio, bool failed) +{ + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + struct folio *folio = bvec_folio(&sio->bvecs[p]); + + if (!failed) { + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); + count_memcg_folio_events(folio, PSWPIN, + folio_nr_pages(folio)); + folio_mark_uptodate(folio); + } + folio_unlock(folio); + } + + if (!failed) + count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); + + mempool_free(sio, sio_pool); +} + +static void swap_fs_read_complete(struct kiocb *iocb, long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); + bool failed = ret != sio->len; + + if (failed) + pr_alert_ratelimited("Read-error on swap-device\n"); + swap_read_end(sio, failed); +} + +static void swap_bio_read_end_io(struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); + bool failed = !!bio->bi_status; + + if (failed) + pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); + bio_uninit(bio); + swap_read_end(sio, failed); +} + +static void swap_bdev_submit_write(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct bio *bio = &sio->bio; + + bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs), + REQ_OP_WRITE | REQ_SWAP); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); + bio_associate_blkg_from_page(bio, bio_first_folio_all(bio)); + + if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { + submit_bio_wait(bio); + end_swap_bio_write(bio); + } else { + bio->bi_end_io = end_swap_bio_write; + submit_bio(bio); + } +} + +static void swap_bdev_submit_read(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct bio *bio = &sio->bio; + + bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs), + REQ_OP_READ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); + + if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { + /* + * Keep this task valid during swap readpage because the oom + * killer may attempt to access it in the page fault retry + * time check. + */ + get_task_struct(current); + submit_bio_wait(bio); + swap_bio_read_end_io(bio); + put_task_struct(current); + } else { + bio->bi_end_io = swap_bio_read_end_io; + submit_bio(bio); + } +} + +static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw) { - struct iov_iter from; - struct address_space *mapping = sio->iocb.ki_filp->f_mapping; + if (swap_folio_sector(folio) != + swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT)) + return false; + if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio)) + return false; + return true; +} + +const struct swap_ops swap_bdev_ops = { + .submit_write = swap_bdev_submit_write, + .submit_read = swap_bdev_submit_read, + .can_merge = swap_bdev_can_merge, +}; + +static void swap_fs_submit(struct swap_io_ctx *ctx, int rw) +{ + struct swap_iocb *sio = ctx->sio; + struct iov_iter iter; int ret; - iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw(&sio->iocb, &from); + init_sync_kiocb(&sio->iocb, ctx->sis->swap_file); + sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap); + if (rw == WRITE) + sio->iocb.ki_complete = swap_fs_write_complete; + else + sio->iocb.ki_complete = swap_fs_read_complete; + + iov_iter_bvec(&iter, rw == WRITE ? ITER_SOURCE : ITER_DEST, + sio->bvecs, sio->nr_bvecs, sio->len); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw(&sio->iocb, &iter); if (ret != -EIOCBQUEUED) - sio_read_complete(&sio->iocb, ret); + sio->iocb.ki_complete(&sio->iocb, ret); +} + +static void swap_fs_submit_write(struct swap_io_ctx *ctx) +{ + swap_fs_submit(ctx, WRITE); +} + +static void swap_fs_submit_read(struct swap_io_ctx *ctx) +{ + swap_fs_submit(ctx, READ); +} + +static bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw) +{ + return swap_dev_pos(folio->swap) == + swap_dev_pos(prev_folio->swap) + prev_folio_size; +} + +static const struct swap_ops swap_fs_ops = { + .flags = SWAP_OPS_F_REQUIRE_NOFS, + .submit_write = swap_fs_submit_write, + .submit_read = swap_fs_submit_read, + .can_merge = swap_fs_can_merge, +}; + +int swap_fs_activate(struct swap_info_struct *sis) +{ + sis->ops = &swap_fs_ops; + return add_swap_extent(sis, 0, sis->max, 0); +} +EXPORT_SYMBOL_GPL(swap_fs_activate); + +void swap_write_submit(struct swap_io_ctx *ctx) +{ + if (!ctx->sio) + return; + count_vm_events(NRSWPOUT, 1); + ctx->sis->ops->submit_write(ctx); + ctx->sio = NULL; + ctx->sis = NULL; +} + +void swap_read_submit(struct swap_io_ctx *ctx) +{ + if (!ctx->sio) + return; + count_vm_events(NRSWPIN, 1); + ctx->sis->ops->submit_read(ctx); + ctx->sio = NULL; + ctx->sis = NULL; } diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 32ce8a7d9df3..e5dfc7bf4944 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -11,6 +11,7 @@ #include <linux/page_owner.h> #include <linux/migrate.h> #include "internal.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include <trace/events/page_isolation.h> diff --git a/mm/page_owner.c b/mm/page_owner.c index 2dddcb6510aa..fbbda7ba914b 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -13,7 +13,7 @@ #include <linux/memcontrol.h> #include <linux/sched/clock.h> -#include "internal.h" +#include "page_alloc.h" /* * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) @@ -54,6 +54,24 @@ struct stack_print_ctx { u8 flags; }; +enum page_owner_print_mode { + PAGE_OWNER_PRINT_STACK, + PAGE_OWNER_PRINT_HANDLE, + PAGE_OWNER_PRINT_STACK_HANDLE, +}; + +static const char * const page_owner_print_mode_strings[] = { + [PAGE_OWNER_PRINT_STACK] = "stack", + [PAGE_OWNER_PRINT_HANDLE] = "handle", + [PAGE_OWNER_PRINT_STACK_HANDLE] = "stack_handle", +}; + +struct page_owner_filter_state { + enum page_owner_print_mode print_mode; + nodemask_t nid_filter; + bool nid_filter_enabled; +}; + static bool page_owner_enabled __initdata; DEFINE_STATIC_KEY_FALSE(page_owner_inited); @@ -339,13 +357,13 @@ noinline void __set_page_owner(struct page *page, unsigned short order, depot_stack_handle_t handle; handle = save_stack(gfp_mask); - __update_page_owner_handle(page, handle, order, gfp_mask, -1, + __update_page_owner_handle(page, handle, order, gfp_mask, MR_NEVER, ts_nsec, current->pid, current->tgid, current->comm); inc_stack_record_count(handle, gfp_mask, 1 << order); } -void __folio_set_owner_migrate_reason(struct folio *folio, int reason) +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { struct page_ext *page_ext = page_ext_get(&folio->page); struct page_owner *page_owner; @@ -422,6 +440,39 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old) rcu_read_unlock(); } +/* + * Check if a page is a buddy page and advance @pfn past the entire buddy block. + * This safely reads the buddy order without the zone lock, which may cause us + * to skip less than the full buddy block, but that is acceptable for page owner + * iteration purposes. + * + * The lockless read of buddy_order_unsafe() can also return a garbage order if + * the page is concurrently allocated and PageBuddy is cleared between the check + * and the read. Clamp the advance at the next MAX_ORDER_NR_PAGES boundary so + * that a bogus order cannot carry @pfn into an unvalidated memory section, + * which would break callers that rely on boundary-aligned pfn_valid() checks. + * + * Return: true if the page was skipped (caller should continue its loop), + * false if the page is not a buddy page and should be processed normally. + */ +static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) +{ + unsigned long order; + + if (!PageBuddy(page)) + return false; + + order = buddy_order_unsafe(page); + if (order <= MAX_PAGE_ORDER) { + unsigned long new_pfn = *pfn + (1UL << order); + unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES); + + *pfn = min(new_pfn, boundary) - 1; + } + + return true; +} + void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { @@ -461,14 +512,8 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, if (page_zone(page) != zone) continue; - if (PageBuddy(page)) { - unsigned long freepage_order; - - freepage_order = buddy_order_unsafe(page); - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -505,14 +550,15 @@ ext_put_continue: seq_putc(m, '\n'); } +#ifdef CONFIG_MEMCG /* * Looking for memcg information and print it out */ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, struct page *page) { -#ifdef CONFIG_MEMCG unsigned long memcg_data; + struct obj_cgroup *objcg; struct mem_cgroup *memcg; bool online; char name[80]; @@ -522,11 +568,14 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, if (!memcg_data || PageTail(page)) goto out_unlock; - if (memcg_data & MEMCG_DATA_OBJEXTS) + if (memcg_data & MEMCG_DATA_OBJEXTS) { ret += scnprintf(kbuf + ret, count - ret, "Slab cache page\n"); + goto out_unlock; + } - memcg = page_memcg_check(page); + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); + memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; if (!memcg) goto out_unlock; @@ -534,29 +583,39 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, cgroup_name(memcg->css.cgroup, name, sizeof(name)); ret += scnprintf(kbuf + ret, count - ret, "Charged %sto %smemcg %s\n", - PageMemcgKmem(page) ? "(via objcg) " : "", + (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "", online ? "" : "offline ", name); out_unlock: rcu_read_unlock(); -#endif /* CONFIG_MEMCG */ return ret; } +#else +static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, + struct page *page) +{ + return ret; +} +#endif static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, struct page *page, struct page_owner *page_owner, - depot_stack_handle_t handle) + depot_stack_handle_t handle, + struct page_owner_filter_state *state) { int ret, pageblock_mt, page_mt; char *kbuf; + enum page_owner_print_mode print_mode; count = min_t(size_t, count, PAGE_SIZE); kbuf = kmalloc(count, GFP_KERNEL); if (!kbuf) return -ENOMEM; + print_mode = state->print_mode; + ret = scnprintf(kbuf, count, "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n", page_owner->order, page_owner->gfp_mask, @@ -575,11 +634,20 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn, migratetype_names[pageblock_mt], &page->flags.f); - ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); - if (ret >= count) - goto err; + if (print_mode != PAGE_OWNER_PRINT_HANDLE) { + ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); + if (ret >= count) + goto err; + } + + if (print_mode != PAGE_OWNER_PRINT_STACK) { + ret += scnprintf(kbuf + ret, count - ret, "handle: %u\n", + handle); + if (ret >= count) + goto err; + } - if (page_owner->last_migrate_reason != -1) { + if (page_owner->last_migrate_reason != MR_NEVER) { ret += scnprintf(kbuf + ret, count - ret, "Page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); @@ -630,10 +698,10 @@ void __dump_page_owner(const struct page *page) else pr_alert("page_owner tracks the page as freed\n"); - pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", + pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu\n", page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, page_owner->pid, page_owner->tgid, page_owner->comm, - page_owner->ts_nsec, page_owner->free_ts_nsec); + page_owner->ts_nsec); handle = READ_ONCE(page_owner->handle); if (!handle) @@ -645,12 +713,13 @@ void __dump_page_owner(const struct page *page) if (!handle) { pr_alert("page_owner free stack trace missing\n"); } else { - pr_alert("page last free pid %d tgid %d stack trace:\n", - page_owner->free_pid, page_owner->free_tgid); + pr_alert("page last free pid %d tgid %d ts %llu stack trace:\n", + page_owner->free_pid, page_owner->free_tgid, + page_owner->free_ts_nsec); stack_depot_print(handle); } - if (page_owner->last_migrate_reason != -1) + if (page_owner->last_migrate_reason != MR_NEVER) pr_alert("page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); page_ext_put(page_ext); @@ -664,6 +733,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) struct page_ext *page_ext; struct page_owner *page_owner; depot_stack_handle_t handle; + struct page_owner_filter_state *state = file->private_data; if (!static_branch_unlikely(&page_owner_inited)) return -EINVAL; @@ -697,13 +767,8 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) } page = pfn_to_page(pfn); - if (PageBuddy(page)) { - unsigned long freepage_order = buddy_order_unsafe(page); - - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } page_ext = page_ext_get(page); if (unlikely(!page_ext)) @@ -740,15 +805,31 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) if (!handle) goto ext_put_continue; + if (state->nid_filter_enabled) { + int nid; + memdesc_flags_t page_flags = READ_ONCE(page->flags); + + /* + * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid + * VM_BUG_ON when accessing poisoned pages. + */ + if (page_flags.f == PAGE_POISON_PATTERN) + goto ext_put_continue; + nid = memdesc_nid(&page_flags); + if (!node_isset(nid, state->nid_filter)) + goto ext_put_continue; + } + /* Record the next PFN to read in the file offset */ *ppos = pfn + 1; page_owner_tmp = *page_owner; page_ext_put(page_ext); return print_page_owner(buf, count, pfn, page, - &page_owner_tmp, handle); + &page_owner_tmp, handle, state); ext_put_continue: page_ext_put(page_ext); + cond_resched(); } return 0; @@ -798,20 +879,8 @@ static void init_pages_in_zone(struct zone *zone) if (page_zone(page) != zone) continue; - /* - * To avoid having to grab zone->lock, be a little - * careful when reading buddy page order. The only - * danger is that we skip too much and potentially miss - * some early allocated pages, which is better than - * heavy lock contention. - */ - if (PageBuddy(page)) { - unsigned long order = buddy_order_unsafe(page); - - if (order > 0 && order <= MAX_PAGE_ORDER) - pfn += (1UL << order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -826,7 +895,7 @@ static void init_pages_in_zone(struct zone *zone) /* Found early allocated page */ __update_page_owner_handle(page, early_handle, 0, 0, - -1, local_clock(), current->pid, + MR_NEVER, local_clock(), current->pid, current->tgid, current->comm); count++; ext_put_continue: @@ -847,7 +916,113 @@ static void init_early_allocated_pages(void) init_pages_in_zone(zone); } +static int page_owner_open(struct inode *inode, struct file *file) +{ + struct page_owner_filter_state *state; + + state = kzalloc_obj(*state); + if (!state) + return -ENOMEM; + + state->print_mode = PAGE_OWNER_PRINT_STACK; + nodes_clear(state->nid_filter); + state->nid_filter_enabled = false; + file->private_data = state; + return 0; +} + +static int page_owner_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t page_owner_write(struct file *file, + const char __user *buf, + size_t count, loff_t *ppos) +{ + char *kbuf; + char *orig; + char *token; + int ret; + struct page_owner_filter_state *state = file->private_data; + enum page_owner_print_mode new_print_mode; + nodemask_t new_nid_filter; + bool new_nid_filter_enabled; + + /* + * Maximum input length for filter commands: + * - 32: print_mode command max length is 17 ("mode=stack_handle") + * with sufficient buffer + * - 6 * MAX_NUMNODES: worst case for nid list + * Worst case per node: ",NNNNN" (comma + 5-digit node number) = 6 bytes + */ + if (count > 32 + 6 * MAX_NUMNODES) + return -EINVAL; + + kbuf = memdup_user_nul(buf, count); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + orig = kbuf; + + new_print_mode = state->print_mode; + new_nid_filter = state->nid_filter; + new_nid_filter_enabled = state->nid_filter_enabled; + + while ((token = strsep(&kbuf, " \t\n")) != NULL) { + if (*token == '\0') + continue; + + if (!strncmp(token, "mode=", 5)) { + ret = sysfs_match_string(page_owner_print_mode_strings, + token + 5); + if (ret < 0) + goto out_free; + new_print_mode = ret; + } else if (!strncmp(token, "nid=", 4)) { + ret = nodelist_parse(token + 4, new_nid_filter); + if (ret < 0) + goto out_free; + + if (nodes_empty(new_nid_filter)) { + ret = -EINVAL; + goto out_free; + } + + /* + * We want to filter memory allocations by numa nodes, so make sure + * that the specified nodes have memory. + */ + if (!nodes_subset(new_nid_filter, node_states[N_MEMORY])) { + ret = -EINVAL; + goto out_free; + } + + new_nid_filter_enabled = true; + } else { + ret = -EINVAL; + goto out_free; + } + } + + /* Commit all filter changes */ + state->print_mode = new_print_mode; + state->nid_filter = new_nid_filter; + state->nid_filter_enabled = new_nid_filter_enabled; + + ret = count; + +out_free: + kfree(orig); + return ret; +} + static const struct file_operations page_owner_fops = { + .owner = THIS_MODULE, + .open = page_owner_open, + .release = page_owner_release, + .write = page_owner_write, .read = read_page_owner, .llseek = lseek_page_owner, }; @@ -887,7 +1062,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) return stack; } -static unsigned long page_owner_pages_threshold; +static unsigned long pages_threshold; static int stack_print(struct seq_file *m, void *v) { @@ -904,7 +1079,7 @@ static int stack_print(struct seq_file *m, void *v) nr_base_pages = refcount_read(&stack_record->count) - 1; if (ctx->flags & STACK_PRINT_FLAG_PAGES && - (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold)) + (nr_base_pages < 1 || nr_base_pages < pages_threshold)) return 0; if (ctx->flags & STACK_PRINT_FLAG_STACK) { @@ -926,16 +1101,16 @@ static void stack_stop(struct seq_file *m, void *v) { } -static const struct seq_operations page_owner_stack_op = { +static const struct seq_operations stack_op = { .start = stack_start, .next = stack_next, .stop = stack_stop, .show = stack_print }; -static int page_owner_stack_open(struct inode *inode, struct file *file) +static int stack_open(struct inode *inode, struct file *file) { - int ret = seq_open_private(file, &page_owner_stack_op, + int ret = seq_open_private(file, &stack_op, sizeof(struct stack_print_ctx)); if (!ret) { @@ -948,28 +1123,26 @@ static int page_owner_stack_open(struct inode *inode, struct file *file) return ret; } -static const struct file_operations page_owner_stack_fops = { - .open = page_owner_stack_open, +static const struct file_operations stack_fops = { + .open = stack_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_private, }; -static int page_owner_threshold_get(void *data, u64 *val) +static int threshold_get(void *data, u64 *val) { - *val = READ_ONCE(page_owner_pages_threshold); + *val = READ_ONCE(pages_threshold); return 0; } -static int page_owner_threshold_set(void *data, u64 val) +static int threshold_set(void *data, u64 val) { - WRITE_ONCE(page_owner_pages_threshold, val); + WRITE_ONCE(pages_threshold, val); return 0; } -DEFINE_SIMPLE_ATTRIBUTE(page_owner_threshold_fops, &page_owner_threshold_get, - &page_owner_threshold_set, "%llu"); - +DEFINE_SIMPLE_ATTRIBUTE(threshold_fops, &threshold_get, &threshold_set, "%llu\n"); static int __init pageowner_init(void) { @@ -980,22 +1153,22 @@ static int __init pageowner_init(void) return 0; } - debugfs_create_file("page_owner", 0400, NULL, NULL, &page_owner_fops); + debugfs_create_file("page_owner", 0600, NULL, NULL, &page_owner_fops); dir = debugfs_create_dir("page_owner_stacks", NULL); debugfs_create_file("show_stacks", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_HANDLE | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_stacks_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_HANDLE), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("count_threshold", 0600, dir, NULL, - &page_owner_threshold_fops); + &threshold_fops); return 0; } late_initcall(pageowner_init) diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 3e30731b940e..1cce8729696e 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -8,6 +8,7 @@ #include <linux/delay.h> #include <linux/scatterlist.h> +#include "page_alloc.h" #include "page_reporting.h" #include "internal.h" diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a11410..6ffc536359cd 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -199,8 +199,8 @@ static inline bool softleaf_cached_writable(softleaf_t entry) static void page_table_check_pte_flags(pte_t pte) { if (pte_present(pte)) { - WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte)); - } else if (pte_swp_uffd_wp(pte)) { + WARN_ON_ONCE(pte_uffd(pte) && pte_write(pte)); + } else if (pte_swp_uffd(pte)) { const softleaf_t entry = softleaf_from_pte(pte); WARN_ON_ONCE(softleaf_cached_writable(entry)); @@ -227,9 +227,9 @@ EXPORT_SYMBOL(__page_table_check_ptes_set); static inline void page_table_check_pmd_flags(pmd_t pmd) { if (pmd_present(pmd)) { - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) WARN_ON_ONCE(pmd_write(pmd)); - } else if (pmd_swp_uffd_wp(pmd)) { + } else if (pmd_swp_uffd(pmd)) { const softleaf_t entry = softleaf_from_pmd(pmd); WARN_ON_ONCE(softleaf_cached_writable(entry)); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index bac2eb5de63d..4e964545e5e8 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -107,7 +107,13 @@ again: static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) { unsigned long pfn; - pte_t ptent = ptep_get(pvmw->pte); + pte_t ptent; + + if (is_vm_hugetlb_page(pvmw->vma)) + ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address, + pvmw->pte); + else + ptent = ptep_get(pvmw->pte); if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -350,6 +356,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + const pgoff_t pgoff = page_pgoff(folio, page); struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -357,7 +364,10 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + if (folio_test_anon(folio)) + pvmw.address = vma_anon_address(vma, pgoff, 1); + else + pvmw.address = vma_filebacked_address(vma, pgoff, 1); if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a255..ed4860c01936 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -816,10 +816,10 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); - vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, + mapping_rmap_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ - vba = vma->vm_pgoff; + vba = vma_start_pgoff(vma); vea = vba + vma_pages(vma); cba = first_index; cba = max(cba, vba); diff --git a/mm/percpu-km.c b/mm/percpu-km.c index dc096b5a6ce4..65fd5580e447 100644 --- a/mm/percpu-km.c +++ b/mm/percpu-km.c @@ -94,8 +94,15 @@ static void pcpu_destroy_chunk(struct pcpu_chunk *chunk) pcpu_stats_chunk_dealloc(); trace_percpu_destroy_chunk(chunk->base_addr); - if (chunk->data) + if (chunk->data) { + struct page *pages = (struct page *)chunk->data; + int i; + + /* clear chunk info from each page before free them */ + for (i = 0; i < nr_pages; i++) + pcpu_set_page_chunk(pages + i, NULL); __free_pages(chunk->data, order_base_2(nr_pages)); + } pcpu_free_chunk(chunk); } diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 4f5937090590..509d8901835c 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -9,6 +9,7 @@ * This is the default chunk allocator. */ #include "internal.h" +#include "vmalloc.h" static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, unsigned int cpu, int page_idx) @@ -21,6 +22,8 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, /** * pcpu_get_pages - get temp pages array + * @gfp: allocation flags passed to the underlying allocator, 0 to only + * return the cached array * * Returns pointer to array of pointers to struct page which can be indexed * with pcpu_page_idx(). Note that there is only one array and accesses @@ -29,18 +32,23 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, * RETURNS: * Pointer to temp pages array on success. */ -static struct page **pcpu_get_pages(void) +static struct page **pcpu_get_pages(gfp_t gfp) { static struct page **pages; size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]); lockdep_assert_held(&pcpu_alloc_mutex); - if (!pages) - pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL); + if (!pages && gfp) + pages = pcpu_mem_zalloc(pages_size, gfp); return pages; } +static struct page **pcpu_get_pages_cached(void) +{ + return pcpu_get_pages(0); +} + /** * pcpu_free_pages - free pages which were allocated for @chunk * @chunk: chunk pages were allocated for @@ -191,10 +199,22 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk, } static int __pcpu_map_pages(unsigned long addr, struct page **pages, - int nr_pages) + int nr_pages, gfp_t gfp) { - return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), - PAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL); + unsigned int flags; + int ret; + + /* + * The vmalloc page table allocation path does not pass @gfp down + * explicitly. Apply the corresponding memalloc scope so implicit + * page table allocations preserve NOFS/NOIO constraints. + */ + flags = memalloc_apply_gfp_scope(gfp); + ret = vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), + PAGE_KERNEL, pages, PAGE_SHIFT, gfp); + memalloc_restore_scope(flags); + + return ret; } /** @@ -203,6 +223,7 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * @pages: pages array containing pages to be mapped * @page_start: page index of the first page to map * @page_end: page index of the last page to map + 1 + * @gfp: allocation flags passed to the underlying allocator * * For each cpu, map pages [@page_start,@page_end) into @chunk. The * caller is responsible for calling pcpu_post_map_flush() after all @@ -211,8 +232,8 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * This function is responsible for setting up whatever is necessary for * reverse lookup (addr -> chunk). */ -static int pcpu_map_pages(struct pcpu_chunk *chunk, - struct page **pages, int page_start, int page_end) +static int pcpu_map_pages(struct pcpu_chunk *chunk, struct page **pages, + int page_start, int page_end, gfp_t gfp) { unsigned int cpu, tcpu; int i, err; @@ -220,7 +241,7 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk, for_each_possible_cpu(cpu) { err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), &pages[pcpu_page_idx(cpu, page_start)], - page_end - page_start); + page_end - page_start, gfp); if (err < 0) goto err; @@ -271,21 +292,21 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk, * @chunk. * * CONTEXT: - * pcpu_alloc_mutex, does GFP_KERNEL allocation. + * pcpu_alloc_mutex, does @gfp allocation. */ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int page_start, int page_end, gfp_t gfp) { struct page **pages; - pages = pcpu_get_pages(); + pages = pcpu_get_pages(gfp); if (!pages) return -ENOMEM; if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp)) return -ENOMEM; - if (pcpu_map_pages(chunk, pages, page_start, page_end)) { + if (pcpu_map_pages(chunk, pages, page_start, page_end, gfp)) { pcpu_free_pages(chunk, pages, page_start, page_end); return -ENOMEM; } @@ -319,7 +340,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, * successful population attempt so the temp pages array must * be available now. */ - pages = pcpu_get_pages(); + pages = pcpu_get_pages_cached(); BUG_ON(!pages); /* unmap and free */ @@ -340,7 +361,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) return NULL; vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes, - pcpu_nr_groups, pcpu_atom_size); + pcpu_nr_groups, pcpu_atom_size, gfp); if (!vms) { pcpu_free_chunk(chunk); return NULL; diff --git a/mm/percpu.c b/mm/percpu.c index b0676b8054ed..a802d72c116f 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1726,9 +1726,8 @@ static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t s * @gfp: allocation flags * * Allocate percpu area of @size bytes aligned at @align. If @gfp doesn't - * contain %GFP_KERNEL, the allocation is atomic. If @gfp has __GFP_NOWARN - * then no warning will be triggered on invalid or failed allocation - * requests. + * allow blocking, the allocation is atomic. If @gfp has __GFP_NOWARN then no + * warning will be triggered on invalid or failed allocation requests. * * RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. @@ -1749,8 +1748,17 @@ void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, size_t bits, bit_align; gfp = current_gfp_context(gfp); - /* whitelisted flags that can be passed to the backing allocators */ - pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + /* + * Allowlisted flags that can be passed to the backing allocators. + * Backing allocations under pcpu_alloc_mutex must not recurse into + * IO/FS reclaim. Otherwise a GFP_KERNEL caller holding the mutex can + * block on reclaim while a GFP_NOIO/NOFS caller holding an IO/FS lock + * waits for the same mutex. + * + * Do not pass __GFP_NOFAIL. A small percpu allocation may need many + * backing pages, making nofail reclaim too costly under NOIO/NOFS. + */ + pcpu_gfp = gfp & (GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); is_atomic = !gfpflags_allow_blocking(gfp); do_warn = !(gfp & __GFP_NOWARN); @@ -3256,7 +3264,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t /* pte already populated, the following shouldn't fail */ rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages], - unit_pages); + unit_pages, GFP_KERNEL); if (rc < 0) panic("failed to map percpu area, err=%d\n", rc); diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9..b7ead3e9f064 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -211,7 +211,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) if (likely(!vma->anon_vma)) { vma->anon_vma = anon_vma; anon_vma_chain_assign(vma, avc, anon_vma); - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_rmap_tree_insert(avc, anon_vma); anon_vma->num_active_vmas++; allocated = NULL; avc = NULL; @@ -354,7 +354,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_rmap_tree_insert(avc, anon_vma); if (operation == VMA_OP_FORK) maybe_reuse_anon_vma(dst, anon_vma); } @@ -434,7 +434,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) anon_vma_chain_assign(vma, avc, anon_vma); /* Now let rmap see it. */ anon_vma_lock_write(anon_vma); - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_rmap_tree_insert(avc, anon_vma); anon_vma->parent->num_children++; anon_vma_unlock_write(anon_vma); @@ -499,7 +499,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); + anon_rmap_tree_remove(avc, anon_vma); /* * Leave empty anon_vmas on the list - we'll need @@ -865,14 +865,15 @@ unsigned long page_address_in_vma(const struct folio *folio, if (!vma->anon_vma || !anon_vma || vma->anon_vma->root != anon_vma->root) return -EFAULT; + /* KSM folios don't reach here because of the !anon_vma check */ + return vma_anon_address(vma, page_pgoff(folio, page), 1); } else if (!vma->vm_file) { return -EFAULT; } else if (vma->vm_file->f_mapping != folio->mapping) { return -EFAULT; } - /* KSM folios don't reach here because of the !anon_vma check */ - return vma_address(vma, page_pgoff(folio, page), 1); + return vma_filebacked_address(vma, page_pgoff(folio, page), 1); } /* @@ -907,7 +908,7 @@ out: struct folio_referenced_arg { int mapcount; int referenced; - vm_flags_t vm_flags; + vma_flags_t vma_flags; struct mem_cgroup *memcg; }; @@ -926,7 +927,7 @@ static bool folio_referenced_one(struct folio *folio, address = pvmw.address; nr = 1; - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { ptes++; pra->mapcount--; @@ -947,7 +948,7 @@ static bool folio_referenced_one(struct folio *folio, /* Restore the mlock which got missed */ mlock_vma_folio(folio, vma); page_vma_mapped_walk_done(&pvmw); - pra->vm_flags |= VM_LOCKED; + vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT); return false; /* To break the loop */ } @@ -1015,8 +1016,11 @@ static bool folio_referenced_one(struct folio *folio, referenced++; if (referenced) { + vma_flags_t vma_flags = vma->flags; + pra->referenced++; - pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; + vma_flags_clear(&vma_flags, VMA_LOCKED_BIT); + vma_flags_set_mask(&pra->vma_flags, vma_flags); } if (!pra->mapcount) @@ -1054,7 +1058,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) * @folio: The folio to test. * @is_locked: Caller holds lock on the folio. * @memcg: target memory cgroup - * @vm_flags: A combination of all the vma->vm_flags which referenced the folio. + * @vma_flags: A combination of all the vma->flags which referenced the folio. * * Quick test_and_clear_referenced for all mappings of a folio, * @@ -1062,7 +1066,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) * the function bailed out due to rmap lock contention. */ int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { bool we_locked = false; struct folio_referenced_arg pra = { @@ -1078,7 +1082,7 @@ int folio_referenced(struct folio *folio, int is_locked, }; VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio); - *vm_flags = 0; + vma_flags_clear_all(vma_flags); if (!pra.mapcount) return 0; @@ -1092,7 +1096,7 @@ int folio_referenced(struct folio *folio, int is_locked, } rmap_walk(folio, &rwc); - *vm_flags = pra.vm_flags; + vma_flags_set_mask(vma_flags, pra.vma_flags); if (we_locked) folio_unlock(folio); @@ -1239,6 +1243,7 @@ static bool mapping_wrprotect_range_one(struct folio *folio, .vma = vma, .address = address, .flags = PVMW_SYNC, + .is_anon_walk = false, }; state->cleaned += page_vma_mkclean_one(&pvmw); @@ -1316,12 +1321,13 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, .pgoff = pgoff, .vma = vma, .flags = PVMW_SYNC, + .is_anon_walk = false, }; if (invalid_mkclean_vma(vma, NULL)) return 0; - pvmw.address = vma_address(vma, pgoff, nr_pages); + pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages); VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); return page_vma_mkclean_one(&pvmw); @@ -1482,7 +1488,7 @@ static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, */ anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON; WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma); - folio->index = linear_page_index(vma, address); + folio->index = linear_virt_page_index(vma, address); } /** @@ -1509,8 +1515,8 @@ static void __page_check_anon_rmap(const struct folio *folio, */ VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, folio); - VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address), - page); + VM_BUG_ON_PAGE(page_pgoff(folio, page) != + linear_virt_page_index(vma, address), page); } static __always_inline void __folio_add_anon_rmap(struct folio *folio, @@ -1965,9 +1971,6 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, if (pte_unused(pte)) return 1; - if (userfaultfd_wp(vma)) - return 1; - /* * If unmap fails, we need to restore the ptes. To avoid accidentally * upgrading write permissions for ptes that were not originally @@ -1978,6 +1981,220 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +static bool try_to_unmap_hugetlb_one(struct folio *folio, + struct vm_area_struct *vma, unsigned long address, void *arg) +{ + DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + const unsigned long hsz = huge_page_size(hstate_vma(vma)); + const enum ttu_flags flags = (enum ttu_flags)(long)arg; + struct mm_struct *mm = vma->vm_mm; + struct mmu_notifier_range range; + bool ret = true; + pte_t pteval; + + /* + * The try_to_unmap() is only passed a hugetlb folio in the case + * where the hugetlb folio is poisoned. + */ + VM_WARN_ON_FOLIO(!folio_test_hwpoison(folio), folio); + VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON)); + + range.end = vma_address_end(&pvmw); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, + address, range.end); + adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); + mmu_notifier_invalidate_range_start(&range); + + /* There is only a single mapping in a VMA. */ + if (!page_vma_mapped_walk(&pvmw)) + goto range_end; + + VM_WARN_ON_ONCE(address != pvmw.address); + + pteval = huge_ptep_get(mm, address, pvmw.pte); + VM_WARN_ON_ONCE(!pte_present(pteval)); + VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio)); + + /* + * huge_pmd_unshare may unmap an entire PMD page. There is no way of + * knowing exactly which PMDs may be cached for this mm, so we must + * flush them all. start/end were already adjusted above to cover this + * range. + */ + flush_cache_range(vma, range.start, range.end); + + /* + * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode. + * Caller needs to explicitly do this outside rmap routines. + * + * We also must hold hugetlb vma_lock in write mode. Lock order dictates + * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and + * fail if unsuccessful. + */ + if (!folio_test_anon(folio)) { + struct mmu_gather tlb; + + VM_WARN_ON(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + ret = false; + goto walk_done; + } + + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); + /* + * The PMD table was unmapped, consequently unmapping + * the folio. + */ + goto walk_done; + } + hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); + } + pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); + if (huge_pte_dirty(pteval)) + folio_mark_dirty(folio); + + pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0))); + hugetlb_count_sub(folio_nr_pages(folio), mm); + set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz); + hugetlb_remove_rmap(folio); + folio_put_refs(folio, 1); + +walk_done: + page_vma_mapped_walk_done(&pvmw); +range_end: + mmu_notifier_invalidate_range_end(&range); + return ret; +} + +static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma, + struct folio *folio, unsigned long nr_pages) +{ + int ref_count, map_count; + + /* + * Synchronize with gup_pte_range(): + * - clear PTE; barrier; read refcount + * - inc refcount; barrier; read PTE + */ + smp_mb(); + + ref_count = folio_ref_count(folio); + map_count = folio_mapcount(folio); + + /* + * Order reads for page refcount and dirty flag + * (see comments in __remove_mapping()). + */ + smp_rmb(); + + if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { + /* + * redirtied either using the page table or a previously + * obtained GUP reference. + */ + folio_set_swapbacked(folio); + return false; + } + + /* + * Additional references could be due to GUP or speculative lookups. + * GUP users must mark the folio dirty if there was a modification. + * This folio cannot be reclaimed right now either way, so act just + * like nothing happened. We'll come back here later and detect if the + * folio was dirtied when the additional reference is gone. + */ + if (ref_count != 1 + map_count) + return false; + + add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages); + return true; +} + +static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte, + bool anon_exclusive) +{ + pte_t swp_pte = swp_entry_to_pte(entry); + + if (anon_exclusive) + swp_pte = pte_swp_mkexclusive(swp_pte); + + if (likely(pte_present(old_pte))) { + if (pte_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } else { + /* Device-exclusive entry */ + if (pte_swp_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_swp_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } + + return swp_pte; +} + +static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, + struct folio *folio, struct page *page, unsigned long address, + pte_t *ptep, pte_t pteval) +{ + const bool anon_exclusive = folio_test_anon(folio) && + PageAnonExclusive(page); + swp_entry_t entry = page_swap_entry(page); + struct mm_struct *mm = vma->vm_mm; + + if (folio_dup_swap(folio, page) < 0) + return false; + + /* + * arch_unmap_one() is expected to be a NOP on + * architectures where we could have PFN swap PTEs, + * so we'll not check/care. + */ + if (arch_unmap_one(mm, vma, address, pteval) < 0) { + folio_put_swap(folio, page); + return false; + } + + /* See folio_try_share_anon_rmap(): clear PTE first. */ + if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) { + folio_put_swap(folio, page); + return false; + } + + mm_prepare_for_swap_entries(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, ptep, + swp_pte_prepare(entry, pteval, anon_exclusive)); + return true; +} + +static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio, + struct page *page, unsigned long address, pte_t *ptep, + pte_t pteval, unsigned long nr_pages) +{ + /* + * Store the swap location in the pte. + * See handle_pte_fault() ... + */ + if (WARN_ON_ONCE(folio_test_swapbacked(folio) != + folio_test_swapcache(folio))) + return false; + + if (!folio_test_swapbacked(folio)) + return ttu_anon_lazyfree_folio(vma, folio, nr_pages); + + /* nr_pages > 1 not supported yet */ + return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep, + pteval); +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -1986,14 +2203,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); - bool anon_exclusive, ret = true; + bool ret = true; pte_t pteval; - struct page *subpage; + struct page *page; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long nr_pages = 1, end_addr; unsigned long pfn; - unsigned long hsz = 0; int ptes = 0; /* @@ -2007,8 +2223,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* * For THP, we have to assume the worse case ie pmd for invalidation. - * For hugetlb, it could be much worse if we need to do pud - * invalidation in the case of pmd sharing. * * Note that the folio can not be freed in this function as call of * try_to_unmap() must hold a reference on the folio. @@ -2016,17 +2230,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, range.end = vma_address_end(&pvmw); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, address, range.end); - if (folio_test_hugetlb(folio)) { - /* - * If sharing is possible, start and end will be adjusted - * accordingly. - */ - adjust_range_if_pmd_sharing_possible(vma, &range.start, - &range.end); - - /* We need the huge page size for set_huge_pte_at() */ - hsz = huge_page_size(hstate_vma(vma)); - } mmu_notifier_invalidate_range_start(&range); while (page_vma_mapped_walk(&pvmw)) { @@ -2095,77 +2298,27 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) { + pteval = huge_ptep_get(mm, address, pvmw.pte); + } else { + pteval = ptep_get(pvmw.pte); + } if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); - VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); } - subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; - anon_exclusive = folio_test_anon(folio) && - PageAnonExclusive(subpage); - - if (folio_test_hugetlb(folio)) { - bool anon = folio_test_anon(folio); - - /* - * The try_to_unmap() is only passed a hugetlb page - * in the case where the hugetlb page is poisoned. - */ - VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage); - /* - * huge_pmd_unshare may unmap an entire PMD page. - * There is no way of knowing exactly which PMDs may - * be cached for this mm, so we must flush them all. - * start/end were already adjusted above to cover this - * range. - */ - flush_cache_range(vma, range.start, range.end); + page = folio_page(folio, pfn - folio_pfn(folio)); - /* - * To call huge_pmd_unshare, i_mmap_rwsem must be - * held in write mode. Caller needs to explicitly - * do this outside rmap routines. - * - * We also must hold hugetlb vma_lock in write mode. - * Lock order dictates acquiring vma_lock BEFORE - * i_mmap_rwsem. We can only try lock here and fail - * if unsuccessful. - */ - if (!anon) { - struct mmu_gather tlb; - - VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); - if (!hugetlb_vma_trylock_write(vma)) - goto walk_abort; - - tlb_gather_mmu_vma(&tlb, vma); - if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { - hugetlb_vma_unlock_write(vma); - huge_pmd_unshare_flush(&tlb, vma); - tlb_finish_mmu(&tlb); - /* - * The PMD table was unmapped, - * consequently unmapping the folio. - */ - goto walk_done; - } - hugetlb_vma_unlock_write(vma); - tlb_finish_mmu(&tlb); - } - pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); - if (pte_dirty(pteval)) - folio_mark_dirty(folio); - } else if (likely(pte_present(pteval))) { + if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); end_addr = address + nr_pages * PAGE_SIZE; flush_cache_range(vma, address, end_addr); @@ -2195,21 +2348,17 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * we may want to replace a none pte with a marker pte if * it's file-backed, so we don't lose the tracking info. */ - pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); + cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval, + nr_pages); /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) { - pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); - if (folio_test_hugetlb(folio)) { - hugetlb_count_sub(folio_nr_pages(folio), mm); - set_huge_pte_at(mm, address, pvmw.pte, pteval, - hsz); - } else { - dec_mm_counter(mm, mm_counter(folio)); - set_pte_at(mm, address, pvmw.pte, pteval); - } + /* With TTU_HWPOISON, we only expect small folios here. */ + if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { + pteval = swp_entry_to_pte(make_hwpoison_entry(page)); + dec_mm_counter(mm, mm_counter(folio)); + set_pte_at(mm, address, pvmw.pte, pteval); } else if (likely(pte_present(pteval)) && pte_unused(pteval) && !userfaultfd_armed(vma)) { /* @@ -2224,109 +2373,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ dec_mm_counter(mm, mm_counter(folio)); } else if (folio_test_anon(folio)) { - swp_entry_t entry = page_swap_entry(subpage); - pte_t swp_pte; - /* - * Store the swap location in the pte. - * See handle_pte_fault() ... - */ - if (unlikely(folio_test_swapbacked(folio) != - folio_test_swapcache(folio))) { - WARN_ON_ONCE(1); - goto walk_abort; - } - - /* MADV_FREE page check */ - if (!folio_test_swapbacked(folio)) { - int ref_count, map_count; - - /* - * Synchronize with gup_pte_range(): - * - clear PTE; barrier; read refcount - * - inc refcount; barrier; read PTE - */ - smp_mb(); - - ref_count = folio_ref_count(folio); - map_count = folio_mapcount(folio); - - /* - * Order reads for page refcount and dirty flag - * (see comments in __remove_mapping()). - */ - smp_rmb(); - - if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { - /* - * redirtied either using the page table or a previously - * obtained GUP reference. - */ - set_ptes(mm, address, pvmw.pte, pteval, nr_pages); - folio_set_swapbacked(folio); - goto walk_abort; - } else if (ref_count != 1 + map_count) { - /* - * Additional reference. Could be a GUP reference or any - * speculative reference. GUP users must mark the folio - * dirty if there was a modification. This folio cannot be - * reclaimed right now either way, so act just like nothing - * happened. - * We'll come back here later and detect if the folio was - * dirtied when the additional reference is gone. - */ - set_ptes(mm, address, pvmw.pte, pteval, nr_pages); - goto walk_abort; - } - add_mm_counter(mm, MM_ANONPAGES, -nr_pages); - goto discard; - } - - if (folio_dup_swap(folio, subpage) < 0) { - set_pte_at(mm, address, pvmw.pte, pteval); + if (!ttu_anon_folio(vma, folio, page, address, + pvmw.pte, pteval, nr_pages)) { + set_ptes(mm, address, pvmw.pte, pteval, nr_pages); goto walk_abort; } - /* - * arch_unmap_one() is expected to be a NOP on - * architectures where we could have PFN swap PTEs, - * so we'll not check/care. - */ - if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - - /* See folio_try_share_anon_rmap(): clear PTE first. */ - if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - if (list_empty(&mm->mmlist)) { - spin_lock(&mmlist_lock); - if (list_empty(&mm->mmlist)) - list_add(&mm->mmlist, &init_mm.mmlist); - spin_unlock(&mmlist_lock); - } - dec_mm_counter(mm, MM_ANONPAGES); - inc_mm_counter(mm, MM_SWAPENTS); - swp_pte = swp_entry_to_pte(entry); - if (anon_exclusive) - swp_pte = pte_swp_mkexclusive(swp_pte); - if (likely(pte_present(pteval))) { - if (pte_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); - } else { - if (pte_swp_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); - } - set_pte_at(mm, address, pvmw.pte, swp_pte); + goto finish_unmap; } else { /* * This is a locked file-backed folio, @@ -2341,12 +2394,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } -discard: - if (unlikely(folio_test_hugetlb(folio))) { - hugetlb_remove_rmap(folio); - } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); - } +finish_unmap: + folio_remove_rmap_ptes(folio, page, nr_pages, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put_refs(folio, nr_pages); @@ -2394,7 +2443,8 @@ static int folio_not_mapped(struct folio *folio) void try_to_unmap(struct folio *folio, enum ttu_flags flags) { struct rmap_walk_control rwc = { - .rmap_one = try_to_unmap_one, + .rmap_one = folio_test_hugetlb(folio) ? + try_to_unmap_hugetlb_one : try_to_unmap_one, .arg = (void *)flags, .done = folio_not_mapped, .anon_lock = folio_lock_anon_vma_read, @@ -2477,7 +2527,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_restart(&pvmw); continue; } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES pmdval = pmdp_get(pvmw.pmd); if (likely(pmd_present(pmdval))) pfn = pmd_pfn(pmdval); @@ -2501,14 +2551,18 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) + pteval = huge_ptep_get(mm, address, pvmw.pte); + else + pteval = ptep_get(pvmw.pte); if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); @@ -2516,7 +2570,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2692,14 +2745,14 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { swp_pte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, @@ -2986,13 +3039,12 @@ static void rmap_walk_anon(struct folio *folio, pgoff_start = folio_pgoff(folio); pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; - anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, - pgoff_start, pgoff_end) { + anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; - unsigned long address = vma_address(vma, pgoff_start, + const unsigned long address = vma_anon_address(vma, pgoff_start, folio_nr_pages(folio)); - VM_BUG_ON_VMA(address == -EFAULT, vma); + VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma); cond_resched(); if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) @@ -3051,9 +3103,9 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, i_mmap_lock_read(mapping); } lookup: - vma_interval_tree_foreach(vma, &mapping->i_mmap, - pgoff_start, pgoff_end) { - unsigned long address = vma_address(vma, pgoff_start, nr_pages); + mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { + unsigned long address = vma_filebacked_address(vma, pgoff_start, + nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); diff --git a/mm/secretmem.c b/mm/secretmem.c index 4877c262cb1f..d29865075b6e 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -202,7 +202,7 @@ static struct file *secretmem_file_create(unsigned long flags) if (IS_ERR(file)) goto err_free_inode; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); mapping_set_unevictable(inode->i_mapping); inode->i_op = &secretmem_iops; diff --git a/mm/shmem.c b/mm/shmem.c index 9001aaf3b7b9..6641823bed16 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1042,6 +1042,8 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) struct inode *inode = file_inode(vma->vm_file); struct shmem_inode_info *info = SHMEM_I(inode); struct address_space *mapping = inode->i_mapping; + const pgoff_t pgoff = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); unsigned long swapped; /* Be careful as we don't hold info->lock */ @@ -1055,12 +1057,11 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) if (!swapped) return 0; - if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) + if (!pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; /* Here comes the more involved part */ - return shmem_partial_swap_usage(mapping, vma->vm_pgoff, - vma->vm_pgoff + vma_pages(vma)); + return shmem_partial_swap_usage(mapping, pgoff, pgoff_end); } /* @@ -1297,7 +1298,8 @@ static int shmem_getattr(struct mnt_idmap *idmap, struct inode *inode = path->dentry->d_inode; struct shmem_inode_info *info = SHMEM_I(inode); - if (info->alloced - info->swapped != inode->i_mapping->nrpages) + /* Fast-path hint; recalc under info->lock corrects any stale read. */ + if (data_race(info->alloced - info->swapped != inode->i_mapping->nrpages)) shmem_recalc_inode(inode, 0, 0); if (info->fsflags & FS_APPEND_FL) @@ -1592,13 +1594,13 @@ start_over: /** * shmem_writeout - Write the folio to swap + * @ctx: swap I/O context * @folio: The folio to write - * @plug: swap plug * @folio_list: list to put back folios on split * * Move the folio from the page cache to the swap cache. */ -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, +int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, struct list_head *folio_list) { struct address_space *mapping = folio->mapping; @@ -1667,7 +1669,7 @@ try_split: * reactivate the folio, and let shmem_fallocate() quit when too many. */ if (!folio_test_uptodate(folio)) { - if (inode->i_private) { + if (READ_ONCE(inode->i_private)) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); shmem_falloc = inode->i_private; @@ -1710,7 +1712,7 @@ try_split: shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap)); BUG_ON(folio_mapped(folio)); - error = swap_writeout(folio, plug); + error = swap_writeout(ctx, folio); if (error != AOP_WRITEPAGE_ACTIVATE) { /* folio has been unlocked */ return error; @@ -1746,7 +1748,17 @@ redirty: folio_mark_dirty(folio); return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ } -EXPORT_SYMBOL_GPL(shmem_writeout); + +int shmem_write_folio(struct folio *folio) +{ + struct swap_io_ctx ctx = {}; + int err; + + err = shmem_writeout(&ctx, folio, NULL); + swap_write_submit(&ctx); + return err; +} +EXPORT_SYMBOL_GPL(shmem_write_folio); #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) @@ -2703,7 +2715,7 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ - if (unlikely(inode->i_private)) { + if (unlikely(READ_ONCE(inode->i_private))) { ret = shmem_falloc_wait(vmf, inode); if (ret) return ret; @@ -2849,7 +2861,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = linear_page_index(vma, addr); return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } @@ -3640,7 +3652,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); if ((u64)unmap_end > (u64)unmap_start) @@ -3650,7 +3662,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, /* No need to unmap again: hole-punching leaves COWed pages */ spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); wake_up_all(&shmem_falloc_waitq); WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); spin_unlock(&inode->i_lock); @@ -3682,7 +3694,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.nr_falloced = 0; shmem_falloc.nr_unswapped = 0; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); /* @@ -3757,7 +3769,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, i_size_write(inode, offset + len); undone: spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); spin_unlock(&inode->i_lock); out: if (!error) @@ -4067,6 +4079,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir, goto out_remove_offset; inode->i_op = &shmem_symlink_inode_operations; memcpy(folio_address(folio), symname, len); + folio_zero_range(folio, len, folio_size(folio) - len); folio_mark_uptodate(folio); folio_mark_dirty(folio); folio_unlock(folio); diff --git a/mm/show_mem.c b/mm/show_mem.c index 43aca5a2ac99..d1288b4c2b64 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -16,6 +16,7 @@ #include <linux/vmstat.h> #include "internal.h" +#include "page_alloc.h" #include "swap.h" atomic_long_t _totalram_pages __read_mostly; @@ -116,7 +117,8 @@ void si_meminfo_node(struct sysinfo *val, int nid) * Determine whether the node should be displayed or not, depending on whether * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). */ -static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) +static bool show_mem_node_skip(unsigned int flags, int nid, + const nodemask_t *nodemask) { if (!(flags & SHOW_MEM_FILTER_NODES)) return false; @@ -177,7 +179,8 @@ static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's * cpuset. */ -static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long free_pcp = 0; int cpu, nid; @@ -402,7 +405,8 @@ static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_z show_swap_cache_info(); } -void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +void __show_mem(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long total = 0, reserved = 0, highmem = 0; struct zone *zone; diff --git a/mm/shuffle.c b/mm/shuffle.c index fb1393b8b3a9..82a2c7725a08 100644 --- a/mm/shuffle.c +++ b/mm/shuffle.c @@ -7,6 +7,7 @@ #include <linux/random.h> #include <linux/moduleparam.h> #include "internal.h" +#include "page_alloc.h" #include "shuffle.h" DEFINE_STATIC_KEY_FALSE(page_alloc_shuffle_key); diff --git a/mm/slab.h b/mm/slab.h index f5e336b6b6b0..c24c3daaa869 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -180,7 +180,7 @@ static inline void *slab_address(const struct slab *slab) static inline int slab_nid(const struct slab *slab) { - return memdesc_nid(slab->flags); + return memdesc_nid(&slab->flags); } static inline pg_data_t *slab_pgdat(const struct slab *slab) diff --git a/mm/slub.c b/mm/slub.c index 0337e60db5ac..422bc3e12c02 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -53,6 +53,7 @@ #include <trace/events/kmem.h> #include "internal.h" +#include "page_alloc.h" /* * Lock order: @@ -3263,7 +3264,8 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node, else if (node == NUMA_NO_NODE) page = alloc_frozen_pages(flags, order); else - page = __alloc_frozen_pages(flags, order, node, NULL); + page = __alloc_frozen_pages(flags, order, node, NULL, + ALLOC_DEFAULT); if (!page) return NULL; @@ -5273,7 +5275,8 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) if (node == NUMA_NO_NODE) page = alloc_frozen_pages_noprof(flags, order); else - page = __alloc_frozen_pages_noprof(flags, order, node, NULL); + page = __alloc_frozen_pages_noprof(flags, order, node, NULL, + ALLOC_DEFAULT); if (page) { ptr = page_address(page); @@ -5377,15 +5380,7 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f if (unlikely(!size)) return ZERO_SIZE_PTR; - /* - * See the comment for the same check in - * alloc_frozen_pages_nolock_noprof() - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return NULL; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + if (!can_spin_trylock()) return NULL; retry: diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index ebd3ac997f64..5a2469fb1838 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -41,6 +41,8 @@ #define VMEMMAP_POPULATE_PAGEREF 0x0001 #include "internal.h" +#include "mm_init.h" +#include "sparse.h" /* * Allocate a block of memory to be used to back the virtual memory map @@ -342,8 +344,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * * * Any initialization done here will be overwritten by memmap_init(). * - * hugetlb_vmemmap_init() will take care of initialization after - * memmap_init(). + * hugetlb_bootmem_struct_page_init() will take care of initialization + * after memmap_init(). */ p = vmemmap_alloc_block_zero(PAGE_SIZE, node); @@ -581,17 +583,6 @@ void __init sparse_vmemmap_init_nid_early(int nid) { hugetlb_vmemmap_init_early(nid); } - -/* - * This is called just before the initialization of page structures - * through memmap_init. Zones are now initialized, so any work that - * needs to be done that needs zone information can be done from - * here. - */ -void __init sparse_vmemmap_init_nid_late(int nid) -{ - hugetlb_vmemmap_init_late(nid); -} #endif static void subsection_mask_set(unsigned long *map, unsigned long pfn, @@ -603,7 +594,7 @@ static void subsection_mask_set(unsigned long *map, unsigned long pfn, bitmap_set(map, idx, end - idx + 1); } -void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages) +static void __init sparse_init_subsection_map_range(unsigned long pfn, unsigned long nr_pages) { int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1); unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn); @@ -626,6 +617,15 @@ void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages } } +void __init sparse_init_subsection_map(void) +{ + int i, nid; + unsigned long start, end; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) + sparse_init_subsection_map_range(start, end - start); +} + #ifdef CONFIG_MEMORY_HOTPLUG /* Mark all memory sections within the pfn range as online */ diff --git a/mm/sparse.c b/mm/sparse.c index 16ac6df3c89f..704a9dec2b9a 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -13,9 +13,10 @@ #include <linux/vmalloc.h> #include <linux/swap.h> #include <linux/swapops.h> -#include <linux/bootmem_info.h> #include <linux/vmstat.h> #include "internal.h" +#include "mm_init.h" +#include "sparse.h" #include <asm/dma.h> /* @@ -43,7 +44,7 @@ static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; #endif -int memdesc_nid(memdesc_flags_t mdf) +int memdesc_nid(const memdesc_flags_t *mdf) { return section_to_node_table[memdesc_section(mdf)]; } @@ -239,15 +240,8 @@ struct page __init *__populate_section_memmap(unsigned long pfn, struct dev_pagemap *pgmap) { unsigned long size = section_map_size(); - struct page *map; - phys_addr_t addr = __pa(MAX_DMA_ADDRESS); - map = memmap_alloc(size, size, addr, nid, false); - if (!map) - panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", - __func__, size, PAGE_SIZE, nid, &addr); - - return map; + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ @@ -300,17 +294,14 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, unsigned long map_count) { unsigned long pnum; - struct page *map; - struct mem_section *ms; - if (sparse_usage_init(nid, map_count)) { - pr_err("%s: node[%d] usemap allocation failed", __func__, nid); - goto failed; - } + if (sparse_usage_init(nid, map_count)) + panic("Failed to allocate usemap for node %d\n", nid); sparse_vmemmap_init_nid_early(nid); for_each_present_section_nr(pnum_begin, pnum) { + struct mem_section *ms; unsigned long pfn = section_nr_to_pfn(pnum); if (pnum >= pnum_end) @@ -318,34 +309,18 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, ms = __nr_to_section(pnum); if (!preinited_vmemmap_section(ms)) { + struct page *map; + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, - nid, NULL, NULL); - if (!map) { - pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", - __func__, nid); - pnum_begin = pnum; - sparse_usage_fini(); - goto failed; - } + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), PAGE_SIZE)); sparse_init_early_section(nid, map, pnum, 0); } } sparse_usage_fini(); - return; -failed: - /* - * We failed to allocate, mark all the following pnums as not present, - * except the ones already initialized earlier. - */ - for_each_present_section_nr(pnum_begin, pnum) { - if (pnum >= pnum_end) - break; - ms = __nr_to_section(pnum); - if (!preinited_vmemmap_section(ms)) - ms->section_mem_map = 0; - } } /* @@ -369,9 +344,6 @@ void __init sparse_init(void) pnum_begin = first_present_section_nr(); nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); - /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ - set_pageblock_order(); - for_each_present_section_nr(pnum_begin + 1, pnum_end) { int nid = sparse_early_nid(__nr_to_section(pnum_end)); @@ -387,5 +359,6 @@ void __init sparse_init(void) } /* cover the last node */ sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); + sparse_init_subsection_map(); vmemmap_populate_print_last(); } diff --git a/mm/sparse.h b/mm/sparse.h new file mode 100644 index 000000000000..95aa031213f2 --- /dev/null +++ b/mm/sparse.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * sparse.h: + * + * mm/ internal sparse and sparse-vmemmap declarations + */ + +#ifndef __MM_SPARSE_H +#define __MM_SPARSE_H + +#include <linux/mmzone.h> + +/* + * mm/sparse.c + */ +#ifdef CONFIG_SPARSEMEM +void sparse_init(void); +int sparse_index_init(unsigned long section_nr, int nid); + +static inline void sparse_init_one_section(struct mem_section *ms, + unsigned long pnum, struct page *mem_map, + struct mem_section_usage *usage, unsigned long flags) +{ + unsigned long coded_mem_map; + + BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); + + /* + * We encode the start PFN of the section into the mem_map such that + * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it + * from the page pointer to obtain the PFN. + */ + coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); + VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); + + ms->section_mem_map &= ~SECTION_MAP_MASK; + ms->section_mem_map |= coded_mem_map; + ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; + ms->usage = usage; +} + +static inline void __section_mark_present(struct mem_section *ms, + unsigned long section_nr) +{ + if (section_nr > __highest_present_section_nr) + __highest_present_section_nr = section_nr; + + ms->section_mem_map |= SECTION_MARKED_PRESENT; +} +#else +static inline void sparse_init(void) {} +#endif /* CONFIG_SPARSEMEM */ + +/* + * mm/sparse-vmemmap.c + */ +#ifdef CONFIG_SPARSEMEM_VMEMMAP +void sparse_init_subsection_map(void); +#else +static inline void sparse_init_subsection_map(void) {} +#endif /* CONFIG_SPARSEMEM_VMEMMAP */ + +#endif /* __MM_SPARSE_H */ diff --git a/mm/swap.h b/mm/swap.h index 77d2d14eda42..abd26588abd2 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -4,12 +4,11 @@ #include <linux/atomic.h> /* for atomic_long_t */ #include <linux/mm.h> /* for PAGE_SHIFT */ + struct mempolicy; struct swap_iocb; struct swap_memcg_table; -extern int page_cluster; - #if defined(MAX_POSSIBLE_PHYSMEM_BITS) #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) #elif defined(MAX_PHYSMEM_BITS) @@ -78,6 +77,28 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; +struct swap_io_ctx { + struct swap_iocb *sio; + struct swap_info_struct *sis; +}; + +/* + * SWAP_OPS_F_REQUIRE_NOFS: + * When set, all reclaim operations must operated as GFS_NOFS and not + * just GFP_NOIO, as GFP_NOIO allocations could recourse into the + * file system backing this swap file. + */ +#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) + +struct swap_ops { + unsigned int flags; + + bool (*can_merge)(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw); + void (*submit_write)(struct swap_io_ctx *ctx); + void (*submit_read)(struct swap_io_ctx *ctx); +}; + #ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ @@ -230,8 +251,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); * folio_put_swap(): does the opposite thing of folio_dup_swap(). */ int folio_alloc_swap(struct folio *folio); -int folio_dup_swap(struct folio *folio, struct page *subpage); -void folio_put_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap(struct folio *folio, struct page *page); +void folio_put_swap(struct folio *folio, struct page *page); /* For internal use */ extern void __swap_cluster_free_entries(struct swap_info_struct *si, @@ -240,17 +261,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); -struct swap_iocb; -void swap_read_folio(struct folio *folio, struct swap_iocb **plug); -void __swap_read_unplug(struct swap_iocb *plug); -static inline void swap_read_unplug(struct swap_iocb *plug) -{ - if (unlikely(plug)) - __swap_read_unplug(plug); -} -void swap_write_unplug(struct swap_iocb *sio); -int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); -void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug); +void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio); +void swap_read_submit(struct swap_io_ctx *ctx); +void swap_write_submit(struct swap_io_ctx *ctx); +int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); +void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio); /* linux/mm/swap_state.c */ extern struct address_space swap_space __read_mostly; @@ -317,9 +332,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, void show_swap_cache_info(void); void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); -struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, - struct vm_area_struct *vma, unsigned long addr, - struct swap_iocb **plug); +struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, + gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr); struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, struct mempolicy *mpol, pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, @@ -329,18 +343,7 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders, void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, unsigned long addr); -static inline unsigned int folio_swap_flags(struct folio *folio) -{ - return __swap_entry_to_info(folio->swap)->flags; -} - #else /* CONFIG_SWAP */ -struct swap_iocb; -static inline struct swap_cluster_info *swap_cluster_lock( - struct swap_info_struct *si, pgoff_t offset, bool irq) -{ - return NULL; -} static inline struct swap_cluster_info *swap_cluster_get_and_lock( struct folio *folio) @@ -381,11 +384,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } -static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) +static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) { } -static inline void swap_write_unplug(struct swap_iocb *sio) +static inline void swap_write_submit(struct swap_io_ctx *ctx) { } @@ -427,8 +430,7 @@ static inline void swap_update_readahead(struct folio *folio, { } -static inline int swap_writeout(struct folio *folio, - struct swap_iocb **swap_plug) +static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { return 0; } @@ -466,11 +468,11 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new) { } +#endif /* CONFIG_SWAP */ -static inline unsigned int folio_swap_flags(struct folio *folio) -{ - return 0; -} +extern const struct swap_ops swap_bdev_ops; + +int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, + struct list_head *folio_list); -#endif /* CONFIG_SWAP */ #endif /* _MM_SWAP_H */ diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf99778..5be825911e64 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -22,10 +22,15 @@ #include <linux/vmalloc.h> #include <linux/huge_mm.h> #include <linux/shmem_fs.h> +#include <linux/sysctl.h> #include "internal.h" #include "swap_table.h" #include "swap.h" +/* Swap readahead cluster size, as a power of 2 pages. */ +static int page_cluster; +static const int page_cluster_max = 31; + /* * swapper_space is a fiction, retained to simplify the path through * vmscan's shrink_folio_list. @@ -633,9 +638,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, } } -static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, - struct mempolicy *mpol, pgoff_t ilx, - struct swap_iocb **plug, bool readahead) +static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx, + swp_entry_t entry, gfp_t gfp, struct mempolicy *mpol, + pgoff_t ilx, bool readahead) { struct folio *folio; @@ -649,7 +654,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, if (IS_ERR_OR_NULL(folio)) return NULL; - swap_read_folio(folio, plug); + swap_read_folio(ctx, folio); if (readahead) { folio_set_readahead(folio); count_vm_event(SWAP_RA); @@ -677,6 +682,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) { + struct swap_io_ctx ctx = {}; struct folio *folio; do { @@ -689,7 +695,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, if (IS_ERR(folio)) return folio; - swap_read_folio(folio, NULL); + swap_read_folio(&ctx, folio); + swap_read_submit(&ctx); return folio; } @@ -699,9 +706,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, * A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */ -struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, - struct vm_area_struct *vma, unsigned long addr, - struct swap_iocb **plug) +struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, + gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr) { struct swap_info_struct *si; struct mempolicy *mpol; @@ -713,13 +719,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, return NULL; mpol = get_vma_policy(vma, addr, 0, &ilx); - folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false); + folio = swap_cache_read_folio(ctx, entry, gfp_mask, mpol, ilx, false); mpol_cond_put(mpol); put_swap_device(si); return folio; } +static struct folio *swap_cache_read_folio_sync(swp_entry_t entry, gfp_t gfp, + struct mempolicy *mpol, pgoff_t ilx) +{ + struct swap_io_ctx ctx = {}; + struct folio *folio; + + folio = swap_cache_read_folio(&ctx, entry, gfp, mpol, ilx, false); + swap_read_submit(&ctx); + return folio; +} + static unsigned int __swapin_nr_pages(unsigned long prev_offset, unsigned long offset, int hits, @@ -808,8 +825,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, unsigned long start_offset, end_offset; unsigned long mask; struct swap_info_struct *si = __swap_entry_to_info(entry); + struct swap_io_ctx ctx = {}; struct blk_plug plug; - struct swap_iocb *splug = NULL; swp_entry_t ra_entry; mask = swapin_nr_pages(offset) - 1; @@ -828,18 +845,16 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, for (offset = start_offset; offset <= end_offset ; offset++) { /* Ok, do the async read-ahead now */ ra_entry = swp_entry(swp_type(entry), offset); - folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx, - &splug, offset != entry_offset); + folio = swap_cache_read_folio(&ctx, ra_entry, gfp_mask, mpol, + ilx, offset != entry_offset); if (!folio) continue; folio_put(folio); } blk_finish_plug(&plug); - swap_read_unplug(splug); - lru_add_drain(); /* Push any new pages onto the LRU now */ + swap_read_submit(&ctx); skip: - /* The page was likely read above, so no need for plugging here */ - return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false); + return swap_cache_read_folio_sync(entry, gfp_mask, mpol, ilx); } static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, @@ -899,8 +914,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf) { + struct swap_io_ctx ctx = {}; struct blk_plug plug; - struct swap_iocb *splug = NULL; struct folio *folio; pte_t *pte = NULL, pentry; int win; @@ -939,8 +954,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, if (!si) continue; } - folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, - &splug, addr != vmf->address); + folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx, + addr != vmf->address); if (si) put_swap_device(si); if (!folio) @@ -950,13 +965,10 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, if (pte) pte_unmap(pte); blk_finish_plug(&plug); - swap_read_unplug(splug); - lru_add_drain(); + swap_read_submit(&ctx); skip: /* The folio was likely read above, so no need for plugging here */ - folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx, - NULL, false); - return folio; + return swap_cache_read_folio_sync(targ_entry, gfp_mask, mpol, targ_ilx); } /** @@ -987,6 +999,35 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, return folio; } +static const struct ctl_table swap_readahead_sysctl_table[] = { + { + .procname = "page-cluster", + .data = &page_cluster, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&page_cluster_max, + } +}; + +static void __init swap_readahead_setup(void) +{ + unsigned long megs = PAGES_TO_MB(totalram_pages()); + + /* Use a smaller cluster for small-memory machines */ + if (megs < 16) + page_cluster = 2; + else + page_cluster = 3; + /* + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ + + register_sysctl_init("vm", swap_readahead_sysctl_table); +} + #ifdef CONFIG_SYSFS static ssize_t vma_ra_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -1016,7 +1057,7 @@ static const struct attribute_group swap_attr_group = { .attrs = swap_attrs, }; -static int __init swap_init(void) +static int __init swap_sysfs_init(void) { int err; struct kobject *swap_kobj; @@ -1039,5 +1080,17 @@ delete_obj: kobject_put(swap_kobj); return err; } -subsys_initcall(swap_init); +#else +static int __init swap_sysfs_init(void) +{ + return 0; +} #endif + +static int __init swap_init(void) +{ + swap_readahead_setup(); + + return swap_sysfs_init(); +} +subsys_initcall(swap_init); diff --git a/mm/swapfile.c b/mm/swapfile.c index 78b49b0658ad..70b90fa9c2a0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1072,6 +1072,12 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force) swap_cluster_unlock(ci); if (to_scan <= 0) break; + + /* + * When 'force' is false, 'to_scan' is initialized to 1. + * The loop breaks above, making this cond_resched() unreachable + * in atomic contexts. + */ cond_resched(); } } @@ -1781,7 +1787,7 @@ again: /** * folio_dup_swap() - Increase swap count of swap entries of a folio. * @folio: folio with swap entries bounded. - * @subpage: if not NULL, only increase the swap count of this subpage. + * @page: if not NULL, only increase the swap count of this page. * * Typically called when the folio is unmapped and have its swap entry to * take its place: Swap entries allocated to a folio has count == 0 and pinned @@ -1795,7 +1801,7 @@ again: * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ -int folio_dup_swap(struct folio *folio, struct page *subpage) +int folio_dup_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1803,8 +1809,8 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } @@ -1815,13 +1821,13 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) /** * folio_put_swap() - Decrease swap count of swap entries of a folio. * @folio: folio with swap entries bounded, must be in swap cache and locked. - * @subpage: if not NULL, only decrease the swap count of this subpage. + * @page: if not NULL, only decrease the swap count of this page. * * This won't free the swap slots even if swap count drops to zero, they are * still pinned by the swap cache. User may call folio_free_swap to free them. * Context: Caller must ensure the folio is locked and in the swap cache. */ -void folio_put_swap(struct folio *folio, struct page *subpage) +void folio_put_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1830,8 +1836,8 @@ void folio_put_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } @@ -2496,8 +2502,13 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot)); if (pte_swp_soft_dirty(old_pte)) new_pte = pte_mksoft_dirty(new_pte); - if (pte_swp_uffd_wp(old_pte)) - new_pte = pte_mkuffd_wp(new_pte); + if (pte_swp_uffd(old_pte)) + new_pte = pte_mkuffd(new_pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma)) + new_pte = pte_modify(new_pte, PAGE_NONE); + setpte: set_pte_at(vma->vm_mm, addr, pte, new_pte); folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry))); @@ -2941,6 +2952,12 @@ static int setup_swap_extents(struct swap_info_struct *sis, struct inode *inode = mapping->host; int ret; + ret = sio_pool_init(); + if (ret) + return ret; + + sis->ops = &swap_bdev_ops; + if (S_ISBLK(inode->i_mode)) { ret = add_swap_extent(sis, 0, sis->max, 0); *span = sis->pages; @@ -2952,11 +2969,6 @@ static int setup_swap_extents(struct swap_info_struct *sis, if (ret < 0) return ret; sis->flags |= SWP_ACTIVATED; - if ((sis->flags & SWP_FS_OPS) && - sio_pool_init() != 0) { - destroy_swap_extents(sis, swap_file); - return -ENOMEM; - } return ret; } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index c3adedaaf7d5..258b03182a78 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -376,7 +376,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (writable) _dst_pte = pte_mkwrite(_dst_pte, dst_vma); if (flags & MFILL_ATOMIC_WP) - _dst_pte = pte_mkuffd_wp(_dst_pte); + _dst_pte = pte_mkuffd(_dst_pte); ret = -EAGAIN; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); @@ -481,7 +481,7 @@ static void mfill_retry_state_save(struct mfill_retry_state *s, { s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); s->ops = vma_uffd_ops(vma); - s->pgoff = vma->vm_pgoff; + s->pgoff = vma_start_pgoff(vma); if (vma->vm_file) s->file = get_file(vma->vm_file); @@ -507,7 +507,7 @@ static bool mfill_retry_state_changed(struct mfill_retry_state *state, /* VMA was file backed, but file, inode or offset has changed */ if (!vma->vm_file || vma->vm_file->f_inode != state->file->f_inode || - state->file != vma->vm_file || vma->vm_pgoff != state->pgoff) + state->file != vma->vm_file || vma_start_pgoff(vma) != state->pgoff) return true; return false; @@ -1164,6 +1164,75 @@ out_unlock: return err; } +int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp) +{ + struct mm_struct *dst_mm = ctx->mm; + unsigned long end = start + len; + struct vm_area_struct *dst_vma; + unsigned int mm_cp_flags; + struct mmu_gather tlb; + bool found = false; + VMA_ITERATOR(vmi, dst_mm, start); + + VM_WARN_ON_ONCE(start & ~PAGE_MASK); + VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start + len <= start); + + guard(mmap_read_lock)(dst_mm); + guard(rwsem_read)(&ctx->map_changing_lock); + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (enable_rwp) + mm_cp_flags = MM_CP_UFFD_RWP; + else + mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE; + + /* + * Pre-scan the range: validate every spanned VMA before applying + * any change_protection() so a partial failure cannot leave the + * process with only a prefix of the range re-protected. + */ + for_each_vma_range(vmi, dst_vma, end) { + if (!userfaultfd_rwp(dst_vma)) + return -ENOENT; + + if (is_vm_hugetlb_page(dst_vma)) { + unsigned long page_mask; + + page_mask = vma_kernel_pagesize(dst_vma) - 1; + if ((start & page_mask) || (len & page_mask)) + return -EINVAL; + } + found = true; + } + if (!found) + return -ENOENT; + + vma_iter_set(&vmi, start); + tlb_gather_mmu(&tlb, dst_mm); + for_each_vma_range(vmi, dst_vma, end) { + unsigned long vma_start = max(dst_vma->vm_start, start); + unsigned long vma_end = min(dst_vma->vm_end, end); + unsigned int flags = mm_cp_flags; + + /* + * On resolve, try to upgrade writability per-VMA -- + * MM_CP_TRY_CHANGE_WRITABLE WARNs in + * maybe_change_pte_writable() if the VMA is not VM_WRITE, + * and RWP can be registered on PROT_READ-only mappings. + */ + if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma)) + flags |= MM_CP_TRY_CHANGE_WRITABLE; + + change_protection(&tlb, dst_vma, vma_start, vma_end, flags); + } + tlb_finish_mmu(&tlb); + + return 0; +} void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2) @@ -1283,7 +1352,8 @@ static long move_present_ptes(struct mm_struct *mm, } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_folio_page_index(src_folio, dst_vma, + dst_addr); orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot); /* Set soft dirty bit so userspace can notice the pte was moved */ @@ -1292,6 +1362,13 @@ static long move_present_ptes(struct mm_struct *mm, if (pte_dirty(orig_src_pte)) orig_dst_pte = pte_mkdirty(orig_dst_pte); orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma); + + /* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + orig_dst_pte = pte_modify(orig_dst_pte, PAGE_NONE); + orig_dst_pte = pte_mkuffd(orig_dst_pte); + } + set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); src_addr += PAGE_SIZE; @@ -1352,7 +1429,8 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, */ if (src_folio) { folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_folio_page_index(src_folio, dst_vma, + dst_addr); } else { /* * Check if the swap entry is cached after acquiring the src_pte @@ -1373,6 +1451,9 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); if (pgtable_supports_soft_dirty()) orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte); + /* Re-arm RWP on the moved swap entry if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) + orig_src_pte = pte_swp_mkuffd(orig_src_pte); set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); double_pt_unlock(dst_ptl, src_ptl); @@ -1399,6 +1480,13 @@ static int move_zeropage_pte(struct mm_struct *mm, zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), dst_vma->vm_page_prot)); + + /* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + zero_pte = pte_modify(zero_pte, PAGE_NONE); + zero_pte = pte_mkuffd(zero_pte); + } + ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); @@ -2190,9 +2278,22 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, if (start == vma->vm_start && end == vma->vm_end) give_up_on_oom = true; - /* Reset ptes for the whole vma range if wr-protected */ - if (userfaultfd_wp(vma)) - uffd_wp_range(vma, start, end - start, false); + /* Clear the uffd bit and/or restore protnone PTEs */ + if (userfaultfd_protected(vma)) { + unsigned int mm_cp_flags = 0; + struct mmu_gather tlb; + + if (userfaultfd_wp(vma)) + mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE; + if (userfaultfd_rwp(vma)) + mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE; + if (vma_wants_manual_pte_write_upgrade(vma)) + mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; + + tlb_gather_mmu(&tlb, vma->vm_mm); + change_protection(&tlb, vma, start, end, mm_cp_flags); + tlb_finish_mmu(&tlb); + } ret = vma_modify_flags_uffd(vmi, prev, vma, start, end, &new_vma_flags, NULL_VM_UFFD_CTX, @@ -2238,13 +2339,29 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, * userfaultfd and with the right tracking mode too. */ if (vma->vm_userfaultfd_ctx.ctx == ctx && - vma_test_all_mask(vma, vma_flags)) + (vma->vm_flags & __VM_UFFD_FLAGS) == vm_flags) goto skip; + /* + * Pre-scan in userfaultfd_register() already rejected mode + * switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a + * stray bit here is a bug. + */ + VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx && + vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags); + if (vma->vm_start > start) start = vma->vm_start; vma_end = min(end, vma->vm_end); + /* + * Re-registering into the same userfaultfd can remove WP mode. + * Clear any per-PTE uffd-wp state before dropping VM_UFFD_WP, + * matching the UFFDIO_UNREGISTER cleanup semantics. + */ + if (userfaultfd_wp(vma) && !(vm_flags & VM_UFFD_WP)) + uffd_wp_range(vma, start, vma_end - start, false); + new_vma_flags = vma->flags; vma_flags_clear_mask(&new_vma_flags, __VMA_UFFD_FLAGS); vma_flags_set_mask(&new_vma_flags, vma_flags); @@ -2371,14 +2488,29 @@ struct userfaultfd_wake_range { /* internal indication that UFFD_API ioctl was successfully executed */ #define UFFD_FEATURE_INITIALIZED (1u << 31) +/* + * UFFDIO_SET_MODE updates ctx->features under mmap_write_lock with + * WRITE_ONCE; readers that run outside mmap_read_lock or the per-VMA + * lock (poll/read_iter/ioctl, fdinfo) must pair with READ_ONCE. + */ +static unsigned int userfaultfd_features(struct userfaultfd_ctx *ctx) +{ + return READ_ONCE(ctx->features); +} + static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) { - return ctx->features & UFFD_FEATURE_INITIALIZED; + return userfaultfd_features(ctx) & UFFD_FEATURE_INITIALIZED; } static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) { - return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); + return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_WP_ASYNC); +} + +static bool userfaultfd_rwp_async_ctx(struct userfaultfd_ctx *ctx) +{ + return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_RWP_ASYNC); } /* @@ -2393,7 +2525,7 @@ bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) if (!ctx) return false; - return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; + return userfaultfd_features(ctx) & UFFD_FEATURE_WP_UNPOPULATED; } static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, @@ -2507,6 +2639,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address, msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; if (reason & VM_UFFD_WP) msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; + if (reason & VM_UFFD_RWP) + msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP; if (reason & VM_UFFD_MINOR) msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR; if (features & UFFD_FEATURE_THREAD_ID) @@ -2560,6 +2694,12 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, */ if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) return true; + /* + * PTE is still RW-protected (protnone with uffd bit), wait for + * resolution. Plain PROT_NONE without the marker is not an RWP fault. + */ + if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP)) + return true; return false; } @@ -2620,8 +2760,14 @@ again: if (!pmd_present(_pmd)) return false; - if (pmd_trans_huge(_pmd)) - return !pmd_write(_pmd) && (reason & VM_UFFD_WP); + if (pmd_trans_huge(_pmd)) { + if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) + return true; + if (pmd_protnone(_pmd) && pmd_uffd(_pmd) && + (reason & VM_UFFD_RWP)) + return true; + return false; + } pte = pte_offset_map(pmd, address); if (!pte) @@ -2657,6 +2803,13 @@ again: */ if (!pte_write(ptent) && (reason & VM_UFFD_WP)) goto out; + /* + * PTE is still RW-protected (protnone with uffd bit), wait for + * userspace to resolve. Plain PROT_NONE without the marker is not + * an RWP fault. + */ + if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP)) + goto out; ret = false; out: @@ -3601,11 +3754,27 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) vm_flags |= VM_UFFD_MISSING; if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { - if (!pgtable_supports_uffd_wp()) + if (!pgtable_supports_uffd()) goto out; vm_flags |= VM_UFFD_WP; } + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) { + if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE) + goto out; + if (!(userfaultfd_features(ctx) & UFFD_FEATURE_RWP)) + goto out; + vm_flags |= VM_UFFD_RWP; + } + + /* + * WP and RWP share the uffd PTE bit and + * cannot coexist in the same VMA — the bit would carry ambiguous + * semantics. Reject the combination up front. + */ + if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP)) + goto out; + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) { #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR goto out; @@ -3661,6 +3830,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, goto out_unlock; /* + * RWP uses protnone as an access-tracking marker. PROT_NONE + * VMAs have vm_page_prot == PAGE_NONE, so RWP resolution + * cannot make a page accessible again. Reject at register + * time only: a VMA that later becomes inaccessible via + * mprotect() must still be unregisterable, so this is not + * part of vma_can_userfault(). + */ + if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur)) + goto out_unlock; + + /* * UFFDIO_COPY will fill file holes even without * PROT_WRITE. This check enforces that if this is a * MAP_SHARED, the process has write permission to the backing @@ -3700,6 +3880,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, goto out_unlock; /* + * Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would + * leave PTE markers without the flag that describes them; + * subsequent mprotect() would then promote stale markers + * into the other mode. Require an unregister first. + */ + if (cur->vm_userfaultfd_ctx.ctx == ctx && + cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags) + goto out_unlock; + + /* * Note vmas containing huge pages */ if (is_vm_hugetlb_page(cur)) @@ -3732,6 +3922,10 @@ out_unlock: if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR)) ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE); + /* RWPROTECT is only supported for RWP ranges */ + if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP)) + ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT); + /* * Now that we scanned all vmas we can already tell * userland which ioctls methods are guaranteed to @@ -4079,6 +4273,158 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, return ret; } +static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + int ret; + struct uffdio_rwprotect uffdio_rwp; + struct userfaultfd_wake_range range; + bool mode_rwp, mode_dontwake; + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (copy_from_user(&uffdio_rwp, (void __user *)arg, + sizeof(uffdio_rwp))) + return -EFAULT; + + ret = validate_range(ctx->mm, uffdio_rwp.range.start, + uffdio_rwp.range.len); + if (ret) + return ret; + + if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE | + UFFDIO_RWPROTECT_MODE_RWP)) + return -EINVAL; + + mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP; + mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE; + + if (mode_rwp && mode_dontwake) + return -EINVAL; + + if (mmget_not_zero(ctx->mm)) { + ret = mrwprotect_range(ctx, uffdio_rwp.range.start, + uffdio_rwp.range.len, mode_rwp); + mmput(ctx->mm); + } else { + return -ESRCH; + } + + if (ret) + return ret; + + if (!mode_rwp && !mode_dontwake) { + range.start = uffdio_rwp.range.start; + range.len = uffdio_rwp.range.len; + wake_userfault(ctx, &range); + } + return ret; +} + +/* Subset of UFFD_API_FEATURES actually supported by this kernel/arch */ +static __u64 uffd_api_available_features(void) +{ + __u64 f = UFFD_API_FEATURES; + + if (!IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_MINOR)) + f &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); + if (!pgtable_supports_uffd()) + f &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; + if (!uffd_supports_wp_marker()) + f &= ~(UFFD_FEATURE_WP_HUGETLBFS_SHMEM | + UFFD_FEATURE_WP_UNPOPULATED | + UFFD_FEATURE_WP_ASYNC); + /* + * RWP needs both PROT_NONE support and the uffd PTE bit. The + * VM_UFFD_RWP check covers compile-time unavailability; the + * pgtable_supports_uffd() check covers runtime (e.g. riscv + * without the SVRSW60T59B extension) where the PTE bit is declared + * but not actually usable. + */ + if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) + f &= ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC); + return f; +} + +/* Async features that can be toggled at runtime via UFFDIO_SET_MODE */ +#define UFFD_FEATURE_TOGGLEABLE UFFD_FEATURE_RWP_ASYNC + +static int userfaultfd_set_mode(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + struct uffdio_set_mode mode; + struct mm_struct *mm = ctx->mm; + + if (copy_from_user(&mode, (void __user *)arg, sizeof(mode))) + return -EFAULT; + + /* enable and disable must not overlap */ + if (mode.enable & mode.disable) + return -EINVAL; + + /* only toggleable features that this kernel/arch actually supports */ + if ((mode.enable | mode.disable) & + ~(uffd_api_available_features() & UFFD_FEATURE_TOGGLEABLE)) + return -EINVAL; + + /* RWP_ASYNC can only be enabled on contexts that negotiated RWP */ + if ((mode.enable & UFFD_FEATURE_RWP_ASYNC) && + !(userfaultfd_features(ctx) & UFFD_FEATURE_RWP)) + return -EINVAL; + + if (!mmget_not_zero(mm)) + return -ESRCH; + + /* + * Drain in-flight faults before flipping features. mmap_write_lock() + * blocks new mmap_read_lock() callers, but per-VMA locked faults + * (lock_vma_under_rcu() + FAULT_FLAG_VMA_LOCK) that acquired before + * this point keep running. Calling vma_start_write() on each UFFD- + * armed VMA waits for those readers to drop, so no in-flight fault + * can observe the old features after mmap_write_unlock(). + */ + mmap_write_lock(mm); + { + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + for_each_vma(vmi, vma) { + if (vma->vm_userfaultfd_ctx.ctx == ctx) + vma_start_write(vma); + } + } + /* + * Single WRITE_ONCE so lockless readers (fdinfo, poll/read_iter + * via userfaultfd_is_initialized(), and the userfaultfd_features() + * helper used elsewhere) can't observe a mid-RMW intermediate + * value. Hot-path readers already serialise through the mmap lock + * + vma_start_write() drain above, so their load doesn't need an + * annotation. + */ + WRITE_ONCE(ctx->features, + (ctx->features | mode.enable) & ~mode.disable); + mmap_write_unlock(mm); + + /* + * If switching to async, wake threads blocked in handle_userfault(). + * They will retry the fault and auto-resolve under the new mode. + * len=0 means wake all pending faults on this context. + */ + if (mode.enable & UFFD_FEATURE_RWP_ASYNC) { + struct userfaultfd_wake_range range = { .len = 0 }; + + spin_lock_irq(&ctx->fault_pending_wqh.lock); + __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, + &range); + __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range); + spin_unlock_irq(&ctx->fault_pending_wqh.lock); + } + + mmput(mm); + return 0; +} + static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) { __s64 ret; @@ -4202,6 +4548,11 @@ bool userfaultfd_wp_async(struct vm_area_struct *vma) return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx); } +bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return userfaultfd_rwp_async_ctx(vma->vm_userfaultfd_ctx.ctx); +} + static inline unsigned int uffd_ctx_features(__u64 user_features) { /* @@ -4305,20 +4656,14 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, if (features & UFFD_FEATURE_WP_ASYNC) features |= UFFD_FEATURE_WP_UNPOPULATED; - /* report all available features and ioctls to userland */ - uffdio_api.features = UFFD_API_FEATURES; -#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR - uffdio_api.features &= - ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); -#endif - if (!pgtable_supports_uffd_wp()) - uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; + ret = -EINVAL; + /* RWP_ASYNC requires RWP */ + if ((features & UFFD_FEATURE_RWP_ASYNC) && + !(features & UFFD_FEATURE_RWP)) + goto err_out; - if (!uffd_supports_wp_marker()) { - uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; - uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; - uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; - } + /* report all available features and ioctls to userland */ + uffdio_api.features = uffd_api_available_features(); ret = -EINVAL; if (features & ~uffdio_api.features) @@ -4385,6 +4730,12 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, case UFFDIO_POISON: ret = userfaultfd_poison(ctx, arg); break; + case UFFDIO_RWPROTECT: + ret = userfaultfd_rwprotect(ctx, arg); + break; + case UFFDIO_SET_MODE: + ret = userfaultfd_set_mode(ctx, arg); + break; } return ret; } @@ -4412,7 +4763,7 @@ static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) * protocols: aa:... bb:... */ seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", - pending, total, UFFD_API, ctx->features, + pending, total, UFFD_API, userfaultfd_features(ctx), UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); } #endif diff --git a/mm/util.c b/mm/util.c index 34cb43b3eaa4..bf0513d1d3d0 100644 --- a/mm/util.c +++ b/mm/util.c @@ -578,8 +578,8 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); + ret = do_mmap(file, addr, len, prot, flag, EMPTY_VMA_FLAGS, pgoff, + &populate, &uf); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -627,20 +627,20 @@ EXPORT_SYMBOL(vm_mmap); unsigned long vm_mmap_shadow_stack(unsigned long addr, unsigned long len, unsigned long flags) { + vma_flags_t vma_flags = VMA_SHADOW_STACK; struct mm_struct *mm = current->mm; unsigned long ret, unused; - vm_flags_t vm_flags = VM_SHADOW_STACK; flags |= MAP_ANONYMOUS | MAP_PRIVATE; if (addr) flags |= MAP_FIXED_NOREPLACE; if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) - vm_flags |= VM_NOHUGEPAGE; + vma_flags_set(&vma_flags, VMA_NOHUGEPAGE_BIT); mmap_write_lock(mm); ret = do_mmap(NULL, addr, len, PROT_READ | PROT_WRITE, flags, - vm_flags, 0, &unused, NULL); + vma_flags, 0, &unused, NULL); mmap_write_unlock(mm); return ret; @@ -1188,7 +1188,7 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->start = vma->vm_start; desc->end = vma->vm_end; - desc->pgoff = vma->vm_pgoff; + desc->pgoff = vma_start_pgoff(vma); desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1379,7 +1379,7 @@ static int call_vma_mapped(struct vm_area_struct *vma) if (!vm_ops || !vm_ops->mapped) return 0; - err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma->vm_pgoff, + err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma_start_pgoff(vma), vma->vm_file, &vm_private_data); if (err) return err; @@ -4,6 +4,10 @@ * VMA-specific functions. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" @@ -14,6 +18,7 @@ struct mmap_state { unsigned long addr; unsigned long end; pgoff_t pgoff; + pgoff_t virt_pgoff; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,17 +47,26 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +static inline pgoff_t map_anon_pgoff(const struct mmap_state *map) +{ + if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT)) + return map->pgoff; + + return map->virt_pgoff; +} + +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, virt_pgoff_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ .pgoff = pgoff_, \ + .virt_pgoff = virt_pgoff_, \ .pglen = PHYS_PFN(len_), \ .vma_flags = vma_flags_, \ .file = file_, \ - .page_prot = vma_get_page_prot(vma_flags_), \ + .page_prot = vma_flags_to_page_prot(vma_flags_), \ } #define VMG_MMAP_STATE(name, map_, vma_) \ @@ -63,6 +77,7 @@ struct mmap_state { .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ .pgoff = (map_)->pgoff, \ + .anon_pgoff = map_anon_pgoff(map_), \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -70,6 +85,21 @@ struct mmap_state { .state = VMA_MERGE_START, \ } +static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + vma->vm_start = start; + vma->vm_end = end; +} + +static void vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, pgoff_t pgoff, pgoff_t virt_pgoff) +{ + __vma_set_range(vma, start, end); + vma_set_pgoff(vma, pgoff); + vma_set_virt_pgoff(vma, virt_pgoff); +} + /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ static bool vma_is_fork_child(struct vm_area_struct *vma) { @@ -197,15 +227,15 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); - - if (is_mergeable_vma(vmg, /* merge_next = */ true) && - is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) - return true; - } - - return false; + if (!is_mergeable_vma(vmg, /* merge_next = */ true)) + return false; + if (!is_mergeable_anon_vma(vmg, /* merge_next = */ true)) + return false; + if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next)) + return false; + if (vmg_end_anon_pgoff(vmg) != vma_start_anon_pgoff(vmg->next)) + return false; + return true; } /* @@ -219,12 +249,15 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) */ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { - if (is_mergeable_vma(vmg, /* merge_next = */ false) && - is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) - return true; - } - return false; + if (!is_mergeable_vma(vmg, /* merge_next = */ false)) + return false; + if (!is_mergeable_anon_vma(vmg, /* merge_next = */ false)) + return false; + if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg)) + return false; + if (vma_end_anon_pgoff(vmg->prev) != vmg_start_anon_pgoff(vmg)) + return false; + return true; } static void __vma_link_file(struct vm_area_struct *vma, @@ -234,7 +267,7 @@ static void __vma_link_file(struct vm_area_struct *vma, mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -248,7 +281,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -258,30 +291,30 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, * * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the * vma must be removed from the anon_vma's interval trees using - * anon_vma_interval_tree_pre_update_vma(). + * anon_rmap_tree_pre_update_vma(). * * After the update, the vma will be reinserted using - * anon_vma_interval_tree_post_update_vma(). + * anon_rmap_tree_post_update_vma(). * * The entire update must be protected by exclusive mmap_lock and by * the root anon_vma's mutex. */ static void -anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) +anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma) { struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + anon_rmap_tree_remove(avc, avc->anon_vma); } static void -anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) +anon_rmap_tree_post_update_vma(struct vm_area_struct *vma) { struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_rmap_tree_insert(avc, avc->anon_vma); } /* @@ -312,17 +345,16 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->anon_vma) { anon_vma_lock_write(vp->anon_vma); - anon_vma_interval_tree_pre_update_vma(vp->vma); + anon_rmap_tree_pre_update_vma(vp->vma); if (vp->adj_next) - anon_vma_interval_tree_pre_update_vma(vp->adj_next); + anon_rmap_tree_pre_update_vma(vp->adj_next); } if (vp->file) { flush_dcache_mmap_lock(vp->mapping); - vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); + mapping_rmap_tree_remove(vp->vma, vp->mapping); if (vp->adj_next) - vma_interval_tree_remove(vp->adj_next, - &vp->mapping->i_mmap); + mapping_rmap_tree_remove(vp->adj_next, vp->mapping); } } @@ -340,9 +372,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, { if (vp->file) { if (vp->adj_next) - vma_interval_tree_insert(vp->adj_next, - &vp->mapping->i_mmap); - vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); + mapping_rmap_tree_insert(vp->adj_next, vp->mapping); + mapping_rmap_tree_insert(vp->vma, vp->mapping); flush_dcache_mmap_unlock(vp->mapping); } @@ -361,9 +392,9 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, } if (vp->anon_vma) { - anon_vma_interval_tree_post_update_vma(vp->vma); + anon_rmap_tree_post_update_vma(vp->vma); if (vp->adj_next) - anon_vma_interval_tree_post_update_vma(vp->adj_next); + anon_rmap_tree_post_update_vma(vp->adj_next); anon_vma_unlock_write(vp->anon_vma); } @@ -521,7 +552,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_add_pgoff(new, linear_page_delta(vma, addr)); } err = -ENOMEM; @@ -560,7 +591,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_add_pgoff(vma, linear_page_delta(new, addr)); } else { vma->vm_end = addr; } @@ -679,7 +710,7 @@ void validate_mm(struct mm_struct *mm) if (anon_vma) { anon_vma_lock_read(anon_vma); list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_verify(avc); + anon_rmap_tree_verify(avc); anon_vma_unlock_read(anon_vma); } #endif @@ -704,20 +735,56 @@ void validate_mm(struct mm_struct *mm) */ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { - struct vm_area_struct *adjust; - pgoff_t pgoff; - if (vmg->__adjust_middle_start) { - adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + /* + * vmg->start vmg->end + * | | + * v merge v + * <-------------> + * delta + * <------> + * |------|----------------| + * | prev | middle | + * |------|----------------| + * ^ + * | + * middle->vm_start + */ + struct vm_area_struct *middle = vmg->middle; + const unsigned long delta = vmg->end - middle->vm_start; + + __vma_set_range(middle, vmg->end, middle->vm_end); + vma_add_pgoff(middle, delta >> PAGE_SHIFT); } else if (vmg->__adjust_next_start) { - adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); - } else { - return; - } + /* + * Originally: + * + * vmg->start vmg->end + * | | + * v merge v + * <------------> + * . . + * merge_existing_range() updates to: + * . . + * vmg->start vmg->end . + * | | . + * v retain v . + * <----------> . + * delta . + * <-----> . + * |----------------|------| + * | middle | next | + * |----------------|------| + * ^ + * | + * next->vm_start + */ + struct vm_area_struct *next = vmg->next; + const unsigned long delta = next->vm_start - vmg->end; - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + __vma_set_range(next, vmg->end, next->vm_end); + vma_sub_pgoff(next, delta >> PAGE_SHIFT); + } } /* @@ -761,7 +828,8 @@ static int commit_merge(struct vma_merge_struct *vmg) */ vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); - vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg), + vmg_start_anon_pgoff(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -928,10 +996,10 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * prev middle next * extend delete delete */ - vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * We already ensured anon_vma compatibility above, so now it's @@ -948,9 +1016,9 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * prev middle * extend shrink/delete */ - vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -964,8 +1032,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * middle next * shrink/delete extend */ - - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + const pgoff_t pglen = vmg_pages(vmg); VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +1040,15 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg->pgoff = vma_start_pgoff(next) - pglen; + vmg->anon_pgoff = vma_start_anon_pgoff(next) - pglen; } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg->pgoff = vma_start_pgoff(middle); + vmg->anon_pgoff = vma_start_anon_pgoff(middle); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1157,8 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * If this merge would result in removal of the next VMA but we @@ -1235,27 +1305,24 @@ nomem: return -ENOMEM; } -/* - * vma_shrink() - Reduce an existing VMAs memory area +/** + * vma_shrink() - Shrink the end of a VMA * @vmi: The vma iterator * @vma: The VMA to modify - * @start: The new start * @end: The new end * + * Note that the caller may only shrink the end of the VMA. + * * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long end) { struct vma_prepare vp; - WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); - - if (vma->vm_start < start) - vma_iter_config(vmi, vma->vm_start, start); - else - vma_iter_config(vmi, end, vma->vm_end); + VM_WARN_ON_ONCE(end > vma->vm_end); + vma_iter_config(vmi, end, vma->vm_end); if (vma_iter_prealloc(vmi, NULL)) return -ENOMEM; @@ -1263,10 +1330,10 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, init_vma_prep(&vp, vma); vma_prepare(&vp); - vma_adjust_trans_huge(vma, start, end, NULL); + vma_adjust_trans_huge(vma, vma->vm_start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + __vma_set_range(vma, vma->vm_start, end); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1863,9 +1930,10 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + pgoff_t virt_pgoff, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; + const bool is_shared = vma_test(vma, VMA_SHARED_BIT); unsigned long vma_start = vma->vm_start; struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *new_vma; @@ -1874,11 +1942,14 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len); /* - * If anonymous vma has not yet been faulted, update new pgoff - * to match new location, to increase its chance of merging. + * If a vma has not yet been faulted, update its virtual pgoff to match + * the new location to increase its chance of merging. */ - if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + if (!vma->anon_vma && !is_shared) { + virt_pgoff = addr >> PAGE_SHIFT; + + if (vma_is_anonymous(vma)) + pgoff = virt_pgoff; faulted_in_anon_vma = false; } @@ -1895,6 +1966,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ vmg.pgoff = pgoff; + vmg.anon_pgoff = is_shared ? pgoff : virt_pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1916,15 +1988,17 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * safe. It is only safe to keep the vm_pgoff * linear if there are no pages mapped yet. */ - VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); + VM_WARN_ON_ONCE_VMA(faulted_in_anon_vma, new_vma); *vmap = vma = new_vma; } - *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); + *need_rmap_locks = + (vma_start_pgoff(new_vma) <= vma_start_pgoff(vma)) || + (vma_start_anon_pgoff(new_vma) <= vma_start_anon_pgoff(vma)); } else { new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pgoff, virt_pgoff); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1972,14 +2046,30 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * { vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags); + /* Ignore flags that mprotect() can change. */ vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS); + /* Ignore flags that do not impact merging. */ vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS); - return a->vm_end == b->vm_start && - mpol_equal(vma_policy(a), vma_policy(b)) && - a->vm_file == b->vm_file && - vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + /* Must be adjacent. */ + if (a->vm_end != b->vm_start) + return false; + /* Must have matching policy. */ + if (!mpol_equal(vma_policy(a), vma_policy(b))) + return false; + /* Must both be anon or map the same file (MAP_PRIVATE case). */ + if (a->vm_file != b->vm_file) + return false; + /* Flags must be equivalent modulo mprotect(). */ + if (!vma_flags_empty(&diff)) + return false; + /* Page offset must align. */ + if (vma_end_pgoff(a) != vma_start_pgoff(b)) + return false; + /* Anon page offset must align. */ + if (vma_end_anon_pgoff(a) != vma_start_anon_pgoff(b)) + return false; + return true; } /* @@ -2119,7 +2209,7 @@ bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) /* The open routine did something to the protections that pgprot_modify * won't preserve? */ if (pgprot_val(vm_page_prot) != - pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) + pgprot_val(vma_pgprot_modify(vm_page_prot, vma->flags))) return false; /* @@ -2522,6 +2612,33 @@ static int __mmap_new_file_vma(struct mmap_state *map, return 0; } +static bool map_is_dev_zero(const struct mmap_state *map) +{ + const struct file *file = map->file; + const struct inode *inode = file_inode(file); + + if (!S_ISCHR(inode->i_mode)) + return false; + return imajor(inode) == MEM_MAJOR && iminor(inode) == DEVZERO_MINOR; +} + +static void map_set_anon(struct mmap_state *map) +{ + map->file = NULL; + map->vm_ops = NULL; + map->pgoff = map->addr >> PAGE_SHIFT; +} + +static bool map_is_private(const struct mmap_state *map) +{ + return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); +} + +static bool map_is_anon(const struct mmap_state *map) +{ + return map_is_private(map) && !map->file; +} + /* * __mmap_new_vma() - Allocate a new VMA for the region, as merging was not * possible. @@ -2535,6 +2652,7 @@ static int __mmap_new_file_vma(struct mmap_state *map, static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, struct mmap_action *action) { + const bool is_anon = map_is_anon(map); struct vma_iterator *vmi = map->vmi; int error = 0; struct vm_area_struct *vma; @@ -2549,7 +2667,11 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + + if (is_anon) + vma_set_anonymous(vma); + + vma_set_range(vma, map->addr, map->end, map->pgoff, map->virt_pgoff); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2558,12 +2680,11 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, goto free_vma; } + /* Invoke callbacks. */ if (map->file) error = __mmap_new_file_vma(map, vma); - else if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT)) + else if (!is_anon) error = shmem_zero_setup(vma); - else - vma_set_anonymous(vma); if (error) goto free_iter_vma; @@ -2673,6 +2794,10 @@ static int call_mmap_prepare(struct mmap_state *map, if (err) return err; + /* Hooks cannot mark themselves anonymous. */ + if (!desc->vm_ops) + return -EINVAL; + err = call_action_prepare(map, desc); if (err) return err; @@ -2689,16 +2814,21 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; + /* + * MAP_PRIVATE-/dev/zero mappings are an ancient way of getting + * anonymous mappings. Rather than allowing these mappings to be odd + * outliers, simply make them truly anonymous. + */ + if (map_is_private(map) && map_is_dev_zero(map)) + map_set_anon(map); + return 0; } static void set_vma_user_defined_fields(struct vm_area_struct *vma, struct mmap_state *map) { - if (map->vm_ops) - vma->vm_ops = map->vm_ops; - else /* Only /dev/zero should do this. */ - vma_set_anonymous(vma); + vma->vm_ops = map->vm_ops; vma->vm_private_data = map->vm_private_data; } @@ -2739,7 +2869,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + const pgoff_t virt_pgoff = addr >> PAGE_SHIFT; + MMAP_STATE(map, mm, &vmi, addr, len, pgoff, virt_pgoff, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2777,7 +2908,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, allocated_new = true; } - if (have_mmap_prepare) + if (have_mmap_prepare && !map_is_anon(&map)) set_vma_user_defined_fields(vma, &map); __mmap_complete(&map, vma); @@ -2821,7 +2952,7 @@ abort_munmap: * file to be mapped, otherwise NULL. * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. - * @vm_flags: The VMA flags which should be applied to the mapping. + * @vma_flags: The VMA flags which should be applied to the mapping. * @pgoff: If @file is specified, the page offset into the file, if not then * the virtual page offset in memory of the anonymous mapping. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap @@ -2831,12 +2962,11 @@ abort_munmap: * been performed. */ unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; - const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); @@ -2845,7 +2975,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return -EACCES; /* Allow architectures to sanity-check the vm_flags. */ - if (!arch_validate_flags(vm_flags)) + if (!arch_validate_flags(vma_flags_to_legacy(vma_flags))) return -EINVAL; /* Map writable and ensure this isn't a sealed memfd. */ @@ -2885,6 +3015,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long addr, unsigned long len, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; + const pgoff_t pgoff = addr >> PAGE_SHIFT; /* * Check against address space limits by the changed size @@ -2909,7 +3040,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, * occur after forking, so the expand will only happen on new VMAs. */ if (vma && vma->vm_end == addr) { - VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, pgoff, pgoff); vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ @@ -2929,7 +3060,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, goto unacct_fail; vma_set_anonymous(vma); - vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_range(vma, addr, addr + len, pgoff, pgoff); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -3167,23 +3298,22 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address > vma->vm_end) { - unsigned long size, grow; - - size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + const unsigned long size = address - vma->vm_start; + const unsigned long grow = (address - vma->vm_end) >> PAGE_SHIFT; + const pgoff_t pgoff = vma_start_pgoff(vma); error = -ENOMEM; - if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { + if (pgoff + (size >> PAGE_SHIFT) >= pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); + anon_rmap_tree_pre_update_vma(vma); vma->vm_end = address; /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); - anon_vma_interval_tree_post_update_vma(vma); + anon_rmap_tree_post_update_vma(vma); perf_event_mmap(vma); } @@ -3246,24 +3376,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { - unsigned long size, grow; - - size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + const unsigned long size = vma->vm_end - address; + const unsigned long grow = (vma->vm_start - address) >> PAGE_SHIFT; error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (grow <= vma_start_pgoff(vma)) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); + anon_rmap_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_sub_pgoff(vma, grow); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); - anon_vma_interval_tree_post_update_vma(vma); + anon_rmap_tree_post_update_vma(vma); perf_event_mmap(vma); } @@ -3293,9 +3421,9 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) return ret; } -/* Insert vm structure into process list sorted by address - * and into the inode's i_mmap tree. If vm_file is non-NULL - * then i_mmap_rwsem is taken here. +/* + * Insert vm structure into process list sorted by address + * and into the inode's i_mmap tree if file-backed. */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { @@ -3321,9 +3449,10 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) * Similarly in do_mmap and in do_brk_flags. */ if (vma_is_anonymous(vma)) { - BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + WARN_ON_ONCE(vma->anon_vma); + vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } + vma_set_virt_pgoff(vma, vma->vm_start >> PAGE_SHIFT); if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) @@ -3354,3 +3483,44 @@ __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) { return vma_kernel_pagesize(vma); } + +struct vm_area_struct *__install_special_mapping( + struct mm_struct *mm, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, void *priv, + const struct vm_operations_struct *ops) +{ + vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); + struct vm_area_struct *vma; + int ret; + + vma = vm_area_alloc(mm); + if (unlikely(!vma)) + return ERR_PTR(-ENOMEM); + + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_DONTEXPAND_BIT); + if (pgtable_supports_soft_dirty()) + vma_flags_set(&vma_flags, VMA_SOFTDIRTY_BIT); + vma_flags_clear_mask(&vma_flags, VMA_LOCKED_MASK); + vma->flags = vma_flags; + vma->vm_page_prot = vma_get_page_prot(vma); + + vma->vm_ops = ops; + vma->vm_private_data = priv; + vma_set_range(vma, addr, addr + len, 0, addr >> PAGE_SHIFT); + + ret = insert_vm_struct(mm, vma); + if (ret) + goto out; + + vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + + perf_event_mmap(vma); + + return vma; + +out: + vm_area_free(vma); + return ERR_PTR(ret); +} @@ -2,7 +2,14 @@ /* * vma.h * - * Core VMA manipulation API implemented in vma.c. + * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c. + * + * Note that, in order for VMA logic to be userland testable, this header + * intentionally includes no dependencies. + * + * This is specifically scoped to mm-only. Users of this functionality (other + * than the core VMA implementation itself) should not include this header + * directly, but rather include internal.h. */ #ifndef __MM_VMA_H #define __MM_VMA_H @@ -97,6 +104,7 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; union { /* Temporary while VMA flags are being converted. */ @@ -230,43 +238,173 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) return vmg->state == VMA_MERGE_ERROR_NOMEM; } -/* Assumes addr >= vma->vm_start. */ -static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, - unsigned long addr) +static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) +{ + const unsigned long size = vmg->end - vmg->start; + + return size >> PAGE_SHIFT; +} + +static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->pgoff; +} + +static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_pgoff(vmg) + vmg_pages(vmg); +} + +static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + /* nommu doesn't set a virtual pgoff for anon VMAs. */ + if (!IS_ENABLED(CONFIG_MMU)) + return; + /* + * File-backed VMAs have arbitrary page offset (either page offset into + * file or for pfnmap the PFN of the start of the range or drivers may + * set arbitrary page offset). + */ + if (!vma_is_anonymous(vma)) + return; + /* If faulted in, could have been remapped. */ + if (vma->anon_vma) + return; + /* OK this is really an anon VMA - expect virtual page offset. */ + VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT); +} + +static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + assert_sane_pgoff(vma, pgoff); + vma->vm_pgoff = pgoff; +} + +static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->anon_pgoff; +} + +static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg); +} + +static inline void __vma_set_virt_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ +#ifdef CONFIG_64BIT + vma->__vm_virt_pgoff_hi = pgoff >> 32; +#endif + vma->__vm_virt_pgoff_lo = pgoff & GENMASK(31, 0); +} + +static inline void vma_set_virt_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + __vma_set_virt_pgoff(vma, pgoff); +} + +static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + vma_assert_can_modify(vma); + vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); + vma_set_virt_pgoff(vma, vma_start_virt_pgoff(vma) + delta); } -#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ +static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) +{ + vma_assert_can_modify(vma); + vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); + vma_set_virt_pgoff(vma, vma_start_virt_pgoff(vma) - delta); +} + +/** + * vma_anon_pgoff_addr() - Calculates the absolute anonymous page offset of + * @address. + * @vma: The VMA whose anonymous page offset is required. + * @address: The address whose absolute page offset is required. + * + * If the VMA is a shared file-backed mapping, then the file-based page offset + * is returned. + * + * Otherwise, the virtual page offset is returned. + * + * This means that shared file-backed mappings are correctly merged based on + * their file page offset compatibility. + * + * Returns: The absolute anonymous page offset of @address within @vma. + */ +static inline pgoff_t vma_anon_pgoff_addr(const struct vm_area_struct *vma, + unsigned long address) +{ + if (vma_test(vma, VMA_SHARED_BIT)) + return linear_page_index(vma, address); + + return linear_virt_page_index(vma, address); +} + +/** + * vma_start_anon_pgoff() - Calculates the absolute anonymous page offset used + * for purposes of merge compatibility. + * @vma: The VMA whose anonymous page offset is required. + * + * See vma_anon_pgoff_addr(). + * + * Returns: The absolute anonymous page offset of @vma for purposes of merging. + */ +static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_anon_pgoff_addr(vma, vma->vm_start); +} + +/** + * vma_end_anon_pgoff() - Calculates the absolute exclusive end anonymous page + * offset used for purposes of merge compatibility. + * @vma: The VMA whose anonymous end page offset is required. + * + * See vma_start_anon_pgoff(). + * + * Returns: The absolute exclusive end anonymous page offset of @vma for + * purposes of merging. + */ +static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_anon_pgoff(vma) + vma_pages(vma); +} + +#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \ + struct vma_merge_struct name = { \ + .mm = mm_, \ + .vmi = vmi_, \ + .start = start_, \ + .end = end_, \ + .vma_flags = vma_flags_, \ + .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ + .state = VMA_MERGE_START, \ + } + +#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ struct vma_merge_struct name = { \ - .mm = mm_, \ + .mm = vma_->vm_mm, \ .vmi = vmi_, \ + .prev = prev_, \ + .middle = vma_, \ + .next = NULL, \ .start = start_, \ .end = end_, \ - .vma_flags = vma_flags_, \ - .pgoff = pgoff_, \ + .vm_flags = vma_->vm_flags, \ + .pgoff = linear_page_index(vma_, start_), \ + .anon_pgoff = vma_anon_pgoff_addr(vma_, start_), \ + .file = vma_->vm_file, \ + .anon_vma = vma_->anon_vma, \ + .policy = vma_policy(vma_), \ + .uffd_ctx = vma_->vm_userfaultfd_ctx, \ + .anon_name = anon_vma_name(vma_), \ .state = VMA_MERGE_START, \ } -#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ - struct vma_merge_struct name = { \ - .mm = vma_->vm_mm, \ - .vmi = vmi_, \ - .prev = prev_, \ - .middle = vma_, \ - .next = NULL, \ - .start = start_, \ - .end = end_, \ - .vm_flags = vma_->vm_flags, \ - .pgoff = vma_pgoff_offset(vma_, start_), \ - .file = vma_->vm_file, \ - .anon_vma = vma_->anon_vma, \ - .policy = vma_policy(vma_), \ - .uffd_ctx = vma_->vm_userfaultfd_ctx, \ - .anon_name = anon_vma_name(vma_), \ - .state = VMA_MERGE_START, \ - } - #ifdef CONFIG_DEBUG_VM_MAPLE_TREE void validate_mm(struct mm_struct *mm); #else @@ -275,8 +413,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, - struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + struct vm_area_struct *vma, unsigned long end); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -310,7 +447,7 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, */ /* Mutable fields. Populated with initial state. */ - vma->vm_pgoff = desc->pgoff; + vma_set_pgoff(vma, desc->pgoff); if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -449,7 +586,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); + pgoff_t anon_pgoff, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +597,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, @@ -484,9 +621,11 @@ static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma } #ifdef CONFIG_MMU -static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, vm_flags_t vm_flags) +static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) { - return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); + const pgprot_t prot = vma_flags_to_page_prot(vma_flags); + + return pgprot_modify(oldprot, prot); } #endif @@ -753,4 +892,9 @@ static inline bool map_deny_write_exec(const vma_flags_t *old, } #endif +struct vm_area_struct *__install_special_mapping(struct mm_struct *mm, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, void *priv, + const struct vm_operations_struct *ops); + #endif /* __MM_VMA_H */ diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f..586c52155942 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -1,10 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Functions explicitly implemented for exec functionality which however are - * explicitly VMA-only logic. + * Functions provided for exec functionality which however are + * specifically VMA-only logic. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" @@ -37,7 +41,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) unsigned long new_end = old_end - shift; VMA_ITERATOR(vmi, mm, new_start); VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS, - vma->vm_pgoff); + vma_start_pgoff(vma), vma_start_anon_pgoff(vma)); struct vm_area_struct *next; struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); @@ -89,7 +93,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_end); } /* @@ -108,15 +112,17 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, unsigned long *top_mem_p) { - unsigned long flags = VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP; + vma_flags_t flags = VMA_STACK_INCOMPLETE_SETUP; + struct vm_area_struct *vma; int err; - struct vm_area_struct *vma = vm_area_alloc(mm); + /* VMA_STACK_FLAGS and VMA_STACK_INCOMPLETE_SETUP must not overlap. */ + VM_WARN_ON_ONCE(vma_flags_test_any_mask(&flags, VMA_STACK_FLAGS)); + + vma = vm_area_alloc(mm); if (!vma) return -ENOMEM; - vma_set_anonymous(vma); - if (mmap_write_lock_killable(mm)) { err = -EINTR; goto err_free; @@ -130,19 +136,21 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, if (err) goto err_ksm; + vma_flags_set_mask(&flags, VMA_STACK_FLAGS); + vma_set_anonymous(vma); + /* * Place the stack at the largest stack address the architecture * supports. Later, we'll move this to an appropriate place. We don't * use STACK_TOP because that can depend on attributes which aren't * configured yet. */ - VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; vma->vm_start = vma->vm_end - PAGE_SIZE; if (pgtable_supports_soft_dirty()) - flags |= VM_SOFTDIRTY; - vm_flags_init(vma, flags); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma_flags_set(&flags, VMA_SOFTDIRTY_BIT); + vma->flags = flags; + vma->vm_page_prot = vma_get_page_prot(vma); err = insert_vm_struct(mm, vma); if (err) diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510..710b18849a36 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -5,6 +5,10 @@ * between CONFIG_MMU and non-CONFIG_MMU kernel configurations. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" @@ -46,7 +50,8 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + dest->vm_pgoff = vma_start_pgoff(src); + __vma_set_virt_pgoff(dest, vma_start_virt_pgoff(src)); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vma_internal.h b/mm/vma_internal.h index 2da6d224c1a8..385c0ab13777 100644 --- a/mm/vma_internal.h +++ b/mm/vma_internal.h @@ -2,8 +2,8 @@ /* * vma_internal.h * - * Headers required by vma.c, which can be substituted accordingly when testing - * VMA functionality. + * Headers required by vma.c, vma_init.c and vma_exec.c, which can be + * substituted accordingly when testing VMA functionality. */ #ifndef __MM_VMA_INTERNAL_H @@ -23,6 +23,7 @@ #include <linux/ksm.h> #include <linux/khugepaged.h> #include <linux/list.h> +#include <linux/major.h> #include <linux/maple_tree.h> #include <linux/mempolicy.h> #include <linux/mm.h> diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f4fa227a8d7f..26f32949c2f2 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -50,6 +50,7 @@ #include "internal.h" #include "pgalloc-track.h" +#include "vmalloc.h" #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; @@ -92,6 +93,33 @@ struct vfree_deferred { static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); /*** Page table manipulation functions ***/ + +/* + * Try contiguous mappings at the PTE level for arches which support them, and if + * requested by the caller. Fall back to PAGE_SIZE mappings otherwise. + * + * Return: mapping size. + */ +static __always_inline unsigned long vmap_set_ptes(pte_t *pte, + unsigned long addr, unsigned long end, u64 pfn, + pgprot_t prot, unsigned int max_page_shift) +{ +#ifdef CONFIG_HUGETLB_PAGE + unsigned long size; + + size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); + if (size != PAGE_SIZE) { + pte_t entry = pfn_pte(pfn, prot); + + entry = arch_make_huge_pte(entry, ilog2(size), 0); + set_huge_pte_at(&init_mm, addr, pte, entry, size); + return size; + } +#endif + set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); + return PAGE_SIZE; +} + static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift, pgtbl_mod_mask *mask) @@ -99,7 +127,8 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pte_t *pte; u64 pfn; struct page *page; - unsigned long size = PAGE_SIZE; + unsigned long size; + unsigned int steps; if (WARN_ON_ONCE(!PAGE_ALIGNED(end - addr))) return -EINVAL; @@ -120,20 +149,9 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, BUG(); } -#ifdef CONFIG_HUGETLB_PAGE - size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); - if (size != PAGE_SIZE) { - pte_t entry = pfn_pte(pfn, prot); - - entry = arch_make_huge_pte(entry, ilog2(size), 0); - set_huge_pte_at(&init_mm, addr, pte, entry, size); - pfn += PFN_DOWN(size); - continue; - } -#endif - set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); - pfn++; - } while (pte += PFN_DOWN(size), addr += size, addr != end); + size = vmap_set_ptes(pte, addr, end, pfn, prot, max_page_shift); + steps = PFN_DOWN(size); + } while (pte += steps, pfn += steps, addr += size, addr != end); lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED; @@ -546,8 +564,10 @@ void vunmap_range(unsigned long addr, unsigned long end) static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { + unsigned long pfn, size; + unsigned int steps; int err = 0; pte_t *pte; @@ -578,9 +598,10 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, break; } - set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); - (*nr)++; - } while (pte++, addr += PAGE_SIZE, addr != end); + pfn = page_to_pfn(page); + size = vmap_set_ptes(pte, addr, end, pfn, prot, shift); + steps = PFN_DOWN(size); + } while (pte += steps, *nr += steps, addr += size, addr != end); lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED; @@ -590,60 +611,90 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { pmd_t *pmd; unsigned long next; + int err; pmd = pmd_alloc_track(&init_mm, pud, addr, mask); if (!pmd) return -ENOMEM; do { next = pmd_addr_end(addr, end); - if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) - return -ENOMEM; + + if (shift >= PMD_SHIFT) { + struct page *page = pages[*nr]; + phys_addr_t phys_addr; + + if (WARN_ON(!page)) + return -ENOMEM; + if (WARN_ON(!pfn_valid(page_to_pfn(page)))) + return -EINVAL; + + phys_addr = page_to_phys(page); + + if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, + shift)) { + *mask |= PGTBL_PMD_MODIFIED; + *nr += 1 << (PMD_SHIFT - PAGE_SHIFT); + continue; + } + } + + err = vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask, shift); + if (err) + return err; } while (pmd++, addr = next, addr != end); return 0; } static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { pud_t *pud; unsigned long next; + int err; pud = pud_alloc_track(&init_mm, p4d, addr, mask); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); - if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) - return -ENOMEM; + err = vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask, shift); + if (err) + return err; } while (pud++, addr = next, addr != end); return 0; } static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { p4d_t *p4d; unsigned long next; + int err; p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end); - if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) - return -ENOMEM; + err = vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask, shift); + if (err) + return err; } while (p4d++, addr = next, addr != end); return 0; } -static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages) +/* + * It can take an array of pages which are not all contiguous, but it + * may have contiguous chunks, as hinted by @shift. + */ +static int vmap_pages_range_noflush_walk(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, unsigned int shift) { unsigned long start = addr; pgd_t *pgd; @@ -658,7 +709,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, next = pgd_addr_end(addr, end); if (pgd_bad(*pgd)) mask |= PGTBL_PGD_MODIFIED; - err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); + err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask, shift); if (err) break; } while (pgd++, addr = next, addr != end); @@ -681,27 +732,12 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift) { - unsigned int i, nr = (end - addr) >> PAGE_SHIFT; - WARN_ON(page_shift < PAGE_SHIFT); - if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || - page_shift == PAGE_SHIFT) - return vmap_small_pages_range_noflush(addr, end, prot, pages); + if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC)) + page_shift = PAGE_SHIFT; - for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { - int err; - - err = vmap_range_noflush(addr, addr + (1UL << page_shift), - page_to_phys(pages[i]), prot, - page_shift); - if (err) - return err; - - addr += 1UL << page_shift; - } - - return 0; + return vmap_pages_range_noflush_walk(addr, end, prot, pages, page_shift); } int vmap_pages_range_noflush(unsigned long addr, unsigned long end, @@ -3301,6 +3337,14 @@ struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, NUMA_NO_NODE, GFP_KERNEL, caller); } +static struct vm_struct *__get_vm_area_node_aligned_caller(unsigned long size, + unsigned long align, unsigned long flags, const void *caller) +{ + return __get_vm_area_node(size, align, PAGE_SHIFT, flags, + VMALLOC_START, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); +} + /** * find_vm_area - find a continuous kernel virtual area * @addr: base address @@ -3541,6 +3585,116 @@ void vunmap(const void *addr) } EXPORT_SYMBOL(vunmap); +static inline unsigned int vm_shift(pgprot_t prot, unsigned long size) +{ + if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) + return PMD_SHIFT; + + return arch_vmap_pte_supported_shift(size); +} + +static inline int get_vmap_batch_order(struct page **pages, + pgprot_t prot, unsigned int nr_pages) +{ + unsigned long pfn; + unsigned int nr_contig; + int order; + + if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) + return 0; + + /* Limit nr_pages by pfn alignment */ + pfn = page_to_pfn(*pages); + if (pfn > 0) + nr_pages = min_t(unsigned int, nr_pages, 1UL << __ffs(pfn)); + + nr_contig = num_pages_contiguous(pages, nr_pages); + if (nr_contig < 2) + return 0; + + order = ilog2(nr_contig); + + if (vm_shift(prot, PAGE_SIZE << order) == PAGE_SHIFT) + return 0; + + return order; +} + +static int vmap_pages_range_batched(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages) +{ + const unsigned int nr_pages = (end - addr) >> PAGE_SHIFT; + unsigned int prev_shift = 0, batch_start = 0; + unsigned long map_addr = addr, batch_end = addr; + int err; + + err = kmsan_vmap_pages_range_noflush(addr, end, prot, pages, + PAGE_SHIFT, GFP_KERNEL); + if (err) + goto out; + + for (unsigned int i = 0; i < nr_pages; ) { + unsigned int shift = PAGE_SHIFT + + get_vmap_batch_order(pages + i, prot, nr_pages - i); + + if (!i) + prev_shift = shift; + + if (shift != prev_shift) { + err = vmap_pages_range_noflush_walk(map_addr, batch_end, + prot, pages + batch_start, prev_shift); + if (err) + goto out; + prev_shift = shift; + map_addr = batch_end; + batch_start = i; + } + + /* + * Once we fail to batch pages, we expect to fail batching + * for all remaining pages, so just give up. + */ + if (shift == PAGE_SHIFT) + break; + + batch_end += 1UL << shift; + i += 1U << (shift - PAGE_SHIFT); + } + + /* Remaining */ + if (map_addr < end) + err = vmap_pages_range_noflush_walk(map_addr, end, prot, + pages + batch_start, prev_shift); + +out: + flush_cache_vmap(addr, end); + return err; +} + +static struct vm_struct *vmap_get_aligned_vm_area(unsigned long size, + unsigned long flags, pgprot_t prot, const void *caller) +{ + struct vm_struct *vm_area; + unsigned int shift; + + if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) { + vm_area = __get_vm_area_node_aligned_caller(size, PMD_SIZE, + flags, caller); + if (vm_area) + return vm_area; + } + + shift = arch_vmap_pte_supported_shift(size); + if (shift > PAGE_SHIFT) { + vm_area = __get_vm_area_node_aligned_caller(size, 1UL << shift, + flags, caller); + if (vm_area) + return vm_area; + } + + return __get_vm_area_node_aligned_caller(size, PAGE_SIZE, flags, caller); +} + /** * vmap - map an array of pages into virtually contiguous space * @pages: array of page pointers @@ -3579,13 +3733,14 @@ void *vmap(struct page **pages, unsigned int count, return NULL; size = (unsigned long)count << PAGE_SHIFT; - area = get_vm_area_caller(size, flags, __builtin_return_address(0)); + area = vmap_get_aligned_vm_area(size, flags, prot, + __builtin_return_address(0)); if (!area) return NULL; addr = (unsigned long)area->addr; - if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), - pages, PAGE_SHIFT) < 0) { + if (vmap_pages_range_batched(addr, addr + size, pgprot_nx(prot), + pages) < 0) { vunmap(area->addr); return NULL; } @@ -4052,11 +4207,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, * supporting them. */ - if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) - shift = PMD_SHIFT; - else - shift = arch_vmap_pte_supported_shift(size); - + shift = vm_shift(prot, size); align = max(original_align, 1UL << shift); } @@ -4067,8 +4218,8 @@ again: if (!area) { bool nofail = gfp_mask & __GFP_NOFAIL; warn_alloc(gfp_mask, NULL, - "vmalloc error: size %lu, vm_struct allocation failed%s", - size, (nofail) ? ". Retrying." : ""); + "vmalloc error: size %lu, align 0x%lx, vm_struct allocation failed%s", + size, align, (nofail) ? ". Retrying." : ""); if (nofail) { schedule_timeout_uninterruptible(1); goto again; @@ -4968,16 +5119,17 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) * @sizes: array containing size of each area * @nr_vms: the number of areas to allocate * @align: alignment, all entries in @offsets and @sizes must be aligned to this + * @gfp: allocation flags passed to the underlying memory allocator * * Returns: kmalloc'd vm_struct pointer array pointing to allocated * vm_structs on success, %NULL on failure * * Percpu allocator wants to use congruent vm areas so that it can * maintain the offsets among percpu areas. This function allocates - * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to - * be scattered pretty far, distance between two areas easily going up - * to gigabytes. To avoid interacting with regular vmallocs, these - * areas are allocated from top. + * congruent vmalloc areas for it. These areas tend to be scattered + * pretty far, distance between two areas easily going up to gigabytes. + * To avoid interacting with regular vmallocs, these areas are allocated + * from top. * * Despite its complicated look, this allocator is rather simple. It * does everything top-down and scans free blocks from the end looking @@ -4988,7 +5140,7 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) */ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); @@ -5026,14 +5178,14 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, return NULL; } - vms = kzalloc_objs(vms[0], nr_vms); - vas = kzalloc_objs(vas[0], nr_vms); + vms = kzalloc_objs(vms[0], nr_vms, gfp); + vas = kzalloc_objs(vas[0], nr_vms, gfp); if (!vas || !vms) goto err_free2; for (area = 0; area < nr_vms; area++) { - vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); - vms[area] = kzalloc_obj(struct vm_struct); + vas[area] = kmem_cache_zalloc(vmap_area_cachep, gfp); + vms[area] = kzalloc_obj(struct vm_struct, gfp); if (!vas[area] || !vms[area]) goto err_free; } @@ -5123,7 +5275,7 @@ retry: /* populate the kasan shadow space */ for (area = 0; area < nr_vms; area++) { - if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], GFP_KERNEL)) + if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], gfp)) goto err_free_shadow; } @@ -5180,7 +5332,7 @@ overflow: continue; vas[area] = kmem_cache_zalloc( - vmap_area_cachep, GFP_KERNEL); + vmap_area_cachep, gfp); if (!vas[area]) goto err_free; } @@ -5220,9 +5372,7 @@ err_free_shadow: kfree(vms[area]); } spin_unlock(&free_vmap_area_lock); - kfree(vas); - kfree(vms); - return NULL; + goto err_free2; } /** diff --git a/mm/vmalloc.h b/mm/vmalloc.h new file mode 100644 index 000000000000..dcfe30eaa80c --- /dev/null +++ b/mm/vmalloc.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal APIs for vmalloc + */ +#ifndef __MM_VMALLOC_H +#define __MM_VMALLOC_H + +#ifdef CONFIG_MMU +void __init vmalloc_init(void); +int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift, gfp_t gfp_mask); +unsigned int get_vm_area_page_order(struct vm_struct *vm); +#else +static inline void vmalloc_init(void) {} + +static inline +int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift, gfp_t gfp_mask) +{ + return -EINVAL; +} + +static inline void vunmap_range_noflush(unsigned long start, unsigned long end) +{ +} +#endif + +struct vm_struct *__get_vm_area_node(unsigned long size, + unsigned long align, unsigned long shift, + unsigned long vm_flags, unsigned long start, + unsigned long end, int node, gfp_t gfp_mask, + const void *caller); + +void clear_vm_uninitialized_flag(struct vm_struct *vm); + +int __must_check __vmap_pages_range_noflush(unsigned long addr, + unsigned long end, pgprot_t prot, + struct page **pages, unsigned int page_shift); + +void vunmap_range_noflush(unsigned long start, unsigned long end); + +void __vunmap_range_noflush(unsigned long start, unsigned long end); + +#endif /* __MM_VMALLOC_H */ diff --git a/mm/vmpressure.c b/mm/vmpressure.c index f053554e5826..9629240d77ad 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -7,16 +7,15 @@ * * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro, * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg. + * + * Tree-mode (cgroup v1 userspace eventfd) bookkeeping lives in + * mm/memcontrol-v1.c; this file holds the shared code and the in-kernel + * (tree=false) socket-pressure path that runs on cgroup v2. */ #include <linux/cgroup.h> -#include <linux/fs.h> #include <linux/log2.h> -#include <linux/sched.h> #include <linux/mm.h> -#include <linux/vmstat.h> -#include <linux/eventfd.h> -#include <linux/slab.h> #include <linux/swap.h> #include <linux/printk.h> #include <linux/vmpressure.h> @@ -35,7 +34,7 @@ * TODO: Make the window size depend on machine size, as we do for vmstat * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). */ -static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; +const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; /* * These thresholds are used when we account memory pressure through @@ -46,68 +45,6 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; static const unsigned int vmpressure_level_med = 60; static const unsigned int vmpressure_level_critical = 95; -/* - * When there are too little pages left to scan, vmpressure() may miss the - * critical pressure as number of pages will be less than "window size". - * However, in that case the vmscan priority will raise fast as the - * reclaimer will try to scan LRUs more deeply. - * - * The vmscan logic considers these special priorities: - * - * prio == DEF_PRIORITY (12): reclaimer starts with that value - * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed - * prio == 0 : close to OOM, kernel scans every page in an lru - * - * Any value in this range is acceptable for this tunable (i.e. from 12 to - * 0). Current value for the vmpressure_level_critical_prio is chosen - * empirically, but the number, in essence, means that we consider - * critical level when scanning depth is ~10% of the lru size (vmscan - * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one - * eights). - */ -static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); - -static struct vmpressure *work_to_vmpressure(struct work_struct *work) -{ - return container_of(work, struct vmpressure, work); -} - -static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) -{ - struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); - - memcg = parent_mem_cgroup(memcg); - if (!memcg) - return NULL; - return memcg_to_vmpressure(memcg); -} - -enum vmpressure_levels { - VMPRESSURE_LOW = 0, - VMPRESSURE_MEDIUM, - VMPRESSURE_CRITICAL, - VMPRESSURE_NUM_LEVELS, -}; - -enum vmpressure_modes { - VMPRESSURE_NO_PASSTHROUGH = 0, - VMPRESSURE_HIERARCHY, - VMPRESSURE_LOCAL, - VMPRESSURE_NUM_MODES, -}; - -static const char * const vmpressure_str_levels[] = { - [VMPRESSURE_LOW] = "low", - [VMPRESSURE_MEDIUM] = "medium", - [VMPRESSURE_CRITICAL] = "critical", -}; - -static const char * const vmpressure_str_modes[] = { - [VMPRESSURE_NO_PASSTHROUGH] = "default", - [VMPRESSURE_HIERARCHY] = "hierarchy", - [VMPRESSURE_LOCAL] = "local", -}; - static enum vmpressure_levels vmpressure_level(unsigned long pressure) { if (pressure >= vmpressure_level_critical) @@ -117,8 +54,8 @@ static enum vmpressure_levels vmpressure_level(unsigned long pressure) return VMPRESSURE_LOW; } -static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, - unsigned long reclaimed) +enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed) { unsigned long scale = scanned + reclaimed; unsigned long pressure = 0; @@ -147,74 +84,6 @@ out: return vmpressure_level(pressure); } -struct vmpressure_event { - struct eventfd_ctx *efd; - enum vmpressure_levels level; - enum vmpressure_modes mode; - struct list_head node; -}; - -static bool vmpressure_event(struct vmpressure *vmpr, - const enum vmpressure_levels level, - bool ancestor, bool signalled) -{ - struct vmpressure_event *ev; - bool ret = false; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ancestor && ev->mode == VMPRESSURE_LOCAL) - continue; - if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) - continue; - if (level < ev->level) - continue; - eventfd_signal(ev->efd); - ret = true; - } - mutex_unlock(&vmpr->events_lock); - - return ret; -} - -static void vmpressure_work_fn(struct work_struct *work) -{ - struct vmpressure *vmpr = work_to_vmpressure(work); - unsigned long scanned; - unsigned long reclaimed; - enum vmpressure_levels level; - bool ancestor = false; - bool signalled = false; - - spin_lock(&vmpr->sr_lock); - /* - * Several contexts might be calling vmpressure(), so it is - * possible that the work was rescheduled again before the old - * work context cleared the counters. In that case we will run - * just after the old work returns, but then scanned might be zero - * here. No need for any locks here since we don't care if - * vmpr->reclaimed is in sync. - */ - scanned = vmpr->tree_scanned; - if (!scanned) { - spin_unlock(&vmpr->sr_lock); - return; - } - - reclaimed = vmpr->tree_reclaimed; - vmpr->tree_scanned = 0; - vmpr->tree_reclaimed = 0; - spin_unlock(&vmpr->sr_lock); - - level = vmpressure_calc_level(scanned, reclaimed); - - do { - if (vmpressure_event(vmpr, level, ancestor, signalled)) - signalled = true; - ancestor = true; - } while ((vmpr = vmpressure_parent(vmpr))); -} - /** * vmpressure() - Account memory pressure through scanned/reclaimed ratio * @gfp: reclaimer's gfp mask @@ -246,11 +115,12 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; /* - * The in-kernel users only care about the reclaim efficiency - * for this @memcg rather than the whole subtree, and there - * isn't and won't be any in-kernel user in a legacy cgroup. + * Only two combinations have a consumer: + * cgroup v2 + tree=false -> in-kernel socket pressure + * cgroup v1 + tree=true -> userspace eventfds (memory.pressure_level) + * Skip the other two: nothing consumes the result. */ - if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !tree) + if (cgroup_subsys_on_dfl(memory_cgrp_subsys) == tree) return; vmpr = memcg_to_vmpressure(memcg); @@ -281,14 +151,7 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; if (tree) { - spin_lock(&vmpr->sr_lock); - scanned = vmpr->tree_scanned += scanned; - vmpr->tree_reclaimed += reclaimed; - spin_unlock(&vmpr->sr_lock); - - if (scanned < vmpressure_win) - return; - schedule_work(&vmpr->work); + vmpressure_v1_account_tree(vmpr, scanned, reclaimed); } else { enum vmpressure_levels level; @@ -331,134 +194,6 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, } /** - * vmpressure_prio() - Account memory pressure through reclaimer priority level - * @gfp: reclaimer's gfp mask - * @memcg: cgroup memory controller handle - * @prio: reclaimer's priority - * - * This function should be called from the reclaim path every time when - * the vmscan's reclaiming priority (scanning depth) changes. - * - * This function does not return any value. - */ -void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) -{ - /* - * We only use prio for accounting critical level. For more info - * see comment for vmpressure_level_critical_prio variable above. - */ - if (prio > vmpressure_level_critical_prio) - return; - - /* - * OK, the prio is below the threshold, updating vmpressure - * information before shrinker dives into long shrinking of long - * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 - * to the vmpressure() basically means that we signal 'critical' - * level. - */ - vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); -} - -#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) - -/** - * vmpressure_register_event() - Bind vmpressure notifications to an eventfd - * @memcg: memcg that is interested in vmpressure notifications - * @eventfd: eventfd context to link notifications with - * @args: event arguments (pressure level threshold, optional mode) - * - * This function associates eventfd context with the vmpressure - * infrastructure, so that the notifications will be delivered to the - * @eventfd. The @args parameter is a comma-delimited string that denotes a - * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", - * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. - * "hierarchy" or "local"). - * - * To be used as memcg event method. - * - * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could - * not be parsed. - */ -int vmpressure_register_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd, const char *args) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; - enum vmpressure_levels level; - char *spec, *spec_orig; - char *token; - int ret = 0; - - spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); - if (!spec) - return -ENOMEM; - - /* Find required level */ - token = strsep(&spec, ","); - ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); - if (ret < 0) - goto out; - level = ret; - - /* Find optional mode */ - token = strsep(&spec, ","); - if (token) { - ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); - if (ret < 0) - goto out; - mode = ret; - } - - ev = kzalloc_obj(*ev); - if (!ev) { - ret = -ENOMEM; - goto out; - } - - ev->efd = eventfd; - ev->level = level; - ev->mode = mode; - - mutex_lock(&vmpr->events_lock); - list_add(&ev->node, &vmpr->events); - mutex_unlock(&vmpr->events_lock); - ret = 0; -out: - kfree(spec_orig); - return ret; -} - -/** - * vmpressure_unregister_event() - Unbind eventfd from vmpressure - * @memcg: memcg handle - * @eventfd: eventfd context that was used to link vmpressure with the @cg - * - * This function does internal manipulations to detach the @eventfd from - * the vmpressure notifications, and then frees internal resources - * associated with the @eventfd (but the @eventfd itself is not freed). - * - * To be used as memcg event method. - */ -void vmpressure_unregister_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ev->efd != eventfd) - continue; - list_del(&ev->node); - kfree(ev); - break; - } - mutex_unlock(&vmpr->events_lock); -} - -/** * vmpressure_init() - Initialize vmpressure control structure * @vmpr: Structure to be initialized * @@ -468,9 +203,7 @@ void vmpressure_unregister_event(struct mem_cgroup *memcg, void vmpressure_init(struct vmpressure *vmpr) { spin_lock_init(&vmpr->sr_lock); - mutex_init(&vmpr->events_lock); - INIT_LIST_HEAD(&vmpr->events); - INIT_WORK(&vmpr->work, vmpressure_work_fn); + vmpressure_v1_init(vmpr); } /** @@ -482,9 +215,5 @@ void vmpressure_init(struct vmpressure *vmpr) */ void vmpressure_cleanup(struct vmpressure *vmpr) { - /* - * Make sure there is no pending work before eventfd infrastructure - * goes away. - */ - flush_work(&vmpr->work); + vmpressure_v1_cleanup(vmpr); } diff --git a/mm/vmscan.c b/mm/vmscan.c index 56708d1d2dfd..26df81e773ff 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -66,6 +66,7 @@ #include <linux/sched/sysctl.h> #include "internal.h" +#include "page_alloc.h" #include "swap.h" #define CREATE_TRACE_POINTS @@ -79,7 +80,7 @@ struct scan_control { * Nodemask of nodes allowed by the caller. If NULL, all nodes * are scanned. */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* * The memory cgroup that hit its limit and as a result is the @@ -267,6 +268,12 @@ static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) } #endif +static inline bool is_exec_file_folio(const struct folio *folio, + const vma_flags_t *vma_flags) +{ + return vma_flags_test(vma_flags, VMA_EXEC_BIT) && folio_is_file_lru(folio); +} + static void set_task_reclaim_state(struct task_struct *task, struct reclaim_state *rs) { @@ -615,8 +622,8 @@ typedef enum { /* * pageout is called by shrink_folio_list() for each dirty folio. */ -static pageout_t pageout(struct folio *folio, struct address_space *mapping, - struct swap_iocb **plug, struct list_head *folio_list) +static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping, + struct folio *folio, struct list_head *folio_list) { int res; @@ -652,9 +659,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping, * the split out folios get added back to folio_list. */ if (shmem_mapping(mapping)) - res = shmem_writeout(folio, plug, folio_list); + res = shmem_writeout(ctx, folio, folio_list); else - res = swap_writeout(folio, plug); + res = swap_writeout(ctx, folio); if (res < 0) handle_write_error(mapping, folio, res); @@ -823,7 +830,6 @@ void folio_putback_lru(struct folio *folio) enum folio_references { FOLIOREF_RECLAIM, - FOLIOREF_RECLAIM_CLEAN, FOLIOREF_KEEP, FOLIOREF_ACTIVATE, }; @@ -835,10 +841,16 @@ enum folio_references { * with PG_active set. In contrast, the aging (page table walk) path uses * folio_update_gen(). */ -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* Activate file-backed executable folios after first usage. */ + if (is_exec_file_folio(folio, vma_flags)) { + set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + return true; + } + set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return false; } @@ -851,7 +863,7 @@ static bool lru_gen_set_refs(struct folio *folio) return true; } #else -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { return false; } @@ -861,16 +873,16 @@ static enum folio_references folio_check_references(struct folio *folio, struct scan_control *sc) { int referenced_ptes, referenced_folio; - vm_flags_t vm_flags; + vma_flags_t vma_flags; referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, - &vm_flags); + &vma_flags); /* * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. * Let the folio, now marked Mlocked, be moved to the unevictable list. */ - if (vm_flags & VM_LOCKED) + if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) return FOLIOREF_ACTIVATE; /* @@ -886,7 +898,7 @@ static enum folio_references folio_check_references(struct folio *folio, if (!referenced_ptes) return FOLIOREF_RECLAIM; - return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; + return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; } referenced_folio = folio_test_clear_referenced(folio); @@ -914,16 +926,12 @@ static enum folio_references folio_check_references(struct folio *folio, /* * Activate file-backed executable folios after first usage. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) + if (is_exec_file_folio(folio, &vma_flags)) return FOLIOREF_ACTIVATE; return FOLIOREF_KEEP; } - /* Reclaim if clean, defer dirty folios to writeback */ - if (referenced_folio && folio_is_file_lru(folio)) - return FOLIOREF_RECLAIM_CLEAN; - return FOLIOREF_RECLAIM; } @@ -1037,16 +1045,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) { if (gfp_mask & __GFP_FS) return true; - if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) - return false; /* - * We can "enter_fs" for swap-cache with only __GFP_IO - * providing this isn't SWP_FS_OPS. - * ->flags can be updated non-atomically, - * but that will never affect SWP_FS_OPS, so the data_race - * is safe. + * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by + * a swapfile that requires GFP_NOFS I/O. */ - return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); + if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) && + !(__swap_entry_to_info(folio->swap)->ops->flags & + SWAP_OPS_F_REQUIRE_NOFS)) + return true; + return false; } /* @@ -1063,7 +1070,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, unsigned int nr_reclaimed = 0, nr_demoted = 0; unsigned int pgactivate = 0; bool do_demote_pass; - struct swap_iocb *plug = NULL; + struct swap_io_ctx ctx = {}; folio_batch_init(&free_folios); memset(stat, 0, sizeof(*stat)); @@ -1235,7 +1242,6 @@ retry: stat->nr_ref_keep += nr_pages; goto keep_locked; case FOLIOREF_RECLAIM: - case FOLIOREF_RECLAIM_CLEAN: ; /* try to reclaim the folio below */ } @@ -1381,8 +1387,6 @@ retry: goto activate_locked; } - if (references == FOLIOREF_RECLAIM_CLEAN) - goto keep_locked; if (!may_enter_fs(folio, sc->gfp_mask)) goto keep_locked; if (!sc->may_writepage) @@ -1394,7 +1398,7 @@ retry: * starts and then write it out here. */ try_to_unmap_flush_dirty(); - switch (pageout(folio, mapping, &plug, folio_list)) { + switch (pageout(&ctx, mapping, folio, folio_list)) { case PAGE_KEEP: goto keep_locked; case PAGE_ACTIVATE: @@ -1584,8 +1588,7 @@ keep: list_splice(&ret_folios, folio_list); count_vm_events(PGACTIVATE, pgactivate); - if (plug) - swap_write_unplug(plug); + swap_write_submit(&ctx); return nr_reclaimed; } @@ -2072,7 +2075,7 @@ static void shrink_active_list(unsigned long nr_to_scan, { unsigned long nr_taken; unsigned long nr_scanned; - vm_flags_t vm_flags; + vma_flags_t vma_flags; LIST_HEAD(l_hold); /* The folios which were snipped off */ LIST_HEAD(l_active); LIST_HEAD(l_inactive); @@ -2116,7 +2119,7 @@ static void shrink_active_list(unsigned long nr_to_scan, /* Referenced or rmap lock contention: rotate */ if (folio_referenced(folio, 0, sc->target_mem_cgroup, - &vm_flags) != 0) { + &vma_flags) != 0) { /* * Identify referenced, file-backed active folios and * give them one more trip around the active list. So @@ -2126,7 +2129,7 @@ static void shrink_active_list(unsigned long nr_to_scan, * IO, plus JVM can create lots of anon VM_EXEC folios, * so we ignore them here. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { + if (is_exec_file_folio(folio, &vma_flags)) { nr_rotated += folio_nr_pages(folio); list_add(&folio->lru, &l_active); continue; @@ -2501,6 +2504,23 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, enum scan_balance scan_balance; enum lru_list lru; + /* + * Proactive reclaim initiated by userspace for anonymous memory only. + * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so + * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g. + * no swap), bail out instead of falling back to evicting file pages, + * which would violate the anon-only semantics. + */ + if (swappiness == SWAPPINESS_ANON_ONLY) { + WARN_ON_ONCE(!sc->proactive); + if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { + memset(nr, 0, sizeof(*nr) * NR_LRU_LISTS); + return; + } + scan_balance = SCAN_ANON; + goto out; + } + /* If we have no swap space, do not bother scanning anon folios. */ if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { scan_balance = SCAN_FILE; @@ -2519,13 +2539,6 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } - /* Proactive reclaim initiated by userspace for anonymous memory only */ - if (swappiness == SWAPPINESS_ANON_ONLY) { - WARN_ON_ONCE(!sc->proactive); - scan_balance = SCAN_ANON; - goto out; - } - /* * Do not apply any pressure balancing cleverness when the * system is close to OOM, scan both anon and file equally @@ -2698,6 +2711,10 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) { struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); + int swappiness = sc_swappiness(sc, memcg); + + if (swappiness == SWAPPINESS_ANON_ONLY) + return swappiness; if (!sc->may_swap) return 0; @@ -2706,7 +2723,7 @@ static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) return 0; - return sc_swappiness(sc, memcg); + return swappiness; } static int get_nr_gens(struct lruvec *lruvec, int type) @@ -3195,14 +3212,19 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen) +static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) { unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* + * See the comment on LRU_REFS_FLAGS, and activate file-backed + * executable folios after first usage to avoid typical IO + * thrashing from reclaiming. + */ + if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && + !is_exec_file_folio(folio, vma_flags)) { set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return -1; } @@ -3435,8 +3457,8 @@ static bool suitable_to_scan(int total, int young) return young * n >= total; } -static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, - int new_gen, bool dirty) +static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, + struct folio *folio, int new_gen, bool dirty) { int old_gen; @@ -3449,10 +3471,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen); + old_gen = folio_update_gen(folio, new_gen, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) update_batch_size(walk, folio, old_gen, new_gen); - } else if (lru_gen_set_refs(folio)) { + } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) folio_activate(folio); @@ -3525,7 +3547,7 @@ restart: continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = folio; dirty = false; @@ -3538,7 +3560,7 @@ restart: walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3616,7 +3638,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area goto next; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -3630,7 +3652,7 @@ next: i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -4265,7 +4287,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -4277,7 +4299,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) young += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); @@ -4908,6 +4930,20 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, struct mem_cgroup *memcg, int swappiness) { unsigned long nr_to_scan, evictable; + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + + /* + * Proactive reclaim initiated by userspace for anonymous memory only. + * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so + * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g. + * no swap), return 0 to skip the scan entirely, avoiding useless scan + * work when there is nothing eligible to reclaim. + */ + if (swappiness == SWAPPINESS_ANON_ONLY) { + WARN_ON_ONCE(!sc->proactive); + if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) + return 0; + } evictable = lruvec_evictable_size(lruvec, swappiness); @@ -6594,7 +6630,7 @@ static bool allow_direct_reclaim(pg_data_t *pgdat) * happens, the page allocator should not consider triggering the OOM killer. */ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, - nodemask_t *nodemask) + const nodemask_t *nodemask) { struct zoneref *z; struct zone *zone; @@ -6674,7 +6710,7 @@ out: } unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *nodemask) + gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned long nr_reclaimed; struct scan_control sc = { @@ -7654,7 +7690,6 @@ static int __init kswapd_init(void) { int nid; - swap_setup(); for_each_node_state(nid, N_MEMORY) kswapd_run(nid); register_sysctl_init("vm", vmscan_sysctl_table); @@ -7738,7 +7773,7 @@ static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) /* * Try to free up some pages from this node through reclaim. */ -static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, +static unsigned long __node_reclaim(struct pglist_data *pgdat, unsigned long nr_pages, struct scan_control *sc) { @@ -7781,9 +7816,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, return sc->nr_reclaimed; } -int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) +unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) { - int ret; + unsigned long ret; /* Minimum pages needed in order to stay on node */ const unsigned long nr_pages = 1 << order; struct scan_control sc = { @@ -7810,13 +7845,13 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= pgdat->min_slab_pages) - return NODE_RECLAIM_FULL; + return 0; /* * Do not scan if the allocation should not be delayed. */ if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) - return NODE_RECLAIM_NOSCAN; + return 0; /* * Only run node reclaim on the local node or on nodes that do not @@ -7825,15 +7860,15 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) * as wide as possible. */ if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) - return NODE_RECLAIM_NOSCAN; + return 0; if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) - return NODE_RECLAIM_NOSCAN; + return 0; - ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc) >= nr_pages; + ret = __node_reclaim(pgdat, nr_pages, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); - if (ret) + if (ret >= nr_pages) count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS); else count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); @@ -7843,7 +7878,7 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) #else -static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, +static unsigned long __node_reclaim(struct pglist_data *pgdat, unsigned long nr_pages, struct scan_control *sc) { @@ -7919,6 +7954,10 @@ int user_proactive_reclaim(char *buf, if (signal_pending(current)) return -ERESTARTSYS; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg && memcg_is_dying(memcg)) + return -EAGAIN; + /* * This is the final attempt, drain percpu lru caches in the * hope of introducing more evictable pages. @@ -7952,8 +7991,7 @@ int user_proactive_reclaim(char *buf, &pgdat->flags)) return -EBUSY; - reclaimed = __node_reclaim(pgdat, gfp_mask, - batch_size, &sc); + reclaimed = __node_reclaim(pgdat, batch_size, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); } @@ -8020,7 +8058,7 @@ static ssize_t reclaim_store(struct device *dev, int ret, nid = dev->id; ret = user_proactive_reclaim((char *)buf, NULL, NODE_DATA(nid)); - return ret ? -EAGAIN : count; + return ret ? ret : count; } static DEVICE_ATTR_WO(reclaim); diff --git a/mm/vmstat.c b/mm/vmstat.c index f534972f517d..4e26e5fd6666 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -30,6 +30,7 @@ #include <linux/sched/isolation.h> #include "internal.h" +#include "page_alloc.h" #ifdef CONFIG_PROC_FS #ifdef CONFIG_NUMA @@ -1488,7 +1489,11 @@ const char * const vmstat_text[] = { #if THREAD_SIZE > 65536 [I(KSTACK_REST)] = "kstack_rest", #endif -#endif +#endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + [I(NRSWPIN)] = "nrswpin", + [I(NRSWPOUT)] = "nrswpout", +#endif /* CONFIG_SWAP */ #undef I #endif /* CONFIG_VM_EVENT_COUNTERS */ }; @@ -1568,7 +1573,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, static int frag_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; - walk_zones_in_node(m, pgdat, true, false, frag_show_print); + walk_zones_in_node(m, pgdat, true, true, frag_show_print); return 0; } diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 83f5820c45f9..b5eadee0e8f0 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -21,6 +21,10 @@ * pool->lock * class->lock * zspage->lock + * + * When ZS_OBJ_CLASS_BITS > 0, zs_free() skips pool->lock; it picks + * the size_class from obj's encoded class_idx and serializes against + * page migration via class->lock. */ #include <linux/module.h> @@ -67,8 +71,8 @@ #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS #else /* - * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just - * be PAGE_SHIFT + * If this definition of MAX_PHYSMEM_BITS is used, ZS_OBJ_PFN_SHIFT will + * just be PAGE_SHIFT */ #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG #endif @@ -88,8 +92,23 @@ #define OBJ_TAG_BITS 1 #define OBJ_TAG_MASK OBJ_ALLOCATED_TAG -#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS) -#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1) +/* + * obj is encoded as [PFN | class_idx | obj_idx] within an unsigned long: + * + * |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->| + * +-----------------+-------------------------+-----------------------+ + * | PFN | class_idx | obj_idx | + * +-----------------+-------------------------+-----------------------+ + * MSB ^ LSB + * | + * +-- ZS_OBJ_PFN_SHIFT + * + * Encoding class_idx into obj lets zs_free() locate the size_class + * without holding pool->lock; class_idx is invariant across page + * migration (only PFN changes), so a lockless read of the obj value + * always yields a valid class_idx. + */ +#define ZS_OBJ_PFN_SHIFT (BITS_PER_LONG - _PFN_BITS) #define HUGE_BITS 1 #define FULLNESS_BITS 4 @@ -98,9 +117,61 @@ #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL)) +/* + * Bits to index a page within a zspage = ceil(log2(ZS_MAX_PAGES_PER_ZSPAGE)). + * Computed at preprocessor time, for use in #if below. Kconfig + * restricts ZSMALLOC_CHAIN_SIZE to [4, 16]. + */ +#if ZS_MAX_PAGES_PER_ZSPAGE <= 4 +#define ZS_PAGES_PER_ZSPAGE_BITS 2 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 8 +#define ZS_PAGES_PER_ZSPAGE_BITS 3 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 16 +#define ZS_PAGES_PER_ZSPAGE_BITS 4 +#else +#error "ZSMALLOC_CHAIN_SIZE out of expected range [4,16]" +#endif + +/* + * Bits to index an object within a single PAGE_SIZE at the smallest + * possible object size: log2(PAGE_SIZE / 32) = PAGE_SHIFT - 5. + * 32 is the hard floor of ZS_MIN_ALLOC_SIZE. + */ +#define ZS_OBJS_PER_PAGE_BITS (PAGE_SHIFT - 5) + +/* + * Bits to index any object in the densest possible zspage. Below this, + * ZS_MIN_ALLOC_SIZE is auto-raised by the MAX(32, ...) formula -- still + * correct, but objects are coarser. + */ +#define ZS_OBJS_PER_ZSPAGE_BITS \ + (ZS_PAGES_PER_ZSPAGE_BITS + ZS_OBJS_PER_PAGE_BITS) + +/* + * Encode class_idx only when obj has spare bits; otherwise + * ZS_OBJ_CLASS_BITS folds to 0 (32-bit, or 64-bit UML/fallback). + */ +#if BITS_PER_LONG >= 64 && \ + ZS_OBJ_PFN_SHIFT >= (CLASS_BITS + 1) + ZS_OBJS_PER_ZSPAGE_BITS +#define ZS_OBJ_CLASS_BITS (CLASS_BITS + 1) +#else +#define ZS_OBJ_CLASS_BITS 0 +#endif +#define ZS_OBJ_CLASS_MASK ((_AC(1, UL) << ZS_OBJ_CLASS_BITS) - 1) + +#define ZS_OBJ_IDX_BITS (ZS_OBJ_PFN_SHIFT - ZS_OBJ_CLASS_BITS) +#define ZS_OBJ_IDX_MASK ((_AC(1, UL) << ZS_OBJ_IDX_BITS) - 1) + +/* + * Belt-and-suspenders: the #if above already guarantees this when + * class_idx is enabled. Catches future tweaks that bypass it. + */ +static_assert(ZS_OBJ_IDX_BITS >= ZS_PAGES_PER_ZSPAGE_BITS, + "zsmalloc: ZS_MIN_ALLOC_SIZE would exceed ZS_MAX_ALLOC_SIZE"); + /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */ #define ZS_MIN_ALLOC_SIZE \ - MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS)) + MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> ZS_OBJ_IDX_BITS)) /* each chunk includes extra space to keep handle */ #define ZS_MAX_ALLOC_SIZE PAGE_SIZE @@ -396,10 +467,13 @@ static void cache_free_zspage(struct zspage *zspage) kmem_cache_free(zspage_cachep, zspage); } -/* class->lock(which owns the handle) synchronizes races */ +/* + * Pairs with READ_ONCE() in handle_to_obj(): zs_free() may read the + * handle locklessly, so prevent store tearing here. + */ static void record_obj(unsigned long handle, unsigned long obj) { - *(unsigned long *)handle = obj; + WRITE_ONCE(*(unsigned long *)handle, obj); } static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc) @@ -720,33 +794,36 @@ static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc) static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc, unsigned int *obj_idx) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); - *obj_idx = (obj & OBJ_INDEX_MASK); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); + *obj_idx = (obj & ZS_OBJ_IDX_MASK); } static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); } /** - * location_to_obj - get obj value encoded from (<zpdesc>, <obj_idx>) + * location_to_obj - encode (<zpdesc>, <obj_idx>, <class_idx>) into obj value * @zpdesc: zpdesc object resides in zspage * @obj_idx: object index + * @class_idx: size class index; ignored when ZS_OBJ_CLASS_BITS == 0 */ -static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx) +static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx, + unsigned int class_idx) { unsigned long obj; - obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS; - obj |= obj_idx & OBJ_INDEX_MASK; + obj = zpdesc_pfn(zpdesc) << ZS_OBJ_PFN_SHIFT; + obj |= (unsigned long)(class_idx & ZS_OBJ_CLASS_MASK) << ZS_OBJ_IDX_BITS; + obj |= obj_idx & ZS_OBJ_IDX_MASK; return obj; } static unsigned long handle_to_obj(unsigned long handle) { - return *(unsigned long *)handle; + return READ_ONCE(*(unsigned long *)handle); } static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, @@ -800,13 +877,26 @@ unlock: return 0; } -static void __free_zspage(struct zs_pool *pool, struct size_class *class, - struct zspage *zspage) +/* + * Three free helpers, kept apart here: + * + * __free_zspage_lockless(): bare core; walks zpdescs and returns pages + * to the buddy allocator. Caller owns all zpdesc locks and has + * removed the zspage from its class list. Used by zs_free() outside + * class->lock so the buddy-side work does not stall the class. + * + * __free_zspage(): __free_zspage_lockless() + per-class accounting, + * under class->lock. Used by async_free_zspage(), the worker for + * zspages whose trylock_zspage() failed. + * + * free_zspage(): full wrapper - trylock zpdescs, remove from class + * list, call __free_zspage(); kicks deferred free on contention. + * Used by compaction. + */ +static inline void __free_zspage_lockless(struct zspage *zspage) { struct zpdesc *zpdesc, *next; - assert_spin_locked(&class->lock); - VM_BUG_ON(get_zspage_inuse(zspage)); VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0); @@ -822,7 +912,13 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class, } while (zpdesc != NULL); cache_free_zspage(zspage); +} +static void __free_zspage(struct zs_pool *pool, struct size_class *class, + struct zspage *zspage) +{ + assert_spin_locked(&class->lock); + __free_zspage_lockless(zspage); class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); } @@ -1275,7 +1371,7 @@ static unsigned long obj_malloc(struct zs_pool *pool, kunmap_local(vaddr); mod_zspage_inuse(zspage, 1); - obj = location_to_obj(m_zpdesc, obj); + obj = location_to_obj(m_zpdesc, obj, zspage->class); record_obj(handle, obj); return obj; @@ -1380,37 +1476,97 @@ static void obj_free(int class_size, unsigned long obj) mod_zspage_inuse(zspage, -1); } +#if (ZS_OBJ_CLASS_BITS > 0) || defined(CONFIG_COMPACTION) +/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */ +static unsigned int obj_to_class_idx(unsigned long obj) +{ + return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK; +} +#endif + +/* + * Resolve @handle to its zspage / size_class and acquire class->lock. + * + * When class_idx is encoded in obj (ZS_OBJ_CLASS_BITS > 0), it is + * invariant under page migration, so the handle can be read locklessly + * to pick the size_class. Once class->lock is held migration is + * blocked and the handle is re-read to obtain a stable PFN. + * + * Otherwise (32-bit, or 64-bit fallback paths like UML where the + * encoding is disabled), fall back to pool->lock for the lookup. + */ +#if ZS_OBJ_CLASS_BITS > 0 +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + obj = handle_to_obj(handle); + *classp = pool->size_class[obj_to_class_idx(obj)]; + spin_lock(&(*classp)->lock); + /* Re-read under class->lock: PFN is now stable vs migration. */ + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *objp = obj; +} +#else +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + read_lock(&pool->lock); + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *classp = zspage_class(pool, *zspagep); + spin_lock(&(*classp)->lock); + read_unlock(&pool->lock); + *objp = obj; +} +#endif + void zs_free(struct zs_pool *pool, unsigned long handle) { struct zspage *zspage; - struct zpdesc *f_zpdesc; unsigned long obj; struct size_class *class; int fullness; + struct zspage *zspage_to_free = NULL; if (IS_ERR_OR_NULL((void *)handle)) return; - /* - * The pool->lock protects the race with zpage's migration - * so it's safe to get the page from handle. - */ - read_lock(&pool->lock); - obj = handle_to_obj(handle); - obj_to_zpdesc(obj, &f_zpdesc); - zspage = get_zspage(f_zpdesc); - class = zspage_class(pool, zspage); - spin_lock(&class->lock); - read_unlock(&pool->lock); + obj_class_get_and_lock(pool, handle, &obj, &zspage, &class); class_stat_sub(class, ZS_OBJS_INUSE, 1); obj_free(class->size, obj); fullness = fix_fullness_group(class, zspage); - if (fullness == ZS_INUSE_RATIO_0) - free_zspage(pool, class, zspage); + if (fullness == ZS_INUSE_RATIO_0) { + if (trylock_zspage(zspage)) { + remove_zspage(class, zspage); + class_stat_sub(class, ZS_OBJS_ALLOCATED, + class->objs_per_zspage); + zspage_to_free = zspage; + } else { + kick_deferred_free(pool); + } + } spin_unlock(&class->lock); + + if (zspage_to_free) { + __free_zspage_lockless(zspage_to_free); + atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); + } cache_free_handle(handle); } EXPORT_SYMBOL_GPL(zs_free); @@ -1643,9 +1799,6 @@ static void lock_zspage(struct zspage *zspage) } zspage_read_unlock(zspage); } -#endif /* CONFIG_COMPACTION */ - -#ifdef CONFIG_COMPACTION static void replace_sub_page(struct size_class *class, struct zspage *zspage, struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc) @@ -1712,8 +1865,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page, pool = zspage->pool; /* - * The pool migrate_lock protects the race between zpage migration - * and zs_free. + * The pool migrate_lock protects against races between zpage migration + * and zs_free(), but only when ZS_OBJ_CLASS_BITS does not apply. */ write_lock(&pool->lock); class = zspage_class(pool, zspage); @@ -1761,7 +1914,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page, old_obj = handle_to_obj(handle); obj_to_location(old_obj, &dummy, &obj_idx); - new_obj = (unsigned long)location_to_obj(newzpdesc, obj_idx); + new_obj = location_to_obj(newzpdesc, obj_idx, + obj_to_class_idx(old_obj)); record_obj(handle, new_obj); } } @@ -1891,8 +2045,9 @@ static unsigned long __zs_compact(struct zs_pool *pool, unsigned long pages_freed = 0; /* - * protect the race between zpage migration and zs_free - * as well as zpage allocation/free + * Protect against races between zpage migration and zs_free() + * (only when ZS_OBJ_CLASS_BITS does not apply), as well as + * zpage allocation and free. */ write_lock(&pool->lock); spin_lock(&class->lock); diff --git a/mm/zswap.c b/mm/zswap.c index 761cd699e0a3..4e76a4a87cdc 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, struct folio *folio; struct mempolicy *mpol; struct swap_info_struct *si; + struct swap_io_ctx ctx = {}; int ret = 0; /* try to allocate swap cache folio */ @@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, folio_set_reclaim(folio); /* start writeback */ - __swap_writepage(folio, NULL); + __swap_writepage(&ctx, folio); + swap_write_submit(&ctx); out: if (ret) { @@ -1217,7 +1219,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, * Without memcg, use the zswap pool-wide metrics. */ if (!mem_cgroup_disabled()) { - mem_cgroup_flush_stats(memcg); + mem_cgroup_flush_stats_ratelimited(memcg); nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); } else { |