diff options
168 files changed, 4524 insertions, 1771 deletions
diff --git a/Documentation/admin-guide/blockdev/zram.rst b/Documentation/admin-guide/blockdev/zram.rst index 2f6bbfd991fe..148b7cf3b924 100644 --- a/Documentation/admin-guide/blockdev/zram.rst +++ b/Documentation/admin-guide/blockdev/zram.rst @@ -109,14 +109,41 @@ path to the `dict` along with other parameters:: #pass path to pre-trained zstd dictionary echo "algo=zstd dict=/etc/dictionary" > /sys/block/zram0/algorithm_params + #pass path to pre-trained zstd dictionary and compression level + echo "algo=zstd level=8 dict=/etc/dictionary" > \ + /sys/block/zram0/algorithm_params + #same, but using algorithm priority + echo "algo=zstd priority=1" > /sys/block/zram0/recomp_algorithm echo "priority=1 dict=/etc/dictionary" > \ /sys/block/zram0/algorithm_params - #pass path to pre-trained zstd dictionary and compression level +Each write to `algorithm_params` replaces the entire set of parameters of +the corresponding algorithm, parameters that are not listed in the write +are reset to their default values. Configure all of the parameters of an +algorithm in one write:: + + #WRONG: the second write resets level back to its default value + echo "algo=zstd level=8" > /sys/block/zram0/algorithm_params + echo "algo=zstd dict=/etc/dictionary" > /sys/block/zram0/algorithm_params + + #RIGHT echo "algo=zstd level=8 dict=/etc/dictionary" > \ /sys/block/zram0/algorithm_params +Select the compression algorithm before configuring its parameters. The +parameters of one algorithm are not necessarily valid for another one, so +changing the algorithm of a particular priority resets that priority's +parameters:: + + #WRONG: comp_algorithm write resets the previously configured level + echo "level=8" > /sys/block/zram0/algorithm_params + echo zstd > /sys/block/zram0/comp_algorithm + + #RIGHT + echo zstd > /sys/block/zram0/comp_algorithm + echo "algo=zstd level=8" > /sys/block/zram0/algorithm_params + Parameters are algorithm specific: not all algorithms support pre-trained dictionaries, not all algorithms support `level`. Furthermore, for certain algorithms `level` controls the compression level (the higher the value the @@ -124,6 +151,11 @@ better the compression ratio, it even can take negatives values for some algorithms), for other algorithms `level` is acceleration level (the higher the value the lower the compression ratio). +Parameters are handed over to the compression algorithm when the device is +initialised, hence invalid parameters (or parameters that the selected +algorithm does not support) are reported by the `disksize` write, and not +by the `algorithm_params` write that has configured them. + Set Disksize ============ diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4a12805a50ba..11eee89e544b 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2075,8 +2075,14 @@ Kernel parameters hugetlb_cma= [HW,CMA,EARLY] The size of a CMA area used for allocation of gigantic hugepages. Or using node format, the size of a CMA area per node can be specified. - Format: nn[KMGTPE] or (node format) - <node>:nn[KMGTPE][,<node>:nn[KMGTPE]] + The size can be an absolute value (e.g., 2G) or a + percentage of the total memory or node memory (e.g., 20%). + Percentage-derived sizes are rounded down to a multiple of + the architecture's gigantic hugepage size and may become + zero. + Format: nn[KMGTPE] or nn% or (node format) + <node>:nn[KMGTPE][,<node>:nn[KMGTPE]] or + <node>:nn%[,<node>:nn%] The size must be a multiple of the gigantic page size. When using node format, this applies to each per-node size. @@ -4844,6 +4850,12 @@ Kernel parameters Adjust the minimal page reporting order. The page reporting is disabled when it exceeds MAX_PAGE_ORDER. + page_reporting.page_reporting_delay_ms= + [KNL] Free page reporting delay in milliseconds + Format: <unsigned integer> + Adjust the delay in milliseconds between free page + reporting intervals. Default is 2000 (2 seconds). + panic= [KNL] Kernel behaviour on panic: delay <timeout> timeout > 0: seconds before rebooting timeout = 0: wait forever diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 16f37135ed80..b187d618452f 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -224,7 +224,7 @@ khugepaged will be automatically started when any THP size is enabled (either of the per-size anon control or the top-level control are set to "always" or "madvise"), and it'll be automatically shutdown when all THP sizes are disabled (when both the per-size anon control and the -top-level control are "never") +top-level control are "never"). process THP controls -------------------- @@ -301,7 +301,9 @@ being replaced by a PMD mapping, or (2) physical pages replaced by one hugepage of various sizes (PMD-sized or mTHP). Each may happen independently, or together, depending on the type of memory and the failures that occur. As such, this value should be interpreted roughly as a sign of progress, -and counters in /proc/vmstat consulted for more accurate accounting):: +and counters in /proc/vmstat consulted for more accurate accounting. +Per-order mTHP collapse statistics are also available under +/sys/kernel/mm/transparent_hugepage/hugepages-<size>kB/stats/):: /sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst index ccdd1615cf97..1eae24a9c18b 100644 --- a/Documentation/core-api/maple_tree.rst +++ b/Documentation/core-api/maple_tree.rst @@ -17,7 +17,8 @@ supports iterating over a range of entries and going to the previous or next entry in a cache-efficient manner. The tree can also be put into an RCU-safe mode of operation which allows reading and writing concurrently. Writers must synchronize on a lock, which can be the default spinlock, or the user can set -the lock to an external lock of a different type. +the lock to an external lock of a different type. Note that external locks may +interfere with allocations in a low memory situation. The Maple Tree maintains a small memory footprint and was designed to use modern processor cache efficiently. The majority of the users will be able to @@ -42,6 +43,15 @@ successful store operation within a given code segment when allocating cannot be done. Allocations of nodes are relatively small at around 256 bytes. +Since the maple tree uses internal nodes that are allocated and has rules on +data density, erasing an entry may cause allocations to occur. That is, +erasing an entry may consume memory. Users must take care to ensure that they +do not violate the larger system constraints on when and how memory is +allocated. Most situations are fine to allocate, but the pre-allocation +support is provided as a mechanism to avoid trickier situations. There is also +the possibility of using special entries and clean up the tree later, in +extreme circumstances. + .. _maple-tree-normal-api: Normal API @@ -63,7 +73,8 @@ success or an error code otherwise. mtree_store_range() works in the same way but takes a range. mtree_load() is used to retrieve the entry stored at a given index. You can use mtree_erase() to erase an entire range by only knowing one value within that range, or mtree_store() call with an entry of -NULL may be used to partially erase a range or many ranges at once. +NULL may be used to partially erase a range or many ranges at once. Note that +mtree_erase() may use GFP_KERNEL on allocations. If you want to only store a new entry to a range (or index) if that range is currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which @@ -163,7 +174,9 @@ You can use mas_erase() to erase an entire range by setting index and last of the maple state to the desired range to erase. This will erase the first range that is found in that range, set the maple state index and last as the range that was erased and return the entry that existed -at that location. +at that location. Note that mas_erase() may allocate with the GFP_KERNEL flag. +If this is not okay, consider using mas_store_gfp() and pass it a ``NULL``, +after setting up the correct range by walking to the entry. You can walk each entry within a range by using mas_for_each(). If you want to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as @@ -211,7 +224,7 @@ Advanced Locking The maple tree uses a spinlock by default, but external locks can be used for tree updates as well. To use an external lock, the tree must be initialized -with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the +with the ``MT_FLAGS_LOCK_EXTERN`` flag, this is usually done with the MTREE_INIT_EXT() #define, which takes an external lock as an argument. Functions and structures diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index 7d784e03f3f9..d1b690b17169 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -198,6 +198,16 @@ systems, because of pointers temporarily stored in CPU registers or stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing the minimum age of an object to be reported as a memory leak. +The ``min_unref_scans`` module parameter requires an object to be seen +unreferenced in that many consecutive scans before it is reported. It +defaults to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the +periodic scan thread confirms a leak on its own, and to 1 otherwise. A +value of 1 preserves the historical behaviour; higher values filter the +transient false positives described above, at the cost of delaying genuine +reports by up to that many scans. It can be set at boot with +``kmemleak.min_unref_scans=<n>`` or at run-time via +``/sys/module/kmemleak/parameters/min_unref_scans``. + Limitations and Drawbacks ------------------------- diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst index f309337fe110..6e2e511a70bb 100644 --- a/Documentation/filesystems/fscrypt.rst +++ b/Documentation/filesystems/fscrypt.rst @@ -1237,6 +1237,10 @@ astute users may notice some differences in behavior: - DAX (Direct Access) is not supported on encrypted files. +- Encrypted files cannot be used directly as swap files. To swap to + an encrypted file, set up a loopback device on top of it. + Alternatively, encrypted swap can use a dm-crypt device. + - The maximum length of an encrypted symlink is 2 bytes shorter than the maximum length of an unencrypted symlink. For example, on an EXT4 filesystem with a 4K block size, unencrypted symlinks can be up diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index c274c5eef733..844d65eb47a5 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -266,7 +266,6 @@ prototypes:: int (*error_remove_folio)(struct address_space *, struct folio *); int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span) int (*swap_deactivate)(struct file *); - int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter); locking rules: All except dirty_folio and free_folio may block @@ -291,7 +290,6 @@ is_partially_uptodate: yes error_remove_folio: yes swap_activate: no swap_deactivate: no -swap_rw: yes, unlocks ====================== ======================== ========= =============== ->write_begin(), ->write_end() and ->read_folio() may be called from @@ -355,14 +353,12 @@ should perform any validation and preparation necessary to ensure that writes can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted through -->swap_rw(), it should set SWP_FS_OPS, otherwise IO will be submitted -directly to the block device ``sis->bdev``. +the file system it should call swap_fs_activate, otherwise IO will be +submitted directly to the block device ``sis->bdev``. ->swap_deactivate() will be called in the sys_swapoff() path after ->swap_activate() returned success. -->swap_rw will be called for swap IO if SWP_FS_OPS was set by ->swap_activate(). - file_lock_operations ==================== diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 651b83b00440..d3a93eec3945 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -776,7 +776,6 @@ cache in your filesystem. The following members are defined: int (*error_remove_folio)(struct mapping *mapping, struct folio *); int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span) int (*swap_deactivate)(struct file *); - int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter); }; ``read_folio`` @@ -977,16 +976,13 @@ cache in your filesystem. The following members are defined: can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted - through ->swap_rw(), it should set SWP_FS_OPS, otherwise IO will - be submitted directly to the block device ``sis->bdev``. + through the file system it should call swap_fs_activate, otherwise IO + will be submitted directly to the block device ``sis->bdev``. ``swap_deactivate`` Called during swapoff on files where swap_activate was successful. -``swap_rw`` - Called to read or write swap pages when SWP_FS_OPS is set. - The File Object =============== diff --git a/Documentation/mm/allocation-profiling.rst b/Documentation/mm/allocation-profiling.rst index 5389d241176a..b2ebcef8af6f 100644 --- a/Documentation/mm/allocation-profiling.rst +++ b/Documentation/mm/allocation-profiling.rst @@ -43,9 +43,25 @@ sysctl: warnings produced by allocations made while profiling is disabled and freed when it's enabled. + /proc/sys/vm/mem_profiling_compressed + + 1: Page alloc tag compression is enabled. + + 0: Page alloc tag compression is disabled. + + This reflects a static boot-time configuration of how page allocation tags are + stored (in page flags when compression is enabled and in page_ext when disabled). + Toggling ``mem_profiling`` at runtime does not change the state of + ``mem_profiling_compressed``. + Runtime info: /proc/allocinfo + Profiling data can be retrieved either by reading `/proc/allocinfo` directly as + text or programmatically via `ioctl()` calls defined in `<uapi/linux/alloc_tag.h>`. + The ioctl interface supports structured binary data extraction as well as filtering + by module name, function, file, line number, accuracy, or allocation size limits. + Example output:: root@moria-kvm:~# sort -g /proc/allocinfo|tail|numfmt --to=iec diff --git a/Documentation/mm/ksm.rst b/Documentation/mm/ksm.rst index 2806e3e4a10e..2b4f72f1f953 100644 --- a/Documentation/mm/ksm.rst +++ b/Documentation/mm/ksm.rst @@ -24,13 +24,13 @@ tree. If a KSM page is shared between less than ``max_page_sharing`` VMAs, the node of the stable tree that represents such KSM page points to a -list of struct ksm_rmap_item and the ``page->mapping`` of the +list of struct ksm_rmap_item and the ``folio->mapping`` of the KSM page points to the stable tree node. When the sharing passes this threshold, KSM adds a second dimension to the stable tree. The tree node becomes a "chain" that links one or more "dups". Each "dup" keeps reverse mapping information for a KSM -page with ``page->mapping`` pointing to that "dup". +page with ``folio->mapping`` pointing to that "dup". Every "chain" and all "dups" linked into a "chain" enforce the invariant that they represent the same write protected memory content, diff --git a/Documentation/mm/page_migration.rst b/Documentation/mm/page_migration.rst index 34602b254aa6..5b8d50308db1 100644 --- a/Documentation/mm/page_migration.rst +++ b/Documentation/mm/page_migration.rst @@ -110,13 +110,13 @@ Steps: 6. The refcount of the page is examined and we back out if references remain. Otherwise, we know that we are the only one referencing this page. -7. The radix tree is checked and if it does not contain the pointer to this - page then we back out because someone else modified the radix tree. +7. The page cache tree is checked and if it does not contain the pointer to this + page then we back out because someone else modified the page cache tree. 8. The new page is prepped with some settings from the old page so that accesses to the new page will discover a page with the correct settings. -9. The radix tree is changed to point to the new page. +9. The page cache tree is changed to point to the new page. 10. The reference count of the old page is dropped because the address space reference is gone. A reference to the new page is established because diff --git a/Documentation/translations/zh_CN/mm/ksm.rst b/Documentation/translations/zh_CN/mm/ksm.rst index f0f458753d0c..822c7a289671 100644 --- a/Documentation/translations/zh_CN/mm/ksm.rst +++ b/Documentation/translations/zh_CN/mm/ksm.rst @@ -31,10 +31,10 @@ KSM维护着稳定树中的KSM页的逆映射信息。 当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了 KSM页的稳定树其中的节点指向了一个ksm_rmap_item结构体类型的列表。同时，这个KSM页 -的 ``page->mapping`` 指向了该稳定树节点。 +的 ``folio->mapping`` 指向了该稳定树节点。 如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多 -个稳定树"副本"的"链"。每个副本都保留KSM页的逆映射信息，其中 ``page->mapping`` +个稳定树"副本"的"链"。每个副本都保留KSM页的逆映射信息，其中 ``folio->mapping`` 指向该"副本"。 每个链以及链接到该链中的所有"副本"强制不变的是，它们代表了相同的写保护内存 diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 3f0ef1e27eb0..2fc53093752d 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -346,6 +346,8 @@ Code Seq# Include File Comments <mailto:luzmaximilian@gmail.com> 0xA5 20-2F linux/surface_aggregator/dtx.h Microsoft Surface DTX driver <mailto:luzmaximilian@gmail.com> +0xA6 00-0F uapi/linux/alloc_tag.h Memory allocation profiling + <mailto:surenb@google.com> 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h diff --git a/MAINTAINERS b/MAINTAINERS index 51336bb51367..4e0e59548b7a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16905,6 +16905,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git fixes F: Documentation/core-api/boot-time-mm.rst F: include/linux/kho/abi/memblock.h F: include/linux/memblock.h +F: mm/arch_numa.c F: mm/memblock.c F: mm/memtest.c F: mm/mm_init.c @@ -16964,7 +16965,9 @@ S: Maintained F: Documentation/mm/allocation-profiling.rst F: include/linux/alloc_tag.h F: include/linux/pgalloc_tag.h +F: include/uapi/linux/alloc_tag.h F: mm/alloc_tag.c +F: tools/testing/selftests/alloc_tag/ MEMORY MANAGEMENT - BALLOON M: Andrew Morton <akpm@linux-foundation.org> @@ -17137,6 +17140,7 @@ F: Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers F: Documentation/ABI/testing/sysfs-kernel-mm-numa F: Documentation/admin-guide/mm/ F: Documentation/mm/ +F: drivers/char/mem.c F: include/linux/cma.h F: include/linux/dmapool.h F: include/linux/ioremap.h @@ -17264,6 +17268,7 @@ S: Maintained F: Documentation/ABI/testing/sysfs-kernel-mm-swap F: Documentation/mm/swap-table.rst F: include/linux/swap.h +F: include/linux/swap_ops.h F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.c @@ -17340,6 +17345,7 @@ L: linux-mm@kvack.org S: Maintained W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +F: drivers/char/mem.c F: include/trace/events/mmap.h F: fs/proc/task_mmu.c F: fs/proc/task_nommu.c diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c index 4bf7c9012feb..cd5fded159c0 100644 --- a/arch/s390/mm/gmap_helpers.c +++ b/arch/s390/mm/gmap_helpers.c @@ -200,7 +200,7 @@ static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr, * currently only works in COW mappings, which is also where * mm_forbids_zeropage() is checked. */ - if (!is_cow_mapping(walk->vma->vm_flags)) + if (!vma_is_cow_mapping(walk->vma)) return -EFAULT; *found_addr = addr; diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 43f20ca95a2a..a9acf4575f4f 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -230,13 +230,6 @@ config GENERIC_ARCH_TOPOLOGY appropriate scaling, sysfs interface for reading capacity values at runtime. -config GENERIC_ARCH_NUMA - bool - select NUMA_MEMBLKS - help - Enable support for generic NUMA implementation. Currently, RISC-V - and ARM64 use it. - config FW_DEVLINK_SYNC_STATE_TIMEOUT bool "sync_state() behavior defaults to timeout instead of strict" help diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 8074a10183dc..435710f643a5 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -25,7 +25,6 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o obj-$(CONFIG_GENERIC_MSI_IRQ) += platform-msi.o obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o -obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-$(CONFIG_ACPI) += physical_location.o obj-y += test/ diff --git a/drivers/block/zram/backend_842.c b/drivers/block/zram/backend_842.c index 10d9d5c60f53..3846a04c69d7 100644 --- a/drivers/block/zram/backend_842.c +++ b/drivers/block/zram/backend_842.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "842: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/sw842.h> @@ -13,6 +15,14 @@ static void release_params_842(struct zcomp_params *params) static int setup_params_842(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index f92a52a720d1..f71b11bcac78 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "deflate: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/vmalloc.h> @@ -22,10 +24,29 @@ static void deflate_release_params(struct zcomp_params *params) static int deflate_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = Z_DEFAULT_COMPRESSION; - if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) + } else if (params->level < Z_DEFAULT_COMPRESSION || + params->level > Z_BEST_COMPRESSION) { + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } + + if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) { params->deflate.winbits = DEFLATE_DEF_WINBITS; + } else { + s32 wb = params->deflate.winbits; + + if ((wb < -15 || wb > -9) && (wb < 9 || wb > 15)) { + pr_err("invalid winbits %d\n", wb); + return -EINVAL; + } + } return 0; } diff --git a/drivers/block/zram/backend_lz4.c b/drivers/block/zram/backend_lz4.c index c449d511ba86..1e28104ad964 100644 --- a/drivers/block/zram/backend_lz4.c +++ b/drivers/block/zram/backend_lz4.c @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define pr_fmt(fmt) "lz4: " fmt + #include <linux/kernel.h> #include <linux/lz4.h> #include <linux/slab.h> @@ -28,8 +32,12 @@ static int lz4_setup_params(struct zcomp_params *params) LZ4_stream_t *dict_stream; int ret; - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = LZ4_ACCELERATION_DEFAULT; + } else if (params->level < LZ4_ACCELERATION_DEFAULT) { + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } if (!params->dict || !params->dict_sz) return 0; diff --git a/drivers/block/zram/backend_lz4hc.c b/drivers/block/zram/backend_lz4hc.c index f6a336acfe20..d8aa01bb258f 100644 --- a/drivers/block/zram/backend_lz4hc.c +++ b/drivers/block/zram/backend_lz4hc.c @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define pr_fmt(fmt) "lz4hc: " fmt + #include <linux/kernel.h> #include <linux/lz4.h> #include <linux/slab.h> @@ -18,8 +22,18 @@ static void lz4hc_release_params(struct zcomp_params *params) static int lz4hc_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = LZ4HC_DEFAULT_CLEVEL; + } else if (params->level < 1 || params->level > LZ4HC_MAX_CLEVEL) { + /* + * Use < 1 rather than < LZ4HC_MIN_CLEVEL here because + * LZ4HC_compress_generic() only clamps levels below 1 + * (levels 1 and 2 are valid). LZ4HC_MIN_CLEVEL (3) is + * advisory and not enforced by the library. + */ + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } return 0; } diff --git a/drivers/block/zram/backend_lzo.c b/drivers/block/zram/backend_lzo.c index 4c906beaae6b..d83f92cf757c 100644 --- a/drivers/block/zram/backend_lzo.c +++ b/drivers/block/zram/backend_lzo.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "lzo: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/lzo.h> @@ -12,6 +14,14 @@ static void lzo_release_params(struct zcomp_params *params) static int lzo_setup_params(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_lzorle.c b/drivers/block/zram/backend_lzorle.c index 10640c96cbfc..1b120d062c92 100644 --- a/drivers/block/zram/backend_lzorle.c +++ b/drivers/block/zram/backend_lzorle.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "lzo-rle: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/lzo.h> @@ -12,6 +14,14 @@ static void lzorle_release_params(struct zcomp_params *params) static int lzorle_setup_params(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_zstd.c b/drivers/block/zram/backend_zstd.c index d00b548056dc..08da3810cffd 100644 --- a/drivers/block/zram/backend_zstd.c +++ b/drivers/block/zram/backend_zstd.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "zstd: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/vmalloc.h> @@ -58,8 +60,13 @@ static int zstd_setup_params(struct zcomp_params *params) return -ENOMEM; params->drv_data = zp; - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = zstd_default_clevel(); + } else if (params->level < zstd_min_clevel() || + params->level > zstd_max_clevel()) { + pr_err("invalid compression level %d\n", params->level); + goto error; + } zp->cprm = zstd_get_params(params->level, PAGE_SIZE); @@ -85,7 +92,6 @@ static int zstd_setup_params(struct zcomp_params *params) return 0; error: - zstd_release_params(params); return -EINVAL; } @@ -161,7 +167,6 @@ static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx) return 0; error: - zstd_release_params(params); zstd_destroy(ctx); return -EINVAL; } diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 4bfe63a5225d..a9b3bb1d3bef 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -56,7 +56,7 @@ static size_t huge_class_size; static const struct block_device_operations zram_devops; -static void slot_free(struct zram *zram, u32 index); +static void slot_free(struct zram *zram, unsigned long index); /* * entry locking rules: @@ -70,11 +70,11 @@ static void slot_free(struct zram *zram, u32 index); * 4) Use TRY lock variant when in atomic context * - must check return value and handle locking failers */ -static __must_check bool slot_trylock(struct zram *zram, u32 index) +static __must_check bool slot_trylock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; - if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK, lock)) { + if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK_BIT, lock)) { mutex_acquire(&zram->table_lock_map, 0, 1, _RET_IP_); lock_acquired(&zram->table_lock_map, _RET_IP_); return true; @@ -83,21 +83,21 @@ static __must_check bool slot_trylock(struct zram *zram, u32 index) return false; } -static void slot_lock(struct zram *zram, u32 index) +static void slot_lock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; mutex_acquire(&zram->table_lock_map, 0, 0, _RET_IP_); - wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE); + wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK_BIT, TASK_UNINTERRUPTIBLE); lock_acquired(&zram->table_lock_map, _RET_IP_); } -static void slot_unlock(struct zram *zram, u32 index) +static void slot_unlock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; mutex_release(&zram->table_lock_map, _RET_IP_); - clear_and_wake_up_bit(ZRAM_ENTRY_LOCK, lock); + clear_and_wake_up_bit(ZRAM_ENTRY_LOCK_BIT, lock); } static inline bool init_done(struct zram *zram) @@ -110,55 +110,56 @@ static inline struct zram *dev_to_zram(struct device *dev) return (struct zram *)dev_to_disk(dev)->private_data; } -static unsigned long get_slot_handle(struct zram *zram, u32 index) +static unsigned long get_slot_handle(struct zram *zram, unsigned long index) { return zram->table[index].handle; } -static void set_slot_handle(struct zram *zram, u32 index, unsigned long handle) +static void set_slot_handle(struct zram *zram, unsigned long index, + unsigned long handle) { zram->table[index].handle = handle; } -static bool test_slot_flag(struct zram *zram, u32 index, +static bool test_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { return zram->table[index].attr.flags & BIT(flag); } -static void set_slot_flag(struct zram *zram, u32 index, +static void set_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { zram->table[index].attr.flags |= BIT(flag); } -static void clear_slot_flag(struct zram *zram, u32 index, +static void clear_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { zram->table[index].attr.flags &= ~BIT(flag); } -static size_t get_slot_size(struct zram *zram, u32 index) +static size_t get_slot_size(struct zram *zram, unsigned long index) { return zram->table[index].attr.flags & (BIT(ZRAM_FLAG_SHIFT) - 1); } -static void set_slot_size(struct zram *zram, u32 index, size_t size) +static void set_slot_size(struct zram *zram, unsigned long index, size_t size) { unsigned long flags = zram->table[index].attr.flags >> ZRAM_FLAG_SHIFT; zram->table[index].attr.flags = (flags << ZRAM_FLAG_SHIFT) | size; } -static inline bool slot_allocated(struct zram *zram, u32 index) +static inline bool slot_allocated(struct zram *zram, unsigned long index) { return get_slot_size(zram, index) || test_slot_flag(zram, index, ZRAM_SAME) || test_slot_flag(zram, index, ZRAM_WB); } -static inline void set_slot_comp_priority(struct zram *zram, u32 index, - u32 prio) +static inline void set_slot_comp_priority(struct zram *zram, + unsigned long index, u32 prio) { prio &= ZRAM_COMP_PRIORITY_MASK; /* @@ -170,14 +171,14 @@ static inline void set_slot_comp_priority(struct zram *zram, u32 index, zram->table[index].attr.flags |= (prio << ZRAM_COMP_PRIORITY_BIT1); } -static inline u32 get_slot_comp_priority(struct zram *zram, u32 index) +static inline u32 get_slot_comp_priority(struct zram *zram, unsigned long index) { u32 prio = zram->table[index].attr.flags >> ZRAM_COMP_PRIORITY_BIT1; return prio & ZRAM_COMP_PRIORITY_MASK; } -static void mark_slot_accessed(struct zram *zram, u32 index) +static void mark_slot_accessed(struct zram *zram, unsigned long index) { clear_slot_flag(zram, index, ZRAM_IDLE); clear_slot_flag(zram, index, ZRAM_PP_SLOT); @@ -284,7 +285,7 @@ static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl) } static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl, - u32 index) + unsigned long index) { struct zram_pp_slot *pps; u32 bid; @@ -418,7 +419,7 @@ static void mark_idle(struct zram *zram, ktime_t cutoff) { int is_idle = 1; unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; - int index; + unsigned long index; for (index = 0; index < nr_pages; index++) { /* @@ -485,8 +486,9 @@ static ssize_t idle_store(struct device *dev, struct device_attribute *attr, #define INVALID_BDEV_BLOCK (~0UL) static int read_from_zspool_raw(struct zram *zram, struct page *page, - u32 index); -static int read_from_zspool(struct zram *zram, struct page *page, u32 index); + unsigned long index); +static int read_from_zspool(struct zram *zram, struct page *page, + unsigned long index); struct zram_wb_ctl { /* idle list is accessed only by the writeback task, no concurency */ @@ -522,7 +524,7 @@ struct zram_rb_req { /* error status (sync read) */ int error; }; - u32 index; + unsigned long index; }; #define FOUR_K(x) ((x) * (1 << (PAGE_SHIFT - 12))) @@ -910,7 +912,7 @@ static void zram_account_writeback_submit(struct zram *zram) static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req) { - u32 index = req->pps->index; + unsigned long index = req->pps->index; int err; err = blk_status_to_errno(req->bio.bi_status); @@ -1032,7 +1034,7 @@ static int zram_writeback_slots(struct zram *zram, struct zram_wb_req *req = NULL; struct zram_pp_slot *pps; int ret = 0, err = 0; - u32 index = 0; + unsigned long index = 0; while ((pps = select_pp_slot(ctl))) { if (zram->wb_limit_enable && !zram->bd_wb_limit) { @@ -1198,7 +1200,7 @@ static void scan_slots_for_writeback(struct zram *zram, u32 mode, unsigned long lo, unsigned long hi, struct zram_pp_ctl *ctl) { - u32 index = lo; + unsigned long index = lo; while (index < hi) { bool ok = true; @@ -1235,8 +1237,8 @@ static ssize_t writeback_store(struct device *dev, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - u64 nr_pages = zram->disksize >> PAGE_SHIFT; - unsigned long lo = 0, hi = nr_pages; + unsigned long nr_pages; + unsigned long lo = 0, hi; struct zram_pp_ctl *pp_ctl = NULL; struct zram_wb_ctl *wb_ctl = NULL; char *args, *param, *val; @@ -1250,6 +1252,9 @@ static ssize_t writeback_store(struct device *dev, if (!zram->backing_dev) return -ENODEV; + nr_pages = zram->disksize >> PAGE_SHIFT; + hi = nr_pages; + pp_ctl = init_pp_ctl(); if (!pp_ctl) return -ENOMEM; @@ -1333,7 +1338,8 @@ out: return ret; } -static int decompress_bdev_page(struct zram *zram, struct page *page, u32 index) +static int decompress_bdev_page(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned int size; @@ -1375,7 +1381,7 @@ static void zram_deferred_decompress(struct work_struct *w) struct zram_rb_req *req = container_of(w, struct zram_rb_req, work); struct page *page = bio_first_page_all(req->bio); struct zram *zram = req->zram; - u32 index = req->index; + unsigned long index = req->index; int ret; ret = decompress_bdev_page(zram, page, index); @@ -1426,7 +1432,7 @@ static void zram_async_read_endio(struct bio *bio) } static int read_from_bdev_async(struct zram *zram, struct page *page, - u32 index, unsigned long blk_idx, + unsigned long index, unsigned long blk_idx, struct bio *parent) { struct zram_rb_req *req; @@ -1476,8 +1482,8 @@ static void zram_sync_read(struct work_struct *w) * chained IO with parent IO in same context, it's a deadlock. To avoid that, * use a worker thread context. */ -static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx) +static int read_from_bdev_sync(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx) { struct zram_rb_req req; @@ -1496,8 +1502,9 @@ static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index, return decompress_bdev_page(zram, page, index); } -static int read_from_bdev(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx, struct bio *parent) +static int read_from_bdev(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { atomic64_inc(&zram->stats.bd_reads); if (!parent) { @@ -1509,8 +1516,9 @@ static int read_from_bdev(struct zram *zram, struct page *page, u32 index, } #else static inline void reset_bdev(struct zram *zram) {}; -static int read_from_bdev(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx, struct bio *parent) +static int read_from_bdev(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { return -EIO; } @@ -1538,9 +1546,10 @@ static ssize_t read_block_state(struct file *file, char __user *buf, size_t count, loff_t *ppos) { char *kbuf; - ssize_t index, written = 0; + unsigned long index; + ssize_t written = 0; struct zram *zram = file->private_data; - unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; + unsigned long nr_pages; kbuf = kvmalloc(count, GFP_KERNEL); if (!kbuf) @@ -1552,6 +1561,8 @@ static ssize_t read_block_state(struct file *file, char __user *buf, return -EINVAL; } + nr_pages = zram->disksize >> PAGE_SHIFT; + for (index = *ppos; index < nr_pages; index++) { int copied; @@ -1560,7 +1571,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf, goto next; copied = snprintf(kbuf + written, count, - "%12zd %12u.%06d %c%c%c%c%c%c\n", + "%12lu %12u.%06d %c%c%c%c%c%c\n", index, zram->table[index].attr.ac_time, 0, test_slot_flag(zram, index, ZRAM_SAME) ? 's' : '.', test_slot_flag(zram, index, ZRAM_WB) ? 'w' : '.', @@ -1652,6 +1663,17 @@ static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg) zram->comp_algs[prio] = alg; } +static void comp_params_reset(struct zram *zram, u32 prio) +{ + struct zcomp_params *params = &zram->params[prio]; + + vfree(params->dict); + params->level = ZCOMP_PARAM_NOT_SET; + params->deflate.winbits = ZCOMP_PARAM_NOT_SET; + params->dict_sz = 0; + params->dict = NULL; +} + static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf) { const char *alg; @@ -1672,20 +1694,10 @@ static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf) } comp_algorithm_set(zram, prio, alg); + comp_params_reset(zram, prio); return 0; } -static void comp_params_reset(struct zram *zram, u32 prio) -{ - struct zcomp_params *params = &zram->params[prio]; - - vfree(params->dict); - params->level = ZCOMP_PARAM_NOT_SET; - params->deflate.winbits = ZCOMP_PARAM_NOT_SET; - params->dict_sz = 0; - params->dict = NULL; -} - static int comp_params_store(struct zram *zram, u32 prio, s32 level, const char *dict_path, struct deflate_params *deflate_params) @@ -1700,8 +1712,16 @@ static int comp_params_store(struct zram *zram, u32 prio, s32 level, INT_MAX, NULL, READING_POLICY); - if (sz < 0) + if (sz < 0) { + pr_err("failed to load dictionary %s (err=%zd)\n", + dict_path, sz); + return sz; + } + if (sz == 0) { + pr_err("failed to load dictionary %s (empty file)\n", + dict_path); return -EINVAL; + } } zram->params[prio].dict_sz = sz; @@ -1958,8 +1978,8 @@ static ssize_t debug_stat_show(struct device *dev, static void zram_meta_free(struct zram *zram, u64 disksize) { - size_t num_pages = disksize >> PAGE_SHIFT; - size_t index; + unsigned long num_pages = disksize >> PAGE_SHIFT; + unsigned long index; if (!zram->table) return; @@ -1976,7 +1996,7 @@ static void zram_meta_free(struct zram *zram, u64 disksize) static bool zram_meta_alloc(struct zram *zram, u64 disksize) { - size_t num_pages; + unsigned long num_pages; num_pages = disksize >> PAGE_SHIFT; zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); @@ -1999,7 +2019,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) return true; } -static void slot_free(struct zram *zram, u32 index) +static void slot_free(struct zram *zram, unsigned long index) { unsigned long handle; @@ -2053,7 +2073,7 @@ out: } static int read_same_filled_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { void *mem; @@ -2064,7 +2084,7 @@ static int read_same_filled_page(struct zram *zram, struct page *page, } static int read_incompressible_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { unsigned long handle; void *src, *dst; @@ -2079,7 +2099,8 @@ static int read_incompressible_page(struct zram *zram, struct page *page, return 0; } -static int read_compressed_page(struct zram *zram, struct page *page, u32 index) +static int read_compressed_page(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned long handle; @@ -2104,7 +2125,8 @@ static int read_compressed_page(struct zram *zram, struct page *page, u32 index) } #if defined CONFIG_ZRAM_WRITEBACK -static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) +static int read_from_zspool_raw(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned long handle; @@ -2136,7 +2158,8 @@ static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) * Reads (decompresses if needed) a page from zspool (zsmalloc). * Corresponding ZRAM slot should be locked. */ -static int read_from_zspool(struct zram *zram, struct page *page, u32 index) +static int read_from_zspool(struct zram *zram, struct page *page, + unsigned long index) { if (test_slot_flag(zram, index, ZRAM_SAME) || !get_slot_handle(zram, index)) @@ -2148,8 +2171,8 @@ static int read_from_zspool(struct zram *zram, struct page *page, u32 index) return read_incompressible_page(zram, page, index); } -static int zram_read_page(struct zram *zram, struct page *page, u32 index, - struct bio *parent) +static int zram_read_page(struct zram *zram, struct page *page, + unsigned long index, struct bio *parent) { int ret; @@ -2171,7 +2194,7 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, /* Should NEVER happen. Return bio error if it does. */ if (WARN_ON(ret < 0)) - pr_err("Decompression failed! err=%d, page=%u\n", ret, index); + pr_err("Decompression failed! err=%d, page=%lu\n", ret, index); return ret; } @@ -2181,7 +2204,7 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, * always expects a full page for the output. */ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { struct page *page = alloc_page(GFP_NOIO); int ret; @@ -2196,7 +2219,7 @@ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, } static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset, struct bio *bio) + unsigned long index, int offset, struct bio *bio) { if (is_partial_io(bvec)) return zram_bvec_read_partial(zram, bvec, index, offset); @@ -2204,7 +2227,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, } static int write_same_filled_page(struct zram *zram, unsigned long fill, - u32 index) + unsigned long index) { slot_lock(zram, index); slot_free(zram, index); @@ -2219,7 +2242,7 @@ static int write_same_filled_page(struct zram *zram, unsigned long fill, } static int write_incompressible_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { unsigned long handle; void *src; @@ -2259,7 +2282,8 @@ static int write_incompressible_page(struct zram *zram, struct page *page, return 0; } -static int zram_write_page(struct zram *zram, struct page *page, u32 index) +static int zram_write_page(struct zram *zram, struct page *page, + unsigned long index) { int ret = 0; unsigned long handle; @@ -2326,7 +2350,7 @@ static int zram_write_page(struct zram *zram, struct page *page, u32 index) * This is a partial IO. Read the full page before writing the changes. */ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { struct page *page = alloc_page(GFP_NOIO); int ret; @@ -2344,7 +2368,7 @@ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, } static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { if (is_partial_io(bvec)) return zram_bvec_write_partial(zram, bvec, index, offset); @@ -2412,8 +2436,9 @@ next: * * Corresponding ZRAM slot should be locked. */ -static int recompress_slot(struct zram *zram, u32 index, struct page *page, - u64 *num_recomp_pages, u32 threshold, u32 prio) +static int recompress_slot(struct zram *zram, unsigned long index, + struct page *page, u64 *num_recomp_pages, + u32 threshold, u32 prio) { struct zcomp_strm *zstrm = NULL; unsigned long handle_old; @@ -2665,7 +2690,7 @@ out: static void zram_bio_discard(struct zram *zram, struct bio *bio) { size_t n = bio->bi_iter.bi_size; - u32 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; @@ -2706,7 +2731,7 @@ static void zram_bio_read(struct zram *zram, struct bio *bio) struct bvec_iter iter = bio->bi_iter; do { - u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; struct bio_vec bv = bio_iter_iovec(bio, iter); @@ -2737,7 +2762,7 @@ static void zram_bio_write(struct zram *zram, struct bio *bio) struct bvec_iter iter = bio->bi_iter; do { - u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; struct bio_vec bv = bio_iter_iovec(bio, iter); @@ -2828,6 +2853,7 @@ static void zram_destroy_comps(struct zram *zram) zram->comp_algs[prio] = NULL; zram_comp_params_reset(zram); + comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static void zram_reset_device(struct zram *zram) @@ -2845,13 +2871,12 @@ static void zram_reset_device(struct zram *zram) zram_destroy_comps(zram); memset(&zram->stats, 0, sizeof(zram->stats)); reset_bdev(zram); - - comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + unsigned long num_pages; u64 disksize; struct zcomp *comp; struct zram *zram = dev_to_zram(dev); @@ -2869,6 +2894,11 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, } disksize = PAGE_ALIGN(disksize); + num_pages = disksize >> PAGE_SHIFT; + /* Slots are addressed by an unsigned long index */ + if (!num_pages || ((u64)num_pages << PAGE_SHIFT) != disksize) + return -EINVAL; + if (!zram_meta_alloc(zram, disksize)) return -ENOMEM; diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 4fddc582f3b8..7a55d751417e 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -15,6 +15,7 @@ #ifndef _ZRAM_DRV_H_ #define _ZRAM_DRV_H_ +#include <asm/byteorder.h> #include <linux/rwsem.h> #include <linux/zsmalloc.h> @@ -58,6 +59,19 @@ enum zram_pageflags { }; /* + * The slot lock is a bit-wait lock on the whole __lock word, while + * flags and ac_time alias that word as two u32s. The lock bit must + * land in the slot that ZRAM_ENTRY_LOCK reserves in attr.flags; on + * 64-bit big-endian the flags word maps to the upper half of __lock, + * so the bit position has to be shifted up. + */ +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) +#define ZRAM_ENTRY_LOCK_BIT (ZRAM_ENTRY_LOCK + 32) +#else +#define ZRAM_ENTRY_LOCK_BIT ZRAM_ENTRY_LOCK +#endif + +/* * Allocated for each disk page. We use bit-lock (ZRAM_ENTRY_LOCK bit * of flags) to save memory. There can be plenty of entries and standard * locking primitives (e.g. mutex) will significantly increase sizeof() diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index f754a4a3a1c2..165344f7197f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -377,9 +377,9 @@ static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_str /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings * for debugger access to invisible VRAM. Should have used MAP_SHARED * instead. Clearing VM_MAYWRITE prevents the mapping from ever - * becoming writable and makes is_cow_mapping(vm_flags) false. + * becoming writable and makes vma_is_cow_mapping(vma) false. */ - if (is_cow_mapping(vma->vm_flags) && + if (vma_is_cow_mapping(vma) && !(vma->vm_flags & VM_ACCESS_FLAGS)) vm_flags_clear(vma, VM_MAYWRITE); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 152014643a2d..27928bced7c5 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -757,7 +757,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; } - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; dma_resv_lock(shmem->base.resv, NULL); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 06543ae60706..ef9440166295 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping) if (folio_mapped(folio)) folio_redirty_for_writepage(&wbc, folio); else - error = shmem_writeout(folio, NULL, NULL); + error = shmem_write_folio(folio); } } diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index fdcbbe984f4e..dfe102a1109c 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -761,7 +761,7 @@ static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *v return ret; } - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) { diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c index 3c067aadc52d..0c2d53a13b2a 100644 --- a/drivers/gpu/drm/ttm/ttm_backup.c +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -160,7 +160,7 @@ ttm_backup_backup_folio(struct file *backup, struct folio *folio, if (writeback && !folio_mapped(to_folio) && folio_clear_dirty_for_io(to_folio)) { folio_set_reclaim(to_folio); - ret = shmem_writeout(to_folio, NULL, NULL); + ret = shmem_write_folio(to_folio); if (!folio_test_writeback(to_folio)) folio_clear_reclaim(to_folio); if (ret == AOP_WRITEPAGE_ACTIVATE) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 88babf435ac2..872bf444b1f0 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -489,7 +489,7 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { /* Enforce no COW since would have really strange behavior with it. */ - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; drm_gem_object_get(&bo->base); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index ad2722e4d6ff..3e70f2d39df2 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -330,7 +330,7 @@ static int xe_pci_barrier_mmap(struct file *filp, if (vma->vm_end - vma->vm_start > SZ_4K) return -EINVAL; - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; if (vma->vm_flags & (VM_READ | VM_EXEC)) diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index e626c72495e6..ccafdc1ce64d 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -145,26 +145,6 @@ static void nfs_direct_file_adjust_size_locked(struct inode *inode, } } -/** - * nfs_swap_rw - NFS address space operation for swap I/O - * @iocb: target I/O control block - * @iter: I/O buffer - * - * Perform IO to the swap-file. This is much like direct IO. - */ -int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter) -{ - ssize_t ret; - - if (iov_iter_rw(iter) == READ) - ret = nfs_file_direct_read(iocb, iter, true); - else - ret = nfs_file_direct_write(iocb, iter, true); - if (ret < 0) - return ret; - return 0; -} - static void nfs_direct_release_pages(struct page **pages, unsigned int npages) { unsigned int i; diff --git a/fs/nfs/file.c b/fs/nfs/file.c index a0d8f1c1cf10..e1bdd10b35f1 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -29,9 +29,8 @@ #include <linux/pagemap.h> #include <linux/gfp.h> #include <linux/rmap.h> -#include <linux/swap.h> #include <linux/compaction.h> - +#include <linux/swap_ops.h> #include <linux/uaccess.h> #include <linux/filelock.h> @@ -575,6 +574,38 @@ static int nfs_launder_folio(struct folio *folio) return ret; } +#ifdef CONFIG_SWAP +static void nfs_swap_submit_write(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct iov_iter iter; + int ret; + + swap_fs_prepare_rw(ctx, WRITE, &iter); + ret = nfs_file_direct_write(&sio->iocb, &iter, true); + if (ret != -EIOCBQUEUED) + sio->iocb.ki_complete(&sio->iocb, ret); +} + +static void nfs_swap_submit_read(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct iov_iter iter; + int ret; + + swap_fs_prepare_rw(ctx, READ, &iter); + ret = nfs_file_direct_read(&sio->iocb, &iter, true); + if (ret != -EIOCBQUEUED) + sio->iocb.ki_complete(&sio->iocb, ret); +} + +static const struct swap_ops nfs_swap_ops = { + .flags = SWAP_OPS_F_REQUIRE_NOFS, + .submit_write = nfs_swap_submit_write, + .submit_read = nfs_swap_submit_read, + .can_merge = swap_fs_can_merge, +}; + static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, sector_t *span) { @@ -597,7 +628,7 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, ret = rpc_clnt_swap_activate(clnt); if (ret) return ret; - ret = add_swap_extent(sis, 0, sis->max, 0); + ret = swap_fs_activate(sis, &nfs_swap_ops); if (ret < 0) { rpc_clnt_swap_deactivate(clnt); return ret; @@ -607,8 +638,6 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, if (cl->rpc_ops->enable_swap) cl->rpc_ops->enable_swap(inode); - - sis->flags |= SWP_FS_OPS; return ret; } @@ -622,6 +651,10 @@ static void nfs_swap_deactivate(struct file *file) if (cl->rpc_ops->disable_swap) cl->rpc_ops->disable_swap(file_inode(file)); } +#else +#define nfs_swap_activate NULL +#define nfs_swap_deactivate NULL +#endif /* CONFIG_SWAP */ const struct address_space_operations nfs_file_aops = { .read_folio = nfs_read_folio, @@ -638,7 +671,6 @@ const struct address_space_operations nfs_file_aops = { .error_remove_folio = generic_error_remove_folio, .swap_activate = nfs_swap_activate, .swap_deactivate = nfs_swap_deactivate, - .swap_rw = nfs_swap_rw, }; /* diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 817e3e0f9194..5c54aebe2118 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1693,7 +1693,7 @@ static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, if (!pte_write(pte)) return false; - if (!is_cow_mapping(vma->vm_flags)) + if (!vma_is_cow_mapping(vma)) return false; if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm))) return false; diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c index 2f79bcb89d2e..fb471bf88ab7 100644 --- a/fs/ramfs/file-nommu.c +++ b/fs/ramfs/file-nommu.c @@ -69,6 +69,9 @@ int ramfs_nommu_expand_for_mapping(struct inode *inode, size_t newsize) gfp_t gfp = mapping_gfp_mask(inode->i_mapping); /* make various checks */ + if (!newsize) + return 0; + order = get_order(newsize); if (unlikely(order > MAX_PAGE_ORDER)) return -EFBIG; diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index ac89c1ba56b1..112c6a663617 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -20,7 +20,7 @@ #include <linux/delay.h> #include <linux/mount.h> #include <linux/slab.h> -#include <linux/swap.h> +#include <linux/swap_ops.h> #include <linux/mm.h> #include <asm/div64.h> #include "cifsfs.h" @@ -3405,6 +3405,38 @@ out: cifs_done_oplock_break(cinode); } +#ifdef CONFIG_SWAP +static void cifs_swap_submit_write(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct iov_iter iter; + int ret; + + swap_fs_prepare_rw(ctx, WRITE, &iter); + ret = netfs_unbuffered_write_iter_locked(&sio->iocb, &iter, NULL); + if (ret != -EIOCBQUEUED) + sio->iocb.ki_complete(&sio->iocb, ret); +} + +static void cifs_swap_submit_read(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct iov_iter iter; + int ret; + + swap_fs_prepare_rw(ctx, READ, &iter); + ret = netfs_unbuffered_read_iter_locked(&sio->iocb, &iter); + if (ret != -EIOCBQUEUED) + sio->iocb.ki_complete(&sio->iocb, ret); +} + +static const struct swap_ops cifs_swap_ops = { + .flags = SWAP_OPS_F_REQUIRE_NOFS, + .submit_write = cifs_swap_submit_write, + .submit_read = cifs_swap_submit_read, + .can_merge = swap_fs_can_merge, +}; + static int cifs_swap_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *span) { @@ -3415,7 +3447,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis, cifs_dbg(FYI, "swap activate\n"); - if (!swap_file->f_mapping->a_ops->swap_rw) + if (swap_file->f_mapping->a_ops != &cifs_addr_ops) /* Cannot support swap */ return -EINVAL; @@ -3446,9 +3478,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis, * but we could add call to grab a byte range lock to prevent others * from reading or writing the file */ - - sis->flags |= SWP_FS_OPS; - return add_swap_extent(sis, 0, sis->max, 0); + return swap_fs_activate(sis, &cifs_swap_ops); } static void cifs_swap_deactivate(struct file *file) @@ -3464,26 +3494,10 @@ static void cifs_swap_deactivate(struct file *file) /* do we need to unpin (or unlock) the file */ } - -/** - * cifs_swap_rw - SMB3 address space operation for swap I/O - * @iocb: target I/O control block - * @iter: I/O buffer - * - * Perform IO to the swap-file. This is much like direct IO. - */ -static int cifs_swap_rw(struct kiocb *iocb, struct iov_iter *iter) -{ - ssize_t ret; - - if (iov_iter_rw(iter) == READ) - ret = netfs_unbuffered_read_iter_locked(iocb, iter); - else - ret = netfs_unbuffered_write_iter_locked(iocb, iter, NULL); - if (ret < 0) - return ret; - return 0; -} +#else +#define cifs_swap_activate NULL +#define cifs_swap_deactivate NULL +#endif /* CONFIG_SWAP */ const struct address_space_operations cifs_addr_ops = { .read_folio = netfs_read_folio, @@ -3500,7 +3514,6 @@ const struct address_space_operations cifs_addr_ops = { */ .swap_activate = cifs_swap_activate, .swap_deactivate = cifs_swap_deactivate, - .swap_rw = cifs_swap_rw, }; /* diff --git a/include/linux/codetag.h b/include/linux/codetag.h index ddae7484ca45..a25a085c2df1 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -77,6 +77,8 @@ struct codetag_iterator { void codetag_lock_module_list(struct codetag_type *cttype); bool codetag_trylock_module_list(struct codetag_type *cttype); void codetag_unlock_module_list(struct codetag_type *cttype); +unsigned long codetag_get_content_id(struct codetag_type *cttype); +unsigned int codetag_get_count(struct codetag_type *cttype); struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); struct codetag *codetag_next_ct(struct codetag_iterator *iter); diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 7de229134e30..1bd34925d1aa 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -89,7 +89,7 @@ static inline int execmem_restore_rox(void *ptr, size_t size) { return 0; } * @end: address space end (inclusive) * @fallback_start: start of the secondary address space range for fallback * allocations on architectures that require it - * @fallback_end: start of the secondary address space (inclusive) + * @fallback_end: end of the secondary address space (inclusive) * @pgprot: permissions for memory in this address space * @alignment: alignment required for text allocations * @flags: options for memory allocations for this range diff --git a/include/linux/fs.h b/include/linux/fs.h index 072d8cd09a0b..f9d1e05e8ae6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -438,7 +438,6 @@ struct address_space_operations { int (*swap_activate)(struct swap_info_struct *sis, struct file *file, sector_t *span); void (*swap_deactivate)(struct file *file); - int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter); }; extern const struct address_space_operations empty_aops; diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 621566345406..a6451ecbbe9a 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie); extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); +extern u32 lock_sequence(struct lockdep_map *lock); +#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map) + #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) #define lockdep_assert(cond) \ diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index eae115a26488..55c4b152fedf 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -253,7 +253,8 @@ struct held_lock { unsigned int hardirqs_off:1; unsigned int sync:1; unsigned int references:11; /* 32 bits */ - unsigned int pin_count; + unsigned int pin_count:24; + unsigned int seq_count:8; }; #else /* !CONFIG_LOCKDEP */ diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 4a5631906aff..b723384b3bf4 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -11,7 +11,6 @@ #include <linux/kernel.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> -/* #define CONFIG_MAPLE_RCU_DISABLED */ /* * Allocated nodes are mutable until they have been inserted into the tree, @@ -29,13 +28,11 @@ #define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) #else /* 32bit sizes */ #define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ #define MAPLE_NODE_MASK 255UL @@ -485,6 +482,12 @@ struct ma_state { unsigned char mas_flags; unsigned char end; /* The end of the node */ enum store_type store_type; /* The type of store needed for this operation */ +#ifdef CONFIG_LOCKDEP + u32 ld_seq; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + unsigned long rcu_gp; +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ +#endif /* CONFIG_LOCKDEP */ }; struct ma_wr_state { @@ -572,7 +575,7 @@ void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); @@ -864,9 +867,6 @@ static inline void mt_init(struct maple_tree *mt) static inline bool mt_in_rcu(struct maple_tree *mt) { -#ifdef CONFIG_MAPLE_RCU_DISABLED - return false; -#endif return mt->ma_flags & MT_FLAGS_USE_RCU; } diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 8170bb8066a2..215e2e87f42b 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -239,8 +239,6 @@ struct mem_cgroup { */ bool oom_group; - int swappiness; - /* memory.events and memory.events.local */ struct cgroup_file events_file; struct cgroup_file events_local_file; @@ -270,10 +268,15 @@ struct mem_cgroup { #endif int kmemcg_id; - struct memcg_vmstats_percpu __percpu *vmstats_percpu; - #ifdef CONFIG_CGROUP_WRITEBACK struct list_head cgwb_list; +#endif + + /* Keep the hot per-CPU stats pointer away from memory event counters. */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu + ____cacheline_aligned_in_smp; + +#ifdef CONFIG_CGROUP_WRITEBACK struct wb_domain cgwb_domain; struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif @@ -318,6 +321,8 @@ struct mem_cgroup { /* List of events which userspace want to receive */ struct list_head event_list; spinlock_t event_list_lock; + + int swappiness; #endif /* CONFIG_MEMCG_V1 */ struct mem_cgroup_per_node *nodeinfo[]; @@ -947,6 +952,8 @@ unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item); bool memcg_stat_item_valid(int idx); bool memcg_vm_event_item_valid(enum vm_event_item idx); unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); +unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, + enum node_stat_item idx); unsigned long lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx); @@ -1399,6 +1406,12 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec, return node_page_state(lruvec_pgdat(lruvec), idx); } +static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, + enum node_stat_item idx) +{ + return node_page_state_monotonic(lruvec_pgdat(lruvec), idx); +} + static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx) { diff --git a/include/linux/mm.h b/include/linux/mm.h index 87feaa5a2b78..dd09c438fa23 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1556,7 +1556,7 @@ static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) desc->vm_ops = NULL; } -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -2271,17 +2271,76 @@ void unpin_user_pages(struct page **pages, unsigned long npages); void unpin_user_folio(struct folio *folio, unsigned long npages); void unpin_folios(struct folio **folios, unsigned long nfolios); -static inline bool is_cow_mapping(vm_flags_t flags) +/** + * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping? + * @flags: The VMA flags to check. + * + * Mappings which could be CoW'd (subject to Copy-On-Write faults) are + * described as CoW mappings. + * + * All mappings backed by anonymous folios (all anonymous mappings and most + * MAP_PRIVATE-file backed ranges) are CoW mappings. + * + * All other mappings (including all MAP_SHARED mappings) are non-CoW. + * + * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT. + * + * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both + * future mprotect() calls which can render a read-only mapping writable, and + * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping. + * + * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT. + * + * - Writes to anonymous mappings do not immediately result in CoW faults but + * may do so after the process is forked or if a read is followed by a + * write. + * + * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may + * do so again after fork. + * + * - MAP_SHARED mappings of a file opened read-only are transformed into + * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain + * non-CoW. + * + * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot + * mark themselves anonymous. Having cleared this flag it is not valid for + * them to leave the VMA_WRITE_BIT flag set. + * + * As a consequence, the anonymous reverse mapping only tracks CoW mappings. + * + * Returns: true if the flags indicate a CoW mapping, otherwise false. + */ +static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags) { - return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; + return vma_flags_test(flags, VMA_MAYWRITE_BIT) && + !vma_flags_test(flags, VMA_SHARED_BIT); } -static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc) +/** + * vma_is_cow_mapping() - Is this VMA a CoW mapping? + * @vma: The VMA to check. + * + * See vma_flags_is_cow_mapping() for details. + * + * Returns: true if the VMA is a CoW mapping, otherwise false. + */ +static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma) { - const vma_flags_t *flags = &desc->vma_flags; + return vma_flags_is_cow_mapping(&vma->flags); +} - return vma_flags_test(flags, VMA_MAYWRITE_BIT) && - !vma_flags_test(flags, VMA_SHARED_BIT); +/** + * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping? + * @desc: The VMA descriptor to check. + * + * See vma_flags_is_cow_mapping() for details. + * + * Returns: true if the VMA descriptor describes a CoW mapping, otherwise + * false. + */ +static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc) +{ + return vma_flags_is_cow_mapping(&desc->vma_flags); } #ifndef CONFIG_MMU @@ -4393,6 +4452,65 @@ static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) return vma_end_pgoff(vma) - 1; } +/** + * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma + * @vma: The VMA whose anonymous page offset is required. + * + * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the + * anonymous page offset at the time of first fault. + * + * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). + * + * This value is used for tracking MAP_PRIVATE file-backed mappings by their + * anonymous page offset. + * + * Returns: The anonymous page offset of the start of @vma. + */ +static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_anon_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_anon_pgoff_lo; + return pgoff; +} + +/** + * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of + * @vma. + * @vma: The VMA whose end anonymous page offset is required. + * + * This returns the anonymous exclusive end page offset of @vma, which is useful + * for expressing page offset ranges. + * + * See the description of vma_start_anon_pgoff() for a description of VMA + * anonymous page offsets. + * + * Returns: The exclusive end anonymous page offset of @vma. + */ +static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_anon_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in + * @vma. + * @vma: The VMA whose last anonymous page offset is required. + * + * See the description of vma_start_anon_pgoff() for a description of VMA + * anonymous page offsets. + * + * Returns: The last anonymous page offset of @vma. + */ +static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_anon_pgoff(vma) - 1; +} + static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) { return desc->end - desc->start; diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 939b5ea8c9e0..6d815f6440c9 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -968,6 +968,11 @@ struct vm_area_struct { unsigned int vm_lock_seq; #endif /* + * Low 32-bits of anonymous page offset. + * See vma_start_anon_pgoff() comment for details. + */ + unsigned int __vm_anon_pgoff_lo; + /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma * list, after a COW of one of the file pages. A MAP_SHARED vma * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack @@ -1042,6 +1047,13 @@ struct vm_area_struct { struct lockdep_map vmlock_dep_map; #endif #endif +#ifdef CONFIG_64BIT + /* + * High 32-bits of anonymous page offset. + * See vma_start_anon_pgoff() comment for details. + */ + unsigned int __vm_anon_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. @@ -1706,20 +1718,20 @@ enum vm_fault_reason { VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON | \ VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK) -#define VM_FAULT_RESULT_TRACE \ - { VM_FAULT_OOM, "OOM" }, \ - { VM_FAULT_SIGBUS, "SIGBUS" }, \ - { VM_FAULT_MAJOR, "MAJOR" }, \ - { VM_FAULT_HWPOISON, "HWPOISON" }, \ - { VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \ - { VM_FAULT_SIGSEGV, "SIGSEGV" }, \ - { VM_FAULT_NOPAGE, "NOPAGE" }, \ - { VM_FAULT_LOCKED, "LOCKED" }, \ - { VM_FAULT_RETRY, "RETRY" }, \ - { VM_FAULT_FALLBACK, "FALLBACK" }, \ - { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ - { VM_FAULT_COMPLETED, "COMPLETED" } +#define VM_FAULT_RESULT_TRACE \ + { (__force u32)VM_FAULT_OOM, "OOM" }, \ + { (__force u32)VM_FAULT_SIGBUS, "SIGBUS" }, \ + { (__force u32)VM_FAULT_MAJOR, "MAJOR" }, \ + { (__force u32)VM_FAULT_HWPOISON, "HWPOISON" }, \ + { (__force u32)VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \ + { (__force u32)VM_FAULT_SIGSEGV, "SIGSEGV" }, \ + { (__force u32)VM_FAULT_NOPAGE, "NOPAGE" }, \ + { (__force u32)VM_FAULT_LOCKED, "LOCKED" }, \ + { (__force u32)VM_FAULT_RETRY, "RETRY" }, \ + { (__force u32)VM_FAULT_FALLBACK, "FALLBACK" }, \ + { (__force u32)VM_FAULT_DONE_COW, "DONE_COW" }, \ + { (__force u32)VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { (__force u32)VM_FAULT_COMPLETED, "COMPLETED" } struct vm_special_mapping { const char *name; /* The name, e.g. "[vdso]". */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 0507193b3ae3..94f9c3ff5416 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -323,6 +323,8 @@ enum node_stat_item { PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, + PGROTATE_ANON, + PGROTATE_FILE, PGREFILL, #ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB, @@ -755,6 +757,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, #endif /* CONFIG_LRU_GEN */ +struct lru_cost { + unsigned long count; + unsigned long last_rotated; + unsigned long last_io; +}; + struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ @@ -763,9 +771,12 @@ struct lruvec { * These track the cost of reclaiming one LRU - file or anon - * over the other. As the observed cost of reclaiming one LRU * increases, the reclaim scan balance tips toward the other. + * Updated and decayed at prepare_scan_control() time; cost_lock + * serialises that update. */ - unsigned long anon_cost; - unsigned long file_cost; + struct lru_cost cost[ANON_AND_FILE]; + /* Protects cost[]. */ + spinlock_t cost_lock; /* Non-resident age, driven by LRU movement */ atomic_long_t nonresident_age; /* Refaults at the time of last reclaim cycle */ @@ -2058,7 +2069,6 @@ static inline struct mem_section *__nr_to_section(unsigned long nr) #endif return &mem_section[root][nr & SECTION_ROOT_MASK]; } -extern size_t mem_section_usage_size(void); /* * We use the lower bits of the mem_map pointer to store a little bit of diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index ec17e602c979..764056498eba 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -548,7 +548,6 @@ static inline const struct cred *nfs_file_cred(struct file *file) /* * linux/fs/nfs/direct.c */ -int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter); ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, bool swap); ssize_t nfs_file_direct_write(struct kiocb *iocb, diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index f23d4b218da0..79c53ec45dfa 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -55,7 +55,6 @@ struct page_ext { extern bool early_page_ext; extern unsigned long page_ext_size; -extern void pgdat_page_ext_init(struct pglist_data *pgdat); static inline bool early_page_ext_enabled(void) { @@ -202,10 +201,6 @@ static inline bool early_page_ext_enabled(void) return false; } -static inline void pgdat_page_ext_init(struct pglist_data *pgdat) -{ -} - static inline void page_ext_init(void) { } diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index c6fc783aaee5..0adfa6605653 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1094,10 +1094,44 @@ static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; + return linear_page_delta(vma, address) + vma_start_pgoff(vma); +} + +static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + return linear_page_delta(vma, address) + vma_start_anon_pgoff(vma); +} + +/** + * linear_anon_page_index() - Determine the absolute anonymous page offset of + * @address within @vma. + * @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * This returns the anonymous page offset of @address, which is the page offset + * the address possessed at the time the VMA was first faulted. + * + * For anonymous mappings, this returns the same value as linear_page_index(). + * + * For MAP_PRIVATE file-backed mappings, this returns the anonymous page offset + * of @address, which is the page offset the address possessed at the time the + * VMA was first faulted. + * + * It is not valid to call this function for shared file-backed mappings. + * + * Returns: The absolute anonymous page offset of @address within @vma. + */ +static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_anon_page_index(vma, address); + + VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); + /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ + if (vma_is_anonymous(vma) && !vma->vm_file) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); - pgoff = linear_page_delta(vma, address); - pgoff += vma_start_pgoff(vma); return pgoff; } diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 2cba7cc2b01f..dbe3267a0a13 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -65,13 +65,8 @@ * * Archs which need weak percpu definitions should set * CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU when necessary. - * - * To ensure that the generic code observes the above two - * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak - * definition is used for all cases. */ -#if (defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)) || \ - defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU) +#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE) /* * __pcpu_scope_* dummy variable is used to enforce scope. It * receives the static modifier when it's used in front of diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f00..0b332770abee 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -843,7 +843,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio, * Called from mm/vmscan.c to handle paging out */ int folio_referenced(struct folio *, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags); + struct mem_cgroup *memcg, vma_flags_t *vma_flags); void try_to_migrate(struct folio *folio, enum ttu_flags flags); void try_to_unmap(struct folio *, enum ttu_flags flags); @@ -864,13 +864,14 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; - pgoff_t pgoff; + pgoff_t pgoff; /* Only meaningful if nr_pages > 1 and not a KSM walk */ struct vm_area_struct *vma; unsigned long address; pmd_t *pmd; pte_t *pte; spinlock_t *ptl; unsigned int flags; + bool pgoff_is_anon : 1; }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -881,6 +882,7 @@ struct page_vma_mapped_walk { .vma = _vma, \ .address = _address, \ .flags = _flags, \ + .pgoff_is_anon = folio_test_anon(_folio), \ } static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) @@ -975,10 +977,9 @@ struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, #define anon_vma_prepare(vma) (0) static inline int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, - vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { - *vm_flags = 0; + vma_flags_clear_all(vma_flags); return 0; } diff --git a/include/linux/sched.h b/include/linux/sched.h index c7cdbb3e4aa6..8b3d47a325cc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1298,6 +1298,7 @@ struct task_struct { u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; + unsigned int lockdep_seq; struct held_lock held_locks[MAX_LOCK_DEPTH]; #endif diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index e729b9b0e38d..5663dff53186 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -12,8 +12,6 @@ #include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb; - /* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping) void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, - struct list_head *folio_list); +int shmem_write_folio(struct folio *folio); void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end); int shmem_unuse(unsigned int type); diff --git a/include/linux/swap.h b/include/linux/swap.h index 696ed01709c2..5658a1634b85 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -202,7 +202,6 @@ enum { SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */ SWP_BLKDEV = (1 << 6), /* its a block device */ SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */ - SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */ SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */ SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */ @@ -276,6 +275,7 @@ struct swap_info_struct { struct work_struct reclaim_work; /* reclaim worker */ struct list_head discard_clusters; /* discard clusters list */ struct plist_node avail_list; /* entry in swap_avail_head */ + const struct swap_ops *ops; }; static inline swp_entry_t page_swap_entry(struct page *page) @@ -298,6 +298,14 @@ void folio_add_lru(struct folio *folio); void folio_mark_accessed(struct folio *folio); void lru_add_drain_all(void); +enum lru_cache_drained { + LRU_CACHE_NOT_DRAINED, + LRU_CACHE_DRAINED, + LRU_CACHE_DRAINED_ALL, +}; +void lru_cache_drain_for_folio(const struct folio *folio, + unsigned int extra_refs, enum lru_cache_drained *drained); + /* linux/mm/folio-compat.c */ void mark_page_accessed(struct page *page); @@ -309,7 +317,6 @@ static inline bool lru_cache_disabled(void) } extern unsigned long shrink_all_memory(unsigned long nr_pages); -extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) @@ -334,7 +341,6 @@ extern void __meminit kswapd_run(int nid); extern void __meminit kswapd_stop(int nid); #ifdef CONFIG_SWAP - int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block); int generic_swapfile_activate(struct swap_info_struct *, struct file *, @@ -468,25 +474,7 @@ static inline int add_swap_extent(struct swap_info_struct *sis, } #endif /* CONFIG_SWAP */ #ifdef CONFIG_MEMCG -static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) -{ - /* Cgroup2 doesn't have per-cgroup swappiness */ - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) - return READ_ONCE(vm_swappiness); - - /* root ? */ - if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg)) - return READ_ONCE(vm_swappiness); - - return READ_ONCE(memcg->swappiness); -} - void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); -#else -static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) -{ - return READ_ONCE(vm_swappiness); -} #endif #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) diff --git a/include/linux/swap_ops.h b/include/linux/swap_ops.h new file mode 100644 index 000000000000..57ac6c703f68 --- /dev/null +++ b/include/linux/swap_ops.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _MM_SWAP_OPS_H +#define _MM_SWAP_OPS_H + +#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */ + +struct swap_iocb { + union { + struct kiocb iocb; + struct bio bio; + }; + struct bio_vec bvecs[SWAP_CLUSTER_MAX]; + int nr_bvecs; + int len; +}; + +struct swap_io_ctx { + struct swap_iocb *sio; + struct swap_info_struct *sis; +}; + +/* + * SWAP_OPS_F_REQUIRE_NOFS: + * When set, all reclaim operations must operated as GFS_NOFS and not + * just GFP_NOIO, as GFP_NOIO allocations could recourse into the + * file system backing this swap file. + */ +#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) + +struct swap_ops { + unsigned int flags; + + bool (*can_merge)(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw); + void (*submit_write)(struct swap_io_ctx *ctx); + void (*submit_read)(struct swap_io_ctx *ctx); +}; + +void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter); +bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw); +int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops); + +#endif /* _MM_SWAP_OPS_H */ diff --git a/include/linux/swapops.h b/include/linux/swapops.h index c956bc445ee0..e7d0d529f3e0 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -5,6 +5,7 @@ #include <linux/radix-tree.h> #include <linux/bug.h> #include <linux/mm_types.h> +#include <linux/swap.h> #ifdef CONFIG_MMU @@ -325,8 +326,8 @@ struct page_vma_mapped_walk; extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); -extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, - struct page *new); +void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, + struct folio *folio); extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); @@ -346,7 +347,7 @@ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, - struct page *new) + struct folio *folio) { BUILD_BUG(); } diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 03fe95f5a020..2628ccda076a 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST, #endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */ NR_VM_EVENT_ITEMS }; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index e4d8d0a9f30f..aed121d729b0 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -62,7 +62,7 @@ struct vm_struct { #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC unsigned int page_order; #endif - unsigned int nr_pages; + unsigned long nr_pages; phys_addr_t phys_addr; const void *caller; unsigned long requested_size; diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 3c9c266cf782..5b31d8e7ae40 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -20,7 +20,6 @@ struct reclaim_stat { unsigned nr_congested; unsigned nr_writeback; unsigned nr_immediate; - unsigned nr_pageout; unsigned nr_activate[ANON_AND_FILE]; unsigned nr_ref_keep; unsigned nr_unmap_fail; @@ -194,6 +193,19 @@ unsigned long global_node_page_state_pages(enum node_stat_item item) return x; } +/* + * Non-clamping variant of global_node_page_state() intended for callers that + * snapshot a monotonically-incremented counter and subtract two samples. + * Returns the raw wrapping value so that unsigned modular subtraction stays + * correct across a signed-long overflow (a real hazard on 32-bit) that the + * clamp in global_node_page_state() would otherwise turn into a huge spurious + * delta. Do NOT use for non-monotonic page-count reads. + */ +static inline unsigned long global_node_page_state_monotonic(enum node_stat_item item) +{ + return (unsigned long)atomic_long_read(&vm_node_stat[item]); +} + static inline unsigned long global_node_page_state(enum node_stat_item item) { VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); @@ -259,11 +271,14 @@ extern unsigned long node_page_state(struct pglist_data *pgdat, enum node_stat_item item); extern unsigned long node_page_state_pages(struct pglist_data *pgdat, enum node_stat_item item); +extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat, + enum node_stat_item item); extern void fold_vm_numa_events(void); #else #define sum_zone_node_page_state(node, item) global_zone_page_state(item) #define node_page_state(node, item) global_node_page_state(item) #define node_page_state_pages(node, item) global_node_page_state_pages(item) +#define node_page_state_monotonic(node, item) global_node_page_state_monotonic(item) static inline void fold_vm_numa_events(void) { } diff --git a/include/rv/da_monitor.h b/include/rv/da_monitor.h index e3cf85c9ce55..6fc5ef8142ac 100644 --- a/include/rv/da_monitor.h +++ b/include/rv/da_monitor.h @@ -25,7 +25,7 @@ /* * Per-cpu variables require a unique name although static in some - * configurations (e.g. CONFIG_DEBUG_FORCE_WEAK_PER_CPU or alpha modules). + * configurations (e.g. alpha modules). */ #define DA_MON_NAME CONCATENATE(da_mon_, MONITOR_NAME) diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h new file mode 100644 index 000000000000..e3ad94444864 --- /dev/null +++ b/include/uapi/linux/alloc_tag.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * alloc_tag IOCTL API definition + * + * Copyright (C) 2026 Google, LLC. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _UAPI_ALLOC_TAG_H +#define _UAPI_ALLOC_TAG_H + +#include <linux/types.h> + +/* + * Function, file and module names often have the same prefixes, therefore + * when filtering by these criteria, we compare the last 64 characters to + * minimize the chances of name collisions + */ +#define ALLOCINFO_STR_SIZE 64 + +struct allocinfo_content_id { + __u64 id; +}; + +struct allocinfo_tag { + /* Longer names are trimmed */ + char modname[ALLOCINFO_STR_SIZE]; + char function[ALLOCINFO_STR_SIZE]; + char filename[ALLOCINFO_STR_SIZE]; + __u64 lineno; +}; + +/* The alignment ensures 32-bit compatible interfaces are not broken */ +struct allocinfo_counter { + __u64 bytes; + __u64 calls; + __u8 accurate; +} __attribute__((aligned(8))); + +struct allocinfo_tag_data { + struct allocinfo_tag tag; + struct allocinfo_counter counter; +}; + +enum { + ALLOCINFO_FILTER_MODNAME, + ALLOCINFO_FILTER_FUNCTION, + ALLOCINFO_FILTER_FILENAME, + ALLOCINFO_FILTER_LINENO, + ALLOCINFO_FILTER_INACCURATE, + ALLOCINFO_FILTER_MIN_SIZE, + ALLOCINFO_FILTER_MAX_SIZE, + __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE +}; + +#define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME) +#define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION) +#define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME) +#define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO) +#define ALLOCINFO_FILTER_MASK_INACCURATE (1 << ALLOCINFO_FILTER_INACCURATE) +#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE) +#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE) + +#define ALLOCINFO_FILTER_MASKS \ + ((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1) + +struct allocinfo_filter { + __u64 mask; /* bitmask of the filter fields used */ + struct allocinfo_tag fields; + __u64 min_size; + __u64 max_size; + /* filter criteria only; see allocinfo_counter.accurate for actual accuracy */ + __u64 inaccurate; +}; + +struct allocinfo_get_at { + /* inputs */ + __u64 pos; + struct allocinfo_filter filter; + /* output */ + struct allocinfo_tag_data data; +}; + +#define _ALLOCINFO_IOC_CONTENT_ID 0 +#define _ALLOCINFO_IOC_GET_AT 1 +#define _ALLOCINFO_IOC_GET_NEXT 2 + +#define ALLOCINFO_IOC_BASE 0xA6 +#define ALLOCINFO_IOC_CONTENT_ID _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID, \ + struct allocinfo_content_id) +#define ALLOCINFO_IOC_GET_AT _IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT, \ + struct allocinfo_get_at) +#define ALLOCINFO_IOC_GET_NEXT _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT, \ + struct allocinfo_tag_data) + +#endif /* _UAPI_ALLOC_TAG_H */ diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index b25531331902..7709ea882477 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -513,7 +513,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, uprobe = container_of(auprobe, struct uprobe, arch); - if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags))) + if (WARN_ON_ONCE(!vma_is_cow_mapping(vma))) return -EINVAL; /* diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 25d77d4a1061..c56a7f91d72e 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -5093,7 +5093,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read); static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, int hardirqs_off, struct lockdep_map *nest_lock, unsigned long ip, - int references, int pin_count, int sync) + int references, int pin_count, int sync, int seq) { struct task_struct *curr = current; struct lock_class *class = NULL; @@ -5199,6 +5199,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, hlock->holdtime_stamp = lockstat_clock(); #endif hlock->pin_count = pin_count; + hlock->seq_count = seq; if (check_wait_context(curr, hlock)) return 0; @@ -5404,7 +5405,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth, hlock->read, hlock->check, hlock->hardirqs_off, hlock->nest_lock, hlock->acquire_ip, - hlock->references, hlock->pin_count, 0)) { + hlock->references, hlock->pin_count, 0, hlock->seq_count)) { case 0: return 1; case 1: @@ -5687,14 +5688,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie struct held_lock *hlock = curr->held_locks + i; if (match_held_lock(hlock, lock)) { + int pin_count; + if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n")) return; - hlock->pin_count -= cookie.val; + pin_count = hlock->pin_count - cookie.val; - if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n")) - hlock->pin_count = 0; + if (WARN(pin_count < 0, "pin count corrupted\n")) + pin_count = 0; + hlock->pin_count = pin_count; return; } } @@ -5702,6 +5706,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie WARN(1, "unpinning an unheld lock\n"); } +static u32 __lock_sequence(struct lockdep_map *lock) +{ + struct task_struct *curr = current; + int i; + + if (unlikely(!debug_locks)) + return ~0; + + for (i = 0; i < curr->lockdep_depth; i++) { + struct held_lock *hlock = curr->held_locks + i; + + if (match_held_lock(hlock, lock)) + return hlock->seq_count; + } + + return ~0; +} + /* * Check whether we follow the irq-flags state precisely: */ @@ -5884,7 +5906,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass, lockdep_recursion_inc(); __lock_acquire(lock, subclass, trylock, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0, + ++current->lockdep_seq); lockdep_recursion_finish(); raw_local_irq_restore(flags); } @@ -5932,7 +5955,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read, lockdep_recursion_inc(); __lock_acquire(lock, subclass, 0, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1, + ++current->lockdep_seq); check_chain_key(current); lockdep_recursion_finish(); raw_local_irq_restore(flags); @@ -6018,6 +6042,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) } EXPORT_SYMBOL_GPL(lock_unpin_lock); +u32 lock_sequence(struct lockdep_map *lock) +{ + unsigned long flags; + u32 seq = ~0; + + if (unlikely(!lockdep_enabled())) + return seq; + + raw_local_irq_save(flags); + check_flags(flags); + + lockdep_recursion_inc(); + seq = __lock_sequence(lock); + lockdep_recursion_finish(); + raw_local_irq_restore(flags); + + return seq; +} +EXPORT_SYMBOL_GPL(lock_sequence); + #ifdef CONFIG_LOCK_STAT static void print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a2f0d3e97889..134b15a44625 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -613,21 +613,6 @@ config BUILTIN_MODULE_RANGES It also records an anchor symbol to determine the load address of the section. -config DEBUG_FORCE_WEAK_PER_CPU - bool "Force weak per-cpu definitions" - depends on DEBUG_KERNEL - help - s390 and alpha require percpu variables in modules to be - defined weak to work around addressing range issue which - puts the following two restrictions on percpu variable - definitions. - - 1. percpu symbols must be unique whether static or not - 2. percpu variables can't be defined inside a function - - To ensure that generic code follows the above rules, this - option forces all percpu variables to be defined as weak. - config WARN_CONTEXT_ANALYSIS bool "Compiler context-analysis warnings" depends on CC_IS_CLANG && CLANG_VERSION >= 230000 diff --git a/lib/codetag.c b/lib/codetag.c index 4001a7ea6675..a9cda4c962a3 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -19,6 +19,8 @@ struct codetag_type { struct codetag_type_desc desc; /* generates unique sequence number for module load */ unsigned long next_mod_seq; + /* bumped on every module load and unload */ + unsigned long content_id; }; struct codetag_range { @@ -50,6 +52,20 @@ void codetag_unlock_module_list(struct codetag_type *cttype) up_read(&cttype->mod_lock); } +unsigned long codetag_get_content_id(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->content_id; +} + +unsigned int codetag_get_count(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->count; +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { @@ -204,6 +220,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) down_write(&cttype->mod_lock); cmod->mod_seq = ++cttype->next_mod_seq; + ++cttype->content_id; mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL); if (mod_id >= 0) { if (cttype->desc.module_load) { @@ -368,6 +385,7 @@ void codetag_unload_module(struct module *mod) cttype->count -= range_size(cttype, &cmod->range); idr_remove(&cttype->mod_idr, mod_id); kfree(cmod); + ++cttype->content_id; } up_write(&cttype->mod_lock); if (found && cttype->desc.free_section_mem) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 529acc056e55..5ee1175af26e 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas) return mas->status == ma_underflow; } +static inline void mas_make_walkable(struct ma_state *mas) +{ + if (!mas_is_active(mas) && !mas_is_start(mas)) + mas->status = ma_start; +} + static __always_inline struct maple_node *mte_to_node( const struct maple_enode *entry) { @@ -356,11 +362,6 @@ static __always_inline bool mte_is_root(const struct maple_enode *node) return ma_is_root(mte_to_node(node)); } -static inline bool mas_is_root_limits(const struct ma_state *mas) -{ - return !mas->min && mas->max == ULONG_MAX; -} - static __always_inline bool mt_is_alloc(struct maple_tree *mt) { return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); @@ -456,46 +457,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode) } /* - * mas_set_parent() - Set the parent node and encode the slot - * @mas: The maple state - * @enode: The encoded maple node. - * @parent: The encoded maple node that is the parent of @enode. - * @slot: The slot that @enode resides in @parent. - * - * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the - * parent type. - */ -static inline -void mas_set_parent(struct ma_state *mas, struct maple_enode *enode, - const struct maple_enode *parent, unsigned char slot) -{ - unsigned long val = (unsigned long)parent; - unsigned long shift; - unsigned long type; - enum maple_type p_type = mte_node_type(parent); - - MAS_BUG_ON(mas, p_type == maple_dense); - MAS_BUG_ON(mas, p_type == maple_leaf_64); - - switch (p_type) { - case maple_range_64: - case maple_arange_64: - shift = MAPLE_PARENT_SLOT_SHIFT; - type = MAPLE_PARENT_RANGE64; - break; - default: - case maple_dense: - case maple_leaf_64: - shift = type = 0; - break; - } - - val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ - val |= (slot << shift) | type; - mte_to_node(enode)->parent = ma_parent_ptr(val); -} - -/* * mte_parent_slot() - get the parent slot of @enode. * @enode: The encoded maple node. * @@ -877,6 +838,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, } /* + * mas_set_parent_slots() - Bulk operation to set many slot parent pointers + * @mas: The maple state + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that of the @enode. + * @start_slot: The offset into @slot + * @count: The number of slots to set (eg: exclusive) + */ +static inline +void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent, + void __rcu **slots, unsigned char start_slot, unsigned char count) +{ + unsigned long val; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + unsigned char i; + + MAS_BUG_ON(mas, p_type != maple_range_64 && + p_type != maple_arange_64); + + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + + val = (unsigned long)parent; + val &= ~MAPLE_NODE_MASK; + + for (i = 0; i < count; i++) { + unsigned long pval = val | ((start_slot + i) << shift) | type; + struct maple_enode *child; + + child = mt_slot_locked(mas->tree, slots, i); + mte_to_node(child)->parent = ma_parent_ptr(pval); + } +} + +/* * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. * @mat: the ma_topiary, a linked list of dead nodes. * @dead_enode: the node to be marked as dead and added to the tail of the list @@ -1153,6 +1150,79 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) ma_free_rcu(mte_to_node(used)); } + +#ifdef CONFIG_LOCKDEP +static struct lockdep_map *mas_lockdep_map(struct ma_state *mas) +{ + struct maple_tree *mt = mas->tree; + + if (mt_external_lock(mt)) + return mt->ma_external_lock; + + return &(mt->ma_lock).dep_map; +} + +#endif + +static void mas_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; + u32 seq; + + if (!mas_is_active(mas)) + return; + +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp)); + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (!map) + return; + + seq = lock_sequence(map); + if (seq != UINT_MAX && mas->ld_seq != UINT_MAX) + WARN_ON_ONCE(mas->ld_seq != seq); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + mas->rcu_gp = get_state_synchronize_rcu(); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map) /* Update regardless of lock state */ + mas->ld_seq = lock_sequence(map); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_may_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (mas_is_start(mas) || mas_is_paused(mas)) { + mas_init_lock_check(mas); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + mas_lock_check(mas); +#endif /* CONFIG_LOCKDEP */ +} + /* * mas_start() - Sets up maple state for operations. * @mas: The maple state. @@ -1171,6 +1241,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas) if (likely(mas_is_start(mas))) { struct maple_enode *root; + mas_init_lock_check(mas); mas->min = 0; mas->max = ULONG_MAX; @@ -1501,14 +1572,26 @@ ascend: goto ascend; } +static __always_inline void mas_update_gap_known(struct ma_state *mas, + unsigned long gap) +{ + unsigned char pslot; + unsigned long p_gap; + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), + mas_parent_type(mas, mas->node))[pslot]; + + if (p_gap != gap) + mas_parent_gap(mas, pslot, gap); +} + /* * mas_update_gap() - Update a nodes gaps and propagate up if necessary. * @mas: the maple state. */ static inline void mas_update_gap(struct ma_state *mas) { - unsigned char pslot; - unsigned long p_gap; unsigned long max_gap; if (!mt_is_alloc(mas->tree)) @@ -1518,13 +1601,7 @@ static inline void mas_update_gap(struct ma_state *mas) return; max_gap = mas_max_gap(mas); - - pslot = mte_parent_slot(mas->node); - p_gap = ma_gaps(mte_parent(mas->node), - mas_parent_type(mas, mas->node))[pslot]; - - if (p_gap != max_gap) - mas_parent_gap(mas, pslot, max_gap); + mas_update_gap_known(mas, max_gap); } /* @@ -1540,14 +1617,10 @@ static inline void mas_adopt_children(struct ma_state *mas, struct maple_node *node = mte_to_node(parent); void __rcu **slots = ma_slots(node, type); unsigned long *pivots = ma_pivots(node, type); - struct maple_enode *child; - unsigned char offset; + unsigned char end; - offset = ma_data_end(node, type, pivots, mas->max); - do { - child = mas_slot_locked(mas, slots, offset); - mas_set_parent(mas, child, parent, offset); - } while (offset--); + end = ma_data_end(node, type, pivots, mas->max); + mas_set_parent_slots(mas, parent, slots, 0, end + 1); } /* @@ -1929,15 +2002,10 @@ unsigned long node_copy(struct ma_state *mas, struct maple_node *src, s_slots = ma_slots(src, s_mt) + start; s_pivots = ma_pivots(src, s_mt) + start; memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); - if (!ma_is_leaf(d_mt) && s_mt == maple_copy) { - struct maple_enode *edst = mt_mk_node(dst, d_mt); - - for (int i = 0; i < size; i++) - mas_set_parent(mas, - mt_slot_locked(mas->tree, d_slots, i), - edst, d_start + i); - } + if (!ma_is_leaf(d_mt) && s_mt == maple_copy) + mas_set_parent_slots(mas, mt_mk_node(dst, d_mt), + d_slots, d_start, size); d_gaps = ma_gaps(dst, d_mt); if (d_gaps) { @@ -2081,8 +2149,8 @@ static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp) mas->node = mt_slot_locked(mas->tree, cp->slot, 0); /* Insert the new data in the tree */ mas_topiary_replace(mas, old_enode, cp->height); - if (!mte_is_leaf(mas->node)) - mas_update_gap(mas); + if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node)) + mas_update_gap_known(mas, cp->gap[0]); mtree_range_walk(mas); } @@ -2928,13 +2996,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, if (r_mas->last < r_wr_mas->r_max) r_mas->last = r_wr_mas->r_max; r_mas->offset++; - } else if ((r_mas->last == r_wr_mas->r_max) && - (r_mas->last < r_mas->max) && - !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { - r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, - r_wr_mas->type, r_mas->offset + 1); - r_mas->offset++; - r_wr_mas->r_max = r_mas->last; } } @@ -3125,7 +3186,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas) static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) { unsigned char dst_offset, offset_end; - unsigned char copy_size, node_pivots; + unsigned char copy_size, node_pivots, node_slots; struct maple_node reuse, *newnode; unsigned long *dst_pivots; void __rcu **dst_slots; @@ -3138,6 +3199,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) in_rcu = mt_in_rcu(mas->tree); offset_end = wr_mas->offset_end; node_pivots = mt_pivots[wr_mas->type]; + node_slots = mt_slots[wr_mas->type]; /* Assume last adds an entry */ new_end = mas->end + 1 - offset_end + mas->offset; if (mas->last == wr_mas->end_piv) { @@ -3149,7 +3211,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) if (in_rcu) { newnode = mas_pop_node(mas); } else { - memset(&reuse, 0, sizeof(struct maple_node)); newnode = &reuse; } @@ -3193,7 +3254,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) dst_pivots[new_end] = mas->max; done: - mas_leaf_set_meta(newnode, maple_leaf_64, new_end); + if (!in_rcu && new_end + 2 < node_slots) { + unsigned char clear_from = new_end + 1; + + /* + * Note that the last slot is never cleared, since the metadata + * will be stored there or it has a value. + */ + memset(dst_slots + clear_from, 0, + sizeof(void __rcu *) * (node_slots - clear_from)); + if (clear_from < node_pivots) + memset(dst_pivots + clear_from, 0, + sizeof(unsigned long) * (node_pivots - clear_from)); + } + + mas_leaf_set_meta(newnode, wr_mas->type, new_end); if (in_rcu) { struct maple_enode *old_enode = mas->node; @@ -3218,7 +3293,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) void __rcu **slots = wr_mas->slots; bool gap = false; - gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !wr_mas->content; gap |= !mt_slot_locked(mas->tree, slots, offset + 1); if (wr_mas->offset_end - offset == 1) { @@ -3595,7 +3670,7 @@ set_content: /** * mas_prealloc_calc() - Calculate number of nodes needed for a - * given store oepration + * given store operation * @wr_mas: The maple write state * @entry: The entry to store into the tree * @@ -3661,6 +3736,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; unsigned char new_end; + bool appending; + bool one_slot; + bool in_rcu; if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) return wr_store_root; @@ -3680,21 +3758,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) return wr_new_root; new_end = mas_wr_new_end(wr_mas); + in_rcu = mt_in_rcu(mas->tree); + appending = mas->offset == mas->end; + one_slot = wr_mas->offset_end - mas->offset == 1; + /* Potential spanning rebalance collapsing a node */ if (new_end < mt_min_slots[wr_mas->type]) { if (!mte_is_root(mas->node)) return wr_rebalance; + if (!in_rcu) { + if (appending) + return wr_append; + else if (mas->end == new_end && one_slot) + return wr_slot_store; + } return wr_node_store; } if (new_end >= mt_slots[wr_mas->type]) return wr_split_store; - if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end)) + if (!in_rcu && appending) return wr_append; - if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) || - (wr_mas->offset_end - mas->offset == 1))) + if (new_end == mas->end && (!in_rcu || one_slot)) return wr_slot_store; return wr_node_store; @@ -3793,35 +3880,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } @@ -4360,8 +4452,8 @@ void *mas_walk(struct ma_state *mas) { void *entry; - if (!mas_is_active(mas) && !mas_is_start(mas)) - mas->status = ma_start; + mas_may_init_lock_check(mas); + mas_make_walkable(mas); retry: entry = mas_state_walk(mas); if (mas_is_start(mas)) { @@ -4831,6 +4923,7 @@ void *mas_store(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); trace_ma_write(TP_FCT, mas, 0, entry); #ifdef CONFIG_DEBUG_MAPLE_TREE if (MAS_WARN_ON(mas, mas->index > mas->last)) @@ -4889,6 +4982,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) MA_WR_STATE(wr_mas, mas, entry); int ret = 0; + mas_may_init_lock_check(mas); retry: mas_wr_preallocate(&wr_mas, entry); if (unlikely(mas_nomem(mas, gfp))) { @@ -4919,6 +5013,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_lock_check(mas); if (mas->store_type == wr_store_root) { mas_wr_prealloc_setup(&wr_mas); goto store; @@ -4951,6 +5046,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); mas_wr_prealloc_setup(&wr_mas); mas->store_type = mas_wr_store_type(&wr_mas); mas_prealloc_calc(&wr_mas, entry); @@ -4997,6 +5093,7 @@ static void mas_may_activate(struct ma_state *mas) mas->status = ma_start; } else { mas->status = ma_active; + mas_lock_check(mas); } } @@ -5074,6 +5171,7 @@ void *mas_next(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5097,6 +5195,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5205,6 +5304,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5228,6 +5328,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5274,6 +5375,7 @@ EXPORT_SYMBOL_GPL(mt_prev); */ void mas_pause(struct ma_state *mas) { + mas_lock_check(mas); mas->status = ma_pause; mas->node = NULL; } @@ -5382,6 +5484,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5409,6 +5512,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5428,7 +5532,6 @@ EXPORT_SYMBOL_GPL(mas_find_range); static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) { - switch (mas->status) { case ma_active: goto active; @@ -5521,6 +5624,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5547,6 +5651,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5564,6 +5669,10 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev); * Searches for @mas->index, sets @mas->index and @mas->last to the range and * erases that range. * + * Note that erase requires allocations and will use GFP_KERNEL to do so if + * necessary. If the allocation fails, the internal lock will be dropped to + * retry. + * * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. */ void *mas_erase(struct ma_state *mas) @@ -5572,13 +5681,19 @@ void *mas_erase(struct ma_state *mas) unsigned long index = mas->index; MA_WR_STATE(wr_mas, mas, NULL); - if (!mas_is_active(mas) || !mas_is_start(mas)) - mas->status = ma_start; + /* + * In low memory situations, the allocation is retried with the gfp flag + * GFP_KERNEL. The internal spinlock is dropped in mas_nomem(), however + * the external lock is not dropped. + */ + if (mt_external_lock(mas->tree)) + might_alloc(GFP_KERNEL); + mas_make_walkable(mas); write_retry: entry = mas_state_walk(mas); if (!entry) - return NULL; + goto out; /* Must reset to ensure spanning writes of last slot are detected */ mas_reset(mas); @@ -5589,8 +5704,10 @@ write_retry: goto write_retry; } - if (mas_is_err(mas)) + if (mas_is_err(mas)) { + entry = NULL; goto out; + } mas_wr_store_entry(&wr_mas); out: @@ -5601,10 +5718,11 @@ EXPORT_SYMBOL_GPL(mas_erase); /** * mas_nomem() - Check if there was an error allocating and do the allocation - * if necessary If there are allocations, then free them. + * if necessary. + * * @mas: The maple state * @gfp: The GFP_FLAGS to use for allocations - * Return: true on allocation, false otherwise. + * Return: False on no memory. True otherwise (partial success as well) */ bool mas_nomem(struct ma_state *mas, gfp_t gfp) __must_hold(mas->tree->ma_lock) @@ -5612,6 +5730,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) if (likely(mas->node != MA_ERROR(-ENOMEM))) return false; + /* Allocations can fail, don't do this. */ + WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) && + mt_external_lock(mas->tree)); + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { mtree_unlock(mas->tree); mas_alloc_nodes(mas, gfp); @@ -5620,10 +5742,17 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) mas_alloc_nodes(mas, gfp); } - if (!mas->sheaf && !mas->alloc) + /* + * Return false on zero forward progress. Partial allocations are kept + * so the retry path will attempt to get the rest. The failure should + * not happen as we try our best to reclaim. The user would need an + * external lock with a non-blocking gfp in a low memory situation - + * which would have triggered the first warning in this function. + */ + if (WARN_ON_ONCE(!mas->sheaf && !mas->alloc)) return false; - mas->status = ma_start; + mas_reset(mas); return true; } @@ -5913,6 +6042,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange); * Erasing is the same as a walk to an entry then a store of a NULL to that * ENTIRE range. In fact, it is implemented as such using the advanced API. * + * Note that erase requires allocations and will use GFP_KERNEL to do so if + * necessary. If the allocation fails, the internal lock will be dropped to + * retry. + * * Return: The entry stored at the @index or %NULL */ void *mtree_erase(struct maple_tree *mt, unsigned long index) @@ -5922,6 +6055,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); trace_ma_op(TP_FCT, &mas); + might_alloc(GFP_KERNEL); mtree_lock(mt); entry = mas_erase(&mas); mtree_unlock(mt); diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 6e72a0d8a53d..6911daa9f854 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1151,10 +1151,9 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, if (!dpage && !order) return VM_FAULT_OOM; - pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", - page_to_pfn(spage), page_to_pfn(dpage)); - if (dpage) { + pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", + page_to_pfn(spage), page_to_pfn(dpage)); lock_page(dpage); *dst |= migrate_pfn(page_to_pfn(dpage)); } diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540c..bfe7bef80f34 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1053,6 +1053,9 @@ void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, if (xas->xa_shift + XA_CHUNK_SHIFT > order) return; + if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) + gfp |= __GFP_ACCOUNT; + do { struct xa_node *node; diff --git a/mm/Kconfig b/mm/Kconfig index 3185500ce7b7..604c58199acb 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -125,8 +125,6 @@ config ZSWAP_COMPRESSOR_DEFAULT config ZSMALLOC tristate -if ZSMALLOC - menu "Zsmalloc allocator options" depends on ZSMALLOC @@ -161,8 +159,6 @@ config ZSMALLOC_CHAIN_SIZE endmenu -endif - menu "Slab allocator options" config SLUB @@ -394,6 +390,7 @@ choice config FLATMEM_MANUAL bool "Flat Memory" + depends on !NUMA depends on !ARCH_SPARSEMEM_ENABLE || ARCH_FLATMEM_ENABLE help This option is best suited for non-NUMA systems with @@ -428,6 +425,7 @@ config SPARSEMEM config FLATMEM def_bool y + depends on !NUMA depends on !SPARSEMEM || FLATMEM_MANUAL # @@ -583,12 +581,10 @@ endchoice config MEMORY_HOTREMOVE bool "Allow for memory hot remove" - depends on MEMORY_HOTPLUG select MIGRATION config MHP_MEMMAP_ON_MEMORY def_bool y - depends on MEMORY_HOTPLUG && SPARSEMEM_VMEMMAP depends on ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE endif # MEMORY_HOTPLUG @@ -771,6 +767,7 @@ config MEMORY_FAILURE depends on ARCH_SUPPORTS_MEMORY_FAILURE bool "Enable recovery from hardware memory errors" select INTERVAL_TREE + select MIGRATION help Enables code to recover from some memory failures on systems with MCA recovery. This allows a system to continue running @@ -1227,9 +1224,7 @@ config ZONE_DMA32 config ZONE_DEVICE bool "Device memory (pmem, HMM, etc...) hotplug support" - depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE - depends on SPARSEMEM_VMEMMAP select XARRAY_MULTI help @@ -1395,17 +1390,15 @@ menuconfig USERFAULTFD Enable the userfaultfd() system call that allows to intercept and handle page faults in userland. -if USERFAULTFD config PTE_MARKER_UFFD_WP bool "Userfaultfd write protection support for shmem/hugetlbfs" default y - depends on HAVE_ARCH_USERFAULTFD_WP + depends on USERFAULTFD && HAVE_ARCH_USERFAULTFD_WP help Allows to create marker PTEs for userfaultfd write protection purposes. It is required to enable userfaultfd write protection on file-backed memory types like shmem and hugetlbfs. -endif # USERFAULTFD # multi-gen LRU { config LRU_GEN @@ -1471,6 +1464,13 @@ config NUMA_EMU into virtual nodes when booted with "numa=fake=N", where N is the number of nodes. This is only useful for debugging. +config GENERIC_ARCH_NUMA + bool + select NUMA_MEMBLKS + help + Enable support for generic NUMA implementation. Currently, RISC-V + and ARM64 use it. + config ARCH_HAS_USER_SHADOW_STACK bool help diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 5737a504efbb..15dca19dd07d 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -326,7 +326,6 @@ config MEM_ALLOC_PROFILING default n depends on MMU depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU select CODE_TAGGING select PAGE_EXTENSION select SLAB_OBJ_EXT diff --git a/mm/Makefile b/mm/Makefile index ab37ef428d98..e7245cb88c66 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -122,6 +122,7 @@ obj-$(CONFIG_CMA) += cma.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_NUMA_MEMBLKS) += numa_memblks.o obj-$(CONFIG_NUMA_EMU) += numa_emulation.o +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-$(CONFIG_BALLOON) += balloon.o obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o obj-$(CONFIG_PAGE_TABLE_CHECK) += page_table_check.o diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c index e93e7fec1f06..b33410310477 100644 --- a/mm/alloc_tag.c +++ b/mm/alloc_tag.c @@ -5,6 +5,8 @@ #include <linux/gfp.h> #include <linux/kallsyms.h> #include <linux/module.h> +#include <linux/mutex.h> +#include <linux/compat.h> #include <linux/page_ext.h> #include <linux/pgalloc_tag.h> #include <linux/proc_fs.h> @@ -14,6 +16,7 @@ #include <linux/string_choices.h> #include <linux/vmalloc.h> #include <linux/kmemleak.h> +#include <uapi/linux/alloc_tag.h> #include "internal.h" #include "page_alloc.h" @@ -59,6 +62,11 @@ struct allocinfo_private { struct codetag_iterator iter; struct codetag_iterator reported_iter; bool print_header; + struct allocinfo_filter filter; + /* ioctl uses a separate iterator not to interfere with reads */ + struct codetag_iterator ioctl_iter; + bool positioned; /* seq_open_private() sets to 0 */ + struct mutex ioctl_lock; }; static void *allocinfo_start(struct seq_file *m, loff_t *pos) @@ -142,6 +150,340 @@ static const struct seq_operations allocinfo_seq_op = { .show = allocinfo_show, }; +/* + * Initializes seq_file operations and allocates private state when opening + * the /proc/allocinfo procfs entry. + */ +static int allocinfo_open(struct inode *inode, struct file *file) +{ + int ret; + + ret = seq_open_private(file, &allocinfo_seq_op, + sizeof(struct allocinfo_private)); + if (!ret) { + struct seq_file *m = file->private_data; + struct allocinfo_private *priv = m->private; + + mutex_init(&priv->ioctl_lock); + } + return ret; +} + +/* + * Cleans up the seq_file state and frees up the private state allocated in + * allocinfo_open() when closing the /proc/allocinfo file descriptor. + */ +static int allocinfo_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + struct allocinfo_private *priv = m->private; + + mutex_destroy(&priv->ioctl_lock); + return seq_release_private(inode, file); +} + +/* + * Returns a pointer to the suffix of a string so that its length fits within + * ALLOCINFO_STR_SIZE, preserving the trailing characters. + * Function, file and module names often have the same prefixes, therefore + * when filtering by these criteria, we compare the last 64 characters to + * minimize the chances of name collisions + */ +static const char *allocinfo_str(const char *str) +{ + size_t len = strlen(str); + + /* Keep an extra space for the trailing NULL. */ + if (len >= ALLOCINFO_STR_SIZE) + str += (len - ALLOCINFO_STR_SIZE) + 1; + return str; +} + +/* Copy a string and trim from the beginning if it's too long */ +static void allocinfo_copy_str(char *dest, const char *src) +{ + strscpy_pad(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE); +} + +/* Compare two strings and only consider the trimmed suffix if s1 is too long */ +static int allocinfo_cmp_str(const char *str, const char *template) +{ + return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE); +} + +/* Fetch the per-CPU counters */ +static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct) +{ + return alloc_tag_read(ct_to_alloc_tag(ct)); +} + +/* + * Populates the UAPI allocinfo_tag_data structure with active runtime + * profiling counters extracted from the given kernel codetag. + */ +static void allocinfo_to_params(struct codetag *ct, + struct allocinfo_tag_data *data, + struct alloc_tag_counters *counters) +{ + if (ct->modname) + allocinfo_copy_str(data->tag.modname, ct->modname); + else + data->tag.modname[0] = '\0'; + allocinfo_copy_str(data->tag.function, ct->function); + allocinfo_copy_str(data->tag.filename, ct->filename); + data->tag.lineno = ct->lineno; + data->counter.bytes = counters->bytes; + data->counter.calls = counters->calls; + data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct)); +} + +/* + * Retrieves the unique content ID representing the current allocation tag module + * layout, allowing userspace to detect if modules were loaded / unloaded. + */ +static int allocinfo_ioctl_get_content_id(struct seq_file *m, void __user *arg) +{ + struct allocinfo_content_id params; + + codetag_lock_module_list(alloc_tag_cttype); + params.id = codetag_get_content_id(alloc_tag_cttype); + codetag_unlock_module_list(alloc_tag_cttype); + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +/* + * Verifies whether a given codetag satisfies the active filtering criteria by + * matching its characteristics against the specified filter. + */ +static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter, + struct alloc_tag_counters *counters, + bool *fetched_counters) +{ + bool inaccurate; + + if (!filter || !filter->mask) + return true; + + if (filter->mask & ALLOCINFO_FILTER_MASK_MODNAME) { + /* user wants to filter by modname but ct->modname is NULL */ + if (!ct->modname) { + /* validate if user was attempting to filter for built-in allocations */ + if (filter->fields.modname[0] != '\0') + return false; + } else if (allocinfo_cmp_str(ct->modname, filter->fields.modname)) + return false; + } + + if ((filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) && + ct->function && allocinfo_cmp_str(ct->function, filter->fields.function)) + return false; + + if ((filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) && + ct->filename && allocinfo_cmp_str(ct->filename, filter->fields.filename)) + return false; + + if ((filter->mask & ALLOCINFO_FILTER_MASK_LINENO) && + ct->lineno != filter->fields.lineno) + return false; + + if (filter->mask & ALLOCINFO_FILTER_MASK_INACCURATE) { + inaccurate = !!(ct->flags & CODETAG_FLAG_INACCURATE); + if (inaccurate != !!(filter->inaccurate)) + return false; + } + + if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE)) { + if (!*fetched_counters) { + *counters = allocinfo_prefetch_counters(ct); + *fetched_counters = true; + } + if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) && + counters->bytes < filter->min_size) + return false; + if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) && + counters->bytes > filter->max_size) + return false; + } + + return true; +} + +/* + * Seeks the ioctl iterator to the specified 0-indexed tag position, reads its + * profiling data and returns it to userspace. + */ +static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg) +{ + struct allocinfo_private *priv; + struct codetag *ct; + struct allocinfo_get_at params = {0}; + __u64 skip_count; + struct alloc_tag_counters counters; + bool fetched_counters; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS) + return -EINVAL; + + if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) && + (params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) && + params.filter.min_size > params.filter.max_size) + return -EINVAL; + + priv = m->private; + + mutex_lock(&priv->ioctl_lock); + codetag_lock_module_list(alloc_tag_cttype); + + if (params.pos >= codetag_get_count(alloc_tag_cttype)) { + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + return -ENOENT; + } + + skip_count = params.pos; + + if (params.filter.mask) + priv->filter = params.filter; + else + priv->filter.mask = 0; + + /* Find the codetag */ + priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype); + ct = codetag_next_ct(&priv->ioctl_iter); + + while (ct) { + fetched_counters = false; + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) { + if (skip_count == 0) + break; + skip_count--; + } + ct = codetag_next_ct(&priv->ioctl_iter); + } + + if (ct) { + if (!fetched_counters) + counters = allocinfo_prefetch_counters(ct); + allocinfo_to_params(ct, ¶ms.data, &counters); + priv->positioned = true; + } + + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + + if (!ct) + return -ENOENT; + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +/* + * Advances the ioctl iterator to the next allocation tag in the sequence and + * returns its profiling data to userspace. + */ +static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg) +{ + struct allocinfo_private *priv; + struct codetag *ct; + struct allocinfo_tag_data params; + int ret = 0; + struct alloc_tag_counters counters; + bool fetched_counters; + + memset(¶ms, 0, sizeof(params)); + priv = m->private; + + mutex_lock(&priv->ioctl_lock); + codetag_lock_module_list(alloc_tag_cttype); + + if (!priv->positioned) { + priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype); + priv->positioned = true; + } + + ct = codetag_next_ct(&priv->ioctl_iter); + while (ct) { + fetched_counters = false; + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) + break; + ct = codetag_next_ct(&priv->ioctl_iter); + } + + if (ct) { + if (!fetched_counters) + counters = allocinfo_prefetch_counters(ct); + allocinfo_to_params(ct, ¶ms, &counters); + } + if (!ct) { + priv->positioned = false; + ret = -ENOENT; + } + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + + if (ret == 0) { + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + } + return ret; +} + +/* + * Entry point ioctl function for /proc/allocinfo routing requests to fetch the + * layout content ID, seek to a specific tag, or read sequential tags. + */ +static long allocinfo_ioctl(struct file *file, unsigned int cmd, + unsigned long __arg) +{ + void __user *arg = (void __user *)__arg; + int ret; + + switch (cmd) { + case ALLOCINFO_IOC_CONTENT_ID: + ret = allocinfo_ioctl_get_content_id(file->private_data, arg); + break; + case ALLOCINFO_IOC_GET_AT: + ret = allocinfo_ioctl_get_at(file->private_data, arg); + break; + case ALLOCINFO_IOC_GET_NEXT: + ret = allocinfo_ioctl_get_next(file->private_data, arg); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + return ret; +} + +#ifdef CONFIG_COMPAT +static long allocinfo_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return allocinfo_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static const struct proc_ops allocinfo_proc_ops = { + .proc_open = allocinfo_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = allocinfo_release, + .proc_ioctl = allocinfo_ioctl, +#ifdef CONFIG_COMPAT + .proc_compat_ioctl = allocinfo_compat_ioctl, +#endif +}; + size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) { struct codetag_iterator iter; @@ -961,6 +1303,12 @@ static const struct ctl_table memory_allocation_profiling_sysctls[] = { .mode = 0644, .proc_handler = proc_mem_profiling_handler, }, + { + .procname = "mem_profiling_compressed", + .data = &mem_profiling_compressed, + .mode = 0444, + .proc_handler = proc_do_static_key, + }, }; static void __init sysctl_init(void) @@ -993,8 +1341,7 @@ static int __init alloc_tag_init(void) return 0; } - if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, - sizeof(struct allocinfo_private), NULL)) { + if (!proc_create(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_proc_ops)) { pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); shutdown_mem_profiling(false); return -ENOMEM; diff --git a/drivers/base/arch_numa.c b/mm/arch_numa.c index 6476227b772c..a4b1732274fa 100644 --- a/drivers/base/arch_numa.c +++ b/mm/arch_numa.c @@ -105,6 +105,18 @@ static void __init setup_node_to_cpumask_map(void) if (nr_node_ids == MAX_NUMNODES) setup_nr_node_ids(); + /* + * This check should never be true but it makes it clear to compilers + * that node_to_cpumask_map is bound by nr_node_ids, avoiding false + * positive fortify warnings when accessing node_to_cpumask_map in the + * for loop below. + */ + if (unlikely(nr_node_ids > MAX_NUMNODES)) { + pr_err("nr_node_ids (%u) is larger than MAX_NUMNODES (%u)\n", + nr_node_ids, MAX_NUMNODES); + return; + } + /* allocate and clear the mapping */ for (node = 0; node < nr_node_ids; node++) { alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); @@ -242,7 +242,7 @@ static int __init cma_new_area(const char *name, phys_addr_t size, if (name) strscpy(cma->name, name); else - snprintf(cma->name, CMA_MAX_NAME, "cma%d\n", cma_area_count); + snprintf(cma->name, CMA_MAX_NAME, "cma%d", cma_area_count); cma->available_count = cma->count = size >> PAGE_SHIFT; cma->order_per_bit = order_per_bit; @@ -2,6 +2,7 @@ #ifndef __MM_CMA_H__ #define __MM_CMA_H__ +#include <linux/cma.h> #include <linux/debugfs.h> #include <linux/kobject.h> diff --git a/mm/debug_page_alloc.c b/mm/debug_page_alloc.c index 6a26eca546c3..fd2664c3c86a 100644 --- a/mm/debug_page_alloc.c +++ b/mm/debug_page_alloc.c @@ -20,14 +20,14 @@ early_param("debug_pagealloc", early_debug_pagealloc); static int __init debug_guardpage_minorder_setup(char *buf) { - unsigned long res; + unsigned int res; - if (kstrtoul(buf, 10, &res) < 0 || res > MAX_PAGE_ORDER / 2) { - pr_err("Bad debug_guardpage_minorder value: %s\n", buf); + if (!buf || kstrtouint(buf, 10, &res) < 0 || res > MAX_PAGE_ORDER / 2) { + pr_err("Bad debug_guardpage_minorder value: %s\n", buf ?: "(missing)"); return 0; } _debug_guardpage_minorder = res; - pr_info("Setting debug_guardpage_minorder to %lu\n", res); + pr_info("Setting debug_guardpage_minorder to %u\n", res); return 0; } early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); diff --git a/mm/folio-compat.c b/mm/folio-compat.c index a02179a0bded..6212fdd6761a 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -41,6 +41,7 @@ void set_page_writeback(struct page *page) } EXPORT_SYMBOL(set_page_writeback); +/* Read the comment above folio_mark_dirty() regarding required locks! */ bool set_page_dirty(struct page *page) { return folio_mark_dirty(page_folio(page)); diff --git a/mm/folio.c b/mm/folio.c index d2937600cf72..59c477120b9a 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -265,73 +265,6 @@ void folio_rotate_reclaimable(struct folio *folio) folio_batch_add_and_move(folio, lru_move_tail); } -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated) - __releases(lruvec->lru_lock) - __releases(rcu) -{ - unsigned long cost; - - /* - * Reflect the relative cost of incurring IO and spending CPU - * time on rotations. This doesn't attempt to make a precise - * comparison, it just says: if reloads are about comparable - * between the LRU lists, or rotations are overwhelmingly - * different between them, adjust scan balance for CPU work. - */ - cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated; - if (!cost) { - spin_unlock_irq(&lruvec->lru_lock); - rcu_read_unlock(); - return; - } - - for (;;) { - unsigned long lrusize; - - /* Record cost event */ - if (file) - lruvec->file_cost += cost; - else - lruvec->anon_cost += cost; - - /* - * Decay previous events - * - * Because workloads change over time (and to avoid - * overflow) we keep these statistics as a floating - * average, which ends up weighing recent refaults - * more than old ones. - */ - lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + - lruvec_page_state(lruvec, NR_ACTIVE_ANON) + - lruvec_page_state(lruvec, NR_INACTIVE_FILE) + - lruvec_page_state(lruvec, NR_ACTIVE_FILE); - - if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { - lruvec->file_cost /= 2; - lruvec->anon_cost /= 2; - } - - spin_unlock_irq(&lruvec->lru_lock); - lruvec = parent_lruvec(lruvec); - if (!lruvec) { - rcu_read_unlock(); - break; - } - spin_lock_irq(&lruvec->lru_lock); - } -} - -void lru_note_cost_refault(struct folio *folio) -{ - struct lruvec *lruvec; - - lruvec = folio_lruvec_lock_irq(folio); - lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio), - folio_nr_pages(folio), 0); -} - static void lru_activate(struct lruvec *lruvec, struct folio *folio) { long nr_pages = folio_nr_pages(folio); @@ -948,6 +881,52 @@ void lru_add_drain_all(void) } #endif /* CONFIG_SMP */ +/** + * lru_cache_drain_for_folio() - drain LRU caches if the caches might hold + * folio references + * @folio: The folio. + * @extra_refs: Extra folio references held by the caller. + * @drained: Drain status for batch folio processing. + * + * Drain LRU caches if the caches might hold folio references. Start + * with a local LRU cache drain, to then drain LRU caches on all CPUs if + * local draining was insufficient. + * + * This function detects LRU cache references by comparing the folio refcount + * with the sum of the expected folio refcount + extra references held by the + * caller. Note that we cannot rely on PG_lru to reliably detect all LRU + * cache references, and there are rare scenarios (concurrent folio (un)mapping) + * where this function might miss detecting LRU cache references. + * + * If @drained is not NULL, the function will avoid re-draining LRU caches + * when processing multiple folios in a row. In that case, the variable + * @drained points at must be initialized to LRU_CACHE_NOT_DRAINED before + * the first invocation by the caller. + */ +void lru_cache_drain_for_folio(const struct folio *folio, + unsigned int extra_refs, enum lru_cache_drained *drained) +{ + if (!folio_may_be_lru_cached(folio)) + return; + + if (!drained || *drained == LRU_CACHE_NOT_DRAINED) { + if (folio_ref_count(folio) == + folio_expected_ref_count(folio) + extra_refs) + return; + lru_add_drain(); + if (drained) + *drained = LRU_CACHE_DRAINED; + } + if (!drained || *drained == LRU_CACHE_DRAINED) { + if (folio_ref_count(folio) == + folio_expected_ref_count(folio) + extra_refs) + return; + lru_add_drain_all(); + if (drained) + *drained = LRU_CACHE_DRAINED_ALL; + } +} + atomic_t lru_disable_count = ATOMIC_INIT(0); /* @@ -1162,8 +1141,6 @@ void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid)); - parent_lruvec->anon_cost += child_lruvec->anon_cost; - parent_lruvec->file_cost += child_lruvec->file_cost; for_each_lru(lru) lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid); @@ -1236,7 +1236,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * Anon pages in shared mappings are surprising: now * just reject it. */ - if (!is_cow_mapping(vm_flags)) + if (!vma_is_cow_mapping(vma)) return -EFAULT; } } else if (!(vm_flags & VM_READ)) { @@ -2266,13 +2266,14 @@ static unsigned long collect_longterm_unpinnable_folios( struct list_head *movable_folio_list, struct pages_or_folios *pofs) { + enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED; unsigned long collected = 0; struct folio *folio; - int drained = 0; long i = 0; for (folio = pofs_get_folio(pofs, i); folio; folio = pofs_next_folio(folio, pofs, &i)) { + const int pin_refs = folio_has_pincount(folio) ? 1 : GUP_PIN_COUNTING_BIAS; if (folio_is_longterm_pinnable(folio)) continue; @@ -2287,18 +2288,12 @@ static unsigned long collect_longterm_unpinnable_folios( continue; } - if (drained == 0 && folio_may_be_lru_cached(folio) && - folio_ref_count(folio) != - folio_expected_ref_count(folio) + 1) { - lru_add_drain(); - drained = 1; - } - if (drained == 1 && folio_may_be_lru_cached(folio) && - folio_ref_count(folio) != - folio_expected_ref_count(folio) + 1) { - lru_add_drain_all(); - drained = 2; - } + /* + * We drain not only to make the folio_isolate_lru() succeed, + * but also to remove any other folio references from LRU + * caches. + */ + lru_cache_drain_for_folio(folio, pin_refs, &drained); if (!folio_isolate_lru(folio)) continue; diff --git a/mm/gup_test.c b/mm/gup_test.c index eb4c9cda16ed..44c1cdfb9c37 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -8,6 +8,12 @@ #include <linux/highmem.h> #include "gup_test.h" +struct gup_test_data { + struct mutex longterm_mutex; + struct page **longterm_pages; + unsigned long longterm_nr_pages; +}; + static void put_back_pages(unsigned int cmd, struct page **pages, unsigned long nr_pages, unsigned int gup_test_flags) { @@ -208,23 +214,20 @@ free_pages: return ret; } -static DEFINE_MUTEX(pin_longterm_test_mutex); -static struct page **pin_longterm_test_pages; -static unsigned long pin_longterm_test_nr_pages; - -static inline void pin_longterm_test_stop(void) +static inline void pin_longterm_test_stop(struct gup_test_data *data) { - if (pin_longterm_test_pages) { - if (pin_longterm_test_nr_pages) - unpin_user_pages(pin_longterm_test_pages, - pin_longterm_test_nr_pages); - kvfree(pin_longterm_test_pages); - pin_longterm_test_pages = NULL; - pin_longterm_test_nr_pages = 0; + if (data->longterm_pages) { + if (data->longterm_nr_pages) + unpin_user_pages(data->longterm_pages, + data->longterm_nr_pages); + kvfree(data->longterm_pages); + data->longterm_pages = NULL; + data->longterm_nr_pages = 0; } } -static inline int pin_longterm_test_start(unsigned long arg) +static inline int pin_longterm_test_start(struct gup_test_data *data, + unsigned long arg) { long nr_pages, cur_pages, addr, remaining_pages; int gup_flags = FOLL_LONGTERM; @@ -233,7 +236,7 @@ static inline int pin_longterm_test_start(unsigned long arg) int ret = 0; bool fast; - if (pin_longterm_test_pages) + if (data->longterm_pages) return -EINVAL; if (copy_from_user(&args, (void __user *)arg, sizeof(args))) @@ -263,12 +266,12 @@ static inline int pin_longterm_test_start(unsigned long arg) return -EINTR; } - pin_longterm_test_pages = pages; - pin_longterm_test_nr_pages = 0; + data->longterm_pages = pages; + data->longterm_nr_pages = 0; - while (nr_pages - pin_longterm_test_nr_pages) { - remaining_pages = nr_pages - pin_longterm_test_nr_pages; - addr = args.addr + pin_longterm_test_nr_pages * PAGE_SIZE; + while (nr_pages - data->longterm_nr_pages) { + remaining_pages = nr_pages - data->longterm_nr_pages; + addr = args.addr + data->longterm_nr_pages * PAGE_SIZE; if (fast) cur_pages = pin_user_pages_fast(addr, remaining_pages, @@ -277,11 +280,11 @@ static inline int pin_longterm_test_start(unsigned long arg) cur_pages = pin_user_pages(addr, remaining_pages, gup_flags, pages); if (cur_pages < 0) { - pin_longterm_test_stop(); + pin_longterm_test_stop(data); ret = cur_pages; break; } - pin_longterm_test_nr_pages += cur_pages; + data->longterm_nr_pages += cur_pages; pages += cur_pages; } @@ -290,19 +293,20 @@ static inline int pin_longterm_test_start(unsigned long arg) return ret; } -static inline int pin_longterm_test_read(unsigned long arg) +static inline int pin_longterm_test_read(struct gup_test_data *data, + unsigned long arg) { __u64 user_addr; unsigned long i; - if (!pin_longterm_test_pages) + if (!data->longterm_pages) return -EINVAL; if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr))) return -EFAULT; - for (i = 0; i < pin_longterm_test_nr_pages; i++) { - void *addr = kmap_local_page(pin_longterm_test_pages[i]); + for (i = 0; i < data->longterm_nr_pages; i++) { + void *addr = kmap_local_page(data->longterm_pages[i]); unsigned long ret; ret = copy_to_user((void __user *)(unsigned long)user_addr, addr, @@ -318,25 +322,26 @@ static inline int pin_longterm_test_read(unsigned long arg) static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { + struct gup_test_data *data = filep->private_data; int ret = -EINVAL; - if (mutex_lock_killable(&pin_longterm_test_mutex)) + if (mutex_lock_killable(&data->longterm_mutex)) return -EINTR; switch (cmd) { case PIN_LONGTERM_TEST_START: - ret = pin_longterm_test_start(arg); + ret = pin_longterm_test_start(data, arg); break; case PIN_LONGTERM_TEST_STOP: - pin_longterm_test_stop(); + pin_longterm_test_stop(data); ret = 0; break; case PIN_LONGTERM_TEST_READ: - ret = pin_longterm_test_read(arg); + ret = pin_longterm_test_read(data, arg); break; } - mutex_unlock(&pin_longterm_test_mutex); + mutex_unlock(&data->longterm_mutex); return ret; } @@ -375,15 +380,40 @@ static long gup_test_ioctl(struct file *filep, unsigned int cmd, return 0; } +static int gup_test_open(struct inode *inode, struct file *file) +{ + struct gup_test_data *data; + int ret; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + ret = nonseekable_open(inode, file); + if (ret) { + kfree(data); + return ret; + } + + mutex_init(&data->longterm_mutex); + file->private_data = data; + return 0; +} + static int gup_test_release(struct inode *inode, struct file *file) { - pin_longterm_test_stop(); + struct gup_test_data *data = file->private_data; + + pin_longterm_test_stop(data); + mutex_destroy(&data->longterm_mutex); + kfree(data); + file->private_data = NULL; return 0; } static const struct file_operations gup_test_fops = { - .open = nonseekable_open, + .open = gup_test_open, .unlocked_ioctl = gup_test_ioctl, .compat_ioctl = compat_ptr_ioctl, .release = gup_test_release, @@ -670,7 +670,10 @@ static int hmm_do_fault(struct mm_struct *mm, ret = handle_mm_fault(vma, addr, fault_flags, NULL); if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) { - *hmm_vma_walk->locked = false; + if (hmm_vma_walk->locked) /* needed by sparse */ + *hmm_vma_walk->locked = false; + else + WARN_ON_ONCE(1); /* broken fault handler */ return HMM_FAULT_UNLOCKED; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4a76a41aa38f..afbb5974bd22 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1681,7 +1681,7 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == (VM_PFNMAP|VM_MIXEDMAP)); - BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); + BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); pfnmap_setup_cachemode_pfn(pfn, &pgprot); @@ -1789,7 +1789,7 @@ vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == (VM_PFNMAP|VM_MIXEDMAP)); - BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); + BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); pfnmap_setup_cachemode_pfn(pfn, &pgprot); @@ -1931,7 +1931,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, * applied special bit, or we made the PRIVATE mapping be * able to wrongly write to the backend MMIO. */ - VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); + VM_WARN_ON_ONCE(vma_is_cow_mapping(src_vma) && pmd_write(pmd)); goto set_pmd; } @@ -2052,7 +2052,7 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, * TODO: once we support anonymous pages, use * folio_try_dup_anon_rmap_*() and split if duplicating fails. */ - if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { + if (vma_is_cow_mapping(vma) && pud_write(pud)) { pudp_set_wrprotect(src_mm, addr, src_pud); pud = pud_wrprotect(pud); } @@ -2932,7 +2932,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ @@ -4107,34 +4107,42 @@ static int __folio_split(struct folio *folio, unsigned int new_order, XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); + struct mem_cgroup *memcg, *old_memcg; struct address_space *mapping = NULL; struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; int nr_shmem_dropped = 0; enum ttu_flags ttu_flags = 0; - int ret; pgoff_t end = 0; + int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { ret = -EINVAL; - goto out; + goto out_no_memcg; } if (new_order >= old_order) { ret = -EINVAL; - goto out; + goto out_no_memcg; } ret = folio_check_splittable(folio, new_order, split_type); if (ret) { VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); - goto out; + goto out_no_memcg; } + /* + * switch to folio's memcg as xarray node allocation can happen and + * needs to charge to it. + */ + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); + if (is_anon) { /* * The caller does not necessarily hold an mmap_lock that would @@ -4277,6 +4285,10 @@ out_unlock: if (mapping) i_mmap_unlock_read(mapping); out: + /* restore to caller's old_memcg */ + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); +out_no_memcg: xas_destroy(&xas); if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); @@ -5079,9 +5091,8 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, return 0; } -void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) +void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct folio *folio) { - struct folio *folio = page_folio(new); struct vm_area_struct *vma = pvmw->vma; struct mm_struct *mm = vma->vm_mm; unsigned long address = pvmw->address; @@ -5117,11 +5128,9 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) swp_entry_t entry; if (pmd_write(pmde)) - entry = make_writable_device_private_entry( - page_to_pfn(new)); + entry = make_writable_device_private_entry(folio_pfn(folio)); else - entry = make_readable_device_private_entry( - page_to_pfn(new)); + entry = make_readable_device_private_entry(folio_pfn(folio)); pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) @@ -5136,11 +5145,12 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) if (!softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; - folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); + folio_add_anon_rmap_pmd(folio, &folio->page, vma, haddr, rmap_flags); } else { - folio_add_file_rmap_pmd(folio, new, vma); + folio_add_file_rmap_pmd(folio, &folio->page, vma); } - VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); + VM_WARN_ON_ONCE(pmd_write(pmde) && folio_test_anon(folio) && + !PageAnonExclusive(&folio->page)); set_pmd_at(mm, haddr, pvmw->pmd, pmde); /* No need to invalidate - it was non-present before */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d49c519e662d..1878edafab23 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4897,7 +4897,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, pte_t *src_pte, *dst_pte, entry; struct folio *pte_folio; unsigned long addr; - bool cow = is_cow_mapping(src_vma->vm_flags); + bool cow = vma_is_cow_mapping(src_vma); struct hstate *h = hstate_vma(src_vma); unsigned long sz = huge_page_size(h); unsigned long npages = pages_per_huge_page(h); @@ -5217,6 +5217,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, bool adjust_reservation; unsigned long last_addr_mask; + i_mmap_assert_write_locked(vma->vm_file->f_mapping); WARN_ON(!is_vm_hugetlb_page(vma)); BUG_ON(start & ~huge_page_mask(h)); BUG_ON(end & ~huge_page_mask(h)); @@ -5308,7 +5309,10 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, /* * Restore the reservation for anonymous page, otherwise the - * backing page could be stolen by someone. + * backing page could be stolen by someone. Restore only on the + * last unmap, otherwise the owner could empty its resv map + * while the folio is still mapped by a child. Note that holding + * i_mmap_lock_write is needed to check the number of mappings. * If there we are freeing a surplus, do not set the restore * reservation bit. */ @@ -5316,7 +5320,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, spin_lock_irq(&hugetlb_lock); if (!h->surplus_huge_pages && __vma_private_lock(vma) && - folio_test_anon(folio)) { + !folio_mapped(folio) && folio_test_anon(folio)) { folio_set_hugetlb_restore_reserve(folio); /* Reservation to be adjusted after the spin lock */ adjust_reservation = true; @@ -7335,14 +7339,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, * There is no need to transfer the per-node surplus state * when we do not cross the node. */ - if (new_nid == old_nid) - return; - spin_lock_irq(&hugetlb_lock); - if (h->surplus_huge_pages_node[old_nid]) { - h->surplus_huge_pages_node[old_nid]--; - h->surplus_huge_pages_node[new_nid]++; + if (new_nid != old_nid) { + spin_lock_irq(&hugetlb_lock); + if (h->surplus_huge_pages_node[old_nid]) { + h->surplus_huge_pages_node[old_nid]--; + h->surplus_huge_pages_node[new_nid]++; + } + spin_unlock_irq(&hugetlb_lock); } - spin_unlock_irq(&hugetlb_lock); } /* diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index ed4e4ae8c01a..95fd2d190f0d 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -10,6 +10,9 @@ #include <asm/setup.h> #include <linux/hugetlb.h> +#include <linux/memblock.h> +#include <linux/math.h> +#include <linux/math64.h> #include "internal.h" #include "hugetlb_cma.h" @@ -19,6 +22,28 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; static bool hugetlb_cma_only __ro_after_init; static unsigned long hugetlb_cma_size __ro_after_init; +static unsigned int hugetlb_cma_percent __initdata; +static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata; + +#ifdef CONFIG_NUMA +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + struct memblock_region *reg; + phys_addr_t size = 0; + + for_each_mem_region(reg) { + if (reg->nid == nid) + size += reg->size; + } + return size; +} +#else +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + return memblock_phys_mem_size(); +} +#endif + void hugetlb_cma_free_frozen_folio(struct folio *folio) { WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], @@ -105,14 +130,31 @@ static int __init cmdline_parse_hugetlb_cma(char *p) break; if (s[count] == ':') { + char *next; + if (tmp >= MAX_NUMNODES) break; nid = array_index_nospec(tmp, MAX_NUMNODES); + hugetlb_cma_size = 0; + hugetlb_cma_percent = 0; + s += count + 1; - tmp = memparse(s, &s); - hugetlb_cma_size_in_node[nid] = tmp; - hugetlb_cma_size += tmp; + tmp = memparse(s, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n", + tmp, nid); + break; + } + hugetlb_cma_percent_in_node[nid] = tmp; + hugetlb_cma_size_in_node[nid] = 0; + s = next + 1; + } else { + hugetlb_cma_size_in_node[nid] = tmp; + hugetlb_cma_percent_in_node[nid] = 0; + s = next; + } /* * Skip the separator if have one, otherwise @@ -123,7 +165,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) else break; } else { - hugetlb_cma_size = memparse(p, &p); + char *next; + + tmp = memparse(p, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp); + } else { + hugetlb_cma_percent = tmp; + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } + } else { + hugetlb_cma_size = tmp; + hugetlb_cma_percent = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } break; } } @@ -149,8 +212,36 @@ void __init hugetlb_cma_reserve(void) { unsigned long size, reserved, per_node, order, gigantic_page_size; bool node_specific_cma_alloc = false; + bool has_node_specific_param = false; int nid; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_size_in_node[nid] || hugetlb_cma_percent_in_node[nid]) { + has_node_specific_param = true; + break; + } + } + + if (has_node_specific_param) { + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_percent_in_node[nid]) { + phys_addr_t node_gfp_mem = memblock_node_memory_size(nid); + u64 s; + + s = mul_u64_u32_div((u64)node_gfp_mem, + hugetlb_cma_percent_in_node[nid], + 100); + + hugetlb_cma_size_in_node[nid] = s; + } + hugetlb_cma_size += hugetlb_cma_size_in_node[nid]; + } + } else if (hugetlb_cma_percent) { + hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(), + hugetlb_cma_percent, 100); + } + if (!hugetlb_cma_size) return; @@ -169,6 +260,32 @@ void __init hugetlb_cma_reserve(void) VM_WARN_ON(order <= MAX_PAGE_ORDER); gigantic_page_size = PAGE_SIZE << order; + if (hugetlb_cma_percent) { + unsigned long orig_size = hugetlb_cma_size; + + hugetlb_cma_size = ALIGN_DOWN(hugetlb_cma_size, PAGE_SIZE << order); + if (orig_size && !hugetlb_cma_size) + pr_warn("hugetlb_cma: reservation size rounded down to 0 from %lu MiB (%u%%)\n", + orig_size / SZ_1M, hugetlb_cma_percent); + } else if (has_node_specific_param) { + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_percent_in_node[nid]) { + unsigned long orig_size = hugetlb_cma_size_in_node[nid]; + + hugetlb_cma_size_in_node[nid] = + ALIGN_DOWN(hugetlb_cma_size_in_node[nid], + PAGE_SIZE << order); + if (orig_size && !hugetlb_cma_size_in_node[nid]) + pr_warn("hugetlb_cma: reservation size rounded down to 0 from %lu MiB (%u%%) on node %d\n", + orig_size / SZ_1M, + hugetlb_cma_percent_in_node[nid], + nid); + } + hugetlb_cma_size += hugetlb_cma_size_in_node[nid]; + } + } + hugetlb_bootmem_set_nodes(); for (nid = 0; nid < MAX_NUMNODES; nid++) { @@ -209,8 +326,13 @@ void __init hugetlb_cma_reserve(void) per_node = DIV_ROUND_UP(hugetlb_cma_size, nodes_weight(hugetlb_bootmem_nodes)); per_node = round_up(per_node, gigantic_page_size); - pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", - hugetlb_cma_size / SZ_1M, per_node / SZ_1M); + if (hugetlb_cma_percent) + pr_info("hugetlb_cma: reserve %lu MiB (%u%%), up to %lu MiB per node\n", + hugetlb_cma_size / SZ_1M, hugetlb_cma_percent, + per_node / SZ_1M); + else + pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", + hugetlb_cma_size / SZ_1M, per_node / SZ_1M); } reserved = 0; @@ -245,8 +367,12 @@ void __init hugetlb_cma_reserve(void) } reserved += size; - pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", - size / SZ_1M, nid); + if (hugetlb_cma_percent_in_node[nid]) + pr_info("hugetlb_cma: reserved %lu MiB (%u%%) on node %d\n", + size / SZ_1M, hugetlb_cma_percent_in_node[nid], nid); + else + pr_info("hugetlb_cma: reserved %lu MiB on node %d\n", + size / SZ_1M, nid); if (reserved >= hugetlb_cma_size) break; diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index 3aa483573d17..730b2b4965b6 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -2,6 +2,8 @@ #ifndef _LINUX_HUGETLB_CMA_H #define _LINUX_HUGETLB_CMA_H +#include <linux/hugetlb.h> + #ifdef CONFIG_CMA void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, diff --git a/mm/internal.h b/mm/internal.h index 68db5abd0a4c..38b1165212c9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -41,12 +41,9 @@ void workingset_refault(struct folio *folio, void *shadow); void workingset_activation(struct folio *folio); /* mm/folio.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated); -void lru_note_cost_refault(struct folio *folio); void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); -static inline bool folio_may_be_lru_cached(struct folio *folio) +static inline bool folio_may_be_lru_cached(const struct folio *folio) { /* * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. @@ -945,7 +942,8 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, return false; pgoff_folio = folio_pgoff(folio); - pgoff_vma_start = vma_start_pgoff(vma); + pgoff_vma_start = folio_test_anon(folio) ? + vma_start_anon_pgoff(vma) : vma_start_pgoff(vma); if (start < vma->vm_start) start = vma->vm_start; @@ -1017,19 +1015,9 @@ void mlock_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); -/** - * vma_address - Find the virtual address a page range is mapped at - * @vma: The vma which maps this object. - * @pgoff: The page offset within its object. - * @nr_pages: The number of pages to consider. - * - * If any page in this range is mapped by this VMA, return the first address - * where any of these pages appear. Otherwise, return -EFAULT. - */ -static inline unsigned long vma_address(const struct vm_area_struct *vma, - pgoff_t pgoff, unsigned long nr_pages) +static inline unsigned long __vma_address(const struct vm_area_struct *vma, + pgoff_t pgoff, pgoff_t pgoff_start, unsigned long nr_pages) { - const pgoff_t pgoff_start = vma_start_pgoff(vma); unsigned long address; if (pgoff >= pgoff_start) { @@ -1047,23 +1035,66 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, return address; } +/** + * vma_filebacked_address - Find the virtual address a file-backed page range is + * mapped at. + * @vma: The vma which maps this object. + * @pgoff: The page offset within its object. + * @nr_pages: The number of pages to consider. + * + * Returns: If any page in this range is mapped by this VMA, return the first + * address where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma, + pgoff_t pgoff, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(vma_is_anonymous(vma)); + + return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages); +} + +/** + * vma_anon_address - Find the virtual address an anonymous page range is mapped + * at. + * @vma: The vma which maps this object. + * @pgoff_anon: The anonymous page index belonging to the folio. + * @nr_pages: The number of pages to consider. + * + * This is only valid for anonymous or MAP_PRIVATE-mapped file-backed VMAs. + * + * Returns: If any page in this range is mapped by this VMA, return the first + * address where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_anon_address(const struct vm_area_struct *vma, + pgoff_t pgoff_anon, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); + + return __vma_address(vma, pgoff_anon, vma_start_anon_pgoff(vma), nr_pages); +} + /* - * Then at what user virtual address will none of the range be found in vma? + * At what user virtual address will none of the range be found in vma? * Assumes that vma_address() already returned a good starting address. */ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) { - struct vm_area_struct *vma = pvmw->vma; - pgoff_t pgoff; + const pgoff_t pgoff_end = pvmw->pgoff + pvmw->nr_pages; + const struct vm_area_struct *vma = pvmw->vma; + pgoff_t pgoff_vma_start; unsigned long address; /* Common case, plus ->pgoff is invalid for KSM */ if (pvmw->nr_pages == 1) return pvmw->address + PAGE_SIZE; - pgoff = pvmw->pgoff + pvmw->nr_pages; + if (pvmw->pgoff_is_anon) + pgoff_vma_start = vma_start_anon_pgoff(vma); + else + pgoff_vma_start = vma_start_pgoff(vma); + address = vma->vm_start + - ((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); + ((pgoff_end - pgoff_vma_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; @@ -1353,7 +1384,7 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, * ... because we only care about writable private ("COW") * mappings where we have to break COW early. */ - return is_cow_mapping(vma->vm_flags); + return vma_is_cow_mapping(vma); } /* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */ diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 3ae9e106d3af..7bbbf15cfbf0 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -83,12 +83,12 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc) { - return vma_start_pgoff(avc->vma); + return vma_start_anon_pgoff(avc->vma); } static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc) { - return vma_last_pgoff(avc->vma); + return vma_last_anon_pgoff(avc->vma); } INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last, diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index 6958aa713c67..c9944fdf48ca 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -355,7 +355,12 @@ void kasan_quarantine_remove_cache(struct kmem_cache *cache) */ on_each_cpu(per_cpu_remove_cache, cache, 1); - for_each_online_cpu(cpu) { + /* + * A CPU can go offline after on_each_cpu() returns, leaving cache + * objects on that CPU's shrink list. Scan all possible CPUs to + * drain those lists. + */ + for_each_possible_cpu(cpu) { sq = per_cpu_ptr(&shrink_qlist, cpu); raw_spin_lock_irqsave(&sq->lock, flags); qlist_move_cache(&sq->qlist, &to_free, cache); @@ -365,9 +370,14 @@ void kasan_quarantine_remove_cache(struct kmem_cache *cache) raw_spin_lock_irqsave(&quarantine_lock, flags); for (i = 0; i < QUARANTINE_BATCHES; i++) { + size_t old_bytes; + if (qlist_empty(&global_quarantine[i])) continue; + old_bytes = global_quarantine[i].bytes; qlist_move_cache(&global_quarantine[i], &to_free, cache); + WRITE_ONCE(quarantine_size, quarantine_size - + (old_bytes - global_quarantine[i].bytes)); /* Scanning whole quarantine can take a while. */ raw_spin_unlock_irqrestore(&quarantine_lock, flags); cond_resched(); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 8b7c3959d193..79effd3f3da4 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -614,7 +614,7 @@ void __khugepaged_exit(struct mm_struct *mm) /* * This is required to serialize against * collapse_test_exit() (which is guaranteed to run - * under mmap sem read mode). Stop here (after we return all + * under mmap_lock read mode). Stop here (after we return all * pagetables will be destroyed) until khugepaged has finished * working on the pagetables under the mmap_lock. */ @@ -623,6 +623,13 @@ void __khugepaged_exit(struct mm_struct *mm) } } +static void collapse_control_init_scan(struct collapse_control *cc) +{ + memset(cc->node_load, 0, sizeof(cc->node_load)); + nodes_clear(cc->alloc_nmask); + bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE); +} + static void release_pte_folio(struct folio *folio) { node_stat_mod_folio(folio, @@ -659,6 +666,30 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte, } } +/* + * folio_pte_referenced() - Check if a folio or its PTE mapping was recently used + * + * Return: true if recent access was observed through either the folio state + * or the current PTE mapping. + */ +static inline bool folio_pte_referenced(struct folio *folio, + struct vm_area_struct *vma, unsigned long addr, pte_t pteval) +{ + /* The folio was referenced previously ... */ + if (folio_test_young(folio) || folio_test_referenced(folio)) + return true; + /* ... or the PTE mapping was recently used */ + return pte_young(pteval) || mmu_notifier_test_young(vma->vm_mm, addr); +} + +static void count_collapse_event(unsigned int order, enum vm_event_item vm_event, + enum mthp_stat_item mthp_event) +{ + if (is_pmd_order(order)) + count_vm_event(vm_event); + count_mthp_stat(order, mthp_event); +} + static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, unsigned long start_addr, pte_t *pte, struct collapse_control *cc, unsigned int order, struct list_head *compound_pagelist) @@ -679,9 +710,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, if (pte_none_or_zero(pteval)) { if (++none_or_zero > max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; - if (is_pmd_order(order)) - count_vm_event(THP_SCAN_EXCEED_NONE_PTE); - count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_NONE); + count_collapse_event(order, THP_SCAN_EXCEED_NONE_PTE, + MTHP_STAT_COLLAPSE_EXCEED_NONE); goto out; } continue; @@ -723,9 +753,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, */ if (++shared > max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; - if (is_pmd_order(order)) - count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); - count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SHARED); + count_collapse_event(order, THP_SCAN_EXCEED_SHARED_PTE, + MTHP_STAT_COLLAPSE_EXCEED_SHARED); goto out; } } @@ -753,8 +782,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, /* * We can do it before folio_isolate_lru because the - * folio can't be freed from under us. NOTE: PG_lock - * is needed to serialize against split_huge_page + * folio can't be freed from under us. NOTE: folio lock + * is needed to serialize against split_huge_page() * when invoked from the VM. */ if (!folio_trylock(folio)) { @@ -780,7 +809,7 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, } /* - * Isolate the page to avoid collapsing an hugepage + * Isolate the folio to avoid collapsing a hugepage * currently in use by the VM. */ if (!folio_isolate_lru(folio)) { @@ -797,14 +826,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, if (folio_test_large(folio)) list_add_tail(&folio->lru, compound_pagelist); next: - /* - * If collapse was initiated by khugepaged, check that there is - * enough young pte to justify collapsing the page - */ if (cc->is_khugepaged && - (pte_young(pteval) || folio_test_young(folio) || - folio_test_referenced(folio) || - mmu_notifier_test_young(vma->vm_mm, addr))) + folio_pte_referenced(folio, vma, addr, pteval)) referenced++; } @@ -898,7 +921,7 @@ static void __collapse_huge_page_copy_failed(pte_t *pte, * Re-establish the PMD to point to the original page table * entry. Restoring PMD needs to be done prior to releasing * pages. Since pages are still isolated and locked here, - * acquiring anon_vma_lock_write is unnecessary. + * acquiring anon_vma_lock_write() is unnecessary. */ pmd_ptl = pmd_lock(vma->vm_mm, pmd); pmd_populate(vma->vm_mm, pmd, pmd_pgtable(orig_pmd)); @@ -1072,9 +1095,9 @@ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned l return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then - * remapped to file after khugepaged reaquired the mmap_lock. + * remapped to file after khugepaged reacquired the mmap_lock. * - * thp_vma_allowable_orders may return true for qualified file + * thp_vma_allowable_orders() may return true for qualified file * vmas. */ if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap))) @@ -1130,7 +1153,7 @@ static enum scan_result check_pmd_still_valid(struct mm_struct *mm, /* * Bring missing pages in from swap, to complete THP collapse. - * Only done if khugepaged_scan_pmd believes it is worthwhile. + * Only done if collapse_scan_pmd() believes it is worthwhile. * * For mTHP orders the function bails on the first swap entry, because * faulting pages back in during collapse could re-populate PTEs that @@ -1198,7 +1221,7 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, pte = NULL; /* - * do_swap_page returns VM_FAULT_RETRY with released mmap_lock. + * do_swap_page() returns VM_FAULT_RETRY with released mmap_lock. * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because * we do not retry here and swap entry will remain in pagetable * resulting in later failure. @@ -1241,15 +1264,12 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); if (!folio) { *foliop = NULL; - if (is_pmd_order(order)) - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); - count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC_FAILED); + count_collapse_event(order, THP_COLLAPSE_ALLOC_FAILED, + MTHP_STAT_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } - if (is_pmd_order(order)) - count_vm_event(THP_COLLAPSE_ALLOC); - count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC); + count_collapse_event(order, THP_COLLAPSE_ALLOC, MTHP_STAT_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); @@ -1265,7 +1285,7 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru } /* - * collapse_huge_page expects the mmap_lock to be unlocked before entering and + * collapse_huge_page() expects the mmap_lock to be unlocked before entering and * will always return with the lock unlocked, to avoid holding the mmap_lock * while allocating a THP, as that could trigger direct reclaim/compaction. * Note that the VMA must be rechecked after grabbing the mmap_lock again. @@ -1312,7 +1332,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s if (unmapped) { /* - * __collapse_huge_page_swapin will return with mmap_lock + * __collapse_huge_page_swapin() will return with mmap_lock * released when it fails. So we jump out_nolock directly in * that case. Continuing to collapse causes inconsistency. */ @@ -1325,8 +1345,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s mmap_read_unlock(mm); /* * Prevent all access to pagetables with the exception of - * gup_fast later handled by the ptep_clear_flush and the VM - * handled by the anon_vma lock + PG_lock. + * gup_fast later handled by the pmdp_collapse_flush() and the VM + * handled by the anon_vma lock + folio lock. * * UFFDIO_MOVE is prevented to race as well thanks to the * mmap_lock. @@ -1383,9 +1403,9 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s spin_lock(pmd_ptl); VM_WARN_ON_ONCE(!pmd_none(*pmd)); /* - * We can only use set_pmd_at when establishing + * We can only use set_pmd_at() when establishing * hugepmds and never for establishing regular pmds that - * points to regular pagetables. Use pmd_populate for that + * points to regular pagetables. Use pmd_populate() for that */ pmd_populate(mm, pmd, pmd_pgtable(_pmd)); spin_unlock(pmd_ptl); @@ -1443,10 +1463,10 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s result = SCAN_SUCCEED; out_up_write: - if (anon_vma_locked) - anon_vma_unlock_write(vma->anon_vma); if (pte) pte_unmap(pte); + if (anon_vma_locked) + anon_vma_unlock_write(vma->anon_vma); mmap_write_unlock(mm); out_nolock: if (folio) @@ -1611,15 +1631,14 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, goto out; } - bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE); - memset(cc->node_load, 0, sizeof(cc->node_load)); - nodes_clear(cc->alloc_nmask); + collapse_control_init_scan(cc); enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags); /* * If PMD is the only enabled order, enforce max_ptes_none, otherwise - * scan all pages to populate the bitmap for mTHP collapse. + * scan all pages to populate the bitmap for mTHP collapse. The bitmap + * is then checked again in mthp_collapse() for each attempted order. */ if (enabled_orders != BIT(HPAGE_PMD_ORDER)) max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT; @@ -1641,9 +1660,8 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (pte_none_or_zero(pteval)) { if (++none_or_zero > max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; - count_vm_event(THP_SCAN_EXCEED_NONE_PTE); - count_mthp_stat(HPAGE_PMD_ORDER, - MTHP_STAT_COLLAPSE_EXCEED_NONE); + count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_NONE_PTE, + MTHP_STAT_COLLAPSE_EXCEED_NONE); goto out_unmap; } continue; @@ -1651,9 +1669,8 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (!pte_present(pteval)) { if (++unmapped > max_ptes_swap) { result = SCAN_EXCEED_SWAP_PTE; - count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); - count_mthp_stat(HPAGE_PMD_ORDER, - MTHP_STAT_COLLAPSE_EXCEED_SWAP); + count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SWAP_PTE, + MTHP_STAT_COLLAPSE_EXCEED_SWAP); goto out_unmap; } /* @@ -1710,9 +1727,8 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, if (folio_maybe_mapped_shared(folio)) { if (++shared > max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; - count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); - count_mthp_stat(HPAGE_PMD_ORDER, - MTHP_STAT_COLLAPSE_EXCEED_SHARED); + count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SHARED_PTE, + MTHP_STAT_COLLAPSE_EXCEED_SHARED); goto out_unmap; } } @@ -1743,26 +1759,17 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, /* * Check if the page has any GUP (or other external) pins. * - * Here the check may be racy: - * it may see folio_mapcount() > folio_ref_count(). - * But such case is ephemeral we could always retry collapse - * later. However it may report false positive if the page - * has excessive GUP pins (i.e. 512). Anyway the same check - * will be done again later the risk seems low. + * Here the check is racy, but such case is ephemeral and + * we could always retry collapse later. Anyway the same + * check will be done again later the risk seems low. */ if (folio_expected_ref_count(folio) != folio_ref_count(folio)) { result = SCAN_PAGE_COUNT; goto out_unmap; } - /* - * If collapse was initiated by khugepaged, check that there is - * enough young pte to justify collapsing the page - */ if (cc->is_khugepaged && - (pte_young(pteval) || folio_test_young(folio) || - folio_test_referenced(folio) || - mmu_notifier_test_young(vma->vm_mm, addr))) + folio_pte_referenced(folio, vma, addr, pteval)) referenced++; } if (cc->is_khugepaged && @@ -1775,7 +1782,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, out_unmap: pte_unmap_unlock(pte, ptl); if (result == SCAN_SUCCEED) { - /* collapse_huge_page expects the lock to be dropped before calling */ + /* collapse_huge_page() expects the lock to be dropped before calling */ mmap_read_unlock(mm); result = mthp_collapse(mm, start_addr, referenced, unmapped, cc, enabled_orders); @@ -2685,8 +2692,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, present = 0; swap = 0; - memset(cc->node_load, 0, sizeof(cc->node_load)); - nodes_clear(cc->alloc_nmask); + collapse_control_init_scan(cc); rcu_read_lock(); xas_for_each(&xas, folio, start + HPAGE_PMD_NR - 1) { if (xas_retry(&xas, folio)) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index e96e9efd19b0..8fa409a4f9fb 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -151,6 +151,8 @@ struct kmemleak_object { int min_count; /* the total number of pointers found pointing to this object */ int count; + /* consecutive scans the object has been seen unreferenced */ + unsigned int unref_scans; /* checksum for detecting modified objects */ u32 checksum; depot_stack_handle_t trace_handle; @@ -175,6 +177,8 @@ struct kmemleak_object { #define OBJECT_PHYS (1 << 4) /* flag set for per-CPU pointers */ #define OBJECT_PERCPU (1 << 5) +/* flag set on an object left unreferenced by the full scan, pending confirmation */ +#define OBJECT_SUSPECT (1 << 6) /* set when __remove_object() called */ #define DELSTATE_REMOVED (1 << 0) @@ -232,9 +236,15 @@ static unsigned long max_percpu_addr; static struct task_struct *scan_thread; /* used to avoid reporting of recently allocated objects */ static unsigned long jiffies_min_age; +/* consecutive scans an object must stay unreferenced before reporting */ +static unsigned int min_unref_scans = + IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 2 : 1; +module_param(min_unref_scans, uint, 0644); static unsigned long jiffies_last_scan; /* delay between automatic memory scannings */ static unsigned long jiffies_scan_wait; +/* number of objects flagged OBJECT_SUSPECT during the current scan */ +static int nr_suspects; /* enables or disables the task stacks scanning */ static int kmemleak_stack_scan = 1; /* protects the memory scanning, parameters and debug/kmemleak file access */ @@ -688,6 +698,7 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp) object->excess_ref = 0; object->count = 0; /* white color initially */ object->checksum = ~0; + object->unref_scans = 0; object->del_state = 0; /* task information */ @@ -1440,6 +1451,11 @@ static void update_refs(struct kmemleak_object *object) */ object->count++; if (color_gray(object)) { + /* referenced after all, no longer a suspect */ + if (object->flags & OBJECT_SUSPECT) { + object->flags &= ~OBJECT_SUSPECT; + nr_suspects--; + } /* put_object() called when removing from gray_list */ WARN_ON(!get_object(object)); list_add_tail(&object->gray_list, &gray_list); @@ -1571,7 +1587,7 @@ static int scan_large_block(void *start, void *end) if (scan_block(start, next, NULL)) return 1; start = next; - cond_resched(); + cond_resched_tasks_rcu_qs(); } return 0; @@ -1608,7 +1624,7 @@ static void scan_object(struct kmemleak_object *object) scan_block(start, end, object); raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); if (!(object->flags & OBJECT_ALLOCATED)) break; @@ -1630,7 +1646,7 @@ static void scan_object(struct kmemleak_object *object) break; raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); } while (object->flags & OBJECT_ALLOCATED); } else { @@ -1658,7 +1674,7 @@ static void scan_gray_list(void) */ object = list_entry(gray_list.next, typeof(*object), gray_list); while (&object->gray_list != &gray_list) { - cond_resched(); + cond_resched_tasks_rcu_qs(); /* may add new objects to the list */ if (!scan_should_stop()) @@ -1693,7 +1709,7 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) raw_spin_unlock_irq(&kmemleak_lock); rcu_read_unlock(); - cond_resched(); + cond_resched_tasks_rcu_qs(); rcu_read_lock(); raw_spin_lock_irq(&kmemleak_lock); @@ -1738,7 +1754,7 @@ static void kmemleak_scan_task_stacks(void) } put_task_struct(p); } - cond_resched(); + cond_resched_tasks_rcu_qs(); } while (pid && !stop); } @@ -1844,16 +1860,16 @@ static void dedup_flush(struct xarray *dedup) * kernel's standard allocators. This function must be called with the * scan_mutex held. */ -static void kmemleak_scan(void) +static int __kmemleak_scan(bool full) { struct kmemleak_object *object; struct zone *zone; int __maybe_unused i; - struct xarray dedup; - int new_leaks = 0; int stop = 0; jiffies_last_scan = jiffies; + if (full) + nr_suspects = 0; /* prepare the kmemleak_object's */ rcu_read_lock(); @@ -1881,8 +1897,13 @@ static void kmemleak_scan(void) __paint_it(object, KMEMLEAK_BLACK); } + /* referenced last scan: restart the unreferenced run */ + if (!color_white(object)) + object->unref_scans = 0; /* reset the reference count (whiten the object) */ object->count = 0; + if (full) + object->flags &= ~OBJECT_SUSPECT; if (color_gray(object) && get_object(object)) list_add_tail(&object->gray_list, &gray_list); @@ -1915,7 +1936,7 @@ static void kmemleak_scan(void) struct page *page = pfn_to_online_page(pfn); if (!(pfn & 63)) - cond_resched(); + cond_resched_tasks_rcu_qs(); if (!page) continue; @@ -1950,6 +1971,10 @@ static void kmemleak_scan(void) scan_gray: scan_gray_list(); + /* a confirmation scan does not look for modified objects */ + if (!full) + return nr_suspects; + /* * Check for new or unreferenced objects modified since the previous * scan and color them gray until the next scan. @@ -1972,6 +1997,11 @@ scan_gray: /* color it gray temporarily */ object->count = object->min_count; list_add_tail(&object->gray_list, &gray_list); + } else if (unreferenced_object(object) && + !(object->flags & OBJECT_REPORTED)) { + /* flag the objects left unreferenced by this scan */ + object->flags |= OBJECT_SUSPECT; + nr_suspects++; } raw_spin_unlock_irq(&object->lock); } @@ -1982,6 +2012,46 @@ scan_gray: */ scan_gray_list(); + return nr_suspects; +} + +/* + * Promote a suspected object to a reported leak once it has stayed + * unreferenced for min_unref_scans consecutive scans. Called with + * object->lock held; returns true when the object is newly reported. + */ +static bool confirm_leak(struct kmemleak_object *object) +{ + if (!unreferenced_object(object) || + !(object->flags & OBJECT_SUSPECT) || + (object->flags & OBJECT_REPORTED)) + return false; + + object->unref_scans += 1; + if (object->unref_scans < min_unref_scans) + return false; + + object->flags |= OBJECT_REPORTED; + return true; +} + +/* + * Scan the memory and report the unreferenced objects as leaks. Must be + * called with the scan_mutex held. + */ +static void kmemleak_scan(void) +{ + struct kmemleak_object *object; + struct xarray dedup; + int new_leaks = 0; + + /* + * Full scan. Objects left unreferenced are flagged OBJECT_SUSPECT and + * counted in the return value; nothing to confirm or report otherwise. + */ + if (!__kmemleak_scan(true)) + return; + /* * If scanning was stopped do not report any new unreferenced objects. */ @@ -1989,6 +2059,16 @@ scan_gray: return; /* + * A live object whose only reference is moved by, for example, a + * concurrent RCU update can be missed for one scan and reported as a + * transient false positive. Scan again and only report the objects + * left unreferenced (still flagged OBJECT_SUSPECT) by both scans. + */ + __kmemleak_scan(false); + if (scan_should_stop()) + return; + + /* * Scanning result reporting. When verbose printing is enabled, dedupe * by stackdepot trace_handle so each unique backtrace is logged once * per scan, annotated with the number of objects that share it. The @@ -2014,9 +2094,8 @@ scan_gray: raw_spin_lock_irq(&object->lock); trace_handle = 0; dedup_print = false; - if (unreferenced_object(object) && - !(object->flags & OBJECT_REPORTED)) { - object->flags |= OBJECT_REPORTED; + + if (confirm_leak(object)) { if (kmemleak_verbose) { trace_handle = object->trace_handle; dedup_print = true; @@ -959,10 +959,9 @@ enum ksm_get_folio_flags { * seconds or even minutes: much too unresponsive. So instead we use a * "keyhole reference": access to the ksm page from the stable node peeps * out through its keyhole to see if that page still holds the right key, - * pointing back to this stable node. This relies on freeing a PageAnon - * page to reset its page->mapping to NULL, and relies on no other use of - * a page to put something that might look like our key in page->mapping. - * is on its way to being freed; but it is an anomaly to bear in mind. + * pointing back to this stable node. This relies on freeing an anon + * folio to reset its mapping to NULL, and relies on no other use of a + * folio to put something that might look like our key in its mapping. */ static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node, enum ksm_get_folio_flags flags) @@ -1625,7 +1624,7 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, * stable_tree, break_cow() will clean it up. */ rmap_item->anon_vma = vma->anon_vma; - rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address); + rmap_item->linear_page_index = linear_anon_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: mmap_read_unlock(mm); @@ -3060,10 +3059,9 @@ int __ksm_enter(struct mm_struct *mm) slot = &mm_slot->slot; + spin_lock(&ksm_mmlist_lock); /* Check ksm_run too? Would need tighter locking */ needs_wakeup = list_empty(&ksm_mm_head.slot.mm_node); - - spin_lock(&ksm_mmlist_lock); mm_slot_insert(mm_slots_hash, mm, slot); /* * When KSM_RUN_MERGE (or KSM_RUN_STOP), @@ -3152,7 +3150,7 @@ struct folio *ksm_might_need_to_copy(struct folio *folio, return folio; /* no need to copy it */ } else if (!anon_vma) { return folio; /* no need to copy it */ - } else if (folio->index == linear_page_index(vma, addr) && + } else if (folio->index == linear_anon_page_index(vma, addr) && anon_vma->root == vma->anon_vma->root) { return folio; /* still no need to copy it */ } @@ -3222,7 +3220,7 @@ again: /* * Currently, KSM folios are always small folios, so it's * sufficient to search for a single page. We can simply use - * the linear_page_index of the original de-duplicate + * the linear_anon_page_index of the original de-duplicate * anonymous page that we remembered in the rmap_item while * de-duplicating. Note that mremap() always de-duplicates KSM * folios: so if there was mremap() in our parent or our child, diff --git a/mm/madvise.c b/mm/madvise.c index 88873686fc41..eeee82cf2b3f 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -32,6 +32,7 @@ #include <linux/leafops.h> #include <linux/shmem_fs.h> #include <linux/mmu_notifier.h> +#include <linux/swap_ops.h> #include <asm/tlb.h> @@ -196,7 +197,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *walk) { struct vm_area_struct *vma = walk->private; - struct swap_iocb *splug = NULL; + struct swap_io_ctx ctx = {}; pte_t *ptep = NULL; spinlock_t *ptl; unsigned long addr; @@ -220,15 +221,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, pte_unmap_unlock(ptep, ptl); ptep = NULL; - folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE, - vma, addr, &splug); + folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE, + vma, addr); if (folio) folio_put(folio); } if (ptep) pte_unmap_unlock(ptep, ptl); - swap_read_unplug(splug); + swap_read_submit(&ctx); cond_resched(); return 0; @@ -246,7 +247,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); pgoff_t end_index = linear_page_index(vma, end) - 1; struct folio *folio; - struct swap_iocb *splug = NULL; + struct swap_io_ctx ctx = {}; rcu_read_lock(); xas_for_each(&xas, folio, end_index) { @@ -265,15 +266,15 @@ static void shmem_swapin_range(struct vm_area_struct *vma, xas_pause(&xas); rcu_read_unlock(); - folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping), - vma, addr, &splug); + folio = read_swap_cache_async(&ctx, entry, + mapping_gfp_mask(mapping), vma, addr); if (folio) folio_put(folio); rcu_read_lock(); } rcu_read_unlock(); - swap_read_unplug(splug); + swap_read_submit(&ctx); } #endif /* CONFIG_SWAP */ diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index d7394dace38d..05ef55cae4dc 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -2297,8 +2297,8 @@ void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) for_each_online_pgdat(pgdat) { mz = memcg->nodeinfo[pgdat->node_id]; - anon_cost += mz->lruvec.anon_cost; - file_cost += mz->lruvec.file_cost; + anon_cost += mz->lruvec.cost[WORKINGSET_ANON].count; + file_cost += mz->lruvec.cost[WORKINGSET_FILE].count; } seq_buf_printf(s, "anon_cost %lu\n", anon_cost); seq_buf_printf(s, "file_cost %lu\n", file_cost); diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 0f703f239c80..1e394269c613 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -4,6 +4,7 @@ #define __MM_MEMCONTROL_V1_H #include <linux/cgroup-defs.h> +#include <linux/memcontrol.h> /* Cgroup v1 and v2 common declarations */ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 69b37f63a307..11b85f4b6828 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -63,6 +63,7 @@ #include <linux/sched/isolation.h> #include <linux/kmemleak.h> #include "internal.h" +#include "swap.h" #include "swap_table.h" #include <net/sock.h> #include <net/ip.h> @@ -398,6 +399,7 @@ static const unsigned int memcg_node_stat_items[] = { NR_SHMEM_THPS, NR_FILE_THPS, NR_ANON_THPS, + NR_VMSCAN_WRITE, NR_VMALLOC, NR_KERNEL_STACK_KB, NR_PAGETABLE, @@ -424,6 +426,8 @@ static const unsigned int memcg_node_stat_items[] = { PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, + PGROTATE_ANON, + PGROTATE_FILE, PGREFILL, #ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB, @@ -507,6 +511,42 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) return x; } +/** + * lruvec_page_state_monotonic - non-clamping lruvec stat read for delta sampling + * @lruvec: the LRU vector to read from + * @idx: the node_stat_item to read + * + * Returns the raw state[idx] value cast to unsigned long, skipping the + * clamp-negative-to-zero step in lruvec_page_state(). Intended for callers + * that snapshot a monotonically-incremented counter and subtract two + * samples: unsigned modular arithmetic then yields the correct delta across + * a signed-long wraparound (a real hazard on 32-bit) that the clamp would + * otherwise turn into a huge spurious delta. + * + * Do NOT use for non-monotonic page-count reads where a transient negative + * reading from per-CPU delta skew must present as zero. + * + * XXX: This helper (and its node/global peers) exists because some + * monotonically-incremented event counters are stored in + * enum node_stat_item. + */ +unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, + enum node_stat_item idx) +{ + struct mem_cgroup_per_node *pn; + int i; + + if (mem_cgroup_disabled()) + return node_page_state_monotonic(lruvec_pgdat(lruvec), idx); + + i = memcg_stats_index(idx); + if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) + return 0; + + pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]); +} + unsigned long lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx) { @@ -2100,7 +2140,12 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) stock_pages = READ_ONCE(stock->nr_pages[i]); if (stock_pages >= nr_pages) { - WRITE_ONCE(stock->nr_pages[i], stock_pages - nr_pages); + stock_pages -= nr_pages; + WRITE_ONCE(stock->nr_pages[i], stock_pages); + if (!stock_pages) { + css_put(&memcg->css); + WRITE_ONCE(stock->cached[i], NULL); + } ret = true; } break; @@ -2653,6 +2698,19 @@ retry: if (!gfpflags_allow_blocking(gfp_mask)) goto nomem; + /* + * OOM victim still needs to charge memory to exit. OOM reaper should + * help but it might fail on mmap_lock contention. If the victim is a + * large thread group then all exiting threads might compete on oom_lock + * just to learn that there is nothing really killable anymore. Bail + * out early and fail the charge to expedite their exit. They are + * considered fully reclaimed by the oom reaper and they shouldn't + * contribute further charges. + */ + if (tsk_is_oom_victim(current) && + mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm)) + goto nomem; + __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); raised_max_event = true; @@ -4176,11 +4234,10 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) #endif page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); if (parent) { - WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); - page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl); page_counter_init(&memcg->swap, &parent->swap, false); #ifdef CONFIG_MEMCG_V1 + WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); memcg->memory.track_failcnt = !memcg_on_dfl; memcg->memsw.track_failcnt = !memcg_on_dfl; WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable)); @@ -4800,6 +4857,9 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, unsigned long nr_pages = page_counter_read(&memcg->memory); unsigned long reclaimed; + if (high != READ_ONCE(memcg->memory.high)) + break; + if (nr_pages <= high) break; @@ -4855,6 +4915,9 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, for (;;) { unsigned long nr_pages = page_counter_read(&memcg->memory); + if (max != READ_ONCE(memcg->memory.max)) + break; + if (nr_pages <= max) break; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index aaf14608b30e..a8b03e2920ba 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -620,7 +620,7 @@ static void add_to_kill_fsdax(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, pgoff_t pgoff) { - unsigned long addr = vma_address(vma, pgoff, 1); + unsigned long addr = vma_filebacked_address(vma, pgoff, 1); __add_to_kill(tsk, p, vma, to_kill, addr); } @@ -2265,7 +2265,7 @@ static void add_to_kill_pgoff(struct task_struct *tsk, } /* Check for pgoff not backed by struct page */ - tk->addr = vma_address(vma, pgoff, 1); + tk->addr = vma_filebacked_address(vma, pgoff, 1); tk->size_shift = PAGE_SHIFT; if (tk->addr == -EFAULT) diff --git a/mm/memory.c b/mm/memory.c index 8da0f945141b..8b0c2c735d3d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -519,9 +519,52 @@ static bool is_bad_page_map_ratelimited(void) return false; } +static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) +{ + if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) { + snprintf(buf, buf_size, "overflow"); + return; + } + + switch (entry_size) { + case sizeof(u32): + snprintf(buf, buf_size, "%08x", *(const u32 *)entry); + break; + case sizeof(u64): + snprintf(buf, buf_size, "%016llx", *(const u64 *)entry); + break; +#if defined(__SIZEOF_INT128__) + case sizeof(u128): + snprintf(buf, buf_size, "%016llx%016llx", + (unsigned long long)(*(const u128 *)entry >> 64), + (unsigned long long)*(const u128 *)entry); + break; +#endif + default: + snprintf(buf, buf_size, "unsupported"); + break; + } +} + +#define ptval_to_str(buf, val) \ + do { \ + auto __val = (val); \ + \ + ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ + } while (0) + +#if defined(__SIZEOF_INT128__) +#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ +#else +#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ +#endif + static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr) { - unsigned long long pgdv, p4dv, pudv, pmdv; + char pgd_str[PTVAL_STR_MAX]; + char p4d_str[PTVAL_STR_MAX]; + char pud_str[PTVAL_STR_MAX]; + char pmd_str[PTVAL_STR_MAX]; p4d_t p4d, *p4dp; pud_t pud, *pudp; pmd_t pmd, *pmdp; @@ -532,34 +575,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * see locking requirements for print_bad_page_map(). */ pgdp = pgd_offset(mm, addr); - pgdv = pgd_val(*pgdp); + ptval_to_str(pgd_str, pgd_val(*pgdp)); if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) { - pr_alert("pgd:%08llx\n", pgdv); + pr_alert("pgd:%s\n", pgd_str); return; } p4dp = p4d_offset(pgdp, addr); p4d = p4dp_get(p4dp); - p4dv = p4d_val(p4d); + ptval_to_str(p4d_str, p4d_val(p4d)); if (!p4d_present(p4d) || p4d_leaf(p4d)) { - pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); + pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str); return; } pudp = pud_offset(p4dp, addr); pud = pudp_get(pudp); - pudv = pud_val(pud); + ptval_to_str(pud_str, pud_val(pud)); if (!pud_present(pud) || pud_leaf(pud)) { - pr_alert("pgd:%08llx p4d:%08llx pud:%08llx\n", pgdv, p4dv, pudv); + pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str); return; } pmdp = pmd_offset(pudp, addr); pmd = pmdp_get(pmdp); - pmdv = pmd_val(pmd); + ptval_to_str(pmd_str, pmd_val(pmd)); /* * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE, @@ -567,8 +610,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * doing another map would be bad. print_bad_page_map() should * already take care of printing the PTE. */ - pr_alert("pgd:%08llx p4d:%08llx pud:%08llx pmd:%08llx\n", pgdv, - p4dv, pudv, pmdv); + pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str); } /* @@ -584,25 +626,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * page table lock. */ static void print_bad_page_map(struct vm_area_struct *vma, - unsigned long addr, unsigned long long entry, struct page *page, - enum pgtable_level level) + unsigned long addr, const void *entry, size_t entry_size, + struct page *page, enum pgtable_level level) { struct address_space *mapping; - pgoff_t index; + char entry_str[PTVAL_STR_MAX]; + pgoff_t index, anon_index; if (is_bad_page_map_ratelimited()) return; mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; index = linear_page_index(vma, addr); + anon_index = __linear_anon_page_index(vma, addr); - pr_alert("BUG: Bad page map in process %s %s:%08llx", current->comm, - pgtable_level_to_str(level), entry); + ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size); + pr_alert("BUG: Bad page map in process %s %s:%s", current->comm, + pgtable_level_to_str(level), entry_str); __print_bad_page_map_pgtable(vma->vm_mm, addr); if (page) dump_page(page, "bad page map"); - pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n", - (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); + pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px", + (void *)addr, vma->vm_flags, vma->anon_vma, mapping); + if (!vma_is_cow_mapping(vma) || index == anon_index) { + pr_cont(" index:%lx\n", index); + } else { + pr_cont(" index:%lx (file) %lx (anon)\n", index, anon_index); + } + pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n", vma->vm_file, vma->vm_ops ? vma->vm_ops->fault : NULL, @@ -627,8 +678,13 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) } } -#define print_bad_pte(vma, addr, pte, page) \ - print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE) +static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, + pte_t pte, struct page *page) +{ + auto entry = pte_val(pte); + + print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE); +} /** * __vm_normal_page() - Get the "struct page" associated with a page table entry. @@ -636,8 +692,9 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) * @addr: The address where the page table entry is mapped. * @pfn: The PFN stored in the page table entry. * @special: Whether the page table entry is marked "special". - * @level: The page table level for error reporting purposes only. * @entry: The page table entry value for error reporting purposes only. + * @entry_size: The size of @entry. + * @level: The page table level for error reporting purposes only. * * "Special" mappings do not wish to be associated with a "struct page" (either * it doesn't exist, or it exists but they don't want to touch it). In this @@ -697,7 +754,7 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) */ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, bool special, - unsigned long long entry, enum pgtable_level level) + const void *entry, size_t entry_size, enum pgtable_level level) { if (pgtable_level_has_pxx_special(level)) { if (unlikely(special)) { @@ -710,7 +767,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return NULL; - print_bad_page_map(vma, addr, entry, NULL, level); + print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* @@ -730,7 +787,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, /* Only CoW'ed anon folios are "normal". */ if (pfn == index) return NULL; - if (!is_cow_mapping(vma->vm_flags)) + if (!vma_is_cow_mapping(vma)) return NULL; } } @@ -741,7 +798,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (unlikely(pfn > highest_memmap_pfn)) { /* Corrupted page table entry. */ - print_bad_page_map(vma, addr, entry, NULL, level); + print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* @@ -767,8 +824,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { + auto entry = pte_val(pte); + return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte), - pte_val(pte), PGTABLE_LEVEL_PTE); + &entry, sizeof(entry), PGTABLE_LEVEL_PTE); } /** @@ -809,8 +868,10 @@ struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd) { + auto entry = pmd_val(pmd); + return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd), - pmd_val(pmd), PGTABLE_LEVEL_PMD); + &entry, sizeof(entry), PGTABLE_LEVEL_PMD); } /** @@ -850,8 +911,10 @@ struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud) { + auto entry = pud_val(pud); + return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud), - pud_val(pud), PGTABLE_LEVEL_PUD); + &entry, sizeof(entry), PGTABLE_LEVEL_PUD); } #endif @@ -946,7 +1009,6 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long addr, int *rss) { - vm_flags_t vm_flags = dst_vma->vm_flags; pte_t orig_pte = ptep_get(src_pte); softleaf_t entry = softleaf_from_pte(orig_pte); pte_t pte = orig_pte; @@ -970,7 +1032,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, rss[mm_counter(folio)]++; if (!softleaf_is_migration_read(entry) && - is_cow_mapping(vm_flags)) { + vma_is_cow_mapping(dst_vma)) { /* * COW mappings require pages in both parent and child * to be set to read. A previously exclusive entry is @@ -1011,7 +1073,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, * save and restore device driver state). */ if (softleaf_is_device_private_write(entry) && - is_cow_mapping(vm_flags)) { + vma_is_cow_mapping(dst_vma)) { entry = make_readable_device_private_entry( swp_offset(entry)); pte = swp_entry_to_pte(entry); @@ -1026,7 +1088,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, * exclusive entries currently only support private writable * (ie. COW) mappings. */ - VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags)); + VM_BUG_ON(!vma_is_cow_mapping(src_vma)); if (try_restore_exclusive_pte(src_vma, addr, src_pte, orig_pte)) return -EBUSY; return -ENOENT; @@ -1125,7 +1187,7 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, } /* If it's a COW mapping, write protect it both processes. */ - if (is_cow_mapping(src_vma->vm_flags) && writable) { + if (vma_is_cow_mapping(src_vma) && writable) { wrprotect_ptes(src_mm, addr, src_pte, nr); pte = pte_wrprotect(pte); } @@ -1546,9 +1608,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) * We need to invalidate the secondary MMU mappings only when * there could be a permission downgrade on the ptes of the * parent mm. And a permission downgrade will only happen if - * is_cow_mapping() returns true. + * vma_is_cow_mapping() returns true. */ - is_cow = is_cow_mapping(src_vma->vm_flags); + is_cow = vma_is_cow_mapping(src_vma); if (is_cow) { mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, @@ -2381,7 +2443,7 @@ static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma) if (mm_forbids_zeropage(vma->vm_mm)) return false; /* zeropages in COW mappings are common and unproblematic. */ - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return true; /* Mappings that do not allow for writable PTEs are unproblematic. */ if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) @@ -2832,7 +2894,7 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == (VM_PFNMAP|VM_MIXEDMAP)); - BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); + BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); if (addr < vma->vm_start || addr >= vma->vm_end) @@ -3244,7 +3306,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long size) { const unsigned long end = addr + PAGE_ALIGN(size); - const bool is_cow = is_cow_mapping(vma->vm_flags); + const bool is_cow = vma_is_cow_mapping(vma); int err; err = get_remap_pgoff(is_cow, addr, end, vma->vm_start, vma->vm_end, @@ -6744,7 +6806,7 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's * just treat it like an ordinary read-fault otherwise. */ - if (!is_cow_mapping(vma->vm_flags)) + if (!vma_is_cow_mapping(vma)) *flags &= ~FAULT_FLAG_UNSHARE; } else if (*flags & FAULT_FLAG_WRITE) { /* Write faults on read-only mappings are impossible ... */ @@ -6752,7 +6814,7 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, return VM_FAULT_SIGSEGV; /* ... and FOLL_FORCE only applies to COW mappings. */ if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) && - !is_cow_mapping(vma->vm_flags))) + !vma_is_cow_mapping(vma))) return VM_FAULT_SIGSEGV; } #ifdef CONFIG_PER_VMA_LOCK diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 501e0b80d7da..3498a5651d50 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -844,7 +844,7 @@ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, return false; /* Also skip shared copy-on-write folios */ - if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio)) + if (vma_is_cow_mapping(vma) && folio_maybe_mapped_shared(folio)) return false; /* Folios are pinned and can't be migrated */ @@ -2060,24 +2060,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma) { struct mempolicy *pol; + pgoff_t ilx; + bool mof; - if (vma->vm_ops && vma->vm_ops->get_policy) { - bool ret = false; - pgoff_t ilx; /* ignored here */ - - pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx); - if (pol && (pol->flags & MPOL_F_MOF)) - ret = true; - mpol_cond_put(pol); - - return ret; - } - - pol = vma->vm_policy; + pol = __get_vma_policy(vma, vma->vm_start, &ilx); if (!pol) pol = get_task_policy(current); - - return pol->flags & MPOL_F_MOF; + mof = pol->flags & MPOL_F_MOF; + mpol_cond_put(pol); + return mof; } bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) diff --git a/mm/migrate.c b/mm/migrate.c index 8aaafcea7bc1..15b45832bcfa 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -356,23 +356,18 @@ static bool remove_migration_pte(struct folio *folio, while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; - pte_t old_pte; - pte_t pte; + unsigned long idx = 0; softleaf_t entry; struct page *new; - unsigned long idx = 0; - - /* pgoff is invalid for ksm pages, but they are never large */ - if (folio_test_large(folio) && !folio_test_hugetlb(folio)) - idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; - new = folio_page(folio, idx); + pte_t old_pte; + pte_t pte; #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || !folio_test_pmd_mappable(folio), folio); - remove_migration_pmd(&pvmw, new); + remove_migration_pmd(&pvmw, folio); continue; } #endif @@ -381,14 +376,18 @@ static bool remove_migration_pte(struct folio *folio, pvmw.pte); else old_pte = ptep_get(pvmw.pte); + + entry = softleaf_from_pte(old_pte); + if (folio_test_large(folio) && !folio_test_hugetlb(folio)) + idx = softleaf_to_pfn(entry) - pvmw.pfn; + if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; folio_get(folio); + new = folio_page(folio, idx); pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); - - entry = softleaf_from_pte(old_pte); if (!softleaf_is_migration_young(entry)) pte = pte_mkold(pte); if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 004dafa37038..009bfa8b212d 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -882,7 +882,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, if (flush) { pte_free(vma->vm_mm, pgtable); - flush_cache_page(vma, addr, addr + HPAGE_PMD_SIZE); + flush_cache_range(vma, addr, addr + HPAGE_PMD_SIZE); pmdp_invalidate(vma, addr, pmdp); } else { pgtable_trans_huge_deposit(vma->vm_mm, pmdp, pgtable); @@ -1193,6 +1193,13 @@ static void __migrate_device_pages(unsigned long *src_pfns, MIGRATE_PFN_COMPOUND); goto next; } + + /* + * reset nr so that only first after-split folio + * is processed below + */ + VM_WARN_ON_ONCE(folio_test_large(folio)); + nr = 1; } else if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) && (dst_pfns[i] & MIGRATE_PFN_COMPOUND) && !(src_pfns[i] & MIGRATE_PFN_COMPOUND)) { @@ -1232,6 +1239,12 @@ static void __migrate_device_pages(unsigned long *src_pfns, folio = page_folio(migrate_pfn_to_page(src_pfns[i+j])); newfolio = page_folio(migrate_pfn_to_page(dst_pfns[i+j])); + /* + * folio_free_swap() removed the folio from the swap + * cache. Refresh the saved mapping before migration. + */ + mapping = folio_mapping(folio); + r = folio_migrate_mapping(mapping, newfolio, folio, extra_cnt); if (r) src_pfns[i+j] &= ~MIGRATE_PFN_MIGRATE; diff --git a/mm/mm_init.c b/mm/mm_init.c index 4471768e262b..d859058c1d14 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1338,7 +1338,6 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) for (i = 0; i < NR_VMSCAN_THROTTLE; i++) init_waitqueue_head(&pgdat->reclaim_wait[i]); - pgdat_page_ext_init(pgdat); lruvec_init(&pgdat->__lruvec); } diff --git a/mm/mmzone.c b/mm/mmzone.c index 59dc3f2076a6..9cc9ef588580 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -79,6 +79,7 @@ void lruvec_init(struct lruvec *lruvec) memset(lruvec, 0, sizeof(struct lruvec)); spin_lock_init(&lruvec->lru_lock); + spin_lock_init(&lruvec->cost_lock); zswap_lruvec_state_init(lruvec); for_each_lru(lru) diff --git a/mm/mremap.c b/mm/mremap.c index b64aa1f6e07e..e8df5cdb0ac9 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -264,7 +264,7 @@ static int move_ptes(struct pagetable_move_control *pmc, for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE, new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) { - VM_WARN_ON_ONCE(!pte_none(*new_ptep)); + VM_WARN_ON_ONCE(!pte_none(ptep_get(new_ptep))); nr_ptes = 1; max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT; @@ -1265,7 +1265,9 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - const unsigned long new_pgoff = linear_page_index(vrm->vma, vrm->addr); + const pgoff_t new_pgoff = linear_page_index(vrm->vma, vrm->addr); + const pgoff_t new_anon_pgoff = + __linear_anon_page_index(vrm->vma, vrm->addr); struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; unsigned long moved_len; @@ -1273,7 +1275,7 @@ static int copy_vma_and_data(struct vma_remap_struct *vrm, PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, - &pmc.need_rmap_locks); + new_anon_pgoff, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); *new_vma_ptr = NULL; diff --git a/mm/nommu.c b/mm/nommu.c index 277f663e1c5b..498e01ee40b0 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1393,6 +1393,10 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, setup_vma_to_mm(vma, mm); setup_vma_to_mm(new, mm); vma_iter_store_new(vmi, new); + + /* vmi should point lower address */ + if (new_below) + vma_next(vmi); mm->map_count++; return 0; diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 6c9c7ba89b8a..eeab25d6ce36 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2762,6 +2762,11 @@ EXPORT_SYMBOL(folio_redirty_for_writepage); * in this folio. Truncation will block on the page table lock as it * unmaps pages before removing the folio from its mapping. * + * .. DANGER:: + * Do not use this on a folio obtained from a function like + * get_user_pages_fast() without holding appropriate locks; you might want to + * use set_page_dirty_lock() or folio_mark_dirty_lock() instead. + * * Return: True if the folio was newly dirtied, false if it was already dirty. */ bool folio_mark_dirty(struct folio *folio) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 083cbcb5bdde..12fac9084c48 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6853,8 +6853,8 @@ static int sysctl_min_slab_ratio_sysctl_handler(const struct ctl_table *table, i /* * lowmem_reserve_ratio_sysctl_handler - just a wrapper around - * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() - * whenever sysctl_lowmem_reserve_ratio changes. + * proc_dointvec_minmax() so that we can call + * setup_per_zone_lowmem_reserve() when the sysctl is written. * * The reserve ratio obviously has absolutely no relation with the * minimum watermarks. The lowmem reserve ratio can only make sense @@ -6863,16 +6863,27 @@ static int sysctl_min_slab_ratio_sysctl_handler(const struct ctl_table *table, i static int lowmem_reserve_ratio_sysctl_handler(const struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { - int i; + struct ctl_table tmp = *table; + int ratio[ARRAY_SIZE(sysctl_lowmem_reserve_ratio)]; + int rc; - proc_dointvec_minmax(table, write, buffer, length, ppos); + if (!write) + return proc_dointvec_minmax(table, write, buffer, length, ppos); - for (i = 0; i < MAX_NR_ZONES; i++) { - if (sysctl_lowmem_reserve_ratio[i] < 1) - sysctl_lowmem_reserve_ratio[i] = 0; - } + /* + * proc_dointvec_max() works incrementally. Use a buffer and only set + * the values if all of them parse cleanly. + */ + memcpy(ratio, sysctl_lowmem_reserve_ratio, sizeof(ratio)); + tmp.data = ratio; + + rc = proc_dointvec_minmax(&tmp, write, buffer, length, ppos); + if (rc) + return rc; + memcpy(sysctl_lowmem_reserve_ratio, ratio, sizeof(ratio)); setup_per_zone_lowmem_reserve(); + return 0; } @@ -6971,6 +6982,7 @@ static const struct ctl_table page_alloc_sysctl_table[] = { .maxlen = sizeof(sysctl_lowmem_reserve_ratio), .mode = 0644, .proc_handler = lowmem_reserve_ratio_sysctl_handler, + .extra1 = SYSCTL_ZERO, }, #ifdef CONFIG_NUMA { diff --git a/mm/page_ext.c b/mm/page_ext.c index e2e92bd27ebd..b679a8c1f7d7 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -164,11 +164,6 @@ void __init page_ext_init_flatmem_late(void) invoke_init_callbacks(); } -void __meminit pgdat_page_ext_init(struct pglist_data *pgdat) -{ - pgdat->node_page_ext = NULL; -} - static struct page_ext *lookup_page_ext(const struct page *page) { unsigned long pfn = page_to_pfn(page); @@ -494,10 +489,6 @@ oom: panic("Out of memory"); } -void __meminit pgdat_page_ext_init(struct pglist_data *pgdat) -{ -} - #endif /** diff --git a/mm/page_io.c b/mm/page_io.c index b23f494fcc83..88962571cb93 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,57 +25,10 @@ #include <linux/sched/task.h> #include <linux/delayacct.h> #include <linux/zswap.h> +#include <linux/swap_ops.h> #include "swap.h" #include "swap_table.h" -static void __end_swap_bio_write(struct bio *bio) -{ - struct folio *folio = bio_first_folio_all(bio); - - if (bio->bi_status) { - /* - * We failed to write the page out to swap-space. - * Re-dirty the page in order to avoid it being reclaimed. - * Also print a dire warning that things will go BAD (tm) - * very quickly. - * - * Also clear PG_reclaim to avoid folio_rotate_reclaimable() - */ - folio_mark_dirty(folio); - pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); - folio_clear_reclaim(folio); - } - folio_end_writeback(folio); -} - -static void end_swap_bio_write(struct bio *bio) -{ - __end_swap_bio_write(bio); - bio_put(bio); -} - -static void __end_swap_bio_read(struct bio *bio) -{ - struct folio *folio = bio_first_folio_all(bio); - - if (bio->bi_status) { - pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); - } else { - folio_mark_uptodate(folio); - } - folio_unlock(folio); -} - -static void end_swap_bio_read(struct bio *bio) -{ - __end_swap_bio_read(bio); - bio_put(bio); -} - int generic_swapfile_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *span) @@ -248,7 +201,7 @@ static void swap_zeromap_folio_clear(struct folio *folio) * We may have stale swap cache pages in memory: notice * them here and get rid of the unnecessary final write. */ -int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) +int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { int ret = 0; @@ -295,39 +248,44 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) } rcu_read_unlock(); - __swap_writepage(folio, swap_plug); + __swap_writepage(ctx, folio); return 0; out_unlock: folio_unlock(folio); return ret; } -static inline void count_swpout_vm_event(struct folio *folio) +#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) +static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (unlikely(folio_test_pmd_mappable(folio))) { - count_memcg_folio_events(folio, THP_SWPOUT, 1); - count_vm_event(THP_SWPOUT); + return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys); +} + +static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +{ + bool can_merge = true; + + if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio)) + return false; + if (folio_memcg_charged(folio)) { + rcu_read_lock(); + if (folio_memcg_blkg_css(folio) != + folio_memcg_blkg_css(prev_folio)) + can_merge = false; + rcu_read_unlock(); } -#endif - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); - count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio)); - count_vm_events(PSWPOUT, folio_nr_pages(folio)); + return can_merge; } -#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) { struct cgroup_subsys_state *css; - struct mem_cgroup *memcg; if (!folio_memcg_charged(folio)) return; - rcu_read_lock(); - memcg = folio_memcg(folio); - css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); - if (!css || !css_tryget(css)) + css = folio_memcg_blkg_css(folio); + if (css && !css_tryget(css)) css = NULL; rcu_read_unlock(); @@ -336,15 +294,13 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) css_put(css); } #else +static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +{ + return true; +} #define bio_associate_blkg_from_page(bio, folio) do { } while (0) #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ -struct swap_iocb { - struct kiocb iocb; - struct bio_vec bvecs[SWAP_CLUSTER_MAX]; - int nr_bvecs; - int len; -}; static mempool_t *sio_pool; int sio_pool_init(void) @@ -360,168 +316,74 @@ int sio_pool_init(void) return 0; } -static void sio_write_complete(struct kiocb *iocb, long ret) +static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio, + int rw) { - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); - struct page *page = sio->bvecs[0].bv_page; - int p; - - if (ret != sio->len) { - /* - * In the case of swap-over-nfs, this can be a - * temporary failure if the system has limited - * memory for allocating transmit buffers. - * Mark the page dirty and avoid - * folio_rotate_reclaimable but rate-limit the - * messages. - */ - pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", - ret, swap_dev_pos(page_swap_entry(page))); - for (p = 0; p < sio->nr_bvecs; p++) { - page = sio->bvecs[p].bv_page; - set_page_dirty(page); - ClearPageReclaim(page); - } - } - - for (p = 0; p < sio->nr_bvecs; p++) - end_page_writeback(sio->bvecs[p].bv_page); + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1]; + struct folio *prev_folio = bvec_folio(last_bv); + size_t prev_folio_size = folio_size(prev_folio); - mempool_free(sio, sio_pool); + if (ctx->sis != sis) + return false; + return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw); } -static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) +static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw) { - struct swap_iocb *sio = swap_plug ? *swap_plug : NULL; struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct file *swap_file = sis->swap_file; - loff_t pos = swap_dev_pos(folio->swap); - - count_swpout_vm_event(folio); - folio_start_writeback(folio); - folio_unlock(folio); - if (sio) { - if (sio->iocb.ki_filp != swap_file || - sio->iocb.ki_pos + sio->len != pos) { - swap_write_unplug(sio); - sio = NULL; - } + struct swap_iocb *sio = ctx->sio; + + if (sio && !swap_can_merge(ctx, folio, rw)) { + if (rw == WRITE) + swap_write_submit(ctx); + else + swap_read_submit(ctx); + sio = ctx->sio; } + if (!sio) { - sio = mempool_alloc(sio_pool, GFP_NOIO); - init_sync_kiocb(&sio->iocb, swap_file); - sio->iocb.ki_complete = sio_write_complete; - sio->iocb.ki_pos = pos; + ctx->sis = sis; + ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO); sio->nr_bvecs = 0; sio->len = 0; } bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0); sio->len += folio_size(folio); - sio->nr_bvecs += 1; - if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !swap_plug) { - swap_write_unplug(sio); - sio = NULL; - } - if (swap_plug) - *swap_plug = sio; -} - -static void swap_writepage_bdev_sync(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio_vec bv; - struct bio bio; - - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_WRITE | REQ_SWAP); - bio.bi_iter.bi_sector = swap_folio_sector(folio); - bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); - - bio_associate_blkg_from_page(&bio, folio); - count_swpout_vm_event(folio); - - folio_start_writeback(folio); - folio_unlock(folio); - submit_bio_wait(&bio); - __end_swap_bio_write(&bio); -} - -static void swap_writepage_bdev_async(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio *bio; - - bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE | REQ_SWAP, GFP_NOIO); - bio->bi_iter.bi_sector = swap_folio_sector(folio); - bio->bi_end_io = end_swap_bio_write; - bio_add_folio_nofail(bio, folio, folio_size(folio), 0); - - bio_associate_blkg_from_page(bio, folio); - count_swpout_vm_event(folio); - folio_start_writeback(folio); - folio_unlock(folio); - submit_bio(bio); -} - -void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) -{ - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - - VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); - /* - * ->flags can be updated non-atomically, - * but that will never affect SWP_FS_OPS, so the data_race - * is safe. - */ - if (data_race(sis->flags & SWP_FS_OPS)) - swap_writepage_fs(folio, swap_plug); /* - * ->flags can be updated non-atomically, - * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race - * is safe. + * Write out the iocb if we filled it, or if the device is synchronous. + * + * The latter is to work around expectations in the classic LRU code + * which make synchronous clearing of the folio writeback flag in the + * reclaim path beneficial. */ - else if (data_race(sis->flags & SWP_SYNCHRONOUS_IO)) - swap_writepage_bdev_sync(folio, sis); - else - swap_writepage_bdev_async(folio, sis); -} - -void swap_write_unplug(struct swap_iocb *sio) -{ - struct iov_iter from; - struct address_space *mapping = sio->iocb.ki_filp->f_mapping; - int ret; - - iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw(&sio->iocb, &from); - if (ret != -EIOCBQUEUED) - sio_write_complete(&sio->iocb, ret); + if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || + (rw == WRITE && (sis->flags & SWP_SYNCHRONOUS_IO))) { + if (rw == WRITE) + swap_write_submit(ctx); + else + swap_read_submit(ctx); + } } -static void sio_read_complete(struct kiocb *iocb, long ret) +void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) { - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); - int p; - - if (ret == sio->len) { - for (p = 0; p < sio->nr_bvecs; p++) { - struct folio *folio = bvec_folio(&sio->bvecs[p]); - - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - folio_mark_uptodate(folio); - folio_unlock(folio); - } - count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); - } else { - for (p = 0; p < sio->nr_bvecs; p++) { - struct folio *folio = bvec_folio(&sio->bvecs[p]); + VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); - folio_unlock(folio); - } - pr_alert_ratelimited("Read-error on swap-device\n"); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (unlikely(folio_test_pmd_mappable(folio))) { + count_memcg_folio_events(folio, THP_SWPOUT, 1); + count_vm_event(THP_SWPOUT); } - mempool_free(sio, sio_pool); +#endif + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); + count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio)); + count_vm_events(PSWPOUT, folio_nr_pages(folio)); + + folio_start_writeback(folio); + folio_unlock(folio); + swap_add_folio(ctx, folio, WRITE); } /* @@ -587,78 +449,7 @@ static bool swap_read_folio_zeromap(struct folio *folio) return true; } -static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug) -{ - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_iocb *sio = NULL; - loff_t pos = swap_dev_pos(folio->swap); - - if (plug) - sio = *plug; - if (sio) { - if (sio->iocb.ki_filp != sis->swap_file || - sio->iocb.ki_pos + sio->len != pos) { - swap_read_unplug(sio); - sio = NULL; - } - } - if (!sio) { - sio = mempool_alloc(sio_pool, GFP_KERNEL); - init_sync_kiocb(&sio->iocb, sis->swap_file); - sio->iocb.ki_pos = pos; - sio->iocb.ki_complete = sio_read_complete; - sio->nr_bvecs = 0; - sio->len = 0; - } - bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0); - sio->len += folio_size(folio); - sio->nr_bvecs += 1; - if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !plug) { - swap_read_unplug(sio); - sio = NULL; - } - if (plug) - *plug = sio; -} - -static void swap_read_folio_bdev_sync(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio_vec bv; - struct bio bio; - - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ); - bio.bi_iter.bi_sector = swap_folio_sector(folio); - bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); - /* - * Keep this task valid during swap readpage because the oom killer may - * attempt to access it in the page fault retry time check. - */ - get_task_struct(current); - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); - __end_swap_bio_read(&bio); - put_task_struct(current); -} - -static void swap_read_folio_bdev_async(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio *bio; - - bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL); - bio->bi_iter.bi_sector = swap_folio_sector(folio); - bio->bi_end_io = end_swap_bio_read; - bio_add_folio_nofail(bio, folio, folio_size(folio), 0); - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); -} - -void swap_read_folio(struct folio *folio, struct swap_iocb **plug) +void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) { struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO; @@ -691,14 +482,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug) /* We have to read from slower devices. Increase zswap protection. */ zswap_folio_swapin(folio); - - if (data_race(sis->flags & SWP_FS_OPS)) { - swap_read_folio_fs(folio, plug); - } else if (synchronous) { - swap_read_folio_bdev_sync(folio, sis); - } else { - swap_read_folio_bdev_async(folio, sis); - } + swap_add_folio(ctx, folio, READ); finish: if (workingset) { @@ -708,14 +492,211 @@ finish: delayacct_swapin_end(); } -void __swap_read_unplug(struct swap_iocb *sio) +static void swap_write_end(struct swap_iocb *sio, bool failed) { - struct iov_iter from; - struct address_space *mapping = sio->iocb.ki_filp->f_mapping; - int ret; + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + struct page *page = sio->bvecs[p].bv_page; + + if (failed) { + set_page_dirty(page); + ClearPageReclaim(page); + } + end_page_writeback(page); + } + mempool_free(sio, sio_pool); +} + +static void swap_fs_write_complete(struct kiocb *iocb, long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); + bool failed = ret != sio->len; + + if (failed) { + struct page *page = sio->bvecs[0].bv_page; + + /* + * In the case of swap-over-nfs, this can be a temporary failure + * if the system has limited memory for allocating transmit + * buffers. Mark the page dirty and avoid + * folio_rotate_reclaimable but rate-limit the messages. + */ + pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", + ret, swap_dev_pos(page_swap_entry(page))); + } + + swap_write_end(sio, failed); +} - iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw(&sio->iocb, &from); - if (ret != -EIOCBQUEUED) - sio_read_complete(&sio->iocb, ret); +static void end_swap_bio_write(struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); + bool failed = !!bio->bi_status; + + if (failed) + pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); + bio_uninit(bio); + swap_write_end(sio, failed); +} + +static void swap_read_end(struct swap_iocb *sio, bool failed) +{ + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + struct folio *folio = bvec_folio(&sio->bvecs[p]); + + if (!failed) { + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); + count_memcg_folio_events(folio, PSWPIN, + folio_nr_pages(folio)); + folio_mark_uptodate(folio); + } + folio_unlock(folio); + } + + if (!failed) + count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); + + mempool_free(sio, sio_pool); +} + +static void swap_fs_read_complete(struct kiocb *iocb, long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); + bool failed = ret != sio->len; + + if (failed) + pr_alert_ratelimited("Read-error on swap-device\n"); + swap_read_end(sio, failed); +} + +static void swap_bio_read_end_io(struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); + bool failed = !!bio->bi_status; + + if (failed) + pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); + bio_uninit(bio); + swap_read_end(sio, failed); +} + +static void swap_bdev_submit_write(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct bio *bio = &sio->bio; + + bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs), + REQ_OP_WRITE | REQ_SWAP); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); + bio_associate_blkg_from_page(bio, bio_first_folio_all(bio)); + + if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { + submit_bio_wait(bio); + end_swap_bio_write(bio); + } else { + bio->bi_end_io = end_swap_bio_write; + submit_bio(bio); + } +} + +static void swap_bdev_submit_read(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct bio *bio = &sio->bio; + + bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs), + REQ_OP_READ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); + + if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { + /* + * Keep this task valid during swap readpage because the oom + * killer may attempt to access it in the page fault retry + * time check. + */ + get_task_struct(current); + submit_bio_wait(bio); + swap_bio_read_end_io(bio); + put_task_struct(current); + } else { + bio->bi_end_io = swap_bio_read_end_io; + submit_bio(bio); + } +} + +static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw) +{ + if (swap_folio_sector(folio) != + swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT)) + return false; + if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio)) + return false; + return true; +} + +const struct swap_ops swap_bdev_ops = { + .submit_write = swap_bdev_submit_write, + .submit_read = swap_bdev_submit_read, + .can_merge = swap_bdev_can_merge, +}; + +void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter) +{ + struct swap_iocb *sio = ctx->sio; + + init_sync_kiocb(&sio->iocb, ctx->sis->swap_file); + sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap); + if (rw == WRITE) + sio->iocb.ki_complete = swap_fs_write_complete; + else + sio->iocb.ki_complete = swap_fs_read_complete; + + iov_iter_bvec(iter, rw == WRITE ? ITER_SOURCE : ITER_DEST, + sio->bvecs, sio->nr_bvecs, sio->len); +} +EXPORT_SYMBOL_GPL(swap_fs_prepare_rw); + +bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw) +{ + return swap_dev_pos(folio->swap) == + swap_dev_pos(prev_folio->swap) + prev_folio_size; +} +EXPORT_SYMBOL_GPL(swap_fs_can_merge); + +int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops) +{ + sis->ops = ops; + return add_swap_extent(sis, 0, sis->max, 0); +} +EXPORT_SYMBOL_GPL(swap_fs_activate); + +void swap_write_submit(struct swap_io_ctx *ctx) +{ + if (!ctx->sio) + return; + count_vm_events(NRSWPOUT, 1); + ctx->sis->ops->submit_write(ctx); + ctx->sio = NULL; + ctx->sis = NULL; +} + +void swap_read_submit(struct swap_io_ctx *ctx) +{ + if (!ctx->sio) + return; + count_vm_events(NRSWPIN, 1); + ctx->sis->ops->submit_read(ctx); + ctx->sio = NULL; + ctx->sis = NULL; } diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 1cce8729696e..de587be17801 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -48,7 +48,11 @@ MODULE_PARM_DESC(page_reporting_order, "Set page reporting order"); */ EXPORT_SYMBOL_GPL(page_reporting_order); -#define PAGE_REPORTING_DELAY (2 * HZ) +static unsigned int page_reporting_delay_ms = 2 * MSEC_PER_SEC; +module_param(page_reporting_delay_ms, uint, 0644); +MODULE_PARM_DESC(page_reporting_delay_ms, + "Set page reporting delay in milliseconds"); + static struct page_reporting_dev_info __rcu *pr_dev_info __read_mostly; enum { @@ -57,6 +61,13 @@ enum { PAGE_REPORTING_ACTIVE }; +/* schedule work for page reporting */ +static void page_reporting_schedule_work(struct page_reporting_dev_info *prdev) +{ + queue_delayed_work(system_freezable_wq, &prdev->work, + msecs_to_jiffies(page_reporting_delay_ms)); +} + /* request page reporting */ static void __page_reporting_request(struct page_reporting_dev_info *prdev) @@ -77,12 +88,10 @@ __page_reporting_request(struct page_reporting_dev_info *prdev) return; /* - * Delay the start of work to allow a sizable queue to build. For - * now we are limiting this to running no more than once every - * couple of seconds. + * Delay the start of work to allow a sizable queue to build. + * We limit this based on page_reporting_delay_ms. */ - queue_delayed_work(system_freezable_wq, &prdev->work, - PAGE_REPORTING_DELAY); + page_reporting_schedule_work(prdev); } /* notify prdev of free page reporting request */ @@ -337,13 +346,12 @@ static void page_reporting_process(struct work_struct *work) err_out: /* * If the state has reverted back to requested then there may be - * additional pages to be processed. We will defer for 2s to allow - * more pages to accumulate. + * additional pages to be processed. We will defer by + * page_reporting_delay_ms to allow more pages to accumulate. */ state = atomic_cmpxchg(&prdev->state, state, PAGE_REPORTING_IDLE); if (state == PAGE_REPORTING_REQUESTED) - queue_delayed_work(system_freezable_wq, &prdev->work, - PAGE_REPORTING_DELAY); + page_reporting_schedule_work(prdev); } static DEFINE_MUTEX(page_reporting_mutex); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index d7670ba4147b..4e964545e5e8 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -356,6 +356,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + const pgoff_t pgoff = page_pgoff(folio, page); struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -363,7 +364,10 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + if (folio_test_anon(folio)) + pvmw.address = vma_anon_address(vma, pgoff, 1); + else + pvmw.address = vma_filebacked_address(vma, pgoff, 1); if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) diff --git a/mm/pgalloc-track.h b/mm/pgalloc-track.h index e9e879de8649..1a6de1358a21 100644 --- a/mm/pgalloc-track.h +++ b/mm/pgalloc-track.h @@ -2,6 +2,9 @@ #ifndef _LINUX_PGALLOC_TRACK_H #define _LINUX_PGALLOC_TRACK_H +#include <linux/mm.h> +#include <linux/pgtable.h> + #if defined(CONFIG_MMU) static inline p4d_t *p4d_alloc_track(struct mm_struct *mm, pgd_t *pgd, unsigned long address, diff --git a/mm/rmap.c b/mm/rmap.c index 1f72d279ba68..d1819fd69938 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -350,7 +350,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, * Now link the anon_vma's back to the newly inserted AVCs. * Note that all anon_vma's share the same root. */ - anon_vma_lock_write(src->anon_vma); + anon_vma_lock_write(active_anon_vma); list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; @@ -865,14 +865,15 @@ unsigned long page_address_in_vma(const struct folio *folio, if (!vma->anon_vma || !anon_vma || vma->anon_vma->root != anon_vma->root) return -EFAULT; + /* KSM folios don't reach here because of the !anon_vma check */ + return vma_anon_address(vma, page_pgoff(folio, page), 1); } else if (!vma->vm_file) { return -EFAULT; } else if (vma->vm_file->f_mapping != folio->mapping) { return -EFAULT; } - /* KSM folios don't reach here because of the !anon_vma check */ - return vma_address(vma, page_pgoff(folio, page), 1); + return vma_filebacked_address(vma, page_pgoff(folio, page), 1); } /* @@ -907,7 +908,7 @@ out: struct folio_referenced_arg { int mapcount; int referenced; - vm_flags_t vm_flags; + vma_flags_t vma_flags; struct mem_cgroup *memcg; }; @@ -926,7 +927,7 @@ static bool folio_referenced_one(struct folio *folio, address = pvmw.address; nr = 1; - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { ptes++; pra->mapcount--; @@ -947,7 +948,7 @@ static bool folio_referenced_one(struct folio *folio, /* Restore the mlock which got missed */ mlock_vma_folio(folio, vma); page_vma_mapped_walk_done(&pvmw); - pra->vm_flags |= VM_LOCKED; + vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT); return false; /* To break the loop */ } @@ -1015,8 +1016,11 @@ static bool folio_referenced_one(struct folio *folio, referenced++; if (referenced) { + vma_flags_t vma_flags = vma->flags; + pra->referenced++; - pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; + vma_flags_clear(&vma_flags, VMA_LOCKED_BIT); + vma_flags_set_mask(&pra->vma_flags, vma_flags); } if (!pra->mapcount) @@ -1054,7 +1058,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) * @folio: The folio to test. * @is_locked: Caller holds lock on the folio. * @memcg: target memory cgroup - * @vm_flags: A combination of all the vma->vm_flags which referenced the folio. + * @vma_flags: A combination of all the vma->flags which referenced the folio. * * Quick test_and_clear_referenced for all mappings of a folio, * @@ -1062,7 +1066,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) * the function bailed out due to rmap lock contention. */ int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { bool we_locked = false; struct folio_referenced_arg pra = { @@ -1078,7 +1082,7 @@ int folio_referenced(struct folio *folio, int is_locked, }; VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio); - *vm_flags = 0; + vma_flags_clear_all(vma_flags); if (!pra.mapcount) return 0; @@ -1092,7 +1096,7 @@ int folio_referenced(struct folio *folio, int is_locked, } rmap_walk(folio, &rwc); - *vm_flags = pra.vm_flags; + vma_flags_set_mask(vma_flags, pra.vma_flags); if (we_locked) folio_unlock(folio); @@ -1239,6 +1243,7 @@ static bool mapping_wrprotect_range_one(struct folio *folio, .vma = vma, .address = address, .flags = PVMW_SYNC, + .pgoff_is_anon = false, }; state->cleaned += page_vma_mkclean_one(&pvmw); @@ -1316,12 +1321,13 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, .pgoff = pgoff, .vma = vma, .flags = PVMW_SYNC, + .pgoff_is_anon = false, }; if (invalid_mkclean_vma(vma, NULL)) return 0; - pvmw.address = vma_address(vma, pgoff, nr_pages); + pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages); VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); return page_vma_mkclean_one(&pvmw); @@ -1482,7 +1488,7 @@ static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, */ anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON; WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma); - folio->index = linear_page_index(vma, address); + folio->index = linear_anon_page_index(vma, address); } /** @@ -1509,8 +1515,8 @@ static void __page_check_anon_rmap(const struct folio *folio, */ VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, folio); - VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address), - page); + VM_BUG_ON_PAGE(page_pgoff(folio, page) != + linear_anon_page_index(vma, address), page); } static __always_inline void __folio_add_anon_rmap(struct folio *folio, @@ -3035,10 +3041,10 @@ static void rmap_walk_anon(struct folio *folio, pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; - unsigned long address = vma_address(vma, pgoff_start, + const unsigned long address = vma_anon_address(vma, pgoff_start, folio_nr_pages(folio)); - VM_BUG_ON_VMA(address == -EFAULT, vma); + VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma); cond_resched(); if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) @@ -3098,7 +3104,8 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, } lookup: mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { - unsigned long address = vma_address(vma, pgoff_start, nr_pages); + unsigned long address = vma_filebacked_address(vma, pgoff_start, + nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); diff --git a/mm/shmem.c b/mm/shmem.c index 89a1495e55f7..897fa2b61346 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -41,6 +41,7 @@ #include <linux/swapfile.h> #include <linux/iversion.h> #include <linux/unicode.h> +#include <linux/swap_ops.h> #include "swap.h" static struct vfsmount *shm_mnt __ro_after_init; @@ -1597,13 +1598,13 @@ start_over: /** * shmem_writeout - Write the folio to swap + * @ctx: swap I/O context * @folio: The folio to write - * @plug: swap plug * @folio_list: list to put back folios on split * * Move the folio from the page cache to the swap cache. */ -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, +int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, struct list_head *folio_list) { struct address_space *mapping = folio->mapping; @@ -1715,7 +1716,7 @@ try_split: shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap)); BUG_ON(folio_mapped(folio)); - error = swap_writeout(folio, plug); + error = swap_writeout(ctx, folio); if (error != AOP_WRITEPAGE_ACTIVATE) { /* folio has been unlocked */ return error; @@ -1751,7 +1752,17 @@ redirty: folio_mark_dirty(folio); return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ } -EXPORT_SYMBOL_GPL(shmem_writeout); + +int shmem_write_folio(struct folio *folio) +{ + struct swap_io_ctx ctx = {}; + int err; + + err = shmem_writeout(&ctx, folio, NULL); + swap_write_submit(&ctx); + return err; +} +EXPORT_SYMBOL_GPL(shmem_write_folio); #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) @@ -3617,6 +3628,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_falloc shmem_falloc; pgoff_t start, index, end, undo_fallocend; + loff_t aligned_end; int error; if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) @@ -3673,8 +3685,15 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, goto out; } + /* Check for wraparound */ + if (check_add_overflow(offset + len, (loff_t)PAGE_SIZE - 1, + &aligned_end)) { + error = -EFBIG; + goto out; + } + start = offset >> PAGE_SHIFT; - end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; + end = aligned_end >> PAGE_SHIFT; /* Try to avoid a swapstorm if len is impossible to satisfy */ if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { error = -ENOSPC; diff --git a/mm/show_mem.c b/mm/show_mem.c index d1288b4c2b64..b938cbcd774a 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -309,8 +309,8 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, " min:%lukB" " low:%lukB" " high:%lukB" - " reserved_highatomic:%luKB" - " free_highatomic:%luKB" + " reserved_highatomic:%lukB" + " free_highatomic:%lukB" " active_anon:%lukB" " inactive_anon:%lukB" " active_file:%lukB" @@ -323,7 +323,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, " mlocked:%lukB" " bounce:%lukB" " free_pcp:%lukB" - " local_pcp:%ukB" + " local_pcp:%lukB" " free_cma:%lukB" "\n", zone->name, @@ -350,7 +350,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, K(zone_page_state(zone, NR_MLOCK)), 0UL, K(free_pcp), - K(this_cpu_read(zone->per_cpu_pageset->count)), + K((unsigned long)this_cpu_read(zone->per_cpu_pageset->count)), K(zone_page_state(zone, NR_FREE_CMA_PAGES))); printk("lowmem_reserve[]:"); for (i = 0; i < MAX_NR_ZONES; i++) @@ -400,7 +400,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, hugetlb_show_meminfo_node(nid); } - printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); + printk("%lu total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); show_swap_cache_info(); } @@ -430,7 +430,7 @@ void __show_mem(unsigned int filter, const nodemask_t *nodemask, printk("%lu pages cma reserved\n", totalcma_pages); #endif #ifdef CONFIG_MEMORY_FAILURE - printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages)); + printk("%ld pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages)); #endif #ifdef CONFIG_MEM_ALLOC_PROFILING static DEFINE_SPINLOCK(mem_alloc_profiling_spinlock); diff --git a/mm/shuffle.h b/mm/shuffle.h index 61bbcddeeee6..11bec7521ab8 100644 --- a/mm/shuffle.h +++ b/mm/shuffle.h @@ -2,7 +2,9 @@ // Copyright(c) 2018 Intel Corporation. All rights reserved. #ifndef _MM_SHUFFLE_H #define _MM_SHUFFLE_H + #include <linux/jump_label.h> +#include <linux/mmzone.h> #define SHUFFLE_ORDER MAX_PAGE_ORDER diff --git a/mm/sparse.c b/mm/sparse.c index 704a9dec2b9a..7c15406e77f5 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -104,7 +104,7 @@ int __meminit sparse_index_init(unsigned long section_nr, int nid) return 0; } #else /* !SPARSEMEM_EXTREME */ -int sparse_index_init(unsigned long section_nr, int nid) +int __meminit sparse_index_init(unsigned long section_nr, int nid) { return 0; } @@ -127,7 +127,7 @@ static inline int sparse_early_nid(struct mem_section *section) } /* Validate the physical addressing limitations of the model */ -static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, +static void __init mminit_validate_memmodel_limits(unsigned long *start_pfn, unsigned long *end_pfn) { unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT; @@ -213,16 +213,6 @@ static void __init memblocks_present(void) memory_present(nid, start, end); } -static unsigned long usemap_size(void) -{ - return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long); -} - -size_t mem_section_usage_size(void) -{ - return sizeof(struct mem_section_usage) + usemap_size(); -} - #ifdef CONFIG_SPARSEMEM_VMEMMAP unsigned long __init section_map_size(void) { @@ -249,8 +239,8 @@ void __weak __meminit vmemmap_populate_print_last(void) { } -static void *sparse_usagebuf __meminitdata; -static void *sparse_usagebuf_end __meminitdata; +static void *sparse_usagebuf __initdata; +static void *sparse_usagebuf_end __initdata; /* * Helper function that is used for generic section initialization, and diff --git a/mm/sparse.h b/mm/sparse.h index 95aa031213f2..3b744667a7e6 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -47,6 +47,12 @@ static inline void __section_mark_present(struct mem_section *ms, ms->section_mem_map |= SECTION_MARKED_PRESENT; } + +static inline size_t mem_section_usage_size(void) +{ + return struct_size_t(struct mem_section_usage, pageblock_flags, + BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS)); +} #else static inline void sparse_init(void) {} #endif /* CONFIG_SPARSEMEM */ diff --git a/mm/swap.h b/mm/swap.h index b51ad3071a73..90a551a88df6 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -4,9 +4,13 @@ #include <linux/atomic.h> /* for atomic_long_t */ #include <linux/mm.h> /* for PAGE_SHIFT */ +#include <linux/memcontrol.h> /* for mem_cgroup_swappiness() */ +#include <linux/swap.h> /* for MAX_SWAPFILES_SHIFT, struct swap_info_struct */ + struct mempolicy; struct swap_iocb; struct swap_memcg_table; +struct swap_io_ctx; #if defined(MAX_POSSIBLE_PHYSMEM_BITS) #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) @@ -76,6 +80,18 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; +extern int vm_swappiness; + +static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) +{ +#ifdef CONFIG_MEMCG_V1 + if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && + !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg)) + return READ_ONCE(memcg->swappiness); +#endif + return READ_ONCE(vm_swappiness); +} + #ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ @@ -238,17 +254,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); -struct swap_iocb; -void swap_read_folio(struct folio *folio, struct swap_iocb **plug); -void __swap_read_unplug(struct swap_iocb *plug); -static inline void swap_read_unplug(struct swap_iocb *plug) -{ - if (unlikely(plug)) - __swap_read_unplug(plug); -} -void swap_write_unplug(struct swap_iocb *sio); -int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); -void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug); +void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio); +void swap_read_submit(struct swap_io_ctx *ctx); +void swap_write_submit(struct swap_io_ctx *ctx); +int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); +void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio); /* linux/mm/swap_state.c */ extern struct address_space swap_space __read_mostly; @@ -315,9 +325,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, void show_swap_cache_info(void); void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); -struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, - struct vm_area_struct *vma, unsigned long addr, - struct swap_iocb **plug); +struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, + gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr); struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, struct mempolicy *mpol, pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, @@ -327,18 +336,7 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders, void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, unsigned long addr); -static inline unsigned int folio_swap_flags(struct folio *folio) -{ - return __swap_entry_to_info(folio->swap)->flags; -} - #else /* CONFIG_SWAP */ -struct swap_iocb; -static inline struct swap_cluster_info *swap_cluster_lock( - struct swap_info_struct *si, pgoff_t offset, bool irq) -{ - return NULL; -} static inline struct swap_cluster_info *swap_cluster_get_and_lock( struct folio *folio) @@ -379,11 +377,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } -static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) +static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) { } -static inline void swap_write_unplug(struct swap_iocb *sio) +static inline void swap_write_submit(struct swap_io_ctx *ctx) { } @@ -425,8 +423,7 @@ static inline void swap_update_readahead(struct folio *folio, { } -static inline int swap_writeout(struct folio *folio, - struct swap_iocb **swap_plug) +static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { return 0; } @@ -464,11 +461,11 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new) { } +#endif /* CONFIG_SWAP */ -static inline unsigned int folio_swap_flags(struct folio *folio) -{ - return 0; -} +extern const struct swap_ops swap_bdev_ops; + +int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, + struct list_head *folio_list); -#endif /* CONFIG_SWAP */ #endif /* _MM_SWAP_H */ diff --git a/mm/swap_state.c b/mm/swap_state.c index 2606504e5573..f3961fdd857d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -23,6 +23,7 @@ #include <linux/huge_mm.h> #include <linux/shmem_fs.h> #include <linux/sysctl.h> +#include <linux/swap_ops.h> #include "internal.h" #include "swap_table.h" #include "swap.h" @@ -638,9 +639,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, } } -static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, - struct mempolicy *mpol, pgoff_t ilx, - struct swap_iocb **plug, bool readahead) +static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx, + swp_entry_t entry, gfp_t gfp, struct mempolicy *mpol, + pgoff_t ilx, bool readahead) { struct folio *folio; @@ -654,7 +655,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, if (IS_ERR_OR_NULL(folio)) return NULL; - swap_read_folio(folio, plug); + swap_read_folio(ctx, folio); if (readahead) { folio_set_readahead(folio); count_vm_event(SWAP_RA); @@ -682,6 +683,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) { + struct swap_io_ctx ctx = {}; struct folio *folio; do { @@ -694,7 +696,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, if (IS_ERR(folio)) return folio; - swap_read_folio(folio, NULL); + swap_read_folio(&ctx, folio); + swap_read_submit(&ctx); return folio; } @@ -704,9 +707,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, * A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */ -struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, - struct vm_area_struct *vma, unsigned long addr, - struct swap_iocb **plug) +struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, + gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr) { struct swap_info_struct *si; struct mempolicy *mpol; @@ -718,13 +720,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, return NULL; mpol = get_vma_policy(vma, addr, 0, &ilx); - folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false); + folio = swap_cache_read_folio(ctx, entry, gfp_mask, mpol, ilx, false); mpol_cond_put(mpol); put_swap_device(si); return folio; } +static struct folio *swap_cache_read_folio_sync(swp_entry_t entry, gfp_t gfp, + struct mempolicy *mpol, pgoff_t ilx) +{ + struct swap_io_ctx ctx = {}; + struct folio *folio; + + folio = swap_cache_read_folio(&ctx, entry, gfp, mpol, ilx, false); + swap_read_submit(&ctx); + return folio; +} + static unsigned int __swapin_nr_pages(unsigned long prev_offset, unsigned long offset, int hits, @@ -813,8 +826,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, unsigned long start_offset, end_offset; unsigned long mask; struct swap_info_struct *si = __swap_entry_to_info(entry); + struct swap_io_ctx ctx = {}; struct blk_plug plug; - struct swap_iocb *splug = NULL; swp_entry_t ra_entry; mask = swapin_nr_pages(offset) - 1; @@ -833,17 +846,16 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, for (offset = start_offset; offset <= end_offset ; offset++) { /* Ok, do the async read-ahead now */ ra_entry = swp_entry(swp_type(entry), offset); - folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx, - &splug, offset != entry_offset); + folio = swap_cache_read_folio(&ctx, ra_entry, gfp_mask, mpol, + ilx, offset != entry_offset); if (!folio) continue; folio_put(folio); } blk_finish_plug(&plug); - swap_read_unplug(splug); + swap_read_submit(&ctx); skip: - /* The page was likely read above, so no need for plugging here */ - return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false); + return swap_cache_read_folio_sync(entry, gfp_mask, mpol, ilx); } static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, @@ -903,8 +915,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf) { + struct swap_io_ctx ctx = {}; struct blk_plug plug; - struct swap_iocb *splug = NULL; struct folio *folio; pte_t *pte = NULL, pentry; int win; @@ -943,8 +955,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, if (!si) continue; } - folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, - &splug, addr != vmf->address); + folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx, + addr != vmf->address); if (si) put_swap_device(si); if (!folio) @@ -954,12 +966,10 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, if (pte) pte_unmap(pte); blk_finish_plug(&plug); - swap_read_unplug(splug); + swap_read_submit(&ctx); skip: /* The folio was likely read above, so no need for plugging here */ - folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx, - NULL, false); - return folio; + return swap_cache_read_folio_sync(targ_entry, gfp_mask, mpol, targ_ilx); } /** diff --git a/mm/swapfile.c b/mm/swapfile.c index 4e07d457e261..53bf01d5f7f1 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1899,11 +1899,11 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry) return si; bad_nofile: - pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); + pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val); out: return NULL; put_out: - pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); + pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val); percpu_ref_put(&si->users); return NULL; } @@ -2959,6 +2959,12 @@ static int setup_swap_extents(struct swap_info_struct *sis, struct inode *inode = mapping->host; int ret; + ret = sio_pool_init(); + if (ret) + return ret; + + sis->ops = &swap_bdev_ops; + if (S_ISBLK(inode->i_mode)) { ret = add_swap_extent(sis, 0, sis->max, 0); *span = sis->pages; @@ -2970,11 +2976,6 @@ static int setup_swap_extents(struct swap_info_struct *sis, if (ret < 0) return ret; sis->flags |= SWP_ACTIVATED; - if ((sis->flags & SWP_FS_OPS) && - sio_pool_init() != 0) { - destroy_swap_extents(sis, swap_file); - return -ENOMEM; - } return ret; } @@ -3668,6 +3669,13 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = -EBUSY; goto bad_swap_unlock_inode; } + if (IS_ENCRYPTED(inode)) { + pr_warn_once( + "Filesystem-level encrypted swapfile '%s' is unsupported. Create a loop device over it, or use dm-crypt\n", + name->name); + error = -EINVAL; + goto bad_swap_unlock_inode; + } /* * The swap subsystem needs a major overhaul to support this. @@ -3875,7 +3883,7 @@ int swap_dup_entry_direct(swp_entry_t entry) si = swap_entry_to_info(entry); if (WARN_ON_ONCE(!si)) { - pr_err("%s%08lx\n", Bad_file, entry.val); + pr_err_ratelimited("%s%08lx\n", Bad_file, entry.val); return -EINVAL; } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 752ca2e3b5e5..74f04c323c50 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1352,7 +1352,7 @@ static long move_present_ptes(struct mm_struct *mm, } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot); /* Set soft dirty bit so userspace can notice the pte was moved */ @@ -1428,7 +1428,7 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, */ if (src_folio) { folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); } else { /* * Check if the swap entry is cached after acquiring the src_pte @@ -18,6 +18,7 @@ struct mmap_state { unsigned long addr; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; unsigned long pglen; union { vm_flags_t vm_flags; @@ -46,13 +47,14 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, anon_pgoff_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ .pglen = PHYS_PFN(len_), \ .vma_flags = vma_flags_, \ .file = file_, \ @@ -67,6 +69,7 @@ struct mmap_state { .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ .pgoff = (map_)->pgoff, \ + .anon_pgoff = (map_)->anon_pgoff, \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -82,10 +85,11 @@ static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, } static void vma_set_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end, pgoff_t pgoff) + unsigned long end, pgoff_t pgoff, pgoff_t anon_pgoff) { __vma_set_range(vma, start, end); vma_set_pgoff(vma, pgoff); + vma_set_anon_pgoff(vma, anon_pgoff); } /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ @@ -201,6 +205,25 @@ static void init_multi_vma_prep(struct vma_prepare *vp, } /* + * Does this merge require that adjacent VMAs must have adjacent anonymous page + * offsets in addition to having adjacent vma->vm_pgoff? + * + * This is only required for MAP_PRIVATE-file backed mappings as the page offset + * for pure anonymous VMAs is equal to the anonymous page offset. + * + * Read-only shared mappings (with VMA_SHARED_BIT cleared) are always unfaulted + * so automatically have correct anonymous page offset (as it is always updated + * on remap). + * + * 'Special' mappings in the sense of VDSO, VVAR etc. have !file but would in + * any case not be candidates for merge nor be mergeable. + */ +static bool needs_adjacent_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->file && vma_flags_is_cow_mapping(&vmg->vma_flags); +} + +/* * Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff) * in front of (at a lower virtual address and file offset than) the vma. * @@ -221,6 +244,9 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) return false; if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next)) return false; + if (needs_adjacent_anon_pgoff(vmg) && + vmg_end_anon_pgoff(vmg) != vma_start_anon_pgoff(vmg->next)) + return false; return true; } @@ -241,6 +267,9 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) return false; if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg)) return false; + if (needs_adjacent_anon_pgoff(vmg) && + vma_end_anon_pgoff(vmg->prev) != vmg_start_anon_pgoff(vmg)) + return false; return true; } @@ -812,7 +841,8 @@ static int commit_merge(struct vma_merge_struct *vmg) */ vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); - vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg)); + vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg), + vmg_start_anon_pgoff(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -982,6 +1012,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * We already ensured anon_vma compatibility above, so now it's @@ -1000,6 +1031,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -1022,12 +1054,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; vmg->pgoff = vma_start_pgoff(next) - pglen; + vmg->anon_pgoff = vma_start_anon_pgoff(next) - pglen; } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; vmg->pgoff = vma_start_pgoff(middle); + vmg->anon_pgoff = vma_start_anon_pgoff(middle); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1137,6 +1171,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) vmg->start = prev->vm_start; vmg->target = prev; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * If this merge would result in removal of the next VMA but we @@ -1908,23 +1943,27 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + pgoff_t anon_pgoff, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; - unsigned long vma_start = vma->vm_start; + unsigned long old_vma_start = vma->vm_start; struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *new_vma; - bool faulted_in_anon_vma = true; + bool can_self_merge = false; VMA_ITERATOR(vmi, mm, addr); VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len); /* - * If anonymous vma has not yet been faulted, update new pgoff - * to match new location, to increase its chance of merging. + * If a vma has not yet been faulted, update its anonymous pgoff to + * match the new location to increase its chance of merging. */ - if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; - faulted_in_anon_vma = false; + if (!vma->anon_vma) { + anon_pgoff = addr >> PAGE_SHIFT; + + if (vma_is_anonymous(vma)) { + pgoff = anon_pgoff; + can_self_merge = true; + } } /* @@ -1940,28 +1979,26 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ vmg.pgoff = pgoff; + vmg.anon_pgoff = anon_pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); if (new_vma) { - /* - * Source vma may have been merged into new_vma - */ - if (unlikely(vma_start >= new_vma->vm_start && - vma_start < new_vma->vm_end)) { + /* Self-merged and VMA replaced. */ + if (unlikely(new_vma->vm_start < old_vma_start && + new_vma->vm_end > old_vma_start)) { /* - * The only way we can get a vma_merge with - * self during an mremap is if the vma hasn't - * been faulted in yet and we were allowed to - * reset the dst vma->vm_pgoff to the - * destination address of the mremap to allow - * the merge to happen. mremap must change the - * vm_pgoff linearity between src and dst vmas - * (in turn preventing a vma_merge) to be - * safe. It is only safe to keep the vm_pgoff - * linear if there are no pages mapped yet. + * The only way a VMA can both self-merge and be + * replaced is if the remap places the new VMA + * immediately prior to its old self ('next') and + * immediately after another VMA ('prev') causing the + * next to be removed and prev to be expanded to cover + * the entire range. + * + * This should only be possible if the anonymous page + * offset was updated, i.e. the VMA is unfaulted. */ - VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); + VM_WARN_ON_ONCE_VMA(!can_self_merge, new_vma); *vmap = vma = new_vma; } *need_rmap_locks = @@ -1970,7 +2007,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pgoff, anon_pgoff); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -2036,7 +2073,12 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * if (!vma_flags_empty(&diff)) return false; /* Page offset must align. */ - return vma_end_pgoff(a) == vma_start_pgoff(b); + if (vma_end_pgoff(a) != vma_start_pgoff(b)) + return false; + /* Only reached from anon path, so either MAP_PRIVATE file or anon. */ + if (vma_end_anon_pgoff(a) != vma_start_anon_pgoff(b)) + return false; + return true; } /* @@ -2612,7 +2654,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, if (is_anon) vma_set_anonymous(vma); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->pgoff, map->anon_pgoff); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2801,7 +2843,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + const pgoff_t anon_pgoff = addr >> PAGE_SHIFT; + MMAP_STATE(map, mm, &vmi, addr, len, pgoff, anon_pgoff, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2946,6 +2989,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long addr, unsigned long len, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; + const pgoff_t pgoff = addr >> PAGE_SHIFT; /* * Check against address space limits by the changed size @@ -2970,7 +3014,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, * occur after forking, so the expand will only happen on new VMAs. */ if (vma && vma->vm_end == addr) { - VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, pgoff, pgoff); vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ @@ -2990,7 +3034,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, goto unacct_fail; vma_set_anonymous(vma); - vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_range(vma, addr, addr + len, pgoff, pgoff); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -3382,6 +3426,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) WARN_ON_ONCE(vma->anon_vma); vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } + vma_set_anon_pgoff(vma, vma->vm_start >> PAGE_SHIFT); if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) @@ -3437,7 +3482,7 @@ struct vm_area_struct *__install_special_mapping( vma->vm_ops = ops; vma->vm_private_data = priv; - vma_set_range(vma, addr, addr + len, 0); + vma_set_range(vma, addr, addr + len, 0, addr >> PAGE_SHIFT); ret = insert_vm_struct(mm, vma); if (ret) @@ -104,6 +104,7 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; union { /* Temporary while VMA flags are being converted. */ @@ -237,11 +238,6 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) return vmg->state == VMA_MERGE_ERROR_NOMEM; } -static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) -{ - return vmg->pgoff; -} - static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) { const unsigned long size = vmg->end - vmg->start; @@ -249,6 +245,11 @@ static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) return size >> PAGE_SHIFT; } +static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->pgoff; +} + static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) { return vmg_start_pgoff(vmg) + vmg_pages(vmg); @@ -283,48 +284,76 @@ static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) vma->vm_pgoff = pgoff; } +static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->anon_pgoff; +} + +static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg); +} + +static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ +#ifdef CONFIG_64BIT + vma->__vm_anon_pgoff_hi = pgoff >> 32; +#endif + vma->__vm_anon_pgoff_lo = pgoff & GENMASK(31, 0); +} + +static inline void vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + __vma_set_anon_pgoff(vma, pgoff); +} + static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); + vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta); } static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); -} + vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta); +} + +#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \ + struct vma_merge_struct name = { \ + .mm = mm_, \ + .vmi = vmi_, \ + .start = start_, \ + .end = end_, \ + .vma_flags = vma_flags_, \ + .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ + .state = VMA_MERGE_START, \ + } -#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ +#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ struct vma_merge_struct name = { \ - .mm = mm_, \ + .mm = vma_->vm_mm, \ .vmi = vmi_, \ + .prev = prev_, \ + .middle = vma_, \ + .next = NULL, \ .start = start_, \ .end = end_, \ - .vma_flags = vma_flags_, \ - .pgoff = pgoff_, \ + .vm_flags = vma_->vm_flags, \ + .pgoff = linear_page_index(vma_, start_), \ + .anon_pgoff = __linear_anon_page_index(vma_, start_), \ + .file = vma_->vm_file, \ + .anon_vma = vma_->anon_vma, \ + .policy = vma_policy(vma_), \ + .uffd_ctx = vma_->vm_userfaultfd_ctx, \ + .anon_name = anon_vma_name(vma_), \ .state = VMA_MERGE_START, \ } -#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ - struct vma_merge_struct name = { \ - .mm = vma_->vm_mm, \ - .vmi = vmi_, \ - .prev = prev_, \ - .middle = vma_, \ - .next = NULL, \ - .start = start_, \ - .end = end_, \ - .vm_flags = vma_->vm_flags, \ - .pgoff = linear_page_index(vma_, start_), \ - .file = vma_->vm_file, \ - .anon_vma = vma_->anon_vma, \ - .policy = vma_policy(vma_), \ - .uffd_ctx = vma_->vm_userfaultfd_ctx, \ - .anon_name = anon_vma_name(vma_), \ - .state = VMA_MERGE_START, \ - } - #ifdef CONFIG_DEBUG_VM_MAPLE_TREE void validate_mm(struct mm_struct *mm); #else @@ -506,7 +535,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); + pgoff_t anon_pgoff, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 7af1260689b9..586c52155942 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -41,7 +41,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) unsigned long new_end = old_end - shift; VMA_ITERATOR(vmi, mm, new_start); VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS, - vma_start_pgoff(vma)); + vma_start_pgoff(vma), vma_start_anon_pgoff(vma)); struct vm_area_struct *next; struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); diff --git a/mm/vma_init.c b/mm/vma_init.c index 715feee283f0..baa7e82f47e3 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -51,6 +51,7 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; dest->vm_pgoff = vma_start_pgoff(src); + __vma_set_anon_pgoff(dest, vma_start_anon_pgoff(src)); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 5c0c2d0d6ae7..bea9f76ed7e7 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1840,8 +1840,10 @@ va_alloc(struct vmap_area *va, /* Update the free vmap_area. */ ret = va_clip(root, head, va, nva_start_addr, size); - if (WARN_ON_ONCE(ret)) + if (ret) { + WARN_ON_ONCE(ret != -ENOMEM); return ret; + } return nva_start_addr; } @@ -1914,12 +1916,9 @@ preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node) /* * Preload this CPU with one extra vmap_area object. It is used - * when fit type of free area is NE_FIT_TYPE. It guarantees that - * a CPU that does an allocation is preloaded. - * - * We do it in non-atomic context, thus it allows us to use more - * permissive allocation masks to be more stable under low memory - * condition and high memory pressure. + * when fit type of free area is NE_FIT_TYPE. It is best effort + * pre-loading. If it fails va_clip() may return -ENOMEM from its + * GFP_NOWAIT fallback. */ if (!this_cpu_read(ne_fit_preload_node)) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); @@ -3361,7 +3360,7 @@ struct vm_struct *remove_vm_area(const void *addr) static inline void set_area_direct_map(const struct vm_struct *area, int (*set_direct_map)(struct page *page)) { - int i; + unsigned long i; /* HUGE_VMALLOC passes small pages to set_direct_map */ for (i = 0; i < area->nr_pages; i++) @@ -3377,7 +3376,7 @@ static void vm_reset_perms(struct vm_struct *area) unsigned long start = ULONG_MAX, end = 0; unsigned int page_order = vm_area_page_order(area); int flush_dmap = 0; - int i; + unsigned long i; /* * Find the start and end range of the direct mappings to make sure that @@ -3450,10 +3449,10 @@ void vfree_atomic(const void *addr) * Caller is responsible for unmapping (vunmap_range) and KASAN * poisoning before calling this. */ -static void vm_area_free_pages(struct vm_struct *vm, unsigned int start_idx, - unsigned int end_idx) +static void vm_area_free_pages(struct vm_struct *vm, unsigned long start_idx, + unsigned long end_idx) { - unsigned int i; + unsigned long i; if (!(vm->flags & VM_MAP_PUT_PAGES)) { for (i = start_idx; i < end_idx; i++) @@ -3665,12 +3664,12 @@ static inline gfp_t vmalloc_gfp_adjust(gfp_t flags, const bool large) return flags; } -static inline unsigned int +static inline unsigned long vm_area_alloc_pages(gfp_t gfp, int nid, - unsigned int order, unsigned int nr_pages, struct page **pages) + unsigned int order, unsigned long nr_pages, struct page **pages) { - unsigned int nr_allocated = 0; - unsigned int nr_remaining = nr_pages; + unsigned long nr_allocated = 0; + unsigned long nr_remaining = nr_pages; unsigned int max_attempt_order = MAX_PAGE_ORDER; struct page *page; int i; @@ -3718,7 +3717,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid, if (!order) { while (nr_allocated < nr_pages) { unsigned int nr, nr_pages_request; - int i; + unsigned long i; /* * A maximum allowed request is hard-coded and is 100 @@ -3726,7 +3725,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid, * long preemption off scenario in the bulk-allocator * so the range is [1:100]. */ - nr_pages_request = min(100U, nr_pages - nr_allocated); + nr_pages_request = min(100UL, nr_pages - nr_allocated); /* memory allocation should consider mempolicy, we can't * wrongly use nearest node when nid == NUMA_NO_NODE, @@ -3872,12 +3871,12 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, unsigned long addr = (unsigned long)area->addr; unsigned long size = get_vm_area_size(area); unsigned long array_size; - unsigned int nr_small_pages = size >> PAGE_SHIFT; + unsigned long nr_small_pages = size >> PAGE_SHIFT; unsigned int page_order; unsigned int flags; int ret; - array_size = (unsigned long)nr_small_pages * sizeof(struct page *); + array_size = nr_small_pages * sizeof(struct page *); /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */ if (!gfpflags_allow_blocking(gfp_mask)) @@ -4375,7 +4374,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align } if (size <= old_size) { - unsigned int new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; /* Zero out "freed" memory, potentially for future realloc. */ if (want_init_on_free() || want_init_on_alloc(flags)) @@ -4404,7 +4403,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align !(vm->flags & (VM_FLUSH_RESET_PERMS | VM_USERMAP)) && gfp_has_io_fs(flags)) { unsigned long addr = (unsigned long)kasan_reset_tag(p); - unsigned int old_nr_pages = vm->nr_pages; + unsigned long old_nr_pages = vm->nr_pages; /* * Use the node lock to synchronize with concurrent @@ -4417,16 +4416,13 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align spin_unlock(&vn->busy.lock); /* Notify kmemleak of the reduced allocation size before unmapping. */ - kmemleak_free_part( - (void *)addr + ((unsigned long)new_nr_pages - << PAGE_SHIFT), - (unsigned long)(old_nr_pages - new_nr_pages) - << PAGE_SHIFT); + kmemleak_free_part((void *)addr + + (new_nr_pages << PAGE_SHIFT), + (old_nr_pages - new_nr_pages) + << PAGE_SHIFT); - vunmap_range(addr + ((unsigned long)new_nr_pages - << PAGE_SHIFT), - addr + ((unsigned long)old_nr_pages - << PAGE_SHIFT)); + vunmap_range(addr + (new_nr_pages << PAGE_SHIFT), + addr + (old_nr_pages << PAGE_SHIFT)); vm_area_free_pages(vm, new_nr_pages, old_nr_pages); } @@ -5250,7 +5246,7 @@ bool vmalloc_dump_obj(void *object) struct vmap_area *va; struct vmap_node *vn; unsigned long addr; - unsigned int nr_pages; + unsigned long nr_pages; addr = PAGE_ALIGN((unsigned long) object); vn = addr_to_node(addr); @@ -5270,7 +5266,7 @@ bool vmalloc_dump_obj(void *object) nr_pages = vm->nr_pages; spin_unlock(&vn->busy.lock); - pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", + pr_cont(" %lu-page vmalloc region starting at %#lx allocated at %pS\n", nr_pages, addr, caller); return true; @@ -5288,16 +5284,17 @@ bool vmalloc_dump_obj(void *object) static void show_numa_info(struct seq_file *m, struct vm_struct *v, unsigned int *counters) { - unsigned int nr; unsigned int step = 1U << vm_area_page_order(v); + unsigned long i; + unsigned int nr; if (!counters) return; memset(counters, 0, nr_node_ids * sizeof(unsigned int)); - for (nr = 0; nr < v->nr_pages; nr += step) - counters[page_to_nid(v->pages[nr])] += step; + for (i = 0; i < v->nr_pages; i += step) + counters[page_to_nid(v->pages[i])] += step; for_each_node_state(nr, N_HIGH_MEMORY) if (counters[nr]) seq_printf(m, " N%u=%u", nr, counters[nr]); @@ -5355,7 +5352,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p) seq_printf(m, " %pS", v->caller); if (v->nr_pages) - seq_printf(m, " pages=%d", v->nr_pages); + seq_printf(m, " pages=%lu", v->nr_pages); if (v->phys_addr) seq_printf(m, " phys=%pa", &v->phys_addr); diff --git a/mm/vmalloc.h b/mm/vmalloc.h index dcfe30eaa80c..8866ddcff668 100644 --- a/mm/vmalloc.h +++ b/mm/vmalloc.h @@ -5,6 +5,8 @@ #ifndef __MM_VMALLOC_H #define __MM_VMALLOC_H +#include <linux/vmalloc.h> + #ifdef CONFIG_MMU void __init vmalloc_init(void); int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, diff --git a/mm/vmscan.c b/mm/vmscan.c index 4ca9775ceee8..fe7f0c52a18c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -58,6 +58,7 @@ #include <linux/random.h> #include <linux/mmu_notifier.h> #include <linux/parser.h> +#include <linux/swap_ops.h> #include <asm/tlbflush.h> #include <asm/div64.h> @@ -199,6 +200,13 @@ struct scan_control { */ int vm_swappiness = 60; +static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) +{ + if (sc->proactive && sc->proactive_swappiness) + return *sc->proactive_swappiness; + return mem_cgroup_swappiness(memcg); +} + #ifdef CONFIG_MEMCG /* Returns true for reclaim through cgroup limits or cgroup interfaces. */ @@ -239,13 +247,6 @@ static bool writeback_throttling_sane(struct scan_control *sc) #endif return false; } - -static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) -{ - if (sc->proactive && sc->proactive_swappiness) - return *sc->proactive_swappiness; - return mem_cgroup_swappiness(memcg); -} #else static bool cgroup_reclaim(struct scan_control *sc) { @@ -261,12 +262,13 @@ static bool writeback_throttling_sane(struct scan_control *sc) { return true; } +#endif -static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) +static inline bool is_exec_file_folio(const struct folio *folio, + const vma_flags_t *vma_flags) { - return READ_ONCE(vm_swappiness); + return vma_flags_test(vma_flags, VMA_EXEC_BIT) && folio_is_file_lru(folio); } -#endif static void set_task_reclaim_state(struct task_struct *task, struct reclaim_state *rs) @@ -616,8 +618,8 @@ typedef enum { /* * pageout is called by shrink_folio_list() for each dirty folio. */ -static pageout_t pageout(struct folio *folio, struct address_space *mapping, - struct swap_iocb **plug, struct list_head *folio_list) +static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping, + struct folio *folio, struct list_head *folio_list) { int res; @@ -653,9 +655,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping, * the split out folios get added back to folio_list. */ if (shmem_mapping(mapping)) - res = shmem_writeout(folio, plug, folio_list); + res = shmem_writeout(ctx, folio, folio_list); else - res = swap_writeout(folio, plug); + res = swap_writeout(ctx, folio); if (res < 0) handle_write_error(mapping, folio, res); @@ -669,7 +671,7 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping, folio_clear_reclaim(folio); trace_mm_vmscan_write_folio(folio); - node_stat_add_folio(folio, NR_VMSCAN_WRITE); + lruvec_stat_mod_folio(folio, NR_VMSCAN_WRITE, folio_nr_pages(folio)); return PAGE_SUCCESS; } @@ -835,10 +837,16 @@ enum folio_references { * with PG_active set. In contrast, the aging (page table walk) path uses * folio_update_gen(). */ -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* Activate file-backed executable folios after first usage. */ + if (is_exec_file_folio(folio, vma_flags)) { + set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + return true; + } + set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return false; } @@ -851,7 +859,7 @@ static bool lru_gen_set_refs(struct folio *folio) return true; } #else -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { return false; } @@ -861,16 +869,16 @@ static enum folio_references folio_check_references(struct folio *folio, struct scan_control *sc) { int referenced_ptes, referenced_folio; - vm_flags_t vm_flags; + vma_flags_t vma_flags; referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, - &vm_flags); + &vma_flags); /* * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. * Let the folio, now marked Mlocked, be moved to the unevictable list. */ - if (vm_flags & VM_LOCKED) + if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) return FOLIOREF_ACTIVATE; /* @@ -886,7 +894,7 @@ static enum folio_references folio_check_references(struct folio *folio, if (!referenced_ptes) return FOLIOREF_RECLAIM; - return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; + return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; } referenced_folio = folio_test_clear_referenced(folio); @@ -914,7 +922,7 @@ static enum folio_references folio_check_references(struct folio *folio, /* * Activate file-backed executable folios after first usage. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) + if (is_exec_file_folio(folio, &vma_flags)) return FOLIOREF_ACTIVATE; return FOLIOREF_KEEP; @@ -1033,16 +1041,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) { if (gfp_mask & __GFP_FS) return true; - if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) - return false; /* - * We can "enter_fs" for swap-cache with only __GFP_IO - * providing this isn't SWP_FS_OPS. - * ->flags can be updated non-atomically, - * but that will never affect SWP_FS_OPS, so the data_race - * is safe. + * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by + * a swapfile that requires GFP_NOFS I/O. */ - return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); + if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) && + !(__swap_entry_to_info(folio->swap)->ops->flags & + SWAP_OPS_F_REQUIRE_NOFS)) + return true; + return false; } /* @@ -1059,7 +1066,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, unsigned int nr_reclaimed = 0, nr_demoted = 0; unsigned int pgactivate = 0; bool do_demote_pass; - struct swap_iocb *plug = NULL; + struct swap_io_ctx ctx = {}; folio_batch_init(&free_folios); memset(stat, 0, sizeof(*stat)); @@ -1387,7 +1394,7 @@ retry: * starts and then write it out here. */ try_to_unmap_flush_dirty(); - switch (pageout(folio, mapping, &plug, folio_list)) { + switch (pageout(&ctx, mapping, folio, folio_list)) { case PAGE_KEEP: goto keep_locked; case PAGE_ACTIVATE: @@ -1406,8 +1413,6 @@ retry: sc->nr_scanned -= (nr_pages - 1); nr_pages = 1; } - stat->nr_pageout += nr_pages; - if (folio_test_writeback(folio)) goto keep; if (folio_test_dirty(folio)) @@ -1577,8 +1582,7 @@ keep: list_splice(&ret_folios, folio_list); count_vm_events(PGACTIVATE, pgactivate); - if (plug) - swap_write_unplug(plug); + swap_write_submit(&ctx); return nr_reclaimed; } @@ -2031,10 +2035,10 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, item = PGSTEAL_KSWAPD + reclaimer_offset(sc); mod_lruvec_state(lruvec, item, nr_reclaimed); mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed); + if (nr_scanned > nr_reclaimed) + mod_lruvec_state(lruvec, PGROTATE_ANON + file, + nr_scanned - nr_reclaimed); - lruvec_lock_irq(lruvec); - lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout, - nr_scanned - nr_reclaimed); handle_reclaim_writeback(nr_taken, pgdat, sc, &stat); trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, nr_scanned, nr_reclaimed, &stat, sc->priority, file); @@ -2065,7 +2069,7 @@ static void shrink_active_list(unsigned long nr_to_scan, { unsigned long nr_taken; unsigned long nr_scanned; - vm_flags_t vm_flags; + vma_flags_t vma_flags; LIST_HEAD(l_hold); /* The folios which were snipped off */ LIST_HEAD(l_active); LIST_HEAD(l_inactive); @@ -2109,7 +2113,7 @@ static void shrink_active_list(unsigned long nr_to_scan, /* Referenced or rmap lock contention: rotate */ if (folio_referenced(folio, 0, sc->target_mem_cgroup, - &vm_flags) != 0) { + &vma_flags) != 0) { /* * Identify referenced, file-backed active folios and * give them one more trip around the active list. So @@ -2119,7 +2123,7 @@ static void shrink_active_list(unsigned long nr_to_scan, * IO, plus JVM can create lots of anon VM_EXEC folios, * so we ignore them here. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { + if (is_exec_file_folio(folio, &vma_flags)) { nr_rotated += folio_nr_pages(folio); list_add(&folio->lru, &l_active); continue; @@ -2140,9 +2144,9 @@ static void shrink_active_list(unsigned long nr_to_scan, count_vm_events(PGDEACTIVATE, nr_deactivate); count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); + if (nr_rotated) + mod_lruvec_state(lruvec, PGROTATE_ANON + file, nr_rotated); - lruvec_lock_irq(lruvec); - lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated); trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, nr_deactivate, nr_rotated, sc->priority, file); } @@ -2275,8 +2279,10 @@ enum scan_balance { static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) { - unsigned long file; + struct lru_cost *anon_cost, *file_cost; struct lruvec *target_lruvec; + unsigned long lrusize; + unsigned long file; if (lru_gen_enabled() && !lru_gen_switching()) return; @@ -2292,11 +2298,69 @@ static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) /* * Determine the scan balance between anon and file LRUs. + * + * The cost model is based on rotations, refaults and + * reclaim-driven writes (anon only) on each side. + * + * These event counters are monotonic, so each reclaim cycle + * the delta since the last scan is extracted and incorporated + * into a decaying average. This ensures currency, as workloads + * change over time, and avoids overflow in the calculations. + * + * Use lruvec_page_state_monotonic() so unsigned subtraction + * yields the correct delta across a signed-long wraparound of + * the underlying counter (a real hazard on 32-bit that the + * clamp in lruvec_page_state() would otherwise turn into a huge + * spurious delta). */ - spin_lock_irq(&target_lruvec->lru_lock); - sc->anon_cost = target_lruvec->anon_cost; - sc->file_cost = target_lruvec->file_cost; - spin_unlock_irq(&target_lruvec->lru_lock); + spin_lock(&target_lruvec->cost_lock); + + for (int f = 0; f <= 1; f++) { + struct lru_cost *cost = &target_lruvec->cost[f]; + unsigned long rotated, io, nr_rotated, nr_io; + + rotated = lruvec_page_state_monotonic(target_lruvec, + PGROTATE_ANON + f); + io = lruvec_page_state_monotonic(target_lruvec, + WORKINGSET_RESTORE_BASE + f); + if (f == WORKINGSET_ANON) + io += lruvec_page_state_monotonic(target_lruvec, + NR_VMSCAN_WRITE); + + nr_rotated = rotated - cost->last_rotated; + nr_io = io - cost->last_io; + + /* + * Reflect the relative cost of incurring IO and spending + * CPU time on rotations. This doesn't attempt to make a + * precise comparison, it just says: if reloads are about + * comparable between the LRU lists, or rotations are + * overwhelmingly different between them, adjust scan + * balance for CPU work. + */ + cost->count += nr_io * SWAP_CLUSTER_MAX + nr_rotated; + + cost->last_rotated = rotated; + cost->last_io = io; + } + + anon_cost = &target_lruvec->cost[WORKINGSET_ANON]; + file_cost = &target_lruvec->cost[WORKINGSET_FILE]; + + lrusize = lruvec_page_state(target_lruvec, NR_INACTIVE_ANON) + + lruvec_page_state(target_lruvec, NR_ACTIVE_ANON) + + lruvec_page_state(target_lruvec, NR_INACTIVE_FILE) + + lruvec_page_state(target_lruvec, NR_ACTIVE_FILE); + + while (anon_cost->count + file_cost->count > lrusize / 4) { + anon_cost->count /= 2; + file_cost->count /= 2; + } + + sc->anon_cost = anon_cost->count; + sc->file_cost = file_cost->count; + + spin_unlock(&target_lruvec->cost_lock); /* * Target desirable inactive:active list ratios for the anon @@ -3202,14 +3266,19 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen) +static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) { unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* + * See the comment on LRU_REFS_FLAGS, and activate file-backed + * executable folios after first usage to avoid typical IO + * thrashing from reclaiming. + */ + if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && + !is_exec_file_folio(folio, vma_flags)) { set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return -1; } @@ -3442,8 +3511,8 @@ static bool suitable_to_scan(int total, int young) return young * n >= total; } -static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, - int new_gen, bool dirty) +static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, + struct folio *folio, int new_gen, bool dirty) { int old_gen; @@ -3456,10 +3525,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen); + old_gen = folio_update_gen(folio, new_gen, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) update_batch_size(walk, folio, old_gen, new_gen); - } else if (lru_gen_set_refs(folio)) { + } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) folio_activate(folio); @@ -3532,7 +3601,7 @@ restart: continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = folio; dirty = false; @@ -3545,7 +3614,7 @@ restart: walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3623,7 +3692,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area goto next; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -3637,7 +3706,7 @@ next: i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -4192,7 +4261,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) unsigned long end; struct lru_gen_mm_walk *walk; struct folio *last = NULL; - int young = 1; + int young = nr; pte_t *pte = pvmw->pte; unsigned long addr = pvmw->address; struct vm_area_struct *vma = pvmw->vma; @@ -4272,7 +4341,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -4284,7 +4353,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) young += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); @@ -4580,7 +4649,6 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, int tier_idx) { - bool success; int gen = folio_lru_gen(folio); int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); @@ -4592,15 +4660,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); - /* unevictable */ - if (!folio_evictable(folio)) { - success = lru_gen_del_folio(lruvec, folio, true); - VM_WARN_ON_ONCE_FOLIO(!success, folio); - folio_set_unevictable(folio); - lruvec_add_folio(lruvec, folio); - __count_vm_events(UNEVICTABLE_PGCULLED, delta); - return true; - } + /* unevictable: let it through and the generic path will cull it */ + if (!folio_evictable(folio)) + return false; /* promoted */ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { @@ -4816,7 +4878,8 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct reclaim_stat stat; struct lru_gen_mm_walk *walk; int scanned, reclaimed; - int isolated = 0, type, type_scanned; + int isolated = 0, nr_isolated = 0, type, type_scanned; + unsigned long total_reclaimed = 0; bool skip_retry = false; struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); @@ -4828,6 +4891,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &list, &isolated, &type, &type_scanned); + nr_isolated = isolated; /* Scanning may have emptied the oldest gen, flush it */ if (scanned) @@ -4840,6 +4904,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, retry: reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg); sc->nr_reclaimed += reclaimed; + total_reclaimed += reclaimed; /* Retry pass is only meant for clean folios without new isolation */ if (isolated) handle_reclaim_writeback(isolated, pgdat, sc, &stat); @@ -4850,11 +4915,9 @@ retry: list_for_each_entry_safe_reverse(folio, next, &list, lru) { DEFINE_MIN_SEQ(lruvec); - if (!folio_evictable(folio)) { - list_del(&folio->lru); - folio_putback_lru(folio); + /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */ + if (!folio_evictable(folio)) continue; - } /* retry folios that may have missed folio_rotate_reclaimable() */ if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) && @@ -4891,6 +4954,10 @@ retry: goto retry; } + if (nr_isolated > total_reclaimed) + mod_lruvec_state(lruvec, PGROTATE_ANON + type, + nr_isolated - total_reclaimed); + return scanned; } diff --git a/mm/vmstat.c b/mm/vmstat.c index 3b5cb1031f72..cb57714539fb 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1025,6 +1025,17 @@ unsigned long node_page_state(struct pglist_data *pgdat, return node_page_state_pages(pgdat, item); } + +/* + * Non-clamping variant of node_page_state() intended for callers that + * snapshot a monotonically-incremented counter and subtract two samples. + * See global_node_page_state_monotonic() for the rationale. + */ +unsigned long node_page_state_monotonic(struct pglist_data *pgdat, + enum node_stat_item item) +{ + return (unsigned long)atomic_long_read(&pgdat->vm_stat[item]); +} #endif /* @@ -1290,6 +1301,8 @@ const char * const vmstat_text[] = { [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", [I(PGSCAN_ANON)] = "pgscan_anon", [I(PGSCAN_FILE)] = "pgscan_file", + [I(PGROTATE_ANON)] = "pgrotate_anon", + [I(PGROTATE_FILE)] = "pgrotate_file", [I(PGREFILL)] = "pgrefill", #ifdef CONFIG_HUGETLB_PAGE [I(NR_HUGETLB)] = "nr_hugetlb", @@ -1489,7 +1502,11 @@ const char * const vmstat_text[] = { #if THREAD_SIZE > 65536 [I(KSTACK_REST)] = "kstack_rest", #endif -#endif +#endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + [I(NRSWPIN)] = "nrswpin", + [I(NRSWPOUT)] = "nrswpout", +#endif /* CONFIG_SWAP */ #undef I #endif /* CONFIG_VM_EVENT_COUNTERS */ }; diff --git a/mm/workingset.c b/mm/workingset.c index f351798e723a..7ac2b88c80ae 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -584,11 +584,6 @@ void workingset_refault(struct folio *folio, void *shadow) /* Folio was active prior to eviction */ if (workingset) { folio_set_workingset(folio); - /* - * XXX: Move to folio_add_lru() when it supports new vs - * putback - */ - lru_note_cost_refault(folio); mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr); } out: diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 8204b76f7830..825022a7a328 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -552,6 +552,11 @@ static int get_size_class_index(int size) return min_t(int, ZS_SIZE_CLASSES - 1, idx); } +static struct size_class *lookup_size_class(struct zs_pool *pool, size_t size) +{ + return pool->size_class[get_size_class_index(size + ZS_HANDLE_SIZE)]; +} + static inline void class_stat_add(struct size_class *class, int type, unsigned long cnt) { @@ -1117,7 +1122,7 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size) { struct size_class *class; - class = pool->size_class[get_size_class_index(size)]; + class = lookup_size_class(pool, size); return class->index; } @@ -1407,9 +1412,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, if (!handle) return (unsigned long)ERR_PTR(-ENOMEM); - /* extra space in chunk to keep the handle */ - size += ZS_HANDLE_SIZE; - class = pool->size_class[get_size_class_index(size)]; + class = lookup_size_class(pool, size); /* class->lock effectively protects the zpage migration */ spin_lock(&class->lock); diff --git a/mm/zswap.c b/mm/zswap.c index 761cd699e0a3..37f34e406c8e 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -20,7 +20,7 @@ #include <linux/spinlock.h> #include <linux/types.h> #include <linux/atomic.h> -#include <linux/swap.h> +#include <linux/swap_ops.h> #include <linux/crypto.h> #include <linux/scatterlist.h> #include <linux/mempolicy.h> @@ -665,7 +665,7 @@ static inline int entry_to_nid(struct zswap_entry *entry) return page_to_nid(virt_to_page(entry)); } -static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry) +static void zswap_lru_add(struct zswap_entry *entry) { int nid = entry_to_nid(entry); struct mem_cgroup *memcg; @@ -684,11 +684,11 @@ static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry) rcu_read_lock(); memcg = mem_cgroup_from_entry(entry); /* will always succeed */ - list_lru_add(list_lru, &entry->lru, nid, memcg); + list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg); rcu_read_unlock(); } -static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry) +static void zswap_lru_del(struct zswap_entry *entry) { int nid = entry_to_nid(entry); struct mem_cgroup *memcg; @@ -696,7 +696,7 @@ static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry) rcu_read_lock(); memcg = mem_cgroup_from_entry(entry); /* will always succeed */ - list_lru_del(list_lru, &entry->lru, nid, memcg); + list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg); rcu_read_unlock(); } @@ -764,7 +764,7 @@ static void zswap_entry_cache_free(struct zswap_entry *entry) */ static void zswap_entry_free(struct zswap_entry *entry) { - zswap_lru_del(&zswap_list_lru, entry); + zswap_lru_del(entry); zs_free(entry->pool->zs_pool, entry->handle); zswap_pool_put(entry->pool); if (entry->objcg) { @@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, struct folio *folio; struct mempolicy *mpol; struct swap_info_struct *si; + struct swap_io_ctx ctx = {}; int ret = 0; /* try to allocate swap cache folio */ @@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, folio_set_reclaim(folio); /* start writeback */ - __swap_writepage(folio, NULL); + __swap_writepage(&ctx, folio); + swap_write_submit(&ctx); out: if (ret) { @@ -1217,7 +1219,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, * Without memcg, use the zswap pool-wide metrics. */ if (!mem_cgroup_disabled()) { - mem_cgroup_flush_stats(memcg); + mem_cgroup_flush_stats_ratelimited(memcg); nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); } else { @@ -1275,6 +1277,14 @@ static struct shrinker *zswap_alloc_shrinker(void) return shrinker; } +/* + * Scan up to SWAP_CLUSTER_MAX pages on each per-node zswap LRU of @memcg + * and write back the reclaimable ones. + * + * Return: 0 if at least one entry was written back, -EAGAIN if entries + * were scanned but none could be written back, or -ENOENT if @memcg has + * writeback disabled, is a zombie cgroup, or has empty zswap LRUs. + */ static int shrink_memcg(struct mem_cgroup *memcg) { int nid, shrunk = 0, scanned = 0; @@ -1290,13 +1300,14 @@ static int shrink_memcg(struct mem_cgroup *memcg) return -ENOENT; for_each_node_state(nid, N_NORMAL_MEMORY) { - unsigned long nr_to_walk = 1; + unsigned long nr_to_walk = SWAP_CLUSTER_MAX; shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg, &shrink_memcg_cb, NULL, &nr_to_walk); - scanned += 1 - nr_to_walk; + scanned += SWAP_CLUSTER_MAX - nr_to_walk; } + /* Nothing was scanned: every LRU under @memcg was empty. */ if (!scanned) return -ENOENT; @@ -1356,11 +1367,12 @@ static void shrink_worker(struct work_struct *w) } while (memcg && !mem_cgroup_tryget_online(memcg)); spin_unlock(&zswap_shrink_lock); - if (!memcg) { - /* - * Continue shrinking without incrementing failures if - * we found candidate memcgs in the last tree walk. - */ + /* + * A NULL memcg ends a full hierarchy pass (except when memcg is + * disabled, where it is always NULL: fall through to the root LRU). + * Count a failure only if the last pass found no candidates. + */ + if (!memcg && !mem_cgroup_disabled()) { if (!attempts && ++failures == MAX_RECLAIM_RETRIES) break; @@ -1379,7 +1391,7 @@ static void shrink_worker(struct work_struct *w) * and failures. */ if (ret == -ENOENT) - continue; + goto resched; ++attempts; if (ret && ++failures == MAX_RECLAIM_RETRIES) @@ -1461,7 +1473,7 @@ static bool zswap_store_page(struct page *page, entry->referenced = true; if (entry->length) { INIT_LIST_HEAD(&entry->lru); - zswap_lru_add(&zswap_list_lru, entry); + zswap_lru_add(entry); } return true; diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index 0607913a3022..d967e76a3c06 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -35234,7 +35234,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) mt_set_non_kernel(1); /* Spanning store */ mas_set_range(&mas, 1, 100); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); @@ -35257,7 +35257,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) MT_BUG_ON(mt, mas_allocated(&mas) != 0); mas_set_range(&mas, 0, 200); mt_set_non_kernel(1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index c62642302c84..2d960626750e 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 TARGETS += acct +TARGETS += alloc_tag TARGETS += alsa TARGETS += amd-pstate TARGETS += arm64 diff --git a/tools/testing/selftests/alloc_tag/Makefile b/tools/testing/selftests/alloc_tag/Makefile new file mode 100644 index 000000000000..c4637f69e9c2 --- /dev/null +++ b/tools/testing/selftests/alloc_tag/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 + +TEST_GEN_PROGS := allocinfo_ioctl_test + +CFLAGS += -Wall +CFLAGS += $(KHDR_INCLUDES) + +include ../lib.mk diff --git a/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c new file mode 100644 index 000000000000..74fd64b2370c --- /dev/null +++ b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* kselftest for allocinfo ioctl + * allocinfo ioctl retrieves allocinfo data through ioctl + * Copyright (C) 2026 Google, Inc. + */ + +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdbool.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <linux/types.h> +#include <linux/alloc_tag.h> +#include "../kselftest.h" + +#define MAX_LINE_LEN 512 +#define ALLOCINFO_PROC "/proc/allocinfo" + +enum ioctl_ret { + IOCTL_SUCCESS = 0, + IOCTL_FAILURE = 1, + IOCTL_INVALID_DATA = 2, +}; + +#define VEC_MAX_ENTRIES 32 + +struct allocinfo_tag_data_vec { + struct allocinfo_tag_data tag[VEC_MAX_ENTRIES]; + __u64 count; +}; + +static inline int __allocinfo_get_content_id(int dev_fd, struct allocinfo_content_id *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_CONTENT_ID, params); +} + +static inline int __allocinfo_get_at(int dev_fd, struct allocinfo_get_at *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_GET_AT, params); +} + +static inline int __allocinfo_get_next(int dev_fd, struct allocinfo_tag_data *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_GET_NEXT, params); +} + +static bool match_entry(const struct allocinfo_tag_data *procfs_entry, + const struct allocinfo_tag_data *tag_data, + bool match_bytes, bool match_calls, bool match_lineno, + bool match_function, bool match_filename) +{ + if (match_bytes && tag_data->counter.bytes != procfs_entry->counter.bytes) { + ksft_print_msg("size retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_calls && tag_data->counter.calls != procfs_entry->counter.calls) { + ksft_print_msg("call count retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_lineno && tag_data->tag.lineno != procfs_entry->tag.lineno) { + ksft_print_msg("lineno retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_function && + strncmp(tag_data->tag.function, procfs_entry->tag.function, ALLOCINFO_STR_SIZE)) { + ksft_print_msg("function retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_filename && + strncmp(tag_data->tag.filename, procfs_entry->tag.filename, ALLOCINFO_STR_SIZE)) { + ksft_print_msg("filename retrieved through ioctl does not match procfs\n"); + return false; + } + return true; +} + +static bool match_entries(const struct allocinfo_tag_data_vec *procfs_entries, + const struct allocinfo_tag_data_vec *tags, + bool match_bytes, bool match_calls, bool match_lineno, + bool match_function, bool match_filename) +{ + __u64 i; + + if (procfs_entries->count != tags->count) { + ksft_print_msg("Entry count mismatch. ioctl entries: %llu, proc entries: %llu\n", + tags->count, procfs_entries->count); + return false; + } + for (i = 0; i < procfs_entries->count; i++) { + if (!match_entry(&procfs_entries->tag[i], &tags->tag[i], + match_bytes, match_calls, match_lineno, + match_function, match_filename)) { + ksft_print_msg("%lluth entry does not match.\n", i); + return false; + } + } + return true; +} + +static const char *allocinfo_str(const char *str) +{ + size_t len = strlen(str); + + if (len >= ALLOCINFO_STR_SIZE) + str += (len - ALLOCINFO_STR_SIZE) + 1; + return str; +} + +static void allocinfo_copy_str(char *dest, const char *src) +{ + strncpy(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE - 1); + dest[ALLOCINFO_STR_SIZE - 1] = '\0'; +} + +static int get_filtered_procfs_entries(struct allocinfo_tag_data_vec *procfs_entries, + const struct allocinfo_filter *filter) +{ + FILE *fp = fopen(ALLOCINFO_PROC, "r"); + char line[MAX_LINE_LEN]; + int matches; + struct allocinfo_tag_data procfs_entry; + + if (!fp) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC " for reading\n"); + return 1; + } + memset(procfs_entries, 0, sizeof(*procfs_entries)); + while (fgets(line, sizeof(line), fp) && procfs_entries->count < VEC_MAX_ENTRIES) { + char filename[MAX_LINE_LEN]; + char function[MAX_LINE_LEN]; + + memset(&procfs_entry, 0, sizeof(procfs_entry)); + matches = sscanf(line, "%llu %llu %[^:]:%llu func:%s", + &procfs_entry.counter.bytes, + &procfs_entry.counter.calls, + filename, + &procfs_entry.tag.lineno, + function); + + if (matches != 5) + continue; + + allocinfo_copy_str(procfs_entry.tag.filename, filename); + allocinfo_copy_str(procfs_entry.tag.function, function); + + if (filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) { + if (strncmp(procfs_entry.tag.filename, + filter->fields.filename, ALLOCINFO_STR_SIZE)) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) { + if (strncmp(procfs_entry.tag.function, + filter->fields.function, ALLOCINFO_STR_SIZE)) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_LINENO) { + if (procfs_entry.tag.lineno != filter->fields.lineno) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) { + if (procfs_entry.counter.bytes < filter->min_size) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) { + if (procfs_entry.counter.bytes > filter->max_size) + continue; + } + + memcpy(&procfs_entries->tag[procfs_entries->count++], &procfs_entry, + sizeof(procfs_entry)); + } + fclose(fp); + return 0; +} + +static enum ioctl_ret get_filtered_ioctl_entries(struct allocinfo_tag_data_vec *tags, + const struct allocinfo_filter *filter, + __u64 start_pos) +{ + int fd = open(ALLOCINFO_PROC, O_RDONLY); + + if (fd < 0) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC " for IOCTL\n"); + return IOCTL_FAILURE; + } + + struct allocinfo_content_id start_cont_id, end_cont_id; + struct allocinfo_get_at get_at_params; + const int max_retries = 10; + int retry_count = 0; + int status; + + /* + * __allocinfo_get_content_id may return different values if a kernel module was loaded + * between the two calls. If that happens, the data gathered cannot be considered consistent + * and hence needs to be fetched again to avoid flakiness. + */ + do { + if (__allocinfo_get_content_id(fd, &start_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + status = IOCTL_FAILURE; + break; + } + + memset(tags, 0, sizeof(*tags)); + memset(&get_at_params, 0, sizeof(get_at_params)); + memcpy(&get_at_params.filter, filter, sizeof(*filter)); + get_at_params.pos = start_pos; + if (__allocinfo_get_at(fd, &get_at_params)) { + ksft_print_msg("allocinfo_get_at failed\n"); + status = IOCTL_FAILURE; + break; + } + memcpy(&tags->tag[tags->count++], &get_at_params.data, sizeof(get_at_params.data)); + + while (tags->count < VEC_MAX_ENTRIES && + __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0) + tags->count++; + + if (__allocinfo_get_content_id(fd, &end_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + status = IOCTL_FAILURE; + break; + } + + if (start_cont_id.id == end_cont_id.id) { + status = IOCTL_SUCCESS; + } else { + ksft_print_msg("allocinfo_get_content_id mismatch, retrying...\n"); + status = IOCTL_INVALID_DATA; + } + } while (status == IOCTL_INVALID_DATA && retry_count++ < max_retries); + + close(fd); + return status; +} + +static int run_filter_test(const struct allocinfo_filter *filter) +{ + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + int ioctl_status; + int ret = KSFT_PASS; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (get_filtered_procfs_entries(procfs_entries, filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + + if (procfs_entries->count == 0) { + ksft_print_msg("No entries found in " ALLOCINFO_PROC ", skipping test\n"); + ret = KSFT_SKIP; + goto exit; + } + + ioctl_status = get_filtered_ioctl_entries(tags, filter, 0); + if (ioctl_status == IOCTL_INVALID_DATA) { + ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n"); + ret = KSFT_SKIP; + goto exit; + } + if (ioctl_status == IOCTL_FAILURE) { + ksft_print_msg("Error retrieving IOCTL entries.\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (!match_entries(procfs_entries, tags, false, false, true, true, true)) + ret = KSFT_FAIL; + +exit: + free(tags); + free(procfs_entries); + return ret; +} + +static int test_filename_filter(void) +{ + struct allocinfo_filter filter; + const char *target_filename = "mm/memory.c"; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FILENAME; + strncpy(filter.fields.filename, target_filename, ALLOCINFO_STR_SIZE); + + return run_filter_test(&filter); +} + +static int test_function_filter(void) +{ + struct allocinfo_filter filter; + const char *target_function = "dup_mm"; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION; + strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE); + + return run_filter_test(&filter); +} + +static int test_size_filter(void) +{ + int fd; + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + struct allocinfo_filter filter; + int ret = KSFT_PASS; + __u64 target_size, i, pos; + struct allocinfo_tag_data *found_tag = NULL; + const char *target_function = "do_init_module"; + struct allocinfo_content_id start_cont_id, end_cont_id; + int retry = 0; + const int max_retries = 10; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto freemem; + } + + fd = open(ALLOCINFO_PROC, O_RDONLY); + if (fd < 0) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC ": %s\n", strerror(errno)); + ret = KSFT_SKIP; + goto freemem; + } + + do { + found_tag = NULL; + pos = 0; + + if (__allocinfo_get_content_id(fd, &start_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + ret = KSFT_FAIL; + goto exit; + } + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION; + strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE); + + if (get_filtered_procfs_entries(procfs_entries, &filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + + if (procfs_entries->count == 0) { + ksft_print_msg("Function %s not found in procfs\n", target_function); + ret = KSFT_SKIP; + goto exit; + } + + target_size = procfs_entries->tag[0].counter.bytes; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE; + filter.min_size = target_size; + filter.max_size = target_size; + + while (1) { + struct allocinfo_get_at get_at_params; + + memset(&get_at_params, 0, sizeof(get_at_params)); + memcpy(&get_at_params.filter, &filter, sizeof(filter)); + get_at_params.pos = pos; + + if (__allocinfo_get_at(fd, &get_at_params)) + break; + + tags->count = 0; + memcpy(&tags->tag[tags->count++], &get_at_params.data, + sizeof(get_at_params.data)); + + while (tags->count < VEC_MAX_ENTRIES && + __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0) + tags->count++; + + for (i = 0; i < tags->count; i++) { + if (strcmp(tags->tag[i].tag.function, target_function) == 0) { + found_tag = &tags->tag[i]; + break; + } + } + + if (found_tag || tags->count < VEC_MAX_ENTRIES) + break; + + pos += tags->count; + } + + if (__allocinfo_get_content_id(fd, &end_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (start_cont_id.id == end_cont_id.id) + break; + + ksft_print_msg("Module load detected during size verification, retrying...\n"); + } while (retry++ < max_retries); + + if (start_cont_id.id == end_cont_id.id && !found_tag) { + ksft_print_msg("Entry with function %s not found in IOCTL results\n", + target_function); + ret = KSFT_FAIL; + } else if (start_cont_id.id != end_cont_id.id) { + ksft_print_msg("Failed to match content_ids for procfs and IOCTL, skipping...\n"); + ret = KSFT_SKIP; + } else if (found_tag && found_tag->counter.bytes != target_size) { + ksft_print_msg("IOCTL entry size %llu does not match target size %llu\n", + found_tag->counter.bytes, target_size); + ret = KSFT_FAIL; + } + +exit: + close(fd); +freemem: + free(tags); + free(procfs_entries); + return ret; +} + +static int test_lineno_filter(void) +{ + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + struct allocinfo_filter filter; + enum ioctl_ret ioctl_status; + int ret = KSFT_PASS; + __u64 target_lineno, i; + struct allocinfo_tag_data *target_tag; + bool found = false; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto exit; + } + + memset(&filter, 0, sizeof(filter)); + + if (get_filtered_procfs_entries(procfs_entries, &filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + if (procfs_entries->count == 0) { + ksft_print_msg("Could not retrieve procfs entries\n"); + ret = KSFT_SKIP; + goto exit; + } + /* + * We depend on the procfs results to determine the line number for the filter before + * making the ioctl query. Hence, we cannot reuse run_filter_test here. + */ + target_tag = &procfs_entries->tag[0]; + target_lineno = target_tag->tag.lineno; + + filter.mask |= ALLOCINFO_FILTER_MASK_LINENO; + filter.fields.lineno = target_lineno; + + ioctl_status = get_filtered_ioctl_entries(tags, &filter, 0); + if (ioctl_status == IOCTL_INVALID_DATA) { + ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n"); + ret = KSFT_SKIP; + goto exit; + } + if (ioctl_status == IOCTL_FAILURE) { + ksft_print_msg("Error retrieving IOCTL entries.\n"); + ret = KSFT_FAIL; + goto exit; + } + + for (i = 0; i < tags->count; i++) { + if (tags->tag[i].tag.lineno != target_lineno) { + ksft_print_msg("IOCTL entry %llu has incorrect lineno %llu.\n", + i, tags->tag[i].tag.lineno); + ret = KSFT_FAIL; + goto exit; + } + + if (strncmp(tags->tag[i].tag.function, target_tag->tag.function, + ALLOCINFO_STR_SIZE) == 0 && + strncmp(tags->tag[i].tag.filename, target_tag->tag.filename, + ALLOCINFO_STR_SIZE) == 0) + found = true; + } + + if (!found) { + ksft_print_msg("Original procfs entry not found in IOCTL lineno filter results.\n"); + ret = KSFT_FAIL; + } + +exit: + free(tags); + free(procfs_entries); + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + ksft_set_plan(4); + + ret = test_filename_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_filename_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_filename_filter\n"); + + ret = test_function_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_function_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_function_filter\n"); + + ret = test_size_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_size_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_size_filter\n"); + + ret = test_lineno_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_lineno_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_lineno_filter\n"); + + ksft_finished(); +} diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h index 8ebb2b4d4ec0..5d39c709ac7a 100644 --- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h +++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h @@ -7,6 +7,7 @@ #endif #define MB(x) (x << 20) +#define GB(x) ((unsigned long long)(x) << 30) #define NSEC_PER_USEC 1000L #define USEC_PER_SEC 1000000L diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 49b36ee79160..f7b4c4370db6 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -20,6 +20,7 @@ static int page_size; #define PATH_ZSWAP "/sys/module/zswap" #define PATH_ZSWAP_ENABLED "/sys/module/zswap/parameters/enabled" +#define PATH_ZSWAP_STORED_PAGES "/sys/kernel/debug/zswap/stored_pages" static int read_int(const char *path, size_t *value) { @@ -55,7 +56,7 @@ static int read_min_free_kb(size_t *value) static int get_zswap_stored_pages(size_t *value) { - return read_int("/sys/kernel/debug/zswap/stored_pages", value); + return read_int(PATH_ZSWAP_STORED_PAGES, value); } static long get_cg_wb_count(const char *cg) @@ -570,8 +571,16 @@ static int test_no_kmem_bypass(const char *root) /* Read sys info and compute test values accordingly */ if (sysinfo(&sys_info) != 0) return KSFT_FAIL; - if (sys_info.totalram > 5000000000) + if (sys_info.totalram > GB(4)) { + ksft_print_msg( + "requires less than 4GB total ram, sys_info.totalram: %.1fGB\n", + (double)sys_info.totalram / GB(1)); return KSFT_SKIP; + } + if (access(PATH_ZSWAP_STORED_PAGES, R_OK)) { + ksft_print_msg("debugfs not mounted at /sys/kernel/debug\n"); + return KSFT_SKIP; + } values = mmap(0, sizeof(struct no_kmem_bypass_child_args), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (values == MAP_FAILED) diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore index 9ccd9e1447e6..fcd892ed21e3 100644 --- a/tools/testing/selftests/mm/.gitignore +++ b/tools/testing/selftests/mm/.gitignore @@ -1,68 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only -cow -hugepage-mmap -hugepage-mremap -hugepage-shm -hugepage-vmemmap -hugetlb-mmap -hugetlb-mremap -hugetlb-shm -hugetlb-vmemmap -hugetlb-madvise -hugetlb-read-hwpoison -hugetlb-soft-offline -khugepaged -map_hugetlb -map_populate -thuge-gen -compaction_test -memory-failure -migration -mlock2-tests -mrelease_test -mremap_dontunmap -mremap_test -on-fault-limit -transhuge-stress -pagemap_ioctl -pfnmap -process_madv -*.tmp* -protection_keys -protection_keys_32 -protection_keys_64 -madv_populate -uffd-stress -uffd-unit-tests -uffd-wp-mremap -mlock-intersect-test -mlock-random-test -virtual_address_range -gup_test -va_128TBswitch -map_fixed_noreplace -write_to_hugetlbfs -hmm-tests -memfd_secret -soft-dirty -split_huge_page_test -ksm_tests -local_config.h -local_config.mk -ksm_functional_tests -mdwe_test -gup_longterm -mkdirty -va_high_addr_switch -hugetlb_fault_after_madv -hugetlb_madv_vs_map -mseal_test -droppable -hugetlb_dio -pkey_sighandler_tests_32 -pkey_sighandler_tests_64 -guard-regions -merge -prctl_thp_disable -rmap -folio_split_race_test +* +!/**/ +!*.c +!*.h +!*.sh +!.gitignore +!Makefile +!config +!settings diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 277a141d662e..2d5366196e30 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -5,7 +5,7 @@ # script so kunit knows to run it, and add it to the list below. # If you do not YOUR TESTS WILL NOT RUN IN THE CI. -LOCAL_HDRS += $(selfdir)/mm/local_config.h $(top_srcdir)/mm/gup_test.h +LOCAL_HDRS += $(selfdir)/mm/local_config.h_gen $(top_srcdir)/mm/gup_test.h LOCAL_HDRS += $(selfdir)/mm/mseal_helpers.h include local_config.mk @@ -149,6 +149,7 @@ TEST_PROGS += ksft_gup_test.sh TEST_PROGS += ksft_hmm.sh TEST_PROGS += ksft_hugetlb.sh TEST_PROGS += ksft_hugevm.sh +TEST_PROGS += ksft_kmemleak_confirm.sh TEST_PROGS += ksft_kmemleak_dedup.sh TEST_PROGS += ksft_ksm.sh TEST_PROGS += ksft_ksm_numa.sh @@ -261,11 +262,11 @@ $(OUTPUT)/migration: LDLIBS += -lnuma $(OUTPUT)/rmap: LDLIBS += -lnuma -local_config.mk local_config.h: check_config.sh +local_config.mk local_config.h_gen: check_config.sh $(call msg,CHK,config,$@) $(Q)CC="$(CC)" CFLAGS="$(CFLAGS)" ./check_config.sh -EXTRA_CLEAN += local_config.mk local_config.h +EXTRA_CLEAN += local_config.mk local_config.h_gen ifeq ($(IOURING_EXTRA_LIBS),) all: warn_missing_liburing diff --git a/tools/testing/selftests/mm/check_config.sh b/tools/testing/selftests/mm/check_config.sh index 32beaefe279e..1c603261e93d 100755 --- a/tools/testing/selftests/mm/check_config.sh +++ b/tools/testing/selftests/mm/check_config.sh @@ -4,7 +4,7 @@ # Probe for libraries and create header files to record the results. Both C # header files and Makefile include fragments are created. -OUTPUT_H_FILE=local_config.h +OUTPUT_H_FILE=local_config.h_gen OUTPUT_MKFILE=local_config.mk tmpname=$(mktemp) diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 5b582588e015..30d4ace7155a 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -29,30 +29,34 @@ struct map_list { int read_memory_info(unsigned long *memfree, unsigned long *hugepagesize) { - char buffer[256] = {0}; - char *cmd = "cat /proc/meminfo | grep -i memfree | grep -o '[0-9]*'"; - FILE *cmdfile = popen(cmd, "r"); + char buffer[256]; + int found = 0; + FILE *file; + int ret = -1; - if (!(fgets(buffer, sizeof(buffer), cmdfile))) { - ksft_print_msg("Failed to read meminfo: %s\n", strerror(errno)); + file = fopen("/proc/meminfo", "r"); + if (!file) { + ksft_print_msg("Failed to open /proc/meminfo: %s\n", + strerror(errno)); return -1; } - pclose(cmdfile); - - *memfree = atoll(buffer); - cmd = "cat /proc/meminfo | grep -i hugepagesize | grep -o '[0-9]*'"; - cmdfile = popen(cmd, "r"); - - if (!(fgets(buffer, sizeof(buffer), cmdfile))) { - ksft_print_msg("Failed to read meminfo: %s\n", strerror(errno)); - return -1; + while (fgets(buffer, sizeof(buffer), file) && found != 2) { + if (sscanf(buffer, "MemFree: %lu kB", memfree) == 1 || + sscanf(buffer, "Hugepagesize: %lu kB", hugepagesize) == 1) + found++; } - pclose(cmdfile); - *hugepagesize = atoll(buffer); + if (ferror(file)) + ksft_print_msg("Failed to read /proc/meminfo: %s\n", + strerror(errno)); + else if (found != 2) + ksft_print_msg("Failed to parse /proc/meminfo\n"); + else + ret = 0; - return 0; + fclose(file); + return ret; } int prereq(void) diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 0c627ea89ff7..8aa5249d9bef 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -21,7 +21,7 @@ #include <sys/wait.h> #include <linux/memfd.h> -#include "local_config.h" +#include "local_config.h_gen" #ifdef LOCAL_CONFIG_HAVE_LIBURING #include <liburing.h> #endif /* LOCAL_CONFIG_HAVE_LIBURING */ @@ -1718,8 +1718,13 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) /* File consists of a single page filled with zeroes. */ if (fallocate(fd, 0, 0, pagesize)) { - ksft_perror("fallocate() failed"); - log_test_result(KSFT_FAIL); + if (errno == EOPNOTSUPP) { + ksft_print_msg("fallocate() not supported by filesystem\n"); + log_test_result(KSFT_SKIP); + } else { + ksft_perror("fallocate() failed"); + log_test_result(KSFT_FAIL); + } goto close; } diff --git a/tools/testing/selftests/mm/folio_split_race_test.c b/tools/testing/selftests/mm/folio_split_race_test.c index 6329e37fff4c..45b84f7b364e 100644 --- a/tools/testing/selftests/mm/folio_split_race_test.c +++ b/tools/testing/selftests/mm/folio_split_race_test.c @@ -182,7 +182,7 @@ static uint64_t run_iteration(void) for (i = 0; i < TOTAL_PAGES; i++) fill_page(mmap_base, i); - if (!check_huge_shmem(mmap_base, NR_PMD_PAGE, pmd_pagesize)) + if (!check_huge_shmem(mmap_base, FILE_SIZE, NR_PMD_PAGE, pmd_pagesize)) ksft_exit_fail_msg("No shmem THP is allocated\n"); if (pthread_barrier_init(&ctl.barrier, NULL, NUM_READER_THREADS + 1) != 0) diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index b21df3040b1c..5c8ec3ca75d7 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -1912,7 +1912,7 @@ TEST_F(guard_regions, hole_punch) { const unsigned long page_size = self->page_size; char *ptr; - int i; + int i, ret; if (variant->backing == ANON_BACKED) SKIP(return, "Truncation test specific to file-backed"); @@ -1944,8 +1944,12 @@ TEST_F(guard_regions, hole_punch) } /* Now hole punch the guarded region. */ - ASSERT_EQ(madvise(&ptr[3 * page_size], 4 * page_size, - MADV_REMOVE), 0); + ret = madvise(&ptr[3 * page_size], 4 * page_size, MADV_REMOVE); + if (ret == -1 && errno == EOPNOTSUPP) { + ASSERT_EQ(munmap(ptr, 10 * page_size), 0); + SKIP(return, "MADV_REMOVE not supported by filesystem"); + } + ASSERT_EQ(ret, 0); /* Ensure guard regions remain. */ for (i = 0; i < 10; i++) { diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index c03b4f8910c0..510de93be681 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -21,7 +21,7 @@ #include <linux/magic.h> #include <linux/memfd.h> -#include "local_config.h" +#include "local_config.h_gen" #ifdef LOCAL_CONFIG_HAVE_LIBURING #include <liburing.h> #endif /* LOCAL_CONFIG_HAVE_LIBURING */ diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 10e8dedcb087..1d2d6bd72fd2 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -26,9 +26,11 @@ #define BASE_ADDR ((void *)(1UL << 30)) static unsigned long hpage_pmd_size; +static int hpage_pmd_order; static unsigned long page_size; static int hpage_pmd_nr; static int anon_order; +static int collapse_order; #define PID_SMAPS "/proc/self/smaps" #define TEST_FILE "collapse_test_file" @@ -51,7 +53,7 @@ struct mem_ops { void *(*setup_area)(int nr_hpages); void (*cleanup_area)(void *p, unsigned long size); void (*fault)(void *p, unsigned long start, unsigned long end); - bool (*check_huge)(void *addr, int nr_hpages); + bool (*check_huge)(void *addr, size_t len, int nr_hpages, unsigned long hpage_size); const char *name; }; @@ -69,6 +71,7 @@ struct collapse_context { }; static struct collapse_context *khugepaged_context; +static struct collapse_context *mthp_khugepaged_context; static struct collapse_context *madvise_context; struct file_info { @@ -121,7 +124,8 @@ static void get_finfo(const char *dir) char *str, *end; finfo.dir = dir; - stat(finfo.dir, &path_stat); + if (stat(finfo.dir, &path_stat)) + ksft_exit_fail_perror("stat()"); if (!S_ISDIR(path_stat.st_mode)) ksft_exit_fail_msg("%s: Not a directory (%s)\n", __func__, finfo.dir); if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE, @@ -138,8 +142,8 @@ static void get_finfo(const char *dir) major(path_stat.st_dev), minor(path_stat.st_dev)) >= sizeof(path)) ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); - if (read_file(path, buf, sizeof(buf)) < 0) - ksft_exit_fail_perror("read_file(read_num)"); + if (!read_file(path, buf, sizeof(buf))) + ksft_exit_fail_perror("read_file(uevent)"); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, @@ -276,7 +280,7 @@ static void *alloc_hpage(struct mem_ops *ops) ksft_print_msg("Allocate huge page..."); if (madvise_collapse_retry(p, hpage_pmd_size)) ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); - if (!ops->check_huge(p, 1)) + if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); @@ -310,9 +314,10 @@ static void anon_fault(void *p, unsigned long start, unsigned long end) fill_memory(p, start, end); } -static bool anon_check_huge(void *addr, int nr_hpages) +static bool anon_check_huge(void *addr, size_t len, int nr_hpages, + unsigned long hpage_size) { - return check_huge_anon(addr, nr_hpages, hpage_pmd_size); + return check_huge_anon(addr, len, nr_hpages, hpage_size); } static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) @@ -412,13 +417,14 @@ static void file_fault_write(void *p, unsigned long start, unsigned long end) ksft_exit_fail_perror("madvise(MADV_POPULATE_WRITE)"); } -static bool file_check_huge(void *addr, int nr_hpages) +static bool file_check_huge(void *addr, size_t len, int nr_hpages, + unsigned long hpage_size) { switch (finfo.type) { case VMA_FILE: - return check_huge_file(addr, nr_hpages, hpage_pmd_size); + return check_huge_file(addr, len, nr_hpages, hpage_size); case VMA_SHMEM: - return check_huge_shmem(addr, nr_hpages, hpage_pmd_size); + return check_huge_shmem(addr, len, nr_hpages, hpage_size); default: exit(EXIT_FAILURE); return false; @@ -448,9 +454,10 @@ static void shmem_cleanup_area(void *p, unsigned long size) close(finfo.fd); } -static bool shmem_check_huge(void *addr, int nr_hpages) +static bool shmem_check_huge(void *addr, size_t len, int nr_hpages, + unsigned long hpage_size) { - return check_huge_shmem(addr, nr_hpages, hpage_pmd_size); + return check_huge_shmem(addr, len, nr_hpages, hpage_size); } static struct mem_ops __anon_ops = { @@ -533,7 +540,7 @@ static void __madvise_collapse(const char *msg, char *p, int nr_hpages, ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size); if (((bool)ret) == expect) fail("Fail: Bad return value"); - else if (!ops->check_huge(p, expect ? nr_hpages : 0)) + else if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, expect ? nr_hpages : 0, hpage_pmd_size)) fail("Fail: check_huge()"); else success("OK"); @@ -545,30 +552,31 @@ static void madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { /* Sanity check */ - if (!ops->check_huge(p, 0)) + if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, 0, hpage_pmd_size)) ksft_exit_fail_msg("Unexpected huge page\n"); __madvise_collapse(msg, p, nr_hpages, ops, expect); } #define TICK 500000 -static bool wait_for_scan(const char *msg, char *p, int nr_hpages, - struct mem_ops *ops) +static bool wait_for_scan(const char *msg, char *p, size_t len, + int nr_hpages, int collap_order, struct mem_ops *ops) { + unsigned long hpage_size = page_size << collap_order; int full_scans; int timeout = 6; /* 3 seconds */ /* Sanity check */ - if (!ops->check_huge(p, 0)) + if (!ops->check_huge(p, len, 0, hpage_size)) ksft_exit_fail_msg("Unexpected huge page\n"); - madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); + madvise(p, len, MADV_HUGEPAGE); /* Wait until the second full_scan completed */ full_scans = thp_read_num("khugepaged/full_scans") + 2; ksft_print_msg("%s...", msg); while (timeout--) { - if (ops->check_huge(p, nr_hpages)) + if (ops->check_huge(p, len, nr_hpages, hpage_size)) break; if (thp_read_num("khugepaged/full_scans") >= full_scans) break; @@ -582,6 +590,8 @@ static bool wait_for_scan(const char *msg, char *p, int nr_hpages, static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { + size_t len = nr_hpages * hpage_pmd_size; + /* * read&write file collapse fails since khugepaged does not flush * the target dirty folios @@ -589,7 +599,7 @@ static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, if (!is_tmpfs(ops) && ops == &__read_write_file_write_ops) expect = false; - if (wait_for_scan(msg, p, nr_hpages, ops)) { + if (wait_for_scan(msg, p, len, nr_hpages, hpage_pmd_order, ops)) { if (expect) fail("Timeout"); else @@ -605,10 +615,54 @@ static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, if (ops != &__anon_ops) ops->fault(p, 0, nr_hpages * hpage_pmd_size); - if (ops->check_huge(p, expect ? nr_hpages : 0)) + if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_pmd_size)) + success("OK"); + else + fail("Fail"); +} + +static void mthp_khugepaged_collapse(const char *msg, char *p, int nr_hpages, + struct mem_ops *ops, bool expect) +{ + unsigned long hpage_size = page_size << collapse_order; + struct thp_settings settings = *thp_current_settings(); + /* mTHP collpase only allocates PMD sized memory */ + size_t len = hpage_pmd_size; + + /* Set mTHP setting for mTHP collapse */ + if (ops == &__anon_ops) { + settings.thp_enabled = THP_NEVER; + settings.hugepages[collapse_order].enabled = THP_MADVISE; + } + + thp_push_settings(&settings); + + if (wait_for_scan(msg, p, len, nr_hpages, collapse_order, ops)) { + if (expect) + fail("Timeout"); + else + success("OK"); + + /* Restore THP settings for mTHP collapse. */ + thp_pop_settings(); + return; + } + + /* + * For file and shmem memory, khugepaged only retracts pte entries after + * putting the new hugepage in the page cache. The hugepage must be + * subsequently refaulted to install the pmd mapping for the mm. + */ + if (ops != &__anon_ops) + ops->fault(p, 0, nr_hpages * hpage_size); + + if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_size)) success("OK"); else fail("Fail"); + + /* Restore THP settings for mTHP collapse. */ + thp_pop_settings(); } static struct collapse_context __khugepaged_context = { @@ -617,6 +671,12 @@ static struct collapse_context __khugepaged_context = { .name = "khugepaged", }; +static struct collapse_context __mthp_khugepaged_context = { + .collapse = &mthp_khugepaged_collapse, + .enforce_pte_scan_limits = true, + .name = "mthp_khugepaged", +}; + static struct collapse_context __madvise_context = { .collapse = &madvise_collapse, .enforce_pte_scan_limits = false, @@ -634,7 +694,7 @@ static void alloc_at_fault(void) p = alloc_mapping(1); *p = 1; ksft_print_msg("Allocate huge page on fault..."); - if (check_huge_anon(p, 1, hpage_pmd_size)) + if (check_huge_anon(p, hpage_pmd_size, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -643,7 +703,7 @@ static void alloc_at_fault(void) madvise(p, page_size, MADV_DONTNEED); ksft_print_msg("Split huge PMD on MADV_DONTNEED..."); - if (check_huge_anon(p, 0, hpage_pmd_size)) + if (check_huge_anon(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -655,10 +715,17 @@ static void alloc_at_fault(void) static void collapse_full(struct collapse_context *c, struct mem_ops *ops) { void *p; - int nr_hpages = 4; + int nr_pmds = 4, nr_hpages = 4; unsigned long size = nr_hpages * hpage_pmd_size; - p = ops->setup_area(nr_hpages); + /* Only try 1 PMD sized range for mTHP collapse. */ + if (c == &__mthp_khugepaged_context) { + nr_pmds = 1; + nr_hpages = 1 << (hpage_pmd_order - collapse_order); + size = hpage_pmd_size; + } + + p = ops->setup_area(nr_pmds); ops->fault(p, 0, size); c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, ops, true); @@ -670,10 +737,31 @@ static void collapse_full(struct collapse_context *c, struct mem_ops *ops) static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) { + int nr_hpages = 1; + void *p; + + if (c == &__mthp_khugepaged_context) + nr_hpages = 1 << (hpage_pmd_order - collapse_order); + + p = ops->setup_area(1); + c->collapse("Do not collapse empty PTE table", p, nr_hpages, ops, false); + ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); +} + +static void collapse_single_mthp(struct collapse_context *c, struct mem_ops *ops) +{ + unsigned long hpage_size = page_size << collapse_order; void *p; p = ops->setup_area(1); - c->collapse("Do not collapse empty PTE table", p, 1, ops, false); + /* + * Only fault collapse_order sized ranges, and only check 1 + * collapse_order sized huge page. + */ + ops->fault(p, 0, hpage_size); + c->collapse("Collapse PTE table with half PTE entries present", + p, 1, ops, true); ops->cleanup_area(p, hpage_pmd_size); ksft_test_result_report(exit_status, "%s\n", __func__); } @@ -815,7 +903,7 @@ static void collapse_single_pte_entry_compound(struct collapse_context *c, struc madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); ksft_print_msg("Split huge page leaving single PTE mapping compound page..."); madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -836,7 +924,7 @@ static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops ksft_print_msg("Split huge page leaving single PTE page table full of compound pages..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -858,7 +946,7 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops for (i = 0; i < hpage_pmd_nr; i++) { madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); ops->fault(BASE_ADDR, 0, hpage_pmd_size); - if (!ops->check_huge(BASE_ADDR, 1)) + if (!ops->check_huge(BASE_ADDR, hpage_pmd_size, 1, hpage_pmd_size)) ksft_exit_fail_msg("Failed to allocate huge page\n"); madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE); @@ -881,7 +969,7 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops ops->cleanup_area(BASE_ADDR, hpage_pmd_size); ops->fault(p, 0, hpage_pmd_size); - if (!ops->check_huge(p, 1)) + if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -903,7 +991,7 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) ksft_print_msg("Allocate small page..."); ops->fault(p, 0, page_size); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -911,7 +999,7 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) ksft_print_msg("Share small page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -929,7 +1017,7 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) exit_status = WEXITSTATUS(wstatus); ksft_print_msg("Check if parent still has small page..."); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -947,7 +1035,7 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o ksft_print_msg("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - if (ops->check_huge(p, 1)) + if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -955,7 +1043,7 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o ksft_print_msg("Split huge page PMD in child process..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -976,7 +1064,7 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o exit_status = WEXITSTATUS(wstatus); ksft_print_msg("Check if parent still has huge page..."); - if (ops->check_huge(p, 1)) + if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -995,7 +1083,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops ksft_print_msg("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - if (ops->check_huge(p, 1)) + if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -1003,7 +1091,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops ksft_print_msg("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -1016,7 +1104,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -1034,7 +1122,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops exit_status = WEXITSTATUS(wstatus); ksft_print_msg("Check if parent still has huge page..."); - if (ops->check_huge(p, 1)) + if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -1075,8 +1163,8 @@ static void madvise_retracted_page_tables(struct collapse_context *c, ops->fault(p, 0, size); /* Let khugepaged collapse and leave pmd cleared */ - if (wait_for_scan("Collapse and leave PMD cleared", p, nr_hpages, - ops)) { + if (wait_for_scan("Collapse and leave PMD cleared", p, size, nr_hpages, + hpage_pmd_order, ops)) { fail("Timeout"); return; } @@ -1092,17 +1180,19 @@ static void usage(void) { fprintf(stderr, "\nUsage: ./khugepaged [OPTIONS] <test type> [dir]\n\n"); fprintf(stderr, "\t<test type>\t: <context>:<mem_type>\n"); - fprintf(stderr, "\t<context>\t: [all|khugepaged|madvise]\n"); + fprintf(stderr, "\t<context>\t: [all|khugepaged|mthp_khugepaged|madvise]\n"); fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n"); fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n"); fprintf(stderr, "\n\t\"file,all\" mem_type requires a file system\n"); fprintf(stderr, "\twith PMD-sized large folio support\n"); fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n"); fprintf(stderr, "\tmounted with huge=advise option for khugepaged tests to work\n"); + fprintf(stderr, "\n\tmthp_khugepaged only supports anon mem_type now.\n"); fprintf(stderr, "\n\tSupported Options:\n"); fprintf(stderr, "\t\t-h: This help message.\n"); fprintf(stderr, "\t\t-s: mTHP size, expressed as page order.\n"); fprintf(stderr, "\t\t Defaults to 0. Use this size for anon or shmem allocations.\n"); + fprintf(stderr, "\t\t-c: collapse order for mTHP collapse, expressed as page order.\n"); exit(1); } @@ -1112,11 +1202,14 @@ static void parse_test_type(int argc, char **argv) char *buf; const char *token; - while ((opt = getopt(argc, argv, "s:h")) != -1) { + while ((opt = getopt(argc, argv, "s:c:h")) != -1) { switch (opt) { case 's': anon_order = atoi(optarg); break; + case 'c': + collapse_order = atoi(optarg); + break; case 'h': default: usage(); @@ -1142,6 +1235,10 @@ static void parse_test_type(int argc, char **argv) madvise_context = &__madvise_context; } else if (!strcmp(token, "khugepaged")) { khugepaged_context = &__khugepaged_context; + } else if (!strcmp(token, "mthp_khugepaged")) { + mthp_khugepaged_context = &__mthp_khugepaged_context; + if (collapse_order <= 0 || collapse_order >= hpage_pmd_order) + usage(); } else if (!strcmp(token, "madvise")) { madvise_context = &__madvise_context; } else { @@ -1157,14 +1254,20 @@ static void parse_test_type(int argc, char **argv) read_write_file_write_ops = &__read_write_file_write_ops; anon_ops = &__anon_ops; shmem_ops = &__shmem_ops; + if (mthp_khugepaged_context) + usage(); } else if (!strcmp(buf, "anon")) { anon_ops = &__anon_ops; } else if (!strcmp(buf, "file")) { read_only_file_ops = &__read_only_file_ops; read_write_file_read_ops = &__read_write_file_read_ops; read_write_file_write_ops = &__read_write_file_write_ops; + if (mthp_khugepaged_context) + usage(); } else if (!strcmp(buf, "shmem")) { shmem_ops = &__shmem_ops; + if (mthp_khugepaged_context) + usage(); } else { usage(); } @@ -1207,7 +1310,6 @@ static int nr_test_cases; int main(int argc, char **argv) { - int hpage_pmd_order; struct thp_settings default_settings = { .thp_enabled = THP_MADVISE, .thp_defrag = THP_DEFRAG_ALWAYS, @@ -1233,10 +1335,6 @@ int main(int argc, char **argv) if (!thp_is_enabled()) ksft_exit_skip("Transparent Hugepages not available\n"); - parse_test_type(argc, argv); - - setbuf(stdout, NULL); - page_size = getpagesize(); hpage_pmd_size = read_pmd_pagesize(); if (!hpage_pmd_size) @@ -1244,6 +1342,10 @@ int main(int argc, char **argv) hpage_pmd_nr = hpage_pmd_size / page_size; hpage_pmd_order = __builtin_ctz(hpage_pmd_nr); + parse_test_type(argc, argv); + + setbuf(stdout, NULL); + default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1; default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8; default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2; @@ -1261,6 +1363,7 @@ int main(int argc, char **argv) TEST(collapse_full, khugepaged_context, read_write_file_read_ops); TEST(collapse_full, khugepaged_context, read_write_file_write_ops); TEST(collapse_full, khugepaged_context, shmem_ops); + TEST(collapse_full, mthp_khugepaged_context, anon_ops); TEST(collapse_full, madvise_context, anon_ops); TEST(collapse_full, madvise_context, read_only_file_ops); TEST(collapse_full, madvise_context, read_write_file_read_ops); @@ -1268,8 +1371,11 @@ int main(int argc, char **argv) TEST(collapse_full, madvise_context, shmem_ops); TEST(collapse_empty, khugepaged_context, anon_ops); + TEST(collapse_empty, mthp_khugepaged_context, anon_ops); TEST(collapse_empty, madvise_context, anon_ops); + TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); + TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops); diff --git a/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh new file mode 100755 index 000000000000..72ded5e6794c --- /dev/null +++ b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Functional test for kmemleak's N-consecutive-scan leak confirmation +# (the min_unref_scans module parameter). +# +# kmemleak only reports an object once it has stayed unreferenced for +# min_unref_scans consecutive scans. A threshold of 1 reports on the first +# scan (historical behaviour); higher values filter transient false +# positives where a live object's only reference is briefly invisible to a +# single scan (e.g. an RCU tree update in flight while the scan runs). The +# test loads samples/kmemleak's helper module to create orphan allocations +# and, counting only those orphans (matched by their [kmemleak_test] +# backtrace so unrelated leaks already present on the system are ignored), +# checks that: +# - a freshly allocated object is greyed on its first scan (its checksum +# settles then), so nothing can be reported before that priming scan; +# each case below primes once first, +# - at min_unref_scans=1 one scan after priming reports the orphans, +# - raising the threshold to 2 needs two scans after priming: one is not +# enough, the second reports, +# - the parameter reads back what was written. +# +# The "one post-prime scan is not enough at min_unref_scans=2" check is the +# core regression test: raising min_unref_scans must push the report +# strictly later. Like ksft_kmemleak_dedup.sh, if the module yields no +# detectable orphan at all in the running environment the test skips rather +# than failing. +# +# Author: Breno Leitao <leitao@debian.org> + +# KTAP output helpers (ktap_skip_all, ktap_exit_fail_msg, ktap_test_pass, ...). +DIR="$(dirname "$(readlink -f "$0")")" +# shellcheck source=../kselftest/ktap_helpers.sh +source "${DIR}"/../kselftest/ktap_helpers.sh + +KMEMLEAK=/sys/kernel/debug/kmemleak +PARAM=/sys/module/kmemleak/parameters/min_unref_scans +MODULE=kmemleak-test +AGE=6 # seconds; must exceed kmemleak's 5s minimum object age + +ktap_print_header + +[ "$(id -u)" -eq 0 ] || { ktap_skip_all "must run as root"; exit "$KSFT_SKIP"; } +[ -r "$KMEMLEAK" ] || + { ktap_skip_all "no kmemleak debugfs (CONFIG_DEBUG_KMEMLEAK)"; exit "$KSFT_SKIP"; } +[ -w "$PARAM" ] || + { ktap_skip_all "min_unref_scans module parameter not present"; exit "$KSFT_SKIP"; } +modinfo "$MODULE" >/dev/null 2>&1 || + { ktap_skip_all "$MODULE not built (CONFIG_SAMPLE_KMEMLEAK)"; exit "$KSFT_SKIP"; } + +# kmemleak can be present but disabled at runtime (kmemleak=off boot arg, +# or it self-disabled after an internal error); a "scan" then returns +# EPERM. Probe once and skip if so. +echo scan > "$KMEMLEAK" 2>/dev/null || + { ktap_skip_all "kmemleak is disabled (check dmesg or kmemleak= boot arg)"; exit "$KSFT_SKIP"; } + +prev=$(cat "$PARAM") +# shellcheck disable=SC2317 # invoked indirectly via trap +cleanup() { + echo "$prev" > "$PARAM" 2>/dev/null # restore the parameter + echo scan=on > "$KMEMLEAK" 2>/dev/null # re-enable auto scan + rmmod "$MODULE" 2>/dev/null + echo clear > "$KMEMLEAK" 2>/dev/null +} +trap cleanup EXIT + +# Stop the automatic scan thread: only our manual scans should advance an +# object's consecutive-unreferenced run. An auto scan landing between two +# manual scans would change the result and make the test flaky. +echo scan=off > "$KMEMLEAK" 2>/dev/null + +# Create a fresh, aged set of orphan objects from the helper module's init +# path (its kmalloc/vmalloc/percpu allocations are dropped right away). +# Pre-existing reported leaks are greyed first ("clear") so only our +# orphans are counted. The module is left loaded on purpose: once it is +# unloaded its symbols are gone, so the orphan backtraces no longer resolve +# to [kmemleak_test] and could not be matched below. +gen_orphans() { + rmmod "$MODULE" 2>/dev/null + echo clear > "$KMEMLEAK" + modprobe "$MODULE" || + { ktap_skip_all "failed to load $MODULE"; exit "$KSFT_SKIP"; } + sleep "$AGE" +} + +scan() { echo scan > "$KMEMLEAK"; } + +# Number of helper-module orphans currently reported by kmemleak. Matching +# the module's own backtrace ([kmemleak_test]) keeps the count immune to +# unrelated leaks on the running system. kmemleak only lists an object here +# once it has been reported, so this reflects the confirmation gating. +count_orphans() { + c=$(grep -c '\[kmemleak_test\]' "$KMEMLEAK" 2>/dev/null) + echo "${c:-0}" +} + +# 0) the parameter reads back what was written. +echo 3 > "$PARAM" +[ "$(cat "$PARAM")" = "3" ] || ktap_exit_fail_msg "min_unref_scans did not read back as 3" + +# Priming scan: kmemleak greys a freshly allocated object on its first scan +# (its checksum settles then), so nothing can be reported until a second +# scan. Every case below runs this priming scan before counting. +prime() { scan; } + +# 1) min_unref_scans=1: one scan after priming reports the orphans. This +# also establishes that the helper produces detectable orphans here. +echo 1 > "$PARAM" +gen_orphans +prime +scan +first=$(count_orphans) +[ "$first" -gt 0 ] || + { ktap_skip_all "$MODULE produced no detectable orphans (cannot test min_unref_scans)"; exit "$KSFT_SKIP"; } + +# 2) min_unref_scans=2: after priming, one scan is not enough (still +# gated), the second reports. The gated-scan-zero check is the core +# regression. +echo 2 > "$PARAM" +gen_orphans +prime +scan; s1=$(count_orphans) +scan; s2=$(count_orphans) +[ "$s1" -eq 0 ] || ktap_exit_fail_msg "min_unref_scans=2: $s1 orphan(s) after 1 post-prime scan (must be 0)" +[ "$s2" -gt 0 ] || ktap_exit_fail_msg "min_unref_scans=2: no report after 2 post-prime scans (false negative)" + +ktap_set_plan 1 +ktap_test_pass "min_unref_scans=1 reported $first orphan(s) one scan after priming; =2 held them one scan longer ($s1 after one scan, $s2 after two); param read-back ok" +ktap_finished diff --git a/tools/testing/selftests/mm/memory-failure.c b/tools/testing/selftests/mm/memory-failure.c index 1a5a32e22cce..f3cb578b1609 100644 --- a/tools/testing/selftests/mm/memory-failure.c +++ b/tools/testing/selftests/mm/memory-failure.c @@ -287,8 +287,10 @@ TEST_F(memory_failure, clean_pagecache) if (fd < 0) SKIP(return, "failed to open test file.\n"); fs_type = get_fs_type(fd); - if (!fs_type || fs_type == TMPFS_MAGIC) + if (!fs_type || fs_type == TMPFS_MAGIC) { + close(fd); SKIP(return, "unsupported filesystem :%x\n", fs_type); + } addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); @@ -327,8 +329,16 @@ TEST_F(memory_failure, dirty_pagecache) if (fd < 0) SKIP(return, "failed to open test file.\n"); fs_type = get_fs_type(fd); - if (!fs_type || fs_type == TMPFS_MAGIC) + /* + * MADV_HARD poisoning of dirty page-cache data records an expected + * -EIO in the file mapping. NFS reports this error on close(), so + * skip this variant. + */ + if (!fs_type || fs_type == TMPFS_MAGIC || + (fs_type == NFS_SUPER_MAGIC && variant->type == MADV_HARD)) { + close(fd); SKIP(return, "unsupported filesystem :%x\n", fs_type); + } addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index 519e5ac02db7..52b8727b6628 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -1305,6 +1305,63 @@ TEST_F(merge, merge_vmas_with_mseal) ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size); } +TEST_F(merge, anon_and_page_offset_mismatch_memfd) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd; + + /* Create a 10 page memfd descriptor. */ + fd = memfd_create("anon_page_offset_test", MFD_CLOEXEC); + ASSERT_NE(fd, -1); + ASSERT_EQ(ftruncate(fd, 10 * page_size), 0); + + /* Map a region using the memfd at page offset 0. */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd, 0); + ASSERT_NE(ptr, MAP_FAILED); + + /* + * Map another separately and trigger a CoW fault at page offset 5: + * + * |-----------| |---------| + * | unfaulted | | faulted | + * |-----------| |---------| + */ + ptr2 = mmap(&carveout[10 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd, 5 * page_size); + ASSERT_NE(ptr2, MAP_FAILED); + ptr2[0] = 'x'; + + /* + * Now move it in place: + * + * |----------| + * | | + * v | + * |-----------| |---------| + * | unfaulted | | faulted | + * |-----------| |---------| + * + * Because the anonymous page offset of the faulted region is now + * &carveout[10 * page_size], despite the two regions being mergeable + * due to file page offset, they are NOT mergeable due to anonymous + * page offset. + */ + ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, + MREMAP_MAYMOVE | MREMAP_FIXED, + &carveout[5 * page_size]); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they did not merge. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size); +} + TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev) { struct procmap_fd *procmap = &self->procmap; diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index 29f7492453d4..f19d53c69576 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -7,7 +7,7 @@ #include "kselftest_harness.h" #include "hugepage_settings.h" -#include <strings.h> +#include <string.h> #include <pthread.h> #include <numa.h> #include <numaif.h> @@ -20,7 +20,6 @@ #define TWOMEG (2<<20) #define RUNTIME (20) -#define MAX_RETRIES 100 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) HUGETLB_SETUP_DEFAULT_PAGES(1) @@ -110,7 +109,7 @@ int migrate(uint64_t *ptr, int n1, int n2) int ret, tmp; int status = 0; struct timespec ts1, ts2; - int failures = 0; + int success = 0; if (clock_gettime(CLOCK_MONOTONIC, &ts1)) return -1; @@ -119,29 +118,33 @@ int migrate(uint64_t *ptr, int n1, int n2) if (clock_gettime(CLOCK_MONOTONIC, &ts2)) return -1; - if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) - return 0; + if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) { + /* Reaching both targets verifies a cross-node move. */ + if (success >= 2) + return 0; + else + return -2; + } ret = move_pages(0, 1, (void **) &ptr, &n2, &status, MPOL_MF_MOVE_ALL); - if (ret) { - if (ret > 0) { - /* Migration is best effort; try again */ - if (++failures < MAX_RETRIES) - continue; - printf("Didn't migrate %d pages\n", ret); - } - else - perror("Couldn't migrate pages"); + if (ret < 0) { + perror("Couldn't migrate pages"); + return ret; + } + /* Migration is best effort. Try again */ + if (ret > 0 || status < 0) + continue; + if (status != n2) { + printf("Page is on node %d instead of target node %d\n", + status, n2); return -2; } - failures = 0; + success++; tmp = n2; n2 = n1; n1 = tmp; } - - return 0; } void *access_mem(void *ptr) diff --git a/tools/testing/selftests/mm/mseal_test.c b/tools/testing/selftests/mm/mseal_test.c index 93c2e13094d4..1a05e6921fed 100644 --- a/tools/testing/selftests/mm/mseal_test.c +++ b/tools/testing/selftests/mm/mseal_test.c @@ -1876,7 +1876,7 @@ int main(void) if (!pkey_supported()) ksft_print_msg("PKEY not supported\n"); - ksft_set_plan(88); + ksft_set_plan(87); test_seal_addseal(); test_seal_unmapped_start(); @@ -1914,7 +1914,6 @@ int main(void) test_seal_mprotect_partial_mprotect(true); test_seal_mprotect_two_vma_with_gap(); - test_seal_mprotect_two_vma_with_gap(); test_seal_mprotect_merge(false); test_seal_mprotect_merge(true); diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 1b2dffcc999b..eadc7159ca5b 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -1085,7 +1085,7 @@ static void unpopulated_written_test(const char *name, char *mem, long size, memset(mem, 1, size); if (use_thp && (madvise(mem, size, MADV_COLLAPSE) || - !check_huge_anon(mem, size / hpage_size, hpage_size))) { + !check_huge_anon(mem, size, size / hpage_size, hpage_size))) { ksft_test_result_skip("%s could not form a THP\n", name); goto out; } @@ -1332,12 +1332,6 @@ int mprotect_tests(void) int ret; char *mem, *mem2; struct page_region vec; - int pagemap_fd = open("/proc/self/pagemap", O_RDONLY); - - if (pagemap_fd < 0) { - fprintf(stderr, "open() failed\n"); - exit(1); - } /* 1. Map two pages */ mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); diff --git a/tools/testing/selftests/mm/prctl_thp_disable.c b/tools/testing/selftests/mm/prctl_thp_disable.c index d8d9d1de57b8..82c6e96ea6eb 100644 --- a/tools/testing/selftests/mm/prctl_thp_disable.c +++ b/tools/testing/selftests/mm/prctl_thp_disable.c @@ -67,7 +67,7 @@ static int test_mmap_thp(enum thp_collapse_type madvise_buf, size_t pmdsize) /* HACK: make sure we have a separate VMA that we can check reliably. */ mprotect(mem, pmdsize, PROT_READ); - ret = check_huge_anon(mem, 1, pmdsize); + ret = check_huge_anon(mem, pmdsize, 1, pmdsize); munmap(mmap_mem, mmap_size); return ret; } diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 687d115e3bd8..d09f9f6a384e 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -410,6 +410,8 @@ CATEGORY="thp" run_test ./khugepaged all:shmem CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem +CATEGORY="thp" run_test ./khugepaged -c 4 mthp_khugepaged:anon + # Try to create XFS if not provided if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then if test_selected "thp"; then diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index fb1864a68e1c..5f278913c4d7 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -103,7 +103,7 @@ static void test_hugepage(int pagemap_fd, int pagesize) for (i = 0; i < hpage_len; i++) map[i] = (char)i; - if (check_huge_anon(map, 1, hpage_len)) { + if (check_huge_anon(map, hpage_len, 1, hpage_len)) { ksft_test_result_pass("Test %s huge page allocation\n", __func__); clear_softdirty(); @@ -152,7 +152,8 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) return; } unlink(fname); - ftruncate(test_fd, pagesize); + if (ftruncate(test_fd, pagesize) != 0) + ksft_exit_fail_msg("ftruncate failed\n"); map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_SHARED, test_fd, 0); if (map == MAP_FAILED) diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 32b991472f74..86a603692826 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -104,129 +104,6 @@ fail: return false; } -static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, - uint64_t *flags) -{ - unsigned long pfn; - - pfn = pagemap_get_pfn(pagemap_fd, vaddr); - - /* non-present PFN */ - if (pfn == -1UL) - return 1; - - if (pageflags_get(pfn, kpageflags_fd, flags)) - return -1; - - return 0; -} - -/* - * gather_after_split_folio_orders - scan through [vaddr_start, len) and record - * folio orders - * - * @vaddr_start: start vaddr - * @len: range length - * @pagemap_fd: file descriptor to /proc/<pid>/pagemap - * @kpageflags_fd: file descriptor to /proc/kpageflags - * @orders: output folio order array - * @nr_orders: folio order array size - * - * gather_after_split_folio_orders() scan through [vaddr_start, len) and check - * all folios within the range and record their orders. All order-0 pages will - * be recorded. Non-present vaddr is skipped. - * - * NOTE: the function is used to check folio orders after a split is performed, - * so it assumes [vaddr_start, len) fully maps to after-split folios within that - * range. - * - * Return: 0 - no error, -1 - unhandled cases - */ -static int gather_after_split_folio_orders(char *vaddr_start, size_t len, - int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) -{ - uint64_t page_flags = 0; - int cur_order = -1; - char *vaddr; - - if (pagemap_fd == -1 || kpageflags_fd == -1) - return -1; - if (!orders) - return -1; - if (nr_orders <= 0) - return -1; - - for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { - char *next_folio_vaddr; - int status; - - status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, - &page_flags); - if (status < 0) - return -1; - - /* skip non present vaddr */ - if (status == 1) { - vaddr += psize(); - continue; - } - - /* all order-0 pages with possible false postive (non folio) */ - if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { - orders[0]++; - vaddr += psize(); - continue; - } - - /* skip non thp compound pages */ - if (!(page_flags & KPF_THP)) { - vaddr += psize(); - continue; - } - - /* vpn points to part of a THP at this point */ - if (page_flags & KPF_COMPOUND_HEAD) - cur_order = 1; - else { - vaddr += psize(); - continue; - } - - next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); - - if (next_folio_vaddr >= vaddr_start + len) - break; - - while ((status = vaddr_pageflags_get(next_folio_vaddr, - pagemap_fd, kpageflags_fd, - &page_flags)) >= 0) { - /* - * non present vaddr, next compound head page, or - * order-0 page - */ - if (status == 1 || - (page_flags & KPF_COMPOUND_HEAD) || - !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { - if (cur_order < nr_orders) { - orders[cur_order]++; - cur_order = -1; - vaddr = next_folio_vaddr; - } - break; - } - - cur_order++; - next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); - } - - if (status < 0) - return status; - } - if (cur_order > 0 && cur_order < nr_orders) - orders[cur_order]++; - return 0; -} - static int check_after_split_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) { @@ -240,7 +117,7 @@ static int check_after_split_folio_orders(char *vaddr_start, size_t len, ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); memset(vaddr_orders, 0, sizeof(int) * nr_orders); - status = gather_after_split_folio_orders(vaddr_start, len, pagemap_fd, + status = gather_folio_orders(vaddr_start, len, pagemap_fd, kpageflags_fd, vaddr_orders, nr_orders); if (status) ksft_exit_fail_msg("gather folio info failed\n"); @@ -296,7 +173,7 @@ static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hp unsigned long rss_anon_before, rss_anon_after; size_t i; - if (!check_huge_anon(one_page, nr_hpages, pmd_pagesize)) + if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize)) ksft_exit_fail_msg("No THP is allocated\n"); rss_anon_before = rss_anon(); @@ -311,7 +188,7 @@ static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hp if (one_page[i] != (char)0) ksft_exit_fail_msg("%ld byte corrupted\n", i); - if (!check_huge_anon(one_page, 0, pmd_pagesize)) + if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize)) ksft_exit_fail_msg("Still AnonHugePages not split\n"); rss_anon_after = rss_anon(); @@ -347,7 +224,7 @@ static void split_pmd_thp_to_order(int order) for (i = 0; i < len; i++) one_page[i] = (char)i; - if (!check_huge_anon(one_page, 4, pmd_pagesize)) + if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize)) ksft_exit_fail_msg("No THP is allocated\n"); /* split all THPs */ @@ -366,7 +243,7 @@ static void split_pmd_thp_to_order(int order) (pmd_order + 1))) ksft_exit_fail_msg("Unexpected THP split\n"); - if (!check_huge_anon(one_page, 0, pmd_pagesize)) + if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize)) ksft_exit_fail_msg("Still AnonHugePages not split\n"); ksft_test_result_pass("Split huge pages to order %d successful\n", order); @@ -393,7 +270,7 @@ static void split_pte_mapped_thp(void) for (i = 0; i < thp_area_size; i++) thp_area[i] = (char)i; - if (!check_huge_anon(thp_area, nr_thps, pmd_pagesize)) { + if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) { ksft_test_result_skip("Not all THPs allocated\n"); goto out; } @@ -657,7 +534,7 @@ static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize); - if (!check_huge_file(*addr, fd_size / pmd_pagesize, pmd_pagesize)) { + if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) { ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); munmap(*addr, fd_size); close(*fd); @@ -735,7 +612,7 @@ static void split_thp_in_pagecache_to_order_at(size_t fd_size, goto out; } - if (!check_huge_file(addr, 0, pmd_pagesize)) { + if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) { ksft_print_msg("Still FilePmdMapped not split\n"); err = EXIT_FAILURE; goto out; diff --git a/tools/testing/selftests/mm/uffd-common.c b/tools/testing/selftests/mm/uffd-common.c index f48f5d4594ab..1fb967ef4985 100644 --- a/tools/testing/selftests/mm/uffd-common.c +++ b/tools/testing/selftests/mm/uffd-common.c @@ -194,7 +194,9 @@ static void shmem_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start, static void shmem_check_pmd_mapping(uffd_global_test_opts_t *gopts, void *p, int expect_nr_hpages) { - if (!check_huge_shmem(gopts->area_dst_alias, expect_nr_hpages, + size_t len = expect_nr_hpages * read_pmd_pagesize(); + + if (!check_huge_shmem(gopts->area_dst_alias, len, expect_nr_hpages, read_pmd_pagesize())) err("Did not find expected %d number of hugepages", expect_nr_hpages); diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index ef1ea11981a7..4821a3563036 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -15,6 +15,9 @@ #define SMAP_FILE_PATH "/proc/self/smaps" #define STATUS_FILE_PATH "/proc/self/status" #define MAX_LINE_LENGTH 500 +#define PAGEMAP_PATH "/proc/self/pagemap" +#define KPAGEFLAGS_PATH "/proc/kpageflags" +#define MAX_NR_ORDERS 20 unsigned int __page_size; unsigned int __page_shift; @@ -31,7 +34,7 @@ uint64_t pagemap_get_entry(int fd, char *start) return entry; } -static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) +static int __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) { struct pm_scan_arg arg; @@ -55,7 +58,7 @@ static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_r static uint64_t pagemap_scan_get_categories(int fd, char *start) { struct page_region r; - long ret; + int ret; ret = __pagemap_scan_get_categories(fd, start, &r); if (ret < 0) @@ -194,6 +197,125 @@ err_out: return rss_anon; } +static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, + uint64_t *flags) +{ + unsigned long pfn; + + pfn = pagemap_get_pfn(pagemap_fd, vaddr); + + /* non-present PFN */ + if (pfn == -1UL) + return 1; + + if (pageflags_get(pfn, kpageflags_fd, flags)) + return -1; + + return 0; +} + +/* + * gather_folio_orders - scan through [vaddr_start, len) and record + * folio orders + * + * @vaddr_start: start vaddr + * @len: range length + * @pagemap_fd: file descriptor to /proc/<pid>/pagemap + * @kpageflags_fd: file descriptor to /proc/kpageflags + * @orders: output folio order array + * @nr_orders: folio order array size + * + * gather_folio_orders() scan through [vaddr_start, len) and check + * all folios within the range and record their orders. All order-0 pages will + * be recorded. Non-present vaddr is skipped. + * + * Return: 0 - no error, -1 - unhandled cases + */ +int gather_folio_orders(char *vaddr_start, size_t len, + int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) +{ + uint64_t page_flags = 0; + int cur_order = -1; + char *vaddr; + + if (pagemap_fd == -1 || kpageflags_fd == -1) + return -1; + if (!orders) + return -1; + if (nr_orders <= 0) + return -1; + + for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { + char *next_folio_vaddr; + int status; + + status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, + &page_flags); + if (status < 0) + return -1; + + /* skip non present vaddr */ + if (status == 1) { + vaddr += psize(); + continue; + } + + /* all order-0 pages with possible false postive (non folio) */ + if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { + orders[0]++; + vaddr += psize(); + continue; + } + + /* skip non thp compound pages */ + if (!(page_flags & KPF_THP)) { + vaddr += psize(); + continue; + } + + /* vpn points to part of a THP at this point */ + if (page_flags & KPF_COMPOUND_HEAD) + cur_order = 1; + else { + vaddr += psize(); + continue; + } + + next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); + + if (next_folio_vaddr >= vaddr_start + len) + break; + + while ((status = vaddr_pageflags_get(next_folio_vaddr, + pagemap_fd, kpageflags_fd, + &page_flags)) >= 0) { + /* + * non present vaddr, next compound head page, or + * order-0 page + */ + if (status == 1 || + (page_flags & KPF_COMPOUND_HEAD) || + !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { + if (cur_order < nr_orders) { + orders[cur_order]++; + cur_order = -1; + vaddr = next_folio_vaddr; + } + break; + } + + cur_order++; + next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); + } + + if (status < 0) + return status; + } + if (cur_order > 0 && cur_order < nr_orders) + orders[cur_order]++; + return 0; +} + char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) { int ret; @@ -229,7 +351,7 @@ err_out: return entry; } -bool __check_huge(void *addr, char *pattern, int nr_hpages, +static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages, uint64_t hpage_size) { char buffer[MAX_LINE_LENGTH]; @@ -247,19 +369,84 @@ err_out: return thp == (nr_hpages * (hpage_size >> 10)); } -bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) +static bool check_large_folios(void *addr, size_t len, int nr_hpages, + uint64_t hpage_size) { - return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); + int order = 0, pagesize = getpagesize(); + unsigned int nr_pages = hpage_size / pagesize; + int orders[MAX_NR_ORDERS], status; + int pagemap_fd, kpageflags_fd; + bool ret = false; + + if (!nr_pages) + ksft_exit_fail_msg("invalid hugepage size\n"); + + order = 31 - __builtin_clz(nr_pages); + if (!order || order >= MAX_NR_ORDERS) + ksft_exit_fail_msg("invalid order\n"); + + memset(orders, 0, sizeof(int) * MAX_NR_ORDERS); + pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); + if (pagemap_fd == -1) + ksft_exit_fail_msg("read pagemap fail\n"); + + kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); + if (kpageflags_fd == -1) { + close(pagemap_fd); + ksft_exit_fail_msg("read kpageflags fail\n"); + } + + status = gather_folio_orders(addr, len, pagemap_fd, + kpageflags_fd, orders, MAX_NR_ORDERS); + if (status) + goto out; + + if (orders[order] == nr_hpages) + ret = true; + +out: + close(pagemap_fd); + close(kpageflags_fd); + return ret; } -bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) +bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) { - return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); + uint64_t pmd_pagesize = read_pmd_pagesize(); + + if (!pmd_pagesize) + ksft_exit_fail_msg("reading PMD pagesize failed\n"); + + if (hpage_size == pmd_pagesize) + return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); + + return check_large_folios(addr, len, nr_hpages, hpage_size); } -bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) +bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) { - return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); + uint64_t pmd_pagesize = read_pmd_pagesize(); + + if (!pmd_pagesize) + ksft_exit_fail_msg("reading PMD pagesize failed\n"); + + if (hpage_size == pmd_pagesize) + return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); + + return check_large_folios(addr, len, nr_hpages, hpage_size); +} + +bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +{ + uint64_t pmd_pagesize = read_pmd_pagesize(); + + if (!pmd_pagesize) + ksft_exit_fail_msg("reading PMD pagesize failed\n"); + + if (hpage_size == pmd_pagesize) + return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); + + return check_large_folios(addr, len, nr_hpages, hpage_size); } int64_t allocate_transhuge(void *ptr, int pagemap_fd) @@ -755,7 +942,7 @@ unsigned long read_num(const char *path) { char buf[21]; - if (read_file(path, buf, sizeof(buf)) < 0) + if (!read_file(path, buf, sizeof(buf))) ksft_exit_fail_perror("read_file()"); return strtoul(buf, NULL, 10); diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 7799154b67ee..9a49af88702e 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -90,11 +90,13 @@ void clear_softdirty(void); bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len); uint64_t read_pmd_pagesize(void); unsigned long rss_anon(void); -bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size); -bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size); -bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size); +bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); +bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); +bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); int64_t allocate_transhuge(void *ptr, int pagemap_fd); int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); +int gather_folio_orders(char *vaddr_start, size_t len, + int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders); int uffd_register(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor); diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index 1026d8c400e1..415eccb70468 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -490,7 +490,8 @@ static bool query_addr_at(int maps_fd, void *addr, static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self) { - return mmap(self->mod_info->addr, self->page_size, self->mod_info->prot | PROT_EXEC, + /* PROT_NONE differs from both readable neighbors. */ + return mmap(self->mod_info->addr, self->page_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED; } diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index cdeb53bbdd1b..4c58487b764e 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -243,7 +243,7 @@ enum { #define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE) #define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE) #define VM_STACK INIT_VM_FLAG(STACK) -#ifdef CONFIG_STACK_GROWS_UP +#ifdef CONFIG_STACK_GROWSUP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) #define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else @@ -577,6 +577,7 @@ struct vm_area_struct { */ unsigned int vm_lock_seq; #endif + unsigned int __vm_anon_pgoff_lo; /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma @@ -613,6 +614,9 @@ struct vm_area_struct { /* Unstable RCU readers are allowed to read this. */ refcount_t vm_refcnt; #endif +#ifdef CONFIG_64BIT + unsigned int __vm_anon_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. @@ -1158,6 +1162,17 @@ static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma) return is_shared_maywrite(&vma->flags); } +static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags) +{ + return vma_flags_test(flags, VMA_MAYWRITE_BIT) && + !vma_flags_test(flags, VMA_SHARED_BIT); +} + +static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma) +{ + return vma_flags_is_cow_mapping(&vma->flags); +} + static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) { /* @@ -1320,6 +1335,28 @@ static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) return vma_start_pgoff(vma) + vma_pages(vma); } +static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_anon_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_anon_pgoff_lo; + return pgoff; +} + +static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_anon_pgoff(vma) + vma_pages(vma); +} + +static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_anon_pgoff(vma) - 1; +} + static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) { return file->f_op->mmap_prepare(desc); @@ -1391,7 +1428,7 @@ static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) mas_set(&vmi->mas, addr); } -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -1584,3 +1621,26 @@ static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) { return vma_flags_to_page_prot(vma->flags); } + +static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_anon_pgoff(vma); + return pgoff; +} + +static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_anon_page_index(vma, address); + + VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); + /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ + if (vma_is_anonymous(vma) && !vma->vm_file) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); + + return pgoff; +} diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index bea9ea6db02a..4a39c9d50489 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -23,7 +23,8 @@ struct vm_area_struct *alloc_vma(struct mm_struct *mm, vma->vm_start = start; vma->vm_end = end; - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); + vma_set_anon_pgoff(vma, start >> PAGE_SHIFT); vma->flags = vma_flags; vma_assert_detached(vma); diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index e357accc8499..acaab282939c 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -45,6 +45,7 @@ void vmg_set_range(struct vma_merge_struct *vmg, unsigned long start, vmg->start = start; vmg->end = end; vmg->pgoff = pgoff; + vmg->anon_pgoff = start >> PAGE_SHIFT; vmg->vma_flags = vma_flags; vmg->just_expand = false; @@ -108,6 +109,7 @@ static bool test_simple_merge(void) .end = 0x2000, .vma_flags = vma_flags, .pgoff = 1, + .anon_pgoff = 1, }; ASSERT_FALSE(attach_vma(&mm, vma_left)); @@ -119,6 +121,7 @@ static bool test_simple_merge(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_FLAGS_SAME_MASK(&vma->flags, vma_flags); detach_free_vma(vma); @@ -151,6 +154,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); ASSERT_EQ(vma_start_pgoff(vma), 1); + ASSERT_EQ(vma_start_anon_pgoff(vma), 1); /* * Now walk through the three split VMAs and make sure they are as @@ -163,6 +167,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -172,6 +177,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); ASSERT_EQ(vma_start_pgoff(vma), 1); + ASSERT_EQ(vma_start_anon_pgoff(vma), 1); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -181,6 +187,7 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); ASSERT_EQ(vma_start_pgoff(vma), 2); + ASSERT_EQ(vma_start_anon_pgoff(vma), 2); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -210,6 +217,7 @@ static bool test_simple_expand(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -232,6 +240,7 @@ static bool test_simple_shrink(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -344,6 +353,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -365,6 +375,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x9000); ASSERT_EQ(vma_start_pgoff(vma), 6); + ASSERT_EQ(vma_start_anon_pgoff(vma), 6); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -385,6 +396,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x9000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -405,6 +417,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_EQ(vma->vm_start, 0xa000); ASSERT_EQ(vma->vm_end, 0xc000); ASSERT_EQ(vma_start_pgoff(vma), 0xa); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0xa); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -424,6 +437,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -444,6 +458,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); detach_free_vma(vma); @@ -640,7 +655,8 @@ static bool test_vma_merge_with_close(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x5000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_EQ(cleanup_mm(&mm, &vmi), 2); @@ -751,7 +767,8 @@ static bool test_vma_merge_with_close(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x5000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_EQ(cleanup_mm(&mm, &vmi), 2); @@ -806,6 +823,7 @@ static bool test_vma_merge_new_with_close(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_EQ(vma->vm_ops, &vm_ops); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -861,11 +879,13 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_next->vm_start, 0x3000); ASSERT_EQ(vma_next->vm_end, 0x9000); - ASSERT_EQ(vma_next->vm_pgoff, 3); + ASSERT_EQ(vma_start_pgoff(vma_next), 3); + ASSERT_EQ(vma_start_anon_pgoff(vma_next), 3); ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); ASSERT_EQ(vma_start_pgoff(vma), 2); + ASSERT_EQ(vma_start_anon_pgoff(vma), 2); ASSERT_TRUE(vma_write_started(vma)); ASSERT_TRUE(vma_write_started(vma_next)); ASSERT_EQ(mm.map_count, 2); @@ -895,7 +915,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_next->vm_start, 0x2000); ASSERT_EQ(vma_next->vm_end, 0x9000); - ASSERT_EQ(vma_next->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma_next), 2); + ASSERT_EQ(vma_start_anon_pgoff(vma_next), 2); ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_next)); ASSERT_EQ(mm.map_count, 1); @@ -927,11 +948,13 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x6000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x7000); ASSERT_EQ(vma_start_pgoff(vma), 6); + ASSERT_EQ(vma_start_anon_pgoff(vma), 6); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -962,7 +985,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x7000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_EQ(mm.map_count, 1); @@ -994,7 +1018,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x9000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_EQ(mm.map_count, 1); @@ -1124,7 +1149,8 @@ static bool test_anon_vma_non_mergeable(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x7000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_FALSE(vma_write_started(vma_next)); @@ -1155,7 +1181,8 @@ static bool test_anon_vma_non_mergeable(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x7000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_FALSE(vma_write_started(vma_next)); @@ -1417,6 +1444,7 @@ static bool test_merge_extend(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_anon_pgoff(vma), 0); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -1431,7 +1459,7 @@ static bool test_expand_only_mode(void) struct mm_struct mm = {}; VMA_ITERATOR(vmi, &mm, 0); struct vm_area_struct *vma_prev, *vma; - VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5); + VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5, 5); /* * Place a VMA prior to the one we're expanding so we assert that we do @@ -1457,6 +1485,7 @@ static bool test_expand_only_mode(void) ASSERT_EQ(vma->vm_start, 0x3000); ASSERT_EQ(vma->vm_end, 0x9000); ASSERT_EQ(vma_start_pgoff(vma), 3); + ASSERT_EQ(vma_start_anon_pgoff(vma), 3); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(vma_iter_addr(&vmi), 0x3000); vma_assert_attached(vma); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da06321..c8ef7b8cd46b 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -33,12 +33,56 @@ static bool test_copy_vma(void) struct mm_struct mm = {}; bool need_locks = false; VMA_ITERATOR(vmi, &mm, 0); - struct vm_area_struct *vma, *vma_new, *vma_next; + struct vm_area_struct *vma, *vma_prev, *vma_new, *vma_next, *vma_orig; + + /* Move forwards, adjacent to old self - self-merge. */ + + vma = alloc_and_link_vma(&mm, 0x1000, 0x2000, 1, vma_flags); + vma_set_anonymous(vma); + vma_orig = vma; + vma_new = copy_vma(&vma, 0x2000, 0x1000, 1, 1, &need_locks); + ASSERT_EQ(vma_new, vma_orig); + ASSERT_EQ(vma, vma_orig); + ASSERT_EQ(vma_new->vm_start, 0x1000); + ASSERT_EQ(vma_new->vm_end, 0x3000); + + cleanup_mm(&mm, &vmi); + + /* Move backwards, adjacent to old self - self-merge. */ + + vma = alloc_and_link_vma(&mm, 0x2000, 0x3000, 2, vma_flags); + vma_set_anonymous(vma); + vma_orig = vma; + vma_new = copy_vma(&vma, 0x1000, 0x1000, 2, 2, &need_locks); + ASSERT_EQ(vma_new, vma_orig); + ASSERT_EQ(vma, vma_orig); + ASSERT_EQ(vma_new->vm_start, 0x1000); + ASSERT_EQ(vma_new->vm_end, 0x3000); + + cleanup_mm(&mm, &vmi); + + /* + * Move backwards between prior VMA and old self - self-merge and vma + * updated to a new VMA. + */ + + vma_prev = alloc_and_link_vma(&mm, 0x1000, 0x2000, 1, vma_flags); + vma_set_anonymous(vma_prev); + vma = alloc_and_link_vma(&mm, 0x3000, 0x4000, 3, vma_flags); + vma_set_anonymous(vma); + vma_orig = vma; + vma_new = copy_vma(&vma, 0x2000, 0x1000, 3, 3, &need_locks); + ASSERT_NE(vma_new, vma_orig); + ASSERT_EQ(vma_new, vma); + ASSERT_EQ(vma_new->vm_start, 0x1000); + ASSERT_EQ(vma_new->vm_end, 0x4000); + + cleanup_mm(&mm, &vmi); /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +95,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index 4f6c5666ac07..8a48b231aa7a 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -53,6 +53,7 @@ typedef __bitwise unsigned int vm_fault_t; #define VM_WARN_ON(_expr) (WARN_ON(_expr)) #define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr)) +#define VM_WARN_ON_ONCE_VMA(_expr, _vma) (WARN_ON_ONCE(_expr)) #define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr)) #define VM_BUG_ON(_expr) (BUG_ON(_expr)) #define VM_BUG_ON_VMA(_expr, _vma) (BUG_ON(_expr)) |