diff options
367 files changed, 14599 insertions, 6854 deletions
@@ -3096,6 +3096,10 @@ N: Jens Osterkamp E: jens@de.ibm.com D: Maintainer of Spidernet network driver for Cell +N: Kent Overstreet +E: kent.overstreet@linux.dev +D: Co-authored and contributed to Memory Allocation Profiling + N: Gadi Oxman E: gadio@netvision.net.il D: Original author and maintainer of IDE/ATAPI floppy/tape drivers diff --git a/Documentation/ABI/testing/sysfs-block-zram b/Documentation/ABI/testing/sysfs-block-zram index 64c03010e951..1ea9018a627f 100644 --- a/Documentation/ABI/testing/sysfs-block-zram +++ b/Documentation/ABI/testing/sysfs-block-zram @@ -152,15 +152,15 @@ Description: compression algorithm parameters. What: /sys/block/zram<id>/compressed_writeback -Date: Decemeber 2025 +Date: December 2025 Contact: Richard Chang <richardycc@google.com> Description: - The compressed_writeback device atrribute toggles compressed + The compressed_writeback device attribute toggles compressed writeback feature. What: /sys/block/zram<id>/writeback_batch_size Date: November 2025 Contact: Sergey Senozhatsky <senozhatsky@chromium.org> Description: - The writeback_batch_size device atrribute sets the maximum + The writeback_batch_size device attribute sets the maximum number of in-flight writeback operations. diff --git a/Documentation/ABI/testing/sysfs-bus-dax b/Documentation/ABI/testing/sysfs-bus-dax index b34266bfae49..60703fffa99e 100644 --- a/Documentation/ABI/testing/sysfs-bus-dax +++ b/Documentation/ABI/testing/sysfs-bus-dax @@ -151,3 +151,27 @@ Description: memmap_on_memory parameter for memory_hotplug. This is typically set on the kernel command line - memory_hotplug.memmap_on_memory set to 'true' or 'force'." + +What: /sys/bus/dax/devices/daxX.Y/state +Contact: nvdimm@lists.linux.dev +Description: + (RW) Controls the state of the memory region. + Applies to all memory blocks associated with the device. + Only applies to dax_kmem devices. + + Reading returns the current state; the writable states mirror + the per-block /sys/devices/system/memory/memoryX/state ABI:: + + "unplugged": memory blocks are not present + "online": memory is online, zone chosen by the kernel + "online_kernel": memory is online in ZONE_NORMAL + "online_movable": memory is online in ZONE_MOVABLE + + "offline" (memory blocks are present but offline) may also be + reported - this happens when the device is bound while the + auto_online_blocks policy is "offline". It cannot be written, + as it's not useful and creates device destruction races. + + A device can only be onlined from the "unplugged" state, so a + device must be returned to "unplugged" before it can be onlined + into a different state. diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index 4fdec63a47d4..e675a57145e3 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -36,7 +36,10 @@ Description: Writing 'on' or 'off' to this file makes the kdamond starts or kdamond. Writing 'clear_schemes_tried_regions' to the file removes contents of the 'tried_regions' directory. Writing 'update_schemes_effective_quotas' to the file updates - '.../quotas/effective_bytes' files of this kdamond. + '.../quotas/effective_bytes' files of this kdamond. Writing + 'update_tuned_intervals' to the file updates 'sample_us' and + 'aggr_us' files under 'monitoring_attrs/intervals' with the + auto-tuned sampling and aggregation intervals. What: /sys/kernel/mm/damon/admin/kdamonds/<K>/pid Date: Mar 2022 @@ -112,7 +115,7 @@ Description: Writing a value to this file sets the update interval of the DAMON context in microseconds as the value. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/access_bp +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/access_bp Date: Feb 2025 Contact: SJ Park <sj@kernel.org> Description: Writing a value to this file sets the monitoring intervals @@ -120,7 +123,7 @@ Description: Writing a value to this file sets the monitoring intervals the given time interval (aggrs in same directory), in bp (1/10,000). Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/aggrs +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/aggrs Date: Feb 2025 Contact: SJ Park <sj@kernel.org> Description: Writing a value to this file sets the time interval to achieve @@ -128,14 +131,14 @@ Description: Writing a value to this file sets the time interval to achieve access events ratio (access_bp in same directory) within. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/min_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/min_sample_us Date: Feb 2025 Contact: SJ Park <sj@kernel.org> Description: Writing a value to this file sets the minimum value of auto-tuned sampling interval in microseconds. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/max_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/max_sample_us Date: Feb 2025 Contact: SJ Park <sj@kernel.org> Description: Writing a value to this file sets the maximum value of @@ -144,7 +147,7 @@ Description: Writing a value to this file sets the maximum value of What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/nr_regions/min -WDate: Mar 2022 +Date: Mar 2022 Contact: SJ Park <sj@kernel.org> Description: Writing a value to this file sets the minimum number of monitoring regions of the DAMON context as the value. Reading @@ -157,6 +160,52 @@ Description: Writing a value to this file sets the maximum number of monitoring regions of the DAMON context as the value. Reading this file returns the value. +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/nr_probes +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probe named '0' to 'N-1' under the + probes/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/weight +Date: Jul 2026 +Contact: SJ Park <sj@kernel.org> +Description: Writing to and reading from this file sets and gets the + per-probe attribute weight. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/nr_filters +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probe filter named '0' to 'N-1' + under the filters/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/type +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Writing to and reading from this file sets and gets the type of + the memory of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/path +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: If 'memcg' is written to the 'type' file, writing to and + reading from this file sets and gets the path to the memory + cgroup of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/matching +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Writing 'Y' or 'N' to this file sets whether the filter is for + the memory of the 'type', or all except the 'type'. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/allow +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Writing 'Y' or 'N' to this file sets whether to allow or reject + hitting the probe for the memory that satisfies the 'type' and + the 'matching' of the directory. + What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/nr_targets Date: Mar 2022 Contact: SJ Park <sj@kernel.org> @@ -241,7 +290,7 @@ Description: Writing to and reading from this file sets and gets the maximum What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/nr_accesses/min Date: Mar 2022 Contact: SJ Park <sj@kernel.org> -Description: Writing to and reading from this file sets and gets the manimum +Description: Writing to and reading from this file sets and gets the minimum 'nr_accesses' of the scheme's target regions. What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/nr_accesses/max @@ -601,3 +650,15 @@ Contact: SJ Park <sj@kernel.org> Description: Reading this file returns the size of the memory in the region that passed DAMON operations layer-handled filters of the scheme in bytes. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/probes/ +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Directory for DAMON data-attribute monitoring probes of the + region. + +What: /sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/probes/<P>/hits +Date: May 2026 +Contact: SJ Park <sj@kernel.org> +Description: Reading this file returns the number of data-attribute + monitoring probe-hit positive samples of the region. diff --git a/Documentation/admin-guide/cgroup-v1/cpusets.rst b/Documentation/admin-guide/cgroup-v1/cpusets.rst index c7909e5ac136..52a213aff04e 100644 --- a/Documentation/admin-guide/cgroup-v1/cpusets.rst +++ b/Documentation/admin-guide/cgroup-v1/cpusets.rst @@ -284,7 +284,7 @@ take action. ==> Unless this feature is enabled by writing "1" to the special file /dev/cpuset/memory_pressure_enabled, the hook in the rebalance - code of __alloc_pages() for this metric reduces to simply noticing + code of the page allocator for this metric reduces to simply noticing that the cpuset_memory_pressure_enabled flag is zero. So only systems that enable this feature will compute the metric. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..53f08950a630 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2064,8 +2064,11 @@ Kernel parameters hugetlb_cma= [HW,CMA,EARLY] The size of a CMA area used for allocation of gigantic hugepages. Or using node format, the size of a CMA area per node can be specified. - Format: nn[KMGTPE] or (node format) - <node>:nn[KMGTPE][,<node>:nn[KMGTPE]] + The size can be an absolute value (e.g., 2G) or a + percentage of the total memory or node memory (e.g., 20%). + Format: nn[KMGTPE] or nn% or (node format) + <node>:nn[KMGTPE][,<node>:nn[KMGTPE]] or + <node>:nn%[,<node>:nn%] The size must be a multiple of the gigantic page size. When using node format, this applies to each per-node size. diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 011296f1e7c2..da5f9afd08ae 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -73,9 +73,10 @@ comma (","). │ │ │ │ │ │ │ intervals_goal/access_bp,aggrs,min_sample_us,max_sample_us │ │ │ │ │ │ nr_regions/min,max │ │ │ │ │ │ :ref:`probes <damon_usage_sysfs_probes>`/nr_probes - │ │ │ │ │ │ │ 0/filters/nr_filters - │ │ │ │ │ │ │ │ 0/type,matching,allow,path - │ │ │ │ │ │ │ │ ... + │ │ │ │ │ │ │ 0/weight + │ │ │ │ │ │ │ │ filters/nr_filters + │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path + │ │ │ │ │ │ │ │ │ ... │ │ │ │ │ │ │ ... │ │ │ │ │ :ref:`targets <sysfs_targets>`/nr_targets │ │ │ │ │ │ :ref:`0 <sysfs_target>`/pid_target,obsolete_target @@ -246,7 +247,7 @@ writing to and reading from the files. Under ``nr_regions`` directory, two files for the lower-bound and upper-bound of DAMON's monitoring regions (``min`` and ``max``, respectively), which controls the monitoring overhead, exist. You can set and get the values by -writing to and rading from the files. +writing to and reading from the files. For more details about the intervals and monitoring regions range, please refer to the Design document (:doc:`/mm/damon/design`). @@ -264,7 +265,7 @@ Please refer to the :ref:`design document of the feature <damon_design_monitoring_intervals_autotuning>` for the internal of the tuning mechanism. Reading and writing the four files under ``intervals_goal`` directory shows and updates the tuning parameters that described in the -:ref:design doc <damon_design_monitoring_intervals_autotuning>` with the same +:ref:`design doc <damon_design_monitoring_intervals_autotuning>` with the same names. The tuning starts with the user-set ``sample_us`` and ``aggr_us``. The tuning-applied current values of the two intervals can be read from the ``sample_us`` and ``aggr_us`` files after writing ``update_tuned_intervals`` to @@ -286,6 +287,10 @@ In each probe directory, one directory, ``filters`` exists. The directory contains files for installing filters for the probe, that is used to determine the data attribute for the probe. +Each probe directory also contains ``weight`` file. Reading from and writing +to the file gets and sets the :ref:`attributes-only monitoring +<damon_design_attrs_only_monitoring>` weight for the attribute of the probe. + In the beginning, ``filters`` directory has only one file, ``nr_filters``. Writing a number (``N``) to the file creates the number of child directories named ``0`` to ``N-1``. Each directory represents each filter and works in a @@ -377,7 +382,7 @@ schemes/<N>/ In each scheme directory, nine directories (``access_pattern``, ``quotas``, ``watermarks``, ``core_filters``, ``ops_filters``, ``filters``, ``dests``, ``stats``, and ``tried_regions``) and three files (``action``, ``target_nid`` -and ``apply_interval``) exist. +and ``apply_interval_us``) exist. The ``action`` file is for setting and getting the scheme's :ref:`action <damon_design_damos_action>`. The keywords that can be written to and read @@ -743,7 +748,7 @@ counter). Finally the tenth field (``X``) shows the ``age`` of the region (refer to :ref:`design <damon_design_age_tracking>` for more details of the counter). -If the event was ``damon:damos_beofre_apply``, the ``perf script`` output would +If the event was ``damon:damos_before_apply``, the ``perf script`` output would be somewhat like below:: kdamond.0 47293 [000] 80801.060214: damon:damos_before_apply: ctx_idx=0 scheme_idx=0 target_idx=0 nr_regions=11 121932607488-135128711168: 0 136 diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst index c57e61b5d8aa..ffa690a171c8 100644 --- a/Documentation/admin-guide/mm/pagemap.rst +++ b/Documentation/admin-guide/mm/pagemap.rst @@ -19,8 +19,11 @@ There are four components to pagemap: * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) * Bit 56 page exclusively mapped (since 4.2) - * Bit 57 pte is uffd-wp write-protected (since 5.13) (see - Documentation/admin-guide/mm/userfaultfd.rst) + * Bit 57 pte is tracked by userfaultfd (since 5.13) — in a + ``VM_UFFD_WP`` VMA this indicates a write-protected PTE; in a + ``VM_UFFD_RWP`` VMA it indicates an RWP-protected PTE. WP and + RWP are mutually exclusive per VMA, so the meaning is + unambiguous. See Documentation/admin-guide/mm/userfaultfd.rst. * Bit 58 pte is a guard region (since 6.15) (see madvise (2) man page) * Bits 59-60 zero * Bit 61 page is file-page or shared-anon (since 3.5) @@ -244,7 +247,8 @@ in this IOCTL: Following flags about pages are currently supported: - ``PAGE_IS_WPALLOWED`` - Page has async-write-protection enabled -- ``PAGE_IS_WRITTEN`` - Page has been written to from the time it was write protected +- ``PAGE_IS_WRITTEN`` - Page in a ``UFFDIO_REGISTER_MODE_WP`` VMA has been + written to since it was write-protected. Only reported inside such VMAs. - ``PAGE_IS_FILE`` - Page is file backed - ``PAGE_IS_PRESENT`` - Page is present in the memory - ``PAGE_IS_SWAPPED`` - Page is in swapped @@ -252,6 +256,9 @@ Following flags about pages are currently supported: - ``PAGE_IS_HUGE`` - Page is PMD-mapped THP or Hugetlb backed - ``PAGE_IS_SOFT_DIRTY`` - Page is soft-dirty - ``PAGE_IS_GUARD`` - Page is a part of a guard region +- ``PAGE_IS_ACCESSED`` - Page in a ``UFFDIO_REGISTER_MODE_RWP`` VMA has been + accessed since RWP was applied. Only reported inside such VMAs. See + Documentation/admin-guide/mm/userfaultfd.rst for the RWP workflow. The ``struct pm_scan_arg`` is used as the argument of the IOCTL. diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 23f8d13c2629..16f37135ed80 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -761,7 +761,7 @@ compact_fail but failed. It is possible to establish how long the stalls were using the function -tracer to record how long was spent in __alloc_pages() and +tracer to record how long was spent in the page allocator and using the mm_page_alloc tracepoint to identify which allocations were for huge pages. diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index e5cc8848dcb3..783d969f0e28 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -131,6 +131,16 @@ userfaults on the range registered. Not all ioctls will necessarily be supported for all memory types (e.g. anonymous memory vs. shmem vs. hugetlbfs), or all types of intercepted faults. +.. note:: + + Re-registering an already-registered range must not drop any of the + modes that install per-PTE markers — currently + ``UFFDIO_REGISTER_MODE_WP`` and ``UFFDIO_REGISTER_MODE_RWP``. Doing + so would strand markers with no flag to describe them, so the call + is rejected with ``-EBUSY``; userspace must issue + ``UFFDIO_UNREGISTER`` first. This differs from older kernels, which + silently replaced the mode bits on re-registration. + Userland can use the ``uffdio_register.ioctls`` to manage the virtual address space in the background (to add or potentially also remove memory from the ``userfaultfd`` registered range). This means a userfault @@ -265,16 +275,16 @@ tracking and it can be different in a few ways: - Dirty information will not get lost if the pte was zapped due to various reasons (e.g. during split of a shmem transparent huge page). - - Due to a reverted meaning of soft-dirty (page clean when uffd-wp bit - set; dirty when uffd-wp bit cleared), it has different semantics on - some of the memory operations. For example: ``MADV_DONTNEED`` on + - Due to a reverted meaning of soft-dirty (page clean when the uffd bit + is set; dirty when the uffd bit is cleared), it has different semantics + on some of the memory operations. For example: ``MADV_DONTNEED`` on anonymous (or ``MADV_REMOVE`` on a file mapping) will be treated as - dirtying of memory by dropping uffd-wp bit during the procedure. + dirtying of memory by dropping the uffd bit during the procedure. The user app can collect the "written/dirty" status by looking up the -uffd-wp bit for the pages being interested in /proc/pagemap. +uffd bit for the pages being interested in /proc/pagemap. -The page will not be under track of uffd-wp async mode until the page is +The page will not be under track of userfaultfd-wp async mode until the page is explicitly write-protected by ``ioctl(UFFDIO_WRITEPROTECT)`` with the mode flag ``UFFDIO_WRITEPROTECT_MODE_WP`` set. Trying to resolve a page fault that was tracked by async mode userfaultfd-wp is invalid. @@ -297,6 +307,253 @@ transparent to the guest, we want that same address range to act as if it was still poisoned, even though it's on a new physical host which ostensibly doesn't have a memory error in the exact same spot. +Read-Write Protection +--------------------- + +``UFFDIO_REGISTER_MODE_RWP`` enables read-write protection tracking on a +memory range. It is similar to (but faster than) ``mprotect(PROT_NONE)`` +combined with a signal handler; unlike ``mprotect(PROT_NONE)``, RWP only +traps accesses to *present* PTEs, so accesses to unpopulated addresses in a +protected range fall through to the normal missing-page path. It uses the +PROT_NONE hinting mechanism (same as NUMA balancing) to make pages +inaccessible while keeping them resident in memory. Works on anonymous, +shmem, and hugetlbfs memory. + +RWP is designed for VM memory managers that need to track the working set +of guest memory for cold page eviction to tiered or remote storage. + +**Setup:** + +1. Open a userfaultfd and enable ``UFFD_FEATURE_RWP`` via ``UFFDIO_API``. + Optionally request ``UFFD_FEATURE_RWP_ASYNC`` as well — it requires + ``UFFD_FEATURE_RWP`` to be set in the same ``UFFDIO_API`` call. + +2. Register the guest memory range with ``UFFDIO_REGISTER_MODE_RWP`` + (and ``UFFDIO_REGISTER_MODE_MISSING`` if evicted pages will need to be + fetched back from storage). + +**Feature availability:** + +RWP is built on top of two kernel primitives: a spare PTE bit owned by +userfaultfd (``CONFIG_HAVE_ARCH_USERFAULTFD_WP``) and architecture support +for present-but-inaccessible PTEs (``CONFIG_ARCH_HAS_PTE_PROTNONE``). When both +are available on a 64-bit kernel, the build selects +``CONFIG_USERFAULTFD_RWP=y`` and the ``VM_UFFD_RWP`` VMA flag becomes +available. + +``UFFD_FEATURE_RWP`` and ``UFFD_FEATURE_RWP_ASYNC`` are unavailable when +the running kernel or architecture does not support them — for example +32-bit kernels (where ``VM_UFFD_RWP`` is unavailable), kernels built +without ``CONFIG_USERFAULTFD_RWP``, and architectures whose ptes cannot +carry the uffd bit at runtime (e.g. riscv without the ``SVRSW60T59B`` +extension). Requesting an unsupported feature in +``uffdio_api.features`` makes ``UFFDIO_API`` fail with ``EINVAL`` and +leaves the userfaultfd context uninitialized; the structure is returned +zeroed, so the error path cannot be used to discover what the kernel +supports. The recommended probe sequence is therefore to open a +throwaway userfaultfd, call ``UFFDIO_API`` once with ``features = 0``, +inspect the returned bitmask, close that fd, then open the real one +and call ``UFFDIO_API`` again with only the supported features set. + +**Protecting and Unprotecting:** + +Use ``UFFDIO_RWPROTECT`` to protect or unprotect a range, mirroring the +``UFFDIO_WRITEPROTECT`` interface:: + + struct uffdio_rwprotect rwp = { + .range = { .start = addr, .len = len }, + .mode = UFFDIO_RWPROTECT_MODE_RWP, /* protect */ + }; + ioctl(uffd, UFFDIO_RWPROTECT, &rwp); + +Setting ``UFFDIO_RWPROTECT_MODE_RWP`` sets PROT_NONE on present PTEs in the +range. Pages stay resident and their physical frames are preserved — only +access permissions are removed. + +Clearing ``UFFDIO_RWPROTECT_MODE_RWP`` restores normal VMA permissions and +wakes any faulting threads (unless ``UFFDIO_RWPROTECT_MODE_DONTWAKE`` is set). + +**Scope of protection:** + +RWP protection is a property of *present* PTEs. ``UFFDIO_RWPROTECT`` only +affects entries that are already populated. Unpopulated addresses within +the range remain unpopulated; when first accessed they fault through the +normal missing path (``do_anonymous_page()``, ``do_swap_page()``, +``finish_fault()``) and the resulting PTE is not RWP-protected. To observe +the population itself, co-register the range with +``UFFDIO_REGISTER_MODE_MISSING``. + +Protection is preserved across page reclaim: a page swapped out while +RWP-protected carries the marker on its swap entry, and swap-in restores +the PROT_NONE state so the first access after swap-in still faults. The +same applies to pages temporarily replaced by migration entries. + +Operations that drop the PTE entirely — ``MADV_DONTNEED`` on anonymous +memory, hole-punch on shmem, truncation of a file mapping — also drop the +RWP marker: the next access re-populates the range without protection. +Unlike WP (which persists via ``PTE_MARKER_UFFD_WP``), there is no +persistent RWP marker today. The user needs to re-arm the range with +``UFFDIO_RWPROTECT`` after any operation that explicitly frees PTEs. + +**Fault Handling:** + +When a protected page is accessed: + +- **Sync mode** (default): The faulting thread blocks and a + ``UFFD_PAGEFAULT_FLAG_RWP`` message is delivered to the userfaultfd + handler. The handler resolves the fault with ``UFFDIO_RWPROTECT`` + (clearing ``MODE_RWP``), which restores the PTE permissions and wakes + the faulting thread. + +- **Async mode** (``UFFD_FEATURE_RWP_ASYNC``): The kernel automatically + restores PTE permissions and the thread continues without blocking. No + message is delivered to the handler. + +**Runtime Mode Switching:** + +``UFFDIO_SET_MODE`` toggles ``UFFD_FEATURE_RWP_ASYNC`` at runtime, allowing +the VMM to switch between lightweight async detection and safe sync +eviction without re-registering. The toggle takes ``mmap_write_lock()`` +and calls ``vma_start_write()`` on each UFFD-armed VMA, draining +in-flight per-VMA-locked faults before the new mode takes effect. + +**Working-set detection with PAGEMAP_SCAN:** + +RWP-protected PTEs carry the uffd PTE bit; an access (and, in async mode, its +auto-resolution) clears it. ``PAGEMAP_SCAN`` reports ``PAGE_IS_ACCESSED`` once +the bit is clear on a ``VM_UFFD_RWP`` VMA, so a *non-inverted* scan reports the +pages that were touched during the interval -- the hot set:: + + struct pm_scan_arg arg = { + .size = sizeof(arg), + .start = guest_mem_start, + .end = guest_mem_end, + .vec = (uint64_t)regions, + .vec_len = regions_len, + .category_mask = PAGE_IS_ACCESSED, + .return_mask = PAGE_IS_ACCESSED, + }; + long n = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg); + +The returned ``page_region`` array lists the hot ranges. ``PAGE_IS_ACCESSED`` +is set on an accessed page whether it is still present or has since been +swapped out, so the hot scan needs no ``PAGE_IS_PRESENT`` filter -- unpopulated +holes carry neither bit and are excluded on their own. + +Track the hot set and reclaim everything else from the backing file (see the +workflow below). Do **not** invert the scan to enumerate "cold" pages +directly: an inverted scan reports only the ``VM_UFFD_RWP`` PTEs that are still +protected, i.e. the resident portion of *this* VMA. For a file mapping the +working set spans the whole file -- pages that live in the page cache but are +not mapped into this VMA (a pre-populated tmpfs file, or memory populated +through another mapping) are ``pte_none`` here, never appear in the scan, and +would never be considered for eviction even though they occupy memory. Driving +eviction from "file offsets minus the hot set" avoids that blind spot; a cold +PTE scan cannot. To additionally record the *first* access to a cached but +unmapped page (e.g. pre-populated content) as hot, co-register the range with +``UFFDIO_REGISTER_MODE_MINOR``: such accesses then fault as minor faults +instead of mapping the page silently. + +**Cleanup:** + +When the userfaultfd is closed or the range is unregistered, all PROT_NONE +PTEs are automatically restored to their normal VMA permissions. This +prevents pages from becoming permanently inaccessible. + +**VMM Working Set Tracking Workflow:** + +A typical VMM lifecycle for cold page eviction to tiered storage. Two +mappings of the same shmem (or hugetlbfs) file are used: ``guest_mem`` is +the RWP-registered mapping that vCPUs access through, and ``io_mem`` is a +private mapping for VMM-side I/O. Reading ``io_mem`` does not go through +the RWP-protected PTEs of ``guest_mem``, so the VMM's own ``pwrite()`` +never traps on its own :: + + /* One-time setup */ + fd = memfd_create("guest", MFD_CLOEXEC); + ftruncate(fd, guest_size); + guest_mem = mmap(NULL, guest_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); /* vCPU view, RWP-registered */ + io_mem = mmap(NULL, guest_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); /* VMM I/O view, unprotected */ + + uffd = userfaultfd(O_CLOEXEC | O_NONBLOCK); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }; + ioctl(uffd, UFFDIO_API, &api); + if (!(api.features & UFFD_FEATURE_RWP)) + /* RWP unavailable on this kernel/arch -- fall back. */ + ioctl(uffd, UFFDIO_REGISTER, &(struct uffdio_register){ + .range = { guest_mem, guest_size }, + .mode = UFFDIO_REGISTER_MODE_RWP | + UFFDIO_REGISTER_MODE_MISSING, + }); + + /* Tracking loop */ + while (vm_running) { + /* 1. Detection phase (async -- no vCPU stalls) */ + ioctl(uffd, UFFDIO_RWPROTECT, &(struct uffdio_rwprotect){ + .range = full_range, + .mode = UFFDIO_RWPROTECT_MODE_RWP }); + sleep(tracking_interval); + + /* + * 2. Switch to sync BEFORE scanning. In async mode a vCPU + * access races eviction: it would auto-resolve and mark the + * page hot just as the VMM writes it out and punches it, + * losing the update. Sync mode makes such accesses block and + * be delivered, freezing the hot snapshot for the rest of the + * iteration. + */ + ioctl(uffd, UFFDIO_SET_MODE, + &(struct uffdio_set_mode){ + .disable = UFFD_FEATURE_RWP_ASYNC }); + + /* 3. Read the hot set: pages touched this interval. */ + ioctl(pagemap_fd, PAGEMAP_SCAN, &(struct pm_scan_arg){ + .category_mask = PAGE_IS_ACCESSED, + .return_mask = PAGE_IS_ACCESSED, + ... + }); + + /* + * 4. Reclaim the file offsets that are NOT in the hot set. + * Driving this from the file's offset space (rather than from a + * cold PTE scan) also reclaims pages that are cached but not + * mapped into guest_mem, e.g. pre-populated content. + */ + for each non-hot offset range: + /* Read from io_mem -- bypasses RWP, no fault. */ + pwrite(storage_fd, (char *)io_mem + off, len, off); + /* Drop the page from the shared file. */ + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, len); + /* + * Wake any vCPU blocked on the RWP fault for this range: + * fallocate() does not iterate ctx->fault_pending_wqh. + */ + ioctl(uffd, UFFDIO_WAKE, &(struct uffdio_range){ + .start = (uintptr_t)guest_mem + off, .len = len }); + + /* 5. Resume async tracking */ + ioctl(uffd, UFFDIO_SET_MODE, + &(struct uffdio_set_mode){ + .enable = UFFD_FEATURE_RWP_ASYNC }); + } + +During step 4, a vCPU that accesses a ``guest_mem`` offset being evicted +blocks with a ``UFFD_PAGEFAULT_FLAG_RWP`` fault while the eviction is in +progress. After ``fallocate()`` punches the page out and ``UFFDIO_WAKE`` +fires, the vCPU retries the access, faults as ``MISSING``, and the +handler resolves it with ``UFFDIO_COPY`` from storage. + +This workflow targets shmem and hugetlbfs (both support a private +``io_mem`` mapping over the same fd). Anonymous-memory backings need a +different inner-loop strategy because the VMM has no way to read the +page without going through the RWP-protected mapping. + QEMU/KVM ======== diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index b9b0c218bfb4..5b318d17aa4b 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -19,9 +19,6 @@ The files in this directory can be used to tune the operation of the virtual memory (VM) subsystem of the Linux kernel and the writeout of dirty data to disk. -Default values and initialization routines for most of these -files can be found in mm/swap.c. - Currently, these files are in /proc/sys/vm: - admin_reserve_kbytes @@ -67,6 +64,7 @@ Currently, these files are in /proc/sys/vm: - page-cluster - page_lock_unfairness - panic_on_oom +- panic_on_unrecoverable_memory_failure - percpu_pagelist_high_fraction - stat_interval - stat_refresh @@ -925,6 +923,85 @@ panic_on_oom=2+kdump gives you very strong tool to investigate why oom happens. You can get snapshot. +panic_on_unrecoverable_memory_failure +====================================== + +When a hardware memory error (e.g. multi-bit ECC) hits a kernel page +that cannot be recovered by the memory failure handler, the default +behaviour is to ignore the error and continue operation. This is +dangerous because the corrupted data remains accessible to the kernel, +risking silent data corruption or a delayed crash when the poisoned +memory is next accessed. + +When enabled, this sysctl triggers a panic on memory failure events +hitting kernel-owned pages that the handler cannot recover: +``PageReserved`` (firmware reservations, kernel image, vDSO, zero +page, and similar memblock-reserved regions), ``PageSlab``, +``PageTable``, and ``PageLargeKmalloc``. These are owned by the +kernel and the memory failure handler cannot reliably evict their +contents. + +Other unrecoverable kernel-owned populations (vmalloc allocations, +kernel stack pages, ...) are not currently covered because the +handler has no page-type signal that distinguishes them from a +userspace folio temporarily off the LRU during migration or +compaction. Such pages still go through the standard +MF_MSG_GET_HWPOISON path: ``PG_hwpoison`` is set on them and a +delayed crash on the next access remains possible. Coverage may +grow as the handler gains stronger kernel-ownership signals. + +Recoverable failure paths are also intentionally left out: in-flight +buddy allocations and other transient races with the page allocator +can reach the same diagnostic, and panicking on them would risk +killing the box for a page destined for userspace where the standard +SIGBUS recovery path applies. Pages whose state could not be +classified at all are not covered either, since an unknown state is +not a sound basis for a panic decision. + +For many environments it is preferable to panic immediately with a clean +crash dump that captures the original error context, rather than to +continue and face a random crash later whose cause is difficult to +diagnose. + +Use cases +--------- + +This option is most useful in environments where unattributed crashes +are expensive to debug or where data integrity must take precedence +over availability: + +* Large fleets, where multi-bit ECC errors on kernel pages are observed + regularly and post-mortem analysis of an unrelated downstream crash + (often seconds to minutes after the original error) consumes + significant engineering effort. + +* Systems configured with kdump, where panicking at the moment of the + hardware error produces a vmcore that still contains the faulting + address, the affected page state, and the originating MCE/GHES + record — context that is typically lost by the time a delayed crash + occurs. + +* High-availability clusters that rely on fast, deterministic node + failure for failover, and prefer an immediate panic over silent data + corruption propagating to replicas or persistent storage. + +* Kernel and platform developers reproducing hwpoison issues with + tools such as ``mce-inject`` or error-injection debugfs interfaces, + where panicking on the unrecoverable path makes regressions + immediately visible instead of surfacing as later, unrelated + failures. + += ===================================================================== +0 Try to continue operation (default). +1 Panic immediately. If the ``panic`` sysctl is also non-zero then the + machine will be rebooted. += ===================================================================== + +Example:: + + echo 1 > /proc/sys/vm/panic_on_unrecoverable_memory_failure + + percpu_pagelist_high_fraction ============================= diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst index ccdd1615cf97..1eae24a9c18b 100644 --- a/Documentation/core-api/maple_tree.rst +++ b/Documentation/core-api/maple_tree.rst @@ -17,7 +17,8 @@ supports iterating over a range of entries and going to the previous or next entry in a cache-efficient manner. The tree can also be put into an RCU-safe mode of operation which allows reading and writing concurrently. Writers must synchronize on a lock, which can be the default spinlock, or the user can set -the lock to an external lock of a different type. +the lock to an external lock of a different type. Note that external locks may +interfere with allocations in a low memory situation. The Maple Tree maintains a small memory footprint and was designed to use modern processor cache efficiently. The majority of the users will be able to @@ -42,6 +43,15 @@ successful store operation within a given code segment when allocating cannot be done. Allocations of nodes are relatively small at around 256 bytes. +Since the maple tree uses internal nodes that are allocated and has rules on +data density, erasing an entry may cause allocations to occur. That is, +erasing an entry may consume memory. Users must take care to ensure that they +do not violate the larger system constraints on when and how memory is +allocated. Most situations are fine to allocate, but the pre-allocation +support is provided as a mechanism to avoid trickier situations. There is also +the possibility of using special entries and clean up the tree later, in +extreme circumstances. + .. _maple-tree-normal-api: Normal API @@ -63,7 +73,8 @@ success or an error code otherwise. mtree_store_range() works in the same way but takes a range. mtree_load() is used to retrieve the entry stored at a given index. You can use mtree_erase() to erase an entire range by only knowing one value within that range, or mtree_store() call with an entry of -NULL may be used to partially erase a range or many ranges at once. +NULL may be used to partially erase a range or many ranges at once. Note that +mtree_erase() may use GFP_KERNEL on allocations. If you want to only store a new entry to a range (or index) if that range is currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which @@ -163,7 +174,9 @@ You can use mas_erase() to erase an entire range by setting index and last of the maple state to the desired range to erase. This will erase the first range that is found in that range, set the maple state index and last as the range that was erased and return the entry that existed -at that location. +at that location. Note that mas_erase() may allocate with the GFP_KERNEL flag. +If this is not okay, consider using mas_store_gfp() and pass it a ``NULL``, +after setting up the correct range by walking to the entry. You can walk each entry within a range by using mas_for_each(). If you want to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as @@ -211,7 +224,7 @@ Advanced Locking The maple tree uses a spinlock by default, but external locks can be used for tree updates as well. To use an external lock, the tree must be initialized -with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the +with the ``MT_FLAGS_LOCK_EXTERN`` flag, this is usually done with the MTREE_INIT_EXT() #define, which takes an external lock as an argument. Functions and structures diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index aabdd3cba58e..c1d03a5a2a19 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -73,6 +73,7 @@ Readahead Writeback --------- +.. kernel-doc:: include/linux/writeback.h .. kernel-doc:: mm/page-writeback.c :export: @@ -117,7 +118,7 @@ More Memory Management Functions .. #kernel-doc:: mm/hmm.c (build warnings) .. kernel-doc:: mm/memremap.c .. kernel-doc:: mm/hugetlb.c -.. kernel-doc:: mm/swap.c +.. kernel-doc:: mm/folio.c .. kernel-doc:: mm/memcontrol.c .. #kernel-doc:: mm/memory-tiers.c (build warnings) .. kernel-doc:: mm/shmem.c diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst index 7d784e03f3f9..a8a83bc69ceb 100644 --- a/Documentation/dev-tools/kmemleak.rst +++ b/Documentation/dev-tools/kmemleak.rst @@ -198,6 +198,14 @@ systems, because of pointers temporarily stored in CPU registers or stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing the minimum age of an object to be reported as a memory leak. +The ``min_unref_scans`` module parameter (default 1) requires an object to +be seen unreferenced in that many consecutive scans before it is reported. +Keeping it at 1 preserves the historical behaviour; higher values filter +the transient false positives described above, at the cost of delaying +genuine reports by up to that many scans. It can be set at boot with +``kmemleak.min_unref_scans=<n>`` or at run-time via +``/sys/module/kmemleak/parameters/min_unref_scans``. + Limitations and Drawbacks ------------------------- diff --git a/Documentation/driver-api/basics.rst b/Documentation/driver-api/basics.rst index 8b6a5888cb11..3b182cfdf135 100644 --- a/Documentation/driver-api/basics.rst +++ b/Documentation/driver-api/basics.rst @@ -90,6 +90,9 @@ Reference counting .. kernel-doc:: lib/refcount.c :export: +.. kernel-doc:: include/linux/percpu-refcount.h +.. kernel-doc:: lib/percpu-refcount.c + Atomics ------- diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 08d01bc62c31..1a50d41a39a1 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -355,13 +355,14 @@ should perform any validation and preparation necessary to ensure that writes can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted through -->swap_rw(), it should set SWP_FS_OPS, otherwise IO will be submitted +->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted directly to the block device ``sis->bdev``. ->swap_deactivate() will be called in the sys_swapoff() path after ->swap_activate() returned success. -->swap_rw will be called for swap IO if SWP_FS_OPS was set by ->swap_activate(). +->swap_rw will be called for swap IO if swap_fs_activate was called by +->swap_activate(). file_lock_operations ==================== diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 2ccd5b2dfdd3..27189f6e004e 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -608,6 +608,7 @@ encoded manner. The codes are the following: um userfaultfd missing tracking uw userfaultfd wr-protect tracking ui userfaultfd minor fault + ur userfaultfd read-write-protect tracking ss shadow/guarded control stack page sl sealed lf lock on fault pages diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 7c753148af88..e7677423a20f 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -977,7 +977,7 @@ cache in your filesystem. The following members are defined: can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted - through ->swap_rw(), it should set SWP_FS_OPS, otherwise IO will + through ->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted directly to the block device ``sis->bdev``. ``swap_deactivate`` @@ -985,7 +985,7 @@ cache in your filesystem. The following members are defined: successful. ``swap_rw`` - Called to read or write swap pages when SWP_FS_OPS is set. + Called to read or write swap pages when swap_fs_activate was called. The File Object =============== diff --git a/Documentation/mm/allocation-profiling.rst b/Documentation/mm/allocation-profiling.rst index 5389d241176a..c3a28467955f 100644 --- a/Documentation/mm/allocation-profiling.rst +++ b/Documentation/mm/allocation-profiling.rst @@ -46,6 +46,11 @@ sysctl: Runtime info: /proc/allocinfo + Profiling data can be retrieved either by reading `/proc/allocinfo` directly as + text or programmatically via `ioctl()` calls defined in `<uapi/linux/alloc_tag.h>`. + The ioctl interface supports structured binary data extraction as well as filtering + by module name, function, file, line number, accuracy, or allocation size limits. + Example output:: root@moria-kvm:~# sort -g /proc/allocinfo|tail|numfmt --to=iec diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 2da7ca0d3d17..aed6cb1cf483 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -86,7 +86,7 @@ To know how user-space can do the configuration via :ref:`DAMON sysfs interface documentation. - .. _damon_design_vaddr_target_regions_construction: +.. _damon_design_vaddr_target_regions_construction: VMA-based Target Address Range Construction ------------------------------------------- @@ -316,6 +316,31 @@ Another way to do this for higher accuracy is using :ref:`DAMOS filter information in page level. But, because it is operated in page level, the overhead is proportional to the size of the memory. +.. _damon_design_attrs_only_monitoring: + +Data Attributes-only Monitoring +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Data access is the primary monitoring information for DAMON. Hence it +:ref:`adjusts <damon_design_data_attrs_monitoring>` regions using the access +:ref:`counter <damon_design_region_based_sampling>` (``nr_accesses``). In some +use cases, however, users may want some of :ref:`attributes +<damon_design_data_attrs_monitoring>` to be the primary information. + +Data attributes-only monitoring mode supports this use case. For the mode, +each attribute probe has their priority weight value. Users can describe by +what combination of the attributes the primary information is decided, by +setting the priority weight value. If the total sum of the weights is not +zero, the mode is enabled. The regions adjustment mechanism uses the weighted +sum of the :ref:`probe hit counts <damon_design_data_attrs_monitoring>` instead +of ``nr_accesses`` in the case. When the mode is enabled, access monitoring is +automatically turned off. The access counter (``nr_accesses``) will always be +zero and not updated. Hence the mode is called Data Attributes "only" +monitoring. + +Refer to the :ref:`admin guide <damon_usage_sysfs_probes>` to know how users +can use the mode. + Dynamic Target Space Updates Handling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -686,9 +711,11 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to (1/10,000). - ``inactive_mem_bp``: Inactive to active + inactive (LRU) memory size ratio in bp (1/10,000). +- ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio + of a node in bp (1/10,000). -``nid`` is optionally required for only ``node_mem_used_bp``, -``node_mem_free_bp``, ``node_memcg_used_bp`` and ``node_memcg_free_bp`` to +``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``, +``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to point the specific NUMA node. ``path`` is optionally required for only ``node_memcg_used_bp`` and @@ -930,11 +957,11 @@ control parameters for the usage would also need to be optimized for the purpose. To support such cases, yet more DAMON API user kernel modules that provide more -simple and optimized user space interfaces are available. Currently, two -modules for proactive reclamation and LRU lists manipulation are provided. For -more detail, please read the usage documents for those -(:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` and -:doc:`/admin-guide/mm/damon/lru_sort`). +simple and optimized user space interfaces are available. Currently, three +modules for access monitoring statistics, proactive reclamation, and LRU lists +manipulation are provided. For more detail, please read the usage documents for +those (:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` +and :doc:`/admin-guide/mm/damon/lru_sort`). .. _damon_design_special_purpose_modules_exclusivity: diff --git a/Documentation/mm/hmm.rst b/Documentation/mm/hmm.rst index 7d61b7a8b65b..fc1b8dc19825 100644 --- a/Documentation/mm/hmm.rst +++ b/Documentation/mm/hmm.rst @@ -156,42 +156,57 @@ During the ops->invalidate() callback the device driver must perform the update action to the range (mark range read only, or fully unmap, etc.). The device must complete the update before the driver callback returns. -When the device driver wants to populate a range of virtual addresses, it can -use:: +When the device driver wants to populate a range of virtual addresses, the +normal interface is:: - int hmm_range_fault(struct hmm_range *range); + int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout); It will trigger a page fault on missing or read-only entries if write access is requested (see below). Page faults use the generic mm page fault code path just -like a CPU page fault. The usage pattern is:: +like a CPU page fault. + +The caller must not hold ``mmap_read_lock`` before the call. +``hmm_range_fault_unlocked_timeout()`` takes the mmap read lock internally and +allows ``handle_mm_fault()`` to drop it during fault handling. This is required +for VMAs whose fault handlers may release the mmap lock, for example regions +managed by ``userfaultfd``. + +If the mmap lock is dropped or the range is invalidated, the function refreshes +``range->notifier_seq`` and restarts the walk internally. ``-EINTR`` is returned +if mmap lock acquisition is interrupted or a fatal signal is pending during +retry handling. + +The timeout is specified in jiffies; passing ``0`` means retry indefinitely. The +timeout exists to preserve caller policy for repeated mmu-notifier invalidation +and is checked between retry attempts. HMM does not interrupt page fault +handling when the timeout expires, but returns ``-EBUSY`` if the retry budget is +exhausted before a stable range is obtained. + +The usage pattern is:: int driver_populate_range(...) { struct hmm_range range; + unsigned long timeout; ... + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); range.notifier = &interval_sub; range.start = ...; range.end = ...; range.hmm_pfns = ...; - if (!mmget_not_zero(interval_sub->notifier.mm)) + if (!mmget_not_zero(interval_sub.mm)) return -EFAULT; again: - range.notifier_seq = mmu_interval_read_begin(&interval_sub); - mmap_read_lock(mm); - ret = hmm_range_fault(&range); - if (ret) { - mmap_read_unlock(mm); - if (ret == -EBUSY) - goto again; - return ret; - } - mmap_read_unlock(mm); + ret = hmm_range_fault_unlocked_timeout(&range, timeout); + if (ret) + goto out_put; take_lock(driver->update); - if (mmu_interval_read_retry(&ni, range.notifier_seq) { + if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) { release_lock(driver->update); goto again; } @@ -200,13 +215,31 @@ like a CPU page fault. The usage pattern is:: * under the update lock */ release_lock(driver->update); - return 0; + ret = 0; + + out_put: + mmput(interval_sub.mm); + return ret; } The driver->update lock is the same lock that the driver takes inside its invalidate() callback. That lock must be held before calling mmu_interval_read_retry() to avoid any race with a concurrent CPU page table -update. +update. The retry check must use the same notifier and sequence number stored +in ``range`` by ``hmm_range_fault_unlocked_timeout()``. + +Holding the mmap lock across HMM faults +======================================= + +Most callers should use ``hmm_range_fault_unlocked_timeout()``. If a driver +really needs to hold the mmap lock across work outside HMM, it can use:: + + int hmm_range_fault(struct hmm_range *range); + +The mmap lock must be held by the caller and will remain held on return. This +interface cannot support VMAs whose fault handlers need to drop the mmap lock. +New callers should prefer ``hmm_range_fault_unlocked_timeout()`` unless they +have a specific requirement to keep the mmap lock held across the call. Leverage default_flags and pfn_flags_mask ========================================= @@ -221,8 +254,8 @@ permission, it sets:: range->default_flags = HMM_PFN_REQ_FAULT; range->pfn_flags_mask = 0; -and calls hmm_range_fault() as described above. This will fill fault all pages -in the range with at least read permission. +and calls the HMM range fault helper as described above. This will fault +all pages in the range with at least read permission. Now let's say the driver wants to do the same except for one page in the range for which it wants to have write permission. Now driver set:: @@ -236,9 +269,9 @@ address == range->start + (index_of_write << PAGE_SHIFT) it will fault with write permission i.e., if the CPU pte does not have write permission set then HMM will call handle_mm_fault(). -After hmm_range_fault completes the flag bits are set to the current state of -the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be set if the page is -writable. +After the HMM range fault helper completes the flag bits are set to the +current state of the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be +set if the page is writable. Represent and manage device memory from core kernel point of view @@ -316,7 +349,7 @@ between device driver specific code and shared common code: system memory and device private memory. One of the first steps migrate_vma_setup() does is to invalidate other - device's MMUs with the ``mmu_notifier_invalidate_range_start(()`` and + device's MMUs with the ``mmu_notifier_invalidate_range_start()`` and ``mmu_notifier_invalidate_range_end()`` calls around the page table walks to fill in the ``args->src`` array with PFNs to be migrated. The ``invalidate_range_start()`` callback is passed a diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst index 6b12f3b007ec..a6bd3fe6423a 100644 --- a/Documentation/mm/page_owner.rst +++ b/Documentation/mm/page_owner.rst @@ -65,7 +65,15 @@ un-tracking state. Usage ===== -1) Build user-space helper:: +1) Build user-space helpers: +:: + + To filter page_owner output: + + cd tools/mm + make page_owner_filter + + To sort and analyze page_owner output: cd tools/mm make page_owner_sort @@ -74,7 +82,11 @@ Usage 3) Do the job that you want to debug. -4) Analyze information from page owner:: +4) (Optional) Filter page_owner output:: + + ./page_owner_filter -m handle -n 0,1,2 > filtered_page_owner.txt + +5) Analyze information from page owner:: cat /sys/kernel/debug/page_owner_stacks/show_stacks > stacks.txt cat stacks.txt @@ -263,3 +275,65 @@ STANDARD FORMAT SPECIFIERS f free whether the page has been released or not st stacktrace stack trace of the page allocation ator allocator memory allocator for pages + +Filtering page_owner output +============================ + +page_owner supports filtering output at the kernel level before reading, +which reduces the amount of data that needs to be processed in userspace. + +The page_owner_filter tool provides a convenient interface for this filtering +capability. It supports two types of filters: + +1. **print_mode filter**: Control what information is printed for each page + - ``stack``: Print full stack traces (default, compatible with existing usage) + - ``handle``: Print only stack handle numbers (much faster, smaller output) + - ``stack_handle``: Print both stack traces and handle numbers + + The ``handle`` mode uses numeric identifiers instead of full stack traces. + The mapping from handles to actual stack traces can be obtained via the + show_stacks_handles interface. + +2. **NUMA node filter**: Filter pages by NUMA node ID + - Supports single node: ``-n 0`` + - Multiple nodes: ``-n 0,1,2`` + - Ranges: ``-n 0-3`` + - Mixed format: ``-n 0,2-3,5`` + +Usage examples:: + + # Filter by print mode + ./page_owner_filter -m handle + ./page_owner_filter -m stack_handle + + # Filter by NUMA node + ./page_owner_filter -n 0 + ./page_owner_filter -n 0-3 + + # Combined filters + ./page_owner_filter -m stack -n 0,1,2 + ./page_owner_filter -m handle -n 0,2-3 + + # Save to file + ./page_owner_filter -m handle -o filtered_output.txt + +The handle mode is particularly useful for monitoring and performance-critical +scenarios as it dramatically reduces output size. Testing shows handle mode can +reduce output size by ~66% (84MB vs 244MB) and improve read performance by ~4.4x +compared to full stack output. + +The NUMA node filter is useful for NUMA-aware memory allocation analysis and debugging. + +Behind the scenes, page_owner_filter opens /sys/kernel/debug/page_owner and +writes filter commands before reading the filtered output. The filtering uses +per-file-descriptor state, allowing each open() to have independent filter settings. + +Each file descriptor maintains its own filter state, so you can have multiple +independent filtering operations running concurrently. For example, in different +terminals you can run different filters simultaneously:: + + # Terminal 1: Filter node 0 + ./page_owner_filter -n 0 > node0_output.txt + + # Terminal 2: Filter node 1 (runs concurrently) + ./page_owner_filter -n 1 > node1_output.txt diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst index b76183545e5b..a09407d72973 100644 --- a/Documentation/mm/physical_memory.rst +++ b/Documentation/mm/physical_memory.rst @@ -262,9 +262,6 @@ General PFN that needs to be initialized. Defined only when ``CONFIG_DEFERRED_STRUCT_PAGE_INIT`` is enabled -``deferred_split_queue`` - Per-node queue of huge pages that their split was deferred. Defined only when ``CONFIG_TRANSPARENT_HUGEPAGE`` is enabled. - ``__lruvec`` Per-node lruvec holding LRU lists and related parameters. Used only when memory cgroups are disabled. It should not be accessed directly, use diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst index 042d64d72421..a7296f251799 100644 --- a/Documentation/mm/process_addrs.rst +++ b/Documentation/mm/process_addrs.rst @@ -724,7 +724,7 @@ the zap and the invocation of :c:func:`!free_pgtables`. Since it is assumed that all such steps have been taken, page table entries are cleared without page table locks (in the :c:func:`!pgd_clear`, :c:func:`!p4d_clear`, -:c:func:`!pud_clear`, and :c:func:`!pmd_clear` functions. +:c:func:`!pud_clear`, and :c:func:`!pmd_clear` functions). .. note:: It is possible for leaf page tables to be torn down independent of the page tables above it as is done by diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 3f0ef1e27eb0..2fc53093752d 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -346,6 +346,8 @@ Code Seq# Include File Comments <mailto:luzmaximilian@gmail.com> 0xA5 20-2F linux/surface_aggregator/dtx.h Microsoft Surface DTX driver <mailto:luzmaximilian@gmail.com> +0xA6 00-0F uapi/linux/alloc_tag.h Memory allocation profiling + <mailto:surenb@google.com> 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h diff --git a/Documentation/userspace-api/mseal.rst b/Documentation/userspace-api/mseal.rst index ea9b11a0bd89..c039d782f9f5 100644 --- a/Documentation/userspace-api/mseal.rst +++ b/Documentation/userspace-api/mseal.rst @@ -50,8 +50,8 @@ mseal syscall signature * The start address (``addr``) is not allocated. * The end address (``addr`` + ``len``) is not allocated. * A gap (unallocated memory) between start and end address. - - **-EPERM**: - * sealing is supported only on 64-bit CPUs, 32-bit is not supported. + - **-ENOSYS**: + * The kernel does not implement ``mseal()``. **Note about error return**: - For above error cases, users can expect the given memory range is @@ -62,7 +62,8 @@ mseal syscall signature memory range could happen. However, those cases should be rare. **Architecture support**: - mseal only works on 64-bit CPUs, not 32-bit CPUs. + mseal is built only for 64-bit kernels. 32-bit kernels return + ``-ENOSYS``. **Idempotent**: users can call mseal multiple times. mseal on an already sealed memory @@ -131,11 +132,11 @@ Use cases - Chrome browser: protect some security sensitive data structures. - System mappings: - The system mappings are created by the kernel and includes vdso, vvar, + The system mappings are created by the kernel and include vdso, vvar, vvar_vclock, vectors (arm compat-mode), sigpage (arm compat-mode), uprobes. Those system mappings are readonly only or execute only, memory sealing can - protect them from ever changing to writable or unmmap/remapped as different + protect them from ever changing to writable or unmapped/remapped as different attributes. This is useful to mitigate memory corruption issues where a corrupted pointer is passed to a memory management system. @@ -143,9 +144,6 @@ Use cases the CONFIG_MSEAL_SYSTEM_MAPPINGS seals all system mappings of this architecture. - The following architectures currently support this feature: x86-64, arm64, - loongarch and s390. - WARNING: This feature breaks programs which rely on relocating or unmapping system mappings. Known broken software at the time of writing includes CHECKPOINT_RESTORE, UML, gVisor, rr. Therefore diff --git a/MAINTAINERS b/MAINTAINERS index 5045562bfdf3..6441e7ae159e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16880,24 +16880,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git fixes F: Documentation/core-api/boot-time-mm.rst F: include/linux/kho/abi/memblock.h F: include/linux/memblock.h -F: mm/bootmem_info.c F: mm/memblock.c F: mm/memtest.c F: mm/mm_init.c +F: mm/mm_init.h F: mm/rodata_test.c F: tools/testing/memblock/ -MEMORY ALLOCATION PROFILING -M: Suren Baghdasaryan <surenb@google.com> -M: Kent Overstreet <kent.overstreet@linux.dev> -R: Hao Ge <hao.ge@linux.dev> -L: linux-mm@kvack.org -S: Maintained -F: Documentation/mm/allocation-profiling.rst -F: include/linux/alloc_tag.h -F: include/linux/pgalloc_tag.h -F: lib/alloc_tag.c - MEMORY CONTROLLER DRIVERS M: Krzysztof Kozlowski <krzk@kernel.org> L: linux-kernel@vger.kernel.org @@ -16942,6 +16931,18 @@ T: quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new F: mm/ F: tools/mm/ +MEMORY MANAGEMENT - ALLOCATION PROFILING (ALLOC TAG) +M: Suren Baghdasaryan <surenb@google.com> +R: Hao Ge <hao.ge@linux.dev> +L: linux-mm@kvack.org +S: Maintained +F: Documentation/mm/allocation-profiling.rst +F: include/linux/alloc_tag.h +F: include/linux/pgalloc_tag.h +F: include/uapi/linux/alloc_tag.h +F: mm/alloc_tag.c +F: tools/testing/selftests/alloc_tag/ + MEMORY MANAGEMENT - BALLOON M: Andrew Morton <akpm@linux-foundation.org> M: David Hildenbrand <david@kernel.org> @@ -17001,6 +17002,7 @@ F: mm/pgtable-generic.c F: mm/ptdump.c F: mm/sparse-vmemmap.c F: mm/sparse.c +F: mm/sparse.h F: mm/util.c F: mm/vmpressure.c F: mm/vmstat.c @@ -17089,7 +17091,7 @@ F: Documentation/admin-guide/mm/multigen_lru.rst F: Documentation/mm/multigen_lru.rst F: include/linux/mm_inline.h F: include/linux/mmzone.h -F: mm/swap.c +F: mm/folio.c F: mm/vmscan.c F: mm/workingset.c @@ -17174,6 +17176,7 @@ F: mm/debug_page_alloc.c F: mm/debug_page_ref.c F: mm/fail_page_alloc.c F: mm/page_alloc.c +F: mm/page_alloc.h F: mm/page_ext.c F: mm/page_frag_cache.c F: mm/page_isolation.c @@ -17211,6 +17214,7 @@ R: Lance Yang <lance.yang@linux.dev> L: linux-mm@kvack.org S: Maintained F: include/linux/rmap.h +F: mm/interval_tree.c F: mm/page_vma_mapped.c F: mm/rmap.c F: tools/testing/selftests/mm/rmap.c @@ -17240,7 +17244,6 @@ F: include/linux/swap.h F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.c -F: mm/swap.c F: mm/swap.h F: mm/swap_table.h F: mm/swap_state.c @@ -17317,7 +17320,6 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: include/trace/events/mmap.h F: fs/proc/task_mmu.c F: fs/proc/task_nommu.c -F: mm/interval_tree.c F: mm/mincore.c F: mm/mlock.c F: mm/mmap.c @@ -28817,6 +28819,7 @@ W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: include/linux/vmalloc.h F: mm/vmalloc.c +F: mm/vmalloc.h F: lib/test_vmalloc.c VME SUBSYSTEM @@ -29786,8 +29789,11 @@ M: Minchan Kim <minchan@kernel.org> M: Sergey Senozhatsky <senozhatsky@chromium.org> L: linux-kernel@vger.kernel.org S: Maintained +F: Documentation/ABI/testing/sysfs-block-zram +F: Documentation/ABI/testing/sysfs-class-zram F: Documentation/admin-guide/blockdev/zram.rst F: drivers/block/zram/ +F: tools/testing/selftests/zram/ ZS DECSTATION Z85C30 SERIAL DRIVER M: "Maciej W. Rozycki" <macro@orcam.me.uk> diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 91e488767783..2769b0f3325f 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -132,7 +132,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, pgoff_t pgoff; int aliases = 0; - pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + pgoff = linear_page_index(vma, addr); /* * If we have any shared mappings that are in the same mm @@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * cache coherency. */ flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(mpnt, mapping, pgoff, pgoff) { /* * If we are using split PTE locks, then we need to take the pte * lock. Otherwise we are using shared mm->page_table_lock which diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 4d7ef5cc36b6..66270904f874 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio * pgoff_end = pgoff + folio_nr_pages(folio) - 1; flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff_end) { unsigned long start, offset, pfn; unsigned int nr; diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..11a9c534b7b4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -17,7 +17,7 @@ config ARM64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION @@ -78,6 +78,7 @@ config ARM64 select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 212ce1b02e15..09d7c00cf405 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -28,11 +28,11 @@ #define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */ -#define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */ +#define PTE_UFFD (_AT(pteval_t, 1) << 58) /* userfaultfd tracking */ +#define PTE_SWP_UFFD (_AT(pteval_t, 1) << 3) /* only for swp ptes */ #else -#define PTE_UFFD_WP (_AT(pteval_t, 0)) -#define PTE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#define PTE_UFFD (_AT(pteval_t, 0)) +#define PTE_SWP_UFFD (_AT(pteval_t, 0)) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..a2681d755358 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -341,19 +341,19 @@ static inline pmd_t pmd_mknoncont(pmd_t pmd) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pte_uffd_wp(pte_t pte) +static inline int pte_uffd(pte_t pte) { - return !!(pte_val(pte) & PTE_UFFD_WP); + return !!(pte_val(pte) & PTE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { - return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP))); + return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD))); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { - return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP)); + return clear_pte_bit(pte, __pgprot(PTE_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -537,26 +537,23 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); + return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } -static inline int pte_swp_uffd_wp(pte_t pte) +static inline int pte_swp_uffd(pte_t pte) { - return !!(pte_val(pte) & PTE_SWP_UFFD_WP); + return !!(pte_val(pte) & PTE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { - return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); + return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/linux/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { /* @@ -575,7 +572,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_present(pmd) pte_present(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) @@ -593,13 +590,13 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd))) #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd)) -#define pmd_mkuffd_wp(pmd) pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))) -#define pmd_clear_uffd_wp(pmd) pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))) -#define pmd_swp_uffd_wp(pmd) pte_swp_uffd_wp(pmd_pte(pmd)) -#define pmd_swp_mkuffd_wp(pmd) pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))) -#define pmd_swp_clear_uffd_wp(pmd) \ - pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))) +#define pmd_uffd(pmd) pte_uffd(pmd_pte(pmd)) +#define pmd_mkuffd(pmd) pte_pmd(pte_mkuffd(pmd_pte(pmd))) +#define pmd_clear_uffd(pmd) pte_pmd(pte_clear_uffd(pmd_pte(pmd))) +#define pmd_swp_uffd(pmd) pte_swp_uffd(pmd_pte(pmd)) +#define pmd_swp_mkuffd(pmd) pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))) +#define pmd_swp_clear_uffd(pmd) \ + pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) @@ -1515,7 +1512,7 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * Encode and decode a swap entry: * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive - * bit 3: remember uffd-wp state + * bit 3: remember uffd state * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset @@ -1534,10 +1531,10 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * Ensure that there are not more swap files than can be encoded in the kernel diff --git a/arch/arm64/include/asm/vmalloc.h b/arch/arm64/include/asm/vmalloc.h index 4ec1acd3c1b3..d665f9d68742 100644 --- a/arch/arm64/include/asm/vmalloc.h +++ b/arch/arm64/include/asm/vmalloc.h @@ -23,10 +23,14 @@ static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr, unsigned long end, u64 pfn, unsigned int max_page_shift) { + unsigned long size; + /* * If the block is at least CONT_PTE_SIZE in size, and is naturally * aligned in both virtual and physical space, then we can pte-map the * block using the PTE_CONT bit for more efficient use of the TLB. + * The returned mapping size may cover multiple CONT_PTE_SIZE blocks, + * capped below PMD_SIZE. */ if (max_page_shift < CONT_PTE_SHIFT) return PAGE_SIZE; @@ -40,7 +44,9 @@ static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr, if (!IS_ALIGNED(PFN_PHYS(pfn), CONT_PTE_SIZE)) return PAGE_SIZE; - return CONT_PTE_SIZE; + size = min3(end - addr, 1UL << max_page_shift, PMD_SIZE >> 1); + size = rounddown_pow_of_two(size); + return size; } #define arch_vmap_pte_range_unmap_size arch_vmap_pte_range_unmap_size diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 8e799c1fe0aa..8429d220660b 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -94,6 +94,11 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, return CONT_PTES; } +/* + * num_contig_ptes(), set_huge_pte_at() and arch_make_huge_pte() can be + * used by non-hugetlbfs(vmalloc) mm code to set multiple huge mappings + * at the PTE level. + */ static inline int num_contig_ptes(unsigned long size, size_t *pgsize) { int contig_ptes = 1; @@ -110,6 +115,12 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize) contig_ptes = CONT_PTES; break; default: + if (size > 0 && size < PMD_SIZE && + IS_ALIGNED(size, CONT_PTE_SIZE)) { + *pgsize = PAGE_SIZE; + contig_ptes = size >> PAGE_SHIFT; + break; + } WARN_ON(!__hugetlb_valid_size(size)); } @@ -359,6 +370,10 @@ pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) case CONT_PTE_SIZE: return pte_mkcont(entry); default: + if (pagesize > 0 && pagesize < PMD_SIZE && + IS_ALIGNED(pagesize, CONT_PTE_SIZE)) + return pte_mkcont(entry); + break; } pr_warn("%s: unrecognized huge page size 0x%lx\n", diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index d011a81575d2..21cc7965c53d 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -42,7 +42,12 @@ static inline void flush_anon_page(struct vm_area_struct *vma, * Use cache_wbinv_all() here and need to be improved in future. */ extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -#define flush_cache_vmap(start, end) cache_wbinv_all() + +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ + cache_wbinv_all(); +} + #define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) cache_wbinv_all() diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d8d252325017..e20acbe5fe7b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -12,7 +12,7 @@ config LOONGARCH select ARCH_NEEDS_DEFER_KASAN select ARCH_DISABLE_KASAN_INLINE select ARCH_ENABLE_MEMORY_HOTPLUG - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER @@ -67,6 +67,7 @@ config LOONGARCH select ARCH_SUPPORTS_LTO_CLANG select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE if 64BIT select ARCH_SUPPORTS_NUMA_BALANCING if NUMA select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 223528c04d73..1952e34bc8ee 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -623,7 +623,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline long pte_protnone(pte_t pte) { return (pte_val(pte) & _PAGE_PROTNONE); @@ -633,7 +633,7 @@ static inline long pmd_protnone(pmd_t pmd) { return (pmd_val(pmd) & _PAGE_PROTNONE); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0) #define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0) diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index bd1fc17d3975..94873775fc0f 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -91,8 +91,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) /* Map delay slot emulation page */ base = do_mmap(NULL, STACK_TOP, PAGE_SIZE, PROT_READ | PROT_EXEC, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, 0, 0, &unused, - NULL); + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, + EMPTY_VMA_FLAGS, 0, &unused, NULL); if (IS_ERR_VALUE(base)) { ret = base; goto out; diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c index 8321182eb927..10232476031f 100644 --- a/arch/nios2/mm/cacheflush.c +++ b/arch/nios2/mm/cacheflush.c @@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio) pgoff = folio->index; flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long start; if (vma->vm_mm != mm) diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 0170b69a21d3..98200b6b5b82 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio) * on machines that support equivalent aliasing */ flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long offset = pgoff - vma->vm_pgoff; unsigned long pfn = folio_pfn(folio); diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index e67e64ac6e8c..f4db7d7fbd5c 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -490,13 +490,13 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte) } #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) == cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_hw_valid(pte_t pte) { @@ -1060,19 +1060,19 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) #define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))) #define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))) #define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd)) #define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))) #endif #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 10aced261cff..cf692b2b5f7b 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1314,7 +1314,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * covering out both edges. */ unsigned long addr; - unsigned long addr_pfn = start_pfn; unsigned long next; pgd_t *pgd; p4d_t *p4d; @@ -1335,7 +1334,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (pmd_leaf(READ_ONCE(*pmd))) { /* existing huge mapping. Skip the range */ - addr_pfn += (PMD_SIZE >> PAGE_SHIFT); next = pmd_addr_end(addr, end); continue; } @@ -1348,11 +1346,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * page whose VMEMMAP_RESERVE_NR pages were mapped and * this request fall in those pages. */ - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } else { unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); pte_t *tail_page_pte; @@ -1376,7 +1374,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte) return -ENOMEM; - addr_pfn += 2; next = addr + 2 * PAGE_SIZE; continue; } @@ -1392,7 +1389,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } @@ -1402,7 +1398,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c index 28bec5bc7879..82127e31dca6 100644 --- a/arch/powerpc/mm/book3s64/slice.c +++ b/arch/powerpc/mm/book3s64/slice.c @@ -659,7 +659,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned int psize; if (radix_enabled()) - return generic_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); + return generic_get_unmapped_area(filp, addr, len, pgoff, flags, + legacy_to_vma_flags(vm_flags)); if (filp && is_file_hugepages(filp)) psize = file_to_psize(filp); @@ -679,7 +680,8 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned int psize; if (radix_enabled()) - return generic_get_unmapped_area_topdown(filp, addr0, len, pgoff, flags, vm_flags); + return generic_get_unmapped_area_topdown(filp, addr0, len, + pgoff, flags, legacy_to_vma_flags(vm_flags)); if (filp && is_file_hugepages(filp)) psize = file_to_psize(filp); diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 558fafb82b8a..a298746dc143 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -104,17 +104,14 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p } } -static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) +static __init void *pseries_alloc_bootmem_huge_page(struct hstate *hstate) { - struct huge_bootmem_page *m; + void *m; if (nr_gpages == 0) - return 0; + return NULL; m = phys_to_virt(gpage_freearray[--nr_gpages]); gpage_freearray[nr_gpages] = 0; - list_add(&m->list, &huge_boot_pages[0]); - m->hstate = hstate; - m->flags = 0; - return 1; + return m; } bool __init hugetlb_node_alloc_supported(void) @@ -124,7 +121,7 @@ bool __init hugetlb_node_alloc_supported(void) #endif -int __init alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) { #ifdef CONFIG_PPC_BOOK3S_64 diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index bac02c83bb3e..f0adbf7ecb7f 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -87,6 +87,7 @@ config PPC_BOOK3S_64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK select ARCH_SUPPORTS_HUGETLBFS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select HAVE_MOVE_PMD select HAVE_MOVE_PUD @@ -112,7 +113,7 @@ config PPC_THP depends on PPC_RADIX_MMU || (PPC_64S_HASH_MMU && PAGE_SIZE_64KB) select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE choice prompt "CPU selection" diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index f7028caaeae0..7b9c373d82fa 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -22,7 +22,7 @@ config RISCV select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CURRENT_STACK_POINTER @@ -71,6 +71,7 @@ config RISCV select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU select ARCH_SUPPORTS_PER_VMA_LOCK if MMU + select ARCH_HAS_PTE_PROTNONE if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_SCHED_MC if SMP diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index b422d9691e60..d5a86b4df3ce 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -40,20 +40,20 @@ #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */ -#define _PAGE_UFFD_WP \ +/* ext_svrsw60t59b: Bit(60) for userfaultfd tracking */ +#define _PAGE_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ (1UL << 60) : 0) /* * Bit 4 is not involved into swap entry computation, so we - * can borrow it for swap page uffd-wp tracking. + * can borrow it for swap page userfaultfd tracking. */ -#define _PAGE_SWP_UFFD_WP \ +#define _PAGE_SWP_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ _PAGE_USER : 0) #else -#define _PAGE_UFFD_WP 0 -#define _PAGE_SWP_UFFD_WP 0 +#define _PAGE_UFFD 0 +#define _PAGE_SWP_UFFD 0 #endif #define _PAGE_TABLE _PAGE_PRESENT diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5d5756bda82e..7f21742b2b6a 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -410,37 +410,37 @@ static inline pte_t pte_wrprotect(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pgtable_supports_uffd_wp() \ +#define pgtable_supports_uffd() \ riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B) -static inline bool pte_uffd_wp(pte_t pte) +static inline bool pte_uffd(pte_t pte) { - return !!(pte_val(pte) & _PAGE_UFFD_WP); + return !!(pte_val(pte) & _PAGE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { - return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP)); + return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } -static inline bool pte_swp_uffd_wp(pte_t pte) +static inline bool pte_swp_uffd(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP); + return !!(pte_val(pte) & _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP); + return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -534,10 +534,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) PAGE_SIZE) #endif -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/asm-generic/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; @@ -547,7 +544,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) @@ -899,34 +896,34 @@ static inline pud_t pud_mkspecial(pud_t pud) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline bool pmd_uffd_wp(pmd_t pmd) +static inline bool pmd_uffd(pmd_t pmd) { - return pte_uffd_wp(pmd_pte(pmd)); + return pte_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { - return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_clear_uffd(pmd_pte(pmd))); } -static inline bool pmd_swp_uffd_wp(pmd_t pmd) +static inline bool pmd_swp_uffd(pmd_t pmd) { - return pte_swp_uffd_wp(pmd_pte(pmd)); + return pte_swp_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { - return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -946,7 +943,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline bool pmd_swp_soft_dirty(pmd_t pmd) { return pte_swp_soft_dirty(pmd_pte(pmd)); @@ -961,7 +958,7 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, @@ -1170,7 +1167,7 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot) * bit 0: _PAGE_PRESENT (zero) * bit 1 to 2: (zero) * bit 3: _PAGE_SWP_SOFT_DIRTY - * bit 4: _PAGE_SWP_UFFD_WP + * bit 4: _PAGE_SWP_UFFD * bit 5: _PAGE_PROT_NONE (zero) * bit 6: exclusive marker * bits 7 to 11: swap type @@ -1208,10 +1205,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * In the RV64 Linux scheme, we give the user half of the virtual-address space diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..ab8fccc2cc4e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -82,7 +82,7 @@ config S390 select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER @@ -148,6 +148,7 @@ config S390 select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index e33a5b587ee4..02821a7d9d91 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -77,20 +77,20 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte)); } -#define __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { return 0; } diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 859ce7c7d454..e882663a58e7 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -842,7 +842,7 @@ static inline int pte_same(pte_t a, pte_t b) return pte_val(a) == pte_val(b); } -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return pte_present(pte) && !(pte_val(pte) & _PAGE_READ); @@ -853,7 +853,7 @@ static inline int pmd_protnone(pmd_t pmd) /* pmd_leaf(pmd) implies pmd_present(pmd) */ return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_swp_exclusive(pte_t pte) { @@ -903,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) #define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) #define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 94ee06ae8913..d2879ce860a1 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -39,7 +39,6 @@ static void __ref *vmem_alloc_pages(unsigned int order) static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap) { - unsigned int nr_pages = 1 << order; struct page *page; if (altmap) { @@ -49,8 +48,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a page = virt_to_page((void *)addr); if (PageReserved(page)) { /* allocated from memblock */ - while (nr_pages--) - free_reserved_page(page++); + free_reserved_pages(page, order); } else { free_pages(addr, order); } diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 74ede706fb32..0837ebbc5dce 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -945,10 +945,6 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, #define pte_clear(mm,addr,ptep) \ set_pte_at((mm), (addr), (ptep), __pte(0UL)) -#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \ - __set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm)) - #ifdef DCACHE_ALIASING_POSSIBLE #define __HAVE_ARCH_MOVE_PTE #define move_pte(pte, old_addr, new_addr) \ diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..fb298e219179 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -70,7 +70,7 @@ config X86 select ARCH_ENABLE_HUGEPAGE_MIGRATION if X86_64 && HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE) - select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if X86_64 && TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_ATTACK_VECTORS if CPU_MITIGATIONS select ARCH_HAS_CACHE_LINE_SIZE @@ -124,6 +124,7 @@ config X86 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64 + select ARCH_HAS_PTE_PROTNONE if X86_64 select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 select ARCH_SUPPORTS_CFI if X86_64 diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 91a093d8cf2e..70be80211d82 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -832,7 +832,7 @@ static void *dsalloc_pages(size_t size, gfp_t flags, int cpu) int node = cpu_to_node(cpu); struct page *page; - page = __alloc_pages_node(node, flags | __GFP_ZERO, order); + page = alloc_pages_node(node, flags | __GFP_ZERO, order); return page ? page_address(page) : NULL; } @@ -1088,9 +1088,9 @@ void init_arch_pebs_on_cpu(int cpu) /* * 4KB-aligned pointer of the output buffer - * (__alloc_pages_node() return page aligned address) + * (alloc_pages_node() returns page aligned address) * Buffer Size = 4KB * 2^SIZE - * contiguous physical buffer (__alloc_pages_node() with order) + * contiguous physical buffer (alloc_pages_node() with order) */ arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT; wrmsrq_on_cpu(cpu, MSR_IA32_PEBS_BASE, arch_pebs_base); diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f..8e0018fadd14 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -411,19 +411,19 @@ static inline pte_t pte_wrprotect(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pte_uffd_wp(pte_t pte) +static inline int pte_uffd(pte_t pte) { - return pte_flags(pte) & _PAGE_UFFD_WP; + return pte_flags(pte) & _PAGE_UFFD; } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { - return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD_WP)); + return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { - return pte_clear_flags(pte, _PAGE_UFFD_WP); + return pte_clear_flags(pte, _PAGE_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -526,19 +526,19 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pmd_uffd_wp(pmd_t pmd) +static inline int pmd_uffd(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_UFFD_WP; + return pmd_flags(pmd) & _PAGE_UFFD; } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { - return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD_WP)); + return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD)); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_UFFD_WP); + return pmd_clear_flags(pmd, _PAGE_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -985,11 +985,7 @@ static inline int pmd_present(pmd_t pmd) return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE); } -#ifdef CONFIG_NUMA_BALANCING -/* - * These work without NUMA balancing but the kernel does not care. See the - * comment in include/linux/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT)) @@ -1001,7 +997,7 @@ static inline int pmd_protnone(pmd_t pmd) return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT)) == _PAGE_PROTNONE; } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline int pmd_none(pmd_t pmd) { @@ -1545,7 +1541,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY); @@ -1564,34 +1560,34 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { - return pte_set_flags(pte, _PAGE_SWP_UFFD_WP); + return pte_set_flags(pte, _PAGE_SWP_UFFD); } -static inline int pte_swp_uffd_wp(pte_t pte) +static inline int pte_swp_uffd(pte_t pte) { - return pte_flags(pte) & _PAGE_SWP_UFFD_WP; + return pte_flags(pte) & _PAGE_SWP_UFFD; } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { - return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP); + return pte_clear_flags(pte, _PAGE_SWP_UFFD); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { - return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP); + return pmd_set_flags(pmd, _PAGE_SWP_UFFD); } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP; + return pmd_flags(pmd) & _PAGE_SWP_UFFD; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP); + return pmd_clear_flags(pmd, _PAGE_SWP_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 2ec250ba467e..af08d98be930 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -31,7 +31,7 @@ #define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 -#define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */ +#define _PAGE_BIT_UFFD _PAGE_BIT_SOFTW2 /* userfaultfd tracking */ #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_KERNEL_4K _PAGE_BIT_SOFTW3 /* page must not be converted to large */ @@ -39,7 +39,7 @@ #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */ #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */ #else -/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */ +/* Shared with _PAGE_BIT_UFFD which is not supported on 32 bit */ #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */ #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */ #endif @@ -111,11 +111,11 @@ #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP) -#define _PAGE_SWP_UFFD_WP _PAGE_USER +#define _PAGE_UFFD (_AT(pteval_t, 1) << _PAGE_BIT_UFFD) +#define _PAGE_SWP_UFFD _PAGE_USER #else -#define _PAGE_UFFD_WP (_AT(pteval_t, 0)) -#define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#define _PAGE_UFFD (_AT(pteval_t, 0)) +#define _PAGE_SWP_UFFD (_AT(pteval_t, 0)) #endif #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) @@ -129,7 +129,7 @@ /* * The hardware requires shadow stack to be Write=0,Dirty=1. However, * there are valid cases where the kernel might create read-only PTEs that - * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In + * are dirty (e.g., fork(), mprotect(), userfaultfd, soft-dirty tracking). In * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit, * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have * (Write=0,SavedDirty=1,Dirty=0) set. @@ -151,7 +151,7 @@ #define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | \ _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \ - _PAGE_CC | _PAGE_UFFD_WP) + _PAGE_CC | _PAGE_UFFD) #define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT) #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE) diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c index db6806c40483..6a1933ddc6fc 100644 --- a/arch/x86/kernel/cpu/sgx/virt.c +++ b/arch/x86/kernel/cpu/sgx/virt.c @@ -9,6 +9,7 @@ #include <linux/miscdevice.h> #include <linux/mm.h> #include <linux/mman.h> +#include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> #include <linux/slab.h> @@ -41,7 +42,7 @@ static int __sgx_vepc_fault(struct sgx_vepc *vepc, WARN_ON(!mutex_is_locked(&vepc->lock)); /* Calculate index of EPC page in virtual EPC's page_array */ - index = vma->vm_pgoff + PFN_DOWN(addr - vma->vm_start); + index = linear_page_index(vma, addr); epc_page = xa_load(&vepc->page_array, index); if (epc_page) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cc75feec05da..6eaa337bd276 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3027,7 +3027,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) struct page *pages; struct vmcs *vmcs; - pages = __alloc_pages_node(node, flags, 0); + pages = alloc_pages_node(node, flags, 0); if (!pages) return NULL; vmcs = page_address(pages); diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index d57f29ca23a5..ab4c5a02326f 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -33,7 +33,6 @@ #include <linux/nmi.h> #include <linux/gfp.h> #include <linux/kcore.h> -#include <linux/bootmem_info.h> #include <asm/processor.h> #include <asm/bios_ebda.h> @@ -1000,32 +999,12 @@ int arch_add_memory(int nid, u64 start, u64 size, return add_pages(nid, start_pfn, nr_pages, params); } -static void free_reserved_pages(struct page *page, unsigned long nr_pages) +static void __meminit free_pagetable(struct page *page) { - while (nr_pages--) - free_reserved_page(page++); -} - -static void __meminit free_pagetable(struct page *page, int order) -{ - /* bootmem page has reserved flag */ - if (PageReserved(page)) { - unsigned long nr_pages = 1 << order; -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE - enum bootmem_type type = bootmem_type(page); - - if (type == MIX_SECTION_INFO) { - while (nr_pages--) - put_page_bootmem(page++); - } else { - free_reserved_pages(page, nr_pages); - } -#else - free_reserved_pages(page, nr_pages); -#endif - } else { + if (PageReserved(page)) + free_reserved_page(page); + else pagetable_free(page_ptdesc(page)); - } } static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, @@ -1033,19 +1012,12 @@ static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, { unsigned long nr_pages = 1u << order; - if (altmap) { + if (altmap) vmem_altmap_free(altmap, nr_pages); - } else if (PageReserved(page)) { - if (IS_ENABLED(CONFIG_HAVE_BOOTMEM_INFO_NODE) && - bootmem_type(page) == SECTION_INFO) { - while (nr_pages--) - put_page_bootmem(page++); - } else { - free_reserved_pages(page, nr_pages); - } - } else { + else if (PageReserved(page)) + free_reserved_pages(page, order); + else __free_pages(page, order); - } } static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) @@ -1060,7 +1032,7 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) } /* free a pte table */ - free_pagetable(pmd_page(*pmd), 0); + free_pagetable(pmd_page(*pmd)); spin_lock(&init_mm.page_table_lock); pmd_clear(pmd); spin_unlock(&init_mm.page_table_lock); @@ -1078,7 +1050,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) } /* free a pmd table */ - free_pagetable(pud_page(*pud), 0); + free_pagetable(pud_page(*pud)); spin_lock(&init_mm.page_table_lock); pud_clear(pud); spin_unlock(&init_mm.page_table_lock); @@ -1096,7 +1068,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) } /* free a pud table */ - free_pagetable(p4d_page(*p4d), 0); + free_pagetable(p4d_page(*p4d)); spin_lock(&init_mm.page_table_lock); p4d_clear(p4d); spin_unlock(&init_mm.page_table_lock); @@ -1313,16 +1285,6 @@ void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap, static struct kcore_list kcore_vsyscall; -static void __init register_page_bootmem_info(void) -{ -#if defined(CONFIG_NUMA) || defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) - int i; - - for_each_online_node(i) - register_page_bootmem_info_node(NODE_DATA(i)); -#endif -} - /* * Pre-allocates page-table pages for the vmalloc area in the kernel page-table. * Only the level which needs to be synchronized between all page-tables is @@ -1385,14 +1347,6 @@ void __init mem_init(void) after_bootmem = 1; x86_init.hyper.init_after_bootmem(); - /* - * Must be done after boot memory is put on freelist, because here we - * might set fields in deferred struct pages that have not yet been - * initialized, and memblock_free_all() initializes all the reserved - * deferred pages for us. - */ - register_page_bootmem_info(); - /* Register memory areas for /proc/kcore */ if (get_gate_vma(&init_mm)) kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); @@ -1591,72 +1545,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return err; } -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void register_page_bootmem_memmap(unsigned long section_nr, - struct page *start_page, unsigned long nr_pages) -{ - unsigned long addr = (unsigned long)start_page; - unsigned long end = (unsigned long)(start_page + nr_pages); - unsigned long next; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - unsigned int nr_pmd_pages; - struct page *page; - - for (; addr < end; addr = next) { - pte_t *pte = NULL; - - pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO); - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO); - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO); - - pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - - if (!boot_cpu_has(X86_FEATURE_PSE) || !pmd_leaf(*pmd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - get_page_bootmem(section_nr, pmd_page(*pmd), - MIX_SECTION_INFO); - - pte = pte_offset_kernel(pmd, addr); - if (pte_none(*pte)) - continue; - get_page_bootmem(section_nr, pte_page(*pte), - SECTION_INFO); - } else { - next = pmd_addr_end(addr, end); - nr_pmd_pages = (next - addr) >> PAGE_SHIFT; - page = pmd_page(*pmd); - while (nr_pmd_pages--) - get_page_bootmem(section_nr, page++, - SECTION_INFO); - } - } -} -#endif - void __meminit vmemmap_populate_print_last(void) { if (p_start) { diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 7e9091c640be..a236447ac7a2 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -196,7 +196,7 @@ static __init int __x86_vmx_init(void) struct page *page; struct vmcs *vmcs; - page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); if (WARN_ON_ONCE(!page)) { x86_vmx_exit(); return -ENOMEM; diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 4b3a62aa8798..0a0ff9a10946 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -1047,7 +1047,7 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) bool found; int ret; - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); again: found = false; down_write(&xdna->notifier_lock); @@ -1072,24 +1072,9 @@ again: return -EFAULT; } - mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); - mmap_read_lock(mm); - ret = hmm_range_fault(&mapp->range); - mmap_read_unlock(mm); - if (ret) { - if (time_after(jiffies, timeout)) { - ret = -ETIME; - goto put_mm; - } - - if (ret == -EBUSY) { - amdxdna_umap_put(mapp); - mmput(mm); - goto again; - } - + ret = hmm_range_fault_unlocked_timeout(&mapp->range, timeout); + if (ret) goto put_mm; - } down_write(&xdna->notifier_lock); if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { @@ -1107,7 +1092,7 @@ again: put_mm: amdxdna_umap_put(mapp); mmput(mm); - return ret; + return ret == -EBUSY ? -ETIME : ret; } int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index c99f2ab105e5..442ea239bba7 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -231,10 +231,6 @@ static int __init numa_init(int (*init_func)(void)) { int ret; - nodes_clear(numa_nodes_parsed); - nodes_clear(node_possible_map); - nodes_clear(node_online_map); - ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); if (ret < 0) goto out_free_distance; diff --git a/drivers/base/memory.c b/drivers/base/memory.c index bcfe2d9f4adb..5eead3346f1e 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -46,6 +46,15 @@ int mhp_online_type_from_str(const char *str) } return -EINVAL; } +EXPORT_SYMBOL_GPL(mhp_online_type_from_str); + +const char *mhp_online_type_to_str(int online_type) +{ + if (online_type < 0 || online_type >= (int)ARRAY_SIZE(online_type_to_str)) + return NULL; + return online_type_to_str[online_type]; +} +EXPORT_SYMBOL_GPL(mhp_online_type_to_str); #define to_memory_block(dev) container_of(dev, struct memory_block, dev) diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 63253d1de5d7..dcfd896b733d 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -506,11 +506,7 @@ static int mmap_zero_prepare(struct vm_area_desc *desc) if (vma_desc_test(desc, VMA_SHARED_BIT)) return shmem_zero_setup_desc(desc); - /* - * This is a highly unique situation where we mark a MAP_PRIVATE mapping - * of /dev/zero anonymous, despite it not being. - */ - vma_desc_set_anonymous(desc); + /* MAP_PRIVATE semantics are taken care for us by core mm. */ return 0; } @@ -698,7 +694,7 @@ static const struct memdev { #ifdef CONFIG_DEVPORT [4] = { "port", &port_fops, 0, 0 }, #endif - [5] = { "zero", &zero_fops, FMODE_NOWAIT, 0666 }, + [DEVZERO_MINOR] = { "zero", &zero_fops, FMODE_NOWAIT, 0666 }, [7] = { "full", &full_fops, 0, 0666 }, [8] = { "random", &random_fops, FMODE_NOWAIT, 0666 }, [9] = { "urandom", &urandom_fops, FMODE_NOWAIT, 0666 }, diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h index 5909171a4428..bfc31923f3ff 100644 --- a/drivers/dax/bus.h +++ b/drivers/dax/bus.h @@ -16,6 +16,8 @@ struct dax_region; #define IORESOURCE_DAX_STATIC BIT(0) #define IORESOURCE_DAX_KMEM BIT(1) +#define DAX_KMEM_UNPLUGGED (-1) /* Do not create memory blocks */ + struct dax_region *alloc_dax_region(struct device *parent, int region_id, struct range *range, int target_node, unsigned int align, unsigned long flags); diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index a18e2b968e4d..a48d699cf344 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -33,9 +33,7 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) struct dev_dax_range *dax_range = &dev_dax->ranges[i]; struct range *range = &dax_range->range; - /* memory-block align the hotplug range */ - r->start = ALIGN(range->start, memory_block_size_bytes()); - r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; + *r = memory_block_aligned_range(range); if (r->start >= r->end) { r->start = range->start; r->end = range->end; @@ -47,6 +45,8 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) struct dax_kmem_data { const char *res_name; int mgid; + int state; + struct mutex lock; /* protects hotplug state transitions */ struct resource *res[]; }; @@ -65,16 +65,331 @@ static void kmem_put_memory_types(void) mt_put_memory_types(&kmem_memory_types); } +/* True for the online states a kmem dax device can hold. */ +static bool dax_kmem_state_is_online(int state) +{ + return state == MMOP_ONLINE || + state == MMOP_ONLINE_KERNEL || + state == MMOP_ONLINE_MOVABLE; +} + +/** + * dax_kmem_do_hotplug - hotplug memory for dax kmem device + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * @online_type: the online policy to use for the memory blocks + * + * Hotplugs all ranges in the dev_dax region as system memory with the + * provided online policy (offline, online, online_movable, online_kernel). + * + * Returns the number of successfully mapped ranges, or negative error. + */ +static int dax_kmem_do_hotplug(struct dev_dax *dev_dax, + struct dax_kmem_data *data, + int online_type) +{ + struct device *dev = &dev_dax->dev; + int i, rc, added = 0; + mhp_t mhp_flags; + + if (dax_kmem_state_is_online(data->state)) + return -EINVAL; + + if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE) + return -EINVAL; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* + * init_resources() is best-effort: if a reservation conflict + * occurs it keeps the range but leaves res[i]=NULL. For hotplug + * on probe systems, this means kmem will partially online. + * + * We have to keep this behavior not to break those systems. + * For those systems - atomicity only applies to valid ranges. + */ + if (!data->res[i]) + continue; + + mhp_flags = MHP_NID_IS_MGID; + if (dev_dax->memmap_on_memory) + mhp_flags |= MHP_MEMMAP_ON_MEMORY; + + /* + * Ensure that future kexec'd kernels will not treat + * this as RAM automatically. + */ + rc = __add_memory_driver_managed(data->mgid, range.start, + range_len(&range), kmem_name, mhp_flags, + online_type); + + if (rc) { + dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", + i, range.start, range.end); + /* + * Release the reservation for the range that failed to + * add so a later hotremove does not try to remove memory + * that was never added. + */ + if (data->res[i]) { + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } + if (added) + continue; + return rc; + } + added++; + } + + return added; +} + +/** + * dax_kmem_init_resources - create memory regions for dax kmem + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Initializes all the resources for the DAX + * + * Returns the number of successfully mapped ranges, or negative error. + */ +static int dax_kmem_init_resources(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + struct device *dev = &dev_dax->dev; + int i, rc, mapped = 0; + + for (i = 0; i < dev_dax->nr_range; i++) { + struct resource *res; + struct range range; + + rc = dax_kmem_range(dev_dax, i, &range); + if (rc) + continue; + + /* Skip ranges already added */ + if (data->res[i]) + continue; + + /* Region is permanently reserved if hotremove fails. */ + res = request_mem_region(range.start, range_len(&range), + data->res_name); + if (!res) { + dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", + i, range.start, range.end); + /* + * Once some memory has been onlined we can't + * assume that it can be un-onlined safely. + */ + if (mapped) + continue; + return -EBUSY; + } + data->res[i] = res; + /* + * Set flags appropriate for System RAM. Leave ..._BUSY clear + * so that add_memory() can add a child resource. Do not + * inherit flags from the parent since it may set new flags + * unknown to us that will break add_memory() later. + */ + res->flags = IORESOURCE_SYSTEM_RAM; + mapped++; + } + return mapped; +} + +#ifdef CONFIG_MEMORY_HOTREMOVE +/** + * dax_kmem_do_hotremove - hot-remove memory for dax kmem device + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Offlines and removes every currently-added range in the dev_dax region + * atomically: either all ranges are offlined and removed, or none are and + * the device is returned to its prior state. + * + * Returns 0 on success, or a negative errno on failure. + */ +static int dax_kmem_do_hotremove(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + struct device *dev = &dev_dax->dev; + struct range *ranges; + int i, nr_ranges = 0, rc; + + ranges = kmalloc_objs(*ranges, dev_dax->nr_range); + if (!ranges) + return -ENOMEM; + + /* Collect the ranges that were actually added during probe. */ + for (i = 0; i < dev_dax->nr_range; i++) { + struct range range; + + if (!data->res[i]) + continue; + if (dax_kmem_range(dev_dax, i, &range)) + continue; + ranges[nr_ranges++] = range; + } + + /* Nothing added means nothing to remove. */ + if (!nr_ranges) { + kfree(ranges); + return 0; + } + + rc = offline_and_remove_memory_ranges(ranges, nr_ranges); + kfree(ranges); + if (rc) { + /* Recoverable: the ranges rolled back, nothing is leaked yet. */ + dev_err(dev, "hotremove failed, device left online: %d\n", rc); + return rc; + } + + /* All ranges removed; release the reserved resources. */ + for (i = 0; i < dev_dax->nr_range; i++) { + if (!data->res[i]) + continue; + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } + + return 0; +} +#else +static int dax_kmem_do_hotremove(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + return -EBUSY; +} +#endif /* CONFIG_MEMORY_HOTREMOVE */ + +/** + * dax_kmem_cleanup_resources - remove the dax memory resources + * @dev_dax: the dev_dax instance + * @data: the dax_kmem_data structure with resource tracking + * + * Removes all resources in the dev_dax region. + */ +static void dax_kmem_cleanup_resources(struct dev_dax *dev_dax, + struct dax_kmem_data *data) +{ + int i; + + /* + * If the device unbind occurs before memory is hotremoved, we can never + * remove the memory (requires reboot). Attempting an offline operation + * here may cause deadlock and a failure to finish the unbind. + * + * Note: This leaks the resources. + */ + if (WARN(((data->state != DAX_KMEM_UNPLUGGED) && + (data->state != MMOP_OFFLINE)), + "Hotplug memory regions stuck online until reboot")) + return; + + for (i = 0; i < dev_dax->nr_range; i++) { + if (!data->res[i]) + continue; + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } +} + +static int dax_kmem_parse_state(const char *buf) +{ + int online_type; + + /* "unplugged" is kmem-specific - the rest map to MMOP_ */ + if (sysfs_streq(buf, "unplugged")) + return DAX_KMEM_UNPLUGGED; + + online_type = mhp_online_type_from_str(buf); + /* Disallow "offline": it's not useful and creates race conditions */ + if (online_type == MMOP_OFFLINE) + return -EINVAL; + return online_type; +} + +static ssize_t state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_kmem_data *data = dev_get_drvdata(dev); + const char *state_str; + + if (data->state == DAX_KMEM_UNPLUGGED) + state_str = "unplugged"; + else + state_str = mhp_online_type_to_str(data->state); + + return sysfs_emit(buf, "%s\n", state_str ?: "unknown"); +} + +static ssize_t state_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_kmem_data *data = dev_get_drvdata(dev); + int online_type; + int rc; + + online_type = dax_kmem_parse_state(buf); + if (online_type < DAX_KMEM_UNPLUGGED) + return online_type; + + guard(mutex)(&data->lock); + + /* Already in requested state */ + if (data->state == online_type) + return len; + + if (online_type == DAX_KMEM_UNPLUGGED) { + rc = dax_kmem_do_hotremove(dev_dax, data); + if (rc) + return rc; + data->state = DAX_KMEM_UNPLUGGED; + return len; + } + + /* Onlining is only allowed from the unplugged state. */ + if (data->state != DAX_KMEM_UNPLUGGED) + return -EBUSY; + + /* Re-acquire resources if previously unplugged, otherwise no-op */ + rc = dax_kmem_init_resources(dev_dax, data); + if (rc < 0) + return rc; + + rc = dax_kmem_do_hotplug(dev_dax, data, online_type); + if (rc < 0) { + /* Total failure, drop the reservations we took. */ + dax_kmem_cleanup_resources(dev_dax, data); + return rc; + } + + data->state = online_type; + return len; +} + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) { struct device *dev = &dev_dax->dev; unsigned long total_len = 0, orig_len = 0; struct dax_kmem_data *data; struct memory_dev_type *mtype; - int i, rc, mapped = 0; - mhp_t mhp_flags; + int i, rc; int numa_node; int adist = MEMTIER_DEFAULT_DAX_ADISTANCE; + int online_type = mhp_get_default_online_type(); /* * Ensure good NUMA information for the persistent memory. @@ -133,68 +448,26 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) if (rc < 0) goto err_reg_mgid; data->mgid = rc; + data->state = DAX_KMEM_UNPLUGGED; + mutex_init(&data->lock); - for (i = 0; i < dev_dax->nr_range; i++) { - struct resource *res; - struct range range; - - rc = dax_kmem_range(dev_dax, i, &range); - if (rc) - continue; - - /* Region is permanently reserved if hotremove fails. */ - res = request_mem_region(range.start, range_len(&range), data->res_name); - if (!res) { - dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", - i, range.start, range.end); - /* - * Once some memory has been onlined we can't - * assume that it can be un-onlined safely. - */ - if (mapped) - continue; - rc = -EBUSY; - goto err_request_mem; - } - data->res[i] = res; - - /* - * Set flags appropriate for System RAM. Leave ..._BUSY clear - * so that add_memory() can add a child resource. Do not - * inherit flags from the parent since it may set new flags - * unknown to us that will break add_memory() below. - */ - res->flags = IORESOURCE_SYSTEM_RAM; - - mhp_flags = MHP_NID_IS_MGID; - if (dev_dax->memmap_on_memory) - mhp_flags |= MHP_MEMMAP_ON_MEMORY; - - /* - * Ensure that future kexec'd kernels will not treat - * this as RAM automatically. - */ - rc = add_memory_driver_managed(data->mgid, range.start, - range_len(&range), kmem_name, mhp_flags); + dev_set_drvdata(dev, data); - if (rc) { - dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", - i, range.start, range.end); - remove_resource(res); - kfree(res); - data->res[i] = NULL; - if (mapped) - continue; - goto err_request_mem; - } - mapped++; - } + rc = dax_kmem_init_resources(dev_dax, data); + if (rc < 0) + goto err_resources; - dev_set_drvdata(dev, data); + rc = dax_kmem_do_hotplug(dev_dax, data, online_type); + if (rc < 0) + goto err_hotplug; + data->state = online_type; return 0; -err_request_mem: +err_hotplug: + dax_kmem_cleanup_resources(dev_dax, data); +err_resources: + dev_set_drvdata(dev, NULL); memory_group_unregister(data->mgid); err_reg_mgid: kfree(data->res_name); @@ -206,61 +479,71 @@ err_dax_kmem_data: } #ifdef CONFIG_MEMORY_HOTREMOVE -static void dev_dax_kmem_remove(struct dev_dax *dev_dax) +/* + * Remove the device's added ranges with remove_memory(). + * Unlike the sysfs unplug path it never offlines and fails if the blocks are + * online (-EBUSY), so it is safe from unbind. Failures leak until reboot. + * + * Returns 0 only if every added range was removed. + */ +static int dax_kmem_remove_ranges(struct dev_dax *dev_dax, + struct dax_kmem_data *data) { - int i, success = 0; - int node = dev_dax->target_node; struct device *dev = &dev_dax->dev; - struct dax_kmem_data *data = dev_get_drvdata(dev); + int i, rc = 0; - /* - * We have one shot for removing memory, if some memory blocks were not - * offline prior to calling this function remove_memory() will fail, and - * there is no way to hotremove this memory until reboot because device - * unbind will succeed even if we return failure. - */ for (i = 0; i < dev_dax->nr_range; i++) { struct range range; - int rc; - rc = dax_kmem_range(dev_dax, i, &range); - if (rc) + if (!data->res[i] || dax_kmem_range(dev_dax, i, &range)) continue; - - /* range was never added during probe */ - if (!data->res[i]) { - success++; + if (remove_memory(range.start, range_len(&range))) { + dev_warn(dev, "mapping%d: %#llx-%#llx stuck online until reboot\n", + i, range.start, range.end); + rc = -EBUSY; continue; } + remove_resource(data->res[i]); + kfree(data->res[i]); + data->res[i] = NULL; + } + return rc; +} - rc = remove_memory(range.start, range_len(&range)); - if (rc == 0) { - remove_resource(data->res[i]); - kfree(data->res[i]); - data->res[i] = NULL; - success++; - continue; - } +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) +{ + int node = dev_dax->target_node; + struct device *dev = &dev_dax->dev; + struct dax_kmem_data *data = dev_get_drvdata(dev); + + /* + * Remove every range that is still added. dax_kmem_remove_ranges() + * uses remove_memory(), which never offlines: an online block fails + * with -EBUSY rather than deadlocking an uninterruptible unbind. + * + * data->state only tracks daxX.Y/state writes, so it can be stale if + * blocks were toggled via memoryX/state. Do not trust it here and + * attempt simply remove_memory() - which reports the true state of + * each range anyway. Anything left online is leaked until reboot. + */ + if (dax_kmem_remove_ranges(dev_dax, data)) { + dev_err(dev, "Hotplug regions stuck online until reboot\n"); any_hotremove_failed = true; - dev_err(dev, - "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n", - i, range.start, range.end); + return; } - if (success >= dev_dax->nr_range) { - memory_group_unregister(data->mgid); - kfree(data->res_name); - kfree(data); - dev_set_drvdata(dev, NULL); - /* - * Clear the memtype association on successful unplug. - * If not, we have memory blocks left which can be - * offlined/onlined later. We need to keep memory_dev_type - * for that. This implies this reference will be around - * till next reboot. - */ - clear_node_memory_type(node, NULL); - } + memory_group_unregister(data->mgid); + kfree(data->res_name); + kfree(data); + dev_set_drvdata(dev, NULL); + /* + * Clear the memtype association on successful unplug. + * If not, we have memory blocks left which can be + * offlined/onlined later. We need to keep memory_dev_type + * for that. This implies this reference will be around + * till next reboot. + */ + clear_node_memory_type(node, NULL); } #else static void dev_dax_kmem_remove(struct dev_dax *dev_dax) @@ -276,10 +559,21 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) } #endif /* CONFIG_MEMORY_HOTREMOVE */ +static DEVICE_ATTR_RW(state); + +static struct attribute *dev_dax_kmem_attrs[] = { + &dev_attr_state.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dev_dax_kmem); + static struct dax_device_driver device_dax_kmem_driver = { .probe = dev_dax_kmem_probe, .remove = dev_dax_kmem_remove, .type = DAXDRV_KMEM_TYPE, + .drv = { + .dev_groups = dev_dax_kmem_groups, + }, }; static int __init dax_kmem_init(void) diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index e3ed684ddcf2..32a05d889b9a 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1252,7 +1252,7 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, } vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); } diff --git a/drivers/gpu/drm/drm_gem_dma_helper.c b/drivers/gpu/drm/drm_gem_dma_helper.c index 1c00a71ab3c9..7d9612075d31 100644 --- a/drivers/gpu/drm/drm_gem_dma_helper.c +++ b/drivers/gpu/drm/drm_gem_dma_helper.c @@ -540,7 +540,7 @@ int drm_gem_dma_mmap(struct drm_gem_dma_object *dma_obj, struct vm_area_struct * vm_flags_mod(vma, VM_DONTDUMP | VM_DONTEXPAND, VM_PFNMAP); if (dma_obj->map_noncoherent) { - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); ret = dma_mmap_pages(drm_dev_dma_dev(dma_obj->base.dev), vma, vma->vm_end - vma->vm_start, diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c989459eb215..06d019d51d3e 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -764,7 +764,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); if (shmem->map_wc) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 18ddef94da47..cdb1521eb1cf 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -773,8 +773,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, .end = end, .dev_private_owner = dev_private_owner, }; - unsigned long timeout = - jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state; unsigned long *pfns; unsigned long npages = npages_in_range(start, end); @@ -788,22 +787,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, hmm_range.hmm_pfns = pfns; retry: - hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - mmap_read_lock(range->gpusvm->mm); - - while (true) { - err = hmm_range_fault(&hmm_range); - if (err == -EBUSY) { - if (time_after(jiffies, timeout)) - break; - - hmm_range.notifier_seq = - mmu_interval_read_begin(notifier); - continue; - } - break; - } - mmap_read_unlock(range->gpusvm->mm); + err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); if (err) goto err_free; @@ -1423,6 +1407,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, void *zdd; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long remaining; unsigned long i, j; unsigned long npages = npages_in_range(pages_start, pages_end); unsigned long num_dma_mapped; @@ -1437,9 +1422,11 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, struct dma_iova_state *state = &svm_pages->state; retry: - if (time_after(jiffies, timeout)) + if (time_after_eq(jiffies, timeout)) return -EBUSY; + remaining = timeout - jiffies; + hmm_range.notifier_seq = mmu_interval_read_begin(notifier); if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages)) goto set_seqno; @@ -1454,21 +1441,7 @@ retry: } hmm_range.hmm_pfns = pfns; - while (true) { - mmap_read_lock(mm); - err = hmm_range_fault(&hmm_range); - mmap_read_unlock(mm); - - if (err == -EBUSY) { - if (time_after(jiffies, timeout)) - break; - - hmm_range.notifier_seq = - mmu_interval_read_begin(notifier); - continue; - } - break; - } + err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining); mmput(mm); if (err) goto err_free; @@ -1741,8 +1714,7 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, .end = drm_gpusvm_range_end(range), .dev_private_owner = NULL, }; - unsigned long timeout = - jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); unsigned long *pfns; unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), drm_gpusvm_range_end(range)); @@ -1759,24 +1731,12 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, } hmm_range.hmm_pfns = pfns; - while (!time_after(jiffies, timeout)) { - hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - if (time_after(jiffies, timeout)) { - err = -ETIME; - break; - } - - mmap_read_lock(mm); - err = hmm_range_fault(&hmm_range); - mmap_read_unlock(mm); - if (err != -EBUSY) - break; - } + err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); kvfree(pfns); mmput(mm); - return err; + return err == -EBUSY ? -ETIME : err; } EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict); diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 4a794544b7dc..f00c27edbfb9 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -12,7 +12,7 @@ #include <drm/drm_pagemap_util.h> #include <drm/drm_print.h> -#if IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES) #define DRM_PAGEMAP_PMD_ORDER HPAGE_PMD_ORDER #else #define DRM_PAGEMAP_PMD_ORDER (-1) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index b0436a1e103f..f9c8b7b2bfc7 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -6,6 +6,7 @@ #include <drm/drm_prime.h> #include <drm/drm_print.h> #include <linux/dma-mapping.h> +#include <linux/pagemap.h> #include <linux/shmem_fs.h> #include <linux/spinlock.h> #include <linux/vmalloc.h> @@ -132,7 +133,7 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vm_page_prot = vm_get_page_prot(vma->vm_flags); + vm_page_prot = vma_get_page_prot(vma); if (etnaviv_obj->flags & ETNA_BO_WC) { vma->vm_page_prot = pgprot_writecombine(vm_page_prot); @@ -188,7 +189,7 @@ static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) } /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); pfn = page_to_pfn(pages[pgoff]); diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c index 9a6270f3dca6..0208c9259572 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_gem.c +++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c @@ -377,13 +377,13 @@ static int exynos_drm_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct /* non-cachable as default. */ if (exynos_gem->flags & EXYNOS_BO_CACHABLE) - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); else if (exynos_gem->flags & EXYNOS_BO_WC) vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); else vma->vm_page_prot = - pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + pgprot_noncached(vma_get_page_prot(vma)); ret = exynos_drm_gem_mmap_buffer(exynos_gem, vma); if (ret) diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c index 88f1e86c8903..2708e8c68f4c 100644 --- a/drivers/gpu/drm/gma500/gem.c +++ b/drivers/gpu/drm/gma500/gem.c @@ -288,7 +288,7 @@ static vm_fault_t psb_gem_fault(struct vm_fault *vmf) /* Page relative to the VMA start - we must calculate this ourselves because vmf->pgoff is the fake GEM offset */ - page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + page_offset = linear_page_delta(vma, vmf->address); /* CPU view of the page, don't go via the GART for CPU writes */ if (pobj->stolen) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 0644f85c6c8e..9ca90c1bb5b4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -112,7 +112,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, vma = find_vma(mm, addr); if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); else addr = -ENOMEM; mmap_write_unlock(mm); @@ -1024,7 +1024,7 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj, fput(anon); if (obj->ops->mmap_ops) { - vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_decrypted(vma_get_page_prot(vma)); vma->vm_ops = obj->ops->mmap_ops; vma->vm_private_data = obj->base.vma_node.driver_private; return 0; @@ -1035,7 +1035,7 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj, switch (mmo->mmap_type) { case I915_MMAP_TYPE_WC: vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_ops = &vm_ops_cpu; break; @@ -1043,19 +1043,19 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj, GEM_WARN_ON(1); fallthrough; case I915_MMAP_TYPE_WB: - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_UC: vma->vm_page_prot = - pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + pgprot_noncached(vma_get_page_prot(vma)); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_GTT: vma->vm_page_prot = - pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_ops = &vm_ops_gtt; break; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 06543ae60706..ef9440166295 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping) if (folio_mapped(folio)) folio_redirty_for_writepage(&wbc, folio); else - error = shmem_writeout(folio, NULL, NULL); + error = shmem_write_folio(folio); } } diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index efd3d3c9a449..6a78e242de7c 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -9,6 +9,7 @@ #include <linux/spinlock.h> #include <linux/shmem_fs.h> #include <linux/dma-buf.h> +#include <linux/pagemap.h> #include <drm/drm_dumb_buffers.h> #include <drm/drm_prime.h> @@ -360,7 +361,7 @@ static vm_fault_t msm_gem_fault(struct vm_fault *vmf) } /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); pfn = page_to_pfn(pages[pgoff]); @@ -1124,7 +1125,7 @@ static int msm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct struct msm_gem_object *msm_obj = to_msm_bo(obj); vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = msm_gem_pgprot(msm_obj, vma_get_page_prot(vma)); return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 20dba02d6175..9a6ee2e880c0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -55,7 +55,7 @@ static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) goto error_unlock; nouveau_bo_del_io_reserve_lru(bo); - prot = vm_get_page_prot(vma->vm_flags); + prot = vma_get_page_prot(vma); ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); nouveau_bo_add_io_reserve_lru(bo); if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index dcc92131488e..58735446d783 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -678,20 +678,22 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, range.end = notifier->notifier.interval_tree.last + 1; while (true) { - if (time_after(jiffies, timeout)) { + long remaining = timeout - jiffies; + + /* + * The HMM timeout only bounds retries while HMM is walking and + * faulting the range. This fault is handled by a kernel worker, + * so fatal signals from the faulting process cannot stop an + * endless stream of invalidations here. + */ + if (time_after_eq(jiffies, timeout)) { ret = -EBUSY; goto out; } - range.notifier_seq = mmu_interval_read_begin(range.notifier); - mmap_read_lock(mm); - ret = hmm_range_fault(&range); - mmap_read_unlock(mm); - if (ret) { - if (ret == -EBUSY) - continue; + ret = hmm_range_fault_unlocked_timeout(&range, remaining); + if (ret) goto out; - } mutex_lock(&svmm->mutex); if (mmu_interval_read_retry(range.notifier, diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c index ca3fb186bf19..4881777642d2 100644 --- a/drivers/gpu/drm/omapdrm/omap_fbdev.c +++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c @@ -84,7 +84,7 @@ fallback: static int omap_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma) { - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); return fb_deferred_io_mmap(info, vma); } diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 8e013e4f2c6b..fb0e6f556b31 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -5,6 +5,7 @@ */ #include <linux/dma-mapping.h> +#include <linux/pagemap.h> #include <linux/seq_file.h> #include <linux/shmem_fs.h> #include <linux/spinlock.h> @@ -359,7 +360,7 @@ static vm_fault_t omap_gem_fault_1d(struct drm_gem_object *obj, pgoff_t pgoff; /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); if (omap_obj->pages) { omap_gem_cpu_sync_page(obj, pgoff); @@ -407,7 +408,7 @@ static vm_fault_t omap_gem_fault_2d(struct drm_gem_object *obj, const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE); /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); /* * Actual address we start mapping at is rounded down to previous slot @@ -537,9 +538,9 @@ static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_IO | VM_MIXEDMAP); if (omap_obj->flags & OMAP_BO_WC) { - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); } else if (omap_obj->flags & OMAP_BO_UNCACHED) { - vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); } else { /* * We do have some private objects, at least for scanout buffers @@ -557,7 +558,7 @@ static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node); vma_set_file(vma, obj->filp); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); } vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index a1e2eb1ca7bb..770556353968 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -776,7 +776,7 @@ static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *v } vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); if (should_map_wc(bo)) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c index b188539dca0b..9a1dc9f12072 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c @@ -255,7 +255,7 @@ static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj, */ vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP); - vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); if (rk_obj->pages) diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 436394e04812..f76af733ea79 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -13,6 +13,7 @@ #include <linux/dma-buf.h> #include <linux/iommu.h> #include <linux/module.h> +#include <linux/pagemap.h> #include <linux/vmalloc.h> #include <drm/drm_drv.h> @@ -564,7 +565,7 @@ static vm_fault_t tegra_bo_fault(struct vm_fault *vmf) if (!bo->pages) return VM_FAULT_SIGBUS; - offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + offset = linear_page_delta(vma, vmf->address); page = bo->pages[offset]; return vmf_insert_page(vma, vmf->address, page); @@ -601,7 +602,7 @@ int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma) vma->vm_pgoff = vm_pgoff; } else { - pgprot_t prot = vm_get_page_prot(vma->vm_flags); + pgprot_t prot = vma_get_page_prot(vma); vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c index 3c067aadc52d..658eeab73255 100644 --- a/drivers/gpu/drm/ttm/ttm_backup.c +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -160,7 +160,7 @@ ttm_backup_backup_folio(struct file *backup, struct folio *folio, if (writeback && !folio_mapped(to_folio) && folio_clear_dirty_for_io(to_folio)) { folio_set_reclaim(to_folio); - ret = shmem_writeout(to_folio, NULL, NULL); + ret = shmem_write_folio(to_folio); if (!folio_test_writeback(to_folio)) folio_clear_reclaim(to_folio); if (ret == AOP_WRITEPAGE_ACTIVATE) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index a80510489c45..88babf435ac2 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -32,6 +32,7 @@ #define pr_fmt(fmt) "[TTM] " fmt #include <linux/export.h> +#include <linux/pagemap.h> #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> @@ -208,9 +209,9 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, if (unlikely(err != 0)) return VM_FAULT_SIGBUS; - page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); - page_last = vma_pages(vma) + vma->vm_pgoff - + page_offset = linear_page_index(vma, address) - + drm_vma_node_start(&bo->base.vma_node); + page_last = vma_end_pgoff(vma) - drm_vma_node_start(&bo->base.vma_node); if (unlikely(page_offset >= PFN_UP(bo->base.size))) diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c index 4ae3cbc35dd3..544a6abddbc8 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vram.c +++ b/drivers/gpu/drm/virtio/virtgpu_vram.c @@ -55,7 +55,7 @@ static int virtio_gpu_vram_mmap(struct drm_gem_object *obj, vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node); vm_flags_set(vma, VM_MIXEDMAP | VM_DONTEXPAND); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); vma->vm_ops = &virtio_gpu_vram_vm_ops; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index 45561bc1c9ef..a9fd4015a0ca 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -481,7 +481,7 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf) if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE) prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED); else - prot = vm_get_page_prot(vma->vm_flags); + prot = vma_get_page_prot(vma); ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault); if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index dcb48caa485d..9d119c95a569 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -281,7 +281,7 @@ static vm_fault_t barrier_fault(struct vm_fault *vmf) pgprot_t prot; int idx; - prot = vm_get_page_prot(vma->vm_flags); + prot = vma_get_page_prot(vma); if (drm_dev_enter(dev, &idx)) { unsigned long pfn; diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 8c803ef233cc..3741ae60f532 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -149,7 +149,7 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct * /* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 */ vma->vm_pgoff = 0; - vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE); diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c index eec4c1da3f9e..dd158443f55f 100644 --- a/drivers/gpu/drm/xen/xen_drm_front_gem.c +++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c @@ -80,7 +80,7 @@ static int xen_drm_front_gem_object_mmap(struct drm_gem_object *gem_obj, * which is mapped as Normal Inner Write-Back Outer Write-Back * Inner-Shareable. */ - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); /* * vm_operations_struct.fault handler will be called if CPU access diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c index 6d65e5b42152..dddaade31b5d 100644 --- a/drivers/hv/mshv_regions.c +++ b/drivers/hv/mshv_regions.c @@ -382,46 +382,6 @@ int mshv_region_get(struct mshv_mem_region *region) } /** - * mshv_region_hmm_fault_and_lock - Handle HMM faults and lock the memory region - * @region: Pointer to the memory region structure - * @range: Pointer to the HMM range structure - * - * This function performs the following steps: - * 1. Reads the notifier sequence for the HMM range. - * 2. Acquires a read lock on the memory map. - * 3. Handles HMM faults for the specified range. - * 4. Releases the read lock on the memory map. - * 5. If successful, locks the memory region mutex. - * 6. Verifies if the notifier sequence has changed during the operation. - * If it has, releases the mutex and returns -EBUSY to match with - * hmm_range_fault() return code for repeating. - * - * Return: 0 on success, a negative error code otherwise. - */ -static int mshv_region_hmm_fault_and_lock(struct mshv_mem_region *region, - struct hmm_range *range) -{ - int ret; - - range->notifier_seq = mmu_interval_read_begin(range->notifier); - mmap_read_lock(region->mreg_mni.mm); - ret = hmm_range_fault(range); - mmap_read_unlock(region->mreg_mni.mm); - if (ret) - return ret; - - mutex_lock(®ion->mreg_mutex); - - if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { - mutex_unlock(®ion->mreg_mutex); - cond_resched(); - return -EBUSY; - } - - return 0; -} - -/** * mshv_region_range_fault - Handle memory range faults for a given region. * @region: Pointer to the memory region structure. * @page_offset: Offset of the page within the region. @@ -452,13 +412,19 @@ static int mshv_region_range_fault(struct mshv_mem_region *region, range.start = region->start_uaddr + page_offset * HV_HYP_PAGE_SIZE; range.end = range.start + page_count * HV_HYP_PAGE_SIZE; - do { - ret = mshv_region_hmm_fault_and_lock(region, &range); - } while (ret == -EBUSY); - +again: + ret = hmm_range_fault_unlocked_timeout(&range, 0); if (ret) goto out; + mutex_lock(®ion->mreg_mutex); + + if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) { + mutex_unlock(®ion->mreg_mutex); + cond_resched(); + goto again; + } + for (i = 0; i < page_count; i++) region->mreg_pages[page_offset + i] = hmm_pfn_to_page(pfns[i]); diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 404fa1cc3254..9cc21cd762d9 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -329,7 +329,7 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, struct mm_struct *owning_mm = umem_odp->umem.owning_mm; int pfn_index, dma_index, ret = 0, start_idx; unsigned int page_shift, hmm_order, pfn_start_idx; - unsigned long num_pfns, current_seq; + unsigned long num_pfns; struct hmm_range range = {}; unsigned long timeout; @@ -363,26 +363,18 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, } range.hmm_pfns = &(umem_odp->map.pfn_list[pfn_start_idx]); - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); retry: - current_seq = range.notifier_seq = - mmu_interval_read_begin(&umem_odp->notifier); - - mmap_read_lock(owning_mm); - ret = hmm_range_fault(&range); - mmap_read_unlock(owning_mm); - if (unlikely(ret)) { - if (ret == -EBUSY && !time_after(jiffies, timeout)) - goto retry; + ret = hmm_range_fault_unlocked_timeout(&range, timeout); + if (unlikely(ret)) goto out_put_mm; - } start_idx = (range.start - ib_umem_start(umem_odp)) >> page_shift; dma_index = start_idx; mutex_lock(&umem_odp->umem_mutex); - if (mmu_interval_read_retry(&umem_odp->notifier, current_seq)) { + if (mmu_interval_read_retry(&umem_odp->notifier, range.notifier_seq)) { mutex_unlock(&umem_odp->umem_mutex); goto retry; } diff --git a/drivers/misc/sgi-xp/xpc_uv.c b/drivers/misc/sgi-xp/xpc_uv.c index 772c78726893..79c2f00ed4d7 100644 --- a/drivers/misc/sgi-xp/xpc_uv.c +++ b/drivers/misc/sgi-xp/xpc_uv.c @@ -170,9 +170,8 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name, mq->mmr_blade = uv_cpu_to_blade_id(cpu); nid = cpu_to_node(cpu); - page = __alloc_pages_node(nid, - GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE, - pg_order); + page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE, + pg_order); if (page == NULL) { dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d " "bytes of memory on nid=%d for GRU mq\n", mq_size, nid); diff --git a/drivers/net/ethernet/fungible/funeth/funeth_rx.c b/drivers/net/ethernet/fungible/funeth/funeth_rx.c index 7e2584895de3..d7000017ac2b 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_rx.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_rx.c @@ -103,7 +103,7 @@ static int funeth_alloc_page(struct funeth_rxq *q, struct funeth_rxbuf *rb, if (cache_get(q, rb)) return 0; - p = __alloc_pages_node(node, gfp | __GFP_NOWARN, 0); + p = alloc_pages_node(node, gfp | __GFP_NOWARN, 0); if (unlikely(!p)) return -ENOMEM; diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index d07dcacb76bd..963fd8ded20d 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -11,6 +11,7 @@ #include <linux/jiffies.h> #include <linux/sched.h> #include <linux/pci-p2pdma.h> +#include <linux/pagemap.h> #include <linux/pm_runtime.h> #include <linux/memory-failure.h> @@ -385,7 +386,7 @@ static unsigned long addr_to_pgoff(struct vm_area_struct *vma, u64 pgoff = vma->vm_pgoff & ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); - return ((addr - vma->vm_start) >> PAGE_SHIFT) + pgoff; + return linear_page_delta(vma, addr) + pgoff; } static vm_fault_t nvgrace_gpu_vfio_pci_huge_fault(struct vm_fault *vmf, diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3f11a9624b9c..a113c55845e1 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -20,6 +20,7 @@ #include <linux/module.h> #include <linux/mutex.h> #include <linux/notifier.h> +#include <linux/pagemap.h> #include <linux/pci.h> #include <linux/pm_runtime.h> #include <linux/slab.h> @@ -1780,7 +1781,7 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf, struct vm_area_struct *vma = vmf->vma; struct vfio_pci_core_device *vdev = vma->vm_private_data; unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1); - unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; + unsigned long pgoff = linear_page_delta(vma, addr); unsigned long pfn = vma_to_pfn(vma) + pgoff; vm_fault_t ret = VM_FAULT_FALLBACK; diff --git a/drivers/video/fbdev/core/fb_io_fops.c b/drivers/video/fbdev/core/fb_io_fops.c index 6ab60fcd0050..6d0a8c8e141a 100644 --- a/drivers/video/fbdev/core/fb_io_fops.c +++ b/drivers/video/fbdev/core/fb_io_fops.c @@ -161,7 +161,7 @@ int fb_io_mmap(struct fb_info *info, struct vm_area_struct *vma) len = info->fix.mmio_len; } - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); vma->vm_page_prot = pgprot_framebuffer(vma->vm_page_prot, vma->vm_start, vma->vm_end, start); @@ -604,7 +604,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, PROT_READ | PROT_WRITE, - MAP_SHARED, 0, 0, &unused, NULL); + MAP_SHARED, EMPTY_VMA_FLAGS, 0, &unused, NULL); mmap_write_unlock(mm); if (IS_ERR((void *)ctx->mmap_base)) { ctx->mmap_size = 0; diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6c..e3131a311995 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1353,11 +1353,8 @@ out_free_interp: emulate the SVr4 behavior. Sigh. */ error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - - retval = do_mseal(0, PAGE_SIZE, 0); - if (retval) - pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", - task_pid_nr(current), retval); + if (!error) + mseal_mmap_page_zero(); } regs = current_pt_regs(); @@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, /* Walk all mappings of a given index of a file and writeprotect them */ i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, index, end) { + mapping_rmap_tree_foreach(vma, mapping, index, end) { pfn_mkclean_range(pfn, count, index, vma); cond_resched(); } diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b2..d71ed62cb783 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -184,7 +184,8 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, if (addr) addr0 = ALIGN(addr, huge_page_size(h)); - return mm_get_unmapped_area_vmflags(file, addr0, len, pgoff, flags, 0); + return mm_get_unmapped_area_vmaflags(file, addr0, len, pgoff, flags, + EMPTY_VMA_FLAGS); } /* @@ -380,7 +381,6 @@ static void hugetlb_unmap_file_folio(struct hstate *h, struct address_space *mapping, struct folio *folio, pgoff_t index) { - struct rb_root_cached *root = &mapping->i_mmap; struct hugetlb_vma_lock *vma_lock; unsigned long pfn = folio_pfn(folio); struct vm_area_struct *vma; @@ -394,7 +394,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h, i_mmap_lock_write(mapping); retry: vma_lock = NULL; - vma_interval_tree_foreach(vma, root, start, end - 1) { + mapping_rmap_tree_foreach(vma, mapping, start, end - 1) { v_start = vma_offset_start(vma, start); v_end = vma_offset_end(vma, end); @@ -460,8 +460,8 @@ retry: } static void -hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, - zap_flags_t zap_flags) +hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start, + pgoff_t end, zap_flags_t zap_flags) { struct vm_area_struct *vma; @@ -470,7 +470,8 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, * unmapped. Note, end is exclusive, whereas the interval tree takes * an inclusive "last". */ - vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) { + mapping_rmap_tree_foreach(vma, mapping, start, + end ? end - 1 : ULONG_MAX) { unsigned long v_start; unsigned long v_end; @@ -615,8 +616,7 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) i_size_write(inode, offset); i_mmap_lock_write(mapping); if (mapping_mapped(mapping)) - hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0, - ZAP_FLAG_DROP_MARKER); + hugetlb_vmdelete_list(mapping, pgoff, 0, ZAP_FLAG_DROP_MARKER); i_mmap_unlock_write(mapping); remove_inode_hugepages(inode, offset, LLONG_MAX); } @@ -676,7 +676,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) /* Unmap users of full pages in the hole. */ if (hole_end > hole_start) { if (mapping_mapped(mapping)) - hugetlb_vmdelete_list(&mapping->i_mmap, + hugetlb_vmdelete_list(mapping, hole_start >> PAGE_SHIFT, hole_end >> PAGE_SHIFT, 0); } @@ -1419,7 +1419,8 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_free; return 0; out_free: - kfree(sbinfo->spool); + if (sbinfo->spool) + hugepage_put_subpool(sbinfo->spool); kfree(sbinfo); return -ENOMEM; } diff --git a/fs/nfs/file.c b/fs/nfs/file.c index a0d8f1c1cf10..851d93a09988 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -597,7 +597,7 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, ret = rpc_clnt_swap_activate(clnt); if (ret) return ret; - ret = add_swap_extent(sis, 0, sis->max, 0); + ret = swap_fs_activate(sis); if (ret < 0) { rpc_clnt_swap_deactivate(clnt); return ret; @@ -607,8 +607,6 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, if (cl->rpc_ops->enable_swap) cl->rpc_ops->enable_swap(inode); - - sis->flags |= SWP_FS_OPS; return ret; } diff --git a/fs/proc/page.c b/fs/proc/page.c index 7d9387143435..260772b20bd9 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -148,8 +148,6 @@ u64 stable_page_flags(const struct page *page) const struct folio *folio; struct page_snapshot ps; unsigned long k; - unsigned long mapping; - bool is_anon; u64 u = 0; /* @@ -157,24 +155,21 @@ u64 stable_page_flags(const struct page *page) * it differentiates a memory hole from a page with no flags */ if (!page) - return 1 << KPF_NOPAGE; + return BIT_ULL(KPF_NOPAGE); snapshot_page(&ps, page); folio = &ps.folio_snapshot; - k = folio->flags.f; - mapping = (unsigned long)folio->mapping; - is_anon = mapping & FOLIO_MAPPING_ANON; /* * pseudo flags for the well known (anonymous) memory mapped pages */ if (folio_mapped(folio)) - u |= 1 << KPF_MMAP; - if (is_anon) { - u |= 1 << KPF_ANON; - if ((mapping & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_KSM) - u |= 1 << KPF_KSM; + u |= BIT_ULL(KPF_MMAP); + if (folio_test_anon(folio)) { + u |= BIT_ULL(KPF_ANON); + if (folio_test_ksm(folio)) + u |= BIT_ULL(KPF_KSM); } /* @@ -184,36 +179,32 @@ u64 stable_page_flags(const struct page *page) if (ps.idx == 0) u |= kpf_copy_bit(k, KPF_COMPOUND_HEAD, PG_head); else - u |= 1 << KPF_COMPOUND_TAIL; + u |= BIT_ULL(KPF_COMPOUND_TAIL); if (folio_test_hugetlb(folio)) - u |= 1 << KPF_HUGE; + u |= BIT_ULL(KPF_HUGE); else if (folio_test_large(folio) && folio_test_large_rmappable(folio)) { /* Note: we indicate any THPs here, not just PMD-sized ones */ - u |= 1 << KPF_THP; + u |= BIT_ULL(KPF_THP); } else if (is_huge_zero_pfn(ps.pfn)) { - u |= 1 << KPF_ZERO_PAGE; - u |= 1 << KPF_THP; + u |= BIT_ULL(KPF_ZERO_PAGE); + u |= BIT_ULL(KPF_THP); } else if (is_zero_pfn(ps.pfn)) { - u |= 1 << KPF_ZERO_PAGE; + u |= BIT_ULL(KPF_ZERO_PAGE); } if (ps.flags & PAGE_SNAPSHOT_PG_BUDDY) - u |= 1 << KPF_BUDDY; + u |= BIT_ULL(KPF_BUDDY); + + if (ps.flags & PAGE_SNAPSHOT_PG_IDLE) + u |= BIT_ULL(KPF_IDLE); if (folio_test_offline(folio)) - u |= 1 << KPF_OFFLINE; + u |= BIT_ULL(KPF_OFFLINE); if (folio_test_pgtable(folio)) - u |= 1 << KPF_PGTABLE; + u |= BIT_ULL(KPF_PGTABLE); if (folio_test_slab(folio)) - u |= 1 << KPF_SLAB; - -#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) - u |= kpf_copy_bit(k, KPF_IDLE, PG_idle); -#else - if (ps.flags & PAGE_SNAPSHOT_PG_IDLE) - u |= 1 << KPF_IDLE; -#endif + u |= BIT_ULL(KPF_SLAB); u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); @@ -225,16 +216,15 @@ u64 stable_page_flags(const struct page *page) u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); -#define SWAPCACHE ((1 << PG_swapbacked) | (1 << PG_swapcache)) - if ((k & SWAPCACHE) == SWAPCACHE) - u |= 1 << KPF_SWAPCACHE; - u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); + if (folio_test_swapcache(folio)) + u |= BIT_ULL(KPF_SWAPCACHE); + u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); #ifdef CONFIG_MEMORY_FAILURE - if (u & (1 << KPF_HUGE)) + if (u & BIT_ULL(KPF_HUGE)) u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); else u |= kpf_copy_bit(ps.page_snapshot.flags.f, KPF_HWPOISON, PG_hwpoison); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 229d1fc3d7f1..817e3e0f9194 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1237,6 +1237,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR [ilog2(VM_UFFD_MINOR)] = "ui", #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ +#ifdef CONFIG_USERFAULTFD_RWP + [ilog2(VM_UFFD_RWP)] = "ur", +#endif #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK [ilog2(VM_SHADOW_STACK)] = "ss", #endif @@ -2035,14 +2038,14 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) flags |= PM_SOFT_DIRTY; - if (pte_uffd_wp(pte)) + if (pte_uffd(pte)) flags |= PM_UFFD_WP; } else { softleaf_t entry; if (pte_swp_soft_dirty(pte)) flags |= PM_SOFT_DIRTY; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) flags |= PM_UFFD_WP; entry = softleaf_from_pte(pte); if (pm->show_pfn) { @@ -2108,7 +2111,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) flags |= PM_SOFT_DIRTY; - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) flags |= PM_UFFD_WP; if (pm->show_pfn) frame = pmd_pfn(pmd) + idx; @@ -2127,9 +2130,10 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SWAP; if (pmd_swp_soft_dirty(pmd)) flags |= PM_SOFT_DIRTY; - if (pmd_swp_uffd_wp(pmd)) + if (pmd_swp_uffd(pmd)) flags |= PM_UFFD_WP; - page = softleaf_to_page(entry); + if (softleaf_has_pfn(entry)) + page = softleaf_to_page(entry); } if (page) { @@ -2232,14 +2236,14 @@ static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, !hugetlb_pmd_shared(ptep)) flags |= PM_MMAP_EXCLUSIVE; - if (huge_pte_uffd_wp(pte)) + if (huge_pte_uffd(pte)) flags |= PM_UFFD_WP; flags |= PM_PRESENT; if (pm->show_pfn) frame = pte_pfn(pte) + ((addr & ~hmask) >> PAGE_SHIFT); - } else if (pte_swp_uffd_wp_any(pte)) { + } else if (pte_swp_uffd_any(pte)) { flags |= PM_UFFD_WP; } @@ -2280,7 +2284,7 @@ static const struct mm_walk_ops pagemap_ops = { * Bits 5-54 swap offset if swapped * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) * Bit 56 page exclusively mapped - * Bit 57 pte is uffd-wp write-protected + * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP) * Bit 58 pte is a guard region * Bits 59-60 zero * Bit 61 page is file-page or shared-anon @@ -2415,7 +2419,7 @@ static int pagemap_release(struct inode *inode, struct file *file) PAGE_IS_FILE | PAGE_IS_PRESENT | \ PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \ - PAGE_IS_GUARD) + PAGE_IS_GUARD | PAGE_IS_ACCESSED) #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) struct pagemap_scan_private { @@ -2434,15 +2438,17 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, if (pte_none(pte)) { /* - * An unpopulated pte carries no uffd-wp marker, i.e. it is not - * write-protected, the same condition under which the present - * and swap cases below report PAGE_IS_WRITTEN. Report it here - * too so this generic path agrees with the PAGE_IS_WRITTEN fast - * path in pagemap_scan_pmd_entry(), which reports pte_none as - * written and, under PM_SCAN_WP_MATCHING, arms a marker. The - * fast path applies no VMA test, so neither does this. + * An unpopulated pte carries no uffd bit, i.e. it is not + * write-protected. The PAGE_IS_WRITTEN fast path in + * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA; + * gate the pte_none report here the same way so the two paths + * still agree. RWP has no such fast path and an unpopulated + * page is not part of the RWP working set, so it is reported as + * neither. */ - return PAGE_IS_WRITTEN; + if (userfaultfd_wp(vma)) + return PAGE_IS_WRITTEN; + return 0; } if (pte_present(pte)) { @@ -2450,8 +2456,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_PRESENT; - if (!pte_uffd_wp(pte)) - categories |= PAGE_IS_WRITTEN; + if (!pte_uffd(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (p->masks_of_interest & PAGE_IS_FILE) { page = vm_normal_page(vma, addr, pte); @@ -2468,8 +2478,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) - categories |= PAGE_IS_WRITTEN; + if (!pte_swp_uffd_any(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } entry = softleaf_from_pte(pte); if (softleaf_is_guard_marker(entry)) @@ -2493,13 +2507,13 @@ static void make_uffd_wp_pte(struct vm_area_struct *vma, pte_t old_pte; old_pte = ptep_modify_prot_start(vma, addr, pte); - ptent = pte_mkuffd_wp(old_pte); + ptent = pte_mkuffd(old_pte); ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); } else if (pte_none(ptent)) { set_pte_at(vma->vm_mm, addr, pte, make_pte_marker(PTE_MARKER_UFFD_WP)); } else { - ptent = pte_swp_mkuffd_wp(ptent); + ptent = pte_swp_mkuffd(ptent); set_pte_at(vma->vm_mm, addr, pte, ptent); } } @@ -2518,8 +2532,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, struct page *page; categories |= PAGE_IS_PRESENT; - if (!pmd_uffd_wp(pmd)) - categories |= PAGE_IS_WRITTEN; + if (!pmd_uffd(pmd)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (p->masks_of_interest & PAGE_IS_FILE) { page = vm_normal_page_pmd(vma, addr, pmd); @@ -2533,8 +2551,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, categories |= PAGE_IS_SOFT_DIRTY; } else { categories |= PAGE_IS_SWAPPED; - if (!pmd_swp_uffd_wp(pmd)) - categories |= PAGE_IS_WRITTEN; + if (!pmd_swp_uffd(pmd)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (pmd_swp_soft_dirty(pmd)) categories |= PAGE_IS_SOFT_DIRTY; @@ -2557,17 +2579,18 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma, if (pmd_present(pmd)) { old = pmdp_invalidate_ad(vma, addr, pmdp); - pmd = pmd_mkuffd_wp(old); + pmd = pmd_mkuffd(old); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } else if (pmd_is_migration_entry(pmd)) { - pmd = pmd_swp_mkuffd_wp(pmd); + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #ifdef CONFIG_HUGETLB_PAGE -static unsigned long pagemap_hugetlb_category(pte_t pte) +static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma, + pte_t pte) { unsigned long categories = PAGE_IS_HUGE; @@ -2582,8 +2605,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) if (pte_present(pte)) { categories |= PAGE_IS_PRESENT; - if (!huge_pte_uffd_wp(pte)) - categories |= PAGE_IS_WRITTEN; + if (!huge_pte_uffd(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (!PageAnon(pte_page(pte))) categories |= PAGE_IS_FILE; if (is_zero_pfn(pte_pfn(pte))) @@ -2593,8 +2620,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) } else { categories |= PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) - categories |= PAGE_IS_WRITTEN; + if (!pte_swp_uffd_any(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } if (pte_swp_soft_dirty(pte)) categories |= PAGE_IS_SOFT_DIRTY; } @@ -2621,12 +2652,12 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, if (softleaf_is_migration(entry)) { set_huge_pte_at(vma->vm_mm, addr, ptep, - pte_swp_mkuffd_wp(ptent), psize); + pte_swp_mkuffd(ptent), psize); } else { pte_t old_pte, new_pte; old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); - new_pte = huge_pte_mkuffd_wp(old_pte); + new_pte = huge_pte_mkuffd(old_pte); huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); } } @@ -2859,8 +2890,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { pte_t ptent = ptep_get(pte); - if ((pte_present(ptent) && pte_uffd_wp(ptent)) || - pte_swp_uffd_wp_any(ptent)) + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) continue; make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) @@ -2870,15 +2901,16 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - if (!p->arg.category_anyof_mask && !p->arg.category_inverted && + if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask && + !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { unsigned long next = addr + PAGE_SIZE; pte_t ptent = ptep_get(pte); - if ((pte_present(ptent) && pte_uffd_wp(ptent)) || - pte_swp_uffd_wp_any(ptent)) + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) continue; ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, p, addr, &next); @@ -2945,7 +2977,8 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, /* Go the short route when not write-protecting pages. */ pte = huge_ptep_get(walk->mm, start, ptep); - categories = p->cur_vma_category | pagemap_hugetlb_category(pte); + categories = p->cur_vma_category | + pagemap_hugetlb_category(vma, pte); if (!pagemap_scan_is_interesting_page(categories, p)) return 0; @@ -2957,7 +2990,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); pte = huge_ptep_get(walk->mm, start, ptep); - categories = p->cur_vma_category | pagemap_hugetlb_category(pte); + categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte); if (!pagemap_scan_is_interesting_page(categories, p)) goto out_unlock; diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index 968740e7c9c3..081bb16abeb4 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -3392,9 +3392,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis, * but we could add call to grab a byte range lock to prevent others * from reading or writing the file */ - - sis->flags |= SWP_FS_OPS; - return add_swap_extent(sis, 0, sis->max, 0); + return swap_fs_activate(sis); } static void cifs_swap_deactivate(struct file *file) diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h index 29cab7947980..3ff832ebcea5 100644 --- a/include/asm-generic/fixmap.h +++ b/include/asm-generic/fixmap.h @@ -21,7 +21,7 @@ #define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) #define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * 'index to address' translation. If anyone tries to use the idx * directly without translation, we catch the bug with a NULL-deference @@ -97,5 +97,5 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) #define set_fixmap_io(idx, phys) \ __set_fixmap(idx, phys, FIXMAP_PAGE_IO) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_FIXMAP_H */ diff --git a/include/asm-generic/getorder.h b/include/asm-generic/getorder.h index f2979e3a96b6..875ccae19683 100644 --- a/include/asm-generic/getorder.h +++ b/include/asm-generic/getorder.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_GETORDER_H #define __ASM_GENERIC_GETORDER_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/compiler.h> #include <linux/log2.h> @@ -47,6 +47,6 @@ static __always_inline __attribute_const__ int get_order(unsigned long size) #endif } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_GETORDER_H */ diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h index e1a2e1b7c8e7..635c41cc3479 100644 --- a/include/asm-generic/hugetlb.h +++ b/include/asm-generic/hugetlb.h @@ -37,24 +37,24 @@ static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) return pte_modify(pte, newprot); } -#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { - return huge_pte_wrprotect(pte_mkuffd_wp(pte)); + return huge_pte_wrprotect(pte_mkuffd(pte)); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { - return pte_clear_uffd_wp(pte); + return pte_clear_uffd(pte); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { - return pte_uffd_wp(pte); + return pte_uffd(pte); } #endif diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h index efa6610acbc7..c6b4eafaf4cb 100644 --- a/include/asm-generic/memory_model.h +++ b/include/asm-generic/memory_model.h @@ -4,7 +4,7 @@ #include <linux/pfn.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * supports 3 memory models. @@ -53,7 +53,7 @@ static inline int pfn_valid(unsigned long pfn) */ #define __page_to_pfn(pg) \ ({ const struct page *__pg = (pg); \ - int __sec = memdesc_section(__pg->flags); \ + int __sec = memdesc_section(&__pg->flags); \ (unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec))); \ }) @@ -86,6 +86,6 @@ static inline int pfn_valid(unsigned long pfn) #endif /* CONFIG_DEBUG_VIRTUAL */ #define phys_to_page(phys) pfn_to_page(PHYS_PFN(phys)) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/include/asm-generic/mmu.h b/include/asm-generic/mmu.h index 061838037542..5f78971e3ac2 100644 --- a/include/asm-generic/mmu.h +++ b/include/asm-generic/mmu.h @@ -6,7 +6,7 @@ * This is the mmu.h header for nommu implementations. * Architectures with an MMU need something more complex. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef struct { unsigned long end_brk; diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index 03b7dae47dd4..89c21f84cffb 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOP4D_H #define _PGTABLE_NOP4D_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __PAGETABLE_P4D_FOLDED 1 @@ -54,5 +54,5 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOP4D_H */ diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 8ffd64e7a24c..36b6490ed180 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPMD_H #define _PGTABLE_NOPMD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm-generic/pgtable-nopud.h> @@ -68,6 +68,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPMD_H */ diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index eb70c6d7ceff..356cbfbaab24 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPUD_H #define _PGTABLE_NOPUD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm-generic/pgtable-nop4d.h> @@ -62,5 +62,5 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) #undef pud_addr_end #define pud_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPUD_H */ diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h index 0d85791efdf7..30e88fc1de2f 100644 --- a/include/asm-generic/pgtable_uffd.h +++ b/include/asm-generic/pgtable_uffd.h @@ -2,79 +2,79 @@ #define _ASM_GENERIC_PGTABLE_UFFD_H /* - * Some platforms can customize the uffd-wp bit, making it unavailable + * Some platforms can customize the uffd PTE bit, making it unavailable * even if the architecture provides the resource. * Adding this API allows architectures to add their own checks for the * devices on which the kernel is running. * Note: When overriding it, please make sure the * CONFIG_HAVE_ARCH_USERFAULTFD_WP is part of this macro. */ -#ifndef pgtable_supports_uffd_wp -#define pgtable_supports_uffd_wp() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) +#ifndef pgtable_supports_uffd +#define pgtable_supports_uffd() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) #endif static inline bool uffd_supports_wp_marker(void) { - return pgtable_supports_uffd_wp() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); + return pgtable_supports_uffd() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); } #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static __always_inline int pte_uffd_wp(pte_t pte) +static __always_inline int pte_uffd(pte_t pte) { return 0; } -static __always_inline int pmd_uffd_wp(pmd_t pmd) +static __always_inline int pmd_uffd(pmd_t pmd) { return 0; } -static __always_inline pte_t pte_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_mkuffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_mkuffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_clear_uffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_clear_uffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_mkuffd(pte_t pte) { return pte; } -static __always_inline int pte_swp_uffd_wp(pte_t pte) +static __always_inline int pte_swp_uffd(pte_t pte) { return 0; } -static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_clear_uffd(pte_t pte) { return pte; } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { return pmd; } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { return 0; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { return pmd; } diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 6ed9f82e639f..7f2d80a59792 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -165,11 +165,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) { WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } -void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags); +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags); #else static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} static inline void alloc_tag_sub_check(union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {} +static inline void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) {} #endif /* Caller should verify both ref and tag to be valid */ diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index 5b7d12b40d5e..c2284466e7aa 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -76,8 +76,6 @@ static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item) return percpu_counter_sum_positive(&wb->stat[item]); } -extern void wb_writeout_inc(struct bdi_writeback *wb); - /* * maximal error of a stat counter. */ diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h deleted file mode 100644 index f724340755e5..000000000000 --- a/include/linux/bootmem_info.h +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_BOOTMEM_INFO_H -#define __LINUX_BOOTMEM_INFO_H - -#include <linux/mm.h> -#include <linux/kmemleak.h> - -/* - * Types for free bootmem stored in the low bits of page->private. - */ -enum bootmem_type { - MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 1, - SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE, - MIX_SECTION_INFO, - NODE_INFO, - MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO, -}; - -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void __init register_page_bootmem_info_node(struct pglist_data *pgdat); -void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, - unsigned long nr_pages); - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type); -void put_page_bootmem(struct page *page); - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return (unsigned long)page->private & 0xf; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return (unsigned long)page->private >> 4; -} - -/* - * Any memory allocated via the memblock allocator and not via the - * buddy will be marked reserved already in the memmap. For those - * pages, we can call this function to free it to buddy allocator. - */ -static inline void free_bootmem_page(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - VM_BUG_ON_PAGE(page_ref_count(page) != 2, page); - - if (type == SECTION_INFO || type == MIX_SECTION_INFO) - put_page_bootmem(page); - else - VM_BUG_ON_PAGE(1, page); -} -#else -static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) -{ -} - -static inline void register_page_bootmem_memmap(unsigned long section_nr, - struct page *map, unsigned long nr_pages) -{ -} - -static inline void put_page_bootmem(struct page *page) -{ -} - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return SECTION_INFO; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return 0; -} - -static inline void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ -} - -static inline void free_bootmem_page(struct page *page) -{ - free_reserved_page(page); -} -#endif - -#endif /* __LINUX_BOOTMEM_INFO_H */ diff --git a/include/linux/codetag.h b/include/linux/codetag.h index ddae7484ca45..a25a085c2df1 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -77,6 +77,8 @@ struct codetag_iterator { void codetag_lock_module_list(struct codetag_type *cttype); bool codetag_trylock_module_list(struct codetag_type *cttype); void codetag_unlock_module_list(struct codetag_type *cttype); +unsigned long codetag_get_content_id(struct codetag_type *cttype); +unsigned int codetag_get_count(struct codetag_type *cttype); struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); struct codetag *codetag_next_ct(struct codetag_iterator *iter); diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..a80d38e752d2 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -83,7 +83,7 @@ extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask); extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask); @@ -224,7 +224,7 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) #define cpuset_current_mems_allowed (node_states[N_MEMORY]) static inline void cpuset_init_current_mems_allowed(void) {} -static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +static inline int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return 1; } diff --git a/include/linux/damon.h b/include/linux/damon.h index cfbbf8ba28f6..327f534696aa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -45,13 +45,11 @@ struct damon_size_range { * @ar: The address range of the region. * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. - * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for - * each sampling interval. * @probe_hits: Number of probe-positive region samples. - * @list: List head for siblings. * @age: Age of this region. * - * For any use case, @ar should be non-zero positive size. + * For any use case, @ar should be non-zero positive size. damon_set_regions() + * does the validation. * * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be * increased for every &damon_attrs->sample_interval if an access to the region @@ -59,13 +57,6 @@ struct damon_size_range { * not be done with direct access but with the helper function, * damon_update_region_access_rate(). * - * @nr_accesses_bp is another representation of @nr_accesses in basis point - * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a - * manner similar to moving sum. By the algorithm, this value becomes - * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can - * be used when the aggregation interval is too huge and therefore cannot wait - * for it before getting the access monitoring results. - * * @age is initially zero, increased for each aggregation interval, and reset * to zero again if the access frequency is significantly changed. If two * regions are merged into a new region, both @nr_accesses and @age of the new @@ -75,21 +66,19 @@ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; - unsigned int nr_accesses_bp; unsigned char probe_hits[DAMON_MAX_PROBES]; - struct list_head list; - unsigned int age; -/* private: Internal value for age calculation. */ +/* private: internal use only. */ + /* List head for siblings. */ + struct list_head list; + /* for age calculation. */ unsigned int last_nr_accesses; + unsigned char last_probe_hits[DAMON_MAX_PROBES]; }; /** * struct damon_target - Represents a monitoring target. * @pid: The PID of the virtual address space to monitor. - * @nr_regions: Number of monitoring target regions of this target. - * @regions_list: Head of the monitoring target regions of this target. - * @list: List head for siblings. * @obsolete: Whether the commit destination target is obsolete. * * Each monitoring context could have multiple targets. For example, a context @@ -103,10 +92,14 @@ struct damon_region { */ struct damon_target { struct pid *pid; + bool obsolete; +/* private: */ + /* Number of monitoring target regions of this target. */ unsigned int nr_regions; + /* Head of the monitoring target regions of this target. */ struct list_head regions_list; + /* List head for siblings. */ struct list_head list; - bool obsolete; }; /** @@ -182,10 +175,8 @@ enum damos_quota_goal_metric { * @metric: Metric to be used for representing the goal. * @target_value: Target value of @metric to achieve with the tuning. * @current_value: Current value of @metric. - * @last_psi_total: Last measured total PSI * @nid: Node id. * @memcg_id: Memcg id. - * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. The * score is calculated by how close @current_value and @target_value are. Then @@ -208,12 +199,17 @@ struct damos_quota_goal { unsigned long current_value; /* metric-dependent fields */ union { - u64 last_psi_total; struct { int nid; u64 memcg_id; }; +/* private: */ + /* Last measured total PSI */ + u64 last_psi_total; + }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; @@ -232,7 +228,6 @@ enum damos_quota_goal_tuner { * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @goals: Head of quota tuning goals (&damos_quota_goal) list. * @goal_tuner: Goal-based @esz tuning algorithm to use. * @esz: Effective size quota in bytes. * @fail_charge_num: Failed regions charge rate numerator. @@ -259,7 +254,7 @@ enum damos_quota_goal_tuner { * the scheme's action. DAMON then compares it against &sz and uses smaller * one as the effective quota. * - * If @goals is not empty, DAMON calculates yet another size quota based on the + * If goals is not empty, DAMON calculates yet another size quota based on the * goals using its internal feedback loop algorithm, for every @reset_interval. * Then, if the new size quota is smaller than the effective quota, it uses the * new size quota as the effective quota. @@ -280,7 +275,7 @@ struct damos_quota { unsigned long reset_interval; unsigned long ms; unsigned long sz; - struct list_head goals; + enum damos_quota_goal_tuner goal_tuner; unsigned long esz; @@ -292,6 +287,8 @@ struct damos_quota { unsigned int weight_age; /* private: */ + /* Head of quota tuning goals (&damos_quota_goal) list. */ + struct list_head goals; /* For throughput estimation */ unsigned long total_charged_sz; unsigned long total_charged_ns; @@ -435,7 +432,6 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. - * @list: List head for siblings. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -453,6 +449,8 @@ struct damos_filter { int target_idx; struct damon_size_range sz_range; }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; @@ -524,12 +522,8 @@ struct damos_migrate_dests { * @wmarks: Watermarks for automated (in)activation of this scheme. * @migrate_dests: Destination nodes if @action is "migrate_{hot,cold}". * @target_nid: Destination node if @action is "migrate_{hot,cold}". - * @core_filters: Additional set of &struct damos_filter for &action. - * @ops_filters: ops layer handling &struct damos_filter objects list. - * @last_applied: Last @action applied ops-managing entity. * @stat: Statistics of this scheme. * @max_nr_snapshots: Upper limit of nr_snapshots stat. - * @list: List head for siblings. * * For each @apply_interval_us, DAMON finds regions which fit in the * &pattern and applies &action to those. To avoid consuming too much @@ -551,16 +545,7 @@ struct damos_migrate_dests { * * Before applying the &action to a memory region, &struct damon_operations * implementation could check pages of the region and skip &action to respect - * &core_filters - * - * The minimum entity that @action can be applied depends on the underlying - * &struct damon_operations. Since it may not be aligned with the core layer - * abstract, namely &struct damon_region, &struct damon_operations could apply - * @action to same entity multiple times. Large folios that underlying on - * multiple &struct damon region objects could be such examples. The &struct - * damon_operations can use @last_applied to avoid that. DAMOS core logic - * unsets @last_applied when each regions walking for applying the scheme is - * finished. + * &struct damos_filter. * * After applying the &action to each region, &stat is updated. * @@ -571,6 +556,16 @@ struct damos { struct damos_access_pattern pattern; enum damos_action action; unsigned long apply_interval_us; + struct damos_quota quota; + struct damos_watermarks wmarks; + union { + struct { + int target_nid; + struct damos_migrate_dests migrate_dests; + }; + }; + struct damos_stat stat; + unsigned long max_nr_snapshots; /* private: internal use only */ /* * number of sample intervals that should be passed before applying @@ -587,20 +582,25 @@ struct damos { /* whether to reject core/ops filters umatched regions */ bool core_filters_default_reject; bool ops_filters_default_reject; -/* public: */ - struct damos_quota quota; - struct damos_watermarks wmarks; - union { - struct { - int target_nid; - struct damos_migrate_dests migrate_dests; - }; - }; + /* Additional set of &struct damos_filter for &action. */ struct list_head core_filters; + /* ops layer handling &struct damos_filter objects list. */ struct list_head ops_filters; + /* + * Last @action applied ops-managing entity. + * + * The minimum entity that @action can be applied depends on the + * underlying &struct damon_operations. Since it may not be aligned + * with the core layer abstract, namely &struct damon_region, &struct + * damon_operations could apply @action to same entity multiple times. + * Large folios that underlying on multiple &struct damon region + * objects could be such examples. The &struct damon_operations can + * use @last_applied to avoid that. DAMOS core logic unsets + * @last_applied when each regions walking for applying the scheme is + * finished. + */ void *last_applied; - struct damos_stat stat; - unsigned long max_nr_snapshots; + /* List head for siblings. */ struct list_head list; }; @@ -656,7 +656,10 @@ enum damon_ops_id { * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. * @apply_probes should apply the data attribute probes to each region and - * accordingly update the probe hits counter of the region. + * accordingly update the probe hits counter of the region. It should also + * set &damon_region->sampling_addr of each region if ``set_samples`` is true. + * It should also return maximum probe hits weighted sum of regions if + * ``return_max_wsum`` is true. * @get_scheme_score should return the priority score of a region for a scheme * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided @@ -674,7 +677,8 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); - void (*apply_probes)(struct damon_ctx *context); + unsigned int (*apply_probes)(struct damon_ctx *context, + bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, struct damon_region *r, struct damos *scheme); unsigned long (*apply_scheme)(struct damon_ctx *context, @@ -754,7 +758,6 @@ enum damon_filter_type { * @matching: Whether this filter is for the type-matching ones. * @allow: Whether the @type-@matching ones should pass this filter. * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. - * @list: Siblings list. */ struct damon_filter { enum damon_filter_type type; @@ -763,17 +766,22 @@ struct damon_filter { union { u64 memcg_id; }; +/* private: */ + /* Siblings list. */ struct list_head list; }; /** * struct damon_probe - Data region attribute probe. * - * @filters: Filters for assessing if a given region is for this probe. - * @list: Siblings list. + * @weight: Relative priority of the attribute for this probe. */ struct damon_probe { + unsigned int weight; +/* private: */ + /* Filters for assessing if a given region is for this probe. */ struct list_head filters; + /* Siblings list. */ struct list_head list; }; @@ -840,14 +848,9 @@ struct damon_attrs { * thread other than the kdamond should be made using safe DAMON APIs, * including damon_call() and damos_walk(). * - * @ops: Set of monitoring operations for given use cases. - * @probes: Head of probes (&damon_probe) list. * @addr_unit: Scale factor for core to ops address conversion. * @min_region_sz: Minimum region size. * @pause: Pause kdamond main loop. - * @adaptive_targets: Head of monitoring targets (&damon_target) list. - * @schemes: Head of schemes (&damos) list. - * @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct damon_ctx { struct damon_attrs attrs; @@ -896,15 +899,21 @@ struct damon_ctx { struct mutex kdamond_lock; /* public: */ - struct damon_operations ops; - struct list_head probes; unsigned long addr_unit; unsigned long min_region_sz; bool pause; +/* private: */ + /* Set of monitoring operations for given use cases. */ + struct damon_operations ops; + /* Head of monitoring targets (&damon_target) list. */ struct list_head adaptive_targets; + /* Head of probes (&damon_probe) list. */ + struct list_head probes; + /* Head of schemes (&damos) list. */ struct list_head schemes; + /* @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct rnd_state rnd_state; }; @@ -1011,11 +1020,16 @@ struct damon_probe *damon_new_probe(void); void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe); struct damon_region *damon_new_region(unsigned long start, unsigned long end); +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx); +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx); +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs); +void damon_update_region_access_rate(struct damon_region *r, bool accessed); struct damos_filter *damos_new_filter(enum damos_filter_type type, bool matching, bool allow); @@ -1062,21 +1076,23 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; } -static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) +/* Returns number of samples per aggregation interval */ +static inline unsigned int damon_nr_samples_per_aggr( + const struct damon_attrs *attrs) { unsigned long sample_interval; - unsigned long max_nr_accesses; + unsigned long nr_samples; sample_interval = attrs->sample_interval ? : 1; - max_nr_accesses = min(attrs->aggr_interval / sample_interval, + nr_samples = min(attrs->aggr_interval / sample_interval, (unsigned long)UINT_MAX); - return max_nr_accesses ? : 1; + return nr_samples ? : 1; } bool damon_initialized(void); int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs); bool damon_is_running(struct damon_ctx *ctx); int damon_kdamond_pid(struct damon_ctx *ctx); diff --git a/include/linux/gfp.h b/include/linux/gfp.h index cdf95a9f0b87..872bc53f32ec 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -17,28 +17,6 @@ struct mempolicy; #define __default_gfp(a,b,...) b #define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL) -/* Convert GFP flags to their corresponding migrate type */ -#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) -#define GFP_MOVABLE_SHIFT 3 - -static inline int gfp_migratetype(const gfp_t gfp_flags) -{ - VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); - BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); - BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); - BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); - BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> - GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); - - if (unlikely(page_group_by_mobility_disabled)) - return MIGRATE_UNMOVABLE; - - /* Group based on mobility */ - return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; -} -#undef GFP_MOVABLE_MASK -#undef GFP_MOVABLE_SHIFT - static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) { return !!(gfp_flags & __GFP_DIRECT_RECLAIM); @@ -226,10 +204,6 @@ static inline void arch_free_page(struct page *page, int order) { } static inline void arch_alloc_page(struct page *page, int order) { } #endif -struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask); -#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) - struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask); #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__)) @@ -278,25 +252,9 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) dump_stack(); } -/* - * Allocate pages, preferring the node given as nid. The node must be valid and - * online. For more general interface, see alloc_pages_node(). - */ -static inline struct page * -__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) -{ - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - warn_if_node_offline(nid, gfp_mask); - - return __alloc_pages_noprof(gfp_mask, order, nid, NULL); -} - -#define __alloc_pages_node(...) alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__)) - static inline struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp); return __folio_alloc_noprof(gfp, order, nid, NULL); @@ -309,14 +267,7 @@ struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) * prefer the current CPU's closest node. Otherwise node must be valid and * online. */ -static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, - unsigned int order) -{ - if (nid == NUMA_NO_NODE) - nid = numa_mem_id(); - - return __alloc_pages_node_noprof(nid, gfp_mask, order); -} +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order); #define alloc_pages_node(...) alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__)) @@ -395,10 +346,6 @@ extern void free_pages(unsigned long addr, unsigned int order); #define __free_page(page) __free_pages((page), 0) #define free_page(addr) free_pages((addr), 0) -void page_alloc_init_cpuhp(void); -bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); -void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); -void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); void page_alloc_init_late(void); diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 54ca0c88bab6..190191411009 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -136,18 +134,14 @@ enum { * %__GFP_THISNODE forces the allocation to be satisfied from the requested * node with no fallbacks or placement policy enforcements. * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. + * %__GFP_ACCOUNT causes the allocation to be accounted to the active + * cgroup context. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers @@ -320,7 +314,7 @@ enum { * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. * * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. + * accounted to the active cgroup context. * * %GFP_NOWAIT is for kernel allocations that should not stall for direct * reclaim, start physical IO or use any filesystem callback. It is very diff --git a/include/linux/hmm.h b/include/linux/hmm.h index db75ffc949a7..6f04e3932f5b 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -123,6 +123,8 @@ struct hmm_range { * Please see Documentation/mm/hmm.rst for how to use the range API. */ int hmm_range_fault(struct hmm_range *range); +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout); /* * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ad20f7f8c179..c745f7ad2298 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -230,6 +230,7 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, /* Don't have to check pgoff for anonymous vma */ if (!vma_is_anonymous(vma)) { + /* vma_start_pgoff() in mm.h so not available. */ if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, hpage_size >> PAGE_SHIFT)) return false; @@ -390,9 +391,9 @@ static inline bool thp_disabled_by_hw(void) unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags); + vma_flags_t vma_flags); enum split_type { SPLIT_TYPE_UNIFORM, @@ -471,6 +472,24 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, unsigned long address); +/** + * pud_is_huge() - Is this PUD either a huge PUD entry or a software leaf entry? + * @pud: The PUD to check. + * + * This is similar to pmd_is_huge(), but it checks at the PUD level. + * + * Returns: true if this PUD is huge, false otherwise. + */ +static inline bool pud_is_huge(pud_t pud) +{ + if (pud_present(pud)) + return pud_trans_huge(pud); + else if (!pud_none(pud)) + return true; + + return false; +} + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, @@ -529,6 +548,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf); + vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf); extern struct folio *huge_zero_folio; @@ -567,7 +588,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, @@ -614,18 +635,13 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, #define thp_get_unmapped_area NULL static inline unsigned long -thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags) { return 0; } -static inline bool -can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) -{ - return false; -} static inline int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order) @@ -723,6 +739,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, return NULL; } +static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + return 0; +} + static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { return 0; @@ -790,22 +811,21 @@ static inline bool pmd_is_huge(pmd_t pmd) { return false; } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline bool is_pmd_order(unsigned int order) +static inline bool pud_is_huge(pud_t pud) { - return order == HPAGE_PMD_ORDER; + return false; } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline int split_folio_to_list_to_order(struct folio *folio, - struct list_head *list, int new_order) +static inline bool is_pmd_order(unsigned int order) { - return split_huge_page_to_list_to_order(&folio->page, list, new_order); + return order == HPAGE_PMD_ORDER; } static inline int split_folio_to_order(struct folio *folio, int new_order) { - return split_folio_to_list_to_order(folio, NULL, new_order); + return split_huge_page_to_list_to_order(&folio->page, NULL, new_order); } /** diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e9..16c4c4caa126 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -2,6 +2,7 @@ #ifndef _LINUX_HUGETLB_H #define _LINUX_HUGETLB_H +#include <linux/mempolicy.h> #include <linux/mm.h> #include <linux/mm_types.h> #include <linux/mmdebug.h> @@ -154,7 +155,8 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end, bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); void folio_putback_hugetlb(struct folio *folio); -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason); void hugetlb_fix_reserve_counts(struct inode *inode); extern struct mutex *hugetlb_fault_mutex_table; u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); @@ -171,6 +173,7 @@ extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); extern nodemask_t hugetlb_bootmem_nodes; void hugetlb_bootmem_set_nodes(void); @@ -424,7 +427,7 @@ static inline void folio_putback_hugetlb(struct folio *folio) } static inline void move_hugetlb_state(struct folio *old_folio, - struct folio *new_folio, int reason) + struct folio *new_folio, enum migrate_reason reason) { } @@ -673,24 +676,30 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -struct cma; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; - struct cma *cma; -}; - #define HUGE_BOOTMEM_HVO 0x0001 #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 -bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); + +struct mempolicy_interpreted { + int nid; + nodemask_t *nodemask; + enum mempolicy_mode mode; +}; + +enum hugetlb_alloc_flag { + HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT = 0, + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT, +}; + +#define HUGETLB_ALLOC_CHARG_CGROUP_RSVD BIT(HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT) +#define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) + +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, @@ -705,8 +714,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct folio *folio); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); -int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid); bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_add_hstate(unsigned order); @@ -792,8 +801,7 @@ static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, { struct hstate *h = hstate_vma(vma); - return ((address - vma->vm_start) >> huge_page_shift(h)) + - (vma->vm_pgoff >> huge_page_order(h)); + return linear_page_index(vma, address) >> huge_page_order(h); } static inline bool order_is_gigantic(unsigned int order) @@ -956,7 +964,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return modified_mask; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { bool allowed_fallback = false; @@ -1137,9 +1145,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, return NULL; } -static inline int __alloc_bootmem_huge_page(struct hstate *h) +static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid) { - return 0; + return NULL; } static inline struct hstate *hstate_file(struct file *f) @@ -1238,7 +1246,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return 0; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { return false; } @@ -1261,6 +1269,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); +unsigned long huge_pte_dirty(pte_t pte); + static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -1293,6 +1304,10 @@ static inline bool hugetlbfs_pagecache_present( static inline void hugetlb_bootmem_alloc(void) { } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index e5d64b8b59c2..16d72c8c71f6 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -267,5 +267,5 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio, { } -#endif /* CONFIG_MEM_RES_CTLR_HUGETLB */ +#endif /* CONFIG_CGROUP_HUGETLB */ #endif diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 992cd8bd8ed0..4c1476ae3234 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -100,14 +100,29 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) if (pmd_swp_soft_dirty(pmd)) pmd = pmd_swp_clear_soft_dirty(pmd); - if (pmd_swp_uffd_wp(pmd)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (pmd_swp_uffd(pmd)) + pmd = pmd_swp_clear_uffd(pmd); arch_entry = __pmd_to_swp_entry(pmd); /* Temporary until swp_entry_t eliminated. */ return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +/** + * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry. + * @entry: Leaf entry. + * + * This generates an architecture-specific PMD entry that can be utilised to + * encode the metadata the leaf entry encodes. + * + * Returns: Architecture-specific PMD entry encoding leaf entry. + */ +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + /* Temporary until swp_entry_t eliminated. */ + return swp_entry_to_pmd(entry); +} + #else static inline softleaf_t softleaf_from_pmd(pmd_t pmd) @@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return softleaf_mk_none(); } +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + return __pmd(0); +} + #endif /** @@ -567,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_PMD_SOFTLEAVES) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -586,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 621566345406..a6451ecbbe9a 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie); extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); +extern u32 lock_sequence(struct lockdep_map *lock); +#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map) + #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) #define lockdep_assert(cond) \ diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index eae115a26488..55c4b152fedf 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -253,7 +253,8 @@ struct held_lock { unsigned int hardirqs_off:1; unsigned int sync:1; unsigned int references:11; /* 32 bits */ - unsigned int pin_count; + unsigned int pin_count:24; + unsigned int seq_count:8; }; #else /* !CONFIG_LOCKDEP */ diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 4a5631906aff..b723384b3bf4 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -11,7 +11,6 @@ #include <linux/kernel.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> -/* #define CONFIG_MAPLE_RCU_DISABLED */ /* * Allocated nodes are mutable until they have been inserted into the tree, @@ -29,13 +28,11 @@ #define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) #else /* 32bit sizes */ #define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ #define MAPLE_NODE_MASK 255UL @@ -485,6 +482,12 @@ struct ma_state { unsigned char mas_flags; unsigned char end; /* The end of the node */ enum store_type store_type; /* The type of store needed for this operation */ +#ifdef CONFIG_LOCKDEP + u32 ld_seq; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + unsigned long rcu_gp; +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ +#endif /* CONFIG_LOCKDEP */ }; struct ma_wr_state { @@ -572,7 +575,7 @@ void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); @@ -864,9 +867,6 @@ static inline void mt_init(struct maple_tree *mt) static inline bool mt_in_rcu(struct maple_tree *mt) { -#ifdef CONFIG_MAPLE_RCU_DISABLED - return false; -#endif return mt->ma_flags & MT_FLAGS_USE_RCU; } diff --git a/include/linux/memfd.h b/include/linux/memfd.h index b4fda09dab9f..c159e40e3f34 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -14,9 +14,9 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); * to sealing, or 0 otherwise. * * We also update VMA flags if appropriate by manipulating the VMA flags pointed - * to by vm_flags_ptr. + * to by vma_flags_ptr. */ -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr); +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr); struct file *memfd_alloc_file(const char *name, unsigned int flags); int memfd_get_seals(struct file *file); int memfd_add_seals(struct file *file, unsigned int seals); @@ -30,7 +30,7 @@ static inline struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) return ERR_PTR(-EINVAL); } static inline int memfd_check_seals_mmap(struct file *file, - vm_flags_t *vm_flags_ptr) + vma_flags_t *vma_flags_ptr) { return 0; } diff --git a/include/linux/memory.h b/include/linux/memory.h index 463dc02f6cff..1783299073e4 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -20,6 +20,7 @@ #include <linux/compiler.h> #include <linux/mutex.h> #include <linux/memory_hotplug.h> +#include <linux/range.h> #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS) @@ -100,6 +101,32 @@ int arch_get_memory_phys_device(unsigned long start_pfn); unsigned long memory_block_size_bytes(void); int set_memory_block_size_order(unsigned int order); +/** + * memory_block_aligned_range - align a physical address range to memory blocks + * @range: the input range to align + * + * Aligns the start address up and the end address down to memory block + * boundaries. This is required for memory hotplug operations which must + * operate on memory-block aligned ranges. + * + * Returns the aligned range. Callers should check that the returned + * range is valid (aligned.start < aligned.end) before using it. + */ +static inline struct range memory_block_aligned_range(const struct range *range) +{ + struct range aligned; + + aligned.start = ALIGN(range->start, memory_block_size_bytes()); + aligned.end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()); + /* No whole block fits (e.g. range below the first boundary): empty. */ + if (aligned.end <= aligned.start) + aligned.start = aligned.end; + else + aligned.end -= 1; + + return aligned; +} + struct memory_notify { unsigned long start_pfn; unsigned long nr_pages; diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 06c58cb05779..b39605d30896 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -127,6 +127,7 @@ extern int arch_add_memory(int nid, u64 start, u64 size, extern u64 max_mem_size; extern int mhp_online_type_from_str(const char *str); +const char *mhp_online_type_to_str(int online_type); /* If movable_node boot option specified */ extern bool movable_node_enabled; @@ -267,6 +268,8 @@ extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages, extern int remove_memory(u64 start, u64 size); extern void __remove_memory(u64 start, u64 size); extern int offline_and_remove_memory(u64 start, u64 size); +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges); #else static inline void try_offline_node(int nid) {} @@ -283,6 +286,12 @@ static inline int remove_memory(u64 start, u64 size) } static inline void __remove_memory(u64 start, u64 size) {} + +static inline int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + return -EBUSY; +} #endif /* CONFIG_MEMORY_HOTREMOVE */ #ifdef CONFIG_MEMORY_HOTPLUG @@ -294,6 +303,9 @@ extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, mhp_t mhp_flags); +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type); extern int add_memory_driver_managed(int nid, u64 start, u64 size, const char *resource_name, mhp_t mhp_flags); @@ -316,6 +328,8 @@ extern struct zone *zone_for_pfn_range(enum mmop online_type, extern int arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params); void arch_remove_linear_mapping(u64 start, u64 size); +#else +static inline enum mmop mhp_get_default_online_type(void) { return MMOP_OFFLINE; } #endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/include/linux/migrate.h b/include/linux/migrate.h index d5af2b7f577b..78424b3824c2 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -57,8 +57,9 @@ void putback_movable_pages(struct list_head *l); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, - unsigned long private, enum migrate_mode mode, int reason, - unsigned int *ret_succeeded); + unsigned long private, enum migrate_mode mode, + enum migrate_reason reason, + unsigned int *ret_succeeded); struct folio *alloc_migration_target(struct folio *src, unsigned long private); bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode); bool isolate_folio_to_list(struct folio *folio, struct list_head *list); @@ -77,7 +78,8 @@ int set_movable_ops(const struct movable_operations *ops, enum pagetype type); static inline void putback_movable_pages(struct list_head *l) {} static inline int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, + unsigned int *ret_succeeded) { return -ENOSYS; } static inline struct folio *alloc_migration_target(struct folio *src, unsigned long private) diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 265c4328b36a..05102d4d2490 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -25,6 +25,7 @@ enum migrate_reason { MR_LONGTERM_PIN, MR_DEMOTION, MR_DAMON, + MR_NEVER, /* page has never been migrated */ MR_TYPES }; diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..7fabe6c66b4b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -37,6 +37,7 @@ #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/iommu-debug-pagealloc.h> +#include <linux/kcsan-checks.h> struct mempolicy; struct anon_vma; @@ -353,6 +354,7 @@ enum { #endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), + DECLARE_VMA_BIT(UFFD_RWP, 43), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), @@ -440,8 +442,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWSUP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -463,15 +467,18 @@ enum { #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI) #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) +#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) #else #define VM_SHADOW_STACK VM_NONE +#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) #endif #if defined(CONFIG_PPC64) #define VM_SAO INIT_VM_FLAG(SAO) #elif defined(CONFIG_PARISC) #define VM_GROWSUP INIT_VM_FLAG(GROWSUP) +#define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT) #elif defined(CONFIG_SPARC64) #define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI) #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) @@ -483,6 +490,7 @@ enum { #endif #ifndef VM_GROWSUP #define VM_GROWSUP VM_NONE +#define VMA_GROWSUP EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARM64_MTE #define VM_MTE INIT_VM_FLAG(MTE) @@ -496,12 +504,17 @@ enum { #else #define VM_UFFD_MINOR VM_NONE #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) +#else +#define VM_UFFD_RWP VM_NONE +#endif /* - * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on - * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits - * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose - * bitmap cannot hold it. + * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are + * gated on the same configs as their VM_* flags above -- both of which imply + * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build + * whose bitmap cannot hold it. */ #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) @@ -510,6 +523,11 @@ enum { #else #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) +#else +#define VMA_UFFD_RWP EMPTY_VMA_FLAGS +#endif #ifdef CONFIG_64BIT #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) @@ -528,6 +546,8 @@ enum { /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VMA_EXEC_BIT : VMA_READ_BIT) @@ -648,29 +668,32 @@ enum { * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the - * source, as VM_UFFD_WP may not be propagated to destination, while all other - * flags will be. + * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination + * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did + * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd - * write protect handler, which cannot be - * reconstructed on page fault. + * VM_UFFD_RWP write- or read-write-protect handler, which + * cannot be reconstructed on page fault. * - * We always copy pgtables when dst_vma has uffd-wp - * enabled even if it's file-backed - * (e.g. shmem). Because when uffd-wp is enabled, - * pgtable contains uffd-wp protection information, - * that's something we can't retrieve from page cache, - * and skip copying will lose those info. + * We always copy pgtables when dst_vma has the + * uffd PTE bit in use even if it's file-backed + * (e.g. shmem). Because when the uffd bit is + * in use, the pgtable contains the protection + * information, that's something we can't + * retrieve from page cache, and skip copying + * will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables * only and thus cannot be reconstructed on page * fault. */ -#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD) +#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ + VM_MAYBE_GUARD) /* * mapping from the currently active vm_flags protection bits (the @@ -717,6 +740,9 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags) { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } +/* /dev/zero minor device number. Special due to MAP_PRIVATE semantics. */ +#define DEVZERO_MINOR 5 + /* * vm_fault is filled by the pagefault handler and passed to the vma's * ->fault function. The vma's ->fault is responsible for returning a bitmask @@ -1528,12 +1554,7 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) -{ - desc->vm_ops = NULL; -} - -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -1563,11 +1584,24 @@ static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) vma->vm_end >= vma->vm_mm->start_stack; } -static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) +static inline bool vma_flags_can_grow(const vma_flags_t *flags) +{ + if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) + return true; + if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) + return true; + + return false; +} + +static inline bool vma_can_grow(const struct vm_area_struct *vma) { - int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); + return vma_flags_can_grow(&vma->flags); +} - if (!maybe_stack) +static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) +{ + if (!vma_can_grow(vma)) return false; if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == @@ -2286,22 +2320,30 @@ static inline int page_zone_id(struct page *page) } #ifdef NODE_NOT_IN_PAGE_FLAGS -int memdesc_nid(memdesc_flags_t mdf); +int memdesc_nid(const memdesc_flags_t *mdf); +#else +#ifdef CONFIG_NUMA +static inline int memdesc_nid(const memdesc_flags_t *mdf) +{ + ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); + return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; +} #else -static inline int memdesc_nid(memdesc_flags_t mdf) +static inline int memdesc_nid(const memdesc_flags_t *mdf) { - return (mdf.f >> NODES_PGSHIFT) & NODES_MASK; + return 0; } #endif +#endif static inline int page_to_nid(const struct page *page) { - return memdesc_nid(PF_POISONED_CHECK(page)->flags); + return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); } static inline int folio_nid(const struct folio *folio) { - return memdesc_nid(folio->flags); + return memdesc_nid(&folio->flags); } #ifdef CONFIG_NUMA_BALANCING @@ -2541,12 +2583,13 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { - return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); + return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; } @@ -3317,6 +3360,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen); #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE) +/* Whether this change is for uffd RWP */ +#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ +#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ +#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ + MM_CP_UFFD_RWP_RESOLVE) bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte); @@ -3967,8 +4015,12 @@ extern unsigned long free_reserved_area(void *start, void *end, extern void adjust_managed_page_count(struct page *page, long count); -/* Free the reserved page into the buddy system, so it gets managed. */ -void free_reserved_page(struct page *page); +void free_reserved_pages(struct page *page, unsigned int order); + +static inline void free_reserved_page(struct page *page) +{ + free_reserved_pages(page, 0); +} static inline void mark_page_reserved(struct page *page) { @@ -4042,7 +4094,7 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); extern void mem_init(void); extern void __init mmap_init(void); -extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); static inline void show_mem(void) { __show_mem(0, NULL, MAX_NR_ZONES - 1); @@ -4052,7 +4104,7 @@ extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); extern __printf(3, 4) -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); extern void setup_per_cpu_pageset(void); @@ -4061,44 +4113,46 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *node, - struct rb_root_cached *root); -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping); +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root); -void vma_interval_tree_remove(struct vm_area_struct *node, - struct rb_root_cached *root); -struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, - unsigned long start, unsigned long last); - -#define vma_interval_tree_foreach(vma, root, start, last) \ - for (vma = vma_interval_tree_iter_first(root, start, last); \ - vma; vma = vma_interval_tree_iter_next(vma, start, last)) - -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root); -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root); + struct address_space *mapping); +void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping); +struct vm_area_struct * +mapping_rmap_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last); +struct vm_area_struct * +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); + +#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ + pgoff_last); \ + vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ + pgoff_last)) + +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +struct anon_vma_chain * +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct anon_vma_chain *anon_vma_interval_tree_iter_next( - struct anon_vma_chain *node, unsigned long start, unsigned long last); +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last); #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node); +void anon_rmap_tree_verify(struct anon_vma_chain *avc); #endif -#define anon_vma_interval_tree_foreach(avc, root, start, last) \ - for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ - avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) +#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ + for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ + avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last)) /* mmap.c */ extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); -extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void exit_mmap(struct mm_struct *); bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, bool write); @@ -4139,18 +4193,20 @@ unsigned long randomize_page(unsigned long start, unsigned long range); unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); + unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags); static inline unsigned long get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return __get_unmapped_area(file, addr, len, pgoff, flags, 0); + return __get_unmapped_area(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } -extern unsigned long do_mmap(struct file *file, unsigned long addr, +unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, @@ -4278,9 +4334,119 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } -static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) +/** + * vma_start_pgoff() - Get the page offset of the start of @vma + * @vma: The VMA whose page offset is required. + * + * If the VMA is file-backed, this is the page offset into the file. + * + * If @vma is anonymous, this is the virtual page offset of the start of the + * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the + * virtual page offset at the time of first fault. + * + * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the + * page offset within the file. + * + * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions + * have their page offset set to the first PFN in the range. + * + * Returns: The page offset of the start of @vma. + */ +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) { - return vma->vm_pgoff + vma_pages(vma) - 1; + return vma->vm_pgoff; +} + +/** + * vma_end_pgoff() - Get the page offset of the exclusive end of @vma + * @vma: The VMA whose end page offset is required. + * + * This returns the exclusive end page offset of @vma, which is useful for + * expressing page offset ranges. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The exclusive end page offset of @vma. + */ +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_pgoff() - Get the page offset of the last page in @vma + * @vma: The VMA whose last page offset is required. + * + * This returns the last page offset contained within @vma. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The last page offset of @vma. + */ +static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_pgoff(vma) - 1; +} + +/** + * vma_start_virt_pgoff() - Get the virtual page offset of the start of @vma + * @vma: The VMA whose virtual page offset is required. + * + * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the + * virtual page offset at the time of first fault. + * + * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). + * + * This value is used for tracking MAP_PRIVATE file-backed mappings by their + * virtual page offset. + * + * Returns: The virtual page offset of the start of @vma. + */ +static inline pgoff_t vma_start_virt_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_virt_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_virt_pgoff_lo; + return pgoff; +} + +/** + * vma_end_virt_pgoff() - Get the virtual page offset of the exclusive end of + * @vma. + * @vma: The VMA whose end virtual page offset is required. + * + * This returns the virtual exclusive end page offset of @vma, which is useful + * for expressing page offset ranges. + * + * See the description of vma_start_virt_pgoff() for a description of VMA + * virtual page offsets. + * + * Returns: The exclusive end virtual page offset of @vma. + */ +static inline pgoff_t vma_end_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_virt_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_virt_pgoff() - Get the virtual page offset of the last page in + * @vma. + * @vma: The VMA whose last virtual page offset is required. + * + * See the description of vma_start_virt_pgoff() for a description of VMA + * virtual page offsets. + * + * Returns: The last virtual page offset of @vma. + */ +static inline pgoff_t vma_last_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_virt_pgoff(vma) - 1; } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4497,26 +4663,35 @@ static inline bool range_in_vma_desc(const struct vm_area_desc *desc, #ifdef CONFIG_MMU pgprot_t vm_get_page_prot(vm_flags_t vm_flags); -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); return vm_get_page_prot(vm_flags); } +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} + void vma_set_page_prot(struct vm_area_struct *vma); #else static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { return __pgprot(0); } -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) +{ + return __pgprot(0); +} +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) { return __pgprot(0); } static inline void vma_set_page_prot(struct vm_area_struct *vma) { - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); } #endif @@ -4625,12 +4800,26 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) /* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether - * a (NUMA hinting) fault is required. + * a (NUMA hinting or userfaultfd RWP) fault is required. */ static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags) { /* + * VM_UFFD_RWP uses protnone as an access-tracking marker, not for + * NUMA hinting. GUP must always take a fault so the access is + * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. + * + * Only do so while the VMA is accessible. If it has been made + * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard + * below: forcing a fault there would loop, as handle_mm_fault() makes + * no progress on protnone in an inaccessible VMA, and the access is + * denied regardless of RWP anyway. + */ + if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) + return false; + + /* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. */ @@ -5158,13 +5347,9 @@ int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t * int reserve_mem_release_by_name(const char *name); #ifdef CONFIG_64BIT -int do_mseal(unsigned long start, size_t len_in, unsigned long flags); +void mseal_mmap_page_zero(void); #else -static inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags) -{ - /* noop on 32 bit */ - return 0; -} +static inline void mseal_mmap_page_zero(void) {} #endif /* diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index a8430a7ae054..b5c4dc0f3fe3 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -600,14 +600,14 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, return false; /* A uffd-wp wr-protected normal pte */ - if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval))) + if (unlikely(pte_present(pteval) && pte_uffd(pteval))) arm_uffd_pte = true; /* * A uffd-wp wr-protected swap pte. Note: this should even cover an * existing pte marker with uffd-wp bit set. */ - if (unlikely(pte_swp_uffd_wp_any(pteval))) + if (unlikely(pte_swp_uffd_any(pteval))) arm_uffd_pte = true; if (unlikely(arm_uffd_pte)) { @@ -650,7 +650,7 @@ static inline bool vma_has_recency(const struct vm_area_struct *vma) static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) { struct page *cur_page = pages[0]; - unsigned long section = memdesc_section(cur_page->flags); + unsigned long section = memdesc_section(&cur_page->flags); size_t i; for (i = 1; i < nr_pages; i++) { @@ -660,7 +660,7 @@ static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) * In unproblematic kernel configs, page_to_section() == 0 and * the whole check will get optimized out. */ - if (memdesc_section(cur_page->flags) != section) + if (memdesc_section(&cur_page->flags) != section) break; } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b18c2b2e7d2c..b5d4cd3b067b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -368,6 +368,8 @@ typedef unsigned short mm_id_t; * dax_associate_entry. * @private: Filesystem per-folio data (see folio_attach_private()). * @swap: Used for swp_entry_t if folio_test_swapcache(). + * @migrate_info: Stores migration state (anon_vma pointer and + * FOLIO_WAS_* markers). * @_mapcount: Do not access this member directly. Use folio_mapcount() to * find out how many times this folio is mapped by userspace. * @_refcount: Do not access this member directly. Use folio_ref_count() @@ -427,6 +429,7 @@ struct folio { union { void *private; swp_entry_t swap; + unsigned long migrate_info; }; atomic_t _mapcount; atomic_t _refcount; @@ -965,6 +968,11 @@ struct vm_area_struct { unsigned int vm_lock_seq; #endif /* + * Low 32-bits of virtual page offset. + * See vma_start_virt_pgoff() comment for details. + */ + unsigned int __vm_virt_pgoff_lo; + /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma * list, after a COW of one of the file pages. A MAP_SHARED vma * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack @@ -1039,6 +1047,13 @@ struct vm_area_struct { struct lockdep_map vmlock_dep_map; #endif #endif +#ifdef CONFIG_64BIT + /* + * High 32-bits of virtual page offset. + * See vma_start_virt_pgoff() comment for details. + */ + unsigned int __vm_virt_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 047f5f5e2c34..87f77e3da77f 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -506,6 +506,8 @@ static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } static inline void vma_assert_write_locked(struct vm_area_struct *vma) { mmap_assert_write_locked(vma->vm_mm); } +static inline bool vma_is_attached(struct vm_area_struct *vma) + { return true; } static inline void vma_assert_attached(struct vm_area_struct *vma) {} static inline void vma_assert_detached(struct vm_area_struct *vma) {} static inline void vma_mark_attached(struct vm_area_struct *vma) {} @@ -530,6 +532,12 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma) #endif /* CONFIG_PER_VMA_LOCK */ +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} + static inline void mmap_write_lock(struct mm_struct *mm) { __mmap_lock_trace_start_locking(mm, true); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ca2712187147..0507193b3ae3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2,7 +2,7 @@ #ifndef _LINUX_MMZONE_H #define _LINUX_MMZONE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __GENERATING_BOUNDS_H #include <linux/spinlock.h> @@ -1272,31 +1272,33 @@ static inline bool zone_is_empty(const struct zone *zone) #define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) -static inline enum zone_type memdesc_zonenum(memdesc_flags_t flags) +static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags) { - ASSERT_EXCLUSIVE_BITS(flags.f, ZONES_MASK << ZONES_PGSHIFT); - return (flags.f >> ZONES_PGSHIFT) & ZONES_MASK; +#if ZONES_WIDTH != 0 + ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT); +#endif + return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK; } static inline enum zone_type page_zonenum(const struct page *page) { - return memdesc_zonenum(page->flags); + return memdesc_zonenum(&page->flags); } static inline enum zone_type folio_zonenum(const struct folio *folio) { - return memdesc_zonenum(folio->flags); + return memdesc_zonenum(&folio->flags); } #ifdef CONFIG_ZONE_DEVICE -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return memdesc_zonenum(mdf) == ZONE_DEVICE; } static inline struct dev_pagemap *page_pgmap(const struct page *page) { - VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(page->flags), page); + VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap; } @@ -1311,9 +1313,9 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b) { - if (memdesc_is_zone_device(a->flags) != memdesc_is_zone_device(b->flags)) + if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; - if (!memdesc_is_zone_device(a->flags)) + if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b); } @@ -1321,7 +1323,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a, extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *); #else -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return false; } @@ -1338,12 +1340,12 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool is_zone_device_page(const struct page *page) { - return memdesc_is_zone_device(page->flags); + return memdesc_is_zone_device(&page->flags); } static inline bool folio_is_zone_device(const struct folio *folio) { - return memdesc_is_zone_device(folio->flags); + return memdesc_is_zone_device(&folio->flags); } static inline bool is_zone_movable_page(const struct page *page) @@ -1815,7 +1817,7 @@ static inline int zonelist_node_idx(const struct zoneref *zoneref) struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes); + const nodemask_t *nodes); /** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point @@ -1834,7 +1836,7 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z, */ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; @@ -1860,7 +1862,7 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, */ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes); @@ -2156,8 +2158,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) } void sparse_vmemmap_init_nid_early(int nid); -void sparse_vmemmap_init_nid_late(int nid); - #else static inline int preinited_vmemmap_section(const struct mem_section *section) { @@ -2166,10 +2166,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) static inline void sparse_vmemmap_init_nid_early(int nid) { } - -static inline void sparse_vmemmap_init_nid_late(int nid) -{ -} #endif static inline int online_section_nr(unsigned long nr) @@ -2374,7 +2370,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #else #define sparse_vmemmap_init_nid_early(_nid) do {} while (0) -#define sparse_vmemmap_init_nid_late(_nid) do {} while (0) #define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ @@ -2389,5 +2384,5 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #endif /* !__GENERATING_BOUNDS.H */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _LINUX_MMZONE_H */ diff --git a/include/linux/oom.h b/include/linux/oom.h index 7b02bc1d0a7e..00da05d227e6 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -30,7 +30,7 @@ struct oom_control { struct zonelist *zonelist; /* Used to determine mempolicy */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* Memory cgroup in which oom is invoked, or NULL for global oom */ struct mem_cgroup *memcg; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7223f6f4e2b4..7a863572adce 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -879,20 +879,6 @@ FOLIO_FLAG_FALSE(partially_mapped) #define PG_head_mask ((1UL << PG_head)) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * PageTransCompound returns true for both transparent huge pages - * and hugetlbfs pages, so it should only be called when it's known - * that hugetlbfs pages aren't involved. - */ -static inline int PageTransCompound(const struct page *page) -{ - return PageCompound(page); -} -#else -TESTPAGEFLAG_FALSE(TransCompound, transcompound) -#endif - #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE) /* * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 3328357f6dba..8188ddc5c412 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -3,6 +3,7 @@ #define __LINUX_PAGE_OWNER_H #include <linux/jump_label.h> +#include <linux/migrate_mode.h> #ifdef CONFIG_PAGE_OWNER extern struct static_key_false page_owner_inited; @@ -14,7 +15,7 @@ extern void __set_page_owner(struct page *page, extern void __split_page_owner(struct page *page, int old_order, int new_order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); -extern void __folio_set_owner_migrate_reason(struct folio *folio, int reason); +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason); extern void __dump_page_owner(const struct page *page); extern void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone); @@ -43,7 +44,7 @@ static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) if (static_branch_unlikely(&page_owner_inited)) __folio_copy_owner(newfolio, old); } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { if (static_branch_unlikely(&page_owner_inited)) __folio_set_owner_migrate_reason(folio, reason); @@ -68,7 +69,7 @@ static inline void split_page_owner(struct page *page, int old_order, static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) { } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { } static inline void dump_page_owner(const struct page *page) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2c3718d592d6..4e8b2b29f6d3 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1063,15 +1063,110 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) return folio->index; } +/** + * linear_page_delta() - Determine the relative page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose relative page offset is required. + * + * The result is identical for both file-backed and anonymous mappings and + * simply determines how many pages @address lies from @vma->vm_start. + * + * Returns: The number of pages @address is offset by within @vma. + */ +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +/** + * linear_page_index() - Determine the absolute page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * See the comment for vma_start_pgoff() for a description of what the page + * offset signifies. + * + * Returns: The absolute page offset of @address within @vma. + */ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_pgoff(vma); + return pgoff; +} + +static inline pgoff_t __linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_virt_pgoff(vma); + return pgoff; +} + +/** + * linear_virt_page_index() - Determine the absolute virtual page offset of + * @address within @vma. + * @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * This returns the virtual page offset of @address, which is the page offset + * the address possessed at the time the VMA was first faulted. + * + * For anonymous mappings, this returns the same value as linear_page_index(). + * + * For MAP_PRIVATE file-backed mappings, this returns the virtual page offset of + * @address, which is the page offset the address possessed at the time the VMA + * was first faulted. + * + * It is not valid to call this function for shared file-backed mappings. + * + * Returns: The absolute virtual page offset of @address within @vma. + */ +static inline pgoff_t linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_virt_page_index(vma, address); + + VM_WARN_ON_ONCE(vma_test(vma, VMA_SHARED_BIT)); + if (vma_is_anonymous(vma)) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); + return pgoff; } +/** + * linear_folio_page_index() - Determine the absolute page offset of + * @address within @vma from @folio. + * @folio: The folio whose linear page index is sought. + * @vma: The VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * Determines whether to obtain the virtual linear page index based on whether + * @folio is anonymous or not. + * + * See the descriptions of linear_virt_page_index() and linear_page_index() for + * details of each. + * + * Returns: The absolute page offset of @address within @vma. + */ +static inline pgoff_t linear_folio_page_index(const struct folio *folio, + const struct vm_area_struct *vma, + const unsigned long address) +{ + if (folio_test_anon(folio)) + return linear_virt_page_index(vma, address); + + return linear_page_index(vma, address); +} + struct wait_page_key { struct folio *folio; int bit_nr; @@ -1219,7 +1314,7 @@ static inline vm_fault_t folio_lock_or_retry(struct folio *folio, void folio_wait_bit(struct folio *folio, int bit_nr); int folio_wait_bit_killable(struct folio *folio, int bit_nr); -/* +/* * Wait for a folio to be unlocked. * * This must be called with the caller "holding" the folio, diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 43c854a273c3..2cba7cc2b01f 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -203,7 +203,7 @@ /* * Accessors and operations. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating @@ -514,5 +514,5 @@ do { \ #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_PERCPU_DEFS_H */ diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index d73a1c08c3e3..1e3212e2c827 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -12,8 +12,8 @@ * start shutting down you call percpu_ref_kill() _before_ dropping the initial * refcount. * - * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less - * than an atomic_t - this is because of the way shutdown works, see + * The refcount will have a range of 0 to LONG_MAX, i.e. one bit less + * than an atomic_long_t - this is because of the way shutdown works, see * percpu_ref_kill()/PERCPU_COUNT_BIAS. * * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the @@ -269,6 +269,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) /** * percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the * caller is responsible for taking RCU. + * @ref: percpu_ref to try-get * * This function is safe to call as long as @ref is between init and exit. */ diff --git a/include/linux/pfn.h b/include/linux/pfn.h index b90ca0b6c331..cfedf0f61bb3 100644 --- a/include/linux/pfn.h +++ b/include/linux/pfn.h @@ -2,7 +2,7 @@ #ifndef _LINUX_PFN_H_ #define _LINUX_PFN_H_ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #endif diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2981e386da7b..8c093c119e5a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -8,7 +8,7 @@ #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT) #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_MMU #include <linux/mm_types.h> @@ -988,49 +988,30 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma, update_mmu_tlb_range(vma, address, ptep, 1); } -/* - * Some architectures may be able to avoid expensive synchronization - * primitives when modifications are made to PTE's which are already - * not present, or in the process of an address space destruction. - */ -#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -static inline void pte_clear_not_present_full(struct mm_struct *mm, - unsigned long address, - pte_t *ptep, - int full) -{ - pte_clear(mm, address, ptep); -} -#endif - -#ifndef clear_not_present_full_ptes /** - * clear_not_present_full_ptes - Clear multiple not present PTEs which are - * consecutive in the pgtable. + * clear_nonpresent_ptes - Clear multiple non-present PTEs which are + * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. - * @full: Whether we are clearing a full mm. - * - * May be overridden by the architecture; otherwise, implemented as a simple - * loop over pte_clear_not_present_full(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */ -static inline void clear_not_present_full_ptes(struct mm_struct *mm, - unsigned long addr, pte_t *ptep, unsigned int nr, int full) +static inline void clear_nonpresent_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) { + (void)addr; + for (;;) { - pte_clear_not_present_full(mm, addr, ptep, full); + pte_clear(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; } } -#endif #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, @@ -1858,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; @@ -2129,18 +2110,26 @@ static inline int pud_trans_unstable(pud_t *pud) return 0; } -#ifndef CONFIG_NUMA_BALANCING +#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE /* - * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is - * perfectly valid to indicate "no" in that case, which is why our default - * implementation defaults to "always no". + * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It + * is perfectly valid to indicate "no" in that case, which is why our + * default implementation defaults to "always no". + * + * In an accessible VMA, pte_protnone() reliably indicates a present + * PROT_NONE page protection. Today the kernel uses such PTEs for two + * purposes: NUMA hinting faults, and userfaultfd RWP tracking on + * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and + * the VMA flag; see include/linux/userfaultfd_k.h. * - * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE - * page protection due to NUMA hinting. NUMA hinting faults only apply in - * accessible VMAs. + * So, to reliably identify PROT_NONE PTEs that require kernel handling, + * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is + * sufficient. * - * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, - * looking at the VMA accessibility is sufficient. + * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero + * stubs below; PAGE_NONE references that survive to runtime fire the + * BUILD_BUG() fallback, since callers should have folded such paths to + * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). */ static inline int pte_protnone(pte_t pte) { @@ -2151,7 +2140,11 @@ static inline int pmd_protnone(pmd_t pmd) { return 0; } -#endif /* CONFIG_NUMA_BALANCING */ + +#ifndef PAGE_NONE +#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; }) +#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #endif /* CONFIG_MMU */ @@ -2320,7 +2313,7 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) #ifdef CONFIG_PHYS_ADDR_T_64BIT diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index eae67015ce51..057edc4cbb6e 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -111,7 +111,7 @@ struct radix_tree_iter { }; /** - * Radix-tree synchronization + * DOC: Radix-tree synchronization * * The radix-tree API requires that users provide all synchronisation (with * specific exceptions, noted below). @@ -182,6 +182,7 @@ static inline void *radix_tree_deref_slot(void __rcu **slot) /** * radix_tree_deref_slot_protected - dereference a slot with tree lock held * @slot: slot pointer, returned by radix_tree_lookup_slot + * @treelock: caller must hold this spinlock * * Similar to radix_tree_deref_slot. The caller does not hold the RCU read * lock but it must hold the tree lock to prevent parallel updates. @@ -306,7 +307,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) * Also it fills @iter with data about chunk: position in the tree (index), * its end (next_index), and constructs a bit mask for tagged iterating (tags). */ -void __rcu **radix_tree_next_chunk(const struct radix_tree_root *, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned flags); /** diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f00..a174758f7777 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -843,7 +843,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio, * Called from mm/vmscan.c to handle paging out */ int folio_referenced(struct folio *, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags); + struct mem_cgroup *memcg, vma_flags_t *vma_flags); void try_to_migrate(struct folio *folio, enum ttu_flags flags); void try_to_unmap(struct folio *, enum ttu_flags flags); @@ -871,6 +871,7 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; + bool is_anon_walk; }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -881,6 +882,7 @@ struct page_vma_mapped_walk { .vma = _vma, \ .address = _address, \ .flags = _flags, \ + .is_anon_walk = folio_test_anon(_folio), \ } static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) @@ -975,10 +977,9 @@ struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, #define anon_vma_prepare(vma) (0) static inline int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, - vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { - *vm_flags = 0; + vma_flags_clear_all(vma_flags); return 0; } diff --git a/include/linux/sched.h b/include/linux/sched.h index 373bcc0598d1..14d5ce8dd613 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1288,6 +1288,7 @@ struct task_struct { u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; + unsigned int lockdep_seq; struct held_lock held_locks[MAX_LOCK_DEPTH]; #endif diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df584..10be8a54b416 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -193,21 +193,18 @@ unsigned long mm_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, - unsigned long addr, - unsigned long len, - unsigned long pgoff, - unsigned long flags, - vm_flags_t vm_flags); +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); #else static inline void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) {} diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index e729b9b0e38d..5663dff53186 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -12,8 +12,6 @@ #include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb; - /* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping) void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, - struct list_head *folio_list); +int shmem_write_folio(struct folio *folio); void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end); int shmem_unuse(unsigned int type); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 22eda1d54a0e..4431b026e429 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3573,7 +3573,7 @@ static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask, * 3. If requesting a order 0 page it will not be compound * due to the check to see if order has a value in prep_new_page * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to - * code in gfp_to_alloc_flags that should be enforcing this. + * code in alloc_flags_slowpath() that should be enforcing this. */ gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; diff --git a/include/linux/swap.h b/include/linux/swap.h index 8f0f68e245ba..0544b2ec4c56 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -16,10 +16,6 @@ #include <uapi/linux/mempolicy.h> #include <asm/page.h> -struct notifier_block; - -struct bio; - #define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */ #define SWAP_FLAG_PRIO_MASK 0x7fff #define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */ @@ -29,7 +25,6 @@ struct bio; #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) -#define SWAP_BATCH 64 static inline int current_is_kswapd(void) { @@ -175,7 +170,6 @@ static inline void mm_account_reclaimed_pages(unsigned long pages) struct address_space; struct sysinfo; -struct writeback_control; struct zone; /* @@ -208,7 +202,6 @@ enum { SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */ SWP_BLKDEV = (1 << 6), /* its a block device */ SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */ - SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */ SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */ SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */ @@ -282,6 +275,7 @@ struct swap_info_struct { struct work_struct reclaim_work; /* reclaim worker */ struct list_head discard_clusters; /* discard clusters list */ struct plist_node avail_list; /* entry in swap_avail_head */ + const struct swap_ops *ops; }; static inline swp_entry_t page_swap_entry(struct page *page) @@ -293,39 +287,19 @@ static inline swp_entry_t page_swap_entry(struct page *page) return entry; } -/* linux/mm/workingset.c */ -bool workingset_test_recent(void *shadow, bool file, bool *workingset, - bool flush); -void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); -void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); -void workingset_refault(struct folio *folio, void *shadow); -void workingset_activation(struct folio *folio); - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; /* Definition of global_zone_page_state not available yet */ #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES) +/* linux/mm/folio.c */ +void folio_add_lru(struct folio *folio); +void folio_mark_accessed(struct folio *folio); +void lru_add_drain_all(void); -/* linux/mm/swap.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated); -void lru_note_cost_refault(struct folio *); -void folio_add_lru(struct folio *); -void folio_add_lru_vma(struct folio *, struct vm_area_struct *); -void mark_page_accessed(struct page *); -void folio_mark_accessed(struct folio *); - -static inline bool folio_may_be_lru_cached(struct folio *folio) -{ - /* - * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. - * Holding small numbers of low-order mTHP folios in per-CPU LRU cache - * will be sensible, but nobody has implemented and tested that yet. - */ - return !folio_test_large(folio); -} +/* linux/mm/folio-compat.c */ +void mark_page_accessed(struct page *page); extern atomic_t lru_disable_count; @@ -334,43 +308,6 @@ static inline bool lru_cache_disabled(void) return atomic_read(&lru_disable_count); } -static inline void lru_cache_enable(void) -{ - atomic_dec(&lru_disable_count); -} - -extern void lru_cache_disable(void); -extern void lru_add_drain(void); -extern void lru_add_drain_cpu(int cpu); -extern void lru_add_drain_cpu_zone(struct zone *zone); -extern void lru_add_drain_all(void); -void folio_deactivate(struct folio *folio); -void folio_mark_lazyfree(struct folio *folio); -extern void swap_setup(void); - -/* linux/mm/vmscan.c */ -extern unsigned long zone_reclaimable_pages(struct zone *zone); -extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *mask); -unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); - -#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) -#define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 - -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) - -extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - unsigned long nr_pages, - gfp_t gfp_mask, - unsigned int reclaim_options, - int *swappiness); -extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); extern unsigned long shrink_all_memory(unsigned long nr_pages); extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); @@ -391,11 +328,6 @@ static inline void reclaim_unregister_node(struct node *node) } #endif /* CONFIG_SYSFS && CONFIG_NUMA */ -#ifdef CONFIG_NUMA -extern int sysctl_min_unmapped_ratio; -extern int sysctl_min_slab_ratio; -#endif - void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); @@ -403,6 +335,7 @@ extern void __meminit kswapd_stop(int nid); #ifdef CONFIG_SWAP +int swap_fs_activate(struct swap_info_struct *sis); int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block); int generic_swapfile_activate(struct swap_info_struct *, struct file *, @@ -442,7 +375,6 @@ extern sector_t swapdev_block(int, pgoff_t); extern int __swap_count(swp_entry_t entry); extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); -struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); @@ -529,6 +461,10 @@ static inline bool folio_free_swap(struct folio *folio) return false; } +static inline int swap_fs_activate(struct swap_info_struct *sis) +{ + return -EINVAL; +} static inline int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block) @@ -552,7 +488,7 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); #else -static inline int mem_cgroup_swappiness(struct mem_cgroup *mem) +static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { return READ_ONCE(vm_swappiness); } diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..c956bc445ee0 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -73,8 +73,8 @@ static inline pte_t pte_swp_clear_flags(pte_t pte) pte = pte_swp_clear_exclusive(pte); if (pte_swp_soft_dirty(pte)) pte = pte_swp_clear_soft_dirty(pte); - if (pte_swp_uffd_wp(pte)) - pte = pte_swp_clear_uffd_wp(pte); + if (pte_swp_uffd(pte)) + pte = pte_swp_clear_uffd(pte); return pte; } @@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); @@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 68edac4dcd78..bfbd6a59909f 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -21,10 +21,11 @@ #include <linux/hugetlb_inline.h> /* The set of all possible UFFD-related VM flags. */ -#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) +#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \ + VM_UFFD_WP | VM_UFFD_RWP) #define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \ - VMA_UFFD_MINOR) + VMA_UFFD_MINOR, VMA_UFFD_RWP) /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining @@ -149,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); +extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp); /* move_pages */ void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); @@ -168,7 +171,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, /* * Never enable huge pmd sharing on some uffd registered vmas: * - * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. + * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access + * tracking information is per pgtable entry. * * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for * VMAs which share huge pmds. (If you have two mappings to the same @@ -178,51 +182,80 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, */ static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } /* - * Don't do fault around for either WP or MINOR registered uffd range. For + * Don't do fault around for WP, RWP or MINOR registered uffd range. For * MINOR registered range, fault around will be a total disaster and ptes can * be installed without notifications; for WP it should mostly be fine as long * as the fault around checks for pte_none() before the installation, however - * to be super safe we just forbid it. + * to be super safe we just forbid it; for RWP, pre-faulted neighbours would + * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED) + * and pollute the tracked working set, so each page must be populated by its + * own fault. */ static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } static inline bool userfaultfd_missing(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MISSING; + return vma_test_any_mask(vma, VMA_UFFD_MISSING); } static inline bool userfaultfd_wp(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_WP; + return vma_test_any_mask(vma, VMA_UFFD_WP); } static inline bool userfaultfd_minor(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MINOR; + return vma_test_any_mask(vma, VMA_UFFD_MINOR); +} + +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return vma_test_single_mask(vma, VMA_UFFD_RWP); +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return userfaultfd_wp(vma) || userfaultfd_rwp(vma); } static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { - return userfaultfd_wp(vma) && pte_uffd_wp(pte); + return userfaultfd_wp(vma) && pte_uffd(pte); } static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd) { - return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); + return userfaultfd_wp(vma) && pmd_uffd(pmd); +} + +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return userfaultfd_rwp(vma) && pte_uffd(pte); +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return userfaultfd_rwp(vma) && pmd_uffd(pmd); } static inline bool userfaultfd_armed(struct vm_area_struct *vma) { - return vma->vm_flags & __VM_UFFD_FLAGS; + return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); } static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) @@ -253,6 +286,7 @@ extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); extern bool userfaultfd_wp_async(struct vm_area_struct *vma); +extern bool userfaultfd_rwp_async(struct vm_area_struct *vma); static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) { @@ -272,10 +306,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { if (!uffd_supports_wp_marker()) return false; @@ -283,7 +317,7 @@ static inline bool pte_swp_uffd_wp_any(pte_t pte) if (pte_present(pte)) return false; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) return true; if (pte_is_uffd_wp_marker(pte)) @@ -328,6 +362,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return false; +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return false; +} + static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { @@ -340,6 +384,17 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, return false; } +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return false; +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return false; +} static inline bool userfaultfd_armed(struct vm_area_struct *vma) { @@ -410,6 +465,11 @@ static inline bool userfaultfd_wp_async(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return false; +} + static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) { return false; @@ -421,10 +481,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { return false; } diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 03fe95f5a020..2628ccda076a 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST, #endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */ NR_VM_EVENT_ITEMS }; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index d87dc7f77f4e..e4d8d0a9f30f 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -310,14 +310,14 @@ static inline void set_vm_flush_reset_perms(void *addr) {} #if defined(CONFIG_MMU) && defined(CONFIG_SMP) struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align); + size_t align, gfp_t gfp); void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms); # else static inline struct vm_struct ** pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { return NULL; } diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h index faecd5522401..b4d13457bc2a 100644 --- a/include/linux/vmpressure.h +++ b/include/linux/vmpressure.h @@ -13,18 +13,31 @@ struct vmpressure { unsigned long scanned; unsigned long reclaimed; + /* The lock is used to keep the scanned/reclaimed in sync. */ + spinlock_t sr_lock; +#ifdef CONFIG_MEMCG_V1 + /* + * tree=true accumulators feed the v1 userspace eventfd interface + * (memory.pressure_level). Drained by @work. v2 has no equivalent + * interface, so this state is omitted on CONFIG_MEMCG_V1=n builds. + */ unsigned long tree_scanned; unsigned long tree_reclaimed; - /* The lock is used to keep the scanned/reclaimed above in sync. */ - spinlock_t sr_lock; - /* The list of vmpressure_event structs. */ struct list_head events; /* Have to grab the lock on events traversal or modifications. */ struct mutex events_lock; struct work_struct work; +#endif +}; + +enum vmpressure_levels { + VMPRESSURE_LOW = 0, + VMPRESSURE_MEDIUM, + VMPRESSURE_CRITICAL, + VMPRESSURE_NUM_LEVELS, }; struct mem_cgroup; @@ -32,18 +45,41 @@ struct mem_cgroup; #ifdef CONFIG_MEMCG void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed); -extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); - extern void vmpressure_init(struct vmpressure *vmpr); extern void vmpressure_cleanup(struct vmpressure *vmpr); extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg); extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr); + +/* Shared with the v1 vmpressure block in mm/memcontrol-v1.c. */ +extern const unsigned long vmpressure_win; +extern enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed); + +#ifdef CONFIG_MEMCG_V1 +extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); extern int vmpressure_register_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); extern void vmpressure_unregister_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); + +/* v1 hooks called from mm/vmpressure.c; no-ops below when !MEMCG_V1. */ +extern void vmpressure_v1_init(struct vmpressure *vmpr); +extern void vmpressure_v1_cleanup(struct vmpressure *vmpr); +extern void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed); #else +static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, + int prio) {} +static inline void vmpressure_v1_init(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_cleanup(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) {} +#endif /* CONFIG_MEMCG_V1 */ + +#else /* !CONFIG_MEMCG */ static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed) {} diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 62552a2ce5b9..b749a9a5a5ee 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -244,7 +244,7 @@ void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, struct inode *inode); /** - * wbc_init_bio - writeback specific initializtion of bio + * wbc_init_bio - writeback specific initialization of bio * @wbc: writeback_control for the writeback in progress * @bio: bio to be initialized * diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index 78388538acf4..8851727ae162 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -78,9 +78,11 @@ TRACE_EVENT_CONDITION(damos_before_apply, TP_PROTO(unsigned int context_idx, unsigned int scheme_idx, unsigned int target_idx, struct damon_region *r, - unsigned int nr_regions, bool do_trace), + unsigned int nr_accesses, unsigned int nr_regions, + bool do_trace), - TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_regions, do_trace), + TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_accesses, + nr_regions, do_trace), TP_CONDITION(do_trace), @@ -101,7 +103,7 @@ TRACE_EVENT_CONDITION(damos_before_apply, __entry->target_idx = target_idx; __entry->start = r->ar.start; __entry->end = r->ar.end; - __entry->nr_accesses = r->nr_accesses_bp / 10000; + __entry->nr_accesses = nr_accesses; __entry->age = r->age; __entry->nr_regions = nr_regions; ), diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 291fae364c62..5a48c5406cce 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -16,7 +16,7 @@ EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \ - EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \ + EM( SCAN_PTE_UFFD, "pte_uffd_wp") \ EM( SCAN_PTE_MAPPED_HUGEPAGE, "pte_mapped_hugepage") \ EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \ EM( SCAN_PAGE_NULL, "page_null") \ diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index cd01dd7b3640..15ee2ef201b5 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h @@ -23,7 +23,8 @@ EM( MR_CONTIG_RANGE, "contig_range") \ EM( MR_LONGTERM_PIN, "longterm_pin") \ EM( MR_DEMOTION, "demotion") \ - EMe(MR_DAMON, "damon") + EM( MR_DAMON, "damon") \ + EMe(MR_NEVER, "never_migrated") /* * First define the enums in the above macros to be exported to userspace @@ -51,7 +52,7 @@ TRACE_EVENT(mm_migrate_pages, TP_PROTO(unsigned long succeeded, unsigned long failed, unsigned long thp_succeeded, unsigned long thp_failed, unsigned long thp_split, unsigned long large_folio_split, - enum migrate_mode mode, int reason), + enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(succeeded, failed, thp_succeeded, thp_failed, thp_split, large_folio_split, mode, reason), @@ -64,7 +65,7 @@ TRACE_EVENT(mm_migrate_pages, __field( unsigned long, thp_split) __field( unsigned long, large_folio_split) __field( enum migrate_mode, mode) - __field( int, reason) + __field( enum migrate_reason, reason) ), TP_fast_assign( @@ -91,13 +92,13 @@ TRACE_EVENT(mm_migrate_pages, TRACE_EVENT(mm_migrate_pages_start, - TP_PROTO(enum migrate_mode mode, int reason), + TP_PROTO(enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(mode, reason), TP_STRUCT__entry( __field(enum migrate_mode, mode) - __field(int, reason) + __field(enum migrate_reason, reason) ), TP_fast_assign( diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index a6e5a44c9b42..935893e5ea53 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -54,18 +54,10 @@ # define TRACE_GFP_FLAGS_LOCKDEP #endif -#ifdef CONFIG_SLAB_OBJ_EXT -# define TRACE_GFP_FLAGS_SLAB \ - TRACE_GFP_EM(NO_OBJ_EXT) -#else -# define TRACE_GFP_FLAGS_SLAB -#endif - #define TRACE_GFP_FLAGS \ TRACE_GFP_FLAGS_GENERAL \ TRACE_GFP_FLAGS_KASAN \ - TRACE_GFP_FLAGS_LOCKDEP \ - TRACE_GFP_FLAGS_SLAB + TRACE_GFP_FLAGS_LOCKDEP #undef TRACE_GFP_EM #define TRACE_GFP_EM(a) TRACE_DEFINE_ENUM(___GFP_##a##_BIT); @@ -194,6 +186,12 @@ IF_HAVE_PG_ARCH_3(arch_3) # define IF_HAVE_UFFD_MINOR(flag, name) #endif +#ifdef CONFIG_USERFAULTFD_RWP +# define IF_HAVE_UFFD_RWP(flag, name) {flag, name}, +#else +# define IF_HAVE_UFFD_RWP(flag, name) +#endif + #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) # define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name}, #else @@ -215,6 +213,7 @@ IF_HAVE_UFFD_MINOR(VM_UFFD_MINOR, "uffd_minor" ) \ {VM_PFNMAP, "pfnmap" }, \ {VM_MAYBE_GUARD, "maybe_guard" }, \ {VM_UFFD_WP, "uffd_wp" }, \ +IF_HAVE_UFFD_RWP(VM_UFFD_RWP, "uffd_rwp" ) \ {VM_LOCKED, "locked" }, \ {VM_IO, "io" }, \ {VM_SEQ_READ, "seqread" }, \ diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 171524d3526d..36c3a90f0acc 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -77,6 +77,14 @@ TRACE_EVENT(mm_lru_activate, TP_printk("folio=%p pfn=0x%lx", __entry->folio, __entry->pfn) ); +DECLARE_TRACE(mm_lru_add_drain, + TP_PROTO(int cpu, unsigned int nr_folios), + TP_ARGS(cpu, nr_folios)); + +DECLARE_TRACE(mm_lru_add_drain_all, + TP_PROTO(bool force_all_cpus), + TP_ARGS(force_all_cpus)); + #endif /* _TRACE_PAGEMAP_H */ /* This part must be outside protection */ diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h new file mode 100644 index 000000000000..270f693b1822 --- /dev/null +++ b/include/uapi/linux/alloc_tag.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * alloc_tag IOCTL API definition + * + * Copyright (C) 2026 Google, LLC. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _UAPI_ALLOC_TAG_H +#define _UAPI_ALLOC_TAG_H + +#include <linux/types.h> + +/* + * Function, file and module names often have the same prefixes, therefore + * when filtering by these criteria, we compare the last 64 characters to + * minimize the chances of name collisions + */ +#define ALLOCINFO_STR_SIZE 64 + +struct allocinfo_content_id { + __u64 id; +}; + +struct allocinfo_tag { + /* Longer names are trimmed */ + char modname[ALLOCINFO_STR_SIZE]; + char function[ALLOCINFO_STR_SIZE]; + char filename[ALLOCINFO_STR_SIZE]; + __u64 lineno; + /* filter criteria only; see allocinfo_counter.accurate for actual accuracy */ + __u64 inaccurate; +}; + +/* The alignment ensures 32-bit compatible interfaces are not broken */ +struct allocinfo_counter { + __u64 bytes; + __u64 calls; + __u8 accurate; +} __attribute__((aligned(8))); + +struct allocinfo_tag_data { + struct allocinfo_tag tag; + struct allocinfo_counter counter; +}; + +enum { + ALLOCINFO_FILTER_MODNAME, + ALLOCINFO_FILTER_FUNCTION, + ALLOCINFO_FILTER_FILENAME, + ALLOCINFO_FILTER_LINENO, + ALLOCINFO_FILTER_INACCURATE, + ALLOCINFO_FILTER_MIN_SIZE, + ALLOCINFO_FILTER_MAX_SIZE, + __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE +}; + +#define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME) +#define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION) +#define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME) +#define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO) +#define ALLOCINFO_FILTER_MASK_INACCURATE (1 << ALLOCINFO_FILTER_INACCURATE) +#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE) +#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE) + +#define ALLOCINFO_FILTER_MASKS \ + ((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1) + +struct allocinfo_filter { + __u64 mask; /* bitmask of the filter fields used */ + struct allocinfo_tag fields; + __u64 min_size; + __u64 max_size; +}; + +struct allocinfo_get_at { + /* inputs */ + __u64 pos; + struct allocinfo_filter filter; + /* output */ + struct allocinfo_tag_data data; +}; + +#define _ALLOCINFO_IOC_CONTENT_ID 0 +#define _ALLOCINFO_IOC_GET_AT 1 +#define _ALLOCINFO_IOC_GET_NEXT 2 + +#define ALLOCINFO_IOC_BASE 0xA6 +#define ALLOCINFO_IOC_CONTENT_ID _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID, \ + struct allocinfo_content_id) +#define ALLOCINFO_IOC_GET_AT _IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT, \ + struct allocinfo_get_at) +#define ALLOCINFO_IOC_GET_NEXT _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT, \ + struct allocinfo_tag_data) + +#endif /* _UAPI_ALLOC_TAG_H */ diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index bd87262f2e34..34c6f219462a 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -471,6 +471,7 @@ typedef int __bitwise __kernel_rwf_t; #define PAGE_IS_HUGE (1 << 6) #define PAGE_IS_SOFT_DIRTY (1 << 7) #define PAGE_IS_GUARD (1 << 8) +#define PAGE_IS_ACCESSED (1 << 9) /* * struct page_region - Page region with flags diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 6c962d866e86..7f6fc9599693 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -16,7 +16,7 @@ */ /* Policies */ -enum { +enum mempolicy_mode { MPOL_DEFAULT, MPOL_PREFERRED, MPOL_BIND, diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 2841e4ea8f2c..cea11aad6b54 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -25,7 +25,8 @@ #define UFFD_API ((__u64)0xAA) #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ UFFDIO_REGISTER_MODE_WP | \ - UFFDIO_REGISTER_MODE_MINOR) + UFFDIO_REGISTER_MODE_MINOR | \ + UFFDIO_REGISTER_MODE_RWP) #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ @@ -42,10 +43,13 @@ UFFD_FEATURE_WP_UNPOPULATED | \ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ - UFFD_FEATURE_MOVE) + UFFD_FEATURE_MOVE | \ + UFFD_FEATURE_RWP | \ + UFFD_FEATURE_RWP_ASYNC) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_SET_MODE | \ (__u64)1 << _UFFDIO_API) #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ @@ -54,13 +58,15 @@ (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) /* * Valid ioctl command number range with this API is from 0x00 to @@ -79,6 +85,8 @@ #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) +#define _UFFDIO_RWPROTECT (0x09) +#define _UFFDIO_SET_MODE (0x0A) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -103,6 +111,10 @@ struct uffdio_continue) #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ struct uffdio_poison) +#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ + struct uffdio_rwprotect) +#define UFFDIO_SET_MODE _IOW(UFFDIO, _UFFDIO_SET_MODE, \ + struct uffdio_set_mode) /* read() structure */ struct uffd_msg { @@ -158,6 +170,7 @@ struct uffd_msg { #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ +#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */ struct uffdio_api { /* userland asks for an API number and the features to enable */ @@ -230,6 +243,18 @@ struct uffdio_api { * * UFFD_FEATURE_MOVE indicates that the kernel supports moving an * existing page contents from userspace. + * + * UFFD_FEATURE_RWP indicates that the kernel supports + * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. + * Pages are made inaccessible via UFFDIO_RWPROTECT and faults + * are delivered when the pages are re-accessed. + * + * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for + * UFFDIO_REGISTER_MODE_RWP. When set, faults on read-write + * protected pages are auto-resolved by the kernel (PTE + * permissions restored immediately) without delivering a message + * to the userfaultfd handler. Use PAGEMAP_SCAN with inverted + * PAGE_IS_ACCESSED to find pages that were not re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -248,6 +273,8 @@ struct uffdio_api { #define UFFD_FEATURE_POISON (1<<14) #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) +#define UFFD_FEATURE_RWP (1<<17) +#define UFFD_FEATURE_RWP_ASYNC (1<<18) __u64 features; __u64 ioctls; @@ -263,6 +290,7 @@ struct uffdio_register { #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) +#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3) __u64 mode; /* @@ -356,6 +384,14 @@ struct uffdio_poison { __s64 updated; }; +struct uffdio_rwprotect { + struct uffdio_range range; + /* !RWP means undo RWP-protection */ +#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0) +#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + struct uffdio_move { __u64 dst; __u64 src; @@ -374,6 +410,16 @@ struct uffdio_move { __s64 move; }; +struct uffdio_set_mode { + /* + * Toggle async mode for features at runtime. + * Supported: UFFD_FEATURE_RWP_ASYNC. + * Setting a bit in both enable and disable is invalid. + */ + __u64 enable; + __u64 disable; +}; + /* * Flags for the userfaultfd(2) system call itself. */ diff --git a/init/Kconfig b/init/Kconfig index 10f2013b5321..13cb6fdd9c37 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -948,6 +948,13 @@ config SCHED_PROXY_EXEC endmenu # +# For architectures that support present-but-inaccessible (PROT_NONE) page +# table entries detectable via pte_protnone() / pmd_protnone(): +# +config ARCH_HAS_PTE_PROTNONE + bool + +# # For architectures that want to enable the support for NUMA-affine scheduler # balancing logic: # @@ -1013,6 +1020,7 @@ config ARCH_WANT_NUMA_VARIABLE_LOCALITY config NUMA_BALANCING bool "Memory placement aware NUMA scheduler" depends on ARCH_SUPPORTS_NUMA_BALANCING + depends on ARCH_HAS_PTE_PROTNONE depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY depends on SMP && NUMA_MIGRATION && !PREEMPT_RT help @@ -2115,7 +2123,7 @@ config ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS from a kernel perspective. After the architecture enables this, a distribution can set - CONFIG_MSEAL_SYSTEM_MAPPING to manage access to the feature. + CONFIG_MSEAL_SYSTEM_MAPPINGS to manage access to the feature. For complete descriptions of memory sealing, please see Documentation/userspace-api/mseal.rst diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d..bb1a721a3e74 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1661,7 +1661,8 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto invalid; } - addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL); + addr = do_mmap(file, addr, size, prot, flags, EMPTY_VMA_FLAGS, 0, + &populate, NULL); *raddr = addr; err = 0; if (IS_ERR_VALUE(addr)) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45944b3e31ca..ee4025b9707a 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4162,7 +4162,7 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk) * * Are any of the nodes in the nodemask allowed in current->mems_allowed? */ -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return nodes_intersects(*nodemask, current->mems_allowed); } @@ -4198,7 +4198,7 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) * nearest enclosing hardwalled ancestor cpuset. * * Scanning up parent cpusets requires callback_lock. The - * __alloc_pages() routine only calls here with __GFP_HARDWALL bit + * page allocator only calls here with __GFP_HARDWALL bit * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the * current tasks mems_allowed came up empty on the first pass over * the zonelist. So only GFP_KERNEL allocations, if all nodes in the @@ -4211,11 +4211,8 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) * come before the __GFP_HARDWALL check, otherwise a dying task * would be blocked on the fast path. * - * The second pass through get_page_from_freelist() doesn't even call - * here for GFP_ATOMIC calls. For those calls, the __alloc_pages() - * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set - * in alloc_flags. That logic and the checks below have the combined - * affect that: + * The second pass through get_page_from_freelist() doesn't even call here for + * GFP_ATOMIC calls. That, and the checks below have the combined affect that: * in_interrupt - any node ok (current task context irrelevant) * GFP_ATOMIC - any node ok * tsk_is_oom_victim - any node ok @@ -4332,8 +4329,8 @@ void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) * should not be possible for the following code to return an * offline node. But if it did, that would be ok, as this routine * is not returning the node where the allocation must be, only - * the node where the search should start. The zonelist passed to - * __alloc_pages() will include all nodes. If the slab allocator + * the node where the search should start. The zonelist used by + * the allocator will include all nodes. If the slab allocator * is passed an offline node, it will fall back to the local node. * See kmem_cache_alloc_node(). */ diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index bcdc0f76d2e8..2d3195eb7e83 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -236,14 +236,15 @@ static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem, { if (mem && vaddr >= mem->virt_base && vaddr + size <= (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) { - unsigned long off = vma->vm_pgoff; + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; unsigned long user_count = vma_pages(vma); int count = PAGE_ALIGN(size) >> PAGE_SHIFT; *ret = -ENXIO; - if (off < count && user_count <= count - off) { - unsigned long pfn = mem->pfn_base + start + off; + if (pgoff_start < count && pgoff_end <= count) { + unsigned long pfn = mem->pfn_base + start + pgoff_start; *ret = remap_pfn_range(vma, vma->vm_start, pfn, user_count << PAGE_SHIFT, vma->vm_page_prot); diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 4391b797d4db..436310d6e4a2 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -534,6 +534,8 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, unsigned long user_count = vma_pages(vma); unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); int ret = -ENXIO; vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); @@ -545,9 +547,9 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) return ret; - if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) + if (pgoff_start >= count || pgoff_end > count) return -ENXIO; - return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, + return remap_pfn_range(vma, vma->vm_start, pfn + pgoff_start, user_count << PAGE_SHIFT, vma->vm_page_prot); } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 4fe04669e5e6..c986639044e9 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -761,12 +761,14 @@ EXPORT_SYMBOL_GPL(dma_free_pages); int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, size_t size, struct page *page) { - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); + const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) + if (pgoff_start >= count || pgoff_end > count) return -ENXIO; return remap_pfn_range(vma, vma->vm_start, - page_to_pfn(page) + vma->vm_pgoff, + page_to_pfn(page) + pgoff_start, vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); } EXPORT_SYMBOL_GPL(dma_mmap_pages); diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 20caf9cabf69..6b5f9208d31c 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c @@ -39,7 +39,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, #ifdef CONFIG_MMU unsigned long user_count = vma_pages(vma); unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long off = vma->vm_pgoff; + unsigned long off = vma_start_pgoff(vma); struct page *page = dma_common_vaddr_to_page(cpu_addr); int ret = -ENXIO; @@ -52,7 +52,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, return -ENXIO; return remap_pfn_range(vma, vma->vm_start, - page_to_pfn(page) + vma->vm_pgoff, + page_to_pfn(page) + vma_start_pgoff(vma), user_count << PAGE_SHIFT, vma->vm_page_prot); #else return -ENXIO; diff --git a/kernel/events/core.c b/kernel/events/core.c index ba5bd6a78fe7..cf78e892a4bb 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6998,7 +6998,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (vma_start_pgoff(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7032,7 +7032,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && vma_start_pgoff(vma) == rb->aux_pgoff && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7192,7 +7192,8 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) */ for (pagenum = 0; pagenum < nr_pages; pagenum++) { unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + struct page *page = perf_mmap_to_page(rb, + vma_start_pgoff(vma) + pagenum); if (page == NULL) { err = -EINVAL; @@ -7346,6 +7347,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { + const pgoff_t pgoff_start = vma_start_pgoff(vma); long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7368,11 +7370,11 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (aux_offset < perf_data_size(rb) + PAGE_SIZE) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != pgoff_start << PAGE_SHIFT) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_pgoff != pgoff_start) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7402,7 +7404,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, pgoff_start, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7459,7 +7461,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + if (!vma_start_pgoff(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else ret = perf_mmap_aux(vma, event, nr_pages); @@ -9886,7 +9888,7 @@ static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter, struct perf_addr_filter_range *fr) { unsigned long vma_size = vma->vm_end - vma->vm_start; - unsigned long off = vma->vm_pgoff << PAGE_SHIFT; + unsigned long off = vma_start_pgoff(vma) << PAGE_SHIFT; struct file *file = vma->vm_file; if (!perf_addr_filter_match(filter, file, off, vma_size)) @@ -9976,7 +9978,7 @@ void perf_event_mmap(struct vm_area_struct *vma) /* .tid */ .start = vma->vm_start, .len = vma->vm_end - vma->vm_start, - .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, + .pgoff = (u64)vma_start_pgoff(vma) << PAGE_SHIFT, }, /* .maj (attr_mmap2 only) */ /* .min (attr_mmap2 only) */ diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e284..ae2f3b9f8d50 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -144,12 +144,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT) + + (vaddr - vma->vm_start); } /** @@ -1210,7 +1212,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (!valid_vma(vma, is_register)) continue; @@ -1482,7 +1484,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = (loff_t)vma_start_pgoff(vma) << PAGE_SHIFT; if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -2453,7 +2455,8 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = (loff_t)(vma_start_pgoff(vma) << PAGE_SHIFT) + + (bp_vaddr - vma->vm_start); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/kcov.c b/kernel/kcov.c index 1df373fb562b..b19b473c366a 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -512,7 +512,7 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) spin_lock_irqsave(&kcov->lock, flags); size = kcov->size * sizeof(unsigned long); - if (kcov->area == NULL || vma->vm_pgoff != 0 || + if (kcov->area == NULL || vma_start_pgoff(vma) || vma->vm_end - vma->vm_start != size) { res = -EINVAL; goto exit; diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 175c08a6e41e..be50c22196b0 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -34,7 +34,8 @@ * KHO is tightly coupled with mm init and needs access to some of mm * internal APIs. */ -#include "../../mm/internal.h" +#include "../../mm/mm_init.h" +#include "../../mm/vmalloc.h" #include "../kexec_internal.h" #include "kexec_handover_internal.h" diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af8..a69567bdd791 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -5077,7 +5077,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read); static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, int hardirqs_off, struct lockdep_map *nest_lock, unsigned long ip, - int references, int pin_count, int sync) + int references, int pin_count, int sync, int seq) { struct task_struct *curr = current; struct lock_class *class = NULL; @@ -5183,6 +5183,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, hlock->holdtime_stamp = lockstat_clock(); #endif hlock->pin_count = pin_count; + hlock->seq_count = seq; if (check_wait_context(curr, hlock)) return 0; @@ -5388,7 +5389,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth, hlock->read, hlock->check, hlock->hardirqs_off, hlock->nest_lock, hlock->acquire_ip, - hlock->references, hlock->pin_count, 0)) { + hlock->references, hlock->pin_count, 0, hlock->seq_count)) { case 0: return 1; case 1: @@ -5669,14 +5670,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie struct held_lock *hlock = curr->held_locks + i; if (match_held_lock(hlock, lock)) { + int pin_count; + if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n")) return; - hlock->pin_count -= cookie.val; + pin_count = hlock->pin_count - cookie.val; - if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n")) - hlock->pin_count = 0; + if (WARN(pin_count < 0, "pin count corrupted\n")) + pin_count = 0; + hlock->pin_count = pin_count; return; } } @@ -5684,6 +5688,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie WARN(1, "unpinning an unheld lock\n"); } +static u32 __lock_sequence(struct lockdep_map *lock) +{ + struct task_struct *curr = current; + int i; + + if (unlikely(!debug_locks)) + return ~0; + + for (i = 0; i < curr->lockdep_depth; i++) { + struct held_lock *hlock = curr->held_locks + i; + + if (match_held_lock(hlock, lock)) + return hlock->seq_count; + } + + return ~0; +} + /* * Check whether we follow the irq-flags state precisely: */ @@ -5866,7 +5888,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass, lockdep_recursion_inc(); __lock_acquire(lock, subclass, trylock, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0, + ++current->lockdep_seq); lockdep_recursion_finish(); raw_local_irq_restore(flags); } @@ -5914,7 +5937,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read, lockdep_recursion_inc(); __lock_acquire(lock, subclass, 0, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1, + ++current->lockdep_seq); check_chain_key(current); lockdep_recursion_finish(); raw_local_irq_restore(flags); @@ -6000,6 +6024,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) } EXPORT_SYMBOL_GPL(lock_unpin_lock); +u32 lock_sequence(struct lockdep_map *lock) +{ + unsigned long flags; + u32 seq = ~0; + + if (unlikely(!lockdep_enabled())) + return seq; + + raw_local_irq_save(flags); + check_flags(flags); + + lockdep_recursion_inc(); + seq = __lock_sequence(lock); + lockdep_recursion_finish(); + raw_local_irq_restore(flags); + + return seq; +} +EXPORT_SYMBOL_GPL(lock_sequence); + #ifdef CONFIG_LOCK_STAT static void print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock, diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 804ccae694d2..e863eb0d10f7 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7611,7 +7611,8 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, struct vm_area_struct *vma) { - unsigned long nr_subbufs, nr_pages, nr_vma_pages, pgoff = vma->vm_pgoff; + unsigned long nr_subbufs, nr_pages, nr_vma_pages; + pgoff_t pgoff = vma_start_pgoff(vma); unsigned int subbuf_pages, subbuf_order; struct page **pages __free(kfree) = NULL; int p = 0, s = 0; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1244dcac2294..b82515cde538 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1048,34 +1048,6 @@ config CODE_TAGGING bool select KALLSYMS -config MEM_ALLOC_PROFILING - bool "Enable memory allocation profiling" - default n - depends on MMU - depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU - select CODE_TAGGING - select PAGE_EXTENSION - select SLAB_OBJ_EXT - help - Track allocation source code and record total allocation size - initiated at that code location. The mechanism can be used to track - memory leaks with a low performance and memory impact. - -config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - bool "Enable memory allocation profiling by default" - default y - depends on MEM_ALLOC_PROFILING - -config MEM_ALLOC_PROFILING_DEBUG - bool "Memory allocation profiler debugging" - default n - depends on MEM_ALLOC_PROFILING - select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - help - Adds warnings with helpful error messages for memory allocation - profiling. - source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" diff --git a/lib/Makefile b/lib/Makefile index 7f75cc6edf94..531a0be88062 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -196,7 +196,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o -obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o lib-$(CONFIG_GENERIC_BUG) += bug.o diff --git a/lib/codetag.c b/lib/codetag.c index 4001a7ea6675..a9cda4c962a3 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -19,6 +19,8 @@ struct codetag_type { struct codetag_type_desc desc; /* generates unique sequence number for module load */ unsigned long next_mod_seq; + /* bumped on every module load and unload */ + unsigned long content_id; }; struct codetag_range { @@ -50,6 +52,20 @@ void codetag_unlock_module_list(struct codetag_type *cttype) up_read(&cttype->mod_lock); } +unsigned long codetag_get_content_id(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->content_id; +} + +unsigned int codetag_get_count(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->count; +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { @@ -204,6 +220,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) down_write(&cttype->mod_lock); cmod->mod_seq = ++cttype->next_mod_seq; + ++cttype->content_id; mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL); if (mod_id >= 0) { if (cttype->desc.module_load) { @@ -368,6 +385,7 @@ void codetag_unload_module(struct module *mod) cttype->count -= range_size(cttype, &cmod->range); idr_remove(&cttype->mod_idr, mod_id); kfree(cmod); + ++cttype->content_id; } up_write(&cttype->mod_lock); if (found && cttype->desc.free_section_mem) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e52876435b77..e8eb2e1219db 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas) return mas->status == ma_underflow; } +static inline void mas_make_walkable(struct ma_state *mas) +{ + if (!mas_is_active(mas) && !mas_is_start(mas)) + mas->status = ma_start; +} + static __always_inline struct maple_node *mte_to_node( const struct maple_enode *entry) { @@ -456,46 +462,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode) } /* - * mas_set_parent() - Set the parent node and encode the slot - * @mas: The maple state - * @enode: The encoded maple node. - * @parent: The encoded maple node that is the parent of @enode. - * @slot: The slot that @enode resides in @parent. - * - * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the - * parent type. - */ -static inline -void mas_set_parent(struct ma_state *mas, struct maple_enode *enode, - const struct maple_enode *parent, unsigned char slot) -{ - unsigned long val = (unsigned long)parent; - unsigned long shift; - unsigned long type; - enum maple_type p_type = mte_node_type(parent); - - MAS_BUG_ON(mas, p_type == maple_dense); - MAS_BUG_ON(mas, p_type == maple_leaf_64); - - switch (p_type) { - case maple_range_64: - case maple_arange_64: - shift = MAPLE_PARENT_SLOT_SHIFT; - type = MAPLE_PARENT_RANGE64; - break; - default: - case maple_dense: - case maple_leaf_64: - shift = type = 0; - break; - } - - val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ - val |= (slot << shift) | type; - mte_to_node(enode)->parent = ma_parent_ptr(val); -} - -/* * mte_parent_slot() - get the parent slot of @enode. * @enode: The encoded maple node. * @@ -877,6 +843,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, } /* + * mas_set_parent_slots() - Bulk operation to set many slot parent pointers + * @mas: The maple state + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that of the @enode. + * @start_slot: The offset into @slot + * @count: The number of slots to set (eg: exclusive) + */ +static inline +void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent, + void __rcu **slots, unsigned char start_slot, unsigned char count) +{ + unsigned long val; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + unsigned char i; + + MAS_BUG_ON(mas, p_type != maple_range_64 && + p_type != maple_arange_64); + + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + + val = (unsigned long)parent; + val &= ~MAPLE_NODE_MASK; + + for (i = 0; i < count; i++) { + unsigned long pval = val | ((start_slot + i) << shift) | type; + struct maple_enode *child; + + child = mt_slot_locked(mas->tree, slots, i); + mte_to_node(child)->parent = ma_parent_ptr(pval); + } +} + +/* * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. * @mat: the ma_topiary, a linked list of dead nodes. * @dead_enode: the node to be marked as dead and added to the tail of the list @@ -1153,6 +1155,73 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) ma_free_rcu(mte_to_node(used)); } + +#ifdef CONFIG_LOCKDEP +static struct lockdep_map *mas_lockdep_map(struct ma_state *mas) +{ + struct maple_tree *mt = mas->tree; + + if (mt_external_lock(mt)) + return mt->ma_external_lock; + + return &(mt->ma_lock).dep_map; +} + +#endif + +static void mas_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; + if (!mas_is_active(mas)) + return; + +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp)); + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map && lock_is_held(map)) + WARN_ON_ONCE(mas->ld_seq != lock_sequence(map)); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + mas->rcu_gp = get_state_synchronize_rcu(); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map && lock_is_held(map)) + mas->ld_seq = lock_sequence(mas_lockdep_map(mas)); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_may_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (mas_is_start(mas) || mas_is_paused(mas)) { + mas_init_lock_check(mas); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + mas_lock_check(mas); +#endif /* CONFIG_LOCKDEP */ +} + /* * mas_start() - Sets up maple state for operations. * @mas: The maple state. @@ -1171,6 +1240,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas) if (likely(mas_is_start(mas))) { struct maple_enode *root; + mas_init_lock_check(mas); mas->min = 0; mas->max = ULONG_MAX; @@ -1501,14 +1571,26 @@ ascend: goto ascend; } +static __always_inline void mas_update_gap_known(struct ma_state *mas, + unsigned long gap) +{ + unsigned char pslot; + unsigned long p_gap; + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), + mas_parent_type(mas, mas->node))[pslot]; + + if (p_gap != gap) + mas_parent_gap(mas, pslot, gap); +} + /* * mas_update_gap() - Update a nodes gaps and propagate up if necessary. * @mas: the maple state. */ static inline void mas_update_gap(struct ma_state *mas) { - unsigned char pslot; - unsigned long p_gap; unsigned long max_gap; if (!mt_is_alloc(mas->tree)) @@ -1518,13 +1600,7 @@ static inline void mas_update_gap(struct ma_state *mas) return; max_gap = mas_max_gap(mas); - - pslot = mte_parent_slot(mas->node); - p_gap = ma_gaps(mte_parent(mas->node), - mas_parent_type(mas, mas->node))[pslot]; - - if (p_gap != max_gap) - mas_parent_gap(mas, pslot, max_gap); + mas_update_gap_known(mas, max_gap); } /* @@ -1540,14 +1616,10 @@ static inline void mas_adopt_children(struct ma_state *mas, struct maple_node *node = mte_to_node(parent); void __rcu **slots = ma_slots(node, type); unsigned long *pivots = ma_pivots(node, type); - struct maple_enode *child; - unsigned char offset; + unsigned char end; - offset = ma_data_end(node, type, pivots, mas->max); - do { - child = mas_slot_locked(mas, slots, offset); - mas_set_parent(mas, child, parent, offset); - } while (offset--); + end = ma_data_end(node, type, pivots, mas->max); + mas_set_parent_slots(mas, parent, slots, 0, end + 1); } /* @@ -1929,15 +2001,10 @@ unsigned long node_copy(struct ma_state *mas, struct maple_node *src, s_slots = ma_slots(src, s_mt) + start; s_pivots = ma_pivots(src, s_mt) + start; memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); - if (!ma_is_leaf(d_mt) && s_mt == maple_copy) { - struct maple_enode *edst = mt_mk_node(dst, d_mt); - - for (int i = 0; i < size; i++) - mas_set_parent(mas, - mt_slot_locked(mas->tree, d_slots, i), - edst, d_start + i); - } + if (!ma_is_leaf(d_mt) && s_mt == maple_copy) + mas_set_parent_slots(mas, mt_mk_node(dst, d_mt), + d_slots, d_start, size); d_gaps = ma_gaps(dst, d_mt); if (d_gaps) { @@ -2081,8 +2148,8 @@ static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp) mas->node = mt_slot_locked(mas->tree, cp->slot, 0); /* Insert the new data in the tree */ mas_topiary_replace(mas, old_enode, cp->height); - if (!mte_is_leaf(mas->node)) - mas_update_gap(mas); + if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node)) + mas_update_gap_known(mas, cp->gap[0]); mtree_range_walk(mas); } @@ -2928,13 +2995,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, if (r_mas->last < r_wr_mas->r_max) r_mas->last = r_wr_mas->r_max; r_mas->offset++; - } else if ((r_mas->last == r_wr_mas->r_max) && - (r_mas->last < r_mas->max) && - !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { - r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, - r_wr_mas->type, r_mas->offset + 1); - r_mas->offset++; - r_wr_mas->r_max = r_mas->last; } } @@ -3125,7 +3185,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas) static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) { unsigned char dst_offset, offset_end; - unsigned char copy_size, node_pivots; + unsigned char copy_size, node_pivots, node_slots; struct maple_node reuse, *newnode; unsigned long *dst_pivots; void __rcu **dst_slots; @@ -3138,6 +3198,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) in_rcu = mt_in_rcu(mas->tree); offset_end = wr_mas->offset_end; node_pivots = mt_pivots[wr_mas->type]; + node_slots = mt_slots[wr_mas->type]; /* Assume last adds an entry */ new_end = mas->end + 1 - offset_end + mas->offset; if (mas->last == wr_mas->end_piv) { @@ -3149,7 +3210,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) if (in_rcu) { newnode = mas_pop_node(mas); } else { - memset(&reuse, 0, sizeof(struct maple_node)); newnode = &reuse; } @@ -3193,7 +3253,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) dst_pivots[new_end] = mas->max; done: - mas_leaf_set_meta(newnode, maple_leaf_64, new_end); + if (!in_rcu && new_end + 2 < node_slots) { + unsigned char clear_from = new_end + 1; + + /* + * Note that the last slot is never cleared, since the metadata + * will be stored there or it has a value. + */ + memset(dst_slots + clear_from, 0, + sizeof(void __rcu *) * (node_slots - clear_from)); + if (clear_from < node_pivots) + memset(dst_pivots + clear_from, 0, + sizeof(unsigned long) * (node_pivots - clear_from)); + } + + mas_leaf_set_meta(newnode, wr_mas->type, new_end); if (in_rcu) { struct maple_enode *old_enode = mas->node; @@ -3218,7 +3292,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) void __rcu **slots = wr_mas->slots; bool gap = false; - gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !wr_mas->content; gap |= !mt_slot_locked(mas->tree, slots, offset + 1); if (wr_mas->offset_end - offset == 1) { @@ -3661,6 +3735,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; unsigned char new_end; + bool appending; + bool one_slot; + bool in_rcu; if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) return wr_store_root; @@ -3680,21 +3757,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) return wr_new_root; new_end = mas_wr_new_end(wr_mas); + in_rcu = mt_in_rcu(mas->tree); + appending = mas->offset == mas->end; + one_slot = wr_mas->offset_end - mas->offset == 1; + /* Potential spanning rebalance collapsing a node */ if (new_end < mt_min_slots[wr_mas->type]) { if (!mte_is_root(mas->node)) return wr_rebalance; + if (!in_rcu) { + if (appending) + return wr_append; + else if (mas->end == new_end && one_slot) + return wr_slot_store; + } return wr_node_store; } if (new_end >= mt_slots[wr_mas->type]) return wr_split_store; - if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end)) + if (!in_rcu && appending) return wr_append; - if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) || - (wr_mas->offset_end - mas->offset == 1))) + if (new_end == mas->end && (!in_rcu || one_slot)) return wr_slot_store; return wr_node_store; @@ -3793,35 +3879,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } @@ -4360,8 +4451,8 @@ void *mas_walk(struct ma_state *mas) { void *entry; - if (!mas_is_active(mas) && !mas_is_start(mas)) - mas->status = ma_start; + mas_may_init_lock_check(mas); + mas_make_walkable(mas); retry: entry = mas_state_walk(mas); if (mas_is_start(mas)) { @@ -4831,6 +4922,7 @@ void *mas_store(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); trace_ma_write(TP_FCT, mas, 0, entry); #ifdef CONFIG_DEBUG_MAPLE_TREE if (MAS_WARN_ON(mas, mas->index > mas->last)) @@ -4889,6 +4981,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) MA_WR_STATE(wr_mas, mas, entry); int ret = 0; + mas_may_init_lock_check(mas); retry: mas_wr_preallocate(&wr_mas, entry); if (unlikely(mas_nomem(mas, gfp))) { @@ -4919,6 +5012,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_lock_check(mas); if (mas->store_type == wr_store_root) { mas_wr_prealloc_setup(&wr_mas); goto store; @@ -4951,6 +5045,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); mas_wr_prealloc_setup(&wr_mas); mas->store_type = mas_wr_store_type(&wr_mas); mas_prealloc_calc(&wr_mas, entry); @@ -4997,6 +5092,7 @@ static void mas_may_activate(struct ma_state *mas) mas->status = ma_start; } else { mas->status = ma_active; + mas_lock_check(mas); } } @@ -5074,6 +5170,7 @@ void *mas_next(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5097,6 +5194,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5205,6 +5303,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5228,6 +5327,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5274,6 +5374,7 @@ EXPORT_SYMBOL_GPL(mt_prev); */ void mas_pause(struct ma_state *mas) { + mas_lock_check(mas); mas->status = ma_pause; mas->node = NULL; } @@ -5382,6 +5483,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5409,6 +5511,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5428,7 +5531,6 @@ EXPORT_SYMBOL_GPL(mas_find_range); static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) { - switch (mas->status) { case ma_active: goto active; @@ -5521,6 +5623,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5547,6 +5650,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5564,6 +5668,10 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev); * Searches for @mas->index, sets @mas->index and @mas->last to the range and * erases that range. * + * Note that erase requires allocations and will use GFP_KERNEL to do so if + * necessary. If the allocation fails, the internal lock will be dropped to + * retry. + * * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. */ void *mas_erase(struct ma_state *mas) @@ -5572,13 +5680,19 @@ void *mas_erase(struct ma_state *mas) unsigned long index = mas->index; MA_WR_STATE(wr_mas, mas, NULL); - if (!mas_is_active(mas) || !mas_is_start(mas)) - mas->status = ma_start; + /* + * In low memory situations, the allocation is retried with the gfp flag + * GFP_KERNEL. The internal spinlock is dropped in mas_nomem(), however + * the external lock is not dropped. + */ + if (mt_external_lock(mas->tree)) + might_alloc(GFP_KERNEL); + mas_make_walkable(mas); write_retry: entry = mas_state_walk(mas); if (!entry) - return NULL; + goto out; /* Must reset to ensure spanning writes of last slot are detected */ mas_reset(mas); @@ -5589,8 +5703,10 @@ write_retry: goto write_retry; } - if (mas_is_err(mas)) + if (mas_is_err(mas)) { + entry = NULL; goto out; + } mas_wr_store_entry(&wr_mas); out: @@ -5601,10 +5717,11 @@ EXPORT_SYMBOL_GPL(mas_erase); /** * mas_nomem() - Check if there was an error allocating and do the allocation - * if necessary If there are allocations, then free them. + * if necessary. + * * @mas: The maple state * @gfp: The GFP_FLAGS to use for allocations - * Return: true on allocation, false otherwise. + * Return: False on no memory. True otherwise (partial success as well) */ bool mas_nomem(struct ma_state *mas, gfp_t gfp) __must_hold(mas->tree->ma_lock) @@ -5612,6 +5729,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) if (likely(mas->node != MA_ERROR(-ENOMEM))) return false; + /* Allocations can fail, don't do this. */ + WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) && + mt_external_lock(mas->tree)); + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { mtree_unlock(mas->tree); mas_alloc_nodes(mas, gfp); @@ -5620,10 +5741,17 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) mas_alloc_nodes(mas, gfp); } - if (!mas->sheaf && !mas->alloc) + /* + * Return false on zero forward progress. Partial allocations are kept + * so the retry path will attempt to get the rest. The failure should + * not happen as we try our best to reclaim. The user would need an + * external lock with a non-blocking gfp in a low memory situation - + * which would have triggered the first warning in this function. + */ + if (WARN_ON_ONCE(!mas->sheaf && !mas->alloc)) return false; - mas->status = ma_start; + mas_reset(mas); return true; } @@ -5913,6 +6041,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange); * Erasing is the same as a walk to an entry then a store of a NULL to that * ENTIRE range. In fact, it is implemented as such using the advanced API. * + * Note that erase requires allocations and will use GFP_KERNEL to do so if + * necessary. If the allocation fails, the internal lock will be dropped to + * retry. + * * Return: The entry stored at the @index or %NULL */ void *mtree_erase(struct maple_tree *mt, unsigned long index) @@ -5922,6 +6054,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); trace_ma_op(TP_FCT, &mas); + might_alloc(GFP_KERNEL); mtree_lock(mt); entry = mas_erase(&mas); mtree_unlock(mt); @@ -6447,7 +6580,7 @@ static void mt_dump_range(unsigned long min, unsigned long max, { static const char spaces[] = " "; - switch(format) { + switch (format) { case mt_dump_hex: if (min == max) pr_info("%.*s%lx: ", depth * 2, spaces, min); @@ -6489,7 +6622,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, pr_cont(" contents: "); for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) { - switch(format) { + switch (format) { case mt_dump_hex: pr_cont(PTR_FMT " %lX ", node->slot[i], node->pivot[i]); break; @@ -6517,7 +6650,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); @@ -6576,7 +6709,7 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 97772e42b9b2..f8d90689af9f 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref, * @confirm_switch: optional confirmation callback * * There's no reason to use this function for the usual reference counting. - * Use percpu_ref_kill[_and_confirm](). + * Use percpu_ref_kill() or percpu_ref_kill_and_confirm(). * * Schedule switching of @ref to atomic mode. All its percpu counts will * be collected to the main atomic counter. On completion, when all CPUs diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 976b9bd02a1b..cb5ca920fddb 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -455,6 +455,8 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, node->slots[0] = (void __rcu *)entry; entry = node_to_entry(node); rcu_assign_pointer(root->xa_head, entry); + /* new head may be missed by an in-progress kmemleak scan */ + kmemleak_transient_leak(node); shift += RADIX_TREE_MAP_SHIFT; } while (shift <= maxshift); out: @@ -495,8 +497,11 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root) if (!node->shift && is_idr(root)) break; - if (radix_tree_is_internal_node(child)) + if (radix_tree_is_internal_node(child)) { entry_to_node(child)->parent = NULL; + /* new head may be missed by an in-progress kmemleak scan */ + kmemleak_transient_leak(entry_to_node(child)); + } /* * We don't need rcu_assign_pointer(), since we are simply diff --git a/lib/test_hmm.c b/lib/test_hmm.c index c4adbf98fac7..6205fb313bd0 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -389,6 +389,67 @@ out: return ret; } +static int dmirror_range_fault_unlocked(struct dmirror *dmirror, + struct hmm_range *range, + unsigned long timeout) +{ + int ret; + + while (true) { + ret = hmm_range_fault_unlocked_timeout(range, timeout); + if (ret) + goto out; + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + ret = dmirror_do_fault(dmirror, range); + + mutex_unlock(&dmirror->mutex); +out: + return ret; +} + +static int dmirror_fault_unlocked(struct dmirror *dmirror, + unsigned long start, + unsigned long end, bool write, + unsigned long timeout) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long addr; + unsigned long pfns[32]; + struct hmm_range range = { + .notifier = &dmirror->notifier, + .hmm_pfns = pfns, + .pfn_flags_mask = 0, + .default_flags = + HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + if (!mmget_not_zero(mm)) + return -EFAULT; + + for (addr = start; addr < end; addr = range.end) { + range.start = addr; + range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + + ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); + if (ret) + break; + } + + mmput(mm); + return ret; +} + static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { @@ -407,7 +468,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; @@ -488,6 +549,48 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) return ret; } +static int dmirror_read_unlocked(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd, + unsigned long timeout) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); + if (ret) + break; + cmd->faults++; + } + + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { @@ -1572,7 +1675,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, dmirror->flags = cmd.npages; ret = 0; break; - + case HMM_DMIRROR_READ_UNLOCKED: + ret = dmirror_read_unlocked(dmirror, &cmd, 0); + break; default: return -EINVAL; } diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index f94c6d457338..ea9b0ec404fb 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -38,6 +38,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd) #define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_READ_UNLOCKED _IOWR('H', 0x09, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0) diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca4824905..c52ab6afcb15 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -125,8 +125,6 @@ config ZSWAP_COMPRESSOR_DEFAULT config ZSMALLOC tristate -if ZSMALLOC - menu "Zsmalloc allocator options" depends on ZSMALLOC @@ -161,8 +159,6 @@ config ZSMALLOC_CHAIN_SIZE endmenu -endif - menu "Slab allocator options" config SLUB @@ -509,13 +505,6 @@ config EXCLUSIVE_SYSTEM_RAM def_bool y depends on !DEVMEM || STRICT_DEVMEM -# -# Only be set on architectures that have completely implemented memory hotplug -# feature. If you are not sure, don't touch it. -# -config HAVE_BOOTMEM_INFO_NODE - def_bool n - config ARCH_ENABLE_MEMORY_HOTPLUG bool @@ -590,13 +579,10 @@ endchoice config MEMORY_HOTREMOVE bool "Allow for memory hot remove" - select HAVE_BOOTMEM_INFO_NODE if X86_64 - depends on MEMORY_HOTPLUG select MIGRATION config MHP_MEMMAP_ON_MEMORY def_bool y - depends on MEMORY_HOTPLUG && SPARSEMEM_VMEMMAP depends on ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE endif # MEMORY_HOTPLUG @@ -703,7 +689,7 @@ config DEVICE_MIGRATION config ARCH_ENABLE_HUGEPAGE_MIGRATION bool -config ARCH_ENABLE_THP_MIGRATION +config ARCH_HAS_PMD_SOFTLEAVES bool config HUGETLB_PAGE_SIZE_VARIABLE @@ -1235,9 +1221,7 @@ config ZONE_DMA32 config ZONE_DEVICE bool "Device memory (pmem, HMM, etc...) hotplug support" - depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE - depends on SPARSEMEM_VMEMMAP select XARRAY_MULTI help @@ -1386,6 +1370,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR help Arch has userfaultfd minor fault support +config USERFAULTFD_RWP + def_bool y + depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP + help + Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a + userfaultfd notification on every access -- read or write -- to a + protected range, letting userspace observe the working set of a + process. + menuconfig USERFAULTFD bool "Enable userfaultfd() system call" depends on MMU @@ -1393,17 +1386,15 @@ menuconfig USERFAULTFD Enable the userfaultfd() system call that allows to intercept and handle page faults in userland. -if USERFAULTFD config PTE_MARKER_UFFD_WP bool "Userfaultfd write protection support for shmem/hugetlbfs" default y - depends on HAVE_ARCH_USERFAULTFD_WP + depends on USERFAULTFD && HAVE_ARCH_USERFAULTFD_WP help Allows to create marker PTEs for userfaultfd write protection purposes. It is required to enable userfaultfd write protection on file-backed memory types like shmem and hugetlbfs. -endif # USERFAULTFD # multi-gen LRU { config LRU_GEN diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 91b3e027b753..5737a504efbb 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -320,3 +320,31 @@ config PER_VMA_LOCK_STATS overhead in the page fault path. If in doubt, say N. + +config MEM_ALLOC_PROFILING + bool "Enable memory allocation profiling" + default n + depends on MMU + depends on PROC_FS + depends on !DEBUG_FORCE_WEAK_PER_CPU + select CODE_TAGGING + select PAGE_EXTENSION + select SLAB_OBJ_EXT + help + Track allocation source code and record total allocation size + initiated at that code location. The mechanism can be used to track + memory leaks with a low performance and memory impact. + +config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + bool "Enable memory allocation profiling by default" + default y + depends on MEM_ALLOC_PROFILING + +config MEM_ALLOC_PROFILING_DEBUG + bool "Memory allocation profiler debugging" + default n + depends on MEM_ALLOC_PROFILING + select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + help + Adds warnings with helpful error messages for memory allocation + profiling. diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061..ab37ef428d98 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -50,7 +50,7 @@ endif obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ maccess.o page-writeback.o folio-compat.o \ - readahead.o swap.o truncate.o vmscan.o shrinker.o \ + readahead.o folio.o truncate.o vmscan.o shrinker.o \ shmem.o util.o mmzone.o vmstat.o backing-dev.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ @@ -141,9 +141,9 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o obj-$(CONFIG_PTDUMP) += ptdump.o obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o -obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o diff --git a/lib/alloc_tag.c b/mm/alloc_tag.c index e5b218176c5a..52aece27b00e 100644 --- a/lib/alloc_tag.c +++ b/mm/alloc_tag.c @@ -5,6 +5,8 @@ #include <linux/gfp.h> #include <linux/kallsyms.h> #include <linux/module.h> +#include <linux/mutex.h> +#include <linux/compat.h> #include <linux/page_ext.h> #include <linux/pgalloc_tag.h> #include <linux/proc_fs.h> @@ -14,6 +16,10 @@ #include <linux/string_choices.h> #include <linux/vmalloc.h> #include <linux/kmemleak.h> +#include <uapi/linux/alloc_tag.h> + +#include "internal.h" +#include "page_alloc.h" #define ALLOCINFO_FILE_NAME "allocinfo" #define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) @@ -56,6 +62,11 @@ struct allocinfo_private { struct codetag_iterator iter; struct codetag_iterator reported_iter; bool print_header; + struct allocinfo_filter filter; + /* ioctl uses a separate iterator not to interfere with reads */ + struct codetag_iterator ioctl_iter; + bool positioned; /* seq_open_private() sets to 0 */ + struct mutex ioctl_lock; }; static void *allocinfo_start(struct seq_file *m, loff_t *pos) @@ -139,6 +150,340 @@ static const struct seq_operations allocinfo_seq_op = { .show = allocinfo_show, }; +/* + * Initializes seq_file operations and allocates private state when opening + * the /proc/allocinfo procfs entry. + */ +static int allocinfo_open(struct inode *inode, struct file *file) +{ + int ret; + + ret = seq_open_private(file, &allocinfo_seq_op, + sizeof(struct allocinfo_private)); + if (!ret) { + struct seq_file *m = file->private_data; + struct allocinfo_private *priv = m->private; + + mutex_init(&priv->ioctl_lock); + } + return ret; +} + +/* + * Cleans up the seq_file state and frees up the private state allocated in + * allocinfo_open() when closing the /proc/allocinfo file descriptor. + */ +static int allocinfo_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + struct allocinfo_private *priv = m->private; + + mutex_destroy(&priv->ioctl_lock); + return seq_release_private(inode, file); +} + +/* + * Returns a pointer to the suffix of a string so that its length fits within + * ALLOCINFO_STR_SIZE, preserving the trailing characters. + * Function, file and module names often have the same prefixes, therefore + * when filtering by these criteria, we compare the last 64 characters to + * minimize the chances of name collisions + */ +static const char *allocinfo_str(const char *str) +{ + size_t len = strlen(str); + + /* Keep an extra space for the trailing NULL. */ + if (len >= ALLOCINFO_STR_SIZE) + str += (len - ALLOCINFO_STR_SIZE) + 1; + return str; +} + +/* Copy a string and trim from the beginning if it's too long */ +static void allocinfo_copy_str(char *dest, const char *src) +{ + strscpy_pad(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE); +} + +/* Compare two strings and only consider the trimmed suffix if s1 is too long */ +static int allocinfo_cmp_str(const char *str, const char *template) +{ + return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE); +} + +/* Fetch the per-CPU counters */ +static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct) +{ + return alloc_tag_read(ct_to_alloc_tag(ct)); +} + +/* + * Populates the UAPI allocinfo_tag_data structure with active runtime + * profiling counters extracted from the given kernel codetag. + */ +static void allocinfo_to_params(struct codetag *ct, + struct allocinfo_tag_data *data, + struct alloc_tag_counters *counters) +{ + if (ct->modname) + allocinfo_copy_str(data->tag.modname, ct->modname); + else + data->tag.modname[0] = '\0'; + allocinfo_copy_str(data->tag.function, ct->function); + allocinfo_copy_str(data->tag.filename, ct->filename); + data->tag.lineno = ct->lineno; + data->counter.bytes = counters->bytes; + data->counter.calls = counters->calls; + data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct)); +} + +/* + * Retrieves the unique content ID representing the current allocation tag module + * layout, allowing userspace to detect if modules were loaded / unloaded. + */ +static int allocinfo_ioctl_get_content_id(struct seq_file *m, void __user *arg) +{ + struct allocinfo_content_id params; + + codetag_lock_module_list(alloc_tag_cttype); + params.id = codetag_get_content_id(alloc_tag_cttype); + codetag_unlock_module_list(alloc_tag_cttype); + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +/* + * Verifies whether a given codetag satisfies the active filtering criteria by + * matching its characteristics against the specified filter. + */ +static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter, + struct alloc_tag_counters *counters, + bool *fetched_counters) +{ + bool inaccurate; + + if (!filter || !filter->mask) + return true; + + if (filter->mask & ALLOCINFO_FILTER_MASK_MODNAME) { + /* user wants to filter by modname but ct->modname is NULL */ + if (!ct->modname) { + /* validate if user was attempting to filter for built-in allocations */ + if (filter->fields.modname[0] != '\0') + return false; + } else if (allocinfo_cmp_str(ct->modname, filter->fields.modname)) + return false; + } + + if ((filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) && + ct->function && allocinfo_cmp_str(ct->function, filter->fields.function)) + return false; + + if ((filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) && + ct->filename && allocinfo_cmp_str(ct->filename, filter->fields.filename)) + return false; + + if ((filter->mask & ALLOCINFO_FILTER_MASK_LINENO) && + ct->lineno != filter->fields.lineno) + return false; + + if (filter->mask & ALLOCINFO_FILTER_MASK_INACCURATE) { + inaccurate = !!(ct->flags & CODETAG_FLAG_INACCURATE); + if (inaccurate != !!(filter->fields.inaccurate)) + return false; + } + + if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE)) { + if (!*fetched_counters) { + *counters = allocinfo_prefetch_counters(ct); + *fetched_counters = true; + } + if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) && + counters->bytes < filter->min_size) + return false; + if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) && + counters->bytes > filter->max_size) + return false; + } + + return true; +} + +/* + * Seeks the ioctl iterator to the specified 0-indexed tag position, reads its + * profiling data and returns it to userspace. + */ +static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg) +{ + struct allocinfo_private *priv; + struct codetag *ct; + struct allocinfo_get_at params = {0}; + __u64 skip_count; + struct alloc_tag_counters counters; + bool fetched_counters; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS) + return -EINVAL; + + if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) && + (params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) && + params.filter.min_size > params.filter.max_size) + return -EINVAL; + + priv = m->private; + + mutex_lock(&priv->ioctl_lock); + codetag_lock_module_list(alloc_tag_cttype); + + if (params.pos >= codetag_get_count(alloc_tag_cttype)) { + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + return -ENOENT; + } + + skip_count = params.pos; + + if (params.filter.mask) + priv->filter = params.filter; + else + priv->filter.mask = 0; + + /* Find the codetag */ + priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype); + ct = codetag_next_ct(&priv->ioctl_iter); + + while (ct) { + fetched_counters = false; + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) { + if (skip_count == 0) + break; + skip_count--; + } + ct = codetag_next_ct(&priv->ioctl_iter); + } + + if (ct) { + if (!fetched_counters) + counters = allocinfo_prefetch_counters(ct); + allocinfo_to_params(ct, ¶ms.data, &counters); + priv->positioned = true; + } + + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + + if (!ct) + return -ENOENT; + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +/* + * Advances the ioctl iterator to the next allocation tag in the sequence and + * returns its profiling data to userspace. + */ +static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg) +{ + struct allocinfo_private *priv; + struct codetag *ct; + struct allocinfo_tag_data params; + int ret = 0; + struct alloc_tag_counters counters; + bool fetched_counters; + + memset(¶ms, 0, sizeof(params)); + priv = m->private; + + mutex_lock(&priv->ioctl_lock); + codetag_lock_module_list(alloc_tag_cttype); + + if (!priv->positioned) { + priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype); + priv->positioned = true; + } + + ct = codetag_next_ct(&priv->ioctl_iter); + while (ct) { + fetched_counters = false; + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) + break; + ct = codetag_next_ct(&priv->ioctl_iter); + } + + if (ct) { + if (!fetched_counters) + counters = allocinfo_prefetch_counters(ct); + allocinfo_to_params(ct, ¶ms, &counters); + } + if (!ct) { + priv->positioned = false; + ret = -ENOENT; + } + codetag_unlock_module_list(alloc_tag_cttype); + mutex_unlock(&priv->ioctl_lock); + + if (ret == 0) { + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + } + return ret; +} + +/* + * Entry point ioctl function for /proc/allocinfo routing requests to fetch the + * layout content ID, seek to a specific tag, or read sequential tags. + */ +static long allocinfo_ioctl(struct file *file, unsigned int cmd, + unsigned long __arg) +{ + void __user *arg = (void __user *)__arg; + int ret; + + switch (cmd) { + case ALLOCINFO_IOC_CONTENT_ID: + ret = allocinfo_ioctl_get_content_id(file->private_data, arg); + break; + case ALLOCINFO_IOC_GET_AT: + ret = allocinfo_ioctl_get_at(file->private_data, arg); + break; + case ALLOCINFO_IOC_GET_NEXT: + ret = allocinfo_ioctl_get_next(file->private_data, arg); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + return ret; +} + +#ifdef CONFIG_COMPAT +static long allocinfo_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return allocinfo_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static const struct proc_ops allocinfo_proc_ops = { + .proc_open = allocinfo_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = allocinfo_release, + .proc_ioctl = allocinfo_ioctl, +#ifdef CONFIG_COMPAT + .proc_compat_ioctl = allocinfo_compat_ioctl, +#endif +}; + size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) { struct codetag_iterator iter; @@ -792,19 +1137,6 @@ struct pfn_pool { #define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ sizeof(unsigned long)) - -/* - * Skip early PFN recording for a page allocation. Reuses the - * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid - * recursion when allocating pages for the early PFN tracking list - * itself. - * - * Codetags of the pages allocated with __GFP_NO_CODETAG should be - * cleared (via clear_page_tag_ref()) before freeing the pages to prevent - * alloc_tag_sub_check() from triggering a warning. - */ -#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT - static struct pfn_pool *current_pfn_pool __initdata; static void __init __alloc_tag_add_early_pfn(unsigned long pfn) @@ -815,7 +1147,8 @@ static void __init __alloc_tag_add_early_pfn(unsigned long pfn) do { pool = READ_ONCE(current_pfn_pool); if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { - struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); + struct page *new_page = __alloc_pages(__GFP_HIGH, 0, numa_mem_id(), + NULL, ALLOC_NO_CODETAG); struct pfn_pool *new; if (!new_page) { @@ -846,7 +1179,7 @@ typedef void alloc_tag_add_func(unsigned long pfn); static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = RCU_INITIALIZER(__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) { alloc_tag_add_func *alloc_tag_add; @@ -854,7 +1187,7 @@ void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) return; /* Skip allocations for the tracking list itself to avoid recursion. */ - if (gfp_flags & __GFP_NO_CODETAG) + if (alloc_flags & ALLOC_NO_CODETAG) return; rcu_read_lock(); @@ -1002,8 +1335,7 @@ static int __init alloc_tag_init(void) return 0; } - if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, - sizeof(struct allocinfo_private), NULL)) { + if (!proc_create(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_proc_ops)) { pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); shutdown_mem_profiling(false); return -ENOMEM; diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c deleted file mode 100644 index 0fa78db7fbc0..000000000000 --- a/mm/bootmem_info.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Bootmem core functions. - * - * Copyright (c) 2020, Bytedance. - * - * Author: Muchun Song <songmuchun@bytedance.com> - * - */ -#include <linux/mm.h> -#include <linux/compiler.h> -#include <linux/memblock.h> -#include <linux/bootmem_info.h> -#include <linux/memory_hotplug.h> -#include <linux/kmemleak.h> - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ - BUG_ON(type > 0xf); - BUG_ON(info > (ULONG_MAX >> 4)); - set_page_private(page, info << 4 | type); - page_ref_inc(page); -} - -void put_page_bootmem(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || - type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); - - if (page_ref_dec_return(page) == 1) { - set_page_private(page, 0); - free_reserved_page(page); - } -} - -static void __init register_page_bootmem_info_section(unsigned long start_pfn) -{ - unsigned long section_nr; - struct mem_section *ms; - - start_pfn = SECTION_ALIGN_DOWN(start_pfn); - section_nr = pfn_to_section_nr(start_pfn); - ms = __nr_to_section(section_nr); - - if (!preinited_vmemmap_section(ms)) - register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn), - PAGES_PER_SECTION); -} - -void __init register_page_bootmem_info_node(struct pglist_data *pgdat) -{ - unsigned long pfn, end_pfn; - int node = pgdat->node_id; - - pfn = pgdat->node_start_pfn; - end_pfn = pgdat_end_pfn(pgdat); - - /* register section info */ - for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { - /* - * Some platforms can assign the same pfn to multiple nodes - on - * node0 as well as nodeN. To avoid registering a pfn against - * multiple nodes we check that this pfn does not already - * reside in some other nodes. - */ - if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) - register_page_bootmem_info_section(pfn); - } -} @@ -33,6 +33,7 @@ #include "internal.h" #include "cma.h" +#include "mm_init.h" struct cma cma_areas[MAX_CMA_AREAS]; unsigned int cma_area_count; @@ -126,7 +127,6 @@ bool cma_validate_zones(struct cma *cma) * to be in the same zone. Simplify by forcing the entire * CMA resv range to be in the same zone. */ - WARN_ON_ONCE(!pfn_valid(base_pfn)); if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) { set_bit(CMA_ZONES_INVALID, &cma->flags); return false; @@ -165,6 +165,8 @@ static void __init cma_activate_area(struct cma *cma) bitmap_set(cmr->bitmap, 0, bitmap_count); } + WARN_ON_ONCE(!pfn_valid(cmr->base_pfn)); + for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count; pfn += pageblock_nr_pages) init_cma_reserved_pageblock(pfn_to_page(pfn)); diff --git a/mm/compaction.c b/mm/compaction.c index f08765ade014..9f81055a358e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -24,6 +24,7 @@ #include <linux/page_owner.h> #include <linux/psi.h> #include <linux/cpuset.h> +#include "page_alloc.h" #include "internal.h" #ifdef CONFIG_COMPACTION @@ -82,7 +83,7 @@ static inline bool is_via_compact_memory(int order) { return false; } static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags) { - post_alloc_hook(page, order, __GFP_MOVABLE); + post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT); set_page_refcounted(page); return page; } @@ -644,7 +645,6 @@ static unsigned long isolate_freepages_block(struct compact_control *cc, isolated = __isolate_free_page(page, order); if (!isolated) break; - set_page_private(page, order); nr_scanned += isolated - 1; total_isolated += isolated; @@ -1617,7 +1617,6 @@ static void fast_isolate_freepages(struct compact_control *cc) /* Isolate the page if available */ if (page) { if (__isolate_free_page(page, order)) { - set_page_private(page, order); nr_isolated = 1 << order; nr_scanned += nr_isolated - 1; total_isolated += nr_isolated; @@ -1846,11 +1845,10 @@ again: size >>= 1; list_add(&freepage[size].lru, &cc->freepages[start_order]); - set_page_private(&freepage[size], start_order); } dst = (struct folio *)freepage; - post_alloc_hook(&dst->page, order, __GFP_MOVABLE); + post_alloc_hook(&dst->page, order, __GFP_MOVABLE, ALLOC_DEFAULT); set_page_refcounted(&dst->page); if (order) prep_compound_page(&dst->page, order); diff --git a/mm/damon/core.c b/mm/damon/core.c index cff932b3317d..e0122376f43d 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -166,6 +166,7 @@ struct damon_probe *damon_new_probe(void) p = kmalloc_obj(*p); if (!p) return NULL; + p->weight = 0; INIT_LIST_HEAD(&p->filters); INIT_LIST_HEAD(&p->list); return p; @@ -208,6 +209,106 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx) return NULL; } +static bool damon_has_probe_weights(struct damon_ctx *c) +{ + struct damon_probe *p; + + damon_for_each_probe(p, c) { + if (p->weight) + return true; + } + return false; +} + +/* + * damon_mvsum() - Returns pseudo moving sum value for a time window. + * @current_nr: The value of the current aggregation window. + * @last_nr: The value of the last aggregation window. + * @left_window_bp: Left time of the current aggregation window. + * + * This function calculates a pseudo moving sum value of a counter that is + * aggregated for each time window. @current_nr is the value of the counter + * that aggregated so far (maybe not yet complete), from the beginning of the + * current aggregation time window. @last_nr is the value of the counter that + * has completely aggregated in the last aggregation time window. + * @left_window_bp represents how much time is left for the current aggregation + * time window in bp (1/10,000). For example, the aggregation time window is + * for every 10 seconds and 7 seconds has passed since the beginning of the + * current window, this parameter will be 3000 ((10 - 7) / 10 * 10000). + * + * The logic assumes the aggregation in the last phase was made in a single + * speed. Based on the assumption, the value from the last window that needs + * to be added to the current value is calculated as a portion of the last + * value based on the remaining time window. + */ +static unsigned long damon_mvsum(unsigned long current_nr, + unsigned long last_nr, unsigned long left_window_bp) +{ + return current_nr + mult_frac(last_nr, left_window_bp, 10000); +} + +/** + * damon_nr_accesses_mvsum() - Returns moving sum access frequency score. + * @r: Region to get the access frequency of. + * @ctx: DAMON context of @r. + * + * This function returns for how many sampling iterations in the last + * aggregation interval (&damon_attrs->aggr_interval) the region was found to + * be accessed. Hence the value can be interpreted as the relative access + * frequency score of the region (@r). The value is calculated as a pseudo + * moving sum, and hence it is not an exact value but just a best-effort + * reasonable estimation. + * + * Return: the pseudo moving sum access frequency score. + */ +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + if (left_window_bp == 10000) + return r->last_nr_accesses; + + return damon_mvsum(r->nr_accesses, r->last_nr_accesses, + left_window_bp); +} + +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + if (left_window_bp == 10000) + return r->last_probe_hits[probe_idx]; + + return damon_mvsum(r->probe_hits[probe_idx], + r->last_probe_hits[probe_idx], left_window_bp); +} + #ifdef CONFIG_DAMON_DEBUG_SANITY static void damon_verify_new_region(unsigned long start, unsigned long end) { @@ -237,9 +338,10 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) region->ar.start = start; region->ar.end = end; region->nr_accesses = 0; - region->nr_accesses_bp = 0; - for (i = 0; i < DAMON_MAX_PROBES; i++) + for (i = 0; i < DAMON_MAX_PROBES; i++) { region->probe_hits[i] = 0; + region->last_probe_hits[i] = 0; + } INIT_LIST_HEAD(®ion->list); region->age = 0; @@ -302,6 +404,30 @@ static bool damon_is_last_region(struct damon_region *r, return list_is_last(&r->list, &t->regions_list); } +/** + * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region. + * @r: region to get the weighted sum of. + * @last: if the request is for last-window aggregated probe hits. + * @ctx: context of &r. + * + * Return: the weighted sum of probe hits of the region. + */ +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx) +{ + struct damon_probe *probe; + unsigned int sum = 0; + int i = 0; + + damon_for_each_probe(probe, ctx) { + if (last) + sum += r->last_probe_hits[i++] * probe->weight; + else + sum += r->probe_hits[i++] * probe->weight; + } + return sum; +} + /* * Check whether a region is intersecting an address range * @@ -579,6 +705,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->ops_filters); scheme->stat = (struct damos_stat){}; scheme->max_nr_snapshots = 0; + scheme->last_applied = NULL; INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); @@ -789,55 +916,66 @@ static unsigned int damon_age_for_new_attrs(unsigned int age, return age * old_attrs->aggr_interval / new_attrs->aggr_interval; } -/* convert access ratio in bp (per 10,000) to nr_accesses */ -static unsigned int damon_accesses_bp_to_nr_accesses( - unsigned int accesses_bp, struct damon_attrs *attrs) +/* convert sample ratio in bp (per 10,000) to count */ +static unsigned int damon_sample_bp_to_count( + unsigned int bp, struct damon_attrs *attrs) { - return accesses_bp * damon_max_nr_accesses(attrs) / 10000; + return bp * damon_nr_samples_per_aggr(attrs) / 10000; } -/* - * Convert nr_accesses to access ratio in bp (per 10,000). - * - * Callers should ensure attrs.aggr_interval is not zero, like - * damon_update_monitoring_results() does . Otherwise, divide-by-zero would - * happen. - */ -static unsigned int damon_nr_accesses_to_accesses_bp( - unsigned int nr_accesses, struct damon_attrs *attrs) +/* convert sample count to ratio in bp (per 10,000) */ +static unsigned int damon_sample_count_to_bp( + unsigned int count, struct damon_attrs *attrs) { - return mult_frac(nr_accesses, 10000, damon_max_nr_accesses(attrs)); + return mult_frac(count, 10000, damon_nr_samples_per_aggr(attrs)); } -static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, +static unsigned int damon_nr_samples_for_new_attrs(unsigned int nr, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) { - return damon_accesses_bp_to_nr_accesses( - damon_nr_accesses_to_accesses_bp( - nr_accesses, old_attrs), - new_attrs); + return damon_sample_bp_to_count( + damon_sample_count_to_bp(nr, old_attrs), new_attrs); +} + +static void damon_update_probe_hits(struct damon_region *r, + struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, + bool aggregating, struct damon_ctx *ctx) +{ + struct damon_probe *p; + int i = 0; + + damon_for_each_probe(p, ctx) { + r->last_probe_hits[i] = damon_nr_samples_for_new_attrs( + r->last_probe_hits[i], old_attrs, new_attrs); + if (!aggregating) + r->probe_hits[i] = damon_nr_samples_for_new_attrs( + r->probe_hits[i], old_attrs, + new_attrs); + else + r->probe_hits[i] = 0; + i++; + } } static void damon_update_monitoring_result(struct damon_region *r, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, - bool aggregating) + bool aggregating, struct damon_ctx *ctx) { - if (!aggregating) { - r->nr_accesses = damon_nr_accesses_for_new_attrs( + damon_update_probe_hits(r, old_attrs, new_attrs, aggregating, ctx); + + r->last_nr_accesses = damon_nr_samples_for_new_attrs( + r->last_nr_accesses, old_attrs, new_attrs); + if (!aggregating) + r->nr_accesses = damon_nr_samples_for_new_attrs( r->nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->nr_accesses * 10000; - } else { + else /* * if this is called in the middle of the aggregation, reset * the aggregations we made so far for this aggregation * interval. In other words, make the status like * kdamond_reset_aggregated() is called. */ - r->last_nr_accesses = damon_nr_accesses_for_new_attrs( - r->last_nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->last_nr_accesses * 10000; r->nr_accesses = 0; - } r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs); } @@ -864,8 +1002,8 @@ static void damon_update_monitoring_results(struct damon_ctx *ctx, damon_for_each_target(t, ctx) damon_for_each_region(r, t) - damon_update_monitoring_result( - r, old_attrs, new_attrs, aggregating); + damon_update_monitoring_result(r, old_attrs, new_attrs, + aggregating, ctx); } /* @@ -1201,6 +1339,34 @@ static void damos_set_filters_default_reject(struct damos *s) damos_filters_default_reject(&s->ops_filters); } +static bool damon_valid_probe_params(struct damon_ctx *ctx) +{ + unsigned long sample_interval; + unsigned char max_probe_hits; + struct damon_probe *probe; + unsigned int wsum, wsum_to_add; + + if (!damon_has_probe_weights(ctx)) + return true; + + sample_interval = ctx->attrs.sample_interval ? : 1; + if (ctx->attrs.aggr_interval / sample_interval > U8_MAX) + return false; + + /* invalid if probe hits weighted sum can overflow */ + max_probe_hits = damon_nr_samples_per_aggr(&ctx->attrs); + wsum = 0; + damon_for_each_probe(probe, ctx) { + if (probe->weight > UINT_MAX / max_probe_hits) + return false; + wsum_to_add = probe->weight * max_probe_hits; + if (UINT_MAX - wsum < wsum_to_add) + return false; + wsum += wsum_to_add; + } + return true; +} + /* * damos_commit_dests() - Copy migration destinations from @src to @dst. * @dst: Destination structure to update. @@ -1360,23 +1526,35 @@ static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx) static int damon_commit_target_regions(struct damon_target *dst, struct damon_target *src, unsigned long src_min_region_sz) { - struct damon_region *src_region; + struct damon_region *src_region, *prev = NULL; struct damon_addr_range *ranges; int i = 0, err; - damon_for_each_region(src_region, src) - i++; + damon_for_each_region(src_region, src) { + if (!prev || prev->ar.end != src_region->ar.start) + i++; + prev = src_region; + } if (!i) return 0; - ranges = kmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); + ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); if (!ranges) return -ENOMEM; + prev = NULL; i = 0; - damon_for_each_region(src_region, src) - ranges[i++] = src_region->ar; + damon_for_each_region(src_region, src) { + if (!prev) { + ranges[i].start = src_region->ar.start; + } else if (prev->ar.end != src_region->ar.start) { + ranges[i++].end = prev->ar.end; + ranges[i].start = src_region->ar.start; + } + prev = src_region; + } + ranges[i++].end = damon_last_region(src)->ar.end; err = damon_set_regions(dst, ranges, i, src_min_region_sz); - kfree(ranges); + kvfree(ranges); return err; } @@ -1545,6 +1723,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) damon_for_each_probe_safe(dst_probe, next, dst) { src_probe = damon_nth_probe(i++, src); if (src_probe) { + dst_probe->weight = src_probe->weight; err = damon_commit_filters(dst_probe, src_probe); if (err) return err; @@ -1561,6 +1740,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) if (!new_probe) return -ENOMEM; damon_add_probe(dst, new_probe); + new_probe->weight = src_probe->weight; err = damon_commit_filters(new_probe, src_probe); if (err) return err; @@ -1568,20 +1748,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src) return 0; } -/** - * damon_commit_ctx() - Commit parameters of a DAMON context to another. - * @dst: The commit destination DAMON context. - * @src: The commit source DAMON context. - * - * This function copies user-specified parameters from @src to @dst and update - * the internal status and results accordingly. Users should use this function - * for context-level parameters update of running context, instead of manual - * in-place updates. - * - * This function should be called from parameters-update safe context, like - * damon_call(). - */ -int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) +static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) { int err; struct damos *scheme; @@ -1604,6 +1771,9 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) } } + if (!damon_valid_probe_params(src)) + return -EINVAL; + err = damon_commit_schemes(dst, src); if (err) return err; @@ -1636,6 +1806,52 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) return 0; } +static struct damon_ctx *damon_new_test_ctx(struct damon_ctx *dst) +{ + struct damon_ctx *test_ctx; + int err; + + test_ctx = damon_new_ctx(); + if (!test_ctx) + return NULL; + err = __damon_commit_ctx(test_ctx, dst); + if (err) { + damon_destroy_ctx(test_ctx); + return NULL; + } + return test_ctx; +} + +/** + * damon_commit_ctx() - Commit parameters of a DAMON context to another. + * @dst: The commit destination DAMON context. + * @src: The commit source DAMON context. + * + * This function copies user-specified parameters from @src to @dst and update + * the internal status and results accordingly. Users should use this function + * for context-level parameters update of running context, instead of manual + * in-place updates. + * + * This function should be called from parameters-update safe context, like + * damon_call(). + */ +int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) +{ + struct damon_ctx *test_ctx; + int err; + + test_ctx = damon_new_test_ctx(dst); + if (!test_ctx) + return -ENOMEM; + err = __damon_commit_ctx(test_ctx, src); + if (err) + goto out; + err = __damon_commit_ctx(dst, src); +out: + damon_destroy_ctx(test_ctx); + return err; +} + /** * damon_nr_running_ctxs() - Return number of currently running contexts. */ @@ -1670,7 +1886,7 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) return sz; } -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r); /* @@ -1696,11 +1912,13 @@ static unsigned long damon_apply_min_nr_regions(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) { while (damon_sz_region(r) > max_region_sz) { - damon_split_region_at(t, r, max_region_sz); + if (damon_split_region_at(t, r, max_region_sz)) + goto out; r = damon_next_region(r); } } } +out: return max_region_sz; } @@ -1736,6 +1954,8 @@ static int __damon_start(struct damon_ctx *ctx) return err; } +static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src); + /** * damon_start() - Starts the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to start monitoring @@ -1757,8 +1977,16 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) int err = 0; for (i = 0; i < nr_ctxs; i++) { - if (!is_power_of_2(ctxs[i]->min_region_sz)) - return -EINVAL; + struct damon_ctx *test_ctx; + + test_ctx = damon_new_ctx(); + if (!test_ctx) + return -ENOMEM; + + err = __damon_commit_ctx(test_ctx, ctxs[i]); + damon_destroy_ctx(test_ctx); + if (err) + return err; } mutex_lock(&damon_lock); @@ -1778,16 +2006,17 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) running_exclusive_ctxs = true; mutex_unlock(&damon_lock); + if (i != nr_ctxs) + damon_stop(ctxs, i); + return err; } /* * __damon_stop() - Stops monitoring of a given context. * @ctx: monitoring context - * - * Return: 0 on success, negative error code otherwise. */ -static int __damon_stop(struct damon_ctx *ctx) +static void __damon_stop(struct damon_ctx *ctx) { struct task_struct *tsk; @@ -1797,31 +2026,23 @@ static int __damon_stop(struct damon_ctx *ctx) get_task_struct(tsk); mutex_unlock(&ctx->kdamond_lock); kthread_stop_put(tsk); - return 0; + return; } mutex_unlock(&ctx->kdamond_lock); - - return -EPERM; } /** * damon_stop() - Stops the monitorings for a given group of contexts. * @ctxs: an array of the pointers for contexts to stop monitoring * @nr_ctxs: size of @ctxs - * - * Return: 0 on success, negative error code otherwise. */ -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs) { - int i, err = 0; + int i; - for (i = 0; i < nr_ctxs; i++) { + for (i = 0; i < nr_ctxs; i++) /* nr_running_ctxs is decremented in kdamond_fn */ - err = __damon_stop(ctxs[i]); - if (err) - break; - } - return err; + __damon_stop(ctxs[i]); } /** @@ -1878,6 +2099,8 @@ int damon_kdamond_pid(struct damon_ctx *ctx) * @ctx has succeeded. Otherwise, this function could fall into an indefinite * wait. * + * When this function is failed, the @ctx is guaranteed to be stopped. + * * Return: 0 on success, negative error code otherwise. */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) @@ -1890,7 +2113,7 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) mutex_lock(&ctx->call_controls_lock); if (ctx->call_controls_obsolete) { mutex_unlock(&ctx->call_controls_lock); - return -ECANCELED; + goto canceled; } list_add_tail(&control->list, &ctx->call_controls); mutex_unlock(&ctx->call_controls_lock); @@ -1898,8 +2121,14 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) return 0; wait_for_completion(&control->completion); if (control->canceled) - return -ECANCELED; + goto canceled; return 0; + +canceled: + while (damon_is_running(ctx)) + schedule_timeout_idle(msecs_to_jiffies(100)); + return -ECANCELED; + } /** @@ -1949,36 +2178,6 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) } /* - * Warn and fix corrupted ->nr_accesses[_bp] for investigations and preventing - * the problem being propagated. - */ -static void damon_warn_fix_nr_accesses_corruption(struct damon_region *r) -{ - if (r->nr_accesses_bp == r->nr_accesses * 10000) - return; - WARN_ONCE(true, "invalid nr_accesses_bp at reset: %u %u\n", - r->nr_accesses_bp, r->nr_accesses); - r->nr_accesses_bp = r->nr_accesses * 10000; -} - -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ - WARN_ONCE(r->nr_accesses_bp != r->last_nr_accesses * 10000, - "nr_accesses_bp %u last_nr_accesses %u sis %lu %lu\n", - r->nr_accesses_bp, r->last_nr_accesses, - c->passed_sample_intervals, c->next_aggregation_sis); -} -#else -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ -} -#endif - - -/* * Reset the aggregated monitoring results ('nr_accesses' of each region). */ static void kdamond_reset_aggregated(struct damon_ctx *c) @@ -2002,12 +2201,12 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) trace_damon_aggregated(ti, r, damon_nr_regions(t)); trace_damon_region_aggregated(ti, r, damon_nr_regions(t), nr_probes); - damon_warn_fix_nr_accesses_corruption(r); r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; - for (i = 0; i < DAMON_MAX_PROBES; i++) + for (i = 0; i < DAMON_MAX_PROBES; i++) { + r->last_probe_hits[i] = r->probe_hits[i]; r->probe_hits[i] = 0; - damon_verify_reset_aggregated(r, c); + } } ti++; } @@ -2052,7 +2251,7 @@ static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c) return adaptation_bp; } -static void kdamond_tune_intervals(struct damon_ctx *c) +static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c) { unsigned long adaptation_bp; struct damon_attrs new_attrs; @@ -2074,10 +2273,11 @@ static void kdamond_tune_intervals(struct damon_ctx *c) damon_set_attrs(c, &new_attrs); } -static bool __damos_valid_target(struct damon_region *r, struct damos *s) +static bool __damos_valid_target(struct damon_region *r, struct damos *s, + struct damon_ctx *c) { unsigned long sz; - unsigned int nr_accesses = r->nr_accesses_bp / 10000; + unsigned int nr_accesses = damon_nr_accesses_mvsum(r, c); sz = damon_sz_region(r); return s->pattern.min_sz_region <= sz && @@ -2103,7 +2303,7 @@ static bool damos_quota_is_set(struct damos_quota *quota) static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r, struct damos *s) { - bool ret = __damos_valid_target(r, s); + bool ret = __damos_valid_target(r, s, c); if (!ret || !damos_quota_is_set(&s->quota) || !c->ops.get_scheme_score) return ret; @@ -2387,7 +2587,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, struct damos *siter; /* schemes iterator */ unsigned int sidx = 0; struct damon_target *titer; /* targets iterator */ - unsigned int tidx = 0; + unsigned int tidx = 0, nr_accesses = 0; bool do_trace = false; /* get indices for trace_damos_before_apply() */ @@ -2402,6 +2602,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, break; tidx++; } + nr_accesses = damon_nr_accesses_mvsum(r, c); do_trace = true; } @@ -2417,7 +2618,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (damos_core_filter_out(c, t, r, s)) return; ktime_get_coarse_ts64(&begin); - trace_damos_before_apply(cidx, sidx, tidx, r, + trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses, damon_nr_regions(t), do_trace); sz_applied = c->ops.apply_scheme(c, t, r, s, &sz_ops_filter_passed); @@ -2689,7 +2890,7 @@ static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c, damon_for_each_region(r, t) { phys_addr_t addr, end_addr; - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; /* Convert from core address units to physical bytes */ @@ -2978,7 +3179,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) (DAMOS_MAX_SCORE + 1)); damon_for_each_target(t, c) { damon_for_each_region(r, t) { - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; if (damos_core_filter_out(c, t, r, s)) continue; @@ -3082,7 +3283,6 @@ static void damon_merge_two_regions(struct damon_target *t, l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / (sz_l + sz_r); - l->nr_accesses_bp = l->nr_accesses * 10000; l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); l->ar.end = r->ar.end; /* todo: do this for only installed probes */ @@ -3093,19 +3293,15 @@ static void damon_merge_two_regions(struct damon_target *t, damon_destroy_region(r, t); } -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_merge_regions_of(struct damon_region *r) -{ - WARN_ONCE(r->nr_accesses != r->nr_accesses_bp / 10000, - "nr_accesses (%u) != nr_accesses_bp (%u)\n", - r->nr_accesses, r->nr_accesses_bp); -} -#else -static void damon_verify_merge_regions_of(struct damon_region *r) +static unsigned int damon_merge_score(struct damon_region *r, bool last, + struct damon_ctx *ctx, bool use_probe_hits) { + if (use_probe_hits) + return damon_probe_hits_wsum(r, last, ctx); + if (last) + return r->last_nr_accesses; + return r->nr_accesses; } -#endif - /* * Merge adjacent regions having similar access frequencies @@ -3115,25 +3311,40 @@ static void damon_verify_merge_regions_of(struct damon_region *r) * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit) + unsigned long sz_limit, struct damon_ctx *ctx, bool count_age) { struct damon_region *r, *prev = NULL, *next; + bool use_probe_hits = damon_has_probe_weights(ctx); damon_for_each_region_safe(r, next, t) { - damon_verify_merge_regions_of(r); - if (abs(r->nr_accesses - r->last_nr_accesses) > thres) - r->age = 0; - else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) - r->age = 0; - else - r->age++; + unsigned int score, last_score, diff; - if (prev && prev->ar.end == r->ar.start && - abs(prev->nr_accesses - r->nr_accesses) <= thres && - damon_sz_region(prev) + damon_sz_region(r) <= sz_limit) - damon_merge_two_regions(t, prev, r); - else - prev = r; + score = damon_merge_score(r, false, ctx, use_probe_hits); + last_score = damon_merge_score(r, true, ctx, use_probe_hits); + + if (count_age) { + if (abs_diff(score, last_score) > thres) + r->age = 0; + else if ((score == 0) != (last_score == 0)) + r->age = 0; + else + r->age++; + } + + if (!prev) + goto set_prev_continue; + if (prev->ar.end != r->ar.start) + goto set_prev_continue; + diff = abs_diff(score, damon_merge_score(prev, false, ctx, + use_probe_hits)); + if (diff > thres) + goto set_prev_continue; + if (damon_sz_region(prev) + damon_sz_region(r) > sz_limit) + goto set_prev_continue; + damon_merge_two_regions(t, prev, r); + continue; +set_prev_continue: + prev = r; } } @@ -3160,18 +3371,26 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, struct damon_target *t; unsigned int nr_regions; unsigned int max_thres; + bool count_age = true; max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); - do { + while (true) { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(t, threshold, sz_limit, c, + count_age); nr_regions += damon_nr_regions(t); } - threshold = max(1, threshold * 2); - } while (nr_regions > c->attrs.max_nr_regions && - threshold / 2 < max_thres); + count_age = false; + if (nr_regions <= c->attrs.max_nr_regions || + max_thres <= threshold) + break; + if (threshold < max_thres / 2) + threshold = max(1, threshold * 2); + else + threshold = max_thres; + } } #ifdef CONFIG_DAMON_DEBUG_SANITY @@ -3194,8 +3413,10 @@ static void damon_verify_split_region_at(struct damon_region *r, * * r the region to be split * sz_r size of the first sub-region that will be made + * + * Return: 0 on success, negative error code otherwise. */ -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r) { struct damon_region *new; @@ -3203,18 +3424,20 @@ static void damon_split_region_at(struct damon_target *t, damon_verify_split_region_at(r, sz_r); new = damon_new_region(r->ar.start + sz_r, r->ar.end); if (!new) - return; + return -ENOMEM; r->ar.end = new->ar.start; new->age = r->age; new->last_nr_accesses = r->last_nr_accesses; - new->nr_accesses_bp = r->nr_accesses_bp; new->nr_accesses = r->nr_accesses; /* todo: do this for only installed probes */ memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); + memcpy(new->last_probe_hits, r->last_probe_hits, + sizeof(r->last_probe_hits)); damon_insert_region(new, r, damon_next_region(r), t); + return 0; } /* Split every region in the given target into 'nr_subs' regions */ @@ -3247,6 +3470,37 @@ static void damon_split_regions_of(struct damon_ctx *ctx, } } +/* Split one in every @split_step regions into two, from a rotating offset */ +static void damon_split_some_regions(struct damon_ctx *ctx, + unsigned long split_step) +{ + static unsigned long rotation; + struct damon_target *t; + struct damon_region *r, *next; + unsigned long offset = rotation++ % split_step; + unsigned long idx = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region_safe(r, next, t) { + unsigned long sz_region, sz_sub; + + if (idx++ % split_step != offset) + continue; + sz_region = damon_sz_region(r); + if (sz_region < 2 * ctx->min_region_sz) + continue; + + sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) * + sz_region / 10, ctx->min_region_sz); + /* Do not allow blank region */ + if (sz_sub == 0 || sz_sub >= sz_region) + continue; + + damon_split_region_at(t, r, sz_sub); + } + } +} + /* * Split every target region into randomly-sized small regions * @@ -3260,25 +3514,33 @@ static void damon_split_regions_of(struct damon_ctx *ctx, static void kdamond_split_regions(struct damon_ctx *ctx) { struct damon_target *t; - unsigned int nr_regions = 0; - static unsigned int last_nr_regions; + unsigned long nr_regions = 0; + unsigned long max_nr_regions = ctx->attrs.max_nr_regions; + static unsigned long last_nr_regions; int nr_subregions = 2; damon_for_each_target(t, ctx) nr_regions += damon_nr_regions(t); - if (nr_regions > ctx->attrs.max_nr_regions / 2) - return; + if (nr_regions >= max_nr_regions) + goto done; + + if (nr_regions > max_nr_regions / 2) { + damon_split_some_regions(ctx, + max_nr_regions / (max_nr_regions - nr_regions)); + goto done; + } /* Maybe the middle of the region has different access frequency */ if (last_nr_regions == nr_regions && - nr_regions < ctx->attrs.max_nr_regions / 3) + nr_regions < max_nr_regions / 3) nr_subregions = 3; damon_for_each_target(t, ctx) damon_split_regions_of(ctx, t, nr_subregions, ctx->min_region_sz); +done: last_nr_regions = nr_regions; } @@ -3490,7 +3752,6 @@ static void kdamond_init_ctx(struct damon_ctx *ctx) static int kdamond_fn(void *data) { struct damon_ctx *ctx = data; - unsigned int max_nr_accesses = 0; unsigned long sz_limit = 0; pr_debug("kdamond (%d) starts\n", current->pid); @@ -3523,25 +3784,38 @@ static int kdamond_fn(void *data) unsigned long next_aggregation_sis = ctx->next_aggregation_sis; unsigned long next_ops_update_sis = ctx->next_ops_update_sis; unsigned long sample_interval = ctx->attrs.sample_interval; + bool access_check_disabled = damon_has_probe_weights(ctx); + unsigned int max_merge_score = 0, max_wsum; + bool get_max_wsum; if (kdamond_wait_activation(ctx)) break; - if (ctx->ops.prepare_access_checks) + if (!access_check_disabled && ctx->ops.prepare_access_checks) ctx->ops.prepare_access_checks(ctx); kdamond_usleep(sample_interval); ctx->passed_sample_intervals++; - if (ctx->ops.check_accesses) - max_nr_accesses = ctx->ops.check_accesses(ctx); - if (ctx->ops.apply_probes) - ctx->ops.apply_probes(ctx); + if (!access_check_disabled && ctx->ops.check_accesses) + max_merge_score = ctx->ops.check_accesses(ctx); + if (ctx->ops.apply_probes) { + if (time_after_eq(ctx->passed_sample_intervals, + next_aggregation_sis) && + access_check_disabled) + get_max_wsum = true; + else + get_max_wsum = false; + max_wsum = ctx->ops.apply_probes(ctx, + access_check_disabled, get_max_wsum); + if (get_max_wsum) + max_merge_score = max_wsum; + } if (time_after_eq(ctx->passed_sample_intervals, next_aggregation_sis)) { kdamond_merge_regions(ctx, - max_nr_accesses / 10, + max_merge_score / 10, sz_limit); /* online updates might be made */ sz_limit = damon_apply_min_nr_regions(ctx); @@ -3584,8 +3858,7 @@ static int kdamond_fn(void *data) * aggregation, and make aggregation * information reset for all regions. Then, * following kdamond_reset_aggregated() call - * will make the region information invalid, - * particularly for ->nr_accesses_bp. + * will make the region information invalid. * * Reset ->next_aggregation_sis to avoid that. * It will anyway correctly updated after this @@ -3712,9 +3985,6 @@ int damon_set_region_system_rams_default(struct damon_target *t, { struct damon_addr_range addr_range; - if (*start > *end) - return -EINVAL; - if (!*start && !*end && !damon_find_system_rams_range(start, end, addr_unit)) return -EINVAL; @@ -3724,72 +3994,18 @@ int damon_set_region_system_rams_default(struct damon_target *t, return damon_set_regions(t, &addr_range, 1, min_region_sz); } -/* - * damon_moving_sum() - Calculate an inferred moving sum value. - * @mvsum: Inferred sum of the last @len_window values. - * @nomvsum: Non-moving sum of the last discrete @len_window window values. - * @len_window: The number of last values to take care of. - * @new_value: New value that will be added to the pseudo moving sum. - * - * Moving sum (moving average * window size) is good for handling noise, but - * the cost of keeping past values can be high for arbitrary window size. This - * function implements a lightweight pseudo moving sum function that doesn't - * keep the past window values. - * - * It simply assumes there was no noise in the past, and get the no-noise - * assumed past value to drop from @nomvsum and @len_window. @nomvsum is a - * non-moving sum of the last window. For example, if @len_window is 10 and we - * have 25 values, @nomvsum is the sum of the 11th to 20th values of the 25 - * values. Hence, this function simply drops @nomvsum / @len_window from - * given @mvsum and add @new_value. - * - * For example, if @len_window is 10 and @nomvsum is 50, the last 10 values for - * the last window could be vary, e.g., 0, 10, 0, 10, 0, 10, 0, 0, 0, 20. For - * calculating next moving sum with a new value, we should drop 0 from 50 and - * add the new value. However, this function assumes it got value 5 for each - * of the last ten times. Based on the assumption, when the next value is - * measured, it drops the assumed past value, 5 from the current sum, and add - * the new value to get the updated pseduo-moving average. - * - * This means the value could have errors, but the errors will be disappeared - * for every @len_window aligned calls. For example, if @len_window is 10, the - * pseudo moving sum with 11th value to 19th value would have an error. But - * the sum with 20th value will not have the error. - * - * Return: Pseudo-moving average after getting the @new_value. - */ -static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum, - unsigned int len_window, unsigned int new_value) -{ - return mvsum - nomvsum / len_window + new_value; -} - /** * damon_update_region_access_rate() - Update the access rate of a region. * @r: The DAMON region to update for its access check result. * @accessed: Whether the region has accessed during last sampling interval. - * @attrs: The damon_attrs of the DAMON context. * * Update the access rate of a region with the region's last sampling interval * access check result. * * Usually this will be called by &damon_operations->check_accesses callback. */ -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs) +void damon_update_region_access_rate(struct damon_region *r, bool accessed) { - unsigned int len_window = 1; - - /* - * sample_interval can be zero, but cannot be larger than - * aggr_interval, owing to validation of damon_set_attrs(). - */ - if (attrs->sample_interval) - len_window = damon_max_nr_accesses(attrs); - r->nr_accesses_bp = damon_moving_sum(r->nr_accesses_bp, - r->last_nr_accesses * 10000, len_window, - accessed ? 10000 : 0); - if (accessed) r->nr_accesses++; } diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 32f41491b726..7e077084cb03 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -284,11 +284,6 @@ static int damon_lru_sort_apply_parameters(void) param_ctx->addr_unit = addr_unit; param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); - if (!is_power_of_2(param_ctx->min_region_sz)) { - err = -EINVAL; - goto out; - } - if (!damon_lru_sort_mon_attrs.sample_interval) { err = -EINVAL; goto out; @@ -308,7 +303,7 @@ static int damon_lru_sort_apply_parameters(void) goto out; err = -ENOMEM; - hot_thres = damon_max_nr_accesses(&attrs) * + hot_thres = damon_nr_samples_per_aggr(&attrs) * hot_thres_access_freq / 1000; hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); if (!hot_scheme) @@ -414,8 +409,10 @@ static int damon_lru_sort_turn(bool on) { int err; - if (!on) - return damon_stop(&ctx, 1); + if (!on) { + damon_stop(&ctx, 1); + return 0; + } err = damon_lru_sort_apply_parameters(); if (err) diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 6bdd1cfd3863..d7d7f100389b 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -111,8 +111,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_max_nr_accesses(&c->attrs); + freq_subscore = mult_frac(damon_nr_accesses_mvsum(r, c), + DAMON_MAX_SUBSCORE, + damon_nr_samples_per_aggr(&c->attrs)); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; if (age_in_sec) @@ -339,8 +340,6 @@ static unsigned int damon_migrate_folio_list(struct list_head *folio_list, LIST_HEAD(migrate_folios); while (!list_empty(folio_list)) { - struct folio *folio; - cond_resched(); folio = lru_to_folio(folio_list); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5c2da45f988c..e4f98d67461f 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -65,7 +65,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -74,31 +74,19 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) return false; accessed = damon_folio_young(folio); - *folio_sz = folio_size(folio); folio_put(folio); return accessed; } static void __damon_pa_check_access(struct damon_region *r, - struct damon_attrs *attrs, unsigned long addr_unit) + unsigned long addr_unit) { - static phys_addr_t last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; phys_addr_t sampling_addr = damon_pa_phys_addr( r->sampling_addr, addr_unit); - /* If the region is in the last checked page, reuse the result */ - if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = sampling_addr; + accessed = damon_pa_young(sampling_addr); + damon_update_region_access_rate(r, accessed); } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) @@ -109,8 +97,7 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { - __damon_pa_check_access( - r, &ctx->attrs, ctx->addr_unit); + __damon_pa_check_access(r, ctx->addr_unit); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } } @@ -167,11 +154,13 @@ static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, return pass; } -static void damon_pa_apply_probes(struct damon_ctx *ctx) +static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx, + bool set_samples, bool return_max_wsum) { struct damon_target *t; struct damon_region *r; struct damon_probe *p; + unsigned int max_wsum = 0; damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { @@ -179,6 +168,9 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx) phys_addr_t pa; struct folio *folio; + if (set_samples) + r->sampling_addr = damon_rand(ctx, r->ar.start, + r->ar.end); pa = damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit); folio = damon_get_folio(PHYS_PFN(pa)); @@ -189,8 +181,12 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx) } if (folio) folio_put(folio); + if (return_max_wsum) + max_wsum = max(damon_probe_hits_wsum(r, false, + ctx), max_wsum); } } + return max_wsum; } /* diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 11b70d0a9a6f..09e941d75f67 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -208,11 +208,6 @@ static int damon_reclaim_apply_parameters(void) param_ctx->addr_unit = addr_unit; param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); - if (!is_power_of_2(param_ctx->min_region_sz)) { - err = -EINVAL; - goto out; - } - if (!damon_reclaim_mon_attrs.aggr_interval) { err = -EINVAL; goto out; @@ -337,8 +332,10 @@ static int damon_reclaim_turn(bool on) { int err; - if (!on) - return damon_stop(&ctx, 1); + if (!on) { + damon_stop(&ctx, 1); + return 0; + } err = damon_reclaim_apply_parameters(); if (err) diff --git a/mm/damon/stat.c b/mm/damon/stat.c index 0e14f5bb8f75..b05b68f73e10 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -138,7 +138,7 @@ static int damon_stat_damon_call_fn(void *data) /* avoid unnecessarily frequent stat update */ if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies + - msecs_to_jiffies(5 * MSEC_PER_SEC))) + secs_to_jiffies(5))) return 0; damon_stat_last_refresh_jiffies = jiffies; diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3cbeccd436e4..32f495a96b17 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -110,7 +110,8 @@ static int damos_sysfs_probes_add_dirs(struct damos_sysfs_probes *probes, struct damos_sysfs_probe *sys_probe; int err; - sys_probe = damos_sysfs_probe_alloc(region->probe_hits[i]); + sys_probe = damos_sysfs_probe_alloc( + damon_probe_hits_mvsum(i, region, ctx)); if (!sys_probe) { damos_sysfs_probes_rm_dirs(probes); return -ENOMEM; @@ -157,7 +158,7 @@ struct damon_sysfs_scheme_region { }; static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( - struct damon_region *region) + struct damon_region *region, struct damon_ctx *ctx) { struct damon_sysfs_scheme_region *sysfs_region = kmalloc_obj(*sysfs_region); @@ -165,7 +166,7 @@ static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( return NULL; sysfs_region->kobj = (struct kobject){}; sysfs_region->ar = region->ar; - sysfs_region->nr_accesses = region->nr_accesses_bp / 10000; + sysfs_region->nr_accesses = damon_nr_accesses_mvsum(region, ctx); sysfs_region->age = region->age; sysfs_region->probes = NULL; INIT_LIST_HEAD(&sysfs_region->list); @@ -332,6 +333,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { damos_sysfs_region_rm_dirs(r); list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } @@ -911,8 +913,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1460,8 +1464,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; @@ -2138,8 +2144,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; @@ -2681,6 +2689,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2722,13 +2731,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); @@ -3112,7 +3123,7 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, if (total_bytes_only) return; - region = damon_sysfs_scheme_region_alloc(r); + region = damon_sysfs_scheme_region_alloc(r, ctx); if (!region) return; region->sz_filter_passed = sz_filter_passed; @@ -3122,12 +3133,14 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, sysfs_regions->nr_regions)) goto out; if (damos_sysfs_region_add_dirs(region, ctx, r)) - goto out; + goto del_out; list_add_tail(®ion->list, &sysfs_regions->regions_list); sysfs_regions->nr_regions++; return; +del_out: + kobject_del(®ion->kobj); out: kobject_put(®ion->kobj); } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index a9e187158067..e3858ffab4b2 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -105,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -331,6 +333,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -369,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); @@ -955,8 +960,10 @@ static void damon_sysfs_filters_rm_dirs(struct damon_sysfs_filters *filters) struct damon_sysfs_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1058,6 +1065,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = { struct damon_sysfs_probe { struct kobject kobj; + unsigned int weight; struct damon_sysfs_filters *filters; }; @@ -1066,7 +1074,7 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void) return kzalloc_obj(struct damon_sysfs_probe); } -static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) +static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe) { struct damon_sysfs_filters *filters; int err; @@ -1074,31 +1082,54 @@ static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) filters = damon_sysfs_filters_alloc(); if (!filters) return -ENOMEM; - attr->filters = filters; + probe->filters = filters; err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype, - &attr->kobj, "filters"); + &probe->kobj, "filters"); if (err) { kobject_put(&filters->kobj); - attr->filters = NULL; + probe->filters = NULL; } return err; } -static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *attr) +static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { - if (attr->filters) { - damon_sysfs_filters_rm_dirs(attr->filters); - kobject_put(&attr->filters->kobj); + if (probe->filters) { + damon_sysfs_filters_rm_dirs(probe->filters); + kobject_put(&probe->filters->kobj); } } +static ssize_t weight_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct damon_sysfs_probe *probe = container_of(kobj, + struct damon_sysfs_probe, kobj); + + return sysfs_emit(buf, "%u\n", probe->weight); +} + +static ssize_t weight_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_probe *probe = container_of(kobj, + struct damon_sysfs_probe, kobj); + int err = kstrtouint(buf, 0, &probe->weight); + + return err ? err : count; +} + static void damon_sysfs_probe_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_probe, kobj)); } +static struct kobj_attribute damon_sysfs_probe_weight_attr = + __ATTR_RW_MODE(weight, 0600); + static struct attribute *damon_sysfs_probe_attrs[] = { + &damon_sysfs_probe_weight_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_probe); @@ -1132,6 +1163,7 @@ static void damon_sysfs_probes_rm_dirs( for (i = 0; i < probes->nr; i++) { damon_sysfs_probe_rm_dirs(probes_arr[i]); + kobject_del(&probes_arr[i]->kobj); kobject_put(&probes_arr[i]->kobj); } probes->nr = 0; @@ -1173,6 +1205,7 @@ static int damon_sysfs_probes_add_dirs( err = damon_sysfs_probe_add_dirs(probe); if (err) { + kobject_del(&probe->kobj); kobject_put(&probe->kobj); damon_sysfs_probes_rm_dirs(probes); return err; @@ -1640,6 +1673,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1678,13 +1712,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -1897,47 +1933,66 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } -static int damon_sysfs_set_probes(struct damon_ctx *ctx, - struct damon_sysfs_probes *sys_probes) +static int damon_sysfs_set_filters(struct damon_probe *probe, + struct damon_sysfs_filters *sys_filters) { int i; - for (i = 0; i < sys_probes->nr; i++) { - struct damon_sysfs_filters *sys_filters = - sys_probes->probes_arr[i]->filters; - struct damon_probe *c; - int j; + for (i = 0; i < sys_filters->nr; i++) { + struct damon_sysfs_filter *sys_filter = + sys_filters->filters_arr[i]; + struct damon_filter *filter; - if (!sys_filters) - continue; - c = damon_new_probe(); - if (!c) + filter = damon_new_filter(sys_filter->type, + sys_filter->matching, + sys_filter->allow); + if (!filter) return -ENOMEM; - damon_add_probe(ctx, c); - - for (j = 0; j < sys_filters->nr; j++) { - struct damon_sysfs_filter *sys_filter = - sys_filters->filters_arr[j]; - struct damon_filter *filter; - - filter = damon_new_filter(sys_filter->type, - sys_filter->matching, - sys_filter->allow); - if (!filter) - return -ENOMEM; - if (filter->type == DAMON_FILTER_TYPE_MEMCG) { - int err; - - err = damon_sysfs_memcg_path_to_id( - sys_filter->path, - &filter->memcg_id); - if (err) { - damon_destroy_filter(filter); - return err; - } + if (filter->type == DAMON_FILTER_TYPE_MEMCG) { + int err; + + err = damon_sysfs_memcg_path_to_id( + sys_filter->path, + &filter->memcg_id); + if (err) { + damon_destroy_filter(filter); + return err; } - damon_add_filter(c, filter); } + damon_add_filter(probe, filter); + } + return 0; +} + +static int damon_sysfs_set_probe(struct damon_probe *probe, + struct damon_sysfs_probe *sys_probe) +{ + struct damon_sysfs_filters *sys_filters; + + sys_filters = sys_probe->filters; + if (!sys_filters) + return 0; + return damon_sysfs_set_filters(probe, sys_filters); +} + +static int damon_sysfs_set_probes(struct damon_ctx *ctx, + struct damon_sysfs_probes *sys_probes) +{ + int i, err; + + for (i = 0; i < sys_probes->nr; i++) { + struct damon_sysfs_probe *sys_probe; + struct damon_probe *p; + + p = damon_new_probe(); + if (!p) + return -ENOMEM; + damon_add_probe(ctx, p); + sys_probe = sys_probes->probes_arr[i]; + p->weight = sys_probe->weight; + err = damon_sysfs_set_probe(p, sys_probe); + if (err) + return err; } return 0; } @@ -1957,9 +2012,6 @@ static int damon_sysfs_set_regions(struct damon_target *t, struct damon_sysfs_region *sys_region = sysfs_regions->regions_arr[i]; - if (sys_region->ar.start > sys_region->ar.end) - goto out; - ranges[i].start = sys_region->ar.start; ranges[i].end = sys_region->ar.end; if (i == 0) @@ -1999,7 +2051,7 @@ static int damon_sysfs_add_targets(struct damon_ctx *ctx, int i, err; /* Multiple physical address space monitoring targets makes no sense */ - if (ctx->ops.id == DAMON_OPS_PADDR && sysfs_targets->nr > 1) + if (!damon_target_has_pid(ctx) && sysfs_targets->nr > 1) return -EINVAL; for (i = 0; i < sysfs_targets->nr; i++) { @@ -2042,16 +2094,18 @@ static inline bool damon_sysfs_kdamond_running( static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, struct damon_sysfs_context *sys_ctx) { + enum damon_ops_id ops_id; int err; - err = damon_select_ops(ctx, sys_ctx->ops_id); + ops_id = READ_ONCE(sys_ctx->ops_id); + err = damon_select_ops(ctx, ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ - if (sys_ctx->ops_id == DAMON_OPS_PADDR) + if (ops_id == DAMON_OPS_PADDR) ctx->min_region_sz = max( - DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1); + DAMON_MIN_REGION_SZ / ctx->addr_unit, 1); ctx->pause = sys_ctx->pause; err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) @@ -2069,26 +2123,6 @@ static struct damon_ctx *damon_sysfs_build_ctx( struct damon_sysfs_context *sys_ctx); /* - * Return a new damon_ctx for testing new parameters to commit. - */ -static struct damon_ctx *damon_sysfs_new_test_ctx( - struct damon_ctx *running_ctx) -{ - struct damon_ctx *test_ctx; - int err; - - test_ctx = damon_new_ctx(); - if (!test_ctx) - return NULL; - err = damon_commit_ctx(test_ctx, running_ctx); - if (err) { - damon_destroy_ctx(test_ctx); - return NULL; - } - return test_ctx; -} - -/* * damon_sysfs_commit_input() - Commit user inputs to a running kdamond. * @kdamond: The kobject wrapper for the associated kdamond. * @@ -2097,7 +2131,7 @@ static struct damon_ctx *damon_sysfs_new_test_ctx( static int damon_sysfs_commit_input(void *data) { struct damon_sysfs_kdamond *kdamond = data; - struct damon_ctx *param_ctx, *test_ctx; + struct damon_ctx *param_ctx; int err; if (!damon_sysfs_kdamond_running(kdamond)) @@ -2109,17 +2143,7 @@ static int damon_sysfs_commit_input(void *data) param_ctx = damon_sysfs_build_ctx(kdamond->contexts->contexts_arr[0]); if (IS_ERR(param_ctx)) return PTR_ERR(param_ctx); - test_ctx = damon_sysfs_new_test_ctx(kdamond->damon_ctx); - if (!test_ctx) { - damon_destroy_ctx(param_ctx); - return -ENOMEM; - } - err = damon_commit_ctx(test_ctx, param_ctx); - if (err) - goto out; err = damon_commit_ctx(kdamond->damon_ctx, param_ctx); -out: - damon_destroy_ctx(test_ctx); damon_destroy_ctx(param_ctx); return err; } @@ -2265,12 +2289,13 @@ static int damon_sysfs_turn_damon_off(struct damon_sysfs_kdamond *kdamond) { if (!kdamond->damon_ctx) return -EINVAL; - return damon_stop(&kdamond->damon_ctx, 1); + damon_stop(&kdamond->damon_ctx, 1); /* * To allow users show final monitoring results of already turned-off * DAMON, we free kdamond->damon_ctx in next * damon_sysfs_turn_damon_on(), or kdamonds_nr_store() */ + return 0; } static int damon_sysfs_damon_call(int (*fn)(void *data), @@ -2499,6 +2524,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -2553,13 +2579,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fcf7c7fadb5f..4a536d41cdb2 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -114,7 +114,6 @@ static void damon_test_aggregate(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = accesses[it][ir]; - r->nr_accesses_bp = accesses[it][ir] * 10000; damon_add_region(r, t); } it++; @@ -151,12 +150,15 @@ static void damon_test_split_at(struct kunit *test) damon_free_target(t); kunit_skip(test, "region alloc fail"); } - r->nr_accesses_bp = 420000; r->nr_accesses = 42; r->last_nr_accesses = 15; r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -164,11 +166,11 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r_new->ar.start, 25ul); KUNIT_EXPECT_EQ(test, r_new->ar.end, 100ul); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, r_new->nr_accesses_bp); KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); +out: damon_free_target(t); } @@ -187,7 +189,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = 10; - r->nr_accesses_bp = 100000; r->age = 9; damon_add_region(r, t); r2 = damon_new_region(100, 300); @@ -196,7 +197,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "second region alloc fail"); } r2->nr_accesses = 20; - r2->nr_accesses_bp = 200000; r2->age = 21; damon_add_region(r2, t); @@ -204,7 +204,6 @@ static void damon_test_merge_two(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, 160000u); KUNIT_EXPECT_EQ(test, r->age, 17u); i = 0; @@ -232,6 +231,7 @@ static struct damon_region *__nth_region_of(struct damon_target *t, int idx) static void damon_test_merge_regions_of(struct kunit *test) { + struct damon_ctx *ctx; struct damon_target *t; struct damon_region *r; unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184, 230}; @@ -242,29 +242,39 @@ static void damon_test_merge_regions_of(struct kunit *test) unsigned long eaddrs[] = {112, 130, 156, 170, 230, 10170}; int i; + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + t = damon_new_target(); - if (!t) + if (!t) { + damon_destroy_ctx(ctx); kunit_skip(test, "target alloc fail"); + } for (i = 0; i < ARRAY_SIZE(sa); i++) { r = damon_new_region(sa[i], ea[i]); if (!r) { damon_free_target(t); + damon_destroy_ctx(ctx); kunit_skip(test, "region alloc fail"); } r->nr_accesses = nrs[i]; - r->nr_accesses_bp = nrs[i] * 10000; damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999); + damon_merge_regions_of(t, 9, 9999, ctx, true); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); + if (damon_nr_regions(t) != 6) + goto out; for (i = 0; i < 6; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } +out: damon_free_target(t); + damon_destroy_ctx(ctx); } static void damon_test_split_regions_of(struct kunit *test) @@ -335,6 +345,69 @@ static void damon_test_split_regions_of(struct kunit *test) damon_destroy_ctx(c); } +/* + * When the total region count is already above max_nr_regions / 2, + * kdamond_split_regions() must keep refining the resolution by splitting a + * fraction of the regions (making progress), without exceeding + * max_nr_regions. + */ +static void damon_test_split_above_half_progresses(struct kunit *test) +{ + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + unsigned long start; + unsigned int nr_before, nr_after, i; + const unsigned int nr_init = 760; + const unsigned long region_sz = 100; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + /* Keep the split arithmetic independent of the page size */ + c->min_region_sz = 1; + c->attrs.min_nr_regions = 10; + c->attrs.max_nr_regions = 1500; + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + + for (i = 0; i < nr_init; i++) { + start = i * region_sz; + r = damon_new_region(start, start + region_sz); + if (!r) { + damon_free_target(t); + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + r->nr_accesses = (i & 1) ? 0 : 100; + r->age = 5; + damon_add_region(r, t); + } + + damon_add_target(c, t); + + nr_before = damon_nr_regions(t); + /* Above max_nr_regions / 2, so the blanket-split path is skipped */ + KUNIT_EXPECT_GT(test, (unsigned long)nr_before, + c->attrs.max_nr_regions / 2); + + kdamond_split_regions(c); + + nr_after = damon_nr_regions(t); + /* Still made progress ... */ + KUNIT_EXPECT_GT(test, nr_after, nr_before); + /* ... but did not overshoot the configured maximum */ + KUNIT_EXPECT_LE(test, (unsigned long)nr_after, + c->attrs.max_nr_regions); + + damon_destroy_ctx(c); +} + static void damon_test_ops_registration(struct kunit *test) { struct damon_ctx *c = damon_new_ctx(); @@ -519,61 +592,45 @@ static void damon_test_set_regions(struct kunit *test) }, 3); } -static void damon_test_nr_accesses_to_accesses_bp(struct kunit *test) -{ - struct damon_attrs attrs = { - .sample_interval = 10, - .aggr_interval = ((unsigned long)UINT_MAX + 1) * 10 - }; - - /* - * In some cases such as 32bit architectures where UINT_MAX is - * ULONG_MAX, attrs.aggr_interval becomes zero. Calling - * damon_nr_accesses_to_accesses_bp() in the case will cause - * divide-by-zero. Such case is prohibited in normal execution since - * the caution is documented on the comment for the function, and - * damon_update_monitoring_results() does the check. Skip the test in - * the case. - */ - if (!attrs.aggr_interval) - kunit_skip(test, "aggr_interval is zero."); - - KUNIT_EXPECT_EQ(test, damon_nr_accesses_to_accesses_bp(123, &attrs), 0); -} - static void damon_test_update_monitoring_result(struct kunit *test) { struct damon_attrs old_attrs = { .sample_interval = 10, .aggr_interval = 1000,}; struct damon_attrs new_attrs; struct damon_region *r = damon_new_region(3, 7); + struct damon_ctx *ctx; if (!r) kunit_skip(test, "region alloc fail"); + ctx = damon_new_ctx(); + if (!ctx) { + damon_free_region(r); + kunit_skip(test, "ctx alloc fail"); + } r->nr_accesses = 15; - r->nr_accesses_bp = 150000; r->age = 20; new_attrs = (struct damon_attrs){ .sample_interval = 100, .aggr_interval = 10000,}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 15); KUNIT_EXPECT_EQ(test, r->age, 2); new_attrs = (struct damon_attrs){ .sample_interval = 1, .aggr_interval = 1000}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 150); KUNIT_EXPECT_EQ(test, r->age, 2); new_attrs = (struct damon_attrs){ .sample_interval = 1, .aggr_interval = 100}; - damon_update_monitoring_result(r, &old_attrs, &new_attrs, false); + damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx); KUNIT_EXPECT_EQ(test, r->nr_accesses, 150); KUNIT_EXPECT_EQ(test, r->age, 20); damon_free_region(r); + damon_destroy_ctx(ctx); } static void damon_test_set_attrs(struct kunit *test) @@ -604,19 +661,76 @@ static void damon_test_set_attrs(struct kunit *test) damon_destroy_ctx(c); } -static void damon_test_moving_sum(struct kunit *test) +static void damon_test_mvsum(struct kunit *test) +{ + unsigned long input_expects[] = { + /* current value, last value, remaining window (bp) */ + 0, 49, 10000, 49, /* 0 + 49 * 1 */ + 3, 10, 7000, 10, /* 3 + 10 * 0.7 */ + 3, 10, 5000, 8, /* 3 + 10 * 0.5 */ + 32, 100, 1000, 42, /* 32 + 100 * 0.1 */ + 42, 49, 0, 42, /* 42 + 49 * 0 */ + }; + + int i; + + for (i = 0; i < ARRAY_SIZE(input_expects); i += 4) { + unsigned long current_nr = input_expects[i]; + unsigned long last_nr = input_expects[i + 1]; + unsigned long left_window_bp = input_expects[i + 2]; + unsigned long expect = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, damon_mvsum(current_nr, last_nr, + left_window_bp), expect); + } +} + +/* + * Test damon_nr_accesses_mvsum(), which wraps damon_mvsum() with the + * monitoring intervals of the context. With a sample interval of 1 and an + * aggregation interval of 10, an aggregation window is 10 sample intervals + * long. Each row below specifies the passed sample intervals, the next + * aggregation time in sample intervals, the current and last nr_accesses of a + * region, and the expected return value. + */ +static void damon_test_nr_accesses_mvsum(struct kunit *test) { - unsigned int mvsum = 50000, nomvsum = 50000, len_window = 10; - unsigned int new_values[] = {10000, 0, 10000, 0, 0, 0, 10000, 0, 0, 0}; - unsigned int expects[] = {55000, 50000, 55000, 50000, 45000, 40000, - 45000, 40000, 35000, 30000}; + unsigned long input_expects[] = { + /* passed, next_aggr, nr_accesses, last_nr_accesses, expect */ + 0, 10, 5, 3, 3, /* full window left, unreset */ + 0, 10, 0, 7, 7, /* full window left, reset */ + 5, 10, 3, 10, 8, /* half window left */ + 8, 10, 3, 10, 5, /* 20% window left */ + 10, 10, 42, 49, 42, /* no window left */ + }; + struct damon_ctx *c = damon_new_ctx(); + struct damon_region *r; int i; - for (i = 0; i < ARRAY_SIZE(new_values); i++) { - mvsum = damon_moving_sum(mvsum, nomvsum, len_window, - new_values[i]); - KUNIT_EXPECT_EQ(test, mvsum, expects[i]); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + r = damon_new_region(0, 4096); + if (!r) { + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + + c->attrs.sample_interval = 1; + c->attrs.aggr_interval = 10; + + for (i = 0; i < ARRAY_SIZE(input_expects); i += 5) { + c->passed_sample_intervals = input_expects[i]; + c->next_aggregation_sis = input_expects[i + 1]; + r->nr_accesses = input_expects[i + 2]; + r->last_nr_accesses = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, (unsigned int)input_expects[i + 4], + damon_nr_accesses_mvsum(r, c)); } + + damon_free_region(r); + damon_destroy_ctx(c); } static void damos_test_new_filter(struct kunit *test) @@ -725,6 +839,7 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, struct damos_quota_goal *goal, *next; bool skip = true; int i; + int nr_dst = 0, nr_src = 0; INIT_LIST_HEAD(&dst.goals); INIT_LIST_HEAD(&src.goals); @@ -747,6 +862,14 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, damos_commit_quota_goals(&dst, &src); + damos_for_each_quota_goal(goal, &dst) + nr_dst++; + damos_for_each_quota_goal(goal, &src) + nr_src++; + KUNIT_EXPECT_EQ(test, nr_dst, nr_src); + if (nr_dst != nr_src) + goto out; + i = 0; damos_for_each_quota_goal(goal, (&dst)) { KUNIT_EXPECT_EQ(test, goal->metric, src_goals[i].metric); @@ -894,6 +1017,8 @@ static void damos_test_commit_dests_for(struct kunit *test, skip = false; KUNIT_EXPECT_EQ(test, dst.nr_dests, src_nr_dests); + if (dst.nr_dests != src_nr_dests) + goto out; for (i = 0; i < dst.nr_dests; i++) { KUNIT_EXPECT_EQ(test, dst.node_id_arr[i], src_node_id_arr[i]); KUNIT_EXPECT_EQ(test, dst.weight_arr[i], src_weight_arr[i]); @@ -1152,14 +1277,19 @@ static void damon_test_commit_target_regions_for(struct kunit *test, kunit_skip(test, "src target setup fail"); } damon_commit_target_regions(dst_target, src_target, 1); + + KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); + if (damon_nr_regions(dst_target) != nr_expect_regions) + goto out; + i = 0; damon_for_each_region(r, dst_target) { KUNIT_EXPECT_EQ(test, r->ar.start, expect_start_end[i][0]); KUNIT_EXPECT_EQ(test, r->ar.end, expect_start_end[i][1]); i++; } - KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); - KUNIT_EXPECT_EQ(test, i, nr_expect_regions); + +out: damon_free_target(dst_target); damon_free_target(src_target); } @@ -1251,6 +1381,8 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 1); KUNIT_EXPECT_EQ(test, r->ar.end, 2); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 2); KUNIT_EXPECT_EQ(test, r2->ar.end, 4); @@ -1265,11 +1397,14 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 2); KUNIT_EXPECT_EQ(test, r->ar.end, 6); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 6); KUNIT_EXPECT_EQ(test, r2->ar.end, 8); damon_destroy_region(r2, t); +out: damon_free_target(t); damos_free_filter(f); } @@ -1456,6 +1591,47 @@ static void damon_test_is_last_region(struct kunit *test) damon_free_target(t); } +/* + * Verify that damos_walk() rejects new requests when + * walk_control_obsolete is set. + * + * This tests the invariant introduced by: + * commit 33c3f6c2b48c ("mm/damon/core: fix damos_walk() vs kdamond_fn() exit race") + */ +static void damon_test_walk_control_obsolete(struct kunit *test) +{ + struct damon_ctx *ctx; + struct damos_walk_control control = {}; + int ret; + + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + + /* Simulate shutdown phase */ + ctx->walk_control_obsolete = true; + + ret = damos_walk(ctx, &control); + + KUNIT_EXPECT_EQ(test, ret, -ECANCELED); + + damon_destroy_ctx(ctx); +} + +static void damon_test_rand(struct kunit *test) +{ + struct damon_ctx ctx; + int i; + + prandom_seed_state(&ctx.rnd_state, get_random_u64()); + for (i = 0; i < 10000; i++) { + unsigned long rnd = damon_rand(&ctx, 0, 10); + + KUNIT_EXPECT_GE(test, rnd, 0); + KUNIT_EXPECT_LE(test, rnd, 9); + } +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_target), KUNIT_CASE(damon_test_regions), @@ -1464,12 +1640,13 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_merge_two), KUNIT_CASE(damon_test_merge_regions_of), KUNIT_CASE(damon_test_split_regions_of), + KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), - KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), - KUNIT_CASE(damon_test_moving_sum), + KUNIT_CASE(damon_test_mvsum), + KUNIT_CASE(damon_test_nr_accesses_mvsum), KUNIT_CASE(damos_test_new_filter), KUNIT_CASE(damos_test_commit_quota_goal), KUNIT_CASE(damos_test_commit_quota_goals), @@ -1485,6 +1662,8 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_set_filters_default_reject), KUNIT_CASE(damon_test_apply_min_nr_regions), KUNIT_CASE(damon_test_is_last_region), + KUNIT_CASE(damon_test_walk_control_obsolete), + KUNIT_CASE(damon_test_rand), {}, }; diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 61f844336ffb..6a95441d193a 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -158,12 +158,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test, kunit_skip(test, "second damon_set_regions() fail"); } + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2); + if (damon_nr_regions(t) != nr_expected / 2) + goto out; + for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } +out: damon_destroy_target(t, NULL); } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index e73ec1ce016e..d487b7a4a104 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -10,6 +10,7 @@ #include <linux/mman.h> #include <linux/mmu_notifier.h> #include <linux/page_idle.h> +#include <linux/pagemap.h> #include <linux/pagewalk.h> #include <linux/sched/mm.h> @@ -382,8 +383,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) } struct damon_young_walk_private { - /* size of the folio for the access checked virtual memory address */ - unsigned long *folio_sz; bool young; }; @@ -410,7 +409,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = HPAGE_PMD_SIZE; huge_out: spin_unlock(ptl); return 0; @@ -429,7 +427,6 @@ huge_out: if (pte_young(ptent) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = folio_size(folio); out: pte_unmap_unlock(pte, ptl); return 0; @@ -457,7 +454,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, if (pte_young(entry) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = huge_page_size(h); folio_put(folio); @@ -469,11 +465,9 @@ out: #define damon_young_hugetlb_entry NULL #endif /* CONFIG_HUGETLB_PAGE */ -static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *folio_sz) +static bool damon_va_young(struct mm_struct *mm, unsigned long addr) { struct damon_young_walk_private arg = { - .folio_sz = folio_sz, .young = false, }; @@ -493,29 +487,17 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, - struct damon_attrs *attrs) + struct damon_region *r) { - static unsigned long last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; if (!mm) { - damon_update_region_access_rate(r, false, attrs); + damon_update_region_access_rate(r, false); return; } - /* If the region is in the last checked page, reuse the result */ - if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = r->sampling_addr; + accessed = damon_va_young(mm, r->sampling_addr); + damon_update_region_access_rate(r, accessed); } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) @@ -524,16 +506,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; - bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); - same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); - same_target = true; } if (mm) mmput(mm); @@ -625,8 +603,8 @@ static void damos_va_migrate_dests_add(struct folio *folio, } order = folio_order(folio); - ilx = vma->vm_pgoff >> order; - ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + ilx = vma_start_pgoff(vma) >> order; + ilx += linear_page_delta(vma, addr) >> order; for (i = 0; i < dests->nr_dests; i++) weight_total += dests->weight_arr[i]; diff --git a/mm/debug.c b/mm/debug.c index 77fa8fe1d641..9a0297b3988d 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -163,7 +163,7 @@ void dump_vma(const struct vm_area_struct *vma) "flags: %#lx(%pGv)\n", vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, (unsigned long)pgprot_val(vma->vm_page_prot), - vma->anon_vma, vma->vm_ops, vma->vm_pgoff, + vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma), vma->vm_file, vma->vm_private_data, #ifdef CONFIG_PER_VMA_LOCK refcount_read(&vma->vm_refcnt), @@ -197,7 +197,7 @@ void dump_mm(const struct mm_struct *mm) "numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n" #endif "tlb_flush_pending %d\n" - "def_flags: %#lx(%pGv)\n", + "def_flags: %*pb(%pGv)\n", mm, mm->task_size, mm->mmap_base, mm->mmap_legacy_base, @@ -226,7 +226,8 @@ void dump_mm(const struct mm_struct *mm) mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq, #endif atomic_read(&mm->tlb_flush_pending), - mm->def_flags, &mm->def_flags + NUM_VMA_FLAG_BITS, mm->def_vma_flags.__vma_flags, + &mm->def_vma_flags ); } EXPORT_SYMBOL(dump_mm); diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 23dc3ee09561..2875fd22d7bb 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -672,7 +672,7 @@ static void __init pte_protnone_tests(struct pgtable_debug_args *args) { pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; pr_debug("Validating PTE protnone\n"); @@ -685,7 +685,7 @@ static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { pmd_t pmd; - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; if (!has_transparent_hugepage()) @@ -751,14 +751,14 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) pmd_t pmd; if (!pgtable_supports_soft_dirty() || - !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) + !IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES)) return; if (!has_transparent_hugepage()) return; pr_debug("Validating PMD swap soft dirty\n"); - pmd = swp_entry_to_pmd(args->leaf_entry); + pmd = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd)); WARN_ON(!pmd_is_valid_softleaf(pmd)); @@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args) WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { swp_entry_t arch_entry; @@ -829,7 +829,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap\n"); - pmd1 = swp_entry_to_pmd(args->leaf_entry); + pmd1 = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd1)); WARN_ON(!pmd_is_valid_softleaf(pmd1)); @@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) pmd2 = __swp_entry_to_pmd(arch_entry); WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1))); } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init swap_migration_tests(struct pgtable_debug_args *args) { diff --git a/mm/execmem.c b/mm/execmem.c index 084a207e4278..74a178a87e75 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -20,6 +20,7 @@ #include <asm/tlbflush.h> #include "internal.h" +#include "vmalloc.h" static struct execmem_info *execmem_info __ro_after_init; static struct execmem_info default_execmem_info __ro_after_init; diff --git a/mm/filemap.c b/mm/filemap.c index 58eb9d240643..1dbb4c6f824e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2270,10 +2270,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, unsigned long nr; struct folio *folio; - rcu_read_lock(); + if (*start > end) + return 0; - for (folio = xas_load(&xas); folio && xas.xa_index <= end; - folio = xas_next(&xas)) { + rcu_read_lock(); + for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; /* @@ -2281,11 +2282,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, * No current caller is looking for DAX entries. */ if (xa_is_value(folio)) - goto update_start; + break; /* If we landed in the middle of a THP, continue at its end. */ if (xa_is_sibling(folio)) - goto update_start; + break; if (!folio_try_get(folio)) goto retry; @@ -2293,29 +2294,27 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, if (unlikely(folio != xas_reload(&xas))) goto put_folio; - if (!folio_batch_add(fbatch, folio)) { - *start = folio_next_index(folio); - goto out; - } + if (!folio_batch_add(fbatch, folio)) + break; + xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= end) + break; continue; + put_folio: folio_put(folio); - retry: xas_reset(&xas); } + rcu_read_unlock(); -update_start: nr = folio_batch_count(fbatch); - if (nr) { folio = fbatch->folios[nr - 1]; *start = folio_next_index(folio); } -out: - rcu_read_unlock(); - return folio_batch_count(fbatch); + return nr; } EXPORT_SYMBOL(filemap_get_folios_contig); @@ -2467,11 +2466,14 @@ static void filemap_get_read_batch(struct address_space *mapping, XA_STATE(xas, &mapping->i_pages, index); struct folio *folio; + if (index > max) + return; + rcu_read_lock(); for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; - if (xas.xa_index > max || xa_is_value(folio)) + if (xa_is_value(folio)) break; if (xa_is_sibling(folio)) break; @@ -2488,6 +2490,8 @@ static void filemap_get_read_batch(struct address_space *mapping, if (folio_test_readahead(folio)) break; xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= max) + break; continue; put_folio: folio_put(folio); @@ -3225,6 +3229,7 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; size_t seek_size; + u64 next; if (start < pos) { if (!seek_data) @@ -3233,7 +3238,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, } seek_size = seek_folio_size(&xas, folio); - pos = round_up((u64)pos + 1, seek_size); + next = round_up((u64)pos + 1, seek_size); + if (next > (u64)end) + pos = end; + else + pos = next; start = folio_seek_hole_data(&xas, mapping, folio, start, pos, seek_data); if (start < pos) @@ -3402,8 +3411,8 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) * of memory. */ struct vm_area_struct *vma = vmf->vma; - unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + const unsigned long start = vma_start_pgoff(vma); + const unsigned long end = vma_end_pgoff(vma); unsigned long ra_end; ra->order = exec_folio_order(); @@ -3921,7 +3930,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, goto out; } - addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + + ((start_pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); if (!vmf->pte) { folio_unlock(folio); diff --git a/mm/swap.c b/mm/folio.c index 588f50d8f1a8..d2937600cf72 100644 --- a/mm/swap.c +++ b/mm/folio.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/mm/swap.c + * linux/mm/folio.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */ /* - * This file contains the default values for the operation of the - * Linux VM subsystem. Fine-tuning documentation can be found in - * Documentation/admin-guide/sysctl/vm.rst. - * Started 18.12.91 - * Swap aging added 23.2.95, Stephen Tweedie. - * Buffermem limits added 12.3.98, Rik van Riel. + * Folio LRU helpers: add/remove folios from LRU lists, batching, + * activation/deactivation, and page cache release paths. */ #include <linux/mm.h> @@ -39,14 +35,11 @@ #include <linux/buffer_head.h> #include "internal.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include <trace/events/pagemap.h> -/* How many pages do we try to swap or page in/out together? As a power of 2 */ -int page_cluster; -static const int page_cluster_max = 31; - struct cpu_fbatches { /* * The following folio batches are grouped together because they are protected @@ -694,9 +687,12 @@ void lru_add_drain_cpu(int cpu) { struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); struct folio_batch *fbatch = &fbatches->lru_add; + unsigned int nr_folios = folio_batch_count(fbatch); - if (folio_batch_count(fbatch)) + if (nr_folios) { folio_batch_move_lru(fbatch, lru_add); + trace_mm_lru_add_drain_tp(cpu, nr_folios); + } fbatch = &fbatches->lru_move_tail; /* Disabling interrupts below acts as a compiler barrier. */ @@ -828,13 +824,13 @@ static bool cpu_needs_drain(unsigned int cpu) struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); /* Check these in order of likelihood that they're not zero */ - return folio_batch_count(&fbatches->lru_add) || - folio_batch_count(&fbatches->lru_move_tail) || - folio_batch_count(&fbatches->lru_deactivate_file) || - folio_batch_count(&fbatches->lru_deactivate) || - folio_batch_count(&fbatches->lru_lazyfree) || - folio_batch_count(&fbatches->lru_activate) || - need_mlock_drain(cpu) || + return data_race(folio_batch_count(&fbatches->lru_add) || + folio_batch_count(&fbatches->lru_move_tail) || + folio_batch_count(&fbatches->lru_deactivate_file) || + folio_batch_count(&fbatches->lru_deactivate) || + folio_batch_count(&fbatches->lru_lazyfree) || + folio_batch_count(&fbatches->lru_activate) || + need_mlock_drain(cpu)) || has_bh_in_lru(cpu, NULL); } @@ -869,6 +865,8 @@ static inline void __lru_add_drain_all(bool force_all_cpus) if (WARN_ON(!mm_percpu_wq)) return; + trace_mm_lru_add_drain_all_tp(force_all_cpus); + /* * Guarantee folio_batch counter stores visible by this CPU * are visible to other CPUs before loading the current drain @@ -1171,35 +1169,3 @@ void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid); } #endif - -static const struct ctl_table swap_sysctl_table[] = { - { - .procname = "page-cluster", - .data = &page_cluster, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&page_cluster_max, - } -}; - -/* - * Perform any setup for the swap system - */ -void __init swap_setup(void) -{ - unsigned long megs = PAGES_TO_MB(totalram_pages()); - - /* Use a smaller cluster for small-memory machines */ - if (megs < 16) - page_cluster = 2; - else - page_cluster = 3; - /* - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ - - register_sysctl_init("vm", swap_sysctl_table); -} @@ -2784,12 +2784,17 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) mapping = READ_ONCE(folio->mapping); /* - * The mapping may have been truncated, in any case we cannot determine - * if this mapping is safe - fall back to slow path to determine how to - * proceed. + * If the mapping is NULL (truncated, or never set), we cannot + * determine whether the folio is file-backed, so a long-term writable + * pin must fall back to the slow path. + * + * Otherwise, a NULL mapping proves this is not a secretmem folio + * (secretmem folios always have a valid mapping to the secretmem + * inode's address_space), so in that case, we can continue with the + * fast path. */ if (!mapping) - return false; + return !reject_file_backed; /* Anonymous folios pose no problem. */ mapping_flags = (unsigned long)mapping & FOLIO_MAPPING_FLAGS; @@ -32,9 +32,27 @@ struct hmm_vma_walk { struct hmm_range *range; + bool *locked; unsigned long last; + unsigned long end; + unsigned int required_fault; }; +/* + * Internal sentinel returned by walk callbacks when they need a page fault. + * The callback stores end/required_fault in hmm_vma_walk; the outer loop + * consumes the sentinel and never propagates it to the caller. + */ +#define HMM_FAULT_PENDING -EAGAIN + +/* + * Internal sentinel returned by hmm_do_fault() when handle_mm_fault() + * completes a page fault with the mmap lock dropped. hmm_do_fault() sets + * *locked = false; the outer loop consumes the sentinel and never propagates + * it to the caller. + */ +#define HMM_FAULT_UNLOCKED -ENOLCK + enum { HMM_NEED_FAULT = 1 << 0, HMM_NEED_WRITE_FAULT = 1 << 1, @@ -60,37 +78,25 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end, } /* - * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s) - * @addr: range virtual start address (inclusive) - * @end: range virtual end address (exclusive) - * @required_fault: HMM_NEED_* flags - * @walk: mm_walk structure - * Return: -EBUSY after page fault, or page fault error + * hmm_record_fault() - record a range that needs to be faulted in * - * This function will be called whenever pmd_none() or pte_none() returns true, - * or whenever there is no page directory covering the virtual address range. + * Called by the walk callbacks when they discover that part of the range + * needs a page fault. The callback records what to fault and returns + * HMM_FAULT_PENDING; the outer loop in hmm_range_fault_locked() drops + * back out of walk_page_range() and invokes handle_mm_fault() from a context + * where no page-table or hugetlb_vma_lock is held. */ -static int hmm_vma_fault(unsigned long addr, unsigned long end, - unsigned int required_fault, struct mm_walk *walk) +static int hmm_record_fault(unsigned long addr, unsigned long end, + unsigned int required_fault, + struct mm_walk *walk) { struct hmm_vma_walk *hmm_vma_walk = walk->private; - struct vm_area_struct *vma = walk->vma; - unsigned int fault_flags = FAULT_FLAG_REMOTE; WARN_ON_ONCE(!required_fault); hmm_vma_walk->last = addr; - - if (required_fault & HMM_NEED_WRITE_FAULT) { - if (!(vma->vm_flags & VM_WRITE)) - return -EPERM; - fault_flags |= FAULT_FLAG_WRITE; - } - - for (; addr < end; addr += PAGE_SIZE) - if (handle_mm_fault(vma, addr, fault_flags, NULL) & - VM_FAULT_ERROR) - return -EFAULT; - return -EBUSY; + hmm_vma_walk->end = end; + hmm_vma_walk->required_fault = required_fault; + return HMM_FAULT_PENDING; } static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, @@ -174,7 +180,7 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR); } if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); return hmm_pfns_fill(addr, end, range, 0); } @@ -209,7 +215,7 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags); if (required_fault) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { @@ -328,10 +334,10 @@ out: fault: pte_unmap(ptep); /* Fault any virtual address we were asked to fault */ - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -371,7 +377,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, npages, 0); if (required_fault) { if (softleaf_is_device_private(entry)) - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); else return -EFAULT; } @@ -391,7 +397,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, @@ -517,7 +523,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, npages, cpu_flags); if (required_fault) { spin_unlock(ptl); - return hmm_vma_fault(addr, end, required_fault, walk); + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); @@ -564,21 +570,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, required_fault = hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags); if (required_fault) { - int ret; - spin_unlock(ptl); - hugetlb_vma_unlock_read(vma); - /* - * Avoid deadlock: drop the vma lock before calling - * hmm_vma_fault(), which will itself potentially take and - * drop the vma lock. This is also correct from a - * protection point of view, because there is no further - * use here of either pte or ptl after dropping the vma - * lock. - */ - ret = hmm_vma_fault(addr, end, required_fault, walk); - hugetlb_vma_lock_read(vma); - return ret; + return hmm_record_fault(addr, end, required_fault, walk); } pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); @@ -637,29 +630,67 @@ static const struct mm_walk_ops hmm_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; -/** - * hmm_range_fault - try to fault some address in a virtual address range - * @range: argument structure - * - * Returns 0 on success or one of the following error codes: +/* + * hmm_do_fault - fault in a range recorded by a walk callback * - * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma - * (e.g., device file vma). - * -ENOMEM: Out of memory. - * -EPERM: Invalid permission (e.g., asking for write and range is read - * only). - * -EBUSY: The range has been invalidated and the caller needs to wait for - * the invalidation to finish. - * -EFAULT: A page was requested to be valid and could not be made valid - * ie it has no backing VMA or it is illegal to access + * Called from the outer loop in hmm_range_fault_locked() after a callback + * returned HMM_FAULT_PENDING. At this point we hold only mmap_lock; + * the page-table spinlock and any hugetlb_vma_lock acquired by the walk + * framework have already been released by the unwind. * - * This is similar to get_user_pages(), except that it can read the page tables - * without mutating them (ie causing faults). + * Returns -EBUSY on success (all pages faulted, caller should re-walk). + * Returns a negative errno on failure. */ -int hmm_range_fault(struct hmm_range *range) +static int hmm_do_fault(struct mm_struct *mm, + struct hmm_vma_walk *hmm_vma_walk) +{ + unsigned long addr = hmm_vma_walk->last; + unsigned long end = hmm_vma_walk->end; + unsigned int required_fault = hmm_vma_walk->required_fault; + unsigned int fault_flags = FAULT_FLAG_REMOTE; + struct vm_area_struct *vma; + + if (hmm_vma_walk->locked) + fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + + vma = vma_lookup(mm, addr); + if (!vma) + return -EFAULT; + + if (required_fault & HMM_NEED_WRITE_FAULT) { + if (!(vma->vm_flags & VM_WRITE)) + return -EPERM; + fault_flags |= FAULT_FLAG_WRITE; + } + + for (; addr < end; addr += PAGE_SIZE) { + vm_fault_t ret; + + ret = handle_mm_fault(vma, addr, fault_flags, NULL); + + if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) { + *hmm_vma_walk->locked = false; + return HMM_FAULT_UNLOCKED; + } + + if (ret & VM_FAULT_ERROR) { + int err = vm_fault_to_errno(ret, 0); + + if (WARN_ON(!err)) + err = -EINVAL; + + return err; + } + } + + return -EBUSY; +} + +static int hmm_range_fault_locked(struct hmm_range *range, bool *locked) { struct hmm_vma_walk hmm_vma_walk = { .range = range, + .locked = locked, .last = range->start, }; struct mm_struct *mm = range->notifier->mm; @@ -675,6 +706,22 @@ int hmm_range_fault(struct hmm_range *range) ret = walk_page_range(mm, hmm_vma_walk.last, range->end, &hmm_walk_ops, &hmm_vma_walk); /* + * When HMM_FAULT_PENDING is returned a walk callback + * recorded a range that needs handle_mm_fault(); + * hmm_do_fault() runs the fault outside walk_page_range() + * (so no page-table or hugetlb_vma_lock is held) and + * returns -EBUSY so the loop re-walks and picks up the + * now-present entries. + */ + if (ret == HMM_FAULT_PENDING) { + ret = hmm_do_fault(mm, &hmm_vma_walk); + if (ret == HMM_FAULT_UNLOCKED) { + if (fatal_signal_pending(current)) + return -EINTR; + return -EBUSY; + } + } + /* * When -EBUSY is returned the loop restarts with * hmm_vma_walk.last set to an address that has not been stored * in pfns. All entries < last in the pfn array are set to their @@ -683,9 +730,98 @@ int hmm_range_fault(struct hmm_range *range) } while (ret == -EBUSY); return ret; } + +/** + * hmm_range_fault - try to fault some address in a virtual address range + * @range: argument structure + * + * Returns 0 on success or one of the following error codes: + * + * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma + * (e.g., device file vma). + * -ENOMEM: Out of memory. + * -EPERM: Invalid permission (e.g., asking for write and range is read + * only). + * -EBUSY: The range has been invalidated and the caller needs to wait for + * the invalidation to finish. + * -EFAULT: A page was requested to be valid and could not be made valid + * ie it has no backing VMA or it is illegal to access + * + * This is similar to get_user_pages(), except that it can read the page tables + * without mutating them (ie causing faults). + * + * The mmap lock must be held by the caller and will remain held on return. + * New users should prefer hmm_range_fault_unlocked_timeout() unless they + * specifically need to keep the mmap lock held across the call. This helper + * cannot support VMAs whose fault handlers need to drop the mmap lock. + */ +int hmm_range_fault(struct hmm_range *range) +{ + return hmm_range_fault_locked(range, NULL); +} EXPORT_SYMBOL(hmm_range_fault); /** + * hmm_range_fault_unlocked_timeout - fault in a range with a retry timeout + * @range: argument structure + * @timeout: timeout in jiffies for internal -EBUSY retries, or 0 to retry + * indefinitely + * + * The caller must not hold the mmap lock. The function takes the mmap read + * lock internally and allows handle_mm_fault() to drop it during faults. If + * the mmap lock is dropped or the range is invalidated, the function refreshes + * range->notifier_seq and restarts the walk internally. + * + * Passing 0 for @timeout retries indefinitely. A non-zero @timeout is a caller + * policy limit for repeated mmu-notifier invalidation retries. HMM does not + * interrupt page fault handling when the timeout expires, but returns -EBUSY + * if the retry budget is exhausted before a stable range is obtained. + * + * Returns 0 on success or one of the error codes documented for + * hmm_range_fault(). -EINTR is returned if mmap_lock acquisition is + * interrupted or a fatal signal is pending during retry handling. + */ +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout) +{ + struct mm_struct *mm = range->notifier->mm; + unsigned long deadline = 0; + bool locked = false; + int ret; + + do { + /* + * If the previous fault dropped mmap_lock, then the fault + * handler made progress. Restart the retry timeout in that + * case, but keep the existing deadline for ordinary -EBUSY + * retries. + */ + if (timeout && !locked) + deadline = jiffies + timeout; + + range->notifier_seq = + mmu_interval_read_begin(range->notifier); + + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + + if (timeout && time_after(jiffies, deadline)) { + mmap_read_unlock(mm); + return -EBUSY; + } + + locked = true; + ret = hmm_range_fault_locked(range, &locked); + if (locked) + mmap_read_unlock(mm); + } while (ret == -EBUSY); + + return ret; +} +EXPORT_SYMBOL(hmm_range_fault_unlocked_timeout); + +/** * hmm_dma_map_alloc - Allocate HMM map structure * @dev: device to allocate structure for * @map: HMM map to allocate diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 58cabe6af33d..04e8a6b55343 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -180,7 +180,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ if (!in_pf && shmem_file(vma->vm_file)) return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), - vma, vma->vm_pgoff, 0, + vma, vma_start_pgoff(vma), 0, forced_collapse); if (!vma_is_anonymous(vma)) { @@ -818,10 +818,8 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent) ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, "hugepages-%lukB", size); - if (ret) { - kfree(thpsize); - goto err; - } + if (ret) + goto err_put; ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); @@ -1196,7 +1194,7 @@ static inline bool is_transparent_hugepage(const struct folio *folio) static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, loff_t off, unsigned long flags, unsigned long size, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { loff_t off_end = off + len; loff_t off_align = round_up(off, size); @@ -1212,8 +1210,9 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, if (len_pad < len || (off + len_pad) < off) return 0; - ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad, - off >> PAGE_SHIFT, flags, vm_flags); + ret = mm_get_unmapped_area_vmaflags(filp, addr, len_pad, + off >> PAGE_SHIFT, flags, + vma_flags); /* * The failure might be due to length padding. The caller will retry @@ -1238,25 +1237,27 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, return ret; } -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { unsigned long ret; loff_t off = (loff_t)pgoff << PAGE_SHIFT; - ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); + ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, + vma_flags); if (ret) return ret; - return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, - vm_flags); + return mm_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + vma_flags); } unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); + return thp_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL_GPL(thp_get_unmapped_area); @@ -1819,11 +1820,11 @@ static void copy_huge_non_present_pmd( if (softleaf_is_migration_write(entry) || softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } else if (softleaf_is_device_private(entry)) { /* @@ -1832,12 +1833,12 @@ static void copy_huge_non_present_pmd( */ if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } @@ -1856,8 +1857,8 @@ static void copy_huge_non_present_pmd( add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); - if (!userfaultfd_wp(dst_vma)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (!userfaultfd_protected(dst_vma)) + pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); } @@ -1951,9 +1952,15 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, out_zero_page: mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); + + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd)) + pmd = pmd_modify(pmd, dst_vma->vm_page_prot); + pmd = pmd_clear_uffd(pmd); + } + pmdp_set_wrprotect(src_mm, addr, src_pmd); - if (!userfaultfd_wp(dst_vma)) - pmd = pmd_clear_uffd_wp(pmd); pmd = pmd_wrprotect(pmd); set_pmd: pmd = pmd_mkold(pmd); @@ -2196,6 +2203,34 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, return pmd_dirty(pmd); } +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + pmd_t pmd; + + if (!userfaultfd_rwp_async(vma)) + return handle_userfault(vmf, VM_UFFD_RWP); + + vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { + spin_unlock(vmf->ptl); + return 0; + } + pmd = pmd_modify(vmf->orig_pmd, vma->vm_page_prot); + /* pmd_modify() preserves _PAGE_UFFD; drop it on resolution */ + pmd = pmd_clear_uffd(pmd); + pmd = pmd_mkyoung(pmd); + if (!pmd_write(pmd) && + vma_wants_manual_pte_write_upgrade(vma) && + can_change_pmd_writable(vma, vmf->address, pmd)) + pmd = pmd_mkwrite(pmd, vma); + set_pmd_at(vma->vm_mm, vmf->address & HPAGE_PMD_MASK, + vmf->pmd, pmd); + update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); + spin_unlock(vmf->ptl); + return 0; +} + /* NUMA hinting page fault entry point for trans huge pmds */ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { @@ -2297,8 +2332,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto out; } @@ -2494,9 +2529,9 @@ static pmd_t clear_uffd_wp_pmd(pmd_t pmd) if (pmd_none(pmd)) return pmd; if (pmd_present(pmd)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); else - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); return pmd; } @@ -2539,8 +2574,19 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pgtable_trans_huge_deposit(mm, new_pmd, pgtable); } pmd = move_soft_dirty_pmd(pmd); - if (vma_has_uffd_without_event_remap(vma)) + if (vma_has_uffd_without_event_remap(vma)) { + /* + * See __copy_present_ptes(): normalise the RWP marker + * so the destination starts accessible instead of + * taking a numa-hinting fault on first access. Only the + * marker (protnone + uffd) needs it; leave other present + * PMDs in the VMA untouched. + */ + if (pmd_present(pmd) && userfaultfd_rwp(vma) && + pmd_uffd(pmd)) + pmd = pmd_modify(pmd, vma->vm_page_prot); pmd = clear_uffd_wp_pmd(pmd); + } set_pmd_at(mm, new_addr, new_pmd, pmd); if (force_flush) flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); @@ -2553,15 +2599,16 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, } static void change_non_present_huge_pmd(struct mm_struct *mm, - unsigned long addr, pmd_t *pmd, bool uffd_wp, - bool uffd_wp_resolve) + unsigned long addr, pmd_t *pmd, bool uffd_prot, + bool uffd_prot_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); - const struct folio *folio = softleaf_to_folio(entry); pmd_t newpmd; VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); if (softleaf_is_migration_write(entry)) { + const struct folio *folio = softleaf_to_folio(entry); + /* * A protection check is difficult so * just be safe and disable write @@ -2570,22 +2617,22 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, entry = make_readable_exclusive_migration_entry(swp_offset(entry)); else entry = make_readable_migration_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pmd)) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); - if (pmd_swp_uffd_wp(*pmd)) - newpmd = pmd_swp_mkuffd_wp(newpmd); + newpmd = softleaf_to_pmd(entry); + if (pmd_swp_uffd(*pmd)) + newpmd = pmd_swp_mkuffd(newpmd); } else { newpmd = *pmd; } - if (uffd_wp) - newpmd = pmd_swp_mkuffd_wp(newpmd); - else if (uffd_wp_resolve) - newpmd = pmd_swp_clear_uffd_wp(newpmd); + if (uffd_prot) + newpmd = pmd_swp_mkuffd(newpmd); + else if (uffd_prot_resolve) + newpmd = pmd_swp_clear_uffd(newpmd); if (!pmd_same(*pmd, newpmd)) set_pmd_at(mm, addr, pmd, newpmd); } @@ -2605,8 +2652,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, spinlock_t *ptl; pmd_t oldpmd, entry; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; - bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); int ret = 1; tlb_change_page_size(tlb, HPAGE_PMD_SIZE); @@ -2619,11 +2667,17 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, return 0; if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) { - change_non_present_huge_pmd(mm, addr, pmd, uffd_wp, - uffd_wp_resolve); + change_non_present_huge_pmd(mm, addr, pmd, uffd_prot, + uffd_prot_resolve); goto unlock; } + /* Already in the desired state */ + if (prot_numa && pmd_protnone(*pmd)) + goto unlock; + if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd)) + goto unlock; + if (prot_numa) { /* @@ -2634,9 +2688,6 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (is_huge_zero_pmd(*pmd)) goto unlock; - if (pmd_protnone(*pmd)) - goto unlock; - if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, vma_is_single_threaded_private(vma))) goto unlock; @@ -2665,15 +2716,19 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, oldpmd = pmdp_invalidate_ad(vma, addr, pmd); entry = pmd_modify(oldpmd, newprot); - if (uffd_wp) - entry = pmd_mkuffd_wp(entry); - else if (uffd_wp_resolve) + if (uffd_prot) + entry = pmd_mkuffd(entry); + else if (uffd_prot_resolve) /* * Leave the write bit to be handled by PF interrupt * handler, then things like COW could be properly * handled. */ - entry = pmd_clear_uffd_wp(entry); + entry = pmd_clear_uffd(entry); + + /* See change_pte_range(): preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && pmd_uffd(entry)) + entry = pmd_modify(entry, PAGE_NONE); /* See change_pte_range(). */ if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && @@ -2713,10 +2768,10 @@ int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, return 1; /* - * Huge entries on userfault-wp only works with anonymous, while we - * don't have anonymous PUDs yet. + * Huge entries on userfault-wp or userfault-rwp only work with + * anonymous, while we don't have anonymous PUDs yet. */ - if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) + if (WARN_ON_ONCE(cp_flags & (MM_CP_UFFD_WP_ALL | MM_CP_UFFD_RWP_ALL))) return 1; ptl = __pud_trans_huge_lock(pudp, vma); @@ -2832,7 +2887,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_folio_page_index(src_folio, dst_vma, + dst_addr); _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ @@ -2842,6 +2898,13 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm _dst_pmd = move_soft_dirty_pmd(src_pmdval); _dst_pmd = clear_uffd_wp_pmd(_dst_pmd); } + + /* Re-arm RWP on the moved PMD if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + _dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE); + _dst_pmd = pmd_mkuffd(_dst_pmd); + } + set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); @@ -3016,8 +3079,13 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, entry = pfn_pte(zero_pfn(addr), vma->vm_page_prot); entry = pte_mkspecial(entry); - if (pmd_uffd_wp(old_pmd)) - entry = pte_mkuffd_wp(entry); + if (pmd_uffd(old_pmd)) + entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && pmd_uffd(old_pmd)) + entry = pte_modify(entry, PAGE_NONE); + VM_BUG_ON(!pte_none(ptep_get(pte))); set_pte_at(mm, addr, pte, entry); pte++; @@ -3103,7 +3171,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_migration_write(entry); if (PageAnon(page)) @@ -3119,7 +3187,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_device_private_write(entry); anon_exclusive = PageAnonExclusive(page); @@ -3176,7 +3244,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); - uffd_wp = pmd_uffd_wp(old_pmd); + uffd_wp = pmd_uffd(old_pmd); VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); @@ -3247,7 +3315,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3274,7 +3342,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3292,7 +3360,11 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && uffd_wp) + entry = pte_modify(entry, PAGE_NONE); for (i = 0; i < HPAGE_PMD_NR; i++) VM_WARN_ON(!pte_none(ptep_get(pte + i))); @@ -3590,6 +3662,13 @@ static void __split_folio_to_order(struct folio *folio, int old_order, new_folio->mapping = folio->mapping; new_folio->index = folio->index + i; + /* + * page->private should not be set in tail pages. Warn once + * if private is unexpectedly set. Do it before swap.val assignment + * since private overlaps with swap.val. + */ + VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head); + if (folio_test_swapcache(folio)) new_folio->swap.val = folio->swap.val + i; @@ -4879,7 +4958,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -4903,7 +4982,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = pmd_write(pmdval); softdirty = pmd_soft_dirty(pmdval); - uffd_wp = pmd_uffd_wp(pmdval); + uffd_wp = pmd_uffd(pmdval); } else { softleaf_t old_entry; @@ -4912,7 +4991,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = softleaf_is_device_private_write(old_entry); softdirty = pmd_swp_soft_dirty(pmdval); - uffd_wp = pmd_swp_uffd_wp(pmdval); + uffd_wp = pmd_swp_uffd(pmdval); } /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ @@ -4939,11 +5018,11 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } /* Set PMD. */ - pmdswp = swp_entry_to_pmd(entry); + pmdswp = softleaf_to_pmd(entry); if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) - pmdswp = pmd_swp_mkuffd_wp(pmdswp); + pmdswp = pmd_swp_mkuffd(pmdswp); set_pmd_at(mm, address, pvmw->pmd, pmdswp); /* Migration entry installed: cleanup rmap, folio. */ @@ -4975,8 +5054,13 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) pmde = pmd_mksoft_dirty(pmde); if (softleaf_is_migration_write(entry)) pmde = pmd_mkwrite(pmde, vma); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_mkuffd(pmde); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pmd_swp_uffd(*pvmw->pmd) && userfaultfd_rwp(vma)) + pmde = pmd_modify(pmde, PAGE_NONE); + if (!softleaf_is_migration_young(entry)) pmde = pmd_mkold(pmde); /* NOTE: this may contain setting soft-dirty on some archs */ @@ -4992,12 +5076,12 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) else entry = make_readable_device_private_entry( page_to_pfn(new)); - pmde = swp_entry_to_pmd(entry); + pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_swp_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_swp_mkuffd(pmde); } if (folio_test_anon(folio)) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 940b52ac17c4..d86a27c8819c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -47,9 +47,11 @@ #include <linux/node.h> #include <linux/page_owner.h> #include "internal.h" +#include "page_alloc.h" #include "hugetlb_vmemmap.h" #include "hugetlb_cma.h" #include "hugetlb_internal.h" +#include "mm_init.h" #include <linux/page-isolation.h> int hugetlb_max_hstate __read_mostly; @@ -58,7 +60,6 @@ struct hstate hstates[HUGE_MAX_HSTATE]; __initdata nodemask_t hugetlb_bootmem_nodes; __initdata struct list_head huge_boot_pages[MAX_NUMNODES]; -static unsigned long hstate_boot_nrinvalid[HUGE_MAX_HSTATE] __initdata; /* * Due to ordering constraints across the init code for various @@ -1011,8 +1012,7 @@ static long region_count(struct resv_map *resv, long f, long t) static pgoff_t vma_hugecache_offset(struct hstate *h, struct vm_area_struct *vma, unsigned long address) { - return ((address - vma->vm_start) >> huge_page_shift(h)) + - (vma->vm_pgoff >> huge_page_order(h)); + return linear_page_index(vma, address) >> huge_page_order(h); } /* @@ -1317,43 +1317,27 @@ static unsigned long available_huge_pages(struct hstate *h) return h->free_huge_pages - h->resv_huge_pages; } -static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, - struct vm_area_struct *vma, - unsigned long address, long gbl_chg) +static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, + struct mempolicy_interpreted *mpoli) { + nodemask_t *nodemask = mpoli->nodemask; struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask; - nodemask_t *nodemask; - int nid; - /* - * gbl_chg==1 means the allocation requires a new page that was not - * reserved before. Making sure there's at least one free page. - */ - if (gbl_chg && !available_huge_pages(h)) - goto err; - - gfp_mask = htlb_alloc_mask(h); - nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); - - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); + mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) + if (!folio) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); - - mpol_cond_put(mpol); + mpoli->nid, + nodemask); + } return folio; - -err: - return NULL; } #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC) @@ -1806,7 +1790,8 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask, if (alloc_try_hard) gfp_mask |= __GFP_RETRY_MAYFAIL; - folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask); + folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, + ALLOC_DEFAULT); /* * If we did not specify __GFP_RETRY_MAYFAIL, but still got a @@ -2160,32 +2145,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas return folio; } -/* - * Use the VMA's mpolicy to allocate a huge page from the buddy. - */ static -struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, - struct vm_area_struct *vma, unsigned long addr) +struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, + gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) { struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask = htlb_alloc_mask(h); - int nid; - nodemask_t *nodemask; + nodemask_t *nodemask = mpoli->nodemask; - nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); + folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) - folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); - mpol_cond_put(mpol); + if (!folio) { + folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid, + nodemask); + } + return folio; } @@ -2838,6 +2819,104 @@ void wait_for_freed_hugetlb_folios(void) flush_work(&free_hpage_work); } +/** + * hugetlb_alloc_folio - Allocate a hugetlb folio. + * @h: Hugetlb state control block. + * @mpoli: Interpreted memory policy to use for allocation. + * @alloc_flags: Flags controlling the allocation behavior. + * + * Allocates a hugetlb folio and handles cgroup charging and global hstate + * reservations. + * + * Return: A pointer to the allocated folio, or an ERR_PTR on failure. + * -ENOSPC if cgroup charging fails or no folio is available. + * -ENOMEM if mem cgroup charging fails. + */ +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags) +{ + bool charge_hugetlb_cgroup_rsvd = alloc_flags & + HUGETLB_ALLOC_CHARG_CGROUP_RSVD; + bool use_global_reservation = alloc_flags & + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; + size_t nr_pages = pages_per_huge_page(h); + struct hugetlb_cgroup *h_cg_rsvd = NULL; + struct hugetlb_cgroup *h_cg = NULL; + gfp_t gfp = htlb_alloc_mask(h); + int idx = hstate_index(h); + struct folio *folio; + int ret; + + if (charge_hugetlb_cgroup_rsvd && + hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd)) + return ERR_PTR(-ENOSPC); + + if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup_rsvd; + } + + spin_lock_irq(&hugetlb_lock); + + folio = NULL; + if (use_global_reservation || available_huge_pages(h)) + folio = dequeue_hugetlb_folio(h, gfp, mpoli); + + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli); + if (!folio) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup; + } + spin_lock_irq(&hugetlb_lock); + list_add(&folio->lru, &h->hugepage_activelist); + folio_ref_unfreeze(folio, 1); + } + + if (use_global_reservation) { + folio_set_hugetlb_restore_reserve(folio); + h->resv_huge_pages--; + } + + hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio); + + if (charge_hugetlb_cgroup_rsvd) { + hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd, + folio); + } + + spin_unlock_irq(&hugetlb_lock); + + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + /* + * Unconditionally increment NR_HUGETLB here because if + * mem_cgroup_charge_hugetlb failed, freeing the page will + * decrement NR_HUGETLB. + */ + lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages); + + if (ret == -ENOMEM) { + free_huge_folio(folio); + /* + * Skip uncharging hugetlb_cgroup since the charges + * were committed to the folio and freeing the folio + * would have cleared those up. + */ + return ERR_PTR(ret); + } + + return folio; + + err_uncharge_hugetlb_cgroup: + hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg); + err_uncharge_hugetlb_cgroup_rsvd: + if (charge_hugetlb_cgroup_rsvd) + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd); + + return ERR_PTR(ret); +} + typedef enum { /* * For either 0/1: we checked the per-vma resv map, and one resv @@ -2872,12 +2951,13 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *folio; long retval, gbl_chg, gbl_reserve; map_chg_state map_chg; - int ret, idx; - struct hugetlb_cgroup *h_cg = NULL; - struct hugetlb_cgroup *h_cg_rsvd = NULL; - gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; - - idx = hstate_index(h); + struct mempolicy_interpreted mpoli; + gfp_t gfp = htlb_alloc_mask(h); + struct mempolicy *mpol; + nodemask_t *nodemask; + u8 alloc_flags = 0; + int nid; + int ret; /* Whether we need a separate per-vma reservation? */ if (cow_from_owner) { @@ -2909,8 +2989,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ if (map_chg) { gbl_chg = hugepage_subpool_get_pages(spool, 1); - if (gbl_chg < 0) + if (gbl_chg < 0) { + ret = -ENOSPC; goto out_end_reservation; + } } else { /* * If we have the vma reservation ready, no need for extra @@ -2920,57 +3002,39 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } /* - * If this allocation is not consuming a per-vma reservation, - * charge the hugetlb cgroup now. + * If allocation doesn't reuse a reservation in the resv_map, + * charge for the reservation. */ - if (map_chg) { - ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg_rsvd); - if (ret) - goto out_subpool_put; - } - - ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_uncharge_cgroup_reservation; + if (map_chg != MAP_CHG_REUSE) + alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD; - spin_lock_irq(&hugetlb_lock); /* - * glb_chg is passed to indicate whether or not a page must be taken - * from the global free pool (global change). gbl_chg == 0 indicates - * a reservation exists for the allocation. + * gbl_chg == 0 indicates a reservation exists for this + * allocation, so try to use it. */ - folio = dequeue_hugetlb_folio_vma(h, vma, addr, gbl_chg); - if (!folio) { - spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); - if (!folio) - goto out_uncharge_cgroup; - spin_lock_irq(&hugetlb_lock); - list_add(&folio->lru, &h->hugepage_activelist); - folio_ref_unfreeze(folio, 1); - /* Fall through */ - } + if (gbl_chg == 0) + alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; - /* - * Either dequeued or buddy-allocated folio needs to add special - * mark to the folio when it consumes a global reservation. - */ - if (!gbl_chg) { - folio_set_hugetlb_restore_reserve(folio); - h->resv_huge_pages--; - } + /* Takes reference on mpol. */ + nid = huge_node(vma, addr, gfp, &mpol, &nodemask); + mpoli = (struct mempolicy_interpreted){ + .nid = nid, +#ifdef CONFIG_NUMA + .mode = mpol ? mpol->mode : MPOL_DEFAULT, +#else + .mode = MPOL_DEFAULT, +#endif + .nodemask = nodemask, + }; - hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); - /* If allocation is not consuming a reservation, also store the - * hugetlb_cgroup pointer on the page. - */ - if (map_chg) { - hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd, folio); - } + folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags); - spin_unlock_irq(&hugetlb_lock); + mpol_cond_put(mpol); + + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_subpool_put; + } hugetlb_set_folio_subpool(folio, spool); @@ -2999,27 +3063,8 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } } - ret = mem_cgroup_charge_hugetlb(folio, gfp); - /* - * Unconditionally increment NR_HUGETLB here. If it turns out that - * mem_cgroup_charge_hugetlb failed, then immediately free the page and - * decrement NR_HUGETLB. - */ - lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h)); - - if (ret == -ENOMEM) { - free_huge_folio(folio); - return ERR_PTR(-ENOMEM); - } - return folio; -out_uncharge_cgroup: - hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); -out_uncharge_cgroup_reservation: - if (map_chg) - hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd); out_subpool_put: /* * put page to subpool iff the quota of subpool's rsv_hpages is used @@ -3030,100 +3075,94 @@ out_subpool_put: hugetlb_acct_memory(h, -gbl_reserve); } - out_end_reservation: if (map_chg != MAP_CHG_ENFORCED) vma_end_reservation(h, vma, addr); - return ERR_PTR(-ENOSPC); + return ERR_PTR(ret); } static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) { - struct huge_bootmem_page *m; - int listnode = nid; - if (hugetlb_early_cma(h)) - m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); - else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), + return hugetlb_cma_alloc_bootmem(h, nid, node_exact); + + if (node_exact) + return memblock_alloc_exact_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), + + return memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - /* - * For pre-HVO to work correctly, pages need to be on - * the list for the node they were actually allocated - * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, - * extract the actual node first. - */ - if (m) - listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); - } - - if (m) { - m->flags = 0; - m->cma = NULL; - } - } - - if (m) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - * - * Put them into a private list first because mem_map - * is not up yet. - */ - INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[listnode]); - m->hstate = h; - } - - return m; } -int alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node = nid; /* do node specific alloc */ - if (nid != NUMA_NO_NODE) { - m = alloc_bootmem(h, node, true); - if (!m) - return 0; - goto found; - } + if (nid != NUMA_NO_NODE) + return alloc_bootmem(h, node, true); /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, - &hugetlb_bootmem_nodes) { - m = alloc_bootmem(h, node, false); - if (!m) - return 0; - goto found; - } + &hugetlb_bootmem_nodes) + return alloc_bootmem(h, node, false); -found: + return NULL; +} +static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) +{ + unsigned long pfn; + unsigned int nid_request = nid; + struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); + + if (!m) + return false; + + pfn = PHYS_PFN(__pa(m)); + nid = early_pfn_to_nid(pfn); /* - * Only initialize the head struct page in memmap_init_reserved_pages, - * rest of the struct pages will be initialized by the HugeTLB - * subsystem itself. - * The head struct page is used to get folio information by the HugeTLB - * subsystem like zone id and node id. + * Use the beginning of the huge page to store the huge_bootmem_page + * struct (until gather_bootmem puts them into the mem_map). + * + * Put them into a private list first because mem_map is not up yet. */ - memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), - huge_page_size(h) - PAGE_SIZE); + INIT_LIST_HEAD(&m->list); + m->hstate = h; + m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0; - return 1; + /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ + if (!hugetlb_early_cma(h) && + pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) { + /* + * If the allocated page is on a different node than requested + * (e.g., on PowerPC LPARs), put it on the requested node's list, + * because hugetlb_free_cross_zone_pages() only frees cross-zone + * pages belonging to the requested node. + */ + if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request)) + list_add(&m->list, &huge_boot_pages[nid_request]); + else + list_add(&m->list, &huge_boot_pages[nid]); + } else { + list_add_tail(&m->list, &huge_boot_pages[nid]); + m->flags |= HUGE_BOOTMEM_ZONES_VALID; + /* + * Only initialize the head struct page in memmap_init_reserved_pages, + * rest of the struct pages will be initialized by the HugeTLB + * subsystem itself. + * The head struct page is used to get folio information by the HugeTLB + * subsystem like zone id and node id. + */ + memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), + huge_page_size(h) - PAGE_SIZE); + } + + return true; } /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */ @@ -3235,57 +3274,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, } } -bool __init hugetlb_bootmem_page_zones_valid(int nid, - struct huge_bootmem_page *m) -{ - unsigned long start_pfn; - bool valid; - - if (m->flags & HUGE_BOOTMEM_ZONES_VALID) { - /* - * Already validated, skip check. - */ - return true; - } - - if (hugetlb_bootmem_page_earlycma(m)) { - valid = cma_validate_zones(m->cma); - goto out; - } - - start_pfn = virt_to_phys(m) >> PAGE_SHIFT; - - valid = !pfn_range_intersects_zones(nid, start_pfn, - pages_per_huge_page(m->hstate)); -out: - if (!valid) - hstate_boot_nrinvalid[hstate_index(m->hstate)]++; - - return valid; -} - -/* - * Free a bootmem page that was found to be invalid (intersecting with - * multiple zones). - * - * Since it intersects with multiple zones, we can't just do a free - * operation on all pages at once, but instead have to walk all - * pages, freeing them one by one. - */ -static void __init hugetlb_bootmem_free_invalid_page(int nid, struct page *page, - struct hstate *h) -{ - unsigned long npages = pages_per_huge_page(h); - unsigned long pfn; - - while (npages--) { - pfn = page_to_pfn(page); - __init_page_from_nid(pfn, nid); - free_reserved_page(page); - page++; - } -} - /* * Put bootmem huge pages into the standard lists after mem_map is up. * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages. @@ -3301,17 +3289,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) struct folio *folio = (void *)page; h = m->hstate; - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Can't use this page. Initialize the - * page structures if that hasn't already - * been done, and give them to the page - * allocator. - */ - hugetlb_bootmem_free_invalid_page(nid, page, h); - continue; - } - /* * It is possible to have multiple huge page sizes (hstates) * in this list. If so, process each size separately. @@ -3365,7 +3342,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start, gather_bootmem_prealloc_node(nid); } -static void __init gather_bootmem_prealloc(void) +void __init hugetlb_bootmem_struct_page_init(void) { struct padata_mt_job job = { .thread_fn = gather_bootmem_prealloc_parallel, @@ -3377,10 +3354,63 @@ static void __init gather_bootmem_prealloc(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + struct zone *zone; + + for_each_zone(zone) { + for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { + struct page *tail, *p; + unsigned int order; + + tail = zone->vmemmap_tails[i]; + if (!tail) + continue; + + order = i + VMEMMAP_TAIL_MIN_ORDER; + p = page_to_virt(tail); + /* + * prep_and_add_bootmem_folios() can access pageblock + * flags on bootmem HugeTLB pages, so initialize the + * shared tail struct pages here before bootmem folios + * start using them. + */ + for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) + init_compound_tail(p + j, NULL, order, zone); + } + } +#endif padata_do_multithreaded(&job); } +static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid) +{ + unsigned long freed = 0; + struct huge_bootmem_page *m, *tmp; + + if (!hstate_is_gigantic(h)) + return freed; + + list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) { + if (m->flags & HUGE_BOOTMEM_ZONES_VALID) + break; + + list_del(&m->list); + memblock_free(m, huge_page_size(h)); + freed++; + } + + if (freed) { + char buf[32]; + + string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf)); + pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n", + freed, buf, nid); + } + + return freed; +} + static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) { unsigned long i; @@ -3411,6 +3441,8 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) cond_resched(); } + i -= hugetlb_free_cross_zone_pages(h, nid); + if (!list_empty(&folio_list)) prep_and_add_allocated_folios(h, &folio_list); @@ -3484,6 +3516,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) { + int nid; unsigned long i; for (i = 0; i < h->max_huge_pages; ++i) { @@ -3492,6 +3525,9 @@ static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) cond_resched(); } + for_each_node(nid) + i -= hugetlb_free_cross_zone_pages(h, nid); + return i; } @@ -3569,7 +3605,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) * - For gigantic pages, this is called early in the boot process and * pages are allocated from memblock allocated or something similar. * Gigantic pages are actually added to pools later with the routine - * gather_bootmem_prealloc. + * hugetlb_bootmem_struct_page_init. * - For non-gigantic pages, this is called later in the boot process after * all of mm is up and functional. Pages are allocated from buddy and * then added to hugetlb pools. @@ -3647,20 +3683,13 @@ static void __init hugetlb_init_hstates(void) static void __init report_hugepages(void) { struct hstate *h; - unsigned long nrinvalid; for_each_hstate(h) { char buf[32]; - nrinvalid = hstate_boot_nrinvalid[hstate_index(h)]; - h->max_huge_pages -= nrinvalid; - string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n", buf, h->nr_huge_pages); - if (nrinvalid) - pr_info("HugeTLB: %s page size: %lu invalid page%s discarded\n", - buf, nrinvalid, str_plural(nrinvalid)); pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n", hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf); } @@ -4139,7 +4168,6 @@ static int __init hugetlb_init(void) } hugetlb_init_hstates(); - gather_bootmem_prealloc(); report_hugepages(); hugetlb_sysfs_init(); @@ -4855,8 +4883,16 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add __folio_mark_uptodate(new_folio); hugetlb_add_new_anon_rmap(new_folio, vma, addr); - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old)) - newpte = huge_pte_mkuffd_wp(newpte); + if (userfaultfd_protected(vma) && huge_pte_uffd(old)) { + newpte = huge_pte_mkuffd(newpte); + /* Restore PAGE_NONE so the RWP marker keeps trapping. */ + if (userfaultfd_rwp(vma)) { + unsigned int shift = huge_page_shift(hstate_vma(vma)); + + newpte = huge_pte_modify(newpte, PAGE_NONE); + newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags); + } + } set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz); hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); folio_set_hugetlb_migratable(new_folio); @@ -4937,7 +4973,7 @@ again: */ set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(softleaf_is_migration(softleaf))) { - bool uffd_wp = pte_swp_uffd_wp(entry); + bool uffd = pte_swp_uffd(entry); if (!softleaf_is_migration_read(softleaf) && cow) { /* @@ -4947,12 +4983,12 @@ again: softleaf = make_readable_migration_entry( swp_offset(softleaf)); entry = swp_entry_to_pte(softleaf); - if (userfaultfd_wp(src_vma) && uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + if (userfaultfd_protected(src_vma) && uffd) + entry = pte_swp_mkuffd(entry); set_huge_pte_at(src, addr, src_pte, entry, sz); } - if (!userfaultfd_wp(dst_vma)) - entry = pte_swp_clear_uffd_wp(entry); + if (!userfaultfd_protected(dst_vma)) + entry = pte_swp_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(pte_is_marker(entry))) { const pte_marker marker = copy_pte_marker(softleaf, dst_vma); @@ -5016,6 +5052,16 @@ again: goto next; } + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) { + entry = huge_pte_modify(entry, dst_vma->vm_page_prot); + entry = arch_make_huge_pte(entry, huge_page_shift(h), + dst_vma->vm_flags); + } + entry = huge_pte_clear_uffd(entry); + } + if (cow) { /* * No need to notify as we are downgrading page @@ -5028,9 +5074,6 @@ again: entry = huge_pte_wrprotect(entry); } - if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); - set_huge_pte_at(dst, addr, dst_pte, entry, sz); hugetlb_count_add(npages, dst); } @@ -5076,10 +5119,23 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, huge_pte_clear(mm, new_addr, dst_pte, sz); } else { if (need_clear_uffd_wp) { - if (pte_present(pte)) - pte = huge_pte_clear_uffd_wp(pte); - else - pte = pte_swp_clear_uffd_wp(pte); + if (pte_present(pte)) { + /* + * See __copy_present_ptes(): normalise the RWP + * marker so the destination starts accessible + * instead of taking a numa-hinting fault on + * first access. Only the marker (protnone + uffd) + * needs it; leave other present PTEs untouched. + */ + if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) { + pte = huge_pte_modify(pte, vma->vm_page_prot); + pte = arch_make_huge_pte(pte, huge_page_shift(h), + vma->vm_flags); + } + pte = huge_pte_clear_uffd(pte); + } else { + pte = pte_swp_clear_uffd(pte); + } } set_huge_pte_at(mm, new_addr, dst_pte, pte, sz); } @@ -5213,7 +5269,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, * drop the uffd-wp bit in this zap, then replace the * pte with a marker. */ - if (pte_swp_uffd_wp_any(pte) && + if (pte_swp_uffd_any(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5249,7 +5305,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, if (huge_pte_dirty(pte)) folio_mark_dirty(folio); /* Leave a uffd-wp pte marker if needed */ - if (huge_pte_uffd_wp(pte) && + if (huge_pte_uffd(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5388,8 +5444,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * from page cache lookup which is in HPAGE_SIZE units. */ address = address & huge_page_mask(h); - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; + pgoff = linear_page_index(vma, address); mapping = vma->vm_file->f_mapping; /* @@ -5398,7 +5453,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * __unmap_hugepage_range() is called as the lock is already held */ i_mmap_lock_write(mapping); - vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) { /* Do not unmap the current VMA */ if (iter_vma == vma) continue; @@ -5453,7 +5508,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf) * can trigger this, because hugetlb_fault() will always resolve * uffd-wp bit first. */ - if (!unshare && huge_pte_uffd_wp(pte)) + if (!unshare && huge_pte_uffd(pte)) return 0; /* Let's take out MAP_SHARED mappings first. */ @@ -5597,8 +5652,8 @@ retry_avoidcopy: huge_ptep_clear_flush(vma, vmf->address, vmf->pte); hugetlb_remove_rmap(old_folio); hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address); - if (huge_pte_uffd_wp(pte)) - newpte = huge_pte_mkuffd_wp(newpte); + if (huge_pte_uffd(pte)) + newpte = huge_pte_mkuffd(newpte); set_huge_pte_at(mm, vmf->address, vmf->pte, newpte, huge_page_size(h)); folio_set_hugetlb_migratable(new_folio); @@ -5876,7 +5931,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, * if populated. */ if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte))) - new_pte = huge_pte_mkuffd_wp(new_pte); + new_pte = huge_pte_mkuffd(new_pte); set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h)); hugetlb_count_add(pages_per_huge_page(h), mm); @@ -6052,6 +6107,47 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, } /* + * Protnone hugetlb PTEs with the uffd bit are used by + * userfaultfd RWP for access tracking. Plain PROT_NONE (without the + * marker) is not an RWP fault and is not expected on hugetlb (no + * NUMA hinting), so let normal hugetlb fault handling proceed. + */ + if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) && + userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) { + spinlock_t *ptl; + pte_t pte; + + /* Sync: drop hugetlb locks before blocking in handle_userfault() */ + if (!userfaultfd_rwp_async(vma)) + return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP); + + ptl = huge_pte_lock(h, mm, vmf.pte); + pte = huge_ptep_get(mm, vmf.address, vmf.pte); + if (pte_protnone(pte) && huge_pte_uffd(pte)) { + unsigned int shift = huge_page_shift(h); + + pte = huge_pte_modify(pte, vma->vm_page_prot); + pte = arch_make_huge_pte(pte, shift, vma->vm_flags); + /* huge_pte_modify() preserves _PAGE_UFFD; drop it on resolution */ + pte = huge_pte_clear_uffd(pte); + pte = pte_mkyoung(pte); + /* + * Unlike do_uffd_rwp(), do not upgrade to writable + * here. Hugetlb lacks a can_change_huge_pte_writable() + * equivalent, so a write access will take a separate + * COW fault — acceptable for the rare private hugetlb + * case. + */ + set_huge_pte_at(mm, vmf.address, vmf.pte, pte, + huge_page_size(h)); + update_mmu_cache(vma, vmf.address, vmf.pte); + } + spin_unlock(ptl); + ret = 0; + goto out_mutex; + } + + /* * If we are going to COW/unshare the mapping later, we examine the * pending reservations for this page now. This will ensure that any * allocations necessary to record that reservation occur outside the @@ -6074,7 +6170,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, goto out_ptl; /* Handle userfault-wp first, before trying to lock more pages */ - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(mm, vmf.address, vmf.pte)) && + if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) && (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) { if (!userfaultfd_wp_async(vma)) { spin_unlock(vmf.ptl); @@ -6083,7 +6179,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, return handle_userfault(&vmf, VM_UFFD_WP); } - vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte); + vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte); set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte, huge_page_size(hstate_vma(vma))); /* Fallthrough to CoW */ @@ -6368,7 +6464,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, _dst_pte = pte_mkyoung(_dst_pte); if (wp_enabled) - _dst_pte = huge_pte_mkuffd_wp(_dst_pte); + _dst_pte = huge_pte_mkuffd(_dst_pte); set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); @@ -6411,6 +6507,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long last_addr_mask; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP; + bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE; struct mmu_gather tlb; /* @@ -6436,6 +6534,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma, ptep = hugetlb_walk(vma, address, psize); if (!ptep) { + /* + * uffd_wp installs a pte marker on the unpopulated + * entry; uffd_rwp does not install markers so the + * allocation is unnecessary for it. + */ if (!uffd_wp) { address |= last_addr_mask; continue; @@ -6457,7 +6560,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma, * shouldn't happen at all. Warn about it if it * happened due to some reason. */ - WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); + WARN_ON_ONCE(uffd_wp || uffd_wp_resolve || + uffd_rwp || uffd_rwp_resolve); pages++; spin_unlock(ptl); address |= last_addr_mask; @@ -6491,10 +6595,10 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pages++; } - if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); - else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + if (uffd_wp || uffd_rwp) + newpte = pte_swp_mkuffd(newpte); + else if (uffd_wp_resolve || uffd_rwp_resolve) + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(pte, newpte)) set_huge_pte_at(mm, address, ptep, newpte, psize); } else if (unlikely(pte_is_marker(pte))) { @@ -6504,20 +6608,32 @@ long hugetlb_change_protection(struct vm_area_struct *vma, * pte_marker_uffd_wp()==true implies !poison * because they're mutual exclusive. */ - if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve) + if (pte_is_uffd_wp_marker(pte) && + (uffd_wp_resolve || uffd_rwp_resolve)) /* Safe to modify directly (non-present->none). */ huge_pte_clear(mm, address, ptep, psize); } else { pte_t old_pte; unsigned int shift = huge_page_shift(hstate_vma(vma)); + /* Already protnone with uffd bit set? Nothing to do. */ + if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte)) + goto next; + old_pte = huge_ptep_modify_prot_start(vma, address, ptep); pte = huge_pte_modify(old_pte, newprot); pte = arch_make_huge_pte(pte, shift, vma->vm_flags); - if (uffd_wp) - pte = huge_pte_mkuffd_wp(pte); - else if (uffd_wp_resolve) - pte = huge_pte_clear_uffd_wp(pte); + if (uffd_wp || uffd_rwp) + pte = huge_pte_mkuffd(pte); + else if (uffd_wp_resolve || uffd_rwp_resolve) + pte = huge_pte_clear_uffd(pte); + + /* Preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) { + pte = huge_pte_modify(pte, PAGE_NONE); + pte = arch_make_huge_pte(pte, shift, vma->vm_flags); + } + huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); @@ -6787,7 +6903,7 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma, struct vm_area_struct *vma, unsigned long addr, pgoff_t idx) { - unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + + unsigned long saddr = ((idx - vma_start_pgoff(svma)) << PAGE_SHIFT) + svma->vm_start; unsigned long sbase = saddr & PUD_MASK; unsigned long s_end = sbase + PUD_SIZE; @@ -6872,15 +6988,14 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pud_t *pud) { struct address_space *mapping = vma->vm_file->f_mapping; - pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; + const pgoff_t idx = linear_page_index(vma, addr); struct vm_area_struct *svma; unsigned long saddr; pte_t *spte = NULL; pte_t *pte; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { + mapping_rmap_tree_foreach(svma, mapping, idx, idx) { if (svma == vma) continue; @@ -7198,7 +7313,8 @@ void folio_putback_hugetlb(struct folio *folio) folio_put(folio); } -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason) +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason) { struct hstate *h = folio_hstate(old_folio); @@ -7227,14 +7343,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re * There is no need to transfer the per-node surplus state * when we do not cross the node. */ - if (new_nid == old_nid) - return; - spin_lock_irq(&hugetlb_lock); - if (h->surplus_huge_pages_node[old_nid]) { - h->surplus_huge_pages_node[old_nid]--; - h->surplus_huge_pages_node[new_nid]++; + if (new_nid != old_nid) { + spin_lock_irq(&hugetlb_lock); + if (h->surplus_huge_pages_node[old_nid]) { + h->surplus_huge_pages_node[old_nid]--; + h->surplus_huge_pages_node[new_nid]++; + } + spin_unlock_irq(&hugetlb_lock); } - spin_unlock_irq(&hugetlb_lock); } /* diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 79dbd0baafa3..07faf625675b 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -9,6 +9,9 @@ #include <asm/setup.h> #include <linux/hugetlb.h> +#include <linux/memblock.h> +#include <linux/math.h> +#include <linux/math64.h> #include "internal.h" #include "hugetlb_cma.h" @@ -18,6 +21,28 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; static bool hugetlb_cma_only __ro_after_init; static unsigned long hugetlb_cma_size __ro_after_init; +static unsigned int hugetlb_cma_percent __initdata; +static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata; + +#ifdef CONFIG_NUMA +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + struct memblock_region *reg; + phys_addr_t size = 0; + + for_each_mem_region(reg) { + if (reg->nid == nid) + size += reg->size; + } + return size; +} +#else +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + return memblock_phys_mem_size(); +} +#endif + void hugetlb_cma_free_frozen_folio(struct folio *folio) { WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], @@ -56,37 +81,27 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, return folio; } -struct huge_bootmem_page * __init -hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) +void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; - struct huge_bootmem_page *m; - int node = *nid; + void *m; + int node; - cma = hugetlb_cma[*nid]; + cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); - if (!m) { - if (node_exact) - return NULL; - - for_each_node_mask(node, hugetlb_bootmem_nodes) { - cma = hugetlb_cma[node]; - if (!cma || node == *nid) - continue; - m = cma_reserve_early(cma, huge_page_size(h)); - if (m) { - *nid = node; - break; - } - } - } + if (m || node_exact) + return m; - if (m) { - m->flags = HUGE_BOOTMEM_CMA; - m->cma = cma; + for_each_node_mask(node, hugetlb_bootmem_nodes) { + cma = hugetlb_cma[node]; + if (!cma || node == nid) + continue; + m = cma_reserve_early(cma, huge_page_size(h)); + if (m) + return m; } - return m; + return NULL; } static int __init cmdline_parse_hugetlb_cma(char *p) @@ -100,14 +115,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) break; if (s[count] == ':') { + char *next; + if (tmp >= MAX_NUMNODES) break; nid = array_index_nospec(tmp, MAX_NUMNODES); s += count + 1; - tmp = memparse(s, &s); - hugetlb_cma_size_in_node[nid] = tmp; - hugetlb_cma_size += tmp; + tmp = memparse(s, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n", + tmp, nid); + break; + } + hugetlb_cma_percent_in_node[nid] = tmp; + hugetlb_cma_size_in_node[nid] = 0; + s = next + 1; + } else { + hugetlb_cma_size_in_node[nid] = tmp; + hugetlb_cma_percent_in_node[nid] = 0; + s = next; + } /* * Skip the separator if have one, otherwise @@ -118,7 +147,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) else break; } else { - hugetlb_cma_size = memparse(p, &p); + char *next; + + tmp = memparse(p, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp); + } else { + hugetlb_cma_percent = tmp; + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } + } else { + hugetlb_cma_size = tmp; + hugetlb_cma_percent = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } break; } } @@ -144,8 +194,36 @@ void __init hugetlb_cma_reserve(void) { unsigned long size, reserved, per_node, order, gigantic_page_size; bool node_specific_cma_alloc = false; + bool has_node_specific_param = false; int nid; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_size_in_node[nid] || hugetlb_cma_percent_in_node[nid]) { + has_node_specific_param = true; + break; + } + } + + if (has_node_specific_param) { + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_percent_in_node[nid]) { + phys_addr_t node_gfp_mem = memblock_node_memory_size(nid); + u64 s; + + s = mul_u64_u32_div((u64)node_gfp_mem, + hugetlb_cma_percent_in_node[nid], + 100); + + hugetlb_cma_size_in_node[nid] = s; + } + hugetlb_cma_size += hugetlb_cma_size_in_node[nid]; + } + } else if (hugetlb_cma_percent) { + hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(), + hugetlb_cma_percent, 100); + } + if (!hugetlb_cma_size) return; @@ -231,9 +309,11 @@ void __init hugetlb_cma_reserve(void) res = cma_declare_contiguous_multi(size, gigantic_page_size, HUGETLB_PAGE_ORDER, name, &hugetlb_cma[nid], nid); - if (res) { - pr_warn("hugetlb_cma: reservation failed: err %d, node %d", + if (res || !cma_validate_zones(hugetlb_cma[nid])) { + pr_warn("hugetlb_cma: %s: err %d, node %d\n", + res ? "reservation failed" : "reserved area spans zones", res, nid); + hugetlb_cma[nid] = NULL; continue; } diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index c619c394b1ae..3aa483573d17 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -6,8 +6,7 @@ void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int nid, nodemask_t *nodemask); -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact); +void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact); bool hugetlb_cma_exclusive_alloc(void); unsigned long hugetlb_cma_total_size(void); void hugetlb_cma_validate_params(void); @@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, return NULL; } -static inline -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact) +static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, + bool node_exact) { return NULL; } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 133b46dfb09f..917db0984143 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -12,7 +12,6 @@ #include <linux/pgtable.h> #include <linux/moduleparam.h> -#include <linux/bootmem_info.h> #include <linux/mmdebug.h> #include <linux/pagewalk.h> #include <linux/pgalloc.h> @@ -177,13 +176,13 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end, * Free a vmemmap page. A vmemmap page can be allocated from the memblock * allocator or buddy allocator. If the PG_reserved flag is set, it means * that it allocated from the memblock allocator, just free it via the - * free_bootmem_page(). Otherwise, use __free_page(). + * free_reserved_page(). Otherwise, use __free_page(). */ static inline void free_vmemmap_page(struct page *page) { if (PageReserved(page)) { memmap_boot_pages_add(-1); - free_bootmem_page(page); + free_reserved_page(page); } else { memmap_pages_add(-1); __free_page(page); @@ -624,6 +623,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, LIST_HEAD(vmemmap_pages); unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; + if (list_empty(folio_list)) + return; + nr_to_optimize = 0; list_for_each_entry(folio, folio_list, lru) { int ret; @@ -635,12 +637,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, * mirrored tail page structs RO. */ spfn = (unsigned long)&folio->page; - epfn = spfn + pages_per_huge_page(h); + epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(spfn), - &folio->page, - HUGETLB_VMEMMAP_RESERVE_SIZE); continue; } @@ -745,6 +744,20 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) return true; } +static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) +{ + struct zone *zone; + enum zone_type zone_type; + + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + zone = &NODE_DATA(nid)->node_zones[zone_type]; + if (zone_spans_pfn(zone, pfn)) + return zone; + } + + return NULL; +} + /* * Initialize memmap section for a gigantic page, HVO-style. */ @@ -752,6 +765,7 @@ void __init hugetlb_vmemmap_init_early(int nid) { unsigned long psize, paddr, section_size; unsigned long ns, i, pnum, pfn, nr_pages; + unsigned long start, end; struct huge_bootmem_page *m = NULL; void *map; @@ -761,6 +775,8 @@ void __init hugetlb_vmemmap_init_early(int nid) section_size = (1UL << PA_SECTION_SHIFT); list_for_each_entry(m, &huge_boot_pages[nid], list) { + struct zone *zone; + if (!vmemmap_should_optimize_bootmem_page(m)) continue; @@ -769,6 +785,14 @@ void __init hugetlb_vmemmap_init_early(int nid) paddr = virt_to_phys(m); pfn = PHYS_PFN(paddr); map = pfn_to_page(pfn); + start = (unsigned long)map; + end = start + hugetlb_vmemmap_size(m->hstate); + zone = pfn_to_zone(nid, pfn); + + if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), + zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) + panic("Failed to allocate memmap for HugeTLB page\n"); + memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE)); pnum = pfn_to_section_nr(pfn); ns = psize / section_size; @@ -783,78 +807,6 @@ void __init hugetlb_vmemmap_init_early(int nid) m->flags |= HUGE_BOOTMEM_HVO; } } - -static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) -{ - struct zone *zone; - enum zone_type zone_type; - - for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { - zone = &NODE_DATA(nid)->node_zones[zone_type]; - if (zone_spans_pfn(zone, pfn)) - return zone; - } - - return NULL; -} - -void __init hugetlb_vmemmap_init_late(int nid) -{ - struct huge_bootmem_page *m, *tm; - unsigned long phys, nr_pages, start, end; - unsigned long pfn, nr_mmap; - struct zone *zone = NULL; - struct hstate *h; - void *map; - - if (!READ_ONCE(vmemmap_optimize_enabled)) - return; - - list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) { - if (!(m->flags & HUGE_BOOTMEM_HVO)) - continue; - - phys = virt_to_phys(m); - h = m->hstate; - pfn = PHYS_PFN(phys); - nr_pages = pages_per_huge_page(h); - map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + nr_pages * sizeof(struct page); - - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Oops, the hugetlb page spans multiple zones. - * Remove it from the list, and populate it normally. - */ - list_del(&m->list); - - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - - memblock_phys_free(phys, huge_page_size(h)); - continue; - } - - if (!zone || !zone_spans_pfn(zone, pfn)) - zone = pfn_to_zone(nid, pfn); - if (WARN_ON_ONCE(!zone)) - continue; - - if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone, - HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) { - /* Fallback if HVO population fails */ - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - } else { - m->flags |= HUGE_BOOTMEM_ZONES_VALID; - nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE; - } - - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - } -} #endif static const struct ctl_table hugetlb_vmemmap_sysctls[] = { @@ -870,27 +822,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = { static int __init hugetlb_vmemmap_init(void) { const struct hstate *h; - struct zone *zone; /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */ BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES); - for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_TAIL_MIN_ORDER; - p = page_to_virt(tail); - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } - for_each_hstate(h) { if (hugetlb_vmemmap_optimizable(h)) { register_sysctl_init("vm", hugetlb_vmemmap_sysctls); diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 18b490825215..7ac49c52457d 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -29,7 +29,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); #ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT void hugetlb_vmemmap_init_early(int nid); -void hugetlb_vmemmap_init_late(int nid); #endif @@ -81,10 +80,6 @@ static inline void hugetlb_vmemmap_init_early(int nid) { } -static inline void hugetlb_vmemmap_init_late(int nid) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; diff --git a/mm/internal.h b/mm/internal.h index 181e79f1d6a2..41561fdeb56d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,80 @@ #include "vma.h" struct folio_batch; +struct hstate; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; +}; + +/* mm/workingset.c */ +bool workingset_test_recent(void *shadow, bool file, bool *workingset, + bool flush); +void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); +void *workingset_eviction(struct folio *folio, + struct mem_cgroup *target_memcg); +void workingset_refault(struct folio *folio, void *shadow); +void workingset_activation(struct folio *folio); + +/* mm/folio.c */ +void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, + unsigned int nr_io, unsigned int nr_rotated); +void lru_note_cost_refault(struct folio *folio); +void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); + +static inline bool folio_may_be_lru_cached(struct folio *folio) +{ + /* + * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. + * Holding small numbers of low-order mTHP folios in per-CPU LRU cache + * will be sensible, but nobody has implemented and tested that yet. + */ + return !folio_test_large(folio); +} + +static inline void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} + +void lru_cache_disable(void); +void lru_add_drain(void); +void lru_add_drain_cpu(int cpu); +void lru_add_drain_cpu_zone(struct zone *zone); +void folio_deactivate(struct folio *folio); +void folio_mark_lazyfree(struct folio *folio); + +/* mm/vmscan.c */ +unsigned long zone_reclaimable_pages(struct zone *zone); +unsigned long try_to_free_pages(struct zonelist *zonelist, int order, + gfp_t gfp_mask, const nodemask_t *mask); +unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx); + +#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) +#define MEMCG_RECLAIM_PROACTIVE (1 << 2) +#define MIN_SWAPPINESS 0 +#define MAX_SWAPPINESS 200 + +/* Just reclaim from anon folios in proactive memory reclaim */ +#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + +unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness); +unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, + gfp_t gfp_mask, bool noswap, + pg_data_t *pgdat, + unsigned long *nr_scanned); + +#ifdef CONFIG_NUMA +extern int sysctl_min_unmapped_ratio; +extern int sysctl_min_slab_ratio; +#endif /* * Maintains state across a page table move. The operation assumes both source @@ -166,6 +240,10 @@ static inline int mmap_file(struct file *file, struct vm_area_struct *vma) { int err = vfs_mmap(file, vma); + /* Hooks cannot mark themselves anonymous. */ + if (WARN_ON_ONCE(vma_is_anonymous(vma))) + err = -EINVAL; + if (likely(!err)) return 0; @@ -412,8 +490,8 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) new = pte_swp_mksoft_dirty(new); if (pte_swp_exclusive(pte)) new = pte_swp_mkexclusive(new); - if (pte_swp_uffd_wp(pte)) - new = pte_swp_mkuffd_wp(new); + if (pte_swp_uffd(pte)) + new = pte_swp_mkuffd(new); return new; } @@ -642,190 +720,16 @@ void set_recommended_min_free_kbytes(void); extern char * const zone_names[MAX_NR_ZONES]; -/* perform sanity checks on struct pages being allocated or freed */ -DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); - extern int min_free_kbytes; extern int defrag_mode; void setup_per_zone_wmarks(void); void calculate_min_free_kbytes(void); int __meminit init_per_zone_wmark_min(void); -void page_alloc_sysctl_init(void); - -/* - * Structure for holding the mostly immutable allocation parameters passed - * between functions involved in allocations, including the alloc_pages* - * family of functions. - * - * nodemask, migratetype and highest_zoneidx are initialized only once in - * __alloc_pages() and then never change. - * - * zonelist, preferred_zone and highest_zoneidx are set first in - * __alloc_pages() for the fast path, and might be later changed - * in __alloc_pages_slowpath(). All other functions pass the whole structure - * by a const pointer. - */ -struct alloc_context { - struct zonelist *zonelist; - nodemask_t *nodemask; - struct zoneref *preferred_zoneref; - int migratetype; - - /* - * highest_zoneidx represents highest usable zone index of - * the allocation request. Due to the nature of the zone, - * memory on lower zone than the highest_zoneidx will be - * protected by lowmem_reserve[highest_zoneidx]. - * - * highest_zoneidx is also used by reclaim/compaction to limit - * the target zone since higher zone than this index cannot be - * usable for this allocation request. - */ - enum zone_type highest_zoneidx; - bool spread_dirty_pages; -}; - -/* - * This function returns the order of a free page in the buddy system. In - * general, page_zone(page)->lock must be held by the caller to prevent the - * page from being allocated in parallel and returning garbage as the order. - * If a caller does not hold page_zone(page)->lock, it must guarantee that the - * page cannot be allocated or merged in parallel. Alternatively, it must - * handle invalid values gracefully, and use buddy_order_unsafe() below. - */ -static inline unsigned int buddy_order(struct page *page) -{ - /* PageBuddy() must be checked by the caller */ - return page_private(page); -} - -/* - * Like buddy_order(), but for callers who cannot afford to hold the zone lock. - * PageBuddy() should be checked first by the caller to minimize race window, - * and invalid values must be handled gracefully. - * - * READ_ONCE is used so that if the caller assigns the result into a local - * variable and e.g. tests it for valid range before using, the compiler cannot - * decide to remove the variable and inline the page_private(page) multiple - * times, potentially observing different values in the tests and the actual - * use of the result. - */ -#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) - -/* - * This function checks whether a page is free && is the buddy - * we can coalesce a page and its buddy if - * (a) the buddy is not in a hole (check before calling!) && - * (b) the buddy is in the buddy system && - * (c) a page and its buddy have the same order && - * (d) a page and its buddy are in the same zone. - * - * For recording whether a page is in the buddy system, we set PageBuddy. - * Setting, clearing, and testing PageBuddy is serialized by zone->lock. - * - * For recording page's order, we use page_private(page). - */ -static inline bool page_is_buddy(struct page *page, struct page *buddy, - unsigned int order) -{ - if (!page_is_guard(buddy) && !PageBuddy(buddy)) - return false; - - if (buddy_order(buddy) != order) - return false; - - /* - * zone check is done late to avoid uselessly calculating - * zone/node ids for pages that could never merge. - */ - if (page_zone_id(page) != page_zone_id(buddy)) - return false; - - VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); - - return true; -} - -/* - * Locate the struct page for both the matching buddy in our - * pair (buddy1) and the combined O(n+1) page they form (page). - * - * 1) Any buddy B1 will have an order O twin B2 which satisfies - * the following equation: - * B2 = B1 ^ (1 << O) - * For example, if the starting buddy (buddy2) is #8 its order - * 1 buddy is #10: - * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 - * - * 2) Any buddy B will have an order O+1 parent P which - * satisfies the following equation: - * P = B & ~(1 << O) - * - * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER - */ -static inline unsigned long -__find_buddy_pfn(unsigned long page_pfn, unsigned int order) -{ - return page_pfn ^ (1 << order); -} - -/* - * Find the buddy of @page and validate it. - * @page: The input page - * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the - * function is used in the performance-critical __free_one_page(). - * @order: The order of the page - * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to - * page_to_pfn(). - * - * The found buddy can be a non PageBuddy, out of @page's zone, or its order is - * not the same as @page. The validation is necessary before use it. - * - * Return: the found buddy page or NULL if not found. - */ -static inline struct page *find_buddy_page_pfn(struct page *page, - unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) -{ - unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); - struct page *buddy; - - buddy = page + (__buddy_pfn - pfn); - if (buddy_pfn) - *buddy_pfn = __buddy_pfn; - - if (page_is_buddy(page, buddy, order)) - return buddy; - return NULL; -} - -extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone); - -static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone) -{ - if (zone->contiguous) - return pfn_to_page(start_pfn); - - return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); -} - -void set_zone_contiguous(struct zone *zone); -bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, - unsigned long nr_pages); - -static inline void clear_zone_contiguous(struct zone *zone) -{ - zone->contiguous = false; -} extern int __isolate_free_page(struct page *page, unsigned int order); extern void __putback_isolated_page(struct page *page, unsigned int order, int mt); -extern void memblock_free_pages(unsigned long pfn, unsigned int order); -extern void __free_pages_core(struct page *page, unsigned int order, - enum meminit_context context); /* * This will have no effect, other than possibly generating a warning, if the @@ -895,7 +799,7 @@ static inline void prep_compound_tail(struct page *tail, { tail->mapping = TAIL_MAPPING; set_compound_head(tail, head, order); - set_page_private(tail, 0); + VM_WARN_ON_ONCE(tail->private); } static inline void init_compound_tail(struct page *tail, @@ -907,101 +811,6 @@ static inline void init_compound_tail(struct page *tail, prep_compound_tail(tail, head, order); } -void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); -extern bool free_pages_prepare(struct page *page, unsigned int order); - -extern int user_min_free_kbytes; - -struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid, - nodemask_t *); -#define __alloc_frozen_pages(...) \ - alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) -void free_frozen_pages(struct page *page, unsigned int order); -void free_unref_folios(struct folio_batch *fbatch); - -#ifdef CONFIG_NUMA -struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); -#else -static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) -{ - return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); -} -#endif - -#define alloc_frozen_pages(...) \ - alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) - -struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); -#define alloc_frozen_pages_nolock(...) \ - alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) -void free_frozen_pages_nolock(struct page *page, unsigned int order); - -extern void zone_pcp_reset(struct zone *zone); -extern void zone_pcp_disable(struct zone *zone); -extern void zone_pcp_enable(struct zone *zone); -extern void zone_pcp_init(struct zone *zone); - -extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, - int nid, bool exact_nid); - -void memmap_init_range(unsigned long, int, unsigned long, unsigned long, - unsigned long, enum meminit_context, struct vmem_altmap *, int, - bool); - -/* - * mm/sparse.c - */ -#ifdef CONFIG_SPARSEMEM -void sparse_init(void); -int sparse_index_init(unsigned long section_nr, int nid); - -static inline void sparse_init_one_section(struct mem_section *ms, - unsigned long pnum, struct page *mem_map, - struct mem_section_usage *usage, unsigned long flags) -{ - unsigned long coded_mem_map; - - BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); - - /* - * We encode the start PFN of the section into the mem_map such that - * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it - * from the page pointer to obtain the PFN. - */ - coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); - VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); - - ms->section_mem_map &= ~SECTION_MAP_MASK; - ms->section_mem_map |= coded_mem_map; - ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; - ms->usage = usage; -} - -static inline void __section_mark_present(struct mem_section *ms, - unsigned long section_nr) -{ - if (section_nr > __highest_present_section_nr) - __highest_present_section_nr = section_nr; - - ms->section_mem_map |= SECTION_MARKED_PRESENT; -} -#else -static inline void sparse_init(void) {} -#endif /* CONFIG_SPARSEMEM */ - -/* - * mm/sparse-vmemmap.c - */ -#ifdef CONFIG_SPARSEMEM_VMEMMAP -void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages); -#else -static inline void sparse_init_subsection_map(unsigned long pfn, - unsigned long nr_pages) -{ -} -#endif /* CONFIG_SPARSEMEM_VMEMMAP */ - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* @@ -1070,9 +879,6 @@ int isolate_migratepages_range(struct compact_control *cc, unsigned long low_pfn, unsigned long end_pfn); -/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ -void init_cma_reserved_pageblock(struct page *page); - #endif /* CONFIG_COMPACTION || CONFIG_CMA */ struct cma; @@ -1080,7 +886,6 @@ struct cma; #ifdef CONFIG_CMA bool cma_validate_zones(struct cma *cma); void *cma_reserve_early(struct cma *cma, unsigned long size); -void init_cma_pageblock(struct page *page); #else static inline bool cma_validate_zones(struct cma *cma) { @@ -1090,28 +895,8 @@ static inline void *cma_reserve_early(struct cma *cma, unsigned long size) { return NULL; } -static inline void init_cma_pageblock(struct page *page) -{ -} #endif -enum fallback_result { - /* Found suitable migratetype, *mt_out is valid. */ - FALLBACK_FOUND, - /* No fallback found in requested order. */ - FALLBACK_EMPTY, - /* Passed @claimable, but claiming whole block is a bad idea. */ - FALLBACK_NOCLAIM, -}; -enum fallback_result -find_suitable_fallback(struct free_area *area, unsigned int order, - int migratetype, bool claimable, int *mt_out); - -static inline bool free_area_empty(struct free_area *area, int migratetype) -{ - return list_empty(&area->free_list[migratetype]); -} - /* mm/util.c */ struct anon_vma *folio_anon_vma(const struct folio *folio); @@ -1143,26 +928,29 @@ static inline bool folio_within_range(struct folio *folio, struct vm_area_struct *vma, unsigned long start, unsigned long end) { - pgoff_t pgoff, addr; - unsigned long vma_pglen = vma_pages(vma); + const unsigned long vma_pglen = vma_pages(vma); + pgoff_t pgoff_folio, pgoff_vma_start; + unsigned long addr; VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); if (start > end) return false; + pgoff_folio = folio_pgoff(folio); + pgoff_vma_start = folio_test_anon(folio) ? + vma_start_virt_pgoff(vma) : vma_start_pgoff(vma); + if (start < vma->vm_start) start = vma->vm_start; if (end > vma->vm_end) end = vma->vm_end; - pgoff = folio_pgoff(folio); - /* if folio start address is not in vma range */ - if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) + if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen)) return false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT); return !(addr < start || end - addr < folio_size(folio)); } @@ -1222,27 +1010,18 @@ void mlock_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); -/** - * vma_address - Find the virtual address a page range is mapped at - * @vma: The vma which maps this object. - * @pgoff: The page offset within its object. - * @nr_pages: The number of pages to consider. - * - * If any page in this range is mapped by this VMA, return the first address - * where any of these pages appear. Otherwise, return -EFAULT. - */ -static inline unsigned long vma_address(const struct vm_area_struct *vma, - pgoff_t pgoff, unsigned long nr_pages) +static inline unsigned long __vma_address(const struct vm_area_struct *vma, + pgoff_t pgoff, pgoff_t pgoff_start, unsigned long nr_pages) { unsigned long address; - if (pgoff >= vma->vm_pgoff) { + if (pgoff >= pgoff_start) { address = vma->vm_start + - ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + ((pgoff - pgoff_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; - } else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) { + } else if (pgoff + nr_pages - 1 >= pgoff_start) { /* Test above avoids possibility of wrap to 0 on 32-bit */ address = vma->vm_start; } else { @@ -1251,22 +1030,68 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, return address; } +/** + * vma_filebacked_address - Find the virtual address a file-backed page range is + * mapped at. + * @vma: The vma which maps this object. + * @pgoff: The page offset within its object. + * @nr_pages: The number of pages to consider. + * + * Returns: If any page in this range is mapped by this VMA, return the first + * address where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma, + pgoff_t pgoff, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(vma_is_anonymous(vma)); + + return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages); +} + +/** + * vma_anon_address - Find the virtual address an anonymous page range is mapped + * at. + * @vma: The vma which maps this object. + * @pgoff_virt: The virtual page index belonging to the folio. + * @nr_pages: The number of pages to consider. + * + * This is only valid for anonymous or MAP_PRIVATE-mapped file-backed VMAs. + * + * Returns: If any page in this range is mapped by this VMA, return the first address + * where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_anon_address(const struct vm_area_struct *vma, + pgoff_t pgoff_virt, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(!vma_is_anonymous(vma) && vma_test(vma, VMA_SHARED_BIT)); + + return __vma_address(vma, pgoff_virt, vma_start_virt_pgoff(vma), nr_pages); +} + /* - * Then at what user virtual address will none of the range be found in vma? + * At what user virtual address will none of the range be found in vma? * Assumes that vma_address() already returned a good starting address. */ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) { - struct vm_area_struct *vma = pvmw->vma; - pgoff_t pgoff; + const struct vm_area_struct *vma = pvmw->vma; + const pgoff_t pgoff = pvmw->pgoff; + pgoff_t pgoff_vma_start; unsigned long address; + pgoff_t pgoff_end; /* Common case, plus ->pgoff is invalid for KSM */ if (pvmw->nr_pages == 1) return pvmw->address + PAGE_SIZE; - pgoff = pvmw->pgoff + pvmw->nr_pages; - address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pvmw->is_anon_walk) + pgoff_vma_start = vma_start_virt_pgoff(vma); + else + pgoff_vma_start = vma_start_pgoff(vma); + + pgoff_end = pgoff + pvmw->nr_pages; + address = vma->vm_start + + ((pgoff_end - pgoff_vma_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; @@ -1311,85 +1136,21 @@ static inline void mlock_new_folio(struct folio *folio) { } static inline bool need_mlock_drain(int cpu) { return false; } static inline void mlock_drain_local(void) { } static inline void mlock_drain_remote(int cpu) { } -static inline void vunmap_range_noflush(unsigned long start, unsigned long end) -{ -} #endif /* !CONFIG_MMU */ -/* Memory initialisation debug and verification */ -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -DECLARE_STATIC_KEY_TRUE(deferred_pages); - -static inline bool deferred_pages_enabled(void) -{ - return static_branch_unlikely(&deferred_pages); -} - -bool __init deferred_grow_zone(struct zone *zone, unsigned int order); -#else -static inline bool deferred_pages_enabled(void) -{ - return false; -} -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - -void init_deferred_page(unsigned long pfn, int nid); - -enum mminit_level { - MMINIT_WARNING, - MMINIT_VERIFY, - MMINIT_TRACE -}; - -#ifdef CONFIG_DEBUG_MEMORY_INIT - -extern int mminit_loglevel; - -#define mminit_dprintk(level, prefix, fmt, arg...) \ -do { \ - if (level < mminit_loglevel) { \ - if (level <= MMINIT_WARNING) \ - pr_warn("mminit::" prefix " " fmt, ##arg); \ - else \ - printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ - } \ -} while (0) - -extern void mminit_verify_pageflags_layout(void); -extern void mminit_verify_zonelist(void); -#else - -static inline void mminit_dprintk(enum mminit_level level, - const char *prefix, const char *fmt, ...) -{ -} - -static inline void mminit_verify_pageflags_layout(void) -{ -} - -static inline void mminit_verify_zonelist(void) -{ -} -#endif /* CONFIG_DEBUG_MEMORY_INIT */ - -#define NODE_RECLAIM_NOSCAN -2 -#define NODE_RECLAIM_FULL -1 -#define NODE_RECLAIM_SOME 0 -#define NODE_RECLAIM_SUCCESS 1 - #ifdef CONFIG_NUMA extern int node_reclaim_mode; -extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int); +extern unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t gfp_mask, unsigned int order); extern int find_next_best_node(int node, nodemask_t *used_node_mask); #else #define node_reclaim_mode 0 -static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask, - unsigned int order) +static inline unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t mask, unsigned int order) { - return NODE_RECLAIM_NOSCAN; + return 0; } static inline int find_next_best_node(int node, nodemask_t *used_node_mask) { @@ -1436,50 +1197,9 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -extern void set_pageblock_order(void); unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); -/* The ALLOC_WMARK bits are used as an index to zone->watermark */ -#define ALLOC_WMARK_MIN WMARK_MIN -#define ALLOC_WMARK_LOW WMARK_LOW -#define ALLOC_WMARK_HIGH WMARK_HIGH -#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ - -/* Mask to get the watermark bits */ -#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) - -/* - * Only MMU archs have async oom victim reclaim - aka oom_reaper so we - * cannot assume a reduced access to memory reserves is sufficient for - * !MMU - */ -#ifdef CONFIG_MMU -#define ALLOC_OOM 0x08 -#else -#define ALLOC_OOM ALLOC_NO_WATERMARKS -#endif - -#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access - * to 25% of the min watermark or - * 62.5% if __GFP_HIGH is set. - */ -#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% - * of the min watermark. - */ -#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ -#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ -#ifdef CONFIG_ZONE_DMA32 -#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ -#else -#define ALLOC_NOFRAGMENT 0x0 -#endif -#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ -#define ALLOC_TRYLOCK 0x400 /* Only use spin_trylock in allocation path */ -#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ - -/* Flags that allow allocations below the min watermark. */ -#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) enum ttu_flags; struct tlbflush_unmap_batch; @@ -1526,37 +1246,6 @@ struct migration_target_control { size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, struct folio *folio, loff_t fpos, size_t size); -/* - * mm/vmalloc.c - */ -#ifdef CONFIG_MMU -void __init vmalloc_init(void); -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask); -unsigned int get_vm_area_page_order(struct vm_struct *vm); -#else -static inline void vmalloc_init(void) -{ -} - -static inline -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask) -{ - return -EINVAL; -} -#endif - -void clear_vm_uninitialized_flag(struct vm_struct *vm); - -int __must_check __vmap_pages_range_noflush(unsigned long addr, - unsigned long end, pgprot_t prot, - struct page **pages, unsigned int page_shift); - -void vunmap_range_noflush(unsigned long start, unsigned long end); - -void __vunmap_range_noflush(unsigned long start, unsigned long end); - static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) { if (vma->vm_flags & VM_SHARED) @@ -1584,12 +1273,6 @@ int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, void free_zone_device_folio(struct folio *folio); int migrate_device_coherent_folio(struct folio *folio); -struct vm_struct *__get_vm_area_node(unsigned long size, - unsigned long align, unsigned long shift, - unsigned long vm_flags, unsigned long start, - unsigned long end, int node, gfp_t gfp_mask, - const void *caller); - /* * mm/gup.c */ @@ -1712,18 +1395,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, return !PageAnonExclusive(page); } -extern bool mirrored_kernelcore; -bool memblock_has_mirror(void); -void memblock_free_all(void); - -static __always_inline void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) { @@ -1753,10 +1424,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte); } -void __meminit __init_single_page(struct page *page, unsigned long pfn, - unsigned long zone, int nid); -void __meminit __init_page_from_nid(unsigned long pfn, int nid); - /* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, int priority); @@ -1951,4 +1618,37 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) +{ + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } +} + +static inline bool can_spin_trylock(void) +{ + /* + * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is + * unsafe in NMI. If spin_trylock() is called from hard IRQ the current + * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will + * mark the task as the owner of another rt_spin_lock which will + * confuse PI logic, so return immediately if called from hard IRQ or + * NMI. + * + * Note, irqs_disabled() case is ok. spin_trylock() can be called + * from raw_spin_lock_irqsave region. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) + return false; + + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return false; + + return true; +} + #endif /* __MM_INTERNAL_H */ diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 32bcfbfcf15f..26b8437e3b1b 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * mm/interval_tree.c - interval tree for mapping->i_mmap + * mm/interval_tree.c - interval tree for address_space->i_mmap and + * anon_vma->rb_root * * Copyright (C) 2012, Michel Lespinasse <walken@google.com> */ @@ -10,25 +11,29 @@ #include <linux/rmap.h> #include <linux/interval_tree_generic.h> -static inline unsigned long vma_start_pgoff(struct vm_area_struct *v) -{ - return v->vm_pgoff; -} +/* File-backed interval tree (address_space->i_mmap) */ INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, - unsigned long, shared.rb_subtree_last, - vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree) + pgoff_t, shared.rb_subtree_last, + vma_start_pgoff, vma_last_pgoff, static, + __mapping_rmap_tree) + +void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __mapping_rmap_tree_insert(vma, &mapping->i_mmap); +} -/* Insert node immediately after prev in the interval tree */ -void vma_interval_tree_insert_after(struct vm_area_struct *node, +/* Insert vma immediately after prev in the interval tree */ +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root) + struct address_space *mapping) { struct rb_node **link; struct vm_area_struct *parent; - unsigned long last = vma_last_pgoff(node); + const pgoff_t pgoff_last = vma_last_pgoff(vma); - VM_BUG_ON_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node); + VM_WARN_ON_ONCE_VMA(vma_start_pgoff(vma) != vma_start_pgoff(prev), vma); if (!prev->shared.rb.rb_right) { parent = prev; @@ -36,71 +41,95 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node, } else { parent = rb_entry(prev->shared.rb.rb_right, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; while (parent->shared.rb.rb_left) { parent = rb_entry(parent->shared.rb.rb_left, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; } link = &parent->shared.rb.rb_left; } - node->shared.rb_subtree_last = last; - rb_link_node(&node->shared.rb, &parent->shared.rb, link); - rb_insert_augmented(&node->shared.rb, &root->rb_root, - &vma_interval_tree_augment); + vma->shared.rb_subtree_last = pgoff_last; + rb_link_node(&vma->shared.rb, &parent->shared.rb, link); + rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root, + &__mapping_rmap_tree_augment); +} + +void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __mapping_rmap_tree_remove(vma, &mapping->i_mmap); } -static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc) +struct vm_area_struct * +mapping_rmap_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __mapping_rmap_tree_iter_first(&mapping->i_mmap, + pgoff_start, pgoff_last); +} + +struct vm_area_struct * +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __mapping_rmap_tree_iter_next(vma, pgoff_start, pgoff_last); +} + +/* Anonymous interval tree (anon_vma->rb_root) */ + +static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc) { - return vma_start_pgoff(avc->vma); + return vma_start_virt_pgoff(avc->vma); } -static inline unsigned long avc_last_pgoff(struct anon_vma_chain *avc) +static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc) { - return vma_last_pgoff(avc->vma); + return vma_last_virt_pgoff(avc->vma); } -INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last, +INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last, avc_start_pgoff, avc_last_pgoff, - static inline, __anon_vma_interval_tree) + static, __anon_rmap_tree) -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root) +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { #ifdef CONFIG_DEBUG_VM_RB - node->cached_vma_start = avc_start_pgoff(node); - node->cached_vma_last = avc_last_pgoff(node); + avc->cached_vma_start = avc_start_pgoff(avc); + avc->cached_vma_last = avc_last_pgoff(avc); #endif - __anon_vma_interval_tree_insert(node, root); + __anon_rmap_tree_insert(avc, &anon_vma->rb_root); } -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root) +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { - __anon_vma_interval_tree_remove(node, root); + __anon_rmap_tree_remove(avc, &anon_vma->rb_root); } struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long first, unsigned long last) +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_first(root, first, last); + return __anon_rmap_tree_iter_first(&anon_vma->rb_root, + pgoff_start, pgoff_last); } struct anon_vma_chain * -anon_vma_interval_tree_iter_next(struct anon_vma_chain *node, - unsigned long first, unsigned long last) +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __anon_vma_interval_tree_iter_next(node, first, last); + return __anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last); } #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node) +void anon_rmap_tree_verify(struct anon_vma_chain *avc) { - WARN_ON_ONCE(node->cached_vma_start != avc_start_pgoff(node)); - WARN_ON_ONCE(node->cached_vma_last != avc_last_pgoff(node)); + WARN_ON_ONCE(avc->cached_vma_start != avc_start_pgoff(avc)); + WARN_ON_ONCE(avc->cached_vma_last != avc_last_pgoff(avc)); } #endif diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index cbef5e450954..a848eb2f9910 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -61,7 +61,7 @@ DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc); EXPORT_SYMBOL_GPL(kasan_flag_vmalloc); /* Whether to check write accesses only. */ -static bool kasan_flag_write_only = false; +static bool kasan_flag_write_only; #define PAGE_ALLOC_SAMPLE_DEFAULT 1 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3 diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 617bca76db49..27e8f3077e80 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -26,6 +26,7 @@ #include <asm/tlb.h> #include "internal.h" +#include "page_alloc.h" #include "mm_slot.h" enum scan_result { @@ -37,7 +38,7 @@ enum scan_result { SCAN_EXCEED_SWAP_PTE, SCAN_EXCEED_SHARED_PTE, SCAN_PTE_NON_PRESENT, - SCAN_PTE_UFFD_WP, + SCAN_PTE_UFFD, SCAN_PTE_MAPPED_HUGEPAGE, SCAN_LACK_REFERENCED_PAGE, SCAN_PAGE_NULL, @@ -605,8 +606,7 @@ void __khugepaged_exit(struct mm_struct *mm) spin_lock(&khugepaged_mm_lock); slot = mm_slot_lookup(mm_slots_hash, mm); if (slot && khugepaged_scan.mm_slot != slot) { - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); free = 1; } spin_unlock(&khugepaged_mm_lock); @@ -695,8 +695,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, result = SCAN_PTE_NON_PRESENT; goto out; } - if (pte_uffd_wp(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_uffd(pteval)) { + result = SCAN_PTE_UFFD; goto out; } page = vm_normal_page(vma, addr, pteval); @@ -1543,7 +1543,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, case SCAN_PAGE_NULL: case SCAN_DEL_PAGE_LRU: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_PAGE_LAZYFREE: last_result = ret; goto next_order; @@ -1664,15 +1664,15 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, /* * Always be strict with uffd-wp * enabled swap entries. Please see - * comment below for pte_uffd_wp(). + * comment below for pte_uffd(). */ - if (pte_swp_uffd_wp_any(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_swp_uffd_any(pteval)) { + result = SCAN_PTE_UFFD; goto out_unmap; } continue; } - if (pte_uffd_wp(pteval)) { + if (pte_uffd(pteval)) { /* * Don't collapse the page if any of the small * PTEs are armed with uffd write protection. @@ -1682,7 +1682,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, * userfault messages that falls outside of * the registered range. So, just be simple. */ - result = SCAN_PTE_UFFD_WP; + result = SCAN_PTE_UFFD; goto out_unmap; } @@ -1801,8 +1801,7 @@ static void collect_mm_slot(struct mm_slot *slot) if (collapse_test_exit(mm)) { /* free mm_slot */ - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); /* * Not strictly needed because the mm exited already. @@ -1892,9 +1891,12 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER)) return SCAN_VMA_CHECK; - /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ - if (userfaultfd_wp(vma)) - return SCAN_PTE_UFFD_WP; + /* + * Keep pmd pgtable while the uffd bit is in use; see comment in + * retract_page_tables(). + */ + if (userfaultfd_protected(vma)) + return SCAN_PTE_UFFD; folio = filemap_lock_folio(vma->vm_file->f_mapping, linear_page_index(vma, haddr)); @@ -2106,13 +2108,14 @@ static bool file_backed_vma_is_retractable(struct vm_area_struct *vma) return false; /* - * When a vma is registered with uffd-wp, we cannot recycle + * When a vma is registered with uffd-wp or RWP, we cannot recycle * the page table because there may be pte markers installed. - * Other vmas can still have the same file mapped hugely, but - * skip this one: it will always be mapped in small page size - * for uffd-wp registered ranges. + * VM_UFFD_RWP ranges similarly rely on per-PTE uffd state + * and cannot be recycled to a shared PMD. Other vmas can still + * have the same file mapped hugely, but skip this one: it will + * always be mapped in small page size for these registrations. */ - if (userfaultfd_wp(vma)) + if (userfaultfd_protected(vma)) return false; /* @@ -2136,7 +2139,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct vm_area_struct *vma; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; @@ -2145,7 +2148,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) spinlock_t *ptl; bool success = false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + + ((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT); if (addr & ~HPAGE_PMD_MASK || vma->vm_end < addr + HPAGE_PMD_SIZE) continue; @@ -2568,7 +2572,7 @@ xa_unlocked: * not be able to observe any missing pages due to the * previously inserted retry entries. */ - vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { + mapping_rmap_tree_foreach(vma, mapping, start, end) { if (userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; @@ -3241,7 +3245,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, /* Whitelisted set of results where continuing OK */ case SCAN_NO_PTE_TABLE: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_LACK_REFERENCED_PAGE: case SCAN_PAGE_NULL: case SCAN_PAGE_COUNT: diff --git a/mm/kmemleak.c b/mm/kmemleak.c index e196f53f9b46..f63dfacee7ca 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -151,6 +151,8 @@ struct kmemleak_object { int min_count; /* the total number of pointers found pointing to this object */ int count; + /* consecutive scans the object has been seen unreferenced */ + unsigned int unref_scans; /* checksum for detecting modified objects */ u32 checksum; depot_stack_handle_t trace_handle; @@ -175,6 +177,8 @@ struct kmemleak_object { #define OBJECT_PHYS (1 << 4) /* flag set for per-CPU pointers */ #define OBJECT_PERCPU (1 << 5) +/* flag set on an object left unreferenced by the full scan, pending confirmation */ +#define OBJECT_SUSPECT (1 << 6) /* set when __remove_object() called */ #define DELSTATE_REMOVED (1 << 0) @@ -232,9 +236,14 @@ static unsigned long max_percpu_addr; static struct task_struct *scan_thread; /* used to avoid reporting of recently allocated objects */ static unsigned long jiffies_min_age; +/* consecutive scans an object must stay unreferenced before reporting */ +static unsigned int min_unref_scans = 1; +module_param(min_unref_scans, uint, 0644); static unsigned long jiffies_last_scan; /* delay between automatic memory scannings */ static unsigned long jiffies_scan_wait; +/* number of objects flagged OBJECT_SUSPECT during the current scan */ +static int nr_suspects; /* enables or disables the task stacks scanning */ static int kmemleak_stack_scan = 1; /* protects the memory scanning, parameters and debug/kmemleak file access */ @@ -688,6 +697,7 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp) object->excess_ref = 0; object->count = 0; /* white color initially */ object->checksum = ~0; + object->unref_scans = 0; object->del_state = 0; /* task information */ @@ -1440,6 +1450,11 @@ static void update_refs(struct kmemleak_object *object) */ object->count++; if (color_gray(object)) { + /* referenced after all, no longer a suspect */ + if (object->flags & OBJECT_SUSPECT) { + object->flags &= ~OBJECT_SUSPECT; + nr_suspects--; + } /* put_object() called when removing from gray_list */ WARN_ON(!get_object(object)); list_add_tail(&object->gray_list, &gray_list); @@ -1525,22 +1540,25 @@ static int scan_should_stop(void) /* * Scan a memory block (exclusive range) for valid pointers and add those - * found to the gray list. + * found to the gray list. Return non-zero if the scan was interrupted. */ -static void scan_block(void *_start, void *_end, - struct kmemleak_object *scanned) +static int scan_block(void *_start, void *_end, + struct kmemleak_object *scanned) { unsigned long *ptr; unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER); unsigned long *end = _end - (BYTES_PER_POINTER - 1); unsigned long flags; + int stop = 0; raw_spin_lock_irqsave(&kmemleak_lock, flags); for (ptr = start; ptr < end; ptr++) { unsigned long pointer; - if (scan_should_stop()) + if (scan_should_stop()) { + stop = 1; break; + } kasan_disable_current(); pointer = *(unsigned long *)kasan_reset_tag((void *)ptr); @@ -1550,22 +1568,28 @@ static void scan_block(void *_start, void *_end, pointer_update_refs(scanned, pointer, OBJECT_PERCPU); } raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + + return stop; } /* * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency. + * Return non-zero if the scan was interrupted. */ #ifdef CONFIG_SMP -static void scan_large_block(void *start, void *end) +static int scan_large_block(void *start, void *end) { void *next; while (start < end) { next = min(start + MAX_SCAN_SIZE, end); - scan_block(start, next, NULL); + if (scan_block(start, next, NULL)) + return 1; start = next; - cond_resched(); + cond_resched_tasks_rcu_qs(); } + + return 0; } #endif @@ -1599,7 +1623,7 @@ static void scan_object(struct kmemleak_object *object) scan_block(start, end, object); raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); if (!(object->flags & OBJECT_ALLOCATED)) break; @@ -1621,7 +1645,7 @@ static void scan_object(struct kmemleak_object *object) break; raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); } while (object->flags & OBJECT_ALLOCATED); } else { @@ -1649,7 +1673,7 @@ static void scan_gray_list(void) */ object = list_entry(gray_list.next, typeof(*object), gray_list); while (&object->gray_list != &gray_list) { - cond_resched(); + cond_resched_tasks_rcu_qs(); /* may add new objects to the list */ if (!scan_should_stop()) @@ -1684,7 +1708,7 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) raw_spin_unlock_irq(&kmemleak_lock); rcu_read_unlock(); - cond_resched(); + cond_resched_tasks_rcu_qs(); rcu_read_lock(); raw_spin_lock_irq(&kmemleak_lock); @@ -1697,6 +1721,43 @@ unlock_put: } /* + * Scan all task kernel stacks, rescheduling between tasks. Each task is looked + * up and pinned within its own RCU read-side section, so no lock is held across + * the scan and the walk cannot trip the soft lockup watchdog. + */ +static void kmemleak_scan_task_stacks(void) +{ + struct pid *pid; + int nr = 1; + int stop = 0; + + do { + struct task_struct *p = NULL; + + rcu_read_lock(); + pid = find_ge_pid(nr, &init_pid_ns); + if (pid) { + nr = pid_nr(pid) + 1; + p = pid_task(pid, PIDTYPE_PID); + if (p) + get_task_struct(p); + } + rcu_read_unlock(); + + if (p) { + void *stack = try_get_task_stack(p); + + if (stack) { + stop = scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } + put_task_struct(p); + } + cond_resched_tasks_rcu_qs(); + } while (pid && !stop); +} + +/* * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it * does not touch user memory that was freed concurrently; the rest of the * report (backtrace, comm, pid) is always emitted since the kmemleak_object @@ -1798,15 +1859,16 @@ static void dedup_flush(struct xarray *dedup) * kernel's standard allocators. This function must be called with the * scan_mutex held. */ -static void kmemleak_scan(void) +static int __kmemleak_scan(bool full) { struct kmemleak_object *object; struct zone *zone; int __maybe_unused i; - struct xarray dedup; - int new_leaks = 0; + int stop = 0; jiffies_last_scan = jiffies; + if (full) + nr_suspects = 0; /* prepare the kmemleak_object's */ rcu_read_lock(); @@ -1834,8 +1896,13 @@ static void kmemleak_scan(void) __paint_it(object, KMEMLEAK_BLACK); } + /* referenced last scan: restart the unreferenced run */ + if (!color_white(object)) + object->unref_scans = 0; /* reset the reference count (whiten the object) */ object->count = 0; + if (full) + object->flags &= ~OBJECT_SUSPECT; if (color_gray(object) && get_object(object)) list_add_tail(&object->gray_list, &gray_list); @@ -1848,9 +1915,11 @@ static void kmemleak_scan(void) #ifdef CONFIG_SMP /* per-cpu sections scanning */ - for_each_possible_cpu(i) - scan_large_block(__per_cpu_start + per_cpu_offset(i), - __per_cpu_end + per_cpu_offset(i)); + for_each_possible_cpu(i) { + if (scan_large_block(__per_cpu_start + per_cpu_offset(i), + __per_cpu_end + per_cpu_offset(i))) + goto scan_gray; + } #endif /* @@ -1866,7 +1935,7 @@ static void kmemleak_scan(void) struct page *page = pfn_to_online_page(pfn); if (!(pfn & 63)) - cond_resched(); + cond_resched_tasks_rcu_qs(); if (!page) continue; @@ -1877,34 +1946,34 @@ static void kmemleak_scan(void) /* only scan if page is in use */ if (page_count(page) == 0) continue; - scan_block(page, page + 1, NULL); + stop = scan_block(page, page + 1, NULL); + if (stop) + break; } + if (stop) + break; } put_online_mems(); + if (stop) + goto scan_gray; /* * Scanning the task stacks (may introduce false negatives). */ - if (kmemleak_stack_scan) { - struct task_struct *p, *g; - - rcu_read_lock(); - for_each_process_thread(g, p) { - void *stack = try_get_task_stack(p); - if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); - put_task_stack(p); - } - } - rcu_read_unlock(); - } + if (kmemleak_stack_scan) + kmemleak_scan_task_stacks(); /* * Scan the objects already referenced from the sections scanned * above. */ +scan_gray: scan_gray_list(); + /* a confirmation scan does not look for modified objects */ + if (!full) + return nr_suspects; + /* * Check for new or unreferenced objects modified since the previous * scan and color them gray until the next scan. @@ -1927,6 +1996,11 @@ static void kmemleak_scan(void) /* color it gray temporarily */ object->count = object->min_count; list_add_tail(&object->gray_list, &gray_list); + } else if (unreferenced_object(object) && + !(object->flags & OBJECT_REPORTED)) { + /* flag the objects left unreferenced by this scan */ + object->flags |= OBJECT_SUSPECT; + nr_suspects++; } raw_spin_unlock_irq(&object->lock); } @@ -1937,6 +2011,46 @@ static void kmemleak_scan(void) */ scan_gray_list(); + return nr_suspects; +} + +/* + * Promote a suspected object to a reported leak once it has stayed + * unreferenced for min_unref_scans consecutive scans. Called with + * object->lock held; returns true when the object is newly reported. + */ +static bool confirm_leak(struct kmemleak_object *object) +{ + if (!unreferenced_object(object) || + !(object->flags & OBJECT_SUSPECT) || + (object->flags & OBJECT_REPORTED)) + return false; + + object->unref_scans += 1; + if (object->unref_scans < min_unref_scans) + return false; + + object->flags |= OBJECT_REPORTED; + return true; +} + +/* + * Scan the memory and report the unreferenced objects as leaks. Must be + * called with the scan_mutex held. + */ +static void kmemleak_scan(void) +{ + struct kmemleak_object *object; + struct xarray dedup; + int new_leaks = 0; + + /* + * Full scan. Objects left unreferenced are flagged OBJECT_SUSPECT and + * counted in the return value; nothing to confirm or report otherwise. + */ + if (!__kmemleak_scan(true)) + return; + /* * If scanning was stopped do not report any new unreferenced objects. */ @@ -1944,6 +2058,16 @@ static void kmemleak_scan(void) return; /* + * A live object whose only reference is moved by, for example, a + * concurrent RCU update can be missed for one scan and reported as a + * transient false positive. Scan again and only report the objects + * left unreferenced (still flagged OBJECT_SUSPECT) by both scans. + */ + __kmemleak_scan(false); + if (scan_should_stop()) + return; + + /* * Scanning result reporting. When verbose printing is enabled, dedupe * by stackdepot trace_handle so each unique backtrace is logged once * per scan, annotated with the number of objects that share it. The @@ -1969,9 +2093,8 @@ static void kmemleak_scan(void) raw_spin_lock_irq(&object->lock); trace_handle = 0; dedup_print = false; - if (unreferenced_object(object) && - !(object->flags & OBJECT_REPORTED)) { - object->flags |= OBJECT_REPORTED; + + if (confirm_leak(object)) { if (kmemleak_verbose) { trace_handle = object->trace_handle; dedup_print = true; diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c index 8f22d1f22981..5f1b8053f9fa 100644 --- a/mm/kmsan/hooks.c +++ b/mm/kmsan/hooks.c @@ -21,6 +21,7 @@ #include <linux/usb.h> #include "../internal.h" +#include "../vmalloc.h" #include "../slab.h" #include "kmsan.h" diff --git a/mm/kmsan/init.c b/mm/kmsan/init.c index b14ce3417e65..4983b6e9f7c9 100644 --- a/mm/kmsan/init.c +++ b/mm/kmsan/init.c @@ -13,7 +13,7 @@ #include <linux/mm.h> #include <linux/memblock.h> -#include "../internal.h" +#include "../page_alloc.h" #define NUM_FUTURE_RANGES 128 struct start_end_pair { diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c index 8fde939784a7..0c88d89bf0d6 100644 --- a/mm/kmsan/shadow.c +++ b/mm/kmsan/shadow.c @@ -17,6 +17,7 @@ #include <linux/stddef.h> #include "../internal.h" +#include "../vmalloc.h" #include "kmsan.h" #define shadow_page_for(page) ((page)->kmsan_shadow) @@ -195,22 +195,28 @@ struct ksm_stable_node { * @node: rb node of this rmap_item in the unstable tree * @head: pointer to stable_node heading this list in the stable tree * @hlist: link into hlist of rmap_items hanging off that stable_node - * @age: number of scan iterations since creation - * @remaining_skips: how many scans to skip + * @age: number of scan iterations since creation (unstable node) + * @remaining_skips: how many scans to skip (unstable node) + * @linear_page_index: the original page's index before merged by KSM (stable node) */ struct ksm_rmap_item { struct ksm_rmap_item *rmap_list; union { - struct anon_vma *anon_vma; /* when stable */ + struct anon_vma *anon_vma; /* for reverse mapping, when stable */ #ifdef CONFIG_NUMA int nid; /* when node of unstable tree */ #endif }; struct mm_struct *mm; unsigned long address; /* + low bits used for flags below */ - unsigned int oldchecksum; /* when unstable */ - rmap_age_t age; - rmap_age_t remaining_skips; + union { + struct { + unsigned int oldchecksum; + rmap_age_t age; + rmap_age_t remaining_skips; + }; /* when unstable */ + unsigned long linear_page_index; /* for reverse mapping, when stable */ + }; union { struct rb_node node; /* when node of unstable tree */ struct { /* when listed from stable tree */ @@ -776,6 +782,11 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, return vma; } +/* + * break_cow: actively break COW, replacing the KSM page by a fresh anonymous + * page. This is called when rmap_item has not yet become stable, but page + * has been merged. + */ static void break_cow(struct ksm_rmap_item *rmap_item) { struct mm_struct *mm = rmap_item->mm; @@ -787,6 +798,11 @@ static void break_cow(struct ksm_rmap_item *rmap_item) * to undo, we also need to drop a reference to the anon_vma. */ put_anon_vma(rmap_item->anon_vma); + /* + * Reset linear_page_index that might overlay age-related + * information. (it's still unstable node) + */ + rmap_item->linear_page_index = 0; mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); @@ -899,6 +915,8 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node) VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->address &= PAGE_MASK; cond_resched(); } @@ -1052,6 +1070,8 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->head = NULL; rmap_item->address &= PAGE_MASK; @@ -1237,8 +1257,7 @@ mm_exiting: struct mm_slot, mm_node); ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); if (ksm_test_exit(mm)) { - hash_del(&mm_slot->slot.hash); - list_del(&mm_slot->slot.mm_node); + mm_slot_remove(&mm_slot->slot); spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); @@ -1598,8 +1617,16 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, /* Unstable nid is in union with stable anon_vma: remove first */ remove_rmap_item_from_tree(rmap_item); - /* Must get reference to anon_vma while still holding mmap_lock */ + /* + * We can consider the VMA only while still holding the mmap lock, + * so lock, so reference the anon_vma and calculate the linear + * page index early, before stable_tree_append(). If anything goes + * wrong that prevents the rmap_item from being added to the + * stable_tree, break_cow() will clean it up. + */ rmap_item->anon_vma = vma->anon_vma; + /* The VMA is always anon/MAP_PRIVATE-file backed so use anon index. */ + rmap_item->linear_page_index = linear_virt_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: mmap_read_unlock(mm); @@ -2327,23 +2354,24 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite tree_rmap_item = unstable_tree_search_insert(rmap_item, page, &tree_page); if (tree_rmap_item) { + struct folio *tree_folio; bool split; kfolio = try_to_merge_two_pages(rmap_item, page, tree_rmap_item, tree_page); + tree_folio = page_folio(tree_page); /* - * If both pages we tried to merge belong to the same compound - * page, then we actually ended up increasing the reference - * count of the same compound page twice, and split_huge_page - * failed. + * If both pages we tried to merge belong to the same (large) + * folio, then we actually ended up increasing the reference + * count of the same folio twice, and split_huge_page failed. + * * Here we set a flag if that happened, and we use it later to - * try split_huge_page again. Since we call put_page right + * try split_huge_page again. Since we call folio_put() right * afterwards, the reference count will be correct and * split_huge_page should succeed. */ - split = PageTransCompound(page) - && compound_head(page) == compound_head(tree_page); - put_page(tree_page); + split = folio == tree_folio; + folio_put(tree_folio); if (kfolio) { /* * The pages were successfully merged: insert new @@ -2458,6 +2486,13 @@ static bool should_skip_rmap_item(struct folio *folio, if (folio_test_ksm(folio)) return false; + /* + * There is no age information in stable-tree nodes. We might end up + * here without a KSM page for example after COW. + */ + if (rmap_item->address & STABLE_FLAG) + return false; + age = rmap_item->age; if (age != U8_MAX) rmap_item->age++; @@ -2737,8 +2772,7 @@ no_vmas: * or when all VM_MERGEABLE areas have been unmapped (and * mmap_lock then protects against race with MADV_MERGEABLE). */ - hash_del(&mm_slot->slot.hash); - list_del(&mm_slot->slot.mm_node); + mm_slot_remove(&mm_slot->slot); spin_unlock(&ksm_mmlist_lock); mm_slot_free(mm_slot_cache, mm_slot); @@ -3081,8 +3115,7 @@ void __ksm_exit(struct mm_struct *mm) if (ksm_scan.mm_slot == mm_slot) goto unlock; if (!mm_slot->rmap_list) { - hash_del(&slot->hash); - list_del(&slot->mm_node); + mm_slot_remove(slot); easy_to_free = 1; } else { list_move(&slot->mm_node, @@ -3120,7 +3153,7 @@ struct folio *ksm_might_need_to_copy(struct folio *folio, return folio; /* no need to copy it */ } else if (!anon_vma) { return folio; /* no need to copy it */ - } else if (folio->index == linear_page_index(vma, addr) && + } else if (folio->index == linear_virt_page_index(vma, addr) && anon_vma->root == vma->anon_vma->root) { return folio; /* still no need to copy it */ } @@ -3173,6 +3206,7 @@ again: hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { /* Ignore the stable/unstable/sqnr flags */ const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct anon_vma *anon_vma = rmap_item->anon_vma; struct anon_vma_chain *vmac; struct vm_area_struct *vma; @@ -3186,8 +3220,17 @@ again: anon_vma_lock_read(anon_vma); } - anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root, - 0, ULONG_MAX) { + /* + * Currently, KSM folios are always small folios, so it's + * sufficient to search for a single page. We can simply use + * the linear_virt_page_index of the original de-duplicate + * anonymous page that we remembered in the rmap_item while + * de-duplicating. Note that mremap() always de-duplicates KSM + * folios: so if there was mremap() in our parent or our child, + * we wouldn't have the KSM folio mapped in these processes + * anymore. + */ + anon_rmap_tree_foreach(vmac, anon_vma, index, index) { cond_resched(); vma = vmac->vma; @@ -3243,17 +3286,16 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page, rcu_read_lock(); for_each_process(tsk) { struct anon_vma_chain *vmac; - unsigned long addr; + const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct task_struct *t = task_early_kill(tsk, force_early); if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0, - ULONG_MAX) + anon_rmap_tree_foreach(vmac, av, index, index) { vma = vmac->vma; if (vma->vm_mm == t->mm) { - addr = rmap_item->address & PAGE_MASK; add_to_kill_ksm(t, page, vma, to_kill, addr); } diff --git a/mm/madvise.c b/mm/madvise.c index 77552b03d318..07a21ca31bad 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *walk) { struct vm_area_struct *vma = walk->private; - struct swap_iocb *splug = NULL; + struct swap_io_ctx ctx = {}; pte_t *ptep = NULL; spinlock_t *ptl; unsigned long addr; @@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, pte_unmap_unlock(ptep, ptl); ptep = NULL; - folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE, - vma, addr, &splug); + folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE, + vma, addr); if (folio) folio_put(folio); } if (ptep) pte_unmap_unlock(ptep, ptl); - swap_read_unplug(splug); + swap_read_submit(&ctx); cond_resched(); return 0; @@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); pgoff_t end_index = linear_page_index(vma, end) - 1; struct folio *folio; - struct swap_iocb *splug = NULL; + struct swap_io_ctx ctx = {}; rcu_read_lock(); xas_for_each(&xas, folio, end_index) { @@ -253,19 +253,19 @@ static void shmem_swapin_range(struct vm_area_struct *vma, continue; addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + ((xas.xa_index - vma_start_pgoff(vma)) << PAGE_SHIFT); xas_pause(&xas); rcu_read_unlock(); - folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping), - vma, addr, &splug); + folio = read_swap_cache_async(&ctx, entry, + mapping_gfp_mask(mapping), vma, addr); if (folio) folio_put(folio); rcu_read_lock(); } rcu_read_unlock(); - swap_read_unplug(splug); + swap_read_submit(&ctx); } #endif /* CONFIG_SWAP */ @@ -318,7 +318,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -388,8 +388,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, goto huge_unlock; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto huge_unlock; } @@ -694,10 +694,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, nr = swap_pte_batch(pte, max_nr, ptent); nr_swap -= nr; swap_put_entries_direct(entry, nr); - clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(mm, addr, pte, nr); } else if (softleaf_is_hwpoison(entry) || softleaf_is_poison_marker(entry)) { - pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); + pte_clear(mm, addr, pte); } continue; } @@ -1022,7 +1022,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1233,7 +1233,7 @@ static int guard_remove_pte_entry(pte_t *pte, unsigned long addr, if (is_guard_pte_marker(ptent)) { /* Simply clear the PTE marker. */ - pte_clear_not_present_full(walk->mm, addr, pte, false); + pte_clear(walk->mm, addr, pte); update_mmu_cache(walk->vma, addr, pte); } diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c index 737c407f4081..e0efa36e0a07 100644 --- a/mm/mapping_dirty_helpers.c +++ b/mm/mapping_dirty_helpers.c @@ -95,7 +95,7 @@ static int clean_record_pte(pte_t *pte, unsigned long addr, if (pte_dirty(ptent)) { pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) + - walk->vma->vm_pgoff - cwalk->bitmap_pgoff; + vma_start_pgoff(walk->vma) - cwalk->bitmap_pgoff; pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte); ptent = pte_mkclean(old_pte); diff --git a/mm/memblock.c b/mm/memblock.c index 6349c48154f4..777c69f05400 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -29,6 +29,7 @@ #include <linux/io.h> #include "internal.h" +#include "mm_init.h" #define INIT_MEMBLOCK_REGIONS 128 #define INIT_PHYSMEM_REGIONS 4 diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 765069211567..2dc599484d00 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -6,6 +6,7 @@ #include <linux/pagewalk.h> #include <linux/backing-dev.h> #include <linux/eventfd.h> +#include <linux/log2.h> #include <linux/poll.h> #include <linux/sort.h> #include <linux/file.h> @@ -751,7 +752,7 @@ static int compare_thresholds(const void *a, const void *b) return 0; } -static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) +static void mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) { struct mem_cgroup_eventfd_list *ev; @@ -761,7 +762,6 @@ static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) eventfd_signal(ev->eventfd); spin_unlock(&memcg_oom_lock); - return 0; } static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) @@ -1181,13 +1181,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, event->unregister_event = mem_cgroup_usage_unregister_event; } else if (!strcmp(name, "memory.oom_control")) { pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org" + "Please report your usecase to linux-mm@kvack.org" " if you depend on this functionality.\n"); event->register_event = mem_cgroup_oom_register_event; event->unregister_event = mem_cgroup_oom_unregister_event; } else if (!strcmp(name, "memory.pressure_level")) { pr_warn_once("pressure_level is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org " + "Please report your usecase to linux-mm@kvack.org " "if you depend on this functionality.\n"); event->register_event = vmpressure_register_event; event->unregister_event = vmpressure_unregister_event; @@ -1476,6 +1476,297 @@ void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked) mem_cgroup_oom_unlock(memcg); } +/* + * cgroup v1 userspace vmpressure interface (memory.pressure_level / + * cgroup.event_control). Kept here so v2-only kernels (CONFIG_MEMCG_V1=n) + * drop the whole eventfd accumulator, its work item, and the per-memcg + * state it requires. + * + * When there are too little pages left to scan, vmpressure() may miss the + * critical pressure as number of pages will be less than "window size". + * However, in that case the vmscan priority will raise fast as the + * reclaimer will try to scan LRUs more deeply. + * + * The vmscan logic considers these special priorities: + * + * prio == DEF_PRIORITY (12): reclaimer starts with that value + * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed + * prio == 0 : close to OOM, kernel scans every page in an lru + * + * Any value in this range is acceptable for this tunable (i.e. from 12 to + * 0). Current value for the vmpressure_level_critical_prio is chosen + * empirically, but the number, in essence, means that we consider + * critical level when scanning depth is ~10% of the lru size (vmscan + * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one + * eights). + */ +static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); + +enum vmpressure_modes { + VMPRESSURE_NO_PASSTHROUGH = 0, + VMPRESSURE_HIERARCHY, + VMPRESSURE_LOCAL, + VMPRESSURE_NUM_MODES, +}; + +static const char * const vmpressure_str_levels[] = { + [VMPRESSURE_LOW] = "low", + [VMPRESSURE_MEDIUM] = "medium", + [VMPRESSURE_CRITICAL] = "critical", +}; + +static const char * const vmpressure_str_modes[] = { + [VMPRESSURE_NO_PASSTHROUGH] = "default", + [VMPRESSURE_HIERARCHY] = "hierarchy", + [VMPRESSURE_LOCAL] = "local", +}; + +struct vmpressure_event { + struct eventfd_ctx *efd; + enum vmpressure_levels level; + enum vmpressure_modes mode; + struct list_head node; +}; + +static struct vmpressure *work_to_vmpressure(struct work_struct *work) +{ + return container_of(work, struct vmpressure, work); +} + +static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) +{ + struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); + + memcg = parent_mem_cgroup(memcg); + if (!memcg) + return NULL; + return memcg_to_vmpressure(memcg); +} + +static bool vmpressure_event(struct vmpressure *vmpr, + const enum vmpressure_levels level, + bool ancestor, bool signalled) +{ + struct vmpressure_event *ev; + bool ret = false; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ancestor && ev->mode == VMPRESSURE_LOCAL) + continue; + if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) + continue; + if (level < ev->level) + continue; + eventfd_signal(ev->efd); + ret = true; + } + mutex_unlock(&vmpr->events_lock); + + return ret; +} + +static void vmpressure_work_fn(struct work_struct *work) +{ + struct vmpressure *vmpr = work_to_vmpressure(work); + unsigned long scanned; + unsigned long reclaimed; + enum vmpressure_levels level; + bool ancestor = false; + bool signalled = false; + + spin_lock(&vmpr->sr_lock); + /* + * Several contexts might be calling vmpressure(), so it is + * possible that the work was rescheduled again before the old + * work context cleared the counters. In that case we will run + * just after the old work returns, but then scanned might be zero + * here. No need for any locks here since we don't care if + * vmpr->reclaimed is in sync. + */ + scanned = vmpr->tree_scanned; + if (!scanned) { + spin_unlock(&vmpr->sr_lock); + return; + } + + reclaimed = vmpr->tree_reclaimed; + vmpr->tree_scanned = 0; + vmpr->tree_reclaimed = 0; + spin_unlock(&vmpr->sr_lock); + + level = vmpressure_calc_level(scanned, reclaimed); + + do { + if (vmpressure_event(vmpr, level, ancestor, signalled)) + signalled = true; + ancestor = true; + } while ((vmpr = vmpressure_parent(vmpr))); +} + +/* + * Tree-mode accumulator: accumulate per-memcg scanned/reclaimed and + * schedule the work that walks the parent chain and signals registered + * eventfd listeners once we cross the window threshold. + */ +void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) +{ + spin_lock(&vmpr->sr_lock); + scanned = vmpr->tree_scanned += scanned; + vmpr->tree_reclaimed += reclaimed; + spin_unlock(&vmpr->sr_lock); + + if (scanned < vmpressure_win) + return; + schedule_work(&vmpr->work); +} + +void vmpressure_v1_init(struct vmpressure *vmpr) +{ + mutex_init(&vmpr->events_lock); + INIT_LIST_HEAD(&vmpr->events); + INIT_WORK(&vmpr->work, vmpressure_work_fn); +} + +void vmpressure_v1_cleanup(struct vmpressure *vmpr) +{ + /* + * Make sure there is no pending work before eventfd infrastructure + * goes away. + */ + flush_work(&vmpr->work); +} + +/** + * vmpressure_prio() - Account memory pressure through reclaimer priority level + * @gfp: reclaimer's gfp mask + * @memcg: cgroup memory controller handle + * @prio: reclaimer's priority + * + * This function should be called from the reclaim path every time when + * the vmscan's reclaiming priority (scanning depth) changes. + * + * This function does not return any value. + */ +void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) +{ + /* + * We only use prio for accounting critical level. For more info + * see comment for vmpressure_level_critical_prio variable above. + */ + if (prio > vmpressure_level_critical_prio) + return; + + /* + * OK, the prio is below the threshold, updating vmpressure + * information before shrinker dives into long shrinking of long + * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 + * to the vmpressure() basically means that we signal 'critical' + * level. + */ + vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); +} + +#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) + +/** + * vmpressure_register_event() - Bind vmpressure notifications to an eventfd + * @memcg: memcg that is interested in vmpressure notifications + * @eventfd: eventfd context to link notifications with + * @args: event arguments (pressure level threshold, optional mode) + * + * This function associates eventfd context with the vmpressure + * infrastructure, so that the notifications will be delivered to the + * @eventfd. The @args parameter is a comma-delimited string that denotes a + * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", + * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. + * "hierarchy" or "local"). + * + * To be used as memcg event method. + * + * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could + * not be parsed. + */ +int vmpressure_register_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd, const char *args) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; + enum vmpressure_levels level; + char *spec, *spec_orig; + char *token; + int ret = 0; + + spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); + if (!spec) + return -ENOMEM; + + /* Find required level */ + token = strsep(&spec, ","); + ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); + if (ret < 0) + goto out; + level = ret; + + /* Find optional mode */ + token = strsep(&spec, ","); + if (token) { + ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); + if (ret < 0) + goto out; + mode = ret; + } + + ev = kzalloc_obj(*ev, GFP_KERNEL_ACCOUNT); + if (!ev) { + ret = -ENOMEM; + goto out; + } + + ev->efd = eventfd; + ev->level = level; + ev->mode = mode; + + mutex_lock(&vmpr->events_lock); + list_add(&ev->node, &vmpr->events); + mutex_unlock(&vmpr->events_lock); + ret = 0; +out: + kfree(spec_orig); + return ret; +} + +/** + * vmpressure_unregister_event() - Unbind eventfd from vmpressure + * @memcg: memcg handle + * @eventfd: eventfd context that was used to link vmpressure with the @cg + * + * This function does internal manipulations to detach the @eventfd from + * the vmpressure notifications, and then frees internal resources + * associated with the @eventfd (but the @eventfd itself is not freed). + * + * To be used as memcg event method. + */ +void vmpressure_unregister_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ev->efd != eventfd) + continue; + list_del(&ev->node); + kfree(ev); + break; + } + mutex_unlock(&vmpr->events_lock); +} + static DEFINE_MUTEX(memcg_max_mutex); static int mem_cgroup_resize_max(struct mem_cgroup *memcg, @@ -1513,6 +1804,10 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg, if (!ret) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -1551,6 +1846,10 @@ static int mem_cgroup_force_empty(struct mem_cgroup *memcg) if (signal_pending(current)) return -EINTR; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL)) nr_retries--; @@ -2040,7 +2339,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, struct mem_cgroup *memcg = mem_cgroup_from_css(css); pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org if you " + "Please report your usecase to linux-mm@kvack.org if you " "depend on this functionality.\n"); /* cannot set to root cgroup and only 0 and 1 are allowed */ diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 4fa6e2bc8413..0f703f239c80 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -17,14 +17,8 @@ iter != NULL; \ iter = mem_cgroup_iter(root, iter, NULL)) -#define for_each_mem_cgroup(iter) \ - for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ - iter != NULL; \ - iter = mem_cgroup_iter(NULL, iter, NULL)) - void drain_all_stock(struct mem_cgroup *root_memcg); -unsigned long memcg_events(struct mem_cgroup *memcg, int event); int memory_stat_show(struct seq_file *m, void *v); struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6dc4888a90f3..8319ad8c5c23 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -137,6 +137,14 @@ bool mem_cgroup_kmem_disabled(void) static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); +static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages) +{ + mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); + memcg1_account_kmem(memcg, -nr_pages); + if (!mem_cgroup_is_root(memcg)) + memcg_uncharge(memcg, nr_pages); +} + static void obj_cgroup_release(struct percpu_ref *ref) { struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); @@ -172,10 +180,7 @@ static void obj_cgroup_release(struct percpu_ref *ref) struct mem_cgroup *memcg; memcg = get_mem_cgroup_from_objcg(objcg); - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); - memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - memcg_uncharge(memcg, nr_pages); + memcg_uncharge_kmem(memcg, nr_pages); mem_cgroup_put(memcg); } @@ -2039,7 +2044,7 @@ struct obj_stock_pcp { /* * On rare archs with 256KiB base page size (hexagon and powerpc 44x) * keep nr_bytes to unsigned int as uint16_t cannot represent the full -e patches/memcg-uint16_t-for-nr_bytes-in-obj_stock_pcp.patch * sub-page remainder. Such archs are not cacheline optimization target. + * sub-page remainder. Such archs are not cacheline optimization targets. */ unsigned int nr_bytes[NR_OBJ_STOCK]; #else @@ -3329,10 +3334,7 @@ static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i) memcg = get_mem_cgroup_from_objcg(old); - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); - memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - memcg_uncharge(memcg, nr_pages); + memcg_uncharge_kmem(memcg, nr_pages); css_put(&memcg->css); } @@ -4217,7 +4219,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) /* * A memcg must be visible for expand_shrinker_info() * by the time the maps are allocated. So, we allocate maps - * here, when for_each_mem_cgroup() can't skip it. + * here, when mem_cgroup_iter() can't skip it. */ if (alloc_shrinker_info(memcg)) goto offline_kmem; @@ -4362,6 +4364,11 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); + WRITE_ONCE(memcg->oom_group, false); +#ifdef CONFIG_ZSWAP + WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX); + WRITE_ONCE(memcg->zswap_writeback, true); +#endif #ifdef CONFIG_MEMCG_V1 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); @@ -4428,8 +4435,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac) } #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC -static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, - int cpu) +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) { int nid; @@ -4438,6 +4444,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; + memcg->vmstats->state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4455,9 +4462,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4466,17 +4475,18 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } } } #else -static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, - int cpu) +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent) {} #endif @@ -4488,7 +4498,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) struct aggregate_control ac; int nid; - flush_nmi_stats(memcg, parent, cpu); + flush_nmi_stats(memcg, parent); statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); @@ -4794,6 +4804,10 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -4845,6 +4859,10 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; diff --git a/mm/memfd.c b/mm/memfd.c index abe13b291ddc..c708d92533f4 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -19,6 +19,7 @@ #include <linux/memfd.h> #include <linux/pid_namespace.h> #include <uapi/linux/memfd.h> +#include "internal.h" #include "swap.h" /* @@ -369,39 +370,36 @@ static inline bool is_write_sealed(unsigned int seals) return seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE); } -static int check_write_seal(vm_flags_t *vm_flags_ptr) +static int check_write_seal(vma_flags_t *vma_flags_ptr) { - vm_flags_t vm_flags = *vm_flags_ptr; - vm_flags_t mask = vm_flags & (VM_SHARED | VM_WRITE); - /* If a private mapping then writability is irrelevant. */ - if (!(mask & VM_SHARED)) + if (!vma_flags_test(vma_flags_ptr, VMA_SHARED_BIT)) return 0; /* * New PROT_WRITE and MAP_SHARED mmaps are not allowed when * write seals are active. */ - if (mask & VM_WRITE) + if (vma_flags_test(vma_flags_ptr, VMA_WRITE_BIT)) return -EPERM; /* * This is a read-only mapping, disallow mprotect() from making a * write-sealed mapping writable in future. */ - *vm_flags_ptr &= ~VM_MAYWRITE; + vma_flags_clear(vma_flags_ptr, VMA_MAYWRITE_BIT); return 0; } -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr) +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr) { int err = 0; unsigned int *seals_ptr = memfd_file_seals_ptr(file); unsigned int seals = seals_ptr ? *seals_ptr : 0; if (is_write_sealed(seals)) - err = check_write_seal(vm_flags_ptr); + err = check_write_seal(vma_flags_ptr); return err; } diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51508a55c405..a8b03e2920ba 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -66,6 +66,7 @@ #include <trace/events/memory-failure.h> #include "swap.h" +#include "page_alloc.h" #include "internal.h" static int sysctl_memory_failure_early_kill __read_mostly; @@ -74,9 +75,11 @@ static int sysctl_memory_failure_recovery __read_mostly = 1; static int sysctl_enable_soft_offline __read_mostly = 1; +static int sysctl_panic_on_unrecoverable_mf __read_mostly; + atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); -static bool hw_memory_failure __read_mostly = false; +static bool hw_memory_failure __read_mostly; static DEFINE_MUTEX(mf_mutex); @@ -155,6 +158,15 @@ static const struct ctl_table memory_failure_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, + }, + { + .procname = "panic_on_unrecoverable_memory_failure", + .data = &sysctl_panic_on_unrecoverable_mf, + .maxlen = sizeof(sysctl_panic_on_unrecoverable_mf), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, } }; @@ -552,8 +564,7 @@ static void collect_procs_anon(const struct folio *folio, if (!t) continue; - anon_vma_interval_tree_foreach(vmac, &av->rb_root, - pgoff, pgoff) { + anon_rmap_tree_foreach(vmac, av, pgoff, pgoff) { vma = vmac->vma; if (vma->vm_mm != t->mm) continue; @@ -586,8 +597,7 @@ static void collect_procs_file(const struct folio *folio, if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, - pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { /* * Send early kill signal to tasks where a vma covers * the page but the corrupted page is not necessarily @@ -610,7 +620,7 @@ static void add_to_kill_fsdax(struct task_struct *tsk, const struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, pgoff_t pgoff) { - unsigned long addr = vma_address(vma, pgoff, 1); + unsigned long addr = vma_filebacked_address(vma, pgoff, 1); __add_to_kill(tsk, p, vma, to_kill, addr); } @@ -638,7 +648,7 @@ static void collect_procs_fsdax(const struct page *page, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (vma->vm_mm == t->mm) add_to_kill_fsdax(t, page, vma, to_kill, pgoff); } @@ -981,17 +991,6 @@ static bool has_extra_refcount(struct page_state *ps, struct page *p, } /* - * Error hit kernel page. - * Do nothing, try to be lucky and not touch this instead. For a few cases we - * could be more sophisticated. - */ -static int me_kernel(struct page_state *ps, struct page *p) -{ - unlock_page(p); - return MF_IGNORED; -} - -/* * Page in unknown state. Do nothing. * This is a catch-all in case we fail to make sense of the page state. */ @@ -1199,10 +1198,8 @@ static int me_huge_page(struct page_state *ps, struct page *p) #define mlock (1UL << PG_mlocked) #define lru (1UL << PG_lru) #define head (1UL << PG_head) -#define reserved (1UL << PG_reserved) static struct page_state error_states[] = { - { reserved, reserved, MF_MSG_KERNEL, me_kernel }, /* * free pages are specially detected outside this table: * PG_buddy pages only make a small fraction of all free pages. @@ -1234,7 +1231,6 @@ static struct page_state error_states[] = { #undef mlock #undef lru #undef head -#undef reserved static void update_per_node_mf_stats(unsigned long pfn, enum mf_result result) @@ -1269,6 +1265,15 @@ static void update_per_node_mf_stats(unsigned long pfn, ++mf_stats->total; } +static bool panic_on_unrecoverable_mf(enum mf_action_page_type type, + enum mf_result result) +{ + if (!sysctl_panic_on_unrecoverable_mf) + return false; + + return type == MF_MSG_KERNEL && result == MF_IGNORED; +} + /* * "Dirty/Clean" indication is not 100% accurate due to the possibility of * setting PG_dirty outside page lock. See also comment above set_page_dirty(). @@ -1286,6 +1291,9 @@ static int action_result(unsigned long pfn, enum mf_action_page_type type, pr_err("%#lx: recovery action for %s: %s\n", pfn, action_page_types[type], action_name[result]); + if (panic_on_unrecoverable_mf(type, result)) + panic("Memory failure: %#lx: unrecoverable page", pfn); + return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; } @@ -1339,6 +1347,38 @@ static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) return PageLRU(page) || is_free_buddy_page(page); } +/* + * Positive identification of pages the hwpoison handler cannot recover: + * pages owned by kernel internals with no userspace mapping to unmap, no + * file mapping to invalidate, and no migration target. + */ +static inline bool is_kernel_owned_page(struct page *page) +{ + struct page *head; + bool kernel_owned; + + /* PG_reserved is a per-page flag, never set on a compound page. */ + if (PageReserved(page)) + return true; + + /* + * Page-type bits live only on the head page, so resolve any tail + * first. The check takes no refcount; recheck the head afterwards + * so a concurrent split or compound free cannot leave us trusting + * a stale view. A residual free->alloc->free cannot be closed here + * (frozen slab and large-kmalloc pages cannot be pinned), but is + * harmless: where a wrong verdict could panic, memory_failure() has + * already set PageHWPoison, which bars the page from the allocator. + */ +retry: + head = compound_head(page); + kernel_owned = PageSlab(head) || PageTable(head) || + PageLargeKmalloc(head); + if (head != compound_head(page)) + goto retry; + return kernel_owned; +} + static int __get_hwpoison_page(struct page *page, unsigned long flags) { struct folio *folio = page_folio(page); @@ -1385,6 +1425,19 @@ static int get_any_page(struct page *p, unsigned long flags) if (flags & MF_COUNT_INCREASED) count_increased = true; + /* + * Page types we know are kernel-owned and cannot be recovered. + * Short-circuit before the shake_page() / retry loop, which + * cannot turn any of these into something HWPoisonHandlable(). + * Drop the caller's reference if MF_COUNT_INCREASED took one. + */ + if (is_kernel_owned_page(p)) { + if (count_increased) + put_page(p); + ret = -ENOTRECOVERABLE; + goto out; + } + try_again: if (!count_increased) { ret = __get_hwpoison_page(p, flags); @@ -1432,7 +1485,7 @@ try_again: ret = -EIO; } out: - if (ret == -EIO) + if (ret == -EIO || ret == -ENOTRECOVERABLE) pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); return ret; @@ -1489,7 +1542,10 @@ static int __get_unpoison_page(struct page *page) * -EIO for pages on which we can not handle memory errors, * -EBUSY when get_hwpoison_page() has raced with page lifecycle * operations like allocation and free, - * -EHWPOISON when the page is hwpoisoned and taken off from buddy. + * -EHWPOISON when the page is hwpoisoned and taken off from buddy, + * -ENOTRECOVERABLE for kernel-owned pages identified by + * is_kernel_owned_page() (PG_reserved, slab, + * page-table, large-kmalloc) that the handler cannot recover. */ static int get_hwpoison_page(struct page *p, unsigned long flags) { @@ -2209,7 +2265,7 @@ static void add_to_kill_pgoff(struct task_struct *tsk, } /* Check for pgoff not backed by struct page */ - tk->addr = vma_address(vma, pgoff, 1); + tk->addr = vma_filebacked_address(vma, pgoff, 1); tk->size_shift = PAGE_SHIFT; if (tk->addr == -EFAULT) @@ -2239,7 +2295,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, mapping, 0, ULONG_MAX) { pgoff_t pgoff; if (vma->vm_mm == t->mm && @@ -2402,7 +2458,8 @@ try_again: * that may make page_ref_freeze()/page_ref_unfreeze() mismatch. */ res = get_hwpoison_page(p, flags); - if (!res) { + switch (res) { + case 0: if (is_free_buddy_page(p)) { if (take_page_off_buddy(p)) { page_ref_inc(p); @@ -2421,7 +2478,19 @@ try_again: res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); } goto unlock_mutex; - } else if (res < 0) { + case 1: + /* Got a refcount on a handlable page. */ + break; + case -ENOTRECOVERABLE: + /* + * Stable unhandlable kernel-owned page (PG_reserved, + * slab, page tables, large-kmalloc). + * No recovery possible. + */ + res = action_result(pfn, MF_MSG_KERNEL, MF_IGNORED); + goto unlock_mutex; + default: + /* Transient lifecycle race with the page allocator. */ res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); goto unlock_mutex; } diff --git a/mm/memory.c b/mm/memory.c index ff338c2abe92..56b244552f13 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -519,9 +519,52 @@ static bool is_bad_page_map_ratelimited(void) return false; } +static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size) +{ + if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) { + snprintf(buf, buf_size, "overflow"); + return; + } + + switch (entry_size) { + case sizeof(u32): + snprintf(buf, buf_size, "%08x", *(const u32 *)entry); + break; + case sizeof(u64): + snprintf(buf, buf_size, "%016llx", *(const u64 *)entry); + break; +#if defined(__SIZEOF_INT128__) + case sizeof(u128): + snprintf(buf, buf_size, "%016llx%016llx", + (unsigned long long)(*(const u128 *)entry >> 64), + (unsigned long long)*(const u128 *)entry); + break; +#endif + default: + snprintf(buf, buf_size, "unsupported"); + break; + } +} + +#define ptval_to_str(buf, val) \ + do { \ + auto __val = (val); \ + \ + ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ + } while (0) + +#if defined(__SIZEOF_INT128__) +#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ +#else +#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ +#endif + static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr) { - unsigned long long pgdv, p4dv, pudv, pmdv; + char pgd_str[PTVAL_STR_MAX]; + char p4d_str[PTVAL_STR_MAX]; + char pud_str[PTVAL_STR_MAX]; + char pmd_str[PTVAL_STR_MAX]; p4d_t p4d, *p4dp; pud_t pud, *pudp; pmd_t pmd, *pmdp; @@ -532,34 +575,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * see locking requirements for print_bad_page_map(). */ pgdp = pgd_offset(mm, addr); - pgdv = pgd_val(*pgdp); + ptval_to_str(pgd_str, pgd_val(*pgdp)); if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) { - pr_alert("pgd:%08llx\n", pgdv); + pr_alert("pgd:%s\n", pgd_str); return; } p4dp = p4d_offset(pgdp, addr); p4d = p4dp_get(p4dp); - p4dv = p4d_val(p4d); + ptval_to_str(p4d_str, p4d_val(p4d)); if (!p4d_present(p4d) || p4d_leaf(p4d)) { - pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); + pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str); return; } pudp = pud_offset(p4dp, addr); pud = pudp_get(pudp); - pudv = pud_val(pud); + ptval_to_str(pud_str, pud_val(pud)); if (!pud_present(pud) || pud_leaf(pud)) { - pr_alert("pgd:%08llx p4d:%08llx pud:%08llx\n", pgdv, p4dv, pudv); + pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str); return; } pmdp = pmd_offset(pudp, addr); pmd = pmdp_get(pmdp); - pmdv = pmd_val(pmd); + ptval_to_str(pmd_str, pmd_val(pmd)); /* * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE, @@ -567,8 +610,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * doing another map would be bad. print_bad_page_map() should * already take care of printing the PTE. */ - pr_alert("pgd:%08llx p4d:%08llx pud:%08llx pmd:%08llx\n", pgdv, - p4dv, pudv, pmdv); + pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str); } /* @@ -584,25 +626,29 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * page table lock. */ static void print_bad_page_map(struct vm_area_struct *vma, - unsigned long addr, unsigned long long entry, struct page *page, - enum pgtable_level level) + unsigned long addr, const void *entry, size_t entry_size, + struct page *page, enum pgtable_level level) { struct address_space *mapping; - pgoff_t index; + char entry_str[PTVAL_STR_MAX]; + pgoff_t index, virt_index; if (is_bad_page_map_ratelimited()) return; mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; index = linear_page_index(vma, addr); + virt_index = __linear_virt_page_index(vma, addr); - pr_alert("BUG: Bad page map in process %s %s:%08llx", current->comm, - pgtable_level_to_str(level), entry); + ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size); + pr_alert("BUG: Bad page map in process %s %s:%s", current->comm, + pgtable_level_to_str(level), entry_str); __print_bad_page_map_pgtable(vma->vm_mm, addr); if (page) dump_page(page, "bad page map"); - pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n", - (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); + pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx virt_index:%lx\n", + (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index, + virt_index); pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n", vma->vm_file, vma->vm_ops ? vma->vm_ops->fault : NULL, @@ -627,8 +673,13 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) } } -#define print_bad_pte(vma, addr, pte, page) \ - print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE) +static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, + pte_t pte, struct page *page) +{ + auto entry = pte_val(pte); + + print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE); +} /** * __vm_normal_page() - Get the "struct page" associated with a page table entry. @@ -636,8 +687,9 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) * @addr: The address where the page table entry is mapped. * @pfn: The PFN stored in the page table entry. * @special: Whether the page table entry is marked "special". - * @level: The page table level for error reporting purposes only. * @entry: The page table entry value for error reporting purposes only. + * @entry_size: The size of @entry. + * @level: The page table level for error reporting purposes only. * * "Special" mappings do not wish to be associated with a "struct page" (either * it doesn't exist, or it exists but they don't want to touch it). In this @@ -697,7 +749,7 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level) */ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, bool special, - unsigned long long entry, enum pgtable_level level) + const void *entry, size_t entry_size, enum pgtable_level level) { if (pgtable_level_has_pxx_special(level)) { if (unlikely(special)) { @@ -710,7 +762,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return NULL; - print_bad_page_map(vma, addr, entry, NULL, level); + print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* @@ -725,10 +777,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (!pfn_valid(pfn)) return NULL; } else { - unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT; + const pgoff_t index = linear_page_index(vma, addr); /* Only CoW'ed anon folios are "normal". */ - if (pfn == vma->vm_pgoff + off) + if (pfn == index) return NULL; if (!is_cow_mapping(vma->vm_flags)) return NULL; @@ -741,7 +793,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, if (unlikely(pfn > highest_memmap_pfn)) { /* Corrupted page table entry. */ - print_bad_page_map(vma, addr, entry, NULL, level); + print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* @@ -767,8 +819,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma, struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { + auto entry = pte_val(pte); + return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte), - pte_val(pte), PGTABLE_LEVEL_PTE); + &entry, sizeof(entry), PGTABLE_LEVEL_PTE); } /** @@ -809,8 +863,10 @@ struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd) { + auto entry = pmd_val(pmd); + return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd), - pmd_val(pmd), PGTABLE_LEVEL_PMD); + &entry, sizeof(entry), PGTABLE_LEVEL_PMD); } /** @@ -850,8 +906,10 @@ struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud) { + auto entry = pud_val(pud); + return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud), - pud_val(pud), PGTABLE_LEVEL_PUD); + &entry, sizeof(entry), PGTABLE_LEVEL_PUD); } #endif @@ -893,8 +951,12 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, if (pte_swp_soft_dirty(orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_mkuffd(pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma)) + pte = pte_modify(pte, PAGE_NONE); if ((vma->vm_flags & VM_WRITE) && can_change_pte_writable(vma, address, pte)) { @@ -953,14 +1015,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (swap_dup_entry_direct(entry) < 0) return -EIO; - /* make sure dst_mm is on swapoff's mmlist. */ - if (unlikely(list_empty(&dst_mm->mmlist))) { - spin_lock(&mmlist_lock); - if (list_empty(&dst_mm->mmlist)) - list_add(&dst_mm->mmlist, - &src_mm->mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(dst_mm); /* Mark the swap entry as shared. */ if (pte_swp_exclusive(orig_pte)) { pte = pte_swp_clear_exclusive(orig_pte); @@ -984,8 +1039,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_private(entry)) { @@ -1018,8 +1073,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_device_private_entry( swp_offset(entry)); pte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_exclusive(entry)) { @@ -1041,8 +1096,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, make_pte_marker(marker)); return 0; } - if (!userfaultfd_wp(dst_vma)) - pte = pte_swp_clear_uffd_wp(pte); + if (!userfaultfd_protected(dst_vma)) + pte = pte_swp_clear_uffd(pte); set_pte_at(dst_mm, addr, dst_pte, pte); return 0; } @@ -1088,9 +1143,13 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma /* All done, just insert the new page copy in the child */ pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); - if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte))) - /* Uffd-wp needs to be delivered to dest pte as well */ - pte = pte_mkuffd_wp(pte); + if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) { + /* The uffd bit needs to be delivered to the dest pte as well */ + pte = pte_mkuffd(pte); + /* Restore PAGE_NONE so the RWP marker keeps trapping */ + if (userfaultfd_rwp(dst_vma)) + pte = pte_modify(pte, PAGE_NONE); + } set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); return 0; } @@ -1100,9 +1159,31 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte_t pte, unsigned long addr, int nr) { struct mm_struct *src_mm = src_vma->vm_mm; + bool writable; + + /* + * Snapshot writability before the RWP-disarm rewrite below: when the + * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can + * silently drop _PAGE_RW from a resolved (no-marker) writable PTE, + * so a later pte_write(pte) check would skip the COW wrprotect and + * leave the parent writable over a folio shared with the child. + */ + writable = pte_write(pte); + + /* + * Child is not RWP-armed: restore accessible protection so the + * inherited PAGE_NONE does not cost a fault on first read. Gate on + * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing) + * are not normalised away. + */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pte_uffd(pte)) + pte = pte_modify(pte, dst_vma->vm_page_prot); + pte = pte_clear_uffd(pte); + } /* If it's a COW mapping, write protect it both processes. */ - if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) { + if (is_cow_mapping(src_vma->vm_flags) && writable) { wrprotect_ptes(src_mm, addr, src_pte, nr); pte = pte_wrprotect(pte); } @@ -1112,9 +1193,6 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte = pte_mkclean(pte); pte = pte_mkold(pte); - if (!userfaultfd_wp(dst_vma)) - pte = pte_clear_uffd_wp(pte); - set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); } @@ -1797,7 +1875,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } - clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr); *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); return nr; @@ -2643,7 +2721,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma_start_pgoff(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -3298,7 +3376,8 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn; int err; - err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + err = __simple_ioremap_prep(vm_len, vma_start_pgoff(vma), start, len, + &pfn); if (err) return err; @@ -3925,8 +4004,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (unlikely(unshare)) { if (pte_soft_dirty(vmf->orig_pte)) entry = pte_mksoft_dirty(entry); - if (pte_uffd_wp(vmf->orig_pte)) - entry = pte_mkuffd_wp(entry); + if (pte_uffd(vmf->orig_pte)) + entry = pte_mkuffd(entry); } else { entry = maybe_mkwrite(pte_mkdirty(entry), vma); } @@ -4181,6 +4260,9 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { + const bool maybe_in_lru_cache = !folio_test_lru(folio); + const bool in_swapcache = folio_test_swapcache(folio); + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); @@ -4191,15 +4273,16 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, * * KSM doesn't necessarily raise the folio refcount. */ - if (folio_test_ksm(folio) || folio_ref_count(folio) > 3) + if (folio_test_ksm(folio) || + folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache) return false; - if (!folio_test_lru(folio)) + if (maybe_in_lru_cache) /* * We cannot easily detect+handle references from * remote LRU caches or references to LRU folios. */ lru_add_drain(); - if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio)) + if (folio_ref_count(folio) > 1 + in_swapcache) return false; if (!folio_trylock(folio)) return false; @@ -4261,7 +4344,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) * etc.) because we're only removing the uffd-wp bit, * which is completely invisible to the user. */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + pte = pte_clear_uffd(ptep_get(vmf->pte)); set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); /* @@ -4336,21 +4419,21 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_copy(vmf); } -static inline void unmap_mapping_range_tree(struct rb_root_cached *root, +static inline void unmap_mapping_range_tree(struct address_space *mapping, pgoff_t first_index, pgoff_t last_index, struct zap_details *details) { struct vm_area_struct *vma; - unsigned long start, size; struct mmu_gather tlb; - vma_interval_tree_foreach(vma, root, first_index, last_index) { - const pgoff_t start_idx = max(first_index, vma->vm_pgoff); + mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) { + const pgoff_t start_idx = max(first_index, vma_start_pgoff(vma)); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; - - start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); - size = (end_idx - start_idx) << PAGE_SHIFT; + const pgoff_t offset = start_idx - vma_start_pgoff(vma); + const unsigned long offset_bytes = offset << PAGE_SHIFT; + const unsigned long start = vma->vm_start + offset_bytes; + const unsigned long size = (end_idx - start_idx) << PAGE_SHIFT; tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); @@ -4387,7 +4470,7 @@ void unmap_mapping_folio(struct folio *folio) i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } @@ -4417,7 +4500,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } @@ -4512,7 +4595,7 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf) static inline bool should_try_to_free_swap(struct swap_info_struct *si, struct folio *folio, struct vm_area_struct *vma, - unsigned int extra_refs, + bool exclusive, unsigned int fault_flags) { if (!folio_test_swapcache(folio)) @@ -4528,14 +4611,12 @@ static inline bool should_try_to_free_swap(struct swap_info_struct *si, if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) || folio_test_mlocked(folio)) return true; + /* - * If we want to map a page that's in the swapcache writable, we - * have to detect via the refcount if we're really the exclusive - * user. Try freeing the swapcache to get rid of the swapcache - * reference only in case it's likely that we'll be the exclusive user. + * Free the swapcache only if we are the exclusive user and + * this is a write fault. */ - return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) && - folio_ref_count(folio) == (extra_refs + folio_nr_pages(folio)); + return (fault_flags & FAULT_FLAG_WRITE) && exclusive; } static vm_fault_t pte_marker_clear(struct vm_fault *vmf) @@ -4752,6 +4833,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) struct swap_info_struct *si = NULL; rmap_t rmap_flags = RMAP_NONE; bool exclusive = false; + bool rwp_restore = false; softleaf_t entry; pte_t pte; vm_fault_t ret = 0; @@ -4897,16 +4979,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) } else if (folio != swapcache) page = folio_page(folio, 0); - /* - * If we want to map a page that's in the swapcache writable, we - * have to detect via the refcount if we're really the exclusive - * owner. Try removing the extra reference from the local LRU - * caches if required. - */ - if ((vmf->flags & FAULT_FLAG_WRITE) && - !folio_test_ksm(folio) && !folio_test_lru(folio)) - lru_add_drain(); - folio_throttle_swaprate(folio, GFP_KERNEL); /* @@ -5038,24 +5110,34 @@ check_folio: pte = mk_pte(page, vma->vm_page_prot); if (pte_swp_soft_dirty(vmf->orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(vmf->orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(vmf->orig_pte)) + pte = pte_mkuffd(pte); /* - * Same logic as in do_wp_page(); however, optimize for pages that are - * certainly not shared either because we just allocated them without - * exposing them to the swapcache or because the swap entry indicates - * exclusivity. + * A page reclaimed while RWP-protected carries the uffd bit on + * its swap entry. Re-apply PAGE_NONE on swap-in so the first access + * still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD. + */ + if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) { + pte = pte_modify(pte, PAGE_NONE); + rwp_restore = true; + } + + /* + * Similar logic as in do_wp_page(); however, optimize for pages that + * are certainly exclusive. + * + * Skip the write upgrade for an RWP-restored pte: it must stay + * PROT_NONE so the access retries through the RWP fault path + * (do_uffd_rwp()) rather than being made writable here. */ - if (!folio_test_ksm(folio) && - (exclusive || folio_ref_count(folio) == 1)) { - if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + if (exclusive) { + if (!rwp_restore && + (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && !pte_needs_soft_dirty_wp(vma, pte)) { pte = pte_mkwrite(pte, vma); - if (vmf->flags & FAULT_FLAG_WRITE) { + if (vmf->flags & FAULT_FLAG_WRITE) pte = pte_mkdirty(pte); - vmf->flags &= ~FAULT_FLAG_WRITE; - } } rmap_flags |= RMAP_EXCLUSIVE; } @@ -5095,7 +5177,7 @@ check_folio: * Do it after mapping, so raced page faults will likely see the folio * in swap cache and wait on the folio lock. */ - if (should_try_to_free_swap(si, folio, vma, nr_pages, vmf->flags)) + if (should_try_to_free_swap(si, folio, vma, exclusive, vmf->flags)) folio_free_swap(folio); folio_unlock(folio); @@ -5112,7 +5194,12 @@ check_folio: folio_put(swapcache); } - if (vmf->flags & FAULT_FLAG_WRITE) { + /* + * For an RWP-restored pte, leave it PROT_NONE and let the write + * retry through the RWP fault path; do not COW it here, which would + * drop the marker for a non-exclusive page. + */ + if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) { ret |= do_wp_page(vmf); if (ret & VM_FAULT_ERROR) ret &= VM_FAULT_ERROR; @@ -5259,7 +5346,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); @@ -5326,7 +5413,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) return handle_userfault(vmf, VM_UFFD_MISSING); } if (vmf_orig_pte_uffd_wp(vmf)) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); set_pte_at(vma->vm_mm, addr, vmf->pte, entry); /* No need to invalidate - it was non-present before */ @@ -5576,7 +5663,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, else if (pte_write(entry) && folio_test_dirty(folio)) entry = pte_mkdirty(entry); if (unlikely(vmf_orig_pte_uffd_wp(vmf))) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { VM_BUG_ON_FOLIO(nr != 1, folio); @@ -5684,7 +5771,7 @@ fallback: } else if (nr_pages > 1) { pgoff_t idx = folio_page_idx(folio, page); /* The page offset of vmf->address within the VMA. */ - pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; + pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); /* The index of the entry in the pagetable for fault page. */ pgoff_t pte_off = pte_index(vmf->address); @@ -5796,7 +5883,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) pgoff_t nr_pages = READ_ONCE(fault_around_pages); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ - pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; + pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); pgoff_t from_pte, to_pte; vm_fault_t ret; @@ -6091,6 +6178,16 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru if (!pte_present(ptent) || !pte_protnone(ptent)) continue; + /* + * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a + * marker. Leave them alone -- rewriting to vm_page_prot would + * stop the RWP trap. Gate on userfaultfd_rwp(vma) too: + * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs + * when applying PROT_NONE, and those still need rebuilding. + */ + if (userfaultfd_rwp(vma) && pte_uffd(ptent)) + continue; + if (pfn_folio(pte_pfn(ptent)) != folio) continue; @@ -6106,6 +6203,35 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru } } +static vm_fault_t do_uffd_rwp(struct vm_fault *vmf) +{ + pte_t pte; + + if (!userfaultfd_rwp_async(vmf->vma)) { + /* Sync mode: unmap PTE and deliver to userfaultfd handler */ + pte_unmap(vmf->pte); + return handle_userfault(vmf, VM_UFFD_RWP); + } + + spin_lock(vmf->ptl); + if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return 0; + } + pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot); + /* pte_modify() preserves _PAGE_UFFD; drop it on resolution */ + pte = pte_clear_uffd(pte); + pte = pte_mkyoung(pte); + if (!pte_write(pte) && + vma_wants_manual_pte_write_upgrade(vmf->vma) && + can_change_pte_writable(vmf->vma, vmf->address, pte)) + pte = pte_mkwrite(pte, vmf->vma); + set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte); + update_mmu_cache(vmf->vma, vmf->address, vmf->pte); + pte_unmap_unlock(vmf->pte, vmf->ptl); + return 0; +} + static vm_fault_t do_numa_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -6381,8 +6507,16 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) if (!pte_present(vmf->orig_pte)) return do_swap_page(vmf); - if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) + if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) { + /* + * RWP-protected PTEs are protnone plus the uffd bit. On a + * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is + * NUMA hinting and must still fall through to do_numa_page(). + */ + if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) + return do_uffd_rwp(vmf); return do_numa_page(vmf); + } spin_lock(vmf->ptl); entry = vmf->orig_pte; @@ -6496,8 +6630,11 @@ retry_pud: return 0; } if (pmd_trans_huge(vmf.orig_pmd)) { - if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) + if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) { + if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd)) + return do_huge_pmd_uffd_rwp(&vmf); return do_huge_pmd_numa_page(&vmf); + } if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && !pmd_write(vmf.orig_pmd)) { @@ -7274,7 +7411,7 @@ void print_vma_addr(char *prefix, unsigned long ip) if (vma && vma->vm_file) { struct file *f = vma->vm_file; ip -= vma->vm_start; - ip += vma->vm_pgoff << PAGE_SHIFT; + ip += vma_start_pgoff(vma) << PAGE_SHIFT; printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip, vma->vm_start, vma->vm_end - vma->vm_start); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 8b137328dcf0..226ab9cb078a 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -40,6 +40,8 @@ #include <asm/tlbflush.h> #include "internal.h" +#include "mm_init.h" +#include "page_alloc.h" #include "shuffle.h" enum { @@ -239,6 +241,7 @@ enum mmop mhp_get_default_online_type(void) return mhp_default_online_type; } +EXPORT_SYMBOL_GPL(mhp_get_default_online_type); void mhp_set_default_online_type(enum mmop online_type) { @@ -1338,7 +1341,9 @@ static int check_hotplug_memory_range(u64 start, u64 size) static int online_memory_block(struct memory_block *mem, void *arg) { - mem->online_type = mhp_get_default_online_type(); + enum mmop *online_type = arg; + + mem->online_type = *online_type; return device_online(&mem->dev); } @@ -1492,7 +1497,8 @@ out: * * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ -int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +static int __add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags, + enum mmop online_type) { struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) }; enum memblock_flags memblock_flags = MEMBLOCK_NONE; @@ -1582,8 +1588,9 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) merge_system_ram_resource(res); /* online pages if requested */ - if (mhp_get_default_online_type() != MMOP_OFFLINE) - walk_memory_blocks(start, size, NULL, online_memory_block); + if (online_type != MMOP_OFFLINE) + walk_memory_blocks(start, size, &online_type, + online_memory_block); return ret; error: @@ -1599,7 +1606,13 @@ error_mem_hotplug_end: return ret; } -/* requires device_hotplug_lock, see add_memory_resource() */ +int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +{ + return __add_memory_resource(nid, res, mhp_flags, + mhp_get_default_online_type()); +} + +/* requires device_hotplug_lock, see __add_memory_resource() */ int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) { struct resource *res; @@ -1627,7 +1640,15 @@ int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) } EXPORT_SYMBOL_GPL(add_memory); -/* +/** + * __add_memory_driver_managed - add driver-managed memory with explicit online_type + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * @online_type: Auto-Online behavior (offline, online, kernel, movable) + * * Add special, driver-managed memory to the system as system RAM. Such * memory is not exposed via the raw firmware-provided memmap as system * RAM, instead, it is detected and added by a driver - during cold boot, @@ -1635,6 +1656,7 @@ EXPORT_SYMBOL_GPL(add_memory); * * Reasons why this memory should not be used for the initial memmap of a * kexec kernel or for placing kexec images: + * * - The booting kernel is in charge of determining how this memory will be * used (e.g., use persistent memory as system RAM) * - Coordination with a hypervisor is required before this memory @@ -1647,9 +1669,12 @@ EXPORT_SYMBOL_GPL(add_memory); * * The resource_name (visible via /proc/iomem) has to have the format * "System RAM ($DRIVER)". + * + * Return: 0 on success, negative error code on failure. */ -int add_memory_driver_managed(int nid, u64 start, u64 size, - const char *resource_name, mhp_t mhp_flags) +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type) { struct resource *res; int rc; @@ -1659,6 +1684,9 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, resource_name[strlen(resource_name) - 1] != ')') return -EINVAL; + if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE) + return -EINVAL; + lock_device_hotplug(); res = register_memory_resource(start, size, resource_name); @@ -1667,7 +1695,7 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, goto out_unlock; } - rc = add_memory_resource(nid, res, mhp_flags); + rc = __add_memory_resource(nid, res, mhp_flags, online_type); if (rc < 0) release_memory_resource(res); @@ -1675,6 +1703,30 @@ out_unlock: unlock_device_hotplug(); return rc; } +EXPORT_SYMBOL_FOR_MODULES(__add_memory_driver_managed, "kmem"); + +/** + * add_memory_driver_managed - add driver-managed memory + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * + * Add driver-managed memory with the system default online type set by + * build config or kernel boot parameter. + * + * See __add_memory_driver_managed for more details. + * + * Return: 0 on success, negative error code on failure. + */ +int add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags) +{ + return __add_memory_driver_managed(nid, start, size, resource_name, + mhp_flags, + mhp_get_default_online_type()); +} EXPORT_SYMBOL_GPL(add_memory_driver_managed); /* @@ -2380,58 +2432,98 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg) */ int offline_and_remove_memory(u64 start, u64 size) { - const unsigned long mb_count = size / memory_block_size_bytes(); + struct range range = { + .start = start, + .end = start + size - 1, + }; + + return offline_and_remove_memory_ranges(&range, 1); +} +EXPORT_SYMBOL_GPL(offline_and_remove_memory); + +/** + * offline_and_remove_memory_ranges - offline and remove multiple memory ranges + * @ranges: array of physical address ranges to offline and remove + * @nr_ranges: number of entries in @ranges + * + * Offline and remove several memory ranges as one operation, serialized + * against other hotplug operations by a single lock_device_hotplug(). + * + * This offlines all ranges before removing any of them. If offlining any + * range fails, the entire process is reverted and nothing is removed. + * This provides a fully atomic semantic for unplugging an entire device. + * + * Each range must be memory-block aligned in start and size. + * + * Return: 0 on success, negative errno on failure (never positive). On + * failure no range has been removed. + */ +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + unsigned long mb_count = 0; uint8_t *online_types, *tmp; - int rc; + unsigned int i; + int rc = 0; - if (!IS_ALIGNED(start, memory_block_size_bytes()) || - !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + if (!ranges || !nr_ranges) return -EINVAL; + for (i = 0; i < nr_ranges; i++) { + const u64 start = ranges[i].start; + const u64 size = range_len(&ranges[i]); + + if (!IS_ALIGNED(start, memory_block_size_bytes()) || + !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + return -EINVAL; + mb_count += size / memory_block_size_bytes(); + } + /* - * We'll remember the old online type of each memory block, so we can - * try to revert whatever we did when offlining one memory block fails - * after offlining some others succeeded. + * Remember the old online type of every memory block across all ranges, + * so we can revert if offlining a later block fails. All entries start + * as MMOP_OFFLINE so blocks we never touched are skipped on rollback. */ online_types = kmalloc_array(mb_count, sizeof(*online_types), GFP_KERNEL); if (!online_types) return -ENOMEM; - /* - * Initialize all states to MMOP_OFFLINE, so when we abort processing in - * try_offline_memory_block(), we'll skip all unprocessed blocks in - * try_reonline_memory_block(). - */ memset(online_types, MMOP_OFFLINE, mb_count); lock_device_hotplug(); - tmp = online_types; - rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block); - /* - * In case we succeeded to offline all memory, remove it. - * This cannot fail as it cannot get onlined in the meantime. + * Phase 1: offline every block in every range. An already-offline + * block folds to success, so out-of-band offlining never blocks unplug. */ - if (!rc) { - rc = try_remove_memory(start, size); + tmp = online_types; + for (i = 0; i < nr_ranges; i++) { + rc = walk_memory_blocks(ranges[i].start, range_len(&ranges[i]), + &tmp, try_offline_memory_block); if (rc) - pr_err("%s: Failed to remove memory: %d", __func__, rc); + break; } - /* - * Rollback what we did. While memory onlining might theoretically fail - * (nacked by a notifier), it barely ever happens. - */ + /* If any failure occurred at all, rollback any changes and bail */ if (rc) { tmp = online_types; - walk_memory_blocks(start, size, &tmp, - try_reonline_memory_block); + for (i = 0; i < nr_ranges; i++) + walk_memory_blocks(ranges[i].start, + range_len(&ranges[i]), &tmp, + try_reonline_memory_block); + goto out_unlock; } + + /* Phase 2: Remove. This should never fail holding the hotplug lock */ + for (i = 0; i < nr_ranges; i++) + WARN_ON_ONCE(try_remove_memory(ranges[i].start, + range_len(&ranges[i]))); + +out_unlock: unlock_device_hotplug(); kfree(online_types); return rc; } -EXPORT_SYMBOL_GPL(offline_and_remove_memory); +EXPORT_SYMBOL_GPL(offline_and_remove_memory_ranges); #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 36699fabd3c2..5720f7f54d94 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -119,6 +119,7 @@ #include <linux/memory.h> #include "internal.h" +#include "page_alloc.h" /* Internal flags */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ @@ -653,12 +654,14 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) { struct folio *folio; struct queue_pages *qp = walk->private; + pmd_t pmdval = pmdp_get(pmd); - if (unlikely(pmd_is_migration_entry(*pmd))) { - qp->nr_failed++; + if (unlikely(!pmd_present(pmdval))) { + if (pmd_is_migration_entry(pmdval)) + qp->nr_failed++; return; } - folio = pmd_folio(*pmd); + folio = pmd_folio(pmdval); if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; @@ -2048,8 +2051,8 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, pol = get_task_policy(current); if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { - *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += vma_start_pgoff(vma) >> order; + *ilx += linear_page_delta(vma, addr) >> order; } return pol; } @@ -2057,24 +2060,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma) { struct mempolicy *pol; + pgoff_t ilx; + bool mof; - if (vma->vm_ops && vma->vm_ops->get_policy) { - bool ret = false; - pgoff_t ilx; /* ignored here */ - - pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx); - if (pol && (pol->flags & MPOL_F_MOF)) - ret = true; - mpol_cond_put(pol); - - return ret; - } - - pol = vma->vm_policy; + pol = __get_vma_policy(vma, vma->vm_start, &ilx); if (!pol) pol = get_task_policy(current); - - return pol->flags & MPOL_F_MOF; + mof = pol->flags & MPOL_F_MOF; + mpol_cond_put(pol); + return mof; } bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) @@ -2425,9 +2419,11 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, */ preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask); + page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, + ALLOC_DEFAULT); if (!page) - page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL); + page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, + ALLOC_DEFAULT); return page; } @@ -2475,7 +2471,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, */ page = __alloc_frozen_pages_noprof( gfp | __GFP_THISNODE | __GFP_NORETRY, order, - nid, NULL); + nid, NULL, ALLOC_DEFAULT); if (page || !(gfp & __GFP_DIRECT_RECLAIM)) return page; /* @@ -2487,7 +2483,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } - page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask); + page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, ALLOC_DEFAULT); if (unlikely(pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) { @@ -3250,16 +3246,17 @@ EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_init, "kvm"); int mpol_set_shared_policy(struct shared_policy *sp, struct vm_area_struct *vma, struct mempolicy *pol) { - int err; + const pgoff_t pgoff = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); struct sp_node *new = NULL; - unsigned long sz = vma_pages(vma); + int err; if (pol) { - new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); + new = sp_alloc(pgoff, pgoff_end, pol); if (!new) return -ENOMEM; } - err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new); + err = shared_policy_replace(sp, pgoff, pgoff_end, new); if (err && new) sp_free(new); return err; diff --git a/mm/migrate.c b/mm/migrate.c index dd15a84b2a52..ab15a4dddd04 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -49,6 +49,7 @@ #include <trace/events/migrate.h> #include "internal.h" +#include "page_alloc.h" #include "swap.h" static const struct movable_operations *offline_movable_ops; @@ -326,8 +327,12 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, if (pte_swp_soft_dirty(old_pte)) newpte = pte_mksoft_dirty(newpte); - if (pte_swp_uffd_wp(old_pte)) - newpte = pte_mkuffd_wp(newpte); + if (pte_swp_uffd(old_pte)) + newpte = pte_mkuffd(newpte); + + /* See remove_migration_pte(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma)) + newpte = pte_modify(newpte, PAGE_NONE); set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); @@ -358,11 +363,13 @@ static bool remove_migration_pte(struct folio *folio, unsigned long idx = 0; /* pgoff is invalid for ksm pages, but they are never large */ - if (folio_test_large(folio) && !folio_test_hugetlb(folio)) - idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; + if (folio_test_large(folio) && !folio_test_hugetlb(folio)) { + idx += linear_folio_page_index(folio, vma, pvmw.address); + idx -= pvmw.pgoff; + } new = folio_page(folio, idx); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || @@ -371,7 +378,11 @@ static bool remove_migration_pte(struct folio *folio, continue; } #endif - old_pte = ptep_get(pvmw.pte); + if (folio_test_hugetlb(folio)) + old_pte = huge_ptep_get(vma->vm_mm, pvmw.address, + pvmw.pte); + else + old_pte = ptep_get(pvmw.pte); if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; @@ -391,8 +402,12 @@ static bool remove_migration_pte(struct folio *folio, if (softleaf_is_migration_write(entry)) pte = pte_mkwrite(pte, vma); - else if (pte_swp_uffd_wp(old_pte)) - pte = pte_mkuffd_wp(pte); + else if (pte_swp_uffd(old_pte)) + pte = pte_mkuffd(pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma)) + pte = pte_modify(pte, PAGE_NONE); if (folio_test_anon(folio) && !softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; @@ -407,8 +422,8 @@ static bool remove_migration_pte(struct folio *folio, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(old_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(old_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(old_pte)) + pte = pte_swp_mkuffd(pte); } #ifdef CONFIG_HUGETLB_PAGE @@ -545,7 +560,7 @@ fail: } #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; @@ -885,7 +900,7 @@ static int __migrate_folio(struct address_space *mapping, struct folio *dst, * @mapping: The address_space containing the folio. * @dst: The folio to migrate the data to. * @src: The folio containing the current data. - * @mode: How to migrate the page. + * @mode: How to migrate the folio. * * Common logic to directly migrate a single LRU folio suitable for * folios that do not have private data. @@ -1131,7 +1146,7 @@ static int move_to_new_folio(struct folio *dst, struct folio *src, } /* - * To record some information during migration, we use unused private + * To record some information during migration, we use the migrate_info * field of struct folio of the newly allocated destination folio. * This is safe because nobody is using it except us. */ @@ -1144,27 +1159,24 @@ enum { static void __migrate_folio_record(struct folio *dst, int old_folio_state, struct anon_vma *anon_vma) { - dst->private = (void *)anon_vma + old_folio_state; + dst->migrate_info = (unsigned long)anon_vma | old_folio_state; } static void __migrate_folio_extract(struct folio *dst, int *old_folio_state, struct anon_vma **anon_vmap) { - unsigned long private = (unsigned long)dst->private; + unsigned long info = dst->migrate_info; - *anon_vmap = (struct anon_vma *)(private & ~FOLIO_OLD_STATES); - *old_folio_state = private & FOLIO_OLD_STATES; - dst->private = NULL; + *anon_vmap = (struct anon_vma *)(info & ~FOLIO_OLD_STATES); + *old_folio_state = info & FOLIO_OLD_STATES; + dst->migrate_info = 0; } /* Restore the source folio to the original state upon failure */ -static void migrate_folio_undo_src(struct folio *src, - int page_was_mapped, - struct anon_vma *anon_vma, - bool locked, - struct list_head *ret) +static void migrate_folio_undo_src(struct folio *src, int was_mapped, + struct anon_vma *anon_vma, bool locked, struct list_head *ret) { - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, src, 0); /* Drop an anon_vma reference if we took one */ if (anon_vma) @@ -1218,7 +1230,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return -ENOMEM; *dstp = dst; - dst->private = NULL; + dst->migrate_info = 0; if (!folio_trylock(src)) { if (mode == MIGRATE_ASYNC) @@ -1450,7 +1462,8 @@ out: } /* - * Counterpart of unmap_and_move_page() for hugepage migration. + * Counterpart of migrate_folio_unmap() and migrate_folio_move() for hugetlb + * folio migration. * * This function doesn't wait the completion of hugepage I/O * because there is no race between I/O and migration for hugepage. @@ -1467,20 +1480,20 @@ out: * because then pte is replaced with migration swap entry and direct I/O code * will wait in the page fault for migration to complete. */ -static int unmap_and_move_huge_page(new_folio_t get_new_folio, +static int unmap_and_move_hugetlb_folio(new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, struct folio *src, int force, enum migrate_mode mode, - int reason, struct list_head *ret) + enum migrate_reason reason, struct list_head *ret) { struct folio *dst; int rc = -EAGAIN; - int page_was_mapped = 0; + int was_mapped = 0; struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; enum ttu_flags ttu = 0; if (folio_ref_count(src) == 1) { - /* page was freed from under us. So we are done. */ + /* folio was freed from under us. So we are done. */ folio_putback_hugetlb(src); return 0; } @@ -1502,8 +1515,8 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } /* - * Check for pages which are in the process of being freed. Without - * folio_mapping() set, hugetlbfs specific move page routine will not + * Check for folios which are in the process of being freed. Without + * folio_mapping() set, hugetlbfs specific move folio routine will not * be called and we could leak usage counts for subpools. */ if (hugetlb_folio_subpool(src) && !folio_mapping(src)) { @@ -1533,13 +1546,13 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } try_to_migrate(src, ttu); - page_was_mapped = 1; + was_mapped = 1; } if (!folio_mapped(src)) rc = move_to_new_folio(dst, src, mode); - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, !rc ? dst : src, ttu); if (ttu & TTU_RMAP_LOCKED) @@ -1627,7 +1640,7 @@ struct migrate_pages_stats { */ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct migrate_pages_stats *stats, struct list_head *ret_folios) { @@ -1664,10 +1677,10 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, continue; } - rc = unmap_and_move_huge_page(get_new_folio, - put_new_folio, private, - folio, pass > 2, mode, - reason, ret_folios); + rc = unmap_and_move_hugetlb_folio(get_new_folio, + put_new_folio, private, + folio, pass > 2, mode, + reason, ret_folios); /* * The rules are: * 0: hugetlb folio will be put back @@ -1717,7 +1730,7 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, static void migrate_folios_move(struct list_head *src_folios, struct list_head *dst_folios, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct migrate_pages_stats *stats, int *retry, int *thp_retry, int *nr_failed, @@ -1742,7 +1755,7 @@ static void migrate_folios_move(struct list_head *src_folios, /* * The rules are: * 0: folio will be freed - * -EAGAIN: stay on the unmap_folios list + * -EAGAIN: stay on the src_folios list * Other errno: put on ret_folios list */ switch (rc) { @@ -1800,7 +1813,7 @@ static void migrate_folios_undo(struct list_head *src_folios, */ static int migrate_pages_batch(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, - unsigned long private, enum migrate_mode mode, int reason, + unsigned long private, enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats, int nr_pass) { @@ -2012,7 +2025,7 @@ out: static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats) { @@ -2089,7 +2102,7 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, */ int migrate_pages(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, unsigned int *ret_succeeded) { int rc, rc_gather; int nr_pages; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 908d2d4ec43a..18d097c38853 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -77,6 +77,9 @@ static int migrate_vma_collect_hole(unsigned long start, * @folio: the folio to split * @fault_page: struct page associated with the fault if any * + * If @folio is not the folio containing @fault_page, the caller must hold a + * reference on @folio. The helper consumes that reference. + * * Returns 0 on success */ static int migrate_vma_split_folio(struct folio *folio, @@ -86,10 +89,8 @@ static int migrate_vma_split_folio(struct folio *folio, struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; struct folio *new_fault_folio = NULL; - if (folio != fault_folio) { - folio_get(folio); + if (folio != fault_folio) folio_lock(folio); - } ret = split_folio(folio); if (ret) { @@ -166,11 +167,14 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); - folio = softleaf_to_folio(entry); - if (!softleaf_is_device_private(entry) || - !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || - (folio->pgmap->owner != migrate->pgmap_owner)) { + !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { + spin_unlock(ptl); + return migrate_vma_collect_skip(start, end, walk); + } + + folio = softleaf_to_folio(entry); + if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } @@ -307,6 +311,9 @@ again: if (folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -350,6 +357,9 @@ again: if (folio && folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -446,13 +456,13 @@ again: if (pte_present(pte)) { if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, addr, ptep, swp_pte); @@ -514,7 +524,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate) migrate->pgmap_owner); mmu_notifier_invalidate_range_start(&range); - walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end, + walk_page_range_vma(migrate->vma, migrate->start, migrate->end, &migrate_vma_walk_ops, migrate); mmu_notifier_invalidate_range_end(&range); @@ -769,7 +779,7 @@ int migrate_vma_setup(struct migrate_vma *args) } EXPORT_SYMBOL(migrate_vma_setup); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm * at @addr. folio is already allocated as a part of the migration process with @@ -836,7 +846,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, else swp_entry = make_readable_device_private_entry( page_to_pfn(page)); - entry = swp_entry_to_pmd(swp_entry); + entry = softleaf_to_pmd(swp_entry); } else { if (folio_is_zone_device(folio) && !folio_is_device_coherent(folio)) { @@ -924,7 +934,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; return ret; } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, unsigned long addr, struct page *page, @@ -945,7 +955,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, static unsigned long migrate_vma_nr_pages(unsigned long *src) { unsigned long nr = 1; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES if (*src & MIGRATE_PFN_COMPOUND) nr = HPAGE_PMD_NR; #else diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf..ff4ac8281768 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -12,6 +12,7 @@ #include <linux/gfp.h> #include <linux/pagewalk.h> #include <linux/mman.h> +#include <linux/slab.h> #include <linux/syscalls.h> #include <linux/swap.h> #include <linux/leafops.h> @@ -27,30 +28,16 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) { #ifdef CONFIG_HUGETLB_PAGE - unsigned char present; - unsigned char *vec = walk->private; + const unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned char resident; spinlock_t *ptl; + pte_t ptep; ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); - - /* - * Hugepages under user process are always in RAM and never - * swapped out, but theoretically it needs to be checked. - */ - if (!pte) { - present = 0; - } else { - const pte_t ptep = huge_ptep_get(walk->mm, addr, pte); - - if (huge_pte_none(ptep) || pte_is_marker(ptep)) - present = 0; - else - present = 1; - } - - for (; addr != end; vec++, addr += PAGE_SIZE) - *vec = present; - walk->private = vec; + ptep = huge_ptep_get(walk->mm, addr, pte); + resident = !huge_pte_none(ptep) && !pte_is_marker(ptep); + memset(walk->private, resident, nr); + walk->private += nr; spin_unlock(ptl); #else BUG(); @@ -90,8 +77,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) folio = swap_cache_get_folio(entry); if (shmem) put_swap_device(si); - /* The swap cache space contains either folio, shadow or NULL */ - if (folio && !xa_is_value(folio)) { + if (folio) { present = folio_test_uptodate(folio); folio_put(folio); } @@ -107,7 +93,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem) */ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) { - unsigned char present = 0; + unsigned char present; struct folio *folio; /* @@ -117,17 +103,16 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) * tmpfs's .fault). So swapped out tmpfs mappings are tested here. */ folio = filemap_get_entry(mapping, index); - if (folio) { - if (xa_is_value(folio)) { - if (shmem_mapping(mapping)) - return mincore_swap(radix_to_swp_entry(folio), - true); - else - return 0; - } - present = folio_test_uptodate(folio); - folio_put(folio); + if (!folio) + return 0; + + if (xa_is_value(folio)) { + if (!shmem_mapping(mapping)) + return 0; + return mincore_swap(radix_to_swp_entry(folio), true); } + present = folio_test_uptodate(folio); + folio_put(folio); return present; } @@ -160,6 +145,20 @@ static int mincore_unmapped_range(unsigned long addr, unsigned long end, return 0; } +static int mincore_pud_entry(pud_t *pudp, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + if (pud_is_huge(pudp_get(pudp))) { + const unsigned long nr = (end - addr) >> PAGE_SHIFT; + + memset(walk->private, 1, nr); + walk->private += nr; + walk->action = ACTION_CONTINUE; + } + + return 0; +} + static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -232,6 +231,7 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) } static const struct mm_walk_ops mincore_walk_ops = { + .pud_entry = mincore_pud_entry, .pmd_entry = mincore_pte_range, .pte_hole = mincore_unmapped_range, .hugetlb_entry = mincore_hugetlb, @@ -258,7 +258,8 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v memset(vec, 1, pages); return pages; } - err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); + + err = walk_page_range_vma(vma, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; return (end - addr) >> PAGE_SHIFT; @@ -312,7 +313,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, if (!access_ok(vec, pages)) return -EFAULT; - tmp = (void *) __get_free_page(GFP_USER); + tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!tmp) return -EAGAIN; @@ -337,6 +338,6 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start += retval << PAGE_SHIFT; retval = 0; } - free_page((unsigned long) tmp); + kfree(tmp); return retval; } diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df..efa6716e4dfb 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -329,7 +329,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, * be split. And the pages are not in VM_LOCKed VMA * can be reclaimed. */ - if (!(vma->vm_flags & VM_LOCKED)) + if (!vma_test(vma, VMA_LOCKED_BIT)) return true; /* folio_within_range() cannot take KSM, but any small folio is OK */ @@ -368,7 +368,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, folio = pmd_folio(*pmd); if (folio_is_zone_device(folio)) goto out; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mlock_folio(folio); else munlock_folio(folio); @@ -393,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mlock_folio(folio); else munlock_folio(folio); @@ -417,8 +417,8 @@ out: * @end - end of range in @vma * @new_vma_flags - the new set of flags for @vma. * - * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED; - * called for munlock() and munlockall(), to clear VM_LOCKED from @vma. + * Called for mlock(), mlock2() and mlockall(), to set @vma VMA_LOCKED_BIT; + * called for munlock() and munlockall(), to clear VMA_LOCKED_BIT from @vma. */ static void mlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, @@ -431,14 +431,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, /* * There is a slight chance that concurrent page migration, - * or page reclaim finding a page of this now-VM_LOCKED vma, + * or page reclaim finding a page of this now-VMA_LOCKED_BIT vma, * will call mlock_vma_folio() and raise page's mlock_count: * double counting, leaving the page unevictable indefinitely. - * Communicate this danger to mlock_vma_folio() with VM_IO, - * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas. + * Communicate this danger to mlock_vma_folio() with VMA_IO_BIT, + * which is a VMA_SPECIAL_FLAGS flag not allowed on VMA_LOCKED_BIT vmas. * mmap_lock is held in write mode here, so this weird * combination should not be visible to other mmap_lock users; - * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED. + * but WRITE_ONCE so rmap walkers must see VMA_IO_BIT if VMA_LOCKED_BIT. */ if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT)) vma_flags_set(new_vma_flags, VMA_IO_BIT); @@ -446,7 +446,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, vma_flags_reset_once(vma, new_vma_flags); lru_add_drain(); - walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL); + walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL); lru_add_drain(); if (vma_flags_test(new_vma_flags, VMA_IO_BIT)) { @@ -458,7 +458,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, /* * mlock_fixup - handle mlock[all]/munlock[all] requests. * - * Filters out "special" vmas -- VM_LOCKED never gets set for these, and + * Filters out "special" vmas -- VMA_LOCKED_BIT never gets set for these, and * munlock is a no-op. However, for some special vmas, we go ahead and * populate the ptes. * @@ -466,24 +466,23 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, */ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, - unsigned long end, vm_flags_t newflags) + unsigned long end, vma_flags_t *new_vma_flags) { - vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; int nr_pages; int ret = 0; - if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || + if (vma_flags_same_pair(&old_vma_flags, new_vma_flags) || vma_is_secretmem(vma) || !vma_supports_mlock(vma)) { /* - * Don't set VM_LOCKED or VM_LOCKONFAULT and don't count. - * For secretmem, don't allow the memory to be unlocked. + * Don't set VMA_LOCKED_BIT or VMA_LOCKONFAULT_BIT and don't + * count. For secretmem, don't allow the memory to be unlocked. */ goto out; } - vma = vma_modify_flags(vmi, *prev, vma, start, end, &new_vma_flags); + vma = vma_modify_flags(vmi, *prev, vma, start, end, new_vma_flags); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto out; @@ -493,7 +492,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, * Keep track of amount of locked VM. */ nr_pages = (end - start) >> PAGE_SHIFT; - if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) + if (!vma_flags_test(new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) nr_pages = 0; @@ -502,15 +501,15 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * vm_flags is protected by the mmap_lock held in write mode. * It's okay if try_to_unmap_one unmaps a page just after we - * set VM_LOCKED, populate_vma_page_range will bring it back. + * set VMA_LOCKED_BIT, populate_vma_page_range will bring it back. */ - if (vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT) && + if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT) && vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) { /* No work to do, and mlocking twice would be wrong */ vma_start_write(vma); - vma->flags = new_vma_flags; + vma->flags = *new_vma_flags; } else { - mlock_vma_pages_range(vma, start, end, &new_vma_flags); + mlock_vma_pages_range(vma, start, end, new_vma_flags); } out: *prev = vma; @@ -518,7 +517,7 @@ out: } static int apply_vma_lock_flags(unsigned long start, size_t len, - vm_flags_t flags) + const vma_flags_t *flags) { unsigned long nstart, end, tmp; struct vm_area_struct *vma, *prev; @@ -543,18 +542,20 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, tmp = vma->vm_start; for_each_vma_range(vmi, vma, end) { int error; - vm_flags_t newflags; + vma_flags_t newflags; if (vma->vm_start != tmp) return -ENOMEM; - newflags = vma->vm_flags & ~VM_LOCKED_MASK; - newflags |= flags; + newflags = vma->flags; + vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK); + vma_flags_set_mask(&newflags, *flags); + /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ tmp = vma->vm_end; if (tmp > end) tmp = end; - error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags); + error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, &newflags); if (error) return error; tmp = vma_iter_end(&vmi); @@ -589,7 +590,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, end = start + len; for_each_vma_range(vmi, vma, end) { - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { if (start > vma->vm_start) count -= (start - vma->vm_start); if (end < vma->vm_end) { @@ -615,7 +616,8 @@ static int __mlock_posix_error_return(long retval) return retval; } -static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags) +static __must_check int do_mlock(unsigned long start, size_t len, + vma_flags_t *flags) { unsigned long locked; unsigned long lock_limit; @@ -664,24 +666,27 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len) { - return do_mlock(start, len, VM_LOCKED); + vma_flags_t flags = mk_vma_flags(VMA_LOCKED_BIT); + + return do_mlock(start, len, &flags); } SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags) { - vm_flags_t vm_flags = VM_LOCKED; + vma_flags_t vma_flags = mk_vma_flags(VMA_LOCKED_BIT); if (flags & ~MLOCK_ONFAULT) return -EINVAL; if (flags & MLOCK_ONFAULT) - vm_flags |= VM_LOCKONFAULT; + vma_flags_set(&vma_flags, VMA_LOCKONFAULT_BIT); - return do_mlock(start, len, vm_flags); + return do_mlock(start, len, &vma_flags); } SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) { + vma_flags_t flags = EMPTY_VMA_FLAGS; int ret; start = untagged_addr(start); @@ -691,7 +696,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) if (mmap_write_lock_killable(current->mm)) return -EINTR; - ret = apply_vma_lock_flags(start, len, 0); + ret = apply_vma_lock_flags(start, len, &flags); mmap_write_unlock(current->mm); return ret; @@ -699,46 +704,48 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) /* * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall) - * and translate into the appropriate modifications to mm->def_flags and/or the - * flags for all current VMAs. + * and translate into the appropriate modifications to mm->def_vma_flags and/or + * the flags for all current VMAs. * * There are a couple of subtleties with this. If mlockall() is called multiple * times with different flags, the values do not necessarily stack. If mlockall * is called once including the MCL_FUTURE flag and then a second time without - * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags. + * it, VMA_LOCKED_BIT and VMA_LOCKONFAULT_BIT will be cleared from + * mm->def_vma_flags. */ static int apply_mlockall_flags(int flags) { VMA_ITERATOR(vmi, current->mm, 0); + struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev = NULL; - vm_flags_t to_add = 0; + vma_flags_t to_add = EMPTY_VMA_FLAGS; - current->mm->def_flags &= ~VM_LOCKED_MASK; + vma_flags_clear_mask(&mm->def_vma_flags, VMA_LOCKED_MASK); if (flags & MCL_FUTURE) { - current->mm->def_flags |= VM_LOCKED; + vma_flags_set(&mm->def_vma_flags, VMA_LOCKED_BIT); if (flags & MCL_ONFAULT) - current->mm->def_flags |= VM_LOCKONFAULT; + vma_flags_set(&mm->def_vma_flags, VMA_LOCKONFAULT_BIT); if (!(flags & MCL_CURRENT)) goto out; } if (flags & MCL_CURRENT) { - to_add |= VM_LOCKED; + vma_flags_set(&to_add, VMA_LOCKED_BIT); if (flags & MCL_ONFAULT) - to_add |= VM_LOCKONFAULT; + vma_flags_set(&to_add, VMA_LOCKONFAULT_BIT); } for_each_vma(vmi, vma) { int error; - vm_flags_t newflags; + vma_flags_t newflags = vma->flags; - newflags = vma->vm_flags & ~VM_LOCKED_MASK; - newflags |= to_add; + vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK); + vma_flags_set_mask(&newflags, to_add); error = mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end, - newflags); + &newflags); /* Ignore errors, but prev needs fixing up. */ if (error) prev = vma; diff --git a/mm/mm_init.c b/mm/mm_init.c index fbd62670650c..18befe575931 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -33,8 +33,12 @@ #include <linux/kexec_handover.h> #include <linux/hugetlb.h> #include "internal.h" +#include "mm_init.h" +#include "page_alloc.h" +#include "sparse.h" #include "slab.h" #include "shuffle.h" +#include "vmalloc.h" #include <asm/setup.h> @@ -674,45 +678,20 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ -#ifdef CONFIG_ZONE_DEVICE +#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) static __meminit void pageblock_migratetype_init_range(unsigned long pfn, - unsigned long nr_pages, int migratetype) + unsigned long nr_pages, int migratetype, bool atomic) { const unsigned long end = pfn + nr_pages; for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false); - if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); } } #endif -/* - * Initialize a reserved page unconditionally, finding its zone first. - */ -void __meminit __init_page_from_nid(unsigned long pfn, int nid) -{ - pg_data_t *pgdat; - int zid; - - pgdat = NODE_DATA(nid); - - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - - if (pageblock_aligned(pfn)) { - enum migratetype mt = - kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); - init_pageblock_migratetype(pfn_to_page(pfn), mt, false); - } -} - #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) { @@ -771,10 +750,25 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) static void __meminit __init_deferred_page(unsigned long pfn, int nid) { + pg_data_t *pgdat = NODE_DATA(nid); + int zid; + if (early_page_initialised(pfn, nid)) return; - __init_page_from_nid(pfn, nid); + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + struct zone *zone = &pgdat->node_zones[zid]; + + if (zone_spans_pfn(zone, pfn)) + break; + } + __init_single_page(pfn_to_page(pfn), pfn, zid, nid); + + if (pageblock_aligned(pfn)) { + enum migratetype mt = + kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); + init_pageblock_migratetype(pfn_to_page(pfn), mt, false); + } } #else static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} @@ -1142,7 +1136,7 @@ void __ref memmap_init_zone_device(struct zone *zone, compound_nr_pages(pfn, altmap, pgmap)); } - pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false); pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); @@ -1495,7 +1489,7 @@ static inline void setup_usemap(struct zone *zone) {} #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -void __init set_pageblock_order(void) +static void __init set_pageblock_order(void) { unsigned int order = PAGE_BLOCK_MAX_ORDER; @@ -1521,7 +1515,7 @@ void __init set_pageblock_order(void) * include/linux/pageblock-flags.h for the values of pageblock_order based on * the kernel config */ -void __init set_pageblock_order(void) +static inline void __init set_pageblock_order(void) { } @@ -1828,7 +1822,6 @@ static void __init free_area_init(void) bool descending; arch_zone_limits_init(max_zone_pfn); - sparse_init(); start_pfn = PHYS_PFN(memblock_start_of_DRAM()); descending = arch_has_descending_max_zone_pfns(); @@ -1878,18 +1871,12 @@ static void __init free_area_init(void) (u64)zone_movable_pfn[i] << PAGE_SHIFT); } - /* - * Print out the early node map, and initialize the - * subsection-map relative to active online memory ranges to - * enable future "sub-section" extensions of the memory map. - */ + /* Print out the early node map. */ pr_info("Early memory node ranges\n"); - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); - sparse_init_subsection_map(start_pfn, end_pfn - start_pfn); - } /* Initialise every node */ mminit_verify_pageflags_layout(); @@ -1923,11 +1910,7 @@ static void __init free_area_init(void) } } - for_each_node_state(nid, N_MEMORY) - sparse_vmemmap_init_nid_late(nid); - calc_nr_kernel_pages(); - memmap_init(); /* disable hash distribution for systems with a single node */ fixup_hashdist(); @@ -1995,12 +1978,12 @@ static void __init deferred_free_pages(unsigned long pfn, if (!nr_pages) return; + pageblock_migratetype_init_range(pfn, nr_pages, mt, true); + page = pfn_to_page(pfn); /* Free a large naturally-aligned chunk if possible */ if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { - for (i = 0; i < nr_pages; i += pageblock_nr_pages) - init_pageblock_migratetype(page + i, mt, false); __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); return; } @@ -2008,11 +1991,8 @@ static void __init deferred_free_pages(unsigned long pfn, /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE); - for (i = 0; i < nr_pages; i++, page++, pfn++) { - if (pageblock_aligned(pfn)) - init_pageblock_migratetype(page, mt, false); - __free_pages_core(page, 0, MEMINIT_EARLY); - } + for (i = 0; i < nr_pages; i++) + __free_pages_core(page + i, 0, MEMINIT_EARLY); } /* Completion tracking for deferred_init_memmap() threads */ @@ -2337,6 +2317,7 @@ void __init page_alloc_init_late(void) /* Reinit limits that are based on free pages after the kernel is up */ files_maxfiles_init(); #endif + hugetlb_bootmem_struct_page_init(); /* Accounting of total+free memory is stable at this point. */ mem_init_print_info(); @@ -2701,10 +2682,13 @@ void __init __weak mem_init(void) void __init mm_core_init_early(void) { + free_area_init(); + hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); - free_area_init(); + sparse_init(); + memmap_init(); } /* diff --git a/mm/mm_init.h b/mm/mm_init.h new file mode 100644 index 000000000000..39f75df9be1c --- /dev/null +++ b/mm/mm_init.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm_init.h: + * + * mm/ internal mm_init and memblock declarations + */ + +#ifndef __MM_MM_INIT_H +#define __MM_MM_INIT_H + +#include <linux/types.h> +#include <linux/init.h> +#include <linux/mmzone.h> +#include <linux/jump_label.h> +#include <linux/printk.h> + +struct page; +struct vmem_altmap; + +/* perform sanity checks on struct pages being allocated or freed */ +DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); + +void set_zone_contiguous(struct zone *zone); +bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, + unsigned long nr_pages); + +static inline void clear_zone_contiguous(struct zone *zone) +{ + zone->contiguous = false; +} + +void memblock_free_pages(unsigned long pfn, unsigned int order); + +void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, + int nid, bool exact_nid); + +void memmap_init_range(unsigned long size, int nid, unsigned long zone, + unsigned long start_pfn, unsigned long zone_end_pfn, + enum meminit_context context, + struct vmem_altmap *altmap, int migratetype, + bool isolate_pageblock); + +#if defined CONFIG_COMPACTION || defined CONFIG_CMA +/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ +void init_cma_reserved_pageblock(struct page *page); +#endif + +#ifdef CONFIG_CMA +void init_cma_pageblock(struct page *page); +#else +static inline void init_cma_pageblock(struct page *page) +{ +} +#endif + +/* Memory initialisation debug and verification */ +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +DECLARE_STATIC_KEY_TRUE(deferred_pages); + +static inline bool deferred_pages_enabled(void) +{ + return static_branch_unlikely(&deferred_pages); +} + +bool __init deferred_grow_zone(struct zone *zone, unsigned int order); +#else +static inline bool deferred_pages_enabled(void) +{ + return false; +} +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + +void init_deferred_page(unsigned long pfn, int nid); + +enum mminit_level { + MMINIT_WARNING, + MMINIT_VERIFY, + MMINIT_TRACE +}; + +#ifdef CONFIG_DEBUG_MEMORY_INIT + +extern int mminit_loglevel; + +#define mminit_dprintk(level, prefix, fmt, arg...) \ +do { \ + if (level < mminit_loglevel) { \ + if (level <= MMINIT_WARNING) \ + pr_warn("mminit::" prefix " " fmt, ##arg); \ + else \ + printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ + } \ +} while (0) + +void mminit_verify_pageflags_layout(void); +void mminit_verify_zonelist(void); +#else + +static inline void mminit_dprintk(enum mminit_level level, + const char *prefix, const char *fmt, ...) +{ +} + +static inline void mminit_verify_pageflags_layout(void) +{ +} + +static inline void mminit_verify_zonelist(void) +{ +} +#endif /* CONFIG_DEBUG_MEMORY_INIT */ + +extern bool mirrored_kernelcore; +bool memblock_has_mirror(void); +void memblock_free_all(void); + +void __meminit __init_single_page(struct page *page, unsigned long pfn, + unsigned long zone, int nid); + +#endif /* __MM_MM_INIT_H */ diff --git a/mm/mm_slot.h b/mm/mm_slot.h index 83f18ed1c4bd..9b09b68e5742 100644 --- a/mm/mm_slot.h +++ b/mm/mm_slot.h @@ -33,6 +33,12 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp) kmem_cache_free(cache, objp); } +/* + * Note: mm_slot_lookup and mm_slot_insert cannot be converted to static inline + * functions because the hash helpers (hash_for_each_possible and hash_add) rely + * on the actual array argument 'hashtable' for sizeof() instead of pointers. + */ + #define mm_slot_lookup(_hashtable, _mm) \ ({ \ struct mm_slot *tmp_slot, *mm_slot = NULL; \ @@ -52,4 +58,9 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp) hash_add(_hashtable, &_mm_slot->hash, (unsigned long)_mm); \ }) +static inline void mm_slot_remove(struct mm_slot *slot) +{ + hash_del(&slot->hash); + list_del(&slot->mm_node); +} #endif /* _LINUX_MM_SLOT_H */ diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff4..4bf26b0f1e6e 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -80,13 +80,13 @@ core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); /* Update vma->vm_page_prot to reflect vma->vm_flags. */ void vma_set_page_prot(struct vm_area_struct *vma) { - vm_flags_t vm_flags = vma->vm_flags; + vma_flags_t vma_flags = vma->flags; pgprot_t vm_page_prot; - vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); + vm_page_prot = vma_pgprot_modify(vma->vm_page_prot, vma_flags); if (vma_wants_writenotify(vma, vm_page_prot)) { - vm_flags &= ~VM_SHARED; - vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); + vma_flags_clear(&vma_flags, VMA_SHARED_BIT); + vm_page_prot = vma_pgprot_modify(vm_page_prot, vma_flags); } /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ WRITE_ONCE(vma->vm_page_prot, vm_page_prot); @@ -102,15 +102,16 @@ void vma_set_page_prot(struct vm_area_struct *vma) */ static int check_brk_limits(unsigned long addr, unsigned long len) { + const struct mm_struct *mm = current->mm; + const bool is_def_locked = + vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT); unsigned long mapped_addr; mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); if (IS_ERR_VALUE(mapped_addr)) return mapped_addr; - return mlock_future_ok(current->mm, - current->mm->def_flags & VM_LOCKED, len) - ? 0 : -EAGAIN; + return mlock_future_ok(mm, is_def_locked, len) ? 0 : -EAGAIN; } SYSCALL_DEFINE1(brk, unsigned long, brk) @@ -197,7 +198,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) goto out; mm->brk = brk; - if (mm->def_flags & VM_LOCKED) + if (vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT)) populate = true; success: @@ -280,7 +281,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * do_mmap() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to - * apply @vm_flags. If this is a file-backed mapping then the file is specified + * apply @vma_flags. If this is a file-backed mapping then the file is specified * in @file and page offset into the file via @pgoff. * * This function does not perform security checks on the file and assumes, if @@ -320,7 +321,8 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * (2) for details. * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. - * @vm_flags: VMA flags which should be set by default, or 0 otherwise. + * @vma_flags: VMA flags which should be set by default, or EMPTY_VMA_FLAGS + * otherwise. * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be @@ -335,7 +337,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, */ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flags, vm_flags_t vm_flags, + unsigned long flags, vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { @@ -399,13 +401,18 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * to. we assume access permissions have been handled by the open * of the memory object, so we don't do any here. */ - vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) | - mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_prot_bits(prot, pkey))); + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_flag_bits(file, flags))); + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, + VMA_MAYEXEC_BIT); /* Obtain the address to map to. we verify (or select) it and ensure * that it represents a valid section of the address space. */ - addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags); + addr = __get_unmapped_area(file, addr, len, pgoff, flags, vma_flags); if (IS_ERR_VALUE(addr)) return addr; @@ -418,7 +425,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (!can_do_mlock()) return -EPERM; - if (!mlock_future_ok(mm, vm_flags & VM_LOCKED, len)) + if (!mlock_future_ok(mm, vma_flags_test(&vma_flags, VMA_LOCKED_BIT), len)) return -EAGAIN; if (file) { @@ -461,22 +468,23 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) return -EACCES; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); if (!(file->f_mode & FMODE_WRITE)) - vm_flags &= ~(VM_MAYWRITE | VM_SHARED); + vma_flags_clear(&vma_flags, VMA_MAYWRITE_BIT, + VMA_SHARED_BIT); fallthrough; case MAP_PRIVATE: if (!(file->f_mode & FMODE_READ)) return -EACCES; if (path_noexec(&file->f_path)) { - if (vm_flags & VM_EXEC) + if (vma_flags_test(&vma_flags, VMA_EXEC_BIT)) return -EPERM; - vm_flags &= ~VM_MAYEXEC; + vma_flags_clear(&vma_flags, VMA_MAYEXEC_BIT); } if (!can_mmap_file(file)) return -ENODEV; - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; break; @@ -488,23 +496,27 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Check to see if we are violating any seals and update VMA * flags if necessary to avoid future seal violations. */ - err = memfd_check_seals_mmap(file, &vm_flags); + err = memfd_check_seals_mmap(file, &vma_flags); if (err) return (unsigned long)err; } else { switch (flags & MAP_TYPE) { case MAP_SHARED: - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; /* * Ignore pgoff. */ pgoff = 0; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); break; - case MAP_DROPPABLE: - if (VM_DROPPABLE == VM_NONE) + case MAP_DROPPABLE: { + vma_flags_t droppable = VMA_DROPPABLE; + + if (vma_flags_empty(&droppable)) return -EOPNOTSUPP; + vma_flags_set_mask(&vma_flags, droppable); + /* * A locked or stack area makes no sense to be droppable. * @@ -515,23 +527,24 @@ unsigned long do_mmap(struct file *file, unsigned long addr, */ if (flags & (MAP_LOCKED | MAP_HUGETLB)) return -EINVAL; - if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; - vm_flags |= VM_DROPPABLE; - /* * If the pages can be dropped, then it doesn't make * sense to reserve them. */ - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* * Likewise, they're volatile enough that they * shouldn't survive forks or coredumps. */ - vm_flags |= VM_WIPEONFORK | VM_DONTDUMP; + vma_flags_set(&vma_flags, VMA_WIPEONFORK_BIT, + VMA_DONTDUMP_BIT); + fallthrough; + } case MAP_PRIVATE: /* * Set pgoff according to addr for anon_vma. @@ -544,22 +557,22 @@ unsigned long do_mmap(struct file *file, unsigned long addr, } /* - * Set 'VM_NORESERVE' if we should not account for the - * memory use of this mapping. + * Set VMA_NORESERVE_BIT if we should not account for the memory use + * of this mapping. */ if (flags & MAP_NORESERVE) { /* We honor MAP_NORESERVE if allowed to overcommit */ if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* hugetlb applies strict overcommit unless MAP_NORESERVE */ if (file && is_file_hugepages(file)) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vma_flags, pgoff, uf); if (!IS_ERR_VALUE(addr) && - ((vm_flags & VM_LOCKED) || + (vma_flags_test(&vma_flags, VMA_LOCKED_BIT) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) *populate = len; return addr; @@ -645,9 +658,9 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) * Determine if the allocation needs to ensure that there is no * existing mapping within it's guard gaps, for use as start_gap. */ -static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) +static inline unsigned long stack_guard_placement(vma_flags_t vma_flags) { - if (vm_flags & VM_SHADOW_STACK) + if (vma_flags_test_single_mask(&vma_flags, VMA_SHADOW_STACK)) return PAGE_SIZE; return 0; @@ -689,7 +702,7 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev; @@ -714,7 +727,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = mmap_end; - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); return vm_unmapped_area(&info); @@ -727,7 +740,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif @@ -738,7 +751,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct vm_area_struct *vma, *prev; struct mm_struct *mm = current->mm; @@ -766,7 +779,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); addr = vm_unmapped_area(&info); @@ -795,23 +808,24 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags) { if (mm_flags_test(MMF_TOPDOWN, current->mm)) return arch_get_unmapped_area_topdown(filp, addr, len, pgoff, - flags, vm_flags); - return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); + flags, vma_flags_to_legacy(vma_flags)); + return arch_get_unmapped_area(filp, addr, len, pgoff, flags, + vma_flags_to_legacy(vma_flags)); } unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) + unsigned long pgoff, unsigned long flags, vma_flags_t vma_flags) { unsigned long (*get_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long) @@ -846,11 +860,11 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, && !addr /* no hint */ && IS_ALIGNED(len, PMD_SIZE)) { /* Ensures that larger anonymous mappings are THP aligned. */ - addr = thp_get_unmapped_area_vmflags(file, addr, len, - pgoff, flags, vm_flags); + addr = thp_get_unmapped_area_vmaflags(file, addr, len, + pgoff, flags, vma_flags); } else { - addr = mm_get_unmapped_area_vmflags(file, addr, len, - pgoff, flags, vm_flags); + addr = mm_get_unmapped_area_vmaflags(file, addr, len, + pgoff, flags, vma_flags); } if (IS_ERR_VALUE(addr)) return addr; @@ -868,7 +882,8 @@ unsigned long mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return mm_get_unmapped_area_vmflags(file, addr, len, pgoff, flags, 0); + return mm_get_unmapped_area_vmaflags(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL(mm_get_unmapped_area); @@ -970,7 +985,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon return NULL; if (expand_stack_locked(prev, addr)) return NULL; - if (prev->vm_flags & VM_LOCKED) + if (vma_test(prev, VMA_LOCKED_BIT)) populate_vma_page_range(prev, addr, prev->vm_end, NULL); return prev; } @@ -994,7 +1009,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon start = vma->vm_start; if (expand_stack_locked(vma, addr)) return NULL; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) populate_vma_page_range(vma, addr, start, NULL); return vma; } @@ -1119,18 +1134,18 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, */ vma = vma_lookup(mm, start); - if (!vma || !(vma->vm_flags & VM_SHARED)) { + if (!vma || !vma_test(vma, VMA_SHARED_BIT)) { mmap_read_unlock(mm); return -EINVAL; } - prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; - prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; - prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; + prot |= vma_test(vma, VMA_READ_BIT) ? PROT_READ : 0; + prot |= vma_test(vma, VMA_WRITE_BIT) ? PROT_WRITE : 0; + prot |= vma_test(vma, VMA_EXEC_BIT) ? PROT_EXEC : 0; flags &= MAP_NONBLOCK; flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE; - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) flags |= MAP_LOCKED; /* Save vm_flags used to calculate prot and flags, and recheck later. */ @@ -1191,7 +1206,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } ret = do_mmap(vma->vm_file, start, size, - prot, flags, 0, pgoff, &populate, NULL); + prot, flags, EMPTY_VMA_FLAGS, pgoff, &populate, NULL); out: mmap_write_unlock(mm); fput(file); @@ -1233,7 +1248,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) vma = vma_prev(&vmi); ret = do_brk_flags(&vmi, vma, addr, len, vma_flags); - populate = ((mm->def_flags & VM_LOCKED) != 0); + populate = vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate && !ret) @@ -1256,7 +1271,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, mmap_assert_write_locked(mm); vma_iter_set(vmi, vma->vm_end); do { - if (vma->vm_flags & VM_ACCOUNT) + if (vma_test(vma, VMA_ACCOUNT_BIT)) nr_accounted += vma_pages(vma); vma_mark_detached(vma); remove_vma(vma); @@ -1405,7 +1420,7 @@ static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) { /* * Forbid splitting special mappings - kernel has expectations over - * the number of pages in mapping. Together with VM_DONTEXPAND + * the number of pages in mapping. Together with VMA_DONTEXPAND_BIT * the size of vma should stay the same over the special mapping's * lifetime. */ @@ -1447,44 +1462,6 @@ static vm_fault_t special_mapping_fault(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -static struct vm_area_struct *__install_special_mapping( - struct mm_struct *mm, - unsigned long addr, unsigned long len, - vm_flags_t vm_flags, void *priv, - const struct vm_operations_struct *ops) -{ - int ret; - struct vm_area_struct *vma; - - vma = vm_area_alloc(mm); - if (unlikely(vma == NULL)) - return ERR_PTR(-ENOMEM); - - vma_set_range(vma, addr, addr + len, 0); - vm_flags |= mm->def_flags | VM_DONTEXPAND; - if (pgtable_supports_soft_dirty()) - vm_flags |= VM_SOFTDIRTY; - vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); - - vma->vm_ops = ops; - vma->vm_private_data = priv; - - ret = insert_vm_struct(mm, vma); - if (ret) - goto out; - - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); - - perf_event_mmap(vma); - - return vma; - -out: - vm_area_free(vma); - return ERR_PTR(ret); -} - bool vma_is_special_mapping(const struct vm_area_struct *vma, const struct vm_special_mapping *sm) { @@ -1715,7 +1692,7 @@ bool mmap_read_lock_maybe_expand(struct mm_struct *mm, return true; } - if (!(new_vma->vm_flags & VM_GROWSDOWN)) + if (!vma_test(new_vma, VMA_GROWSDOWN_BIT)) return false; mmap_write_lock(mm); @@ -1765,7 +1742,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) retval = vma_start_write_killable(mpnt); if (retval < 0) goto loop_out; - if (mpnt->vm_flags & VM_DONTCOPY) { + if (vma_test(mpnt, VMA_DONTCOPY_BIT)) { retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, mpnt->vm_end, GFP_KERNEL); if (retval) @@ -1775,7 +1752,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) continue; } charge = 0; - if (mpnt->vm_flags & VM_ACCOUNT) { + if (vma_test(mpnt, VMA_ACCOUNT_BIT)) { unsigned long len = vma_pages(mpnt); if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ @@ -1793,16 +1770,19 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) retval = dup_userfaultfd(tmp, &uf); if (retval) goto fail_nomem_anon_vma_fork; - if (tmp->vm_flags & VM_WIPEONFORK) { + + if (vma_test(tmp, VMA_WIPEONFORK_BIT)) { /* - * VM_WIPEONFORK gets a clean slate in the child. + * VMA_WIPEONFORK_BIT gets a clean slate in the child. * Don't prepare anon_vma until fault since we don't * copy page for current vma. */ tmp->anon_vma = NULL; } else if (anon_vma_fork(tmp, mpnt)) goto fail_nomem_anon_vma_fork; - vm_flags_clear(tmp, VM_LOCKED_MASK); + + vma_start_write(tmp); + vma_clear_flags_mask(tmp, VMA_LOCKED_MASK); /* * Copy/update hugetlb private vma information. */ @@ -1830,13 +1810,12 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, - &mapping->i_mmap); + mapping_rmap_tree_insert_after(tmp, mpnt, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } - if (!(tmp->vm_flags & VM_WIPEONFORK)) + if (!vma_test(tmp, VMA_WIPEONFORK_BIT)) retval = copy_page_range(tmp, mpnt); if (retval) { diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 245b74f39f91..df69ba6e797f 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -19,7 +19,7 @@ #include <linux/sched/mm.h> #include <linux/slab.h> -#include "vma.h" +#include "internal.h" /* global SRCU for all MMs */ DEFINE_STATIC_SRCU(srcu); diff --git a/mm/mmzone.c b/mm/mmzone.c index 0c8f181d9d50..59dc3f2076a6 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -43,7 +43,8 @@ struct zone *next_zone(struct zone *zone) return zone; } -static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) +static inline int zref_in_nodemask(struct zoneref *zref, + const nodemask_t *nodes) { #ifdef CONFIG_NUMA return node_isset(zonelist_node_idx(zref), *nodes); @@ -55,7 +56,7 @@ static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) /* Returns the next zone at or below highest_zoneidx in a zonelist */ struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { /* * Find the next suitable zone to use for the allocation. diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028c..2888ee638d87 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -40,7 +40,7 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) { - if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) + if (WARN_ON_ONCE(!vma_test(vma, VMA_WRITE_BIT))) return false; /* Don't touch entries that are not even readable. */ @@ -97,7 +97,7 @@ static bool can_change_shared_pte_writable(struct vm_area_struct *vma, bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { - if (!(vma->vm_flags & VM_SHARED)) + if (!vma_test(vma, VMA_SHARED_BIT)) return can_change_private_pte_writable(vma, addr, pte); return can_change_shared_pte_writable(vma, pte); @@ -143,7 +143,7 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce * that the ptes point to consecutive pages of the same anon large folio. */ -static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len, +static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len, struct page *first_page, bool expected_anon_exclusive) { int idx; @@ -174,16 +174,16 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb) { bool expected_anon_exclusive; - int sub_batch_idx = 0; + int batch_idx = 0; int len; while (nr_ptes) { - expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); - len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, + expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx); + len = page_anon_exclusive_batch(batch_idx, nr_ptes, first_page, expected_anon_exclusive); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, - sub_batch_idx, expected_anon_exclusive, tlb); - sub_batch_idx += len; + batch_idx, expected_anon_exclusive, tlb); + batch_idx += len; nr_ptes -= len; } } @@ -194,7 +194,7 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru { bool set_write; - if (vma->vm_flags & VM_SHARED) { + if (vma_test(vma, VMA_SHARED_BIT)) { set_write = can_change_shared_pte_writable(vma, ptent); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes, /* idx = */ 0, set_write, tlb); @@ -214,8 +214,9 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru static long change_softleaf_pte(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags) { - const bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + const bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); softleaf_t entry = softleaf_from_pte(oldpte); pte_t newpte; @@ -240,8 +241,8 @@ static long change_softleaf_pte(struct vm_area_struct *vma, */ entry = make_readable_device_private_entry(swp_offset(entry)); newpte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(oldpte)) - newpte = pte_swp_mkuffd_wp(newpte); + if (pte_swp_uffd(oldpte)) + newpte = pte_swp_mkuffd(newpte); } else if (softleaf_is_marker(entry)) { /* * Ignore error swap entries unconditionally, @@ -256,7 +257,7 @@ static long change_softleaf_pte(struct vm_area_struct *vma, * to unprotect it, drop it; the next page * fault will trigger without uffd trapping. */ - if (uffd_wp_resolve) { + if (uffd_prot_resolve) { pte_clear(vma->vm_mm, addr, pte); return 1; } @@ -265,10 +266,10 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } - if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); - else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + if (uffd_prot) + newpte = pte_swp_mkuffd(newpte); + else if (uffd_prot_resolve) + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(oldpte, newpte)) { set_pte_at(vma->vm_mm, addr, pte, newpte); @@ -282,17 +283,28 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb, int nr_ptes, unsigned long end, pgprot_t newprot, struct folio *folio, struct page *page, unsigned long cp_flags) { - const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; - const bool uffd_wp = cp_flags & MM_CP_UFFD_WP; + const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + const bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); pte_t ptent, oldpte; oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes); ptent = pte_modify(oldpte, newprot); - if (uffd_wp) - ptent = pte_mkuffd_wp(ptent); - else if (uffd_wp_resolve) - ptent = pte_clear_uffd_wp(ptent); + if (uffd_prot) + ptent = pte_mkuffd(ptent); + else if (uffd_prot_resolve) + ptent = pte_clear_uffd(ptent); + + /* + * The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE + * semantics. If mprotect() or NUMA hinting changed the + * base protection, restore PAGE_NONE so the PTE still + * traps on any access. pte_modify() preserves + * _PAGE_UFFD. + */ + if (userfaultfd_rwp(vma) && pte_uffd(ptent)) + ptent = pte_modify(ptent, PAGE_NONE); /* * In some writable, shared mappings, we might want @@ -325,6 +337,7 @@ static long change_pte_range(struct mmu_gather *tlb, long pages = 0; bool is_private_single_threaded; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; + bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; @@ -350,6 +363,14 @@ static long change_pte_range(struct mmu_gather *tlb, /* Already in the desired state. */ if (prot_numa && pte_protnone(oldpte)) continue; + /* + * RWP-protected PTEs carry _PAGE_UFFD as a marker on + * top of PROT_NONE. Skip only entries already in that + * exact state; plain PROT_NONE from mprotect() still needs + * to be promoted so future faults can be distinguished. + */ + if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte)) + continue; page = vm_normal_page(vma, addr, oldpte); if (page) @@ -358,6 +379,8 @@ static long change_pte_range(struct mmu_gather *tlb, /* * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. + * Skip this filter for uffd RWP which + * must set protnone regardless of NUMA placement. */ if (prot_numa && !folio_can_map_prot_numa(folio, vma, @@ -428,7 +451,7 @@ pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags) * (e.g. 2M shmem) because file thp is handled differently when * split by erasing the pmd so far. */ - return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma); + return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma); } /* @@ -667,7 +690,16 @@ long change_protection(struct mmu_gather *tlb, pgprot_t newprot = vma->vm_page_prot; long pages; - BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); + /* + * MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one + * change, and WP and RWP cannot mix. Miswired callers get a warn and + * a no-op; userspace cannot reach this state. + */ + if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL || + (cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL || + ((cp_flags & MM_CP_UFFD_WP_ALL) && + (cp_flags & MM_CP_UFFD_RWP_ALL)))) + return 0; #ifdef CONFIG_NUMA_BALANCING /* @@ -681,6 +713,10 @@ long change_protection(struct mmu_gather *tlb, WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA); #endif + if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) && + (cp_flags & MM_CP_UFFD_RWP)) + newprot = PAGE_NONE; + if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot, cp_flags); @@ -699,25 +735,24 @@ static int prot_none_pte_entry(pte_t *pte, unsigned long addr, 0 : -EACCES; } +#ifdef CONFIG_HUGETLB_PAGE static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long next, struct mm_walk *walk) { - return pfn_modify_allowed(pte_pfn(ptep_get(pte)), - *(pgprot_t *)(walk->private)) ? - 0 : -EACCES; -} + const pte_t entry = huge_ptep_get(walk->mm, addr, pte); -static int prot_none_test(unsigned long addr, unsigned long next, - struct mm_walk *walk) -{ - return 0; + if (pfn_modify_allowed(pte_pfn(entry), *(pgprot_t *)(walk->private))) + return 0; + return -EACCES; } +#else +#define prot_none_hugetlb_entry NULL +#endif static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, .hugetlb_entry = prot_none_hugetlb_entry, - .test_walk = prot_none_test, .walk_lock = PGWALK_WRLOCK, }; @@ -753,7 +788,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) { pgprot_t new_pgprot = vm_get_page_prot(newflags); - error = walk_page_range(current->mm, start, end, + error = walk_page_range_vma(vma, start, end, &prot_none_walk_ops, &new_pgprot); if (error) return error; @@ -811,8 +846,8 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, vm_unacct_memory(nrpages); /* - * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major - * fault on access. + * Private VMA_LOCKED_BIT VMA becoming writable: trigger COW to avoid + * major fault on access. */ if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT) && vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT) && @@ -886,7 +921,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, goto out; start = vma->vm_start; error = -EINVAL; - if (!(vma->vm_flags & VM_GROWSDOWN)) + if (!vma_test(vma, VMA_GROWSDOWN_BIT)) goto out; } else { if (vma->vm_start > start) @@ -894,7 +929,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (unlikely(grows & PROT_GROWSUP)) { end = vma->vm_end; error = -EINVAL; - if (!(vma->vm_flags & VM_GROWSUP)) + if (!vma_test_single_mask(vma, VMA_GROWSUP)) goto out; } } @@ -918,7 +953,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, } /* Does the application expect PROT_READ to imply PROT_EXEC */ - if (rier && (vma->vm_flags & VM_MAYEXEC)) + if (rier && vma_test(vma, VMA_MAYEXEC_BIT)) prot |= PROT_EXEC; /* diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832..f07fc4e3ef2e 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -68,7 +68,7 @@ struct vma_remap_struct { bool populate_expand; /* mlock()'d expanded, must populate. */ enum mremap_type remap_type; /* expand, shrink, etc. */ bool mmap_locked; /* Is mm currently write-locked? */ - unsigned long charged; /* If VM_ACCOUNT, # pages to account. */ + unsigned long charged; /* If VMA_ACCOUNT_BIT, # pgs to account */ bool vmi_needs_invalidate; /* Is the VMA iterator invalidated? */ }; @@ -296,10 +296,19 @@ static int move_ptes(struct pagetable_move_control *pmc, pte_clear(mm, new_addr, new_ptep); else { if (need_clear_uffd_wp) { - if (pte_present(pte)) - pte = pte_clear_uffd_wp(pte); - else - pte = pte_swp_clear_uffd_wp(pte); + if (pte_present(pte)) { + /* + * See __copy_present_ptes(): normalise + * RWP PTEs so the destination starts + * accessible instead of taking a + * numa-hinting fault on first access. + */ + if (userfaultfd_rwp(vma) && pte_uffd(pte)) + pte = pte_modify(pte, vma->vm_page_prot); + pte = pte_clear_uffd(pte); + } else { + pte = pte_swp_clear_uffd(pte); + } } set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); } @@ -948,14 +957,13 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + const pgoff_t pgoff = linear_page_index(vma, vrm->addr); unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; if (vrm->flags & MREMAP_FIXED) map_flags |= MAP_FIXED; - if (vma->vm_flags & VM_MAYSHARE) + if (vma_test(vma, VMA_MAYSHARE_BIT)) map_flags |= MAP_SHARED; res = get_unmapped_area(vma->vm_file, new_addr, vrm->new_len, pgoff, @@ -977,7 +985,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) { unsigned long charged; - if (!(vrm->vma->vm_flags & VM_ACCOUNT)) + if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT)) return true; /* @@ -1004,7 +1012,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) */ static void vrm_uncharge(struct vma_remap_struct *vrm) { - if (!(vrm->vma->vm_flags & VM_ACCOUNT)) + if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT)) return; vm_unacct_memory(vrm->charged); @@ -1024,7 +1032,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm, struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += pages; } @@ -1168,7 +1176,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * arose, in which case we _do_ wish to unmap the _new_ VMA, which means * we actually _do_ want it be unaccounted. */ - bool accountable_move = (vma->vm_flags & VM_ACCOUNT) && + bool accountable_move = vma_test(vma, VMA_ACCOUNT_BIT) && !(vrm->flags & MREMAP_DONTUNMAP); /* @@ -1187,7 +1195,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * portions of the original VMA that remain. */ if (accountable_move) { - vm_flags_clear(vma, VM_ACCOUNT); + vma_clear_flags(vma, VMA_ACCOUNT_BIT); /* We are about to split vma, so store the start/end. */ vm_start = vma->vm_start; vm_end = vma->vm_end; @@ -1212,8 +1220,8 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) * | | * |-------------| * - * Having cleared VM_ACCOUNT from the whole VMA, after we unmap above - * we'll end up with: + * Having cleared VMA_ACCOUNT_BIT from the whole VMA, after we unmap + * above we'll end up with: * * addr end * | | @@ -1233,13 +1241,15 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) if (vm_start < addr) { struct vm_area_struct *prev = vma_prev(&vmi); - vm_flags_set(prev, VM_ACCOUNT); /* Acquires VMA lock. */ + vma_start_write(prev); + vma_set_flags(prev, VMA_ACCOUNT_BIT); } if (vm_end > end) { struct vm_area_struct *next = vma_next(&vmi); - vm_flags_set(next, VM_ACCOUNT); /* Acquires VMA lock. */ + vma_start_write(next); + vma_set_flags(next, VMA_ACCOUNT_BIT); } } } @@ -1255,17 +1265,17 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; - unsigned long moved_len; + const pgoff_t new_pgoff = linear_page_index(vrm->vma, vrm->addr); + const pgoff_t new_virt_pgoff = + __linear_virt_page_index(vrm->vma, vrm->addr); struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; + unsigned long moved_len; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, - &pmc.need_rmap_locks); + new_virt_pgoff, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); *new_vma_ptr = NULL; @@ -1324,8 +1334,8 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, unsigned long old_start = vrm->vma->vm_start; unsigned long old_end = vrm->vma->vm_end; - /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ - vm_flags_clear(vrm->vma, VM_LOCKED_MASK); + /* We always clear VMA_LOCKED[ONFAULT]_BIT on the old VMA. */ + vma_clear_flags_mask(vrm->vma, VMA_LOCKED_MASK); /* * anon_vma links of the old vma is no longer needed after its page @@ -1761,14 +1771,14 @@ static int check_prep_vma(struct vma_remap_struct *vrm) * based on the original. There are no known use cases for this * behavior. As a result, fail such attempts. */ - if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) { + if (!old_len && !vma_test_any(vma, VMA_SHARED_BIT, VMA_MAYSHARE_BIT)) { pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid); return -EINVAL; } if ((vrm->flags & MREMAP_DONTUNMAP) && - (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))) + vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT)) return -EINVAL; /* @@ -1798,19 +1808,18 @@ static int check_prep_vma(struct vma_remap_struct *vrm) return 0; /* We are expanding and the VMA is mlock()'d so we need to populate. */ - if (vma->vm_flags & VM_LOCKED) + if (vma_test(vma, VMA_LOCKED_BIT)) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; + pgoff = linear_page_index(vma, addr); if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) return -EINVAL; - if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) + if (vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT)) return -EFAULT; - if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) + if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), vrm->delta)) return -EAGAIN; if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) diff --git a/mm/mseal.c b/mm/mseal.c index 9781647483d1..7a8ac66dc215 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -16,32 +16,11 @@ #include <linux/sched.h> #include "internal.h" -/* - * mseal() disallows an input range which contain unmapped ranges (VMA holes). - * - * It disallows unmapped regions from start to end whether they exist at the - * start, in the middle, or at the end of the range, or any combination thereof. - * - * This is because after sealing a range, there's nothing to stop memory mapping - * of ranges in the remaining gaps later, meaning that the user might then - * wrongly consider the entirety of the mseal()'d range to be sealed when it - * in fact isn't. - */ - -/* - * Does the [start, end) range contain any unmapped memory? - * - * We ensure that: - * - start is part of a valid VMA. - * - end is part of a valid VMA. - * - no gap (unallocated memory) exists between start and end. - */ -static bool range_contains_unmapped(struct mm_struct *mm, - unsigned long start, unsigned long end) +static bool range_contains_unmapped(unsigned long start, unsigned long end) { - struct vm_area_struct *vma; - unsigned long prev_end = start; VMA_ITERATOR(vmi, current->mm, start); + unsigned long prev_end = start; + struct vm_area_struct *vma; for_each_vma_range(vmi, vma, end) { if (vma->vm_start > prev_end) @@ -53,11 +32,10 @@ static bool range_contains_unmapped(struct mm_struct *mm, return prev_end < end; } -static int mseal_apply(struct mm_struct *mm, - unsigned long start, unsigned long end) +static int __mseal_range(unsigned long start, unsigned long end) { + VMA_ITERATOR(vmi, current->mm, start); struct vm_area_struct *vma, *prev; - VMA_ITERATOR(vmi, mm, start); /* We know there are no gaps so this will be non-NULL. */ vma = vma_iter_load(&vmi); @@ -88,64 +66,58 @@ static int mseal_apply(struct mm_struct *mm, return 0; } +static int mseal_range(unsigned long start, unsigned long end) +{ + int err; + + err = mmap_write_lock_killable(current->mm); + if (err) + return err; + if (range_contains_unmapped(start, end)) + err = -ENOMEM; + else + err = __mseal_range(start, end); + mmap_write_unlock(current->mm); + return err; +} + +/** + * mseal_mmap_page_zero() - If the MMAP_PAGE_ZERO personality is set, mseal() + * the page mapped at address zero. + */ +void mseal_mmap_page_zero(void) +{ + int err; + + if (WARN_ON_ONCE(!(current->personality & MMAP_PAGE_ZERO))) + return; + + err = mseal_range(0, PAGE_SIZE); + if (err) + pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", + task_pid_nr(current), err); +} + /* - * mseal(2) seals the VM's meta data from - * selected syscalls. - * - * addr/len: VM address range. - * - * The address range by addr/len must meet: - * start (addr) must be in a valid VMA. - * end (addr + len) must be in a valid VMA. - * no gap (unallocated memory) between start and end. - * start (addr) must be page aligned. + * Seal VMAs in the specified input range to prevent an attacker replacing what + * is mapped in the range with something else. * - * len: len will be page aligned implicitly. + * Disallows: + * - VMA unmapping, remapping or shrinking. + * - Overwriting the VMA with another one via mmap(), mremap() or similar. + * - Alteration of properties via mprotect()/pkey_mprotect(). + * - Destructive madvise() behaviours (like MADV_DONTNEED) on anonymous read-only + * ranges. * - * Below VMA operations are blocked after sealing. - * 1> Unmapping, moving to another location, and shrinking - * the size, via munmap() and mremap(), can leave an empty - * space, therefore can be replaced with a VMA with a new - * set of attributes. - * 2> Moving or expanding a different vma into the current location, - * via mremap(). - * 3> Modifying a VMA via mmap(MAP_FIXED). - * 4> Size expansion, via mremap(), does not appear to pose any - * specific risks to sealed VMAs. It is included anyway because - * the use case is unclear. In any case, users can rely on - * merging to expand a sealed VMA. - * 5> mprotect and pkey_mprotect. - * 6> Some destructive madvice() behavior (e.g. MADV_DONTNEED) - * for anonymous memory, when users don't have write permission to the - * memory. Those behaviors can alter region contents by discarding pages, - * effectively a memset(0) for anonymous memory. + * Since unmapped ranges can be mapped at any time, the input range must span + * mapped ranges only. * - * flags: reserved. - * - * return values: - * zero: success. - * -EINVAL: - * invalid input flags. - * start address is not page aligned. - * Address range (start + len) overflow. - * -ENOMEM: - * addr is not a valid address (not allocated). - * end (start + len) is not a valid address. - * a gap (unallocated memory) between start and end. - * -EPERM: - * - In 32 bit architecture, sealing is not supported. - * Note: - * user can call mseal(2) multiple times, adding a seal on an - * already sealed memory is a no-action (no error). - * - * unseal() is not supported. + * The flags parameter is currently reserved. */ -int do_mseal(unsigned long start, size_t len_in, unsigned long flags) +SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, flags) { - size_t len; - int ret = 0; + size_t len_aligned; unsigned long end; - struct mm_struct *mm = current->mm; /* Verify flags not set. */ if (flags) @@ -155,41 +127,17 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) if (!PAGE_ALIGNED(start)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len_aligned = PAGE_ALIGN(len); /* Check to see whether len was rounded up from small -ve to zero. */ - if (len_in && !len) + if (len && !len_aligned) return -EINVAL; - end = start + len; + end = start + len_aligned; if (end < start) return -EINVAL; if (end == start) return 0; - if (mmap_write_lock_killable(mm)) - return -EINTR; - - if (range_contains_unmapped(mm, start, end)) { - ret = -ENOMEM; - goto out; - } - - /* - * Second pass, this should success, unless there are errors - * from vma_modify_flags, e.g. merge/split error, or process - * reaching the max supported VMAs, however, those cases shall - * be rare. - */ - ret = mseal_apply(mm, start, end); - -out: - mmap_write_unlock(mm); - return ret; -} - -SYSCALL_DEFINE3(mseal, unsigned long, start, size_t, len, unsigned long, - flags) -{ - return do_mseal(start, len, flags); + return mseal_range(start, end); } diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7..90b491a27a14 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -12,6 +12,7 @@ #include <linux/mm.h> #include <linux/mman.h> #include <linux/file.h> +#include <linux/pagemap.h> #include <linux/syscalls.h> #include <linux/sched.h> @@ -85,8 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) goto out_unlock; } file = vma->vm_file; - fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + fstart = (loff_t)linear_page_index(vma, start) << PAGE_SHIFT; fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/nommu.c b/mm/nommu.c index ed3934bc2de4..498e01ee40b0 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -975,7 +975,7 @@ static int do_mmap_private(struct vm_area_struct *vma, /* read the contents of a file into the copy */ loff_t fpos; - fpos = vma->vm_pgoff; + fpos = vma_start_pgoff(vma); fpos <<= PAGE_SHIFT; ret = kernel_read(vma->vm_file, base, len, &fpos); @@ -1014,11 +1014,12 @@ unsigned long do_mmap(struct file *file, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { + vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); struct vm_area_struct *vma; struct vm_region *region; struct rb_node *rb; @@ -1035,6 +1036,9 @@ unsigned long do_mmap(struct file *file, if (ret < 0) return ret; + if (current->mm->map_count >= get_sysctl_max_map_count()) + return -ENOMEM; + /* we ignore the address hint */ addr = 0; len = PAGE_ALIGN(len); @@ -1058,7 +1062,7 @@ unsigned long do_mmap(struct file *file, region->vm_pgoff = pgoff; vm_flags_init(vma, vm_flags); - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); if (file) { region->vm_file = get_file(file); @@ -1178,7 +1182,6 @@ unsigned long do_mmap(struct file *file, ret = do_mmap_private(vma, region, len, capabilities); if (ret < 0) goto error_just_free; - add_nommu_region(region); /* clear anonymous mappings that don't ask for uninitialized data */ if (!vma->vm_file && @@ -1196,7 +1199,9 @@ share: BUG_ON(!vma->vm_region); vma_iter_config(&vmi, vma->vm_start, vma->vm_end); if (vma_iter_prealloc(&vmi, vma)) - goto error_just_free; + goto error_vma_iter_prealloc; + + add_nommu_region(region); setup_vma_to_mm(vma, current->mm); current->mm->map_count++; @@ -1215,22 +1220,41 @@ share: return result; error_just_free: + vma_close(vma); + /* if the error was from shared mapping/existing region, don't free the region. + * this has to be before releasing semaphore. + */ + if (region->vm_usage == 1) { + if (region->vm_file) + fput(region->vm_file); + kmem_cache_free(vm_region_jar, region); + + } else + region->vm_usage--; + up_write(&nommu_region_sem); -error: vma_iter_free(&vmi); - if (region->vm_file) - fput(region->vm_file); - kmem_cache_free(vm_region_jar, region); + if (vma->vm_file) fput(vma->vm_file); vm_area_free(vma); return ret; sharing_violation: - up_write(&nommu_region_sem); pr_warn("Attempt to share mismatched mappings\n"); ret = -EINVAL; - goto error; + goto error_just_free; + +error_vma_iter_prealloc: + pr_warn("Allocation of vma iterator for process %d failed\n", current->pid); + show_mem(); + ret = -ENOMEM; + + /* in case that the region is allocated via do_mmap_private() */ + if ((region->vm_usage == 1) && (region->vm_flags & VM_MAPPED_COPY)) + free_page_series(region->vm_start, region->vm_top); + + goto error_just_free; error_getting_vma: kmem_cache_free(vm_region_jar, region); @@ -1332,13 +1356,14 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, *region = *vma->vm_region; new->vm_region = region; - npages = (addr - vma->vm_start) >> PAGE_SHIFT; + npages = linear_page_delta(vma, addr); if (new_below) { region->vm_top = region->vm_end = new->vm_end = addr; } else { region->vm_start = new->vm_start = addr; - region->vm_pgoff = new->vm_pgoff += npages; + vma_add_pgoff(new, npages); + region->vm_pgoff = vma_start_pgoff(new); } vma_iter_config(vmi, new->vm_start, new->vm_end); @@ -1355,7 +1380,8 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, delete_nommu_region(vma->vm_region); if (new_below) { vma->vm_region->vm_start = vma->vm_start = addr; - vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; + vma_add_pgoff(vma, npages); + vma->vm_region->vm_pgoff = vma_start_pgoff(vma); } else { vma->vm_region->vm_end = vma->vm_end = addr; vma->vm_region->vm_top = addr; @@ -1367,6 +1393,10 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, setup_vma_to_mm(vma, mm); setup_vma_to_mm(new, mm); vma_iter_store_new(vmi, new); + + /* vmi should point lower address */ + if (new_below) + vma_next(vmi); mm->map_count++; return 0; @@ -1603,7 +1633,7 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn = start >> PAGE_SHIFT; unsigned long vm_len = vma->vm_end - vma->vm_start; - pfn += vma->vm_pgoff; + pfn += vma_start_pgoff(vma); return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); } EXPORT_SYMBOL(vm_iomap_memory); @@ -1816,7 +1846,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, i_mmap_lock_read(inode->i_mapping); /* search for VMAs that fall within the dead zone */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, low, high) { /* found one - only interested if it's shared out of the page * cache */ if (vma->vm_flags & VM_SHARED) { @@ -1832,7 +1862,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, * we don't check for any regions that start beyond the EOF as there * shouldn't be any */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { if (!(vma->vm_flags & VM_SHARED)) continue; diff --git a/mm/page-writeback.c b/mm/page-writeback.c index e98748112d1e..47495be68598 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -584,16 +584,6 @@ static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr) wb->bdi->max_prop_frac, nr); } -void wb_writeout_inc(struct bdi_writeback *wb) -{ - unsigned long flags; - - local_irq_save(flags); - __wb_writeout_add(wb, 1); - local_irq_restore(flags); -} -EXPORT_SYMBOL_GPL(wb_writeout_inc); - /* * On idle system, we can be called long after we scheduled because we use * deferred timers so count with missed periods. diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee902a468c2f..f93a6bb9a872 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -56,6 +56,8 @@ #include <linux/pgalloc_tag.h> #include <asm/div64.h> #include "internal.h" +#include "mm_init.h" +#include "page_alloc.h" #include "shuffle.h" #include "page_reporting.h" @@ -88,7 +90,7 @@ typedef int __bitwise fpi_t; #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* Free the page without taking locks. Rely on trylock only. */ -#define FPI_TRYLOCK ((__force fpi_t)BIT(2)) +#define FPI_NOLOCK ((__force fpi_t)BIT(2)) /* free_pages_prepare() has already been called for page(s) being freed. */ #define FPI_PREPARED ((__force fpi_t)BIT(3)) @@ -165,7 +167,9 @@ DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); #endif +#ifdef CONFIG_NUMA DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); +#endif #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* @@ -1246,7 +1250,7 @@ void __clear_page_tag_ref(struct page *page) /* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) + unsigned int nr, unsigned int alloc_flags) { union pgtag_ref_handle handle; union codetag_ref ref; @@ -1260,17 +1264,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task, * page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ - alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags); + alloc_tag_add_early_pfn(page_to_pfn(page), alloc_flags); if (task->alloc_tag) alloc_tag_set_inaccurate(task->alloc_tag); } } static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) + unsigned int nr, unsigned int alloc_flags) { if (mem_alloc_profiling_enabled()) - __pgalloc_tag_add(page, task, nr, gfp_flags); + __pgalloc_tag_add(page, task, nr, alloc_flags); } /* Should be called only if mem_alloc_profiling_enabled() */ @@ -1303,7 +1307,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr, gfp_t gfp_flags) {} + unsigned int nr, unsigned int alloc_flags) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} @@ -1374,15 +1378,23 @@ static __always_inline bool __free_pages_prepare(struct page *page, #endif } for (i = 1; i < (1 << order); i++) { + struct page *tail_page = page + i; + if (compound) - bad += free_tail_page_prepare(page, page + i); + bad += free_tail_page_prepare(page, tail_page); if (is_check_pages_enabled()) { - if (free_page_is_bad(page + i)) { + if (free_page_is_bad(tail_page)) { + bad++; + continue; + } + + if (tail_page->private) { + bad_page(tail_page, "nonzero private"); bad++; continue; } } - (page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; + tail_page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; } } if (folio_test_anon(folio)) { @@ -1407,7 +1419,7 @@ static __always_inline bool __free_pages_prepare(struct page *page, page_table_check_free(page, order); pgalloc_tag_sub(page, 1 << order); - if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { + if (!PageHighMem(page) && !(fpi_flags & FPI_NOLOCK)) { debug_check_no_locks_freed(page_address(page), PAGE_SIZE << order); debug_check_no_obj_freed(page_address(page), @@ -1546,8 +1558,8 @@ static void free_one_page(struct zone *zone, struct page *page, struct llist_head *llhead; unsigned long flags; - if (unlikely(fpi_flags & FPI_TRYLOCK)) { - if (!spin_trylock_irqsave(&zone->lock, flags)) { + if (unlikely(fpi_flags & FPI_NOLOCK)) { + if (!can_spin_trylock() || !spin_trylock_irqsave(&zone->lock, flags)) { add_page_to_zone_llist(zone, page, order); return; } @@ -1557,7 +1569,7 @@ static void free_one_page(struct zone *zone, struct page *page, /* The lock succeeded. Process deferred pages. */ llhead = &zone->trylock_free_pages; - if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_TRYLOCK))) { + if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_NOLOCK))) { struct llist_node *llnode; struct page *p, *tmp; @@ -1807,7 +1819,7 @@ static inline bool should_skip_init(gfp_t flags) } inline void post_alloc_hook(struct page *page, unsigned int order, - gfp_t gfp_flags) + gfp_t gfp_flags, unsigned int alloc_flags) { const bool zero_tags = gfp_flags & __GFP_ZEROTAGS; bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && @@ -1858,13 +1870,13 @@ inline void post_alloc_hook(struct page *page, unsigned int order, set_page_owner(page, order, gfp_flags); page_table_check_alloc(page, order); - pgalloc_tag_add(page, current, 1 << order, gfp_flags); + pgalloc_tag_add(page, current, 1 << order, alloc_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, unsigned int alloc_flags) { - post_alloc_hook(page, order, gfp_flags); + post_alloc_hook(page, order, gfp_flags, alloc_flags); if (order && (gfp_flags & __GFP_COMP)) prep_compound_page(page, order); @@ -2528,7 +2540,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, unsigned long flags; int i; - if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { + if (unlikely(alloc_flags & ALLOC_NOLOCK)) { if (!spin_trylock_irqsave(&zone->lock, flags)) return 0; } else { @@ -2870,7 +2882,7 @@ static bool free_frozen_page_commit(struct zone *zone, if (pcp->free_count < (batch << CONFIG_PCP_BATCH_SCALE_MAX)) pcp->free_count += (1 << order); - if (unlikely(fpi_flags & FPI_TRYLOCK)) { + if (unlikely(fpi_flags & FPI_NOLOCK)) { /* * Do not attempt to take a zone lock. Let pcp->count get * over high mark temporarily. @@ -2967,8 +2979,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, migratetype = MIGRATE_MOVABLE; } - if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT) - && (in_nmi() || in_hardirq()))) { + if (unlikely((fpi_flags & FPI_NOLOCK) && !can_spin_trylock())) { add_page_to_zone_llist(zone, page, order); return; } @@ -2990,7 +3001,7 @@ void free_frozen_pages(struct page *page, unsigned int order) void free_frozen_pages_nolock(struct page *page, unsigned int order) { - __free_frozen_pages(page, order, FPI_TRYLOCK); + __free_frozen_pages(page, order, FPI_NOLOCK); } /* @@ -3216,7 +3227,7 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, do { page = NULL; - if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { + if (unlikely(alloc_flags & ALLOC_NOLOCK)) { if (!spin_trylock_irqsave(&zone->lock, flags)) return NULL; } else { @@ -3247,10 +3258,11 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, } while (check_new_pages(page, order)); /* - * If this is a high-order atomic allocation then check - * if the pageblock should be reserved for the future + * Slowpath (precarious) high-atomic allocations may reserve + * a pageblock for future use. */ - if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) + if (unlikely((alloc_flags & ALLOC_HIGHATOMIC) && + ((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN))) reserve_highatomic_pageblock(page, order, zone); __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); @@ -3739,13 +3751,10 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) static inline unsigned int alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) { - unsigned int alloc_flags; + unsigned int alloc_flags = 0; - /* - * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save a branch. - */ - alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; if (defrag_mode) { alloc_flags |= ALLOC_NOFRAGMENT; @@ -3774,14 +3783,13 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) } /* Must be called after current_gfp_context() which can change gfp_mask */ -static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, - unsigned int alloc_flags) +static inline unsigned int alloc_flags_cma(gfp_t gfp_mask) { #ifdef CONFIG_CMA if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) - alloc_flags |= ALLOC_CMA; + return ALLOC_CMA; #endif - return alloc_flags; + return ALLOC_DEFAULT; } /* @@ -3899,8 +3907,6 @@ check_alloc_wmark: if (!zone_watermark_fast(zone, order, mark, ac->highest_zoneidx, alloc_flags, gfp_mask)) { - int ret; - if (cond_accept_memory(zone, order, alloc_flags)) goto try_this_zone; @@ -3921,22 +3927,13 @@ check_alloc_wmark: !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone)) continue; - ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); - switch (ret) { - case NODE_RECLAIM_NOSCAN: - /* did not scan */ + if (!node_reclaim(zone->zone_pgdat, gfp_mask, order)) continue; - case NODE_RECLAIM_FULL: - /* scanned but unreclaimable */ - continue; - default: - /* did we reclaim enough */ - if (zone_watermark_ok(zone, order, mark, - ac->highest_zoneidx, alloc_flags)) - goto try_this_zone; + /* did we reclaim enough */ + if (!zone_watermark_ok(zone, order, mark, + ac->highest_zoneidx, alloc_flags)) continue; - } } try_this_zone: @@ -3979,7 +3976,7 @@ try_this_zone: return NULL; } -static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) +static void warn_alloc_show_mem(gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned int filter = SHOW_MEM_FILTER_NODES; @@ -3999,7 +3996,7 @@ static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) mem_cgroup_show_protected_memory(NULL); } -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...) { struct va_format vaf; va_list args; @@ -4077,7 +4074,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, */ page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & ~__GFP_DIRECT_RECLAIM, order, - ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); + ac->alloc_flags|ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); if (page) goto out; @@ -4123,7 +4120,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, */ if (gfp_mask & __GFP_NOFAIL) page = __alloc_pages_cpuset_fallback(gfp_mask, order, - ALLOC_NO_WATERMARKS, ac); + ac->alloc_flags|ALLOC_NO_WATERMARKS, ac); } out: mutex_unlock(&oom_lock); @@ -4474,17 +4471,32 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, } static inline unsigned int -gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) +alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) { - unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; + unsigned int alloc_flags = 0; + + if (gfp_mask & __GFP_DIRECT_RECLAIM) + return 0; /* - * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE - * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save two branches. + * Not worth trying to allocate harder for __GFP_NOMEMALLOC even + * if it can't schedule. */ - BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE); - BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); + if (gfp_mask & __GFP_NOMEMALLOC) + return 0; + + alloc_flags |= ALLOC_NON_BLOCK; + + if (order > 0 && (gfp_mask & __GFP_HIGH)) + alloc_flags |= ALLOC_HIGHATOMIC; + + return alloc_flags; +} + +static inline unsigned int +alloc_flags_slowpath(gfp_t gfp_mask, unsigned int order) +{ + unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; /* * The caller may dip into page reserves a bit more if the caller @@ -4492,21 +4504,14 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH). */ - alloc_flags |= (__force int) - (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); - - if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { - /* - * Not worth trying to allocate harder for __GFP_NOMEMALLOC even - * if it can't schedule. - */ - if (!(gfp_mask & __GFP_NOMEMALLOC)) { - alloc_flags |= ALLOC_NON_BLOCK; + if (gfp_mask & __GFP_HIGH) + alloc_flags |= ALLOC_MIN_RESERVE; + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; - if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE)) - alloc_flags |= ALLOC_HIGHATOMIC; - } + alloc_flags |= alloc_flags_nonblocking(gfp_mask, order); + if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { /* * Ignore cpuset mems for non-blocking __GFP_HIGH (probably * GFP_ATOMIC) rather than fail, see the comment for @@ -4517,7 +4522,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) } else if (unlikely(rt_or_dl_task(current)) && in_task()) alloc_flags |= ALLOC_MIN_RESERVE; - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); + alloc_flags |= alloc_flags_cma(gfp_mask); if (defrag_mode) alloc_flags |= ALLOC_NOFRAGMENT; @@ -4687,7 +4692,7 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) return false; } -static void check_alloc_stall_warn(gfp_t gfp_mask, nodemask_t *nodemask, +static void check_alloc_stall_warn(gfp_t gfp_mask, const nodemask_t *nodemask, unsigned int order, unsigned long alloc_start_time) { static DEFINE_SPINLOCK(alloc_stall_lock); @@ -4782,8 +4787,12 @@ restart: * The fast path uses conservative alloc_flags to succeed only until * kswapd needs to be woken up, and to avoid the cost of setting up * alloc_flags precisely. So we do that now. + * + * Can't just or alloc_flags if it contains WMARK bits, but those flags + * shouldn't be set in ac->alloc_flags. */ - alloc_flags = gfp_to_alloc_flags(gfp_mask, order); + VM_WARN_ON(ac->alloc_flags & ALLOC_WMARK_MASK); + alloc_flags = ac->alloc_flags | alloc_flags_slowpath(gfp_mask, order); /* * We need to recalculate the starting point for the zonelist iterator @@ -4824,8 +4833,8 @@ retry: reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); if (reserve_flags) - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | - (alloc_flags & ALLOC_KSWAPD); + alloc_flags = alloc_flags_cma(gfp_mask) | reserve_flags | + ac->alloc_flags | (alloc_flags & ALLOC_KSWAPD); /* * Reset the nodemask and zonelist iterators if memory policies can be @@ -4994,6 +5003,8 @@ nopage: * we always retry */ if (unlikely(nofail)) { + unsigned int alloc_flags = ac->alloc_flags | ALLOC_MIN_RESERVE; + /* * Lacking direct_reclaim we can't do anything to reclaim memory, * we disregard these unreasonable nofail requests and still @@ -5009,7 +5020,7 @@ nopage: * could deplete whole memory reserves which would just make * the situation worse. */ - page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac); + page = __alloc_pages_cpuset_fallback(gfp_mask, order, alloc_flags, ac); if (page) goto got_pg; @@ -5051,11 +5062,11 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, * Don't invoke should_fail logic, since it may call * get_random_u32() and printk() which need to spin_lock. */ - if (!(*alloc_flags & ALLOC_TRYLOCK) && + if (!(*alloc_flags & ALLOC_NOLOCK) && should_fail_alloc_page(gfp_mask, order)) return false; - *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); + *alloc_flags |= alloc_flags_cma(gfp_mask); /* Dirty zone balancing only done in the fast path */ ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); @@ -5213,7 +5224,7 @@ retry_this_zone: } nr_account++; - prep_new_page(page, 0, gfp, 0); + prep_new_page(page, 0, gfp, ALLOC_DEFAULT); set_page_refcounted(page); page_array[nr_populated++] = page; } @@ -5227,7 +5238,7 @@ out: return nr_populated; failed: - page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask); + page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask, ALLOC_DEFAULT); if (page) page_array[nr_populated++] = page; goto out; @@ -5262,24 +5273,87 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages) } } +static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order, + unsigned int alloc_flags) +{ + if (alloc_flags & ALLOC_NOLOCK) + return pcp_allowed_order(order); + + /* + * There are several places where we assume that the order value is sane + * so bail out early if the request is out of bound. + */ + return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)); +} + +static inline bool alloc_nolock_allowed(void) +{ + if (!can_spin_trylock()) + return false; + + /* Bailout, since _deferred_grow_zone() needs to take a lock */ + if (deferred_pages_enabled()) + return false; + + return true; +} + +/* + * GFP flags to set for ALLOC_NOLOCK i.e. alloc_pages_nolock(). + * + * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. + * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd + * is not safe in arbitrary context. + * + * These two are the conditions for gfpflags_allow_spinning() being true. + * + * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason + * to warn. Also warn would trigger printk() which is unsafe from + * various contexts. We cannot use printk_deferred_enter() to mitigate, + * since the running context is unknown. + * + * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below + * is safe in any context. Also zeroing the page is mandatory for + * BPF use cases. + * + * Though __GFP_NOMEMALLOC is not checked in the code path below, + * specify it here to highlight that alloc_pages_nolock() + * doesn't want to deplete reserves. + */ +static const gfp_t gfp_nolock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | + __GFP_COMP; + /* * This is the 'heart' of the zoned buddy allocator. */ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, - int preferred_nid, nodemask_t *nodemask) + int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags) { struct page *page; - unsigned int alloc_flags = ALLOC_WMARK_LOW; gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ - struct alloc_context ac = { }; + struct alloc_context ac = { + .alloc_flags = alloc_flags, + }; + unsigned int fastpath_alloc_flags = alloc_flags; - /* - * There are several places where we assume that the order value is sane - * so bail out early if the request is out of bound. - */ - if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)) + /* Other flags could be supported later if needed. */ + if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG))) + return NULL; + + if (!alloc_order_allowed(gfp, order, alloc_flags)) return NULL; + if (alloc_flags & ALLOC_NOLOCK) { + /* Certain other flags could be supported later if needed. */ + VM_WARN_ON_ONCE(gfp & ~(__GFP_ACCOUNT | gfp_nolock)); + if (!alloc_nolock_allowed()) + return NULL; + gfp |= gfp_nolock; + fastpath_alloc_flags |= ALLOC_WMARK_MIN; + } else { + fastpath_alloc_flags |= ALLOC_WMARK_LOW; + } + gfp &= gfp_allowed_mask; /* * Apply scoped allocation constraints. This is mainly about GFP_NOFS @@ -5291,18 +5365,22 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, gfp = current_gfp_context(gfp); alloc_gfp = gfp; if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, - &alloc_gfp, &alloc_flags)) + &alloc_gfp, &fastpath_alloc_flags)) return NULL; - /* - * Forbid the first pass from falling back to types that fragment - * memory until all local zones are considered. - */ - alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + if (!(alloc_flags & ALLOC_NOLOCK)) { + /* + * Forbid the first pass from falling back to types that + * fragment memory until all local zones are considered. + */ + fastpath_alloc_flags |= alloc_flags_nofragment( + zonelist_zone(ac.preferred_zoneref), gfp); + } + fastpath_alloc_flags |= alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; - /* First allocation attempt */ - page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); - if (likely(page)) + /* First allocation attempt (or, for nolock, only attempt) */ + page = get_page_from_freelist(alloc_gfp, order, fastpath_alloc_flags, &ac); + if (likely(page) || (alloc_flags & ALLOC_NOLOCK)) goto out; alloc_gfp = gfp; @@ -5319,7 +5397,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, out: if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { - free_frozen_pages(page, order); + __free_frozen_pages(page, order, + alloc_flags & ALLOC_NOLOCK ? FPI_NOLOCK : 0); page = NULL; } @@ -5331,22 +5410,33 @@ out: EXPORT_SYMBOL(__alloc_frozen_pages_noprof); struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, - int preferred_nid, nodemask_t *nodemask) + int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags) { struct page *page; - page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask); + page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, + alloc_flags); if (page) set_page_refcounted(page); return page; } -EXPORT_SYMBOL(__alloc_pages_noprof); + +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) +{ + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + warn_if_node_offline(nid, gfp_mask); + + return __alloc_pages_noprof(gfp_mask, order, nid, NULL, ALLOC_DEFAULT); +} +EXPORT_SYMBOL(alloc_pages_node_noprof); struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask) { struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order, - preferred_nid, nodemask); + preferred_nid, nodemask, ALLOC_DEFAULT); return page_rmappable_folio(page); } EXPORT_SYMBOL(__folio_alloc_noprof); @@ -5430,7 +5520,7 @@ EXPORT_SYMBOL(__free_pages); */ void free_pages_nolock(struct page *page, unsigned int order) { - ___free_pages(page, order, FPI_TRYLOCK); + ___free_pages(page, order, FPI_NOLOCK); } /** @@ -6287,15 +6377,34 @@ void adjust_managed_page_count(struct page *page, long count) } EXPORT_SYMBOL(adjust_managed_page_count); -void free_reserved_page(struct page *page) +/** + * free_reserved_pages - free reserved pages + * @page: First page to free. + * @order: The page order to free. + * + * Free pages allocated through memblock during boot, letting the buddy + * manage them from now on. + * + * @page must be naturally aligned to the order and the order must not + * exceed MAX_PAGE_ORDER. All pages must be reserved. + */ +void free_reserved_pages(struct page *page, unsigned int order) { - clear_page_tag_ref(page); - ClearPageReserved(page); - init_page_count(page); - __free_page(page); - adjust_managed_page_count(page, 1); + const unsigned long nr_pages = 1UL << order; + int i; + + VM_WARN_ON_ONCE(!IS_ALIGNED(page_to_pfn(page), nr_pages)); + VM_WARN_ON_ONCE(order > MAX_PAGE_ORDER); + + for (i = 0; i < nr_pages; i++) { + clear_page_tag_ref(page + i); + set_page_count(page + i, 0); + ClearPageReserved(page + i); + } + adjust_managed_page_count(page, nr_pages); + __free_frozen_pages(page, order, FPI_NONE); } -EXPORT_SYMBOL(free_reserved_page); +EXPORT_SYMBOL(free_reserved_pages); static int page_alloc_cpu_dead(unsigned int cpu) { @@ -6904,15 +7013,15 @@ static void __free_contig_range_common(unsigned long pfn, unsigned long nr_pages continue; } - if (start && memdesc_section(page->flags) != start_sec) { + if (start && memdesc_section(&page->flags) != start_sec) { free_prepared_contig_range(start, i - nr_start); start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } else if (!start) { start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } } @@ -7028,7 +7137,7 @@ static void split_free_frozen_pages(struct list_head *list, gfp_t gfp_mask) list_for_each_entry_safe(page, next, &list[order], lru) { int i; - post_alloc_hook(page, order, gfp_mask); + post_alloc_hook(page, order, gfp_mask, ALLOC_DEFAULT); if (!order) continue; @@ -7233,7 +7342,7 @@ int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end, struct page *head = pfn_to_page(start); check_new_pages(head, order); - prep_new_page(head, order, gfp_mask, 0); + prep_new_page(head, order, gfp_mask, ALLOC_DEFAULT); } else { ret = -EINVAL; WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n", @@ -7795,7 +7904,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, return false; /* Bailout, since try_to_accept_memory_one() needs to take a lock */ - if (alloc_flags & ALLOC_TRYLOCK) + if (alloc_flags & ALLOC_NOLOCK) return false; wmark = promo_wmark_pages(zone); @@ -7865,84 +7974,15 @@ static bool __free_unaccepted(struct page *page) struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order) { - /* - * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. - * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd - * is not safe in arbitrary context. - * - * These two are the conditions for gfpflags_allow_spinning() being true. - * - * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason - * to warn. Also warn would trigger printk() which is unsafe from - * various contexts. We cannot use printk_deferred_enter() to mitigate, - * since the running context is unknown. - * - * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below - * is safe in any context. Also zeroing the page is mandatory for - * BPF use cases. - * - * Though __GFP_NOMEMALLOC is not checked in the code path below, - * specify it here to highlight that alloc_pages_nolock() - * doesn't want to deplete reserves. - */ - gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP - | gfp_flags; - unsigned int alloc_flags = ALLOC_TRYLOCK; - struct alloc_context ac = { }; - struct page *page; - - VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT); - /* - * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is - * unsafe in NMI. If spin_trylock() is called from hard IRQ the current - * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will - * mark the task as the owner of another rt_spin_lock which will - * confuse PI logic, so return immediately if called from hard IRQ or - * NMI. - * - * Note, irqs_disabled() case is ok. This function can be called - * from raw_spin_lock_irqsave region. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return NULL; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) - return NULL; - - if (!pcp_allowed_order(order)) - return NULL; - - /* Bailout, since _deferred_grow_zone() needs to take a lock */ - if (deferred_pages_enabled()) - return NULL; - if (nid == NUMA_NO_NODE) nid = numa_node_id(); - prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac, - &alloc_gfp, &alloc_flags); - - /* - * Best effort allocation from percpu free list. - * If it's empty attempt to spin_trylock zone->lock. - */ - page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); - - /* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */ - - if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) && - unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) { - __free_frozen_pages(page, order, FPI_TRYLOCK); - page = NULL; - } - trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); - kmsan_alloc_page(page, order, alloc_gfp); - return page; + return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_NOLOCK); } /** * alloc_pages_nolock - opportunistic reentrant allocation from any context - * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed. + * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, plus some flags that get set + * internally regardless (see %gfp_nolock) are allowed. * @nid: node to allocate from * @order: allocation order size * diff --git a/mm/page_alloc.h b/mm/page_alloc.h new file mode 100644 index 000000000000..b9259deddb59 --- /dev/null +++ b/mm/page_alloc.h @@ -0,0 +1,312 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal API for the page (buddy) allocator. Public API lives in + * include/linux/gfp.h. + */ +#ifndef __MM_PAGE_ALLOC_H +#define __MM_PAGE_ALLOC_H + +#include <linux/mm.h> +#include <linux/mmzone.h> +#include <linux/nodemask.h> +#include <linux/types.h> + +#define ALLOC_DEFAULT 0 +/* The ALLOC_WMARK bits are used as an index to zone->watermark */ +#define ALLOC_WMARK_MIN WMARK_MIN +#define ALLOC_WMARK_LOW WMARK_LOW +#define ALLOC_WMARK_HIGH WMARK_HIGH +#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ + +/* Mask to get the watermark bits */ +#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) + +/* + * Only MMU archs have async oom victim reclaim - aka oom_reaper so we + * cannot assume a reduced access to memory reserves is sufficient for + * !MMU + */ +#ifdef CONFIG_MMU +#define ALLOC_OOM 0x08 +#else +#define ALLOC_OOM ALLOC_NO_WATERMARKS +#endif + +#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access + * to 25% of the min watermark or + * 62.5% if __GFP_HIGH is set. + */ +#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% + * of the min watermark. + */ +#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ +#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ +#ifdef CONFIG_ZONE_DMA32 +#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ +#else +#define ALLOC_NOFRAGMENT 0x0 +#endif +#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ +#define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */ +#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ +/* + * Avoid alloc_tag recursion for internal allocations. + * + * Callers must clear_page_tag_ref() before freeing to avoid warnings from + * alloc_tag_sub_check(). + */ +#define ALLOC_NO_CODETAG 0x1000 + +/* Flags that allow allocations below the min watermark. */ +#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) + +/* + * Structure for holding the mostly immutable allocation parameters passed + * between functions involved in allocations, including the alloc_pages* + * family of functions. + * + * nodemask, migratetype and highest_zoneidx are initialized only once in + * __alloc_pages() and then never change. + * + * zonelist, preferred_zone and highest_zoneidx are set first in + * __alloc_pages() for the fast path, and might be later changed + * in __alloc_pages_slowpath(). All other functions pass the whole structure + * by a const pointer. + */ +struct alloc_context { + struct zonelist *zonelist; + const nodemask_t *nodemask; + struct zoneref *preferred_zoneref; + int migratetype; + + /* + * highest_zoneidx represents highest usable zone index of + * the allocation request. Due to the nature of the zone, + * memory on lower zone than the highest_zoneidx will be + * protected by lowmem_reserve[highest_zoneidx]. + * + * highest_zoneidx is also used by reclaim/compaction to limit + * the target zone since higher zone than this index cannot be + * usable for this allocation request. + */ + enum zone_type highest_zoneidx; + bool spread_dirty_pages; + /* Only flags that are global to the whole allocation go here. */ + unsigned int alloc_flags; +}; + +/* + * This function returns the order of a free page in the buddy system. In + * general, page_zone(page)->lock must be held by the caller to prevent the + * page from being allocated in parallel and returning garbage as the order. + * If a caller does not hold page_zone(page)->lock, it must guarantee that the + * page cannot be allocated or merged in parallel. Alternatively, it must + * handle invalid values gracefully, and use buddy_order_unsafe() below. + */ +static inline unsigned int buddy_order(struct page *page) +{ + /* PageBuddy() must be checked by the caller */ + return page_private(page); +} + +/* + * Like buddy_order(), but for callers who cannot afford to hold the zone lock. + * PageBuddy() should be checked first by the caller to minimize race window, + * and invalid values must be handled gracefully. + * + * READ_ONCE is used so that if the caller assigns the result into a local + * variable and e.g. tests it for valid range before using, the compiler cannot + * decide to remove the variable and inline the page_private(page) multiple + * times, potentially observing different values in the tests and the actual + * use of the result. + */ +#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) + +/* + * This function checks whether a page is free && is the buddy + * we can coalesce a page and its buddy if + * (a) the buddy is not in a hole (check before calling!) && + * (b) the buddy is in the buddy system && + * (c) a page and its buddy have the same order && + * (d) a page and its buddy are in the same zone. + * + * For recording whether a page is in the buddy system, we set PageBuddy. + * Setting, clearing, and testing PageBuddy is serialized by zone->lock. + * + * For recording page's order, we use page_private(page). + */ +static inline bool page_is_buddy(struct page *page, struct page *buddy, + unsigned int order) +{ + if (!page_is_guard(buddy) && !PageBuddy(buddy)) + return false; + + if (buddy_order(buddy) != order) + return false; + + /* + * zone check is done late to avoid uselessly calculating + * zone/node ids for pages that could never merge. + */ + if (page_zone_id(page) != page_zone_id(buddy)) + return false; + + VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); + + return true; +} + +/* + * Locate the struct page for both the matching buddy in our + * pair (buddy1) and the combined O(n+1) page they form (page). + * + * 1) Any buddy B1 will have an order O twin B2 which satisfies + * the following equation: + * B2 = B1 ^ (1 << O) + * For example, if the starting buddy (buddy2) is #8 its order + * 1 buddy is #10: + * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 + * + * 2) Any buddy B will have an order O+1 parent P which + * satisfies the following equation: + * P = B & ~(1 << O) + * + * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER + */ +static inline unsigned long +__find_buddy_pfn(unsigned long page_pfn, unsigned int order) +{ + return page_pfn ^ (1 << order); +} + +/* + * Find the buddy of @page and validate it. + * @page: The input page + * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the + * function is used in the performance-critical __free_one_page(). + * @order: The order of the page + * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to + * page_to_pfn(). + * + * The found buddy can be a non PageBuddy, out of @page's zone, or its order is + * not the same as @page. The validation is necessary before use it. + * + * Return: the found buddy page or NULL if not found. + */ +static inline struct page *find_buddy_page_pfn(struct page *page, + unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) +{ + unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); + struct page *buddy; + + buddy = page + (__buddy_pfn - pfn); + if (buddy_pfn) + *buddy_pfn = __buddy_pfn; + + if (page_is_buddy(page, buddy, order)) + return buddy; + return NULL; +} + +extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, + unsigned long end_pfn, struct zone *zone); + +static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, + unsigned long end_pfn, struct zone *zone) +{ + if (zone->contiguous) + return pfn_to_page(start_pfn); + + return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); +} + +extern void __free_pages_core(struct page *page, unsigned int order, + enum meminit_context context); + +void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags, + unsigned int alloc_flags); +extern bool free_pages_prepare(struct page *page, unsigned int order); + +extern int user_min_free_kbytes; + +struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, + nodemask_t *nodemask, unsigned int alloc_flags); +#define __alloc_frozen_pages(...) \ + alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) +void free_frozen_pages(struct page *page, unsigned int order); +void free_unref_folios(struct folio_batch *fbatch); + +#ifdef CONFIG_NUMA +struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); +#else +static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) +{ + return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, + ALLOC_DEFAULT); +} +#endif + +#define alloc_frozen_pages(...) \ + alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) + +struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); +#define alloc_frozen_pages_nolock(...) \ + alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) +void free_frozen_pages_nolock(struct page *page, unsigned int order); + +struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, + nodemask_t *nodemask, unsigned int alloc_flags); +#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) + +extern void zone_pcp_reset(struct zone *zone); +extern void zone_pcp_disable(struct zone *zone); +extern void zone_pcp_enable(struct zone *zone); +extern void zone_pcp_init(struct zone *zone); + +enum fallback_result { + /* Found suitable migratetype, *mt_out is valid. */ + FALLBACK_FOUND, + /* No fallback found in requested order. */ + FALLBACK_EMPTY, + /* Passed @claimable, but claiming whole block is a bad idea. */ + FALLBACK_NOCLAIM, +}; +enum fallback_result +find_suitable_fallback(struct free_area *area, unsigned int order, + int migratetype, bool claimable, int *mt_out); + +static inline bool free_area_empty(struct free_area *area, int migratetype) +{ + return list_empty(&area->free_list[migratetype]); +} + +/* Convert GFP flags to their corresponding migrate type */ +#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) +#define GFP_MOVABLE_SHIFT 3 + +static inline int gfp_migratetype(const gfp_t gfp_flags) +{ + VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); + BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); + BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); + BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); + BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> + GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); + + if (unlikely(page_group_by_mobility_disabled)) + return MIGRATE_UNMOVABLE; + + /* Group based on mobility */ + return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; +} +#undef GFP_MOVABLE_MASK +#undef GFP_MOVABLE_SHIFT + +bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); +void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); +void drain_all_pages(struct zone *zone); + +void page_alloc_init_cpuhp(void); +void page_alloc_sysctl_init(void); + +#endif /* __MM_PAGE_ALLOC_H */ diff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c index d2423f30577e..e63efe78b7d4 100644 --- a/mm/page_frag_cache.c +++ b/mm/page_frag_cache.c @@ -18,7 +18,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/page_frag_cache.h> -#include "internal.h" +#include "page_alloc.h" static unsigned long encoded_page_create(struct page *page, unsigned int order, bool pfmemalloc) @@ -57,10 +57,10 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER, - numa_mem_id(), NULL); + numa_mem_id(), NULL, ALLOC_DEFAULT); #endif if (unlikely(!page)) { - page = __alloc_pages(gfp, 0, numa_mem_id(), NULL); + page = __alloc_pages(gfp, 0, numa_mem_id(), NULL, ALLOC_DEFAULT); order = 0; } diff --git a/mm/page_io.c b/mm/page_io.c index b23f494fcc83..e4fa7ffffe8b 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -28,54 +28,6 @@ #include "swap.h" #include "swap_table.h" -static void __end_swap_bio_write(struct bio *bio) -{ - struct folio *folio = bio_first_folio_all(bio); - - if (bio->bi_status) { - /* - * We failed to write the page out to swap-space. - * Re-dirty the page in order to avoid it being reclaimed. - * Also print a dire warning that things will go BAD (tm) - * very quickly. - * - * Also clear PG_reclaim to avoid folio_rotate_reclaimable() - */ - folio_mark_dirty(folio); - pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); - folio_clear_reclaim(folio); - } - folio_end_writeback(folio); -} - -static void end_swap_bio_write(struct bio *bio) -{ - __end_swap_bio_write(bio); - bio_put(bio); -} - -static void __end_swap_bio_read(struct bio *bio) -{ - struct folio *folio = bio_first_folio_all(bio); - - if (bio->bi_status) { - pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); - } else { - folio_mark_uptodate(folio); - } - folio_unlock(folio); -} - -static void end_swap_bio_read(struct bio *bio) -{ - __end_swap_bio_read(bio); - bio_put(bio); -} - int generic_swapfile_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *span) @@ -248,7 +200,7 @@ static void swap_zeromap_folio_clear(struct folio *folio) * We may have stale swap cache pages in memory: notice * them here and get rid of the unnecessary final write. */ -int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) +int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { int ret = 0; @@ -295,39 +247,44 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) } rcu_read_unlock(); - __swap_writepage(folio, swap_plug); + __swap_writepage(ctx, folio); return 0; out_unlock: folio_unlock(folio); return ret; } -static inline void count_swpout_vm_event(struct folio *folio) +#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) +static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (unlikely(folio_test_pmd_mappable(folio))) { - count_memcg_folio_events(folio, THP_SWPOUT, 1); - count_vm_event(THP_SWPOUT); + return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys); +} + +static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +{ + bool can_merge = true; + + if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio)) + return false; + if (folio_memcg_charged(folio)) { + rcu_read_lock(); + if (folio_memcg_blkg_css(folio) != + folio_memcg_blkg_css(prev_folio)) + can_merge = false; + rcu_read_unlock(); } -#endif - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); - count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio)); - count_vm_events(PSWPOUT, folio_nr_pages(folio)); + return can_merge; } -#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) { struct cgroup_subsys_state *css; - struct mem_cgroup *memcg; if (!folio_memcg_charged(folio)) return; - rcu_read_lock(); - memcg = folio_memcg(folio); - css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); - if (!css || !css_tryget(css)) + css = folio_memcg_blkg_css(folio); + if (css && !css_tryget(css)) css = NULL; rcu_read_unlock(); @@ -336,11 +293,18 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) css_put(css); } #else +static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio) +{ + return true; +} #define bio_associate_blkg_from_page(bio, folio) do { } while (0) #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ struct swap_iocb { - struct kiocb iocb; + union { + struct kiocb iocb; + struct bio bio; + }; struct bio_vec bvecs[SWAP_CLUSTER_MAX]; int nr_bvecs; int len; @@ -360,168 +324,65 @@ int sio_pool_init(void) return 0; } -static void sio_write_complete(struct kiocb *iocb, long ret) +static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio, + int rw) { - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); - struct page *page = sio->bvecs[0].bv_page; - int p; - - if (ret != sio->len) { - /* - * In the case of swap-over-nfs, this can be a - * temporary failure if the system has limited - * memory for allocating transmit buffers. - * Mark the page dirty and avoid - * folio_rotate_reclaimable but rate-limit the - * messages. - */ - pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", - ret, swap_dev_pos(page_swap_entry(page))); - for (p = 0; p < sio->nr_bvecs; p++) { - page = sio->bvecs[p].bv_page; - set_page_dirty(page); - ClearPageReclaim(page); - } - } - - for (p = 0; p < sio->nr_bvecs; p++) - end_page_writeback(sio->bvecs[p].bv_page); + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1]; + struct folio *prev_folio = bvec_folio(last_bv); + size_t prev_folio_size = folio_size(prev_folio); - mempool_free(sio, sio_pool); + if (ctx->sis != sis) + return false; + return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw); } -static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) +static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw) { - struct swap_iocb *sio = swap_plug ? *swap_plug : NULL; struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct file *swap_file = sis->swap_file; - loff_t pos = swap_dev_pos(folio->swap); - - count_swpout_vm_event(folio); - folio_start_writeback(folio); - folio_unlock(folio); - if (sio) { - if (sio->iocb.ki_filp != swap_file || - sio->iocb.ki_pos + sio->len != pos) { - swap_write_unplug(sio); - sio = NULL; - } + struct swap_iocb *sio = ctx->sio; + + if (sio && !swap_can_merge(ctx, folio, rw)) { + if (rw == WRITE) + swap_write_submit(ctx); + else + swap_read_submit(ctx); + sio = ctx->sio; } + if (!sio) { - sio = mempool_alloc(sio_pool, GFP_NOIO); - init_sync_kiocb(&sio->iocb, swap_file); - sio->iocb.ki_complete = sio_write_complete; - sio->iocb.ki_pos = pos; + ctx->sis = sis; + ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO); sio->nr_bvecs = 0; sio->len = 0; } bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0); sio->len += folio_size(folio); - sio->nr_bvecs += 1; - if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !swap_plug) { - swap_write_unplug(sio); - sio = NULL; + if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs)) { + if (rw == WRITE) + swap_write_submit(ctx); + else + swap_read_submit(ctx); } - if (swap_plug) - *swap_plug = sio; -} - -static void swap_writepage_bdev_sync(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio_vec bv; - struct bio bio; - - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_WRITE | REQ_SWAP); - bio.bi_iter.bi_sector = swap_folio_sector(folio); - bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); - - bio_associate_blkg_from_page(&bio, folio); - count_swpout_vm_event(folio); - - folio_start_writeback(folio); - folio_unlock(folio); - - submit_bio_wait(&bio); - __end_swap_bio_write(&bio); } -static void swap_writepage_bdev_async(struct folio *folio, - struct swap_info_struct *sis) +void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) { - struct bio *bio; + VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); - bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE | REQ_SWAP, GFP_NOIO); - bio->bi_iter.bi_sector = swap_folio_sector(folio); - bio->bi_end_io = end_swap_bio_write; - bio_add_folio_nofail(bio, folio, folio_size(folio), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (unlikely(folio_test_pmd_mappable(folio))) { + count_memcg_folio_events(folio, THP_SWPOUT, 1); + count_vm_event(THP_SWPOUT); + } +#endif + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT); + count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio)); + count_vm_events(PSWPOUT, folio_nr_pages(folio)); - bio_associate_blkg_from_page(bio, folio); - count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); -} - -void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) -{ - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - - VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); - /* - * ->flags can be updated non-atomically, - * but that will never affect SWP_FS_OPS, so the data_race - * is safe. - */ - if (data_race(sis->flags & SWP_FS_OPS)) - swap_writepage_fs(folio, swap_plug); - /* - * ->flags can be updated non-atomically, - * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race - * is safe. - */ - else if (data_race(sis->flags & SWP_SYNCHRONOUS_IO)) - swap_writepage_bdev_sync(folio, sis); - else - swap_writepage_bdev_async(folio, sis); -} - -void swap_write_unplug(struct swap_iocb *sio) -{ - struct iov_iter from; - struct address_space *mapping = sio->iocb.ki_filp->f_mapping; - int ret; - - iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw(&sio->iocb, &from); - if (ret != -EIOCBQUEUED) - sio_write_complete(&sio->iocb, ret); -} - -static void sio_read_complete(struct kiocb *iocb, long ret) -{ - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); - int p; - - if (ret == sio->len) { - for (p = 0; p < sio->nr_bvecs; p++) { - struct folio *folio = bvec_folio(&sio->bvecs[p]); - - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - folio_mark_uptodate(folio); - folio_unlock(folio); - } - count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); - } else { - for (p = 0; p < sio->nr_bvecs; p++) { - struct folio *folio = bvec_folio(&sio->bvecs[p]); - - folio_unlock(folio); - } - pr_alert_ratelimited("Read-error on swap-device\n"); - } - mempool_free(sio, sio_pool); + swap_add_folio(ctx, folio, WRITE); } /* @@ -587,78 +448,7 @@ static bool swap_read_folio_zeromap(struct folio *folio) return true; } -static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug) -{ - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_iocb *sio = NULL; - loff_t pos = swap_dev_pos(folio->swap); - - if (plug) - sio = *plug; - if (sio) { - if (sio->iocb.ki_filp != sis->swap_file || - sio->iocb.ki_pos + sio->len != pos) { - swap_read_unplug(sio); - sio = NULL; - } - } - if (!sio) { - sio = mempool_alloc(sio_pool, GFP_KERNEL); - init_sync_kiocb(&sio->iocb, sis->swap_file); - sio->iocb.ki_pos = pos; - sio->iocb.ki_complete = sio_read_complete; - sio->nr_bvecs = 0; - sio->len = 0; - } - bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0); - sio->len += folio_size(folio); - sio->nr_bvecs += 1; - if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !plug) { - swap_read_unplug(sio); - sio = NULL; - } - if (plug) - *plug = sio; -} - -static void swap_read_folio_bdev_sync(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio_vec bv; - struct bio bio; - - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ); - bio.bi_iter.bi_sector = swap_folio_sector(folio); - bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); - /* - * Keep this task valid during swap readpage because the oom killer may - * attempt to access it in the page fault retry time check. - */ - get_task_struct(current); - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); - __end_swap_bio_read(&bio); - put_task_struct(current); -} - -static void swap_read_folio_bdev_async(struct folio *folio, - struct swap_info_struct *sis) -{ - struct bio *bio; - - bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL); - bio->bi_iter.bi_sector = swap_folio_sector(folio); - bio->bi_end_io = end_swap_bio_read; - bio_add_folio_nofail(bio, folio, folio_size(folio), 0); - count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); - count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); - count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); -} - -void swap_read_folio(struct folio *folio, struct swap_iocb **plug) +void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) { struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO; @@ -691,14 +481,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug) /* We have to read from slower devices. Increase zswap protection. */ zswap_folio_swapin(folio); - - if (data_race(sis->flags & SWP_FS_OPS)) { - swap_read_folio_fs(folio, plug); - } else if (synchronous) { - swap_read_folio_bdev_sync(folio, sis); - } else { - swap_read_folio_bdev_async(folio, sis); - } + swap_add_folio(ctx, folio, READ); finish: if (workingset) { @@ -708,14 +491,231 @@ finish: delayacct_swapin_end(); } -void __swap_read_unplug(struct swap_iocb *sio) +static void swap_write_end(struct swap_iocb *sio, bool failed) +{ + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + struct page *page = sio->bvecs[p].bv_page; + + if (failed) { + set_page_dirty(page); + ClearPageReclaim(page); + } + end_page_writeback(page); + } + mempool_free(sio, sio_pool); +} + +static void swap_fs_write_complete(struct kiocb *iocb, long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); + bool failed = ret != sio->len; + + if (failed) { + struct page *page = sio->bvecs[0].bv_page; + + /* + * In the case of swap-over-nfs, this can be a temporary failure + * if the system has limited memory for allocating transmit + * buffers. Mark the page dirty and avoid + * folio_rotate_reclaimable but rate-limit the messages. + */ + pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", + ret, swap_dev_pos(page_swap_entry(page))); + } + + swap_write_end(sio, failed); +} + +static void end_swap_bio_write(struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); + bool failed = !!bio->bi_status; + + if (failed) + pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); + bio_uninit(bio); + swap_write_end(sio, failed); +} + +static void swap_read_end(struct swap_iocb *sio, bool failed) +{ + int p; + + for (p = 0; p < sio->nr_bvecs; p++) { + struct folio *folio = bvec_folio(&sio->bvecs[p]); + + if (!failed) { + count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); + count_memcg_folio_events(folio, PSWPIN, + folio_nr_pages(folio)); + folio_mark_uptodate(folio); + } + folio_unlock(folio); + } + + if (!failed) + count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); + + mempool_free(sio, sio_pool); +} + +static void swap_fs_read_complete(struct kiocb *iocb, long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); + bool failed = ret != sio->len; + + if (failed) + pr_alert_ratelimited("Read-error on swap-device\n"); + swap_read_end(sio, failed); +} + +static void swap_bio_read_end_io(struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); + bool failed = !!bio->bi_status; + + if (failed) + pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); + bio_uninit(bio); + swap_read_end(sio, failed); +} + +static void swap_bdev_submit_write(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct bio *bio = &sio->bio; + + bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs), + REQ_OP_WRITE | REQ_SWAP); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); + bio_associate_blkg_from_page(bio, bio_first_folio_all(bio)); + + if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { + submit_bio_wait(bio); + end_swap_bio_write(bio); + } else { + bio->bi_end_io = end_swap_bio_write; + submit_bio(bio); + } +} + +static void swap_bdev_submit_read(struct swap_io_ctx *ctx) +{ + struct swap_iocb *sio = ctx->sio; + struct bio *bio = &sio->bio; + + bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs), + REQ_OP_READ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio)); + + if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) { + /* + * Keep this task valid during swap readpage because the oom + * killer may attempt to access it in the page fault retry + * time check. + */ + get_task_struct(current); + submit_bio_wait(bio); + swap_bio_read_end_io(bio); + put_task_struct(current); + } else { + bio->bi_end_io = swap_bio_read_end_io; + submit_bio(bio); + } +} + +static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw) { - struct iov_iter from; - struct address_space *mapping = sio->iocb.ki_filp->f_mapping; + if (swap_folio_sector(folio) != + swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT)) + return false; + if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio)) + return false; + return true; +} + +const struct swap_ops swap_bdev_ops = { + .submit_write = swap_bdev_submit_write, + .submit_read = swap_bdev_submit_read, + .can_merge = swap_bdev_can_merge, +}; + +static void swap_fs_submit(struct swap_io_ctx *ctx, int rw) +{ + struct swap_iocb *sio = ctx->sio; + struct iov_iter iter; int ret; - iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw(&sio->iocb, &from); + init_sync_kiocb(&sio->iocb, ctx->sis->swap_file); + sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap); + if (rw == WRITE) + sio->iocb.ki_complete = swap_fs_write_complete; + else + sio->iocb.ki_complete = swap_fs_read_complete; + + iov_iter_bvec(&iter, rw == WRITE ? ITER_SOURCE : ITER_DEST, + sio->bvecs, sio->nr_bvecs, sio->len); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw(&sio->iocb, &iter); if (ret != -EIOCBQUEUED) - sio_read_complete(&sio->iocb, ret); + sio->iocb.ki_complete(&sio->iocb, ret); +} + +static void swap_fs_submit_write(struct swap_io_ctx *ctx) +{ + swap_fs_submit(ctx, WRITE); +} + +static void swap_fs_submit_read(struct swap_io_ctx *ctx) +{ + swap_fs_submit(ctx, READ); +} + +static bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw) +{ + return swap_dev_pos(folio->swap) == + swap_dev_pos(prev_folio->swap) + prev_folio_size; +} + +static const struct swap_ops swap_fs_ops = { + .flags = SWAP_OPS_F_REQUIRE_NOFS, + .submit_write = swap_fs_submit_write, + .submit_read = swap_fs_submit_read, + .can_merge = swap_fs_can_merge, +}; + +int swap_fs_activate(struct swap_info_struct *sis) +{ + sis->ops = &swap_fs_ops; + return add_swap_extent(sis, 0, sis->max, 0); +} +EXPORT_SYMBOL_GPL(swap_fs_activate); + +void swap_write_submit(struct swap_io_ctx *ctx) +{ + if (!ctx->sio) + return; + count_vm_events(NRSWPOUT, 1); + ctx->sis->ops->submit_write(ctx); + ctx->sio = NULL; + ctx->sis = NULL; +} + +void swap_read_submit(struct swap_io_ctx *ctx) +{ + if (!ctx->sio) + return; + count_vm_events(NRSWPIN, 1); + ctx->sis->ops->submit_read(ctx); + ctx->sio = NULL; + ctx->sis = NULL; } diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 32ce8a7d9df3..e5dfc7bf4944 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -11,6 +11,7 @@ #include <linux/page_owner.h> #include <linux/migrate.h> #include "internal.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include <trace/events/page_isolation.h> diff --git a/mm/page_owner.c b/mm/page_owner.c index 2dddcb6510aa..fbbda7ba914b 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -13,7 +13,7 @@ #include <linux/memcontrol.h> #include <linux/sched/clock.h> -#include "internal.h" +#include "page_alloc.h" /* * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) @@ -54,6 +54,24 @@ struct stack_print_ctx { u8 flags; }; +enum page_owner_print_mode { + PAGE_OWNER_PRINT_STACK, + PAGE_OWNER_PRINT_HANDLE, + PAGE_OWNER_PRINT_STACK_HANDLE, +}; + +static const char * const page_owner_print_mode_strings[] = { + [PAGE_OWNER_PRINT_STACK] = "stack", + [PAGE_OWNER_PRINT_HANDLE] = "handle", + [PAGE_OWNER_PRINT_STACK_HANDLE] = "stack_handle", +}; + +struct page_owner_filter_state { + enum page_owner_print_mode print_mode; + nodemask_t nid_filter; + bool nid_filter_enabled; +}; + static bool page_owner_enabled __initdata; DEFINE_STATIC_KEY_FALSE(page_owner_inited); @@ -339,13 +357,13 @@ noinline void __set_page_owner(struct page *page, unsigned short order, depot_stack_handle_t handle; handle = save_stack(gfp_mask); - __update_page_owner_handle(page, handle, order, gfp_mask, -1, + __update_page_owner_handle(page, handle, order, gfp_mask, MR_NEVER, ts_nsec, current->pid, current->tgid, current->comm); inc_stack_record_count(handle, gfp_mask, 1 << order); } -void __folio_set_owner_migrate_reason(struct folio *folio, int reason) +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { struct page_ext *page_ext = page_ext_get(&folio->page); struct page_owner *page_owner; @@ -422,6 +440,39 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old) rcu_read_unlock(); } +/* + * Check if a page is a buddy page and advance @pfn past the entire buddy block. + * This safely reads the buddy order without the zone lock, which may cause us + * to skip less than the full buddy block, but that is acceptable for page owner + * iteration purposes. + * + * The lockless read of buddy_order_unsafe() can also return a garbage order if + * the page is concurrently allocated and PageBuddy is cleared between the check + * and the read. Clamp the advance at the next MAX_ORDER_NR_PAGES boundary so + * that a bogus order cannot carry @pfn into an unvalidated memory section, + * which would break callers that rely on boundary-aligned pfn_valid() checks. + * + * Return: true if the page was skipped (caller should continue its loop), + * false if the page is not a buddy page and should be processed normally. + */ +static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) +{ + unsigned long order; + + if (!PageBuddy(page)) + return false; + + order = buddy_order_unsafe(page); + if (order <= MAX_PAGE_ORDER) { + unsigned long new_pfn = *pfn + (1UL << order); + unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES); + + *pfn = min(new_pfn, boundary) - 1; + } + + return true; +} + void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { @@ -461,14 +512,8 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, if (page_zone(page) != zone) continue; - if (PageBuddy(page)) { - unsigned long freepage_order; - - freepage_order = buddy_order_unsafe(page); - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -505,14 +550,15 @@ ext_put_continue: seq_putc(m, '\n'); } +#ifdef CONFIG_MEMCG /* * Looking for memcg information and print it out */ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, struct page *page) { -#ifdef CONFIG_MEMCG unsigned long memcg_data; + struct obj_cgroup *objcg; struct mem_cgroup *memcg; bool online; char name[80]; @@ -522,11 +568,14 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, if (!memcg_data || PageTail(page)) goto out_unlock; - if (memcg_data & MEMCG_DATA_OBJEXTS) + if (memcg_data & MEMCG_DATA_OBJEXTS) { ret += scnprintf(kbuf + ret, count - ret, "Slab cache page\n"); + goto out_unlock; + } - memcg = page_memcg_check(page); + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); + memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; if (!memcg) goto out_unlock; @@ -534,29 +583,39 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, cgroup_name(memcg->css.cgroup, name, sizeof(name)); ret += scnprintf(kbuf + ret, count - ret, "Charged %sto %smemcg %s\n", - PageMemcgKmem(page) ? "(via objcg) " : "", + (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "", online ? "" : "offline ", name); out_unlock: rcu_read_unlock(); -#endif /* CONFIG_MEMCG */ return ret; } +#else +static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, + struct page *page) +{ + return ret; +} +#endif static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, struct page *page, struct page_owner *page_owner, - depot_stack_handle_t handle) + depot_stack_handle_t handle, + struct page_owner_filter_state *state) { int ret, pageblock_mt, page_mt; char *kbuf; + enum page_owner_print_mode print_mode; count = min_t(size_t, count, PAGE_SIZE); kbuf = kmalloc(count, GFP_KERNEL); if (!kbuf) return -ENOMEM; + print_mode = state->print_mode; + ret = scnprintf(kbuf, count, "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n", page_owner->order, page_owner->gfp_mask, @@ -575,11 +634,20 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn, migratetype_names[pageblock_mt], &page->flags.f); - ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); - if (ret >= count) - goto err; + if (print_mode != PAGE_OWNER_PRINT_HANDLE) { + ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); + if (ret >= count) + goto err; + } + + if (print_mode != PAGE_OWNER_PRINT_STACK) { + ret += scnprintf(kbuf + ret, count - ret, "handle: %u\n", + handle); + if (ret >= count) + goto err; + } - if (page_owner->last_migrate_reason != -1) { + if (page_owner->last_migrate_reason != MR_NEVER) { ret += scnprintf(kbuf + ret, count - ret, "Page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); @@ -630,10 +698,10 @@ void __dump_page_owner(const struct page *page) else pr_alert("page_owner tracks the page as freed\n"); - pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", + pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu\n", page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, page_owner->pid, page_owner->tgid, page_owner->comm, - page_owner->ts_nsec, page_owner->free_ts_nsec); + page_owner->ts_nsec); handle = READ_ONCE(page_owner->handle); if (!handle) @@ -645,12 +713,13 @@ void __dump_page_owner(const struct page *page) if (!handle) { pr_alert("page_owner free stack trace missing\n"); } else { - pr_alert("page last free pid %d tgid %d stack trace:\n", - page_owner->free_pid, page_owner->free_tgid); + pr_alert("page last free pid %d tgid %d ts %llu stack trace:\n", + page_owner->free_pid, page_owner->free_tgid, + page_owner->free_ts_nsec); stack_depot_print(handle); } - if (page_owner->last_migrate_reason != -1) + if (page_owner->last_migrate_reason != MR_NEVER) pr_alert("page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); page_ext_put(page_ext); @@ -664,6 +733,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) struct page_ext *page_ext; struct page_owner *page_owner; depot_stack_handle_t handle; + struct page_owner_filter_state *state = file->private_data; if (!static_branch_unlikely(&page_owner_inited)) return -EINVAL; @@ -697,13 +767,8 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) } page = pfn_to_page(pfn); - if (PageBuddy(page)) { - unsigned long freepage_order = buddy_order_unsafe(page); - - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } page_ext = page_ext_get(page); if (unlikely(!page_ext)) @@ -740,15 +805,31 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) if (!handle) goto ext_put_continue; + if (state->nid_filter_enabled) { + int nid; + memdesc_flags_t page_flags = READ_ONCE(page->flags); + + /* + * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid + * VM_BUG_ON when accessing poisoned pages. + */ + if (page_flags.f == PAGE_POISON_PATTERN) + goto ext_put_continue; + nid = memdesc_nid(&page_flags); + if (!node_isset(nid, state->nid_filter)) + goto ext_put_continue; + } + /* Record the next PFN to read in the file offset */ *ppos = pfn + 1; page_owner_tmp = *page_owner; page_ext_put(page_ext); return print_page_owner(buf, count, pfn, page, - &page_owner_tmp, handle); + &page_owner_tmp, handle, state); ext_put_continue: page_ext_put(page_ext); + cond_resched(); } return 0; @@ -798,20 +879,8 @@ static void init_pages_in_zone(struct zone *zone) if (page_zone(page) != zone) continue; - /* - * To avoid having to grab zone->lock, be a little - * careful when reading buddy page order. The only - * danger is that we skip too much and potentially miss - * some early allocated pages, which is better than - * heavy lock contention. - */ - if (PageBuddy(page)) { - unsigned long order = buddy_order_unsafe(page); - - if (order > 0 && order <= MAX_PAGE_ORDER) - pfn += (1UL << order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -826,7 +895,7 @@ static void init_pages_in_zone(struct zone *zone) /* Found early allocated page */ __update_page_owner_handle(page, early_handle, 0, 0, - -1, local_clock(), current->pid, + MR_NEVER, local_clock(), current->pid, current->tgid, current->comm); count++; ext_put_continue: @@ -847,7 +916,113 @@ static void init_early_allocated_pages(void) init_pages_in_zone(zone); } +static int page_owner_open(struct inode *inode, struct file *file) +{ + struct page_owner_filter_state *state; + + state = kzalloc_obj(*state); + if (!state) + return -ENOMEM; + + state->print_mode = PAGE_OWNER_PRINT_STACK; + nodes_clear(state->nid_filter); + state->nid_filter_enabled = false; + file->private_data = state; + return 0; +} + +static int page_owner_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t page_owner_write(struct file *file, + const char __user *buf, + size_t count, loff_t *ppos) +{ + char *kbuf; + char *orig; + char *token; + int ret; + struct page_owner_filter_state *state = file->private_data; + enum page_owner_print_mode new_print_mode; + nodemask_t new_nid_filter; + bool new_nid_filter_enabled; + + /* + * Maximum input length for filter commands: + * - 32: print_mode command max length is 17 ("mode=stack_handle") + * with sufficient buffer + * - 6 * MAX_NUMNODES: worst case for nid list + * Worst case per node: ",NNNNN" (comma + 5-digit node number) = 6 bytes + */ + if (count > 32 + 6 * MAX_NUMNODES) + return -EINVAL; + + kbuf = memdup_user_nul(buf, count); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + orig = kbuf; + + new_print_mode = state->print_mode; + new_nid_filter = state->nid_filter; + new_nid_filter_enabled = state->nid_filter_enabled; + + while ((token = strsep(&kbuf, " \t\n")) != NULL) { + if (*token == '\0') + continue; + + if (!strncmp(token, "mode=", 5)) { + ret = sysfs_match_string(page_owner_print_mode_strings, + token + 5); + if (ret < 0) + goto out_free; + new_print_mode = ret; + } else if (!strncmp(token, "nid=", 4)) { + ret = nodelist_parse(token + 4, new_nid_filter); + if (ret < 0) + goto out_free; + + if (nodes_empty(new_nid_filter)) { + ret = -EINVAL; + goto out_free; + } + + /* + * We want to filter memory allocations by numa nodes, so make sure + * that the specified nodes have memory. + */ + if (!nodes_subset(new_nid_filter, node_states[N_MEMORY])) { + ret = -EINVAL; + goto out_free; + } + + new_nid_filter_enabled = true; + } else { + ret = -EINVAL; + goto out_free; + } + } + + /* Commit all filter changes */ + state->print_mode = new_print_mode; + state->nid_filter = new_nid_filter; + state->nid_filter_enabled = new_nid_filter_enabled; + + ret = count; + +out_free: + kfree(orig); + return ret; +} + static const struct file_operations page_owner_fops = { + .owner = THIS_MODULE, + .open = page_owner_open, + .release = page_owner_release, + .write = page_owner_write, .read = read_page_owner, .llseek = lseek_page_owner, }; @@ -887,7 +1062,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) return stack; } -static unsigned long page_owner_pages_threshold; +static unsigned long pages_threshold; static int stack_print(struct seq_file *m, void *v) { @@ -904,7 +1079,7 @@ static int stack_print(struct seq_file *m, void *v) nr_base_pages = refcount_read(&stack_record->count) - 1; if (ctx->flags & STACK_PRINT_FLAG_PAGES && - (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold)) + (nr_base_pages < 1 || nr_base_pages < pages_threshold)) return 0; if (ctx->flags & STACK_PRINT_FLAG_STACK) { @@ -926,16 +1101,16 @@ static void stack_stop(struct seq_file *m, void *v) { } -static const struct seq_operations page_owner_stack_op = { +static const struct seq_operations stack_op = { .start = stack_start, .next = stack_next, .stop = stack_stop, .show = stack_print }; -static int page_owner_stack_open(struct inode *inode, struct file *file) +static int stack_open(struct inode *inode, struct file *file) { - int ret = seq_open_private(file, &page_owner_stack_op, + int ret = seq_open_private(file, &stack_op, sizeof(struct stack_print_ctx)); if (!ret) { @@ -948,28 +1123,26 @@ static int page_owner_stack_open(struct inode *inode, struct file *file) return ret; } -static const struct file_operations page_owner_stack_fops = { - .open = page_owner_stack_open, +static const struct file_operations stack_fops = { + .open = stack_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_private, }; -static int page_owner_threshold_get(void *data, u64 *val) +static int threshold_get(void *data, u64 *val) { - *val = READ_ONCE(page_owner_pages_threshold); + *val = READ_ONCE(pages_threshold); return 0; } -static int page_owner_threshold_set(void *data, u64 val) +static int threshold_set(void *data, u64 val) { - WRITE_ONCE(page_owner_pages_threshold, val); + WRITE_ONCE(pages_threshold, val); return 0; } -DEFINE_SIMPLE_ATTRIBUTE(page_owner_threshold_fops, &page_owner_threshold_get, - &page_owner_threshold_set, "%llu"); - +DEFINE_SIMPLE_ATTRIBUTE(threshold_fops, &threshold_get, &threshold_set, "%llu\n"); static int __init pageowner_init(void) { @@ -980,22 +1153,22 @@ static int __init pageowner_init(void) return 0; } - debugfs_create_file("page_owner", 0400, NULL, NULL, &page_owner_fops); + debugfs_create_file("page_owner", 0600, NULL, NULL, &page_owner_fops); dir = debugfs_create_dir("page_owner_stacks", NULL); debugfs_create_file("show_stacks", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_HANDLE | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_stacks_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_HANDLE), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("count_threshold", 0600, dir, NULL, - &page_owner_threshold_fops); + &threshold_fops); return 0; } late_initcall(pageowner_init) diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 3e30731b940e..1cce8729696e 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -8,6 +8,7 @@ #include <linux/delay.h> #include <linux/scatterlist.h> +#include "page_alloc.h" #include "page_reporting.h" #include "internal.h" diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a11410..6ffc536359cd 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -199,8 +199,8 @@ static inline bool softleaf_cached_writable(softleaf_t entry) static void page_table_check_pte_flags(pte_t pte) { if (pte_present(pte)) { - WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte)); - } else if (pte_swp_uffd_wp(pte)) { + WARN_ON_ONCE(pte_uffd(pte) && pte_write(pte)); + } else if (pte_swp_uffd(pte)) { const softleaf_t entry = softleaf_from_pte(pte); WARN_ON_ONCE(softleaf_cached_writable(entry)); @@ -227,9 +227,9 @@ EXPORT_SYMBOL(__page_table_check_ptes_set); static inline void page_table_check_pmd_flags(pmd_t pmd) { if (pmd_present(pmd)) { - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) WARN_ON_ONCE(pmd_write(pmd)); - } else if (pmd_swp_uffd_wp(pmd)) { + } else if (pmd_swp_uffd(pmd)) { const softleaf_t entry = softleaf_from_pmd(pmd); WARN_ON_ONCE(softleaf_cached_writable(entry)); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index bac2eb5de63d..4e964545e5e8 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -107,7 +107,13 @@ again: static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) { unsigned long pfn; - pte_t ptent = ptep_get(pvmw->pte); + pte_t ptent; + + if (is_vm_hugetlb_page(pvmw->vma)) + ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address, + pvmw->pte); + else + ptent = ptep_get(pvmw->pte); if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -350,6 +356,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + const pgoff_t pgoff = page_pgoff(folio, page); struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -357,7 +364,10 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + if (folio_test_anon(folio)) + pvmw.address = vma_anon_address(vma, pgoff, 1); + else + pvmw.address = vma_filebacked_address(vma, pgoff, 1); if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a255..ed4860c01936 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -816,10 +816,10 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); - vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, + mapping_rmap_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ - vba = vma->vm_pgoff; + vba = vma_start_pgoff(vma); vea = vba + vma_pages(vma); cba = first_index; cba = max(cba, vba); diff --git a/mm/percpu-km.c b/mm/percpu-km.c index dc096b5a6ce4..65fd5580e447 100644 --- a/mm/percpu-km.c +++ b/mm/percpu-km.c @@ -94,8 +94,15 @@ static void pcpu_destroy_chunk(struct pcpu_chunk *chunk) pcpu_stats_chunk_dealloc(); trace_percpu_destroy_chunk(chunk->base_addr); - if (chunk->data) + if (chunk->data) { + struct page *pages = (struct page *)chunk->data; + int i; + + /* clear chunk info from each page before free them */ + for (i = 0; i < nr_pages; i++) + pcpu_set_page_chunk(pages + i, NULL); __free_pages(chunk->data, order_base_2(nr_pages)); + } pcpu_free_chunk(chunk); } diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 4f5937090590..509d8901835c 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -9,6 +9,7 @@ * This is the default chunk allocator. */ #include "internal.h" +#include "vmalloc.h" static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, unsigned int cpu, int page_idx) @@ -21,6 +22,8 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, /** * pcpu_get_pages - get temp pages array + * @gfp: allocation flags passed to the underlying allocator, 0 to only + * return the cached array * * Returns pointer to array of pointers to struct page which can be indexed * with pcpu_page_idx(). Note that there is only one array and accesses @@ -29,18 +32,23 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, * RETURNS: * Pointer to temp pages array on success. */ -static struct page **pcpu_get_pages(void) +static struct page **pcpu_get_pages(gfp_t gfp) { static struct page **pages; size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]); lockdep_assert_held(&pcpu_alloc_mutex); - if (!pages) - pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL); + if (!pages && gfp) + pages = pcpu_mem_zalloc(pages_size, gfp); return pages; } +static struct page **pcpu_get_pages_cached(void) +{ + return pcpu_get_pages(0); +} + /** * pcpu_free_pages - free pages which were allocated for @chunk * @chunk: chunk pages were allocated for @@ -191,10 +199,22 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk, } static int __pcpu_map_pages(unsigned long addr, struct page **pages, - int nr_pages) + int nr_pages, gfp_t gfp) { - return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), - PAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL); + unsigned int flags; + int ret; + + /* + * The vmalloc page table allocation path does not pass @gfp down + * explicitly. Apply the corresponding memalloc scope so implicit + * page table allocations preserve NOFS/NOIO constraints. + */ + flags = memalloc_apply_gfp_scope(gfp); + ret = vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), + PAGE_KERNEL, pages, PAGE_SHIFT, gfp); + memalloc_restore_scope(flags); + + return ret; } /** @@ -203,6 +223,7 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * @pages: pages array containing pages to be mapped * @page_start: page index of the first page to map * @page_end: page index of the last page to map + 1 + * @gfp: allocation flags passed to the underlying allocator * * For each cpu, map pages [@page_start,@page_end) into @chunk. The * caller is responsible for calling pcpu_post_map_flush() after all @@ -211,8 +232,8 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * This function is responsible for setting up whatever is necessary for * reverse lookup (addr -> chunk). */ -static int pcpu_map_pages(struct pcpu_chunk *chunk, - struct page **pages, int page_start, int page_end) +static int pcpu_map_pages(struct pcpu_chunk *chunk, struct page **pages, + int page_start, int page_end, gfp_t gfp) { unsigned int cpu, tcpu; int i, err; @@ -220,7 +241,7 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk, for_each_possible_cpu(cpu) { err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), &pages[pcpu_page_idx(cpu, page_start)], - page_end - page_start); + page_end - page_start, gfp); if (err < 0) goto err; @@ -271,21 +292,21 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk, * @chunk. * * CONTEXT: - * pcpu_alloc_mutex, does GFP_KERNEL allocation. + * pcpu_alloc_mutex, does @gfp allocation. */ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int page_start, int page_end, gfp_t gfp) { struct page **pages; - pages = pcpu_get_pages(); + pages = pcpu_get_pages(gfp); if (!pages) return -ENOMEM; if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp)) return -ENOMEM; - if (pcpu_map_pages(chunk, pages, page_start, page_end)) { + if (pcpu_map_pages(chunk, pages, page_start, page_end, gfp)) { pcpu_free_pages(chunk, pages, page_start, page_end); return -ENOMEM; } @@ -319,7 +340,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, * successful population attempt so the temp pages array must * be available now. */ - pages = pcpu_get_pages(); + pages = pcpu_get_pages_cached(); BUG_ON(!pages); /* unmap and free */ @@ -340,7 +361,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) return NULL; vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes, - pcpu_nr_groups, pcpu_atom_size); + pcpu_nr_groups, pcpu_atom_size, gfp); if (!vms) { pcpu_free_chunk(chunk); return NULL; diff --git a/mm/percpu.c b/mm/percpu.c index b0676b8054ed..a802d72c116f 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1726,9 +1726,8 @@ static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t s * @gfp: allocation flags * * Allocate percpu area of @size bytes aligned at @align. If @gfp doesn't - * contain %GFP_KERNEL, the allocation is atomic. If @gfp has __GFP_NOWARN - * then no warning will be triggered on invalid or failed allocation - * requests. + * allow blocking, the allocation is atomic. If @gfp has __GFP_NOWARN then no + * warning will be triggered on invalid or failed allocation requests. * * RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. @@ -1749,8 +1748,17 @@ void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, size_t bits, bit_align; gfp = current_gfp_context(gfp); - /* whitelisted flags that can be passed to the backing allocators */ - pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + /* + * Allowlisted flags that can be passed to the backing allocators. + * Backing allocations under pcpu_alloc_mutex must not recurse into + * IO/FS reclaim. Otherwise a GFP_KERNEL caller holding the mutex can + * block on reclaim while a GFP_NOIO/NOFS caller holding an IO/FS lock + * waits for the same mutex. + * + * Do not pass __GFP_NOFAIL. A small percpu allocation may need many + * backing pages, making nofail reclaim too costly under NOIO/NOFS. + */ + pcpu_gfp = gfp & (GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); is_atomic = !gfpflags_allow_blocking(gfp); do_warn = !(gfp & __GFP_NOWARN); @@ -3256,7 +3264,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t /* pte already populated, the following shouldn't fail */ rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages], - unit_pages); + unit_pages, GFP_KERNEL); if (rc < 0) panic("failed to map percpu area, err=%d\n", rc); diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9..58e3c5f7d9ab 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -211,7 +211,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) if (likely(!vma->anon_vma)) { vma->anon_vma = anon_vma; anon_vma_chain_assign(vma, avc, anon_vma); - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_rmap_tree_insert(avc, anon_vma); anon_vma->num_active_vmas++; allocated = NULL; avc = NULL; @@ -354,7 +354,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src, list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_rmap_tree_insert(avc, anon_vma); if (operation == VMA_OP_FORK) maybe_reuse_anon_vma(dst, anon_vma); } @@ -434,7 +434,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) anon_vma_chain_assign(vma, avc, anon_vma); /* Now let rmap see it. */ anon_vma_lock_write(anon_vma); - anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); + anon_rmap_tree_insert(avc, anon_vma); anon_vma->parent->num_children++; anon_vma_unlock_write(anon_vma); @@ -499,7 +499,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); + anon_rmap_tree_remove(avc, anon_vma); /* * Leave empty anon_vmas on the list - we'll need @@ -865,14 +865,15 @@ unsigned long page_address_in_vma(const struct folio *folio, if (!vma->anon_vma || !anon_vma || vma->anon_vma->root != anon_vma->root) return -EFAULT; + /* KSM folios don't reach here because of the !anon_vma check */ + return vma_anon_address(vma, page_pgoff(folio, page), 1); } else if (!vma->vm_file) { return -EFAULT; } else if (vma->vm_file->f_mapping != folio->mapping) { return -EFAULT; } - /* KSM folios don't reach here because of the !anon_vma check */ - return vma_address(vma, page_pgoff(folio, page), 1); + return vma_filebacked_address(vma, page_pgoff(folio, page), 1); } /* @@ -907,7 +908,7 @@ out: struct folio_referenced_arg { int mapcount; int referenced; - vm_flags_t vm_flags; + vma_flags_t vma_flags; struct mem_cgroup *memcg; }; @@ -926,7 +927,7 @@ static bool folio_referenced_one(struct folio *folio, address = pvmw.address; nr = 1; - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { ptes++; pra->mapcount--; @@ -947,7 +948,7 @@ static bool folio_referenced_one(struct folio *folio, /* Restore the mlock which got missed */ mlock_vma_folio(folio, vma); page_vma_mapped_walk_done(&pvmw); - pra->vm_flags |= VM_LOCKED; + vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT); return false; /* To break the loop */ } @@ -1015,8 +1016,11 @@ static bool folio_referenced_one(struct folio *folio, referenced++; if (referenced) { + vma_flags_t vma_flags = vma->flags; + pra->referenced++; - pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; + vma_flags_clear(&vma_flags, VMA_LOCKED_BIT); + vma_flags_set_mask(&pra->vma_flags, vma_flags); } if (!pra->mapcount) @@ -1054,7 +1058,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) * @folio: The folio to test. * @is_locked: Caller holds lock on the folio. * @memcg: target memory cgroup - * @vm_flags: A combination of all the vma->vm_flags which referenced the folio. + * @vma_flags: A combination of all the vma->flags which referenced the folio. * * Quick test_and_clear_referenced for all mappings of a folio, * @@ -1062,7 +1066,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) * the function bailed out due to rmap lock contention. */ int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { bool we_locked = false; struct folio_referenced_arg pra = { @@ -1078,7 +1082,7 @@ int folio_referenced(struct folio *folio, int is_locked, }; VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio); - *vm_flags = 0; + vma_flags_clear_all(vma_flags); if (!pra.mapcount) return 0; @@ -1092,7 +1096,7 @@ int folio_referenced(struct folio *folio, int is_locked, } rmap_walk(folio, &rwc); - *vm_flags = pra.vm_flags; + vma_flags_set_mask(vma_flags, pra.vma_flags); if (we_locked) folio_unlock(folio); @@ -1239,6 +1243,7 @@ static bool mapping_wrprotect_range_one(struct folio *folio, .vma = vma, .address = address, .flags = PVMW_SYNC, + .is_anon_walk = false, }; state->cleaned += page_vma_mkclean_one(&pvmw); @@ -1316,12 +1321,13 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, .pgoff = pgoff, .vma = vma, .flags = PVMW_SYNC, + .is_anon_walk = false, }; if (invalid_mkclean_vma(vma, NULL)) return 0; - pvmw.address = vma_address(vma, pgoff, nr_pages); + pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages); VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); return page_vma_mkclean_one(&pvmw); @@ -1482,7 +1488,7 @@ static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma, */ anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON; WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma); - folio->index = linear_page_index(vma, address); + folio->index = linear_virt_page_index(vma, address); } /** @@ -1509,8 +1515,8 @@ static void __page_check_anon_rmap(const struct folio *folio, */ VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root, folio); - VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address), - page); + VM_BUG_ON_PAGE(page_pgoff(folio, page) != + linear_virt_page_index(vma, address), page); } static __always_inline void __folio_add_anon_rmap(struct folio *folio, @@ -1965,7 +1971,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, if (pte_unused(pte)) return 1; - if (userfaultfd_wp(vma)) + if (userfaultfd_protected(vma)) return 1; /* @@ -1978,6 +1984,220 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +static bool try_to_unmap_hugetlb_one(struct folio *folio, + struct vm_area_struct *vma, unsigned long address, void *arg) +{ + DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + const unsigned long hsz = huge_page_size(hstate_vma(vma)); + const enum ttu_flags flags = (enum ttu_flags)(long)arg; + struct mm_struct *mm = vma->vm_mm; + struct mmu_notifier_range range; + bool ret = true; + pte_t pteval; + + /* + * The try_to_unmap() is only passed a hugetlb folio in the case + * where the hugetlb folio is poisoned. + */ + VM_WARN_ON_FOLIO(!folio_test_hwpoison(folio), folio); + VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON)); + + range.end = vma_address_end(&pvmw); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, + address, range.end); + adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); + mmu_notifier_invalidate_range_start(&range); + + /* There is only a single mapping in a VMA. */ + if (!page_vma_mapped_walk(&pvmw)) + goto range_end; + + VM_WARN_ON_ONCE(address != pvmw.address); + + pteval = huge_ptep_get(mm, address, pvmw.pte); + VM_WARN_ON_ONCE(!pte_present(pteval)); + VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio)); + + /* + * huge_pmd_unshare may unmap an entire PMD page. There is no way of + * knowing exactly which PMDs may be cached for this mm, so we must + * flush them all. start/end were already adjusted above to cover this + * range. + */ + flush_cache_range(vma, range.start, range.end); + + /* + * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode. + * Caller needs to explicitly do this outside rmap routines. + * + * We also must hold hugetlb vma_lock in write mode. Lock order dictates + * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and + * fail if unsuccessful. + */ + if (!folio_test_anon(folio)) { + struct mmu_gather tlb; + + VM_WARN_ON(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + ret = false; + goto walk_done; + } + + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); + /* + * The PMD table was unmapped, consequently unmapping + * the folio. + */ + goto walk_done; + } + hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); + } + pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); + if (huge_pte_dirty(pteval)) + folio_mark_dirty(folio); + + pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0))); + hugetlb_count_sub(folio_nr_pages(folio), mm); + set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz); + hugetlb_remove_rmap(folio); + folio_put_refs(folio, 1); + +walk_done: + page_vma_mapped_walk_done(&pvmw); +range_end: + mmu_notifier_invalidate_range_end(&range); + return ret; +} + +static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma, + struct folio *folio, unsigned long nr_pages) +{ + int ref_count, map_count; + + /* + * Synchronize with gup_pte_range(): + * - clear PTE; barrier; read refcount + * - inc refcount; barrier; read PTE + */ + smp_mb(); + + ref_count = folio_ref_count(folio); + map_count = folio_mapcount(folio); + + /* + * Order reads for page refcount and dirty flag + * (see comments in __remove_mapping()). + */ + smp_rmb(); + + if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { + /* + * redirtied either using the page table or a previously + * obtained GUP reference. + */ + folio_set_swapbacked(folio); + return false; + } + + /* + * Additional references could be due to GUP or speculative lookups. + * GUP users must mark the folio dirty if there was a modification. + * This folio cannot be reclaimed right now either way, so act just + * like nothing happened. We'll come back here later and detect if the + * folio was dirtied when the additional reference is gone. + */ + if (ref_count != 1 + map_count) + return false; + + add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages); + return true; +} + +static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte, + bool anon_exclusive) +{ + pte_t swp_pte = swp_entry_to_pte(entry); + + if (anon_exclusive) + swp_pte = pte_swp_mkexclusive(swp_pte); + + if (likely(pte_present(old_pte))) { + if (pte_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } else { + /* Device-exclusive entry */ + if (pte_swp_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_swp_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } + + return swp_pte; +} + +static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, + struct folio *folio, struct page *page, unsigned long address, + pte_t *ptep, pte_t pteval) +{ + const bool anon_exclusive = folio_test_anon(folio) && + PageAnonExclusive(page); + swp_entry_t entry = page_swap_entry(page); + struct mm_struct *mm = vma->vm_mm; + + if (folio_dup_swap(folio, page) < 0) + return false; + + /* + * arch_unmap_one() is expected to be a NOP on + * architectures where we could have PFN swap PTEs, + * so we'll not check/care. + */ + if (arch_unmap_one(mm, vma, address, pteval) < 0) { + folio_put_swap(folio, page); + return false; + } + + /* See folio_try_share_anon_rmap(): clear PTE first. */ + if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) { + folio_put_swap(folio, page); + return false; + } + + mm_prepare_for_swap_entries(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, ptep, + swp_pte_prepare(entry, pteval, anon_exclusive)); + return true; +} + +static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio, + struct page *page, unsigned long address, pte_t *ptep, + pte_t pteval, unsigned long nr_pages) +{ + /* + * Store the swap location in the pte. + * See handle_pte_fault() ... + */ + if (WARN_ON_ONCE(folio_test_swapbacked(folio) != + folio_test_swapcache(folio))) + return false; + + if (!folio_test_swapbacked(folio)) + return ttu_anon_lazyfree_folio(vma, folio, nr_pages); + + /* nr_pages > 1 not supported yet */ + return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep, + pteval); +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -1986,14 +2206,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); - bool anon_exclusive, ret = true; + bool ret = true; pte_t pteval; - struct page *subpage; + struct page *page; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long nr_pages = 1, end_addr; unsigned long pfn; - unsigned long hsz = 0; int ptes = 0; /* @@ -2007,8 +2226,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* * For THP, we have to assume the worse case ie pmd for invalidation. - * For hugetlb, it could be much worse if we need to do pud - * invalidation in the case of pmd sharing. * * Note that the folio can not be freed in this function as call of * try_to_unmap() must hold a reference on the folio. @@ -2016,17 +2233,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, range.end = vma_address_end(&pvmw); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, address, range.end); - if (folio_test_hugetlb(folio)) { - /* - * If sharing is possible, start and end will be adjusted - * accordingly. - */ - adjust_range_if_pmd_sharing_possible(vma, &range.start, - &range.end); - - /* We need the huge page size for set_huge_pte_at() */ - hsz = huge_page_size(hstate_vma(vma)); - } mmu_notifier_invalidate_range_start(&range); while (page_vma_mapped_walk(&pvmw)) { @@ -2095,77 +2301,27 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) { + pteval = huge_ptep_get(mm, address, pvmw.pte); + } else { + pteval = ptep_get(pvmw.pte); + } if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); - VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); } - subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; - anon_exclusive = folio_test_anon(folio) && - PageAnonExclusive(subpage); - - if (folio_test_hugetlb(folio)) { - bool anon = folio_test_anon(folio); - - /* - * The try_to_unmap() is only passed a hugetlb page - * in the case where the hugetlb page is poisoned. - */ - VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage); - /* - * huge_pmd_unshare may unmap an entire PMD page. - * There is no way of knowing exactly which PMDs may - * be cached for this mm, so we must flush them all. - * start/end were already adjusted above to cover this - * range. - */ - flush_cache_range(vma, range.start, range.end); + page = folio_page(folio, pfn - folio_pfn(folio)); - /* - * To call huge_pmd_unshare, i_mmap_rwsem must be - * held in write mode. Caller needs to explicitly - * do this outside rmap routines. - * - * We also must hold hugetlb vma_lock in write mode. - * Lock order dictates acquiring vma_lock BEFORE - * i_mmap_rwsem. We can only try lock here and fail - * if unsuccessful. - */ - if (!anon) { - struct mmu_gather tlb; - - VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); - if (!hugetlb_vma_trylock_write(vma)) - goto walk_abort; - - tlb_gather_mmu_vma(&tlb, vma); - if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { - hugetlb_vma_unlock_write(vma); - huge_pmd_unshare_flush(&tlb, vma); - tlb_finish_mmu(&tlb); - /* - * The PMD table was unmapped, - * consequently unmapping the folio. - */ - goto walk_done; - } - hugetlb_vma_unlock_write(vma); - tlb_finish_mmu(&tlb); - } - pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); - if (pte_dirty(pteval)) - folio_mark_dirty(folio); - } else if (likely(pte_present(pteval))) { + if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); end_addr = address + nr_pages * PAGE_SIZE; flush_cache_range(vma, address, end_addr); @@ -2200,16 +2356,11 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) { - pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); - if (folio_test_hugetlb(folio)) { - hugetlb_count_sub(folio_nr_pages(folio), mm); - set_huge_pte_at(mm, address, pvmw.pte, pteval, - hsz); - } else { - dec_mm_counter(mm, mm_counter(folio)); - set_pte_at(mm, address, pvmw.pte, pteval); - } + /* With TTU_HWPOISON, we only expect small folios here. */ + if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { + pteval = swp_entry_to_pte(make_hwpoison_entry(page)); + dec_mm_counter(mm, mm_counter(folio)); + set_pte_at(mm, address, pvmw.pte, pteval); } else if (likely(pte_present(pteval)) && pte_unused(pteval) && !userfaultfd_armed(vma)) { /* @@ -2224,109 +2375,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ dec_mm_counter(mm, mm_counter(folio)); } else if (folio_test_anon(folio)) { - swp_entry_t entry = page_swap_entry(subpage); - pte_t swp_pte; - /* - * Store the swap location in the pte. - * See handle_pte_fault() ... - */ - if (unlikely(folio_test_swapbacked(folio) != - folio_test_swapcache(folio))) { - WARN_ON_ONCE(1); - goto walk_abort; - } - - /* MADV_FREE page check */ - if (!folio_test_swapbacked(folio)) { - int ref_count, map_count; - - /* - * Synchronize with gup_pte_range(): - * - clear PTE; barrier; read refcount - * - inc refcount; barrier; read PTE - */ - smp_mb(); - - ref_count = folio_ref_count(folio); - map_count = folio_mapcount(folio); - - /* - * Order reads for page refcount and dirty flag - * (see comments in __remove_mapping()). - */ - smp_rmb(); - - if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { - /* - * redirtied either using the page table or a previously - * obtained GUP reference. - */ - set_ptes(mm, address, pvmw.pte, pteval, nr_pages); - folio_set_swapbacked(folio); - goto walk_abort; - } else if (ref_count != 1 + map_count) { - /* - * Additional reference. Could be a GUP reference or any - * speculative reference. GUP users must mark the folio - * dirty if there was a modification. This folio cannot be - * reclaimed right now either way, so act just like nothing - * happened. - * We'll come back here later and detect if the folio was - * dirtied when the additional reference is gone. - */ - set_ptes(mm, address, pvmw.pte, pteval, nr_pages); - goto walk_abort; - } - add_mm_counter(mm, MM_ANONPAGES, -nr_pages); - goto discard; - } - - if (folio_dup_swap(folio, subpage) < 0) { - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - - /* - * arch_unmap_one() is expected to be a NOP on - * architectures where we could have PFN swap PTEs, - * so we'll not check/care. - */ - if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); + if (!ttu_anon_folio(vma, folio, page, address, + pvmw.pte, pteval, nr_pages)) { + set_ptes(mm, address, pvmw.pte, pteval, nr_pages); goto walk_abort; } - /* See folio_try_share_anon_rmap(): clear PTE first. */ - if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - if (list_empty(&mm->mmlist)) { - spin_lock(&mmlist_lock); - if (list_empty(&mm->mmlist)) - list_add(&mm->mmlist, &init_mm.mmlist); - spin_unlock(&mmlist_lock); - } - dec_mm_counter(mm, MM_ANONPAGES); - inc_mm_counter(mm, MM_SWAPENTS); - swp_pte = swp_entry_to_pte(entry); - if (anon_exclusive) - swp_pte = pte_swp_mkexclusive(swp_pte); - if (likely(pte_present(pteval))) { - if (pte_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); - } else { - if (pte_swp_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); - } - set_pte_at(mm, address, pvmw.pte, swp_pte); + goto finish_unmap; } else { /* * This is a locked file-backed folio, @@ -2341,12 +2396,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } -discard: - if (unlikely(folio_test_hugetlb(folio))) { - hugetlb_remove_rmap(folio); - } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); - } +finish_unmap: + folio_remove_rmap_ptes(folio, page, nr_pages, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put_refs(folio, nr_pages); @@ -2394,7 +2445,8 @@ static int folio_not_mapped(struct folio *folio) void try_to_unmap(struct folio *folio, enum ttu_flags flags) { struct rmap_walk_control rwc = { - .rmap_one = try_to_unmap_one, + .rmap_one = folio_test_hugetlb(folio) ? + try_to_unmap_hugetlb_one : try_to_unmap_one, .arg = (void *)flags, .done = folio_not_mapped, .anon_lock = folio_lock_anon_vma_read, @@ -2477,7 +2529,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_restart(&pvmw); continue; } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES pmdval = pmdp_get(pvmw.pmd); if (likely(pmd_present(pmdval))) pfn = pmd_pfn(pmdval); @@ -2501,14 +2553,18 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) + pteval = huge_ptep_get(mm, address, pvmw.pte); + else + pteval = ptep_get(pvmw.pte); if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); @@ -2516,7 +2572,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2692,14 +2747,14 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { swp_pte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, @@ -2986,13 +3041,12 @@ static void rmap_walk_anon(struct folio *folio, pgoff_start = folio_pgoff(folio); pgoff_end = pgoff_start + folio_nr_pages(folio) - 1; - anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, - pgoff_start, pgoff_end) { + anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; - unsigned long address = vma_address(vma, pgoff_start, + const unsigned long address = vma_anon_address(vma, pgoff_start, folio_nr_pages(folio)); - VM_BUG_ON_VMA(address == -EFAULT, vma); + VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma); cond_resched(); if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) @@ -3051,9 +3105,9 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, i_mmap_lock_read(mapping); } lookup: - vma_interval_tree_foreach(vma, &mapping->i_mmap, - pgoff_start, pgoff_end) { - unsigned long address = vma_address(vma, pgoff_start, nr_pages); + mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { + unsigned long address = vma_filebacked_address(vma, pgoff_start, + nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); diff --git a/mm/secretmem.c b/mm/secretmem.c index 4877c262cb1f..d29865075b6e 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -202,7 +202,7 @@ static struct file *secretmem_file_create(unsigned long flags) if (IS_ERR(file)) goto err_free_inode; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); mapping_set_unevictable(inode->i_mapping); inode->i_op = &secretmem_iops; diff --git a/mm/shmem.c b/mm/shmem.c index b51f83c970bb..5071177059a9 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1032,6 +1032,8 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) struct inode *inode = file_inode(vma->vm_file); struct shmem_inode_info *info = SHMEM_I(inode); struct address_space *mapping = inode->i_mapping; + const pgoff_t pgoff = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); unsigned long swapped; /* Be careful as we don't hold info->lock */ @@ -1045,12 +1047,11 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) if (!swapped) return 0; - if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) + if (!pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; /* Here comes the more involved part */ - return shmem_partial_swap_usage(mapping, vma->vm_pgoff, - vma->vm_pgoff + vma_pages(vma)); + return shmem_partial_swap_usage(mapping, pgoff, pgoff_end); } /* @@ -1287,7 +1288,8 @@ static int shmem_getattr(struct mnt_idmap *idmap, struct inode *inode = path->dentry->d_inode; struct shmem_inode_info *info = SHMEM_I(inode); - if (info->alloced - info->swapped != inode->i_mapping->nrpages) + /* Fast-path hint; recalc under info->lock corrects any stale read. */ + if (data_race(info->alloced - info->swapped != inode->i_mapping->nrpages)) shmem_recalc_inode(inode, 0, 0); if (info->fsflags & FS_APPEND_FL) @@ -1582,13 +1584,13 @@ start_over: /** * shmem_writeout - Write the folio to swap + * @ctx: swap I/O context * @folio: The folio to write - * @plug: swap plug * @folio_list: list to put back folios on split * * Move the folio from the page cache to the swap cache. */ -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, +int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, struct list_head *folio_list) { struct address_space *mapping = folio->mapping; @@ -1657,7 +1659,7 @@ try_split: * reactivate the folio, and let shmem_fallocate() quit when too many. */ if (!folio_test_uptodate(folio)) { - if (inode->i_private) { + if (READ_ONCE(inode->i_private)) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); shmem_falloc = inode->i_private; @@ -1700,7 +1702,7 @@ try_split: shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap)); BUG_ON(folio_mapped(folio)); - error = swap_writeout(folio, plug); + error = swap_writeout(ctx, folio); if (error != AOP_WRITEPAGE_ACTIVATE) { /* folio has been unlocked */ return error; @@ -1736,7 +1738,17 @@ redirty: folio_mark_dirty(folio); return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ } -EXPORT_SYMBOL_GPL(shmem_writeout); + +int shmem_write_folio(struct folio *folio) +{ + struct swap_io_ctx ctx = {}; + int err; + + err = shmem_writeout(&ctx, folio, NULL); + swap_write_submit(&ctx); + return err; +} +EXPORT_SYMBOL_GPL(shmem_write_folio); #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) @@ -2693,7 +2705,7 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ - if (unlikely(inode->i_private)) { + if (unlikely(READ_ONCE(inode->i_private))) { ret = shmem_falloc_wait(vmf, inode); if (ret) return ret; @@ -2839,7 +2851,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = linear_page_index(vma, addr); return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } @@ -3630,7 +3642,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); if ((u64)unmap_end > (u64)unmap_start) @@ -3640,7 +3652,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, /* No need to unmap again: hole-punching leaves COWed pages */ spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); wake_up_all(&shmem_falloc_waitq); WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); spin_unlock(&inode->i_lock); @@ -3672,7 +3684,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.nr_falloced = 0; shmem_falloc.nr_unswapped = 0; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); /* @@ -3747,7 +3759,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, i_size_write(inode, offset + len); undone: spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); spin_unlock(&inode->i_lock); out: if (!error) @@ -4057,6 +4069,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir, goto out_remove_offset; inode->i_op = &shmem_symlink_inode_operations; memcpy(folio_address(folio), symname, len); + folio_zero_range(folio, len, folio_size(folio) - len); folio_mark_uptodate(folio); folio_mark_dirty(folio); folio_unlock(folio); diff --git a/mm/show_mem.c b/mm/show_mem.c index 43aca5a2ac99..d1288b4c2b64 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -16,6 +16,7 @@ #include <linux/vmstat.h> #include "internal.h" +#include "page_alloc.h" #include "swap.h" atomic_long_t _totalram_pages __read_mostly; @@ -116,7 +117,8 @@ void si_meminfo_node(struct sysinfo *val, int nid) * Determine whether the node should be displayed or not, depending on whether * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). */ -static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) +static bool show_mem_node_skip(unsigned int flags, int nid, + const nodemask_t *nodemask) { if (!(flags & SHOW_MEM_FILTER_NODES)) return false; @@ -177,7 +179,8 @@ static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's * cpuset. */ -static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long free_pcp = 0; int cpu, nid; @@ -402,7 +405,8 @@ static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_z show_swap_cache_info(); } -void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +void __show_mem(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long total = 0, reserved = 0, highmem = 0; struct zone *zone; diff --git a/mm/shuffle.c b/mm/shuffle.c index fb1393b8b3a9..82a2c7725a08 100644 --- a/mm/shuffle.c +++ b/mm/shuffle.c @@ -7,6 +7,7 @@ #include <linux/random.h> #include <linux/moduleparam.h> #include "internal.h" +#include "page_alloc.h" #include "shuffle.h" DEFINE_STATIC_KEY_FALSE(page_alloc_shuffle_key); diff --git a/mm/slab.h b/mm/slab.h index f5e336b6b6b0..c24c3daaa869 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -180,7 +180,7 @@ static inline void *slab_address(const struct slab *slab) static inline int slab_nid(const struct slab *slab) { - return memdesc_nid(slab->flags); + return memdesc_nid(&slab->flags); } static inline pg_data_t *slab_pgdat(const struct slab *slab) diff --git a/mm/slub.c b/mm/slub.c index 0337e60db5ac..422bc3e12c02 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -53,6 +53,7 @@ #include <trace/events/kmem.h> #include "internal.h" +#include "page_alloc.h" /* * Lock order: @@ -3263,7 +3264,8 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node, else if (node == NUMA_NO_NODE) page = alloc_frozen_pages(flags, order); else - page = __alloc_frozen_pages(flags, order, node, NULL); + page = __alloc_frozen_pages(flags, order, node, NULL, + ALLOC_DEFAULT); if (!page) return NULL; @@ -5273,7 +5275,8 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) if (node == NUMA_NO_NODE) page = alloc_frozen_pages_noprof(flags, order); else - page = __alloc_frozen_pages_noprof(flags, order, node, NULL); + page = __alloc_frozen_pages_noprof(flags, order, node, NULL, + ALLOC_DEFAULT); if (page) { ptr = page_address(page); @@ -5377,15 +5380,7 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f if (unlikely(!size)) return ZERO_SIZE_PTR; - /* - * See the comment for the same check in - * alloc_frozen_pages_nolock_noprof() - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) - return NULL; - - /* On UP, spin_trylock() always succeeds even when it is locked */ - if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + if (!can_spin_trylock()) return NULL; retry: diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index ebd3ac997f64..5a2469fb1838 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -41,6 +41,8 @@ #define VMEMMAP_POPULATE_PAGEREF 0x0001 #include "internal.h" +#include "mm_init.h" +#include "sparse.h" /* * Allocate a block of memory to be used to back the virtual memory map @@ -342,8 +344,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * * * Any initialization done here will be overwritten by memmap_init(). * - * hugetlb_vmemmap_init() will take care of initialization after - * memmap_init(). + * hugetlb_bootmem_struct_page_init() will take care of initialization + * after memmap_init(). */ p = vmemmap_alloc_block_zero(PAGE_SIZE, node); @@ -581,17 +583,6 @@ void __init sparse_vmemmap_init_nid_early(int nid) { hugetlb_vmemmap_init_early(nid); } - -/* - * This is called just before the initialization of page structures - * through memmap_init. Zones are now initialized, so any work that - * needs to be done that needs zone information can be done from - * here. - */ -void __init sparse_vmemmap_init_nid_late(int nid) -{ - hugetlb_vmemmap_init_late(nid); -} #endif static void subsection_mask_set(unsigned long *map, unsigned long pfn, @@ -603,7 +594,7 @@ static void subsection_mask_set(unsigned long *map, unsigned long pfn, bitmap_set(map, idx, end - idx + 1); } -void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages) +static void __init sparse_init_subsection_map_range(unsigned long pfn, unsigned long nr_pages) { int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1); unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn); @@ -626,6 +617,15 @@ void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages } } +void __init sparse_init_subsection_map(void) +{ + int i, nid; + unsigned long start, end; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) + sparse_init_subsection_map_range(start, end - start); +} + #ifdef CONFIG_MEMORY_HOTPLUG /* Mark all memory sections within the pfn range as online */ diff --git a/mm/sparse.c b/mm/sparse.c index 16ac6df3c89f..704a9dec2b9a 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -13,9 +13,10 @@ #include <linux/vmalloc.h> #include <linux/swap.h> #include <linux/swapops.h> -#include <linux/bootmem_info.h> #include <linux/vmstat.h> #include "internal.h" +#include "mm_init.h" +#include "sparse.h" #include <asm/dma.h> /* @@ -43,7 +44,7 @@ static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; #endif -int memdesc_nid(memdesc_flags_t mdf) +int memdesc_nid(const memdesc_flags_t *mdf) { return section_to_node_table[memdesc_section(mdf)]; } @@ -239,15 +240,8 @@ struct page __init *__populate_section_memmap(unsigned long pfn, struct dev_pagemap *pgmap) { unsigned long size = section_map_size(); - struct page *map; - phys_addr_t addr = __pa(MAX_DMA_ADDRESS); - map = memmap_alloc(size, size, addr, nid, false); - if (!map) - panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", - __func__, size, PAGE_SIZE, nid, &addr); - - return map; + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ @@ -300,17 +294,14 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, unsigned long map_count) { unsigned long pnum; - struct page *map; - struct mem_section *ms; - if (sparse_usage_init(nid, map_count)) { - pr_err("%s: node[%d] usemap allocation failed", __func__, nid); - goto failed; - } + if (sparse_usage_init(nid, map_count)) + panic("Failed to allocate usemap for node %d\n", nid); sparse_vmemmap_init_nid_early(nid); for_each_present_section_nr(pnum_begin, pnum) { + struct mem_section *ms; unsigned long pfn = section_nr_to_pfn(pnum); if (pnum >= pnum_end) @@ -318,34 +309,18 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, ms = __nr_to_section(pnum); if (!preinited_vmemmap_section(ms)) { + struct page *map; + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, - nid, NULL, NULL); - if (!map) { - pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", - __func__, nid); - pnum_begin = pnum; - sparse_usage_fini(); - goto failed; - } + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), PAGE_SIZE)); sparse_init_early_section(nid, map, pnum, 0); } } sparse_usage_fini(); - return; -failed: - /* - * We failed to allocate, mark all the following pnums as not present, - * except the ones already initialized earlier. - */ - for_each_present_section_nr(pnum_begin, pnum) { - if (pnum >= pnum_end) - break; - ms = __nr_to_section(pnum); - if (!preinited_vmemmap_section(ms)) - ms->section_mem_map = 0; - } } /* @@ -369,9 +344,6 @@ void __init sparse_init(void) pnum_begin = first_present_section_nr(); nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); - /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ - set_pageblock_order(); - for_each_present_section_nr(pnum_begin + 1, pnum_end) { int nid = sparse_early_nid(__nr_to_section(pnum_end)); @@ -387,5 +359,6 @@ void __init sparse_init(void) } /* cover the last node */ sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); + sparse_init_subsection_map(); vmemmap_populate_print_last(); } diff --git a/mm/sparse.h b/mm/sparse.h new file mode 100644 index 000000000000..95aa031213f2 --- /dev/null +++ b/mm/sparse.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * sparse.h: + * + * mm/ internal sparse and sparse-vmemmap declarations + */ + +#ifndef __MM_SPARSE_H +#define __MM_SPARSE_H + +#include <linux/mmzone.h> + +/* + * mm/sparse.c + */ +#ifdef CONFIG_SPARSEMEM +void sparse_init(void); +int sparse_index_init(unsigned long section_nr, int nid); + +static inline void sparse_init_one_section(struct mem_section *ms, + unsigned long pnum, struct page *mem_map, + struct mem_section_usage *usage, unsigned long flags) +{ + unsigned long coded_mem_map; + + BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); + + /* + * We encode the start PFN of the section into the mem_map such that + * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it + * from the page pointer to obtain the PFN. + */ + coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); + VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); + + ms->section_mem_map &= ~SECTION_MAP_MASK; + ms->section_mem_map |= coded_mem_map; + ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; + ms->usage = usage; +} + +static inline void __section_mark_present(struct mem_section *ms, + unsigned long section_nr) +{ + if (section_nr > __highest_present_section_nr) + __highest_present_section_nr = section_nr; + + ms->section_mem_map |= SECTION_MARKED_PRESENT; +} +#else +static inline void sparse_init(void) {} +#endif /* CONFIG_SPARSEMEM */ + +/* + * mm/sparse-vmemmap.c + */ +#ifdef CONFIG_SPARSEMEM_VMEMMAP +void sparse_init_subsection_map(void); +#else +static inline void sparse_init_subsection_map(void) {} +#endif /* CONFIG_SPARSEMEM_VMEMMAP */ + +#endif /* __MM_SPARSE_H */ diff --git a/mm/swap.h b/mm/swap.h index 77d2d14eda42..abd26588abd2 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -4,12 +4,11 @@ #include <linux/atomic.h> /* for atomic_long_t */ #include <linux/mm.h> /* for PAGE_SHIFT */ + struct mempolicy; struct swap_iocb; struct swap_memcg_table; -extern int page_cluster; - #if defined(MAX_POSSIBLE_PHYSMEM_BITS) #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) #elif defined(MAX_PHYSMEM_BITS) @@ -78,6 +77,28 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; +struct swap_io_ctx { + struct swap_iocb *sio; + struct swap_info_struct *sis; +}; + +/* + * SWAP_OPS_F_REQUIRE_NOFS: + * When set, all reclaim operations must operated as GFS_NOFS and not + * just GFP_NOIO, as GFP_NOIO allocations could recourse into the + * file system backing this swap file. + */ +#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) + +struct swap_ops { + unsigned int flags; + + bool (*can_merge)(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw); + void (*submit_write)(struct swap_io_ctx *ctx); + void (*submit_read)(struct swap_io_ctx *ctx); +}; + #ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ @@ -230,8 +251,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); * folio_put_swap(): does the opposite thing of folio_dup_swap(). */ int folio_alloc_swap(struct folio *folio); -int folio_dup_swap(struct folio *folio, struct page *subpage); -void folio_put_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap(struct folio *folio, struct page *page); +void folio_put_swap(struct folio *folio, struct page *page); /* For internal use */ extern void __swap_cluster_free_entries(struct swap_info_struct *si, @@ -240,17 +261,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); -struct swap_iocb; -void swap_read_folio(struct folio *folio, struct swap_iocb **plug); -void __swap_read_unplug(struct swap_iocb *plug); -static inline void swap_read_unplug(struct swap_iocb *plug) -{ - if (unlikely(plug)) - __swap_read_unplug(plug); -} -void swap_write_unplug(struct swap_iocb *sio); -int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); -void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug); +void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio); +void swap_read_submit(struct swap_io_ctx *ctx); +void swap_write_submit(struct swap_io_ctx *ctx); +int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio); +void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio); /* linux/mm/swap_state.c */ extern struct address_space swap_space __read_mostly; @@ -317,9 +332,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, void show_swap_cache_info(void); void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); -struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, - struct vm_area_struct *vma, unsigned long addr, - struct swap_iocb **plug); +struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, + gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr); struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, struct mempolicy *mpol, pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, @@ -329,18 +343,7 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders, void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, unsigned long addr); -static inline unsigned int folio_swap_flags(struct folio *folio) -{ - return __swap_entry_to_info(folio->swap)->flags; -} - #else /* CONFIG_SWAP */ -struct swap_iocb; -static inline struct swap_cluster_info *swap_cluster_lock( - struct swap_info_struct *si, pgoff_t offset, bool irq) -{ - return NULL; -} static inline struct swap_cluster_info *swap_cluster_get_and_lock( struct folio *folio) @@ -381,11 +384,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } -static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) +static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio) { } -static inline void swap_write_unplug(struct swap_iocb *sio) +static inline void swap_write_submit(struct swap_io_ctx *ctx) { } @@ -427,8 +430,7 @@ static inline void swap_update_readahead(struct folio *folio, { } -static inline int swap_writeout(struct folio *folio, - struct swap_iocb **swap_plug) +static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) { return 0; } @@ -466,11 +468,11 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new) { } +#endif /* CONFIG_SWAP */ -static inline unsigned int folio_swap_flags(struct folio *folio) -{ - return 0; -} +extern const struct swap_ops swap_bdev_ops; + +int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, + struct list_head *folio_list); -#endif /* CONFIG_SWAP */ #endif /* _MM_SWAP_H */ diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf99778..5be825911e64 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -22,10 +22,15 @@ #include <linux/vmalloc.h> #include <linux/huge_mm.h> #include <linux/shmem_fs.h> +#include <linux/sysctl.h> #include "internal.h" #include "swap_table.h" #include "swap.h" +/* Swap readahead cluster size, as a power of 2 pages. */ +static int page_cluster; +static const int page_cluster_max = 31; + /* * swapper_space is a fiction, retained to simplify the path through * vmscan's shrink_folio_list. @@ -633,9 +638,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, } } -static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, - struct mempolicy *mpol, pgoff_t ilx, - struct swap_iocb **plug, bool readahead) +static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx, + swp_entry_t entry, gfp_t gfp, struct mempolicy *mpol, + pgoff_t ilx, bool readahead) { struct folio *folio; @@ -649,7 +654,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, if (IS_ERR_OR_NULL(folio)) return NULL; - swap_read_folio(folio, plug); + swap_read_folio(ctx, folio); if (readahead) { folio_set_readahead(folio); count_vm_event(SWAP_RA); @@ -677,6 +682,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) { + struct swap_io_ctx ctx = {}; struct folio *folio; do { @@ -689,7 +695,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, if (IS_ERR(folio)) return folio; - swap_read_folio(folio, NULL); + swap_read_folio(&ctx, folio); + swap_read_submit(&ctx); return folio; } @@ -699,9 +706,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, * A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */ -struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, - struct vm_area_struct *vma, unsigned long addr, - struct swap_iocb **plug) +struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry, + gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr) { struct swap_info_struct *si; struct mempolicy *mpol; @@ -713,13 +719,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, return NULL; mpol = get_vma_policy(vma, addr, 0, &ilx); - folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false); + folio = swap_cache_read_folio(ctx, entry, gfp_mask, mpol, ilx, false); mpol_cond_put(mpol); put_swap_device(si); return folio; } +static struct folio *swap_cache_read_folio_sync(swp_entry_t entry, gfp_t gfp, + struct mempolicy *mpol, pgoff_t ilx) +{ + struct swap_io_ctx ctx = {}; + struct folio *folio; + + folio = swap_cache_read_folio(&ctx, entry, gfp, mpol, ilx, false); + swap_read_submit(&ctx); + return folio; +} + static unsigned int __swapin_nr_pages(unsigned long prev_offset, unsigned long offset, int hits, @@ -808,8 +825,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, unsigned long start_offset, end_offset; unsigned long mask; struct swap_info_struct *si = __swap_entry_to_info(entry); + struct swap_io_ctx ctx = {}; struct blk_plug plug; - struct swap_iocb *splug = NULL; swp_entry_t ra_entry; mask = swapin_nr_pages(offset) - 1; @@ -828,18 +845,16 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, for (offset = start_offset; offset <= end_offset ; offset++) { /* Ok, do the async read-ahead now */ ra_entry = swp_entry(swp_type(entry), offset); - folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx, - &splug, offset != entry_offset); + folio = swap_cache_read_folio(&ctx, ra_entry, gfp_mask, mpol, + ilx, offset != entry_offset); if (!folio) continue; folio_put(folio); } blk_finish_plug(&plug); - swap_read_unplug(splug); - lru_add_drain(); /* Push any new pages onto the LRU now */ + swap_read_submit(&ctx); skip: - /* The page was likely read above, so no need for plugging here */ - return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false); + return swap_cache_read_folio_sync(entry, gfp_mask, mpol, ilx); } static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, @@ -899,8 +914,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start, static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf) { + struct swap_io_ctx ctx = {}; struct blk_plug plug; - struct swap_iocb *splug = NULL; struct folio *folio; pte_t *pte = NULL, pentry; int win; @@ -939,8 +954,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, if (!si) continue; } - folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, - &splug, addr != vmf->address); + folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx, + addr != vmf->address); if (si) put_swap_device(si); if (!folio) @@ -950,13 +965,10 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, if (pte) pte_unmap(pte); blk_finish_plug(&plug); - swap_read_unplug(splug); - lru_add_drain(); + swap_read_submit(&ctx); skip: /* The folio was likely read above, so no need for plugging here */ - folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx, - NULL, false); - return folio; + return swap_cache_read_folio_sync(targ_entry, gfp_mask, mpol, targ_ilx); } /** @@ -987,6 +999,35 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, return folio; } +static const struct ctl_table swap_readahead_sysctl_table[] = { + { + .procname = "page-cluster", + .data = &page_cluster, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&page_cluster_max, + } +}; + +static void __init swap_readahead_setup(void) +{ + unsigned long megs = PAGES_TO_MB(totalram_pages()); + + /* Use a smaller cluster for small-memory machines */ + if (megs < 16) + page_cluster = 2; + else + page_cluster = 3; + /* + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ + + register_sysctl_init("vm", swap_readahead_sysctl_table); +} + #ifdef CONFIG_SYSFS static ssize_t vma_ra_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -1016,7 +1057,7 @@ static const struct attribute_group swap_attr_group = { .attrs = swap_attrs, }; -static int __init swap_init(void) +static int __init swap_sysfs_init(void) { int err; struct kobject *swap_kobj; @@ -1039,5 +1080,17 @@ delete_obj: kobject_put(swap_kobj); return err; } -subsys_initcall(swap_init); +#else +static int __init swap_sysfs_init(void) +{ + return 0; +} #endif + +static int __init swap_init(void) +{ + swap_readahead_setup(); + + return swap_sysfs_init(); +} +subsys_initcall(swap_init); diff --git a/mm/swapfile.c b/mm/swapfile.c index 78b49b0658ad..70b90fa9c2a0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1072,6 +1072,12 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force) swap_cluster_unlock(ci); if (to_scan <= 0) break; + + /* + * When 'force' is false, 'to_scan' is initialized to 1. + * The loop breaks above, making this cond_resched() unreachable + * in atomic contexts. + */ cond_resched(); } } @@ -1781,7 +1787,7 @@ again: /** * folio_dup_swap() - Increase swap count of swap entries of a folio. * @folio: folio with swap entries bounded. - * @subpage: if not NULL, only increase the swap count of this subpage. + * @page: if not NULL, only increase the swap count of this page. * * Typically called when the folio is unmapped and have its swap entry to * take its place: Swap entries allocated to a folio has count == 0 and pinned @@ -1795,7 +1801,7 @@ again: * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ -int folio_dup_swap(struct folio *folio, struct page *subpage) +int folio_dup_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1803,8 +1809,8 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } @@ -1815,13 +1821,13 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) /** * folio_put_swap() - Decrease swap count of swap entries of a folio. * @folio: folio with swap entries bounded, must be in swap cache and locked. - * @subpage: if not NULL, only decrease the swap count of this subpage. + * @page: if not NULL, only decrease the swap count of this page. * * This won't free the swap slots even if swap count drops to zero, they are * still pinned by the swap cache. User may call folio_free_swap to free them. * Context: Caller must ensure the folio is locked and in the swap cache. */ -void folio_put_swap(struct folio *folio, struct page *subpage) +void folio_put_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1830,8 +1836,8 @@ void folio_put_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } @@ -2496,8 +2502,13 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot)); if (pte_swp_soft_dirty(old_pte)) new_pte = pte_mksoft_dirty(new_pte); - if (pte_swp_uffd_wp(old_pte)) - new_pte = pte_mkuffd_wp(new_pte); + if (pte_swp_uffd(old_pte)) + new_pte = pte_mkuffd(new_pte); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma)) + new_pte = pte_modify(new_pte, PAGE_NONE); + setpte: set_pte_at(vma->vm_mm, addr, pte, new_pte); folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry))); @@ -2941,6 +2952,12 @@ static int setup_swap_extents(struct swap_info_struct *sis, struct inode *inode = mapping->host; int ret; + ret = sio_pool_init(); + if (ret) + return ret; + + sis->ops = &swap_bdev_ops; + if (S_ISBLK(inode->i_mode)) { ret = add_swap_extent(sis, 0, sis->max, 0); *span = sis->pages; @@ -2952,11 +2969,6 @@ static int setup_swap_extents(struct swap_info_struct *sis, if (ret < 0) return ret; sis->flags |= SWP_ACTIVATED; - if ((sis->flags & SWP_FS_OPS) && - sio_pool_init() != 0) { - destroy_swap_extents(sis, swap_file); - return -ENOMEM; - } return ret; } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index c3adedaaf7d5..258b03182a78 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -376,7 +376,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (writable) _dst_pte = pte_mkwrite(_dst_pte, dst_vma); if (flags & MFILL_ATOMIC_WP) - _dst_pte = pte_mkuffd_wp(_dst_pte); + _dst_pte = pte_mkuffd(_dst_pte); ret = -EAGAIN; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); @@ -481,7 +481,7 @@ static void mfill_retry_state_save(struct mfill_retry_state *s, { s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS); s->ops = vma_uffd_ops(vma); - s->pgoff = vma->vm_pgoff; + s->pgoff = vma_start_pgoff(vma); if (vma->vm_file) s->file = get_file(vma->vm_file); @@ -507,7 +507,7 @@ static bool mfill_retry_state_changed(struct mfill_retry_state *state, /* VMA was file backed, but file, inode or offset has changed */ if (!vma->vm_file || vma->vm_file->f_inode != state->file->f_inode || - state->file != vma->vm_file || vma->vm_pgoff != state->pgoff) + state->file != vma->vm_file || vma_start_pgoff(vma) != state->pgoff) return true; return false; @@ -1164,6 +1164,75 @@ out_unlock: return err; } +int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp) +{ + struct mm_struct *dst_mm = ctx->mm; + unsigned long end = start + len; + struct vm_area_struct *dst_vma; + unsigned int mm_cp_flags; + struct mmu_gather tlb; + bool found = false; + VMA_ITERATOR(vmi, dst_mm, start); + + VM_WARN_ON_ONCE(start & ~PAGE_MASK); + VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start + len <= start); + + guard(mmap_read_lock)(dst_mm); + guard(rwsem_read)(&ctx->map_changing_lock); + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (enable_rwp) + mm_cp_flags = MM_CP_UFFD_RWP; + else + mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE; + + /* + * Pre-scan the range: validate every spanned VMA before applying + * any change_protection() so a partial failure cannot leave the + * process with only a prefix of the range re-protected. + */ + for_each_vma_range(vmi, dst_vma, end) { + if (!userfaultfd_rwp(dst_vma)) + return -ENOENT; + + if (is_vm_hugetlb_page(dst_vma)) { + unsigned long page_mask; + + page_mask = vma_kernel_pagesize(dst_vma) - 1; + if ((start & page_mask) || (len & page_mask)) + return -EINVAL; + } + found = true; + } + if (!found) + return -ENOENT; + + vma_iter_set(&vmi, start); + tlb_gather_mmu(&tlb, dst_mm); + for_each_vma_range(vmi, dst_vma, end) { + unsigned long vma_start = max(dst_vma->vm_start, start); + unsigned long vma_end = min(dst_vma->vm_end, end); + unsigned int flags = mm_cp_flags; + + /* + * On resolve, try to upgrade writability per-VMA -- + * MM_CP_TRY_CHANGE_WRITABLE WARNs in + * maybe_change_pte_writable() if the VMA is not VM_WRITE, + * and RWP can be registered on PROT_READ-only mappings. + */ + if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma)) + flags |= MM_CP_TRY_CHANGE_WRITABLE; + + change_protection(&tlb, dst_vma, vma_start, vma_end, flags); + } + tlb_finish_mmu(&tlb); + + return 0; +} void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2) @@ -1283,7 +1352,8 @@ static long move_present_ptes(struct mm_struct *mm, } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_folio_page_index(src_folio, dst_vma, + dst_addr); orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot); /* Set soft dirty bit so userspace can notice the pte was moved */ @@ -1292,6 +1362,13 @@ static long move_present_ptes(struct mm_struct *mm, if (pte_dirty(orig_src_pte)) orig_dst_pte = pte_mkdirty(orig_dst_pte); orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma); + + /* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + orig_dst_pte = pte_modify(orig_dst_pte, PAGE_NONE); + orig_dst_pte = pte_mkuffd(orig_dst_pte); + } + set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); src_addr += PAGE_SIZE; @@ -1352,7 +1429,8 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, */ if (src_folio) { folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_folio_page_index(src_folio, dst_vma, + dst_addr); } else { /* * Check if the swap entry is cached after acquiring the src_pte @@ -1373,6 +1451,9 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); if (pgtable_supports_soft_dirty()) orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte); + /* Re-arm RWP on the moved swap entry if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) + orig_src_pte = pte_swp_mkuffd(orig_src_pte); set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); double_pt_unlock(dst_ptl, src_ptl); @@ -1399,6 +1480,13 @@ static int move_zeropage_pte(struct mm_struct *mm, zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), dst_vma->vm_page_prot)); + + /* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + zero_pte = pte_modify(zero_pte, PAGE_NONE); + zero_pte = pte_mkuffd(zero_pte); + } + ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); @@ -2190,9 +2278,22 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, if (start == vma->vm_start && end == vma->vm_end) give_up_on_oom = true; - /* Reset ptes for the whole vma range if wr-protected */ - if (userfaultfd_wp(vma)) - uffd_wp_range(vma, start, end - start, false); + /* Clear the uffd bit and/or restore protnone PTEs */ + if (userfaultfd_protected(vma)) { + unsigned int mm_cp_flags = 0; + struct mmu_gather tlb; + + if (userfaultfd_wp(vma)) + mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE; + if (userfaultfd_rwp(vma)) + mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE; + if (vma_wants_manual_pte_write_upgrade(vma)) + mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; + + tlb_gather_mmu(&tlb, vma->vm_mm); + change_protection(&tlb, vma, start, end, mm_cp_flags); + tlb_finish_mmu(&tlb); + } ret = vma_modify_flags_uffd(vmi, prev, vma, start, end, &new_vma_flags, NULL_VM_UFFD_CTX, @@ -2238,13 +2339,29 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, * userfaultfd and with the right tracking mode too. */ if (vma->vm_userfaultfd_ctx.ctx == ctx && - vma_test_all_mask(vma, vma_flags)) + (vma->vm_flags & __VM_UFFD_FLAGS) == vm_flags) goto skip; + /* + * Pre-scan in userfaultfd_register() already rejected mode + * switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a + * stray bit here is a bug. + */ + VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx && + vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags); + if (vma->vm_start > start) start = vma->vm_start; vma_end = min(end, vma->vm_end); + /* + * Re-registering into the same userfaultfd can remove WP mode. + * Clear any per-PTE uffd-wp state before dropping VM_UFFD_WP, + * matching the UFFDIO_UNREGISTER cleanup semantics. + */ + if (userfaultfd_wp(vma) && !(vm_flags & VM_UFFD_WP)) + uffd_wp_range(vma, start, vma_end - start, false); + new_vma_flags = vma->flags; vma_flags_clear_mask(&new_vma_flags, __VMA_UFFD_FLAGS); vma_flags_set_mask(&new_vma_flags, vma_flags); @@ -2371,14 +2488,29 @@ struct userfaultfd_wake_range { /* internal indication that UFFD_API ioctl was successfully executed */ #define UFFD_FEATURE_INITIALIZED (1u << 31) +/* + * UFFDIO_SET_MODE updates ctx->features under mmap_write_lock with + * WRITE_ONCE; readers that run outside mmap_read_lock or the per-VMA + * lock (poll/read_iter/ioctl, fdinfo) must pair with READ_ONCE. + */ +static unsigned int userfaultfd_features(struct userfaultfd_ctx *ctx) +{ + return READ_ONCE(ctx->features); +} + static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) { - return ctx->features & UFFD_FEATURE_INITIALIZED; + return userfaultfd_features(ctx) & UFFD_FEATURE_INITIALIZED; } static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) { - return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); + return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_WP_ASYNC); +} + +static bool userfaultfd_rwp_async_ctx(struct userfaultfd_ctx *ctx) +{ + return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_RWP_ASYNC); } /* @@ -2393,7 +2525,7 @@ bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) if (!ctx) return false; - return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; + return userfaultfd_features(ctx) & UFFD_FEATURE_WP_UNPOPULATED; } static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, @@ -2507,6 +2639,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address, msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; if (reason & VM_UFFD_WP) msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; + if (reason & VM_UFFD_RWP) + msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP; if (reason & VM_UFFD_MINOR) msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR; if (features & UFFD_FEATURE_THREAD_ID) @@ -2560,6 +2694,12 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, */ if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) return true; + /* + * PTE is still RW-protected (protnone with uffd bit), wait for + * resolution. Plain PROT_NONE without the marker is not an RWP fault. + */ + if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP)) + return true; return false; } @@ -2620,8 +2760,14 @@ again: if (!pmd_present(_pmd)) return false; - if (pmd_trans_huge(_pmd)) - return !pmd_write(_pmd) && (reason & VM_UFFD_WP); + if (pmd_trans_huge(_pmd)) { + if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) + return true; + if (pmd_protnone(_pmd) && pmd_uffd(_pmd) && + (reason & VM_UFFD_RWP)) + return true; + return false; + } pte = pte_offset_map(pmd, address); if (!pte) @@ -2657,6 +2803,13 @@ again: */ if (!pte_write(ptent) && (reason & VM_UFFD_WP)) goto out; + /* + * PTE is still RW-protected (protnone with uffd bit), wait for + * userspace to resolve. Plain PROT_NONE without the marker is not + * an RWP fault. + */ + if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP)) + goto out; ret = false; out: @@ -3601,11 +3754,27 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) vm_flags |= VM_UFFD_MISSING; if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { - if (!pgtable_supports_uffd_wp()) + if (!pgtable_supports_uffd()) goto out; vm_flags |= VM_UFFD_WP; } + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) { + if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE) + goto out; + if (!(userfaultfd_features(ctx) & UFFD_FEATURE_RWP)) + goto out; + vm_flags |= VM_UFFD_RWP; + } + + /* + * WP and RWP share the uffd PTE bit and + * cannot coexist in the same VMA — the bit would carry ambiguous + * semantics. Reject the combination up front. + */ + if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP)) + goto out; + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) { #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR goto out; @@ -3661,6 +3830,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, goto out_unlock; /* + * RWP uses protnone as an access-tracking marker. PROT_NONE + * VMAs have vm_page_prot == PAGE_NONE, so RWP resolution + * cannot make a page accessible again. Reject at register + * time only: a VMA that later becomes inaccessible via + * mprotect() must still be unregisterable, so this is not + * part of vma_can_userfault(). + */ + if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur)) + goto out_unlock; + + /* * UFFDIO_COPY will fill file holes even without * PROT_WRITE. This check enforces that if this is a * MAP_SHARED, the process has write permission to the backing @@ -3700,6 +3880,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, goto out_unlock; /* + * Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would + * leave PTE markers without the flag that describes them; + * subsequent mprotect() would then promote stale markers + * into the other mode. Require an unregister first. + */ + if (cur->vm_userfaultfd_ctx.ctx == ctx && + cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags) + goto out_unlock; + + /* * Note vmas containing huge pages */ if (is_vm_hugetlb_page(cur)) @@ -3732,6 +3922,10 @@ out_unlock: if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR)) ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE); + /* RWPROTECT is only supported for RWP ranges */ + if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP)) + ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT); + /* * Now that we scanned all vmas we can already tell * userland which ioctls methods are guaranteed to @@ -4079,6 +4273,158 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, return ret; } +static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + int ret; + struct uffdio_rwprotect uffdio_rwp; + struct userfaultfd_wake_range range; + bool mode_rwp, mode_dontwake; + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (copy_from_user(&uffdio_rwp, (void __user *)arg, + sizeof(uffdio_rwp))) + return -EFAULT; + + ret = validate_range(ctx->mm, uffdio_rwp.range.start, + uffdio_rwp.range.len); + if (ret) + return ret; + + if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE | + UFFDIO_RWPROTECT_MODE_RWP)) + return -EINVAL; + + mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP; + mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE; + + if (mode_rwp && mode_dontwake) + return -EINVAL; + + if (mmget_not_zero(ctx->mm)) { + ret = mrwprotect_range(ctx, uffdio_rwp.range.start, + uffdio_rwp.range.len, mode_rwp); + mmput(ctx->mm); + } else { + return -ESRCH; + } + + if (ret) + return ret; + + if (!mode_rwp && !mode_dontwake) { + range.start = uffdio_rwp.range.start; + range.len = uffdio_rwp.range.len; + wake_userfault(ctx, &range); + } + return ret; +} + +/* Subset of UFFD_API_FEATURES actually supported by this kernel/arch */ +static __u64 uffd_api_available_features(void) +{ + __u64 f = UFFD_API_FEATURES; + + if (!IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_MINOR)) + f &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); + if (!pgtable_supports_uffd()) + f &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; + if (!uffd_supports_wp_marker()) + f &= ~(UFFD_FEATURE_WP_HUGETLBFS_SHMEM | + UFFD_FEATURE_WP_UNPOPULATED | + UFFD_FEATURE_WP_ASYNC); + /* + * RWP needs both PROT_NONE support and the uffd PTE bit. The + * VM_UFFD_RWP check covers compile-time unavailability; the + * pgtable_supports_uffd() check covers runtime (e.g. riscv + * without the SVRSW60T59B extension) where the PTE bit is declared + * but not actually usable. + */ + if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) + f &= ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC); + return f; +} + +/* Async features that can be toggled at runtime via UFFDIO_SET_MODE */ +#define UFFD_FEATURE_TOGGLEABLE UFFD_FEATURE_RWP_ASYNC + +static int userfaultfd_set_mode(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + struct uffdio_set_mode mode; + struct mm_struct *mm = ctx->mm; + + if (copy_from_user(&mode, (void __user *)arg, sizeof(mode))) + return -EFAULT; + + /* enable and disable must not overlap */ + if (mode.enable & mode.disable) + return -EINVAL; + + /* only toggleable features that this kernel/arch actually supports */ + if ((mode.enable | mode.disable) & + ~(uffd_api_available_features() & UFFD_FEATURE_TOGGLEABLE)) + return -EINVAL; + + /* RWP_ASYNC can only be enabled on contexts that negotiated RWP */ + if ((mode.enable & UFFD_FEATURE_RWP_ASYNC) && + !(userfaultfd_features(ctx) & UFFD_FEATURE_RWP)) + return -EINVAL; + + if (!mmget_not_zero(mm)) + return -ESRCH; + + /* + * Drain in-flight faults before flipping features. mmap_write_lock() + * blocks new mmap_read_lock() callers, but per-VMA locked faults + * (lock_vma_under_rcu() + FAULT_FLAG_VMA_LOCK) that acquired before + * this point keep running. Calling vma_start_write() on each UFFD- + * armed VMA waits for those readers to drop, so no in-flight fault + * can observe the old features after mmap_write_unlock(). + */ + mmap_write_lock(mm); + { + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + for_each_vma(vmi, vma) { + if (vma->vm_userfaultfd_ctx.ctx == ctx) + vma_start_write(vma); + } + } + /* + * Single WRITE_ONCE so lockless readers (fdinfo, poll/read_iter + * via userfaultfd_is_initialized(), and the userfaultfd_features() + * helper used elsewhere) can't observe a mid-RMW intermediate + * value. Hot-path readers already serialise through the mmap lock + * + vma_start_write() drain above, so their load doesn't need an + * annotation. + */ + WRITE_ONCE(ctx->features, + (ctx->features | mode.enable) & ~mode.disable); + mmap_write_unlock(mm); + + /* + * If switching to async, wake threads blocked in handle_userfault(). + * They will retry the fault and auto-resolve under the new mode. + * len=0 means wake all pending faults on this context. + */ + if (mode.enable & UFFD_FEATURE_RWP_ASYNC) { + struct userfaultfd_wake_range range = { .len = 0 }; + + spin_lock_irq(&ctx->fault_pending_wqh.lock); + __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, + &range); + __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range); + spin_unlock_irq(&ctx->fault_pending_wqh.lock); + } + + mmput(mm); + return 0; +} + static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) { __s64 ret; @@ -4202,6 +4548,11 @@ bool userfaultfd_wp_async(struct vm_area_struct *vma) return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx); } +bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return userfaultfd_rwp_async_ctx(vma->vm_userfaultfd_ctx.ctx); +} + static inline unsigned int uffd_ctx_features(__u64 user_features) { /* @@ -4305,20 +4656,14 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, if (features & UFFD_FEATURE_WP_ASYNC) features |= UFFD_FEATURE_WP_UNPOPULATED; - /* report all available features and ioctls to userland */ - uffdio_api.features = UFFD_API_FEATURES; -#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR - uffdio_api.features &= - ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); -#endif - if (!pgtable_supports_uffd_wp()) - uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; + ret = -EINVAL; + /* RWP_ASYNC requires RWP */ + if ((features & UFFD_FEATURE_RWP_ASYNC) && + !(features & UFFD_FEATURE_RWP)) + goto err_out; - if (!uffd_supports_wp_marker()) { - uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; - uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; - uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; - } + /* report all available features and ioctls to userland */ + uffdio_api.features = uffd_api_available_features(); ret = -EINVAL; if (features & ~uffdio_api.features) @@ -4385,6 +4730,12 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, case UFFDIO_POISON: ret = userfaultfd_poison(ctx, arg); break; + case UFFDIO_RWPROTECT: + ret = userfaultfd_rwprotect(ctx, arg); + break; + case UFFDIO_SET_MODE: + ret = userfaultfd_set_mode(ctx, arg); + break; } return ret; } @@ -4412,7 +4763,7 @@ static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) * protocols: aa:... bb:... */ seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", - pending, total, UFFD_API, ctx->features, + pending, total, UFFD_API, userfaultfd_features(ctx), UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); } #endif diff --git a/mm/util.c b/mm/util.c index 34cb43b3eaa4..bf0513d1d3d0 100644 --- a/mm/util.c +++ b/mm/util.c @@ -578,8 +578,8 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); + ret = do_mmap(file, addr, len, prot, flag, EMPTY_VMA_FLAGS, pgoff, + &populate, &uf); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -627,20 +627,20 @@ EXPORT_SYMBOL(vm_mmap); unsigned long vm_mmap_shadow_stack(unsigned long addr, unsigned long len, unsigned long flags) { + vma_flags_t vma_flags = VMA_SHADOW_STACK; struct mm_struct *mm = current->mm; unsigned long ret, unused; - vm_flags_t vm_flags = VM_SHADOW_STACK; flags |= MAP_ANONYMOUS | MAP_PRIVATE; if (addr) flags |= MAP_FIXED_NOREPLACE; if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) - vm_flags |= VM_NOHUGEPAGE; + vma_flags_set(&vma_flags, VMA_NOHUGEPAGE_BIT); mmap_write_lock(mm); ret = do_mmap(NULL, addr, len, PROT_READ | PROT_WRITE, flags, - vm_flags, 0, &unused, NULL); + vma_flags, 0, &unused, NULL); mmap_write_unlock(mm); return ret; @@ -1188,7 +1188,7 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->start = vma->vm_start; desc->end = vma->vm_end; - desc->pgoff = vma->vm_pgoff; + desc->pgoff = vma_start_pgoff(vma); desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1379,7 +1379,7 @@ static int call_vma_mapped(struct vm_area_struct *vma) if (!vm_ops || !vm_ops->mapped) return 0; - err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma->vm_pgoff, + err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma_start_pgoff(vma), vma->vm_file, &vm_private_data); if (err) return err; @@ -4,6 +4,10 @@ * VMA-specific functions. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" @@ -14,6 +18,7 @@ struct mmap_state { unsigned long addr; unsigned long end; pgoff_t pgoff; + pgoff_t virt_pgoff; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,17 +47,26 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +static inline pgoff_t map_anon_pgoff(const struct mmap_state *map) +{ + if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT)) + return map->pgoff; + + return map->virt_pgoff; +} + +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, virt_pgoff_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ .pgoff = pgoff_, \ + .virt_pgoff = virt_pgoff_, \ .pglen = PHYS_PFN(len_), \ .vma_flags = vma_flags_, \ .file = file_, \ - .page_prot = vma_get_page_prot(vma_flags_), \ + .page_prot = vma_flags_to_page_prot(vma_flags_), \ } #define VMG_MMAP_STATE(name, map_, vma_) \ @@ -63,6 +77,7 @@ struct mmap_state { .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ .pgoff = (map_)->pgoff, \ + .anon_pgoff = map_anon_pgoff(map_), \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -70,6 +85,21 @@ struct mmap_state { .state = VMA_MERGE_START, \ } +static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + vma->vm_start = start; + vma->vm_end = end; +} + +static void vma_set_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, pgoff_t pgoff, pgoff_t virt_pgoff) +{ + __vma_set_range(vma, start, end); + vma_set_pgoff(vma, pgoff); + vma_set_virt_pgoff(vma, virt_pgoff); +} + /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ static bool vma_is_fork_child(struct vm_area_struct *vma) { @@ -197,15 +227,15 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); - - if (is_mergeable_vma(vmg, /* merge_next = */ true) && - is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) - return true; - } - - return false; + if (!is_mergeable_vma(vmg, /* merge_next = */ true)) + return false; + if (!is_mergeable_anon_vma(vmg, /* merge_next = */ true)) + return false; + if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next)) + return false; + if (vmg_end_anon_pgoff(vmg) != vma_start_anon_pgoff(vmg->next)) + return false; + return true; } /* @@ -219,12 +249,15 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) */ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { - if (is_mergeable_vma(vmg, /* merge_next = */ false) && - is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) - return true; - } - return false; + if (!is_mergeable_vma(vmg, /* merge_next = */ false)) + return false; + if (!is_mergeable_anon_vma(vmg, /* merge_next = */ false)) + return false; + if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg)) + return false; + if (vma_end_anon_pgoff(vmg->prev) != vmg_start_anon_pgoff(vmg)) + return false; + return true; } static void __vma_link_file(struct vm_area_struct *vma, @@ -234,7 +267,7 @@ static void __vma_link_file(struct vm_area_struct *vma, mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -248,7 +281,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -258,30 +291,30 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, * * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the * vma must be removed from the anon_vma's interval trees using - * anon_vma_interval_tree_pre_update_vma(). + * anon_rmap_tree_pre_update_vma(). * * After the update, the vma will be reinserted using - * anon_vma_interval_tree_post_update_vma(). + * anon_rmap_tree_post_update_vma(). * * The entire update must be protected by exclusive mmap_lock and by * the root anon_vma's mutex. */ static void -anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma) +anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma) { struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + anon_rmap_tree_remove(avc, avc->anon_vma); } static void -anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) +anon_rmap_tree_post_update_vma(struct vm_area_struct *vma) { struct anon_vma_chain *avc; list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_rmap_tree_insert(avc, avc->anon_vma); } /* @@ -312,17 +345,16 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->anon_vma) { anon_vma_lock_write(vp->anon_vma); - anon_vma_interval_tree_pre_update_vma(vp->vma); + anon_rmap_tree_pre_update_vma(vp->vma); if (vp->adj_next) - anon_vma_interval_tree_pre_update_vma(vp->adj_next); + anon_rmap_tree_pre_update_vma(vp->adj_next); } if (vp->file) { flush_dcache_mmap_lock(vp->mapping); - vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); + mapping_rmap_tree_remove(vp->vma, vp->mapping); if (vp->adj_next) - vma_interval_tree_remove(vp->adj_next, - &vp->mapping->i_mmap); + mapping_rmap_tree_remove(vp->adj_next, vp->mapping); } } @@ -340,9 +372,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, { if (vp->file) { if (vp->adj_next) - vma_interval_tree_insert(vp->adj_next, - &vp->mapping->i_mmap); - vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); + mapping_rmap_tree_insert(vp->adj_next, vp->mapping); + mapping_rmap_tree_insert(vp->vma, vp->mapping); flush_dcache_mmap_unlock(vp->mapping); } @@ -361,9 +392,9 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, } if (vp->anon_vma) { - anon_vma_interval_tree_post_update_vma(vp->vma); + anon_rmap_tree_post_update_vma(vp->vma); if (vp->adj_next) - anon_vma_interval_tree_post_update_vma(vp->adj_next); + anon_rmap_tree_post_update_vma(vp->adj_next); anon_vma_unlock_write(vp->anon_vma); } @@ -521,7 +552,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_add_pgoff(new, linear_page_delta(vma, addr)); } err = -ENOMEM; @@ -560,7 +591,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_add_pgoff(vma, linear_page_delta(new, addr)); } else { vma->vm_end = addr; } @@ -679,7 +710,7 @@ void validate_mm(struct mm_struct *mm) if (anon_vma) { anon_vma_lock_read(anon_vma); list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - anon_vma_interval_tree_verify(avc); + anon_rmap_tree_verify(avc); anon_vma_unlock_read(anon_vma); } #endif @@ -704,20 +735,56 @@ void validate_mm(struct mm_struct *mm) */ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { - struct vm_area_struct *adjust; - pgoff_t pgoff; - if (vmg->__adjust_middle_start) { - adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + /* + * vmg->start vmg->end + * | | + * v merge v + * <-------------> + * delta + * <------> + * |------|----------------| + * | prev | middle | + * |------|----------------| + * ^ + * | + * middle->vm_start + */ + struct vm_area_struct *middle = vmg->middle; + const unsigned long delta = vmg->end - middle->vm_start; + + __vma_set_range(middle, vmg->end, middle->vm_end); + vma_add_pgoff(middle, delta >> PAGE_SHIFT); } else if (vmg->__adjust_next_start) { - adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); - } else { - return; - } + /* + * Originally: + * + * vmg->start vmg->end + * | | + * v merge v + * <------------> + * . . + * merge_existing_range() updates to: + * . . + * vmg->start vmg->end . + * | | . + * v retain v . + * <----------> . + * delta . + * <-----> . + * |----------------|------| + * | middle | next | + * |----------------|------| + * ^ + * | + * next->vm_start + */ + struct vm_area_struct *next = vmg->next; + const unsigned long delta = next->vm_start - vmg->end; - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + __vma_set_range(next, vmg->end, next->vm_end); + vma_sub_pgoff(next, delta >> PAGE_SHIFT); + } } /* @@ -761,7 +828,8 @@ static int commit_merge(struct vma_merge_struct *vmg) */ vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); - vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg), + vmg_start_anon_pgoff(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -928,10 +996,10 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * prev middle next * extend delete delete */ - vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * We already ensured anon_vma compatibility above, so now it's @@ -948,9 +1016,9 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * prev middle * extend shrink/delete */ - vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -964,8 +1032,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * middle next * shrink/delete extend */ - - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + const pgoff_t pglen = vmg_pages(vmg); VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +1040,15 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg->pgoff = vma_start_pgoff(next) - pglen; + vmg->anon_pgoff = vma_start_anon_pgoff(next) - pglen; } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg->pgoff = vma_start_pgoff(middle); + vmg->anon_pgoff = vma_start_anon_pgoff(middle); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1157,8 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * If this merge would result in removal of the next VMA but we @@ -1235,27 +1305,24 @@ nomem: return -ENOMEM; } -/* - * vma_shrink() - Reduce an existing VMAs memory area +/** + * vma_shrink() - Shrink the end of a VMA * @vmi: The vma iterator * @vma: The VMA to modify - * @start: The new start * @end: The new end * + * Note that the caller may only shrink the end of the VMA. + * * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long end) { struct vma_prepare vp; - WARN_ON((vma->vm_start != start) && (vma->vm_end != end)); - - if (vma->vm_start < start) - vma_iter_config(vmi, vma->vm_start, start); - else - vma_iter_config(vmi, end, vma->vm_end); + VM_WARN_ON_ONCE(end > vma->vm_end); + vma_iter_config(vmi, end, vma->vm_end); if (vma_iter_prealloc(vmi, NULL)) return -ENOMEM; @@ -1263,10 +1330,10 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, init_vma_prep(&vp, vma); vma_prepare(&vp); - vma_adjust_trans_huge(vma, start, end, NULL); + vma_adjust_trans_huge(vma, vma->vm_start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + __vma_set_range(vma, vma->vm_start, end); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1863,9 +1930,10 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + pgoff_t virt_pgoff, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; + const bool is_shared = vma_test(vma, VMA_SHARED_BIT); unsigned long vma_start = vma->vm_start; struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *new_vma; @@ -1874,11 +1942,14 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len); /* - * If anonymous vma has not yet been faulted, update new pgoff - * to match new location, to increase its chance of merging. + * If a vma has not yet been faulted, update its virtual pgoff to match + * the new location to increase its chance of merging. */ - if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + if (!vma->anon_vma && !is_shared) { + virt_pgoff = addr >> PAGE_SHIFT; + + if (vma_is_anonymous(vma)) + pgoff = virt_pgoff; faulted_in_anon_vma = false; } @@ -1895,6 +1966,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ vmg.pgoff = pgoff; + vmg.anon_pgoff = is_shared ? pgoff : virt_pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1916,15 +1988,17 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * safe. It is only safe to keep the vm_pgoff * linear if there are no pages mapped yet. */ - VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); + VM_WARN_ON_ONCE_VMA(faulted_in_anon_vma, new_vma); *vmap = vma = new_vma; } - *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff); + *need_rmap_locks = + (vma_start_pgoff(new_vma) <= vma_start_pgoff(vma)) || + (vma_start_anon_pgoff(new_vma) <= vma_start_anon_pgoff(vma)); } else { new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pgoff, virt_pgoff); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1972,14 +2046,30 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * { vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags); + /* Ignore flags that mprotect() can change. */ vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS); + /* Ignore flags that do not impact merging. */ vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS); - return a->vm_end == b->vm_start && - mpol_equal(vma_policy(a), vma_policy(b)) && - a->vm_file == b->vm_file && - vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + /* Must be adjacent. */ + if (a->vm_end != b->vm_start) + return false; + /* Must have matching policy. */ + if (!mpol_equal(vma_policy(a), vma_policy(b))) + return false; + /* Must both be anon or map the same file (MAP_PRIVATE case). */ + if (a->vm_file != b->vm_file) + return false; + /* Flags must be equivalent modulo mprotect(). */ + if (!vma_flags_empty(&diff)) + return false; + /* Page offset must align. */ + if (vma_end_pgoff(a) != vma_start_pgoff(b)) + return false; + /* Anon page offset must align. */ + if (vma_end_anon_pgoff(a) != vma_start_anon_pgoff(b)) + return false; + return true; } /* @@ -2119,7 +2209,7 @@ bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) /* The open routine did something to the protections that pgprot_modify * won't preserve? */ if (pgprot_val(vm_page_prot) != - pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) + pgprot_val(vma_pgprot_modify(vm_page_prot, vma->flags))) return false; /* @@ -2522,6 +2612,33 @@ static int __mmap_new_file_vma(struct mmap_state *map, return 0; } +static bool map_is_dev_zero(const struct mmap_state *map) +{ + const struct file *file = map->file; + const struct inode *inode = file_inode(file); + + if (!S_ISCHR(inode->i_mode)) + return false; + return imajor(inode) == MEM_MAJOR && iminor(inode) == DEVZERO_MINOR; +} + +static void map_set_anon(struct mmap_state *map) +{ + map->file = NULL; + map->vm_ops = NULL; + map->pgoff = map->addr >> PAGE_SHIFT; +} + +static bool map_is_private(const struct mmap_state *map) +{ + return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); +} + +static bool map_is_anon(const struct mmap_state *map) +{ + return map_is_private(map) && !map->file; +} + /* * __mmap_new_vma() - Allocate a new VMA for the region, as merging was not * possible. @@ -2535,6 +2652,7 @@ static int __mmap_new_file_vma(struct mmap_state *map, static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, struct mmap_action *action) { + const bool is_anon = map_is_anon(map); struct vma_iterator *vmi = map->vmi; int error = 0; struct vm_area_struct *vma; @@ -2549,7 +2667,11 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + + if (is_anon) + vma_set_anonymous(vma); + + vma_set_range(vma, map->addr, map->end, map->pgoff, map->virt_pgoff); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2558,12 +2680,11 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, goto free_vma; } + /* Invoke callbacks. */ if (map->file) error = __mmap_new_file_vma(map, vma); - else if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT)) + else if (!is_anon) error = shmem_zero_setup(vma); - else - vma_set_anonymous(vma); if (error) goto free_iter_vma; @@ -2673,6 +2794,10 @@ static int call_mmap_prepare(struct mmap_state *map, if (err) return err; + /* Hooks cannot mark themselves anonymous. */ + if (!desc->vm_ops) + return -EINVAL; + err = call_action_prepare(map, desc); if (err) return err; @@ -2689,16 +2814,21 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; + /* + * MAP_PRIVATE-/dev/zero mappings are an ancient way of getting + * anonymous mappings. Rather than allowing these mappings to be odd + * outliers, simply make them truly anonymous. + */ + if (map_is_private(map) && map_is_dev_zero(map)) + map_set_anon(map); + return 0; } static void set_vma_user_defined_fields(struct vm_area_struct *vma, struct mmap_state *map) { - if (map->vm_ops) - vma->vm_ops = map->vm_ops; - else /* Only /dev/zero should do this. */ - vma_set_anonymous(vma); + vma->vm_ops = map->vm_ops; vma->vm_private_data = map->vm_private_data; } @@ -2739,7 +2869,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + const pgoff_t virt_pgoff = addr >> PAGE_SHIFT; + MMAP_STATE(map, mm, &vmi, addr, len, pgoff, virt_pgoff, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2777,7 +2908,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, allocated_new = true; } - if (have_mmap_prepare) + if (have_mmap_prepare && !map_is_anon(&map)) set_vma_user_defined_fields(vma, &map); __mmap_complete(&map, vma); @@ -2821,7 +2952,7 @@ abort_munmap: * file to be mapped, otherwise NULL. * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. - * @vm_flags: The VMA flags which should be applied to the mapping. + * @vma_flags: The VMA flags which should be applied to the mapping. * @pgoff: If @file is specified, the page offset into the file, if not then * the virtual page offset in memory of the anonymous mapping. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap @@ -2831,12 +2962,11 @@ abort_munmap: * been performed. */ unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; - const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); @@ -2845,7 +2975,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return -EACCES; /* Allow architectures to sanity-check the vm_flags. */ - if (!arch_validate_flags(vm_flags)) + if (!arch_validate_flags(vma_flags_to_legacy(vma_flags))) return -EINVAL; /* Map writable and ensure this isn't a sealed memfd. */ @@ -2885,6 +3015,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long addr, unsigned long len, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; + const pgoff_t pgoff = addr >> PAGE_SHIFT; /* * Check against address space limits by the changed size @@ -2909,7 +3040,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, * occur after forking, so the expand will only happen on new VMAs. */ if (vma && vma->vm_end == addr) { - VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, pgoff, pgoff); vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ @@ -2929,7 +3060,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, goto unacct_fail; vma_set_anonymous(vma); - vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_range(vma, addr, addr + len, pgoff, pgoff); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -3167,23 +3298,22 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address > vma->vm_end) { - unsigned long size, grow; - - size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + const unsigned long size = address - vma->vm_start; + const unsigned long grow = (address - vma->vm_end) >> PAGE_SHIFT; + const pgoff_t pgoff = vma_start_pgoff(vma); error = -ENOMEM; - if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { + if (pgoff + (size >> PAGE_SHIFT) >= pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); + anon_rmap_tree_pre_update_vma(vma); vma->vm_end = address; /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); - anon_vma_interval_tree_post_update_vma(vma); + anon_rmap_tree_post_update_vma(vma); perf_event_mmap(vma); } @@ -3246,24 +3376,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { - unsigned long size, grow; - - size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + const unsigned long size = vma->vm_end - address; + const unsigned long grow = (vma->vm_start - address) >> PAGE_SHIFT; error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (grow <= vma_start_pgoff(vma)) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); + anon_rmap_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_sub_pgoff(vma, grow); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); - anon_vma_interval_tree_post_update_vma(vma); + anon_rmap_tree_post_update_vma(vma); perf_event_mmap(vma); } @@ -3293,9 +3421,9 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) return ret; } -/* Insert vm structure into process list sorted by address - * and into the inode's i_mmap tree. If vm_file is non-NULL - * then i_mmap_rwsem is taken here. +/* + * Insert vm structure into process list sorted by address + * and into the inode's i_mmap tree if file-backed. */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { @@ -3321,9 +3449,10 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) * Similarly in do_mmap and in do_brk_flags. */ if (vma_is_anonymous(vma)) { - BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + WARN_ON_ONCE(vma->anon_vma); + vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } + vma_set_virt_pgoff(vma, vma->vm_start >> PAGE_SHIFT); if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) @@ -3354,3 +3483,44 @@ __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) { return vma_kernel_pagesize(vma); } + +struct vm_area_struct *__install_special_mapping( + struct mm_struct *mm, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, void *priv, + const struct vm_operations_struct *ops) +{ + vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); + struct vm_area_struct *vma; + int ret; + + vma = vm_area_alloc(mm); + if (unlikely(!vma)) + return ERR_PTR(-ENOMEM); + + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_DONTEXPAND_BIT); + if (pgtable_supports_soft_dirty()) + vma_flags_set(&vma_flags, VMA_SOFTDIRTY_BIT); + vma_flags_clear_mask(&vma_flags, VMA_LOCKED_MASK); + vma->flags = vma_flags; + vma->vm_page_prot = vma_get_page_prot(vma); + + vma->vm_ops = ops; + vma->vm_private_data = priv; + vma_set_range(vma, addr, addr + len, 0, addr >> PAGE_SHIFT); + + ret = insert_vm_struct(mm, vma); + if (ret) + goto out; + + vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + + perf_event_mmap(vma); + + return vma; + +out: + vm_area_free(vma); + return ERR_PTR(ret); +} @@ -2,7 +2,14 @@ /* * vma.h * - * Core VMA manipulation API implemented in vma.c. + * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c. + * + * Note that, in order for VMA logic to be userland testable, this header + * intentionally includes no dependencies. + * + * This is specifically scoped to mm-only. Users of this functionality (other + * than the core VMA implementation itself) should not include this header + * directly, but rather include internal.h. */ #ifndef __MM_VMA_H #define __MM_VMA_H @@ -97,6 +104,7 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; union { /* Temporary while VMA flags are being converted. */ @@ -230,43 +238,173 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) return vmg->state == VMA_MERGE_ERROR_NOMEM; } -/* Assumes addr >= vma->vm_start. */ -static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, - unsigned long addr) +static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) +{ + const unsigned long size = vmg->end - vmg->start; + + return size >> PAGE_SHIFT; +} + +static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->pgoff; +} + +static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_pgoff(vmg) + vmg_pages(vmg); +} + +static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + /* nommu doesn't set a virtual pgoff for anon VMAs. */ + if (!IS_ENABLED(CONFIG_MMU)) + return; + /* + * File-backed VMAs have arbitrary page offset (either page offset into + * file or for pfnmap the PFN of the start of the range or drivers may + * set arbitrary page offset). + */ + if (!vma_is_anonymous(vma)) + return; + /* If faulted in, could have been remapped. */ + if (vma->anon_vma) + return; + /* OK this is really an anon VMA - expect virtual page offset. */ + VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT); +} + +static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + assert_sane_pgoff(vma, pgoff); + vma->vm_pgoff = pgoff; +} + +static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->anon_pgoff; +} + +static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg); +} + +static inline void __vma_set_virt_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ +#ifdef CONFIG_64BIT + vma->__vm_virt_pgoff_hi = pgoff >> 32; +#endif + vma->__vm_virt_pgoff_lo = pgoff & GENMASK(31, 0); +} + +static inline void vma_set_virt_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + __vma_set_virt_pgoff(vma, pgoff); +} + +static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + vma_assert_can_modify(vma); + vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); + vma_set_virt_pgoff(vma, vma_start_virt_pgoff(vma) + delta); } -#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ +static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) +{ + vma_assert_can_modify(vma); + vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); + vma_set_virt_pgoff(vma, vma_start_virt_pgoff(vma) - delta); +} + +/** + * vma_anon_pgoff_addr() - Calculates the absolute anonymous page offset of + * @address. + * @vma: The VMA whose anonymous page offset is required. + * @address: The address whose absolute page offset is required. + * + * If the VMA is a shared file-backed mapping, then the file-based page offset + * is returned. + * + * Otherwise, the virtual page offset is returned. + * + * This means that shared file-backed mappings are correctly merged based on + * their file page offset compatibility. + * + * Returns: The absolute anonymous page offset of @address within @vma. + */ +static inline pgoff_t vma_anon_pgoff_addr(const struct vm_area_struct *vma, + unsigned long address) +{ + if (vma_test(vma, VMA_SHARED_BIT)) + return linear_page_index(vma, address); + + return linear_virt_page_index(vma, address); +} + +/** + * vma_start_anon_pgoff() - Calculates the absolute anonymous page offset used + * for purposes of merge compatibility. + * @vma: The VMA whose anonymous page offset is required. + * + * See vma_anon_pgoff_addr(). + * + * Returns: The absolute anonymous page offset of @vma for purposes of merging. + */ +static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_anon_pgoff_addr(vma, vma->vm_start); +} + +/** + * vma_end_anon_pgoff() - Calculates the absolute exclusive end anonymous page + * offset used for purposes of merge compatibility. + * @vma: The VMA whose anonymous end page offset is required. + * + * See vma_start_anon_pgoff(). + * + * Returns: The absolute exclusive end anonymous page offset of @vma for + * purposes of merging. + */ +static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_anon_pgoff(vma) + vma_pages(vma); +} + +#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \ + struct vma_merge_struct name = { \ + .mm = mm_, \ + .vmi = vmi_, \ + .start = start_, \ + .end = end_, \ + .vma_flags = vma_flags_, \ + .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ + .state = VMA_MERGE_START, \ + } + +#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ struct vma_merge_struct name = { \ - .mm = mm_, \ + .mm = vma_->vm_mm, \ .vmi = vmi_, \ + .prev = prev_, \ + .middle = vma_, \ + .next = NULL, \ .start = start_, \ .end = end_, \ - .vma_flags = vma_flags_, \ - .pgoff = pgoff_, \ + .vm_flags = vma_->vm_flags, \ + .pgoff = linear_page_index(vma_, start_), \ + .anon_pgoff = vma_anon_pgoff_addr(vma_, start_), \ + .file = vma_->vm_file, \ + .anon_vma = vma_->anon_vma, \ + .policy = vma_policy(vma_), \ + .uffd_ctx = vma_->vm_userfaultfd_ctx, \ + .anon_name = anon_vma_name(vma_), \ .state = VMA_MERGE_START, \ } -#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ - struct vma_merge_struct name = { \ - .mm = vma_->vm_mm, \ - .vmi = vmi_, \ - .prev = prev_, \ - .middle = vma_, \ - .next = NULL, \ - .start = start_, \ - .end = end_, \ - .vm_flags = vma_->vm_flags, \ - .pgoff = vma_pgoff_offset(vma_, start_), \ - .file = vma_->vm_file, \ - .anon_vma = vma_->anon_vma, \ - .policy = vma_policy(vma_), \ - .uffd_ctx = vma_->vm_userfaultfd_ctx, \ - .anon_name = anon_vma_name(vma_), \ - .state = VMA_MERGE_START, \ - } - #ifdef CONFIG_DEBUG_VM_MAPLE_TREE void validate_mm(struct mm_struct *mm); #else @@ -275,8 +413,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, - struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + struct vm_area_struct *vma, unsigned long end); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -310,7 +447,7 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, */ /* Mutable fields. Populated with initial state. */ - vma->vm_pgoff = desc->pgoff; + vma_set_pgoff(vma, desc->pgoff); if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -449,7 +586,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); + pgoff_t anon_pgoff, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +597,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, @@ -484,9 +621,11 @@ static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma } #ifdef CONFIG_MMU -static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, vm_flags_t vm_flags) +static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags) { - return pgprot_modify(oldprot, vm_get_page_prot(vm_flags)); + const pgprot_t prot = vma_flags_to_page_prot(vma_flags); + + return pgprot_modify(oldprot, prot); } #endif @@ -753,4 +892,9 @@ static inline bool map_deny_write_exec(const vma_flags_t *old, } #endif +struct vm_area_struct *__install_special_mapping(struct mm_struct *mm, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, void *priv, + const struct vm_operations_struct *ops); + #endif /* __MM_VMA_H */ diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f..586c52155942 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -1,10 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Functions explicitly implemented for exec functionality which however are - * explicitly VMA-only logic. + * Functions provided for exec functionality which however are + * specifically VMA-only logic. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" @@ -37,7 +41,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) unsigned long new_end = old_end - shift; VMA_ITERATOR(vmi, mm, new_start); VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS, - vma->vm_pgoff); + vma_start_pgoff(vma), vma_start_anon_pgoff(vma)); struct vm_area_struct *next; struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); @@ -89,7 +93,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_end); } /* @@ -108,15 +112,17 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, unsigned long *top_mem_p) { - unsigned long flags = VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP; + vma_flags_t flags = VMA_STACK_INCOMPLETE_SETUP; + struct vm_area_struct *vma; int err; - struct vm_area_struct *vma = vm_area_alloc(mm); + /* VMA_STACK_FLAGS and VMA_STACK_INCOMPLETE_SETUP must not overlap. */ + VM_WARN_ON_ONCE(vma_flags_test_any_mask(&flags, VMA_STACK_FLAGS)); + + vma = vm_area_alloc(mm); if (!vma) return -ENOMEM; - vma_set_anonymous(vma); - if (mmap_write_lock_killable(mm)) { err = -EINTR; goto err_free; @@ -130,19 +136,21 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, if (err) goto err_ksm; + vma_flags_set_mask(&flags, VMA_STACK_FLAGS); + vma_set_anonymous(vma); + /* * Place the stack at the largest stack address the architecture * supports. Later, we'll move this to an appropriate place. We don't * use STACK_TOP because that can depend on attributes which aren't * configured yet. */ - VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; vma->vm_start = vma->vm_end - PAGE_SIZE; if (pgtable_supports_soft_dirty()) - flags |= VM_SOFTDIRTY; - vm_flags_init(vma, flags); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma_flags_set(&flags, VMA_SOFTDIRTY_BIT); + vma->flags = flags; + vma->vm_page_prot = vma_get_page_prot(vma); err = insert_vm_struct(mm, vma); if (err) diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510..710b18849a36 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -5,6 +5,10 @@ * between CONFIG_MMU and non-CONFIG_MMU kernel configurations. */ +/* + * To allow for userland testing we place internal dependencies in + * vma_internal.h and external VMA API declarations in vma.h. + */ #include "vma_internal.h" #include "vma.h" @@ -46,7 +50,8 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + dest->vm_pgoff = vma_start_pgoff(src); + __vma_set_virt_pgoff(dest, vma_start_virt_pgoff(src)); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vma_internal.h b/mm/vma_internal.h index 2da6d224c1a8..385c0ab13777 100644 --- a/mm/vma_internal.h +++ b/mm/vma_internal.h @@ -2,8 +2,8 @@ /* * vma_internal.h * - * Headers required by vma.c, which can be substituted accordingly when testing - * VMA functionality. + * Headers required by vma.c, vma_init.c and vma_exec.c, which can be + * substituted accordingly when testing VMA functionality. */ #ifndef __MM_VMA_INTERNAL_H @@ -23,6 +23,7 @@ #include <linux/ksm.h> #include <linux/khugepaged.h> #include <linux/list.h> +#include <linux/major.h> #include <linux/maple_tree.h> #include <linux/mempolicy.h> #include <linux/mm.h> diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f4fa227a8d7f..26f32949c2f2 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -50,6 +50,7 @@ #include "internal.h" #include "pgalloc-track.h" +#include "vmalloc.h" #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; @@ -92,6 +93,33 @@ struct vfree_deferred { static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); /*** Page table manipulation functions ***/ + +/* + * Try contiguous mappings at the PTE level for arches which support them, and if + * requested by the caller. Fall back to PAGE_SIZE mappings otherwise. + * + * Return: mapping size. + */ +static __always_inline unsigned long vmap_set_ptes(pte_t *pte, + unsigned long addr, unsigned long end, u64 pfn, + pgprot_t prot, unsigned int max_page_shift) +{ +#ifdef CONFIG_HUGETLB_PAGE + unsigned long size; + + size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); + if (size != PAGE_SIZE) { + pte_t entry = pfn_pte(pfn, prot); + + entry = arch_make_huge_pte(entry, ilog2(size), 0); + set_huge_pte_at(&init_mm, addr, pte, entry, size); + return size; + } +#endif + set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); + return PAGE_SIZE; +} + static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift, pgtbl_mod_mask *mask) @@ -99,7 +127,8 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pte_t *pte; u64 pfn; struct page *page; - unsigned long size = PAGE_SIZE; + unsigned long size; + unsigned int steps; if (WARN_ON_ONCE(!PAGE_ALIGNED(end - addr))) return -EINVAL; @@ -120,20 +149,9 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, BUG(); } -#ifdef CONFIG_HUGETLB_PAGE - size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); - if (size != PAGE_SIZE) { - pte_t entry = pfn_pte(pfn, prot); - - entry = arch_make_huge_pte(entry, ilog2(size), 0); - set_huge_pte_at(&init_mm, addr, pte, entry, size); - pfn += PFN_DOWN(size); - continue; - } -#endif - set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); - pfn++; - } while (pte += PFN_DOWN(size), addr += size, addr != end); + size = vmap_set_ptes(pte, addr, end, pfn, prot, max_page_shift); + steps = PFN_DOWN(size); + } while (pte += steps, pfn += steps, addr += size, addr != end); lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED; @@ -546,8 +564,10 @@ void vunmap_range(unsigned long addr, unsigned long end) static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { + unsigned long pfn, size; + unsigned int steps; int err = 0; pte_t *pte; @@ -578,9 +598,10 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, break; } - set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); - (*nr)++; - } while (pte++, addr += PAGE_SIZE, addr != end); + pfn = page_to_pfn(page); + size = vmap_set_ptes(pte, addr, end, pfn, prot, shift); + steps = PFN_DOWN(size); + } while (pte += steps, *nr += steps, addr += size, addr != end); lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED; @@ -590,60 +611,90 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { pmd_t *pmd; unsigned long next; + int err; pmd = pmd_alloc_track(&init_mm, pud, addr, mask); if (!pmd) return -ENOMEM; do { next = pmd_addr_end(addr, end); - if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) - return -ENOMEM; + + if (shift >= PMD_SHIFT) { + struct page *page = pages[*nr]; + phys_addr_t phys_addr; + + if (WARN_ON(!page)) + return -ENOMEM; + if (WARN_ON(!pfn_valid(page_to_pfn(page)))) + return -EINVAL; + + phys_addr = page_to_phys(page); + + if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, + shift)) { + *mask |= PGTBL_PMD_MODIFIED; + *nr += 1 << (PMD_SHIFT - PAGE_SHIFT); + continue; + } + } + + err = vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask, shift); + if (err) + return err; } while (pmd++, addr = next, addr != end); return 0; } static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { pud_t *pud; unsigned long next; + int err; pud = pud_alloc_track(&init_mm, p4d, addr, mask); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); - if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) - return -ENOMEM; + err = vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask, shift); + if (err) + return err; } while (pud++, addr = next, addr != end); return 0; } static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, - pgtbl_mod_mask *mask) + pgtbl_mod_mask *mask, unsigned int shift) { p4d_t *p4d; unsigned long next; + int err; p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end); - if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) - return -ENOMEM; + err = vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask, shift); + if (err) + return err; } while (p4d++, addr = next, addr != end); return 0; } -static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages) +/* + * It can take an array of pages which are not all contiguous, but it + * may have contiguous chunks, as hinted by @shift. + */ +static int vmap_pages_range_noflush_walk(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, unsigned int shift) { unsigned long start = addr; pgd_t *pgd; @@ -658,7 +709,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, next = pgd_addr_end(addr, end); if (pgd_bad(*pgd)) mask |= PGTBL_PGD_MODIFIED; - err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); + err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask, shift); if (err) break; } while (pgd++, addr = next, addr != end); @@ -681,27 +732,12 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift) { - unsigned int i, nr = (end - addr) >> PAGE_SHIFT; - WARN_ON(page_shift < PAGE_SHIFT); - if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || - page_shift == PAGE_SHIFT) - return vmap_small_pages_range_noflush(addr, end, prot, pages); + if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC)) + page_shift = PAGE_SHIFT; - for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { - int err; - - err = vmap_range_noflush(addr, addr + (1UL << page_shift), - page_to_phys(pages[i]), prot, - page_shift); - if (err) - return err; - - addr += 1UL << page_shift; - } - - return 0; + return vmap_pages_range_noflush_walk(addr, end, prot, pages, page_shift); } int vmap_pages_range_noflush(unsigned long addr, unsigned long end, @@ -3301,6 +3337,14 @@ struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, NUMA_NO_NODE, GFP_KERNEL, caller); } +static struct vm_struct *__get_vm_area_node_aligned_caller(unsigned long size, + unsigned long align, unsigned long flags, const void *caller) +{ + return __get_vm_area_node(size, align, PAGE_SHIFT, flags, + VMALLOC_START, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); +} + /** * find_vm_area - find a continuous kernel virtual area * @addr: base address @@ -3541,6 +3585,116 @@ void vunmap(const void *addr) } EXPORT_SYMBOL(vunmap); +static inline unsigned int vm_shift(pgprot_t prot, unsigned long size) +{ + if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) + return PMD_SHIFT; + + return arch_vmap_pte_supported_shift(size); +} + +static inline int get_vmap_batch_order(struct page **pages, + pgprot_t prot, unsigned int nr_pages) +{ + unsigned long pfn; + unsigned int nr_contig; + int order; + + if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) + return 0; + + /* Limit nr_pages by pfn alignment */ + pfn = page_to_pfn(*pages); + if (pfn > 0) + nr_pages = min_t(unsigned int, nr_pages, 1UL << __ffs(pfn)); + + nr_contig = num_pages_contiguous(pages, nr_pages); + if (nr_contig < 2) + return 0; + + order = ilog2(nr_contig); + + if (vm_shift(prot, PAGE_SIZE << order) == PAGE_SHIFT) + return 0; + + return order; +} + +static int vmap_pages_range_batched(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages) +{ + const unsigned int nr_pages = (end - addr) >> PAGE_SHIFT; + unsigned int prev_shift = 0, batch_start = 0; + unsigned long map_addr = addr, batch_end = addr; + int err; + + err = kmsan_vmap_pages_range_noflush(addr, end, prot, pages, + PAGE_SHIFT, GFP_KERNEL); + if (err) + goto out; + + for (unsigned int i = 0; i < nr_pages; ) { + unsigned int shift = PAGE_SHIFT + + get_vmap_batch_order(pages + i, prot, nr_pages - i); + + if (!i) + prev_shift = shift; + + if (shift != prev_shift) { + err = vmap_pages_range_noflush_walk(map_addr, batch_end, + prot, pages + batch_start, prev_shift); + if (err) + goto out; + prev_shift = shift; + map_addr = batch_end; + batch_start = i; + } + + /* + * Once we fail to batch pages, we expect to fail batching + * for all remaining pages, so just give up. + */ + if (shift == PAGE_SHIFT) + break; + + batch_end += 1UL << shift; + i += 1U << (shift - PAGE_SHIFT); + } + + /* Remaining */ + if (map_addr < end) + err = vmap_pages_range_noflush_walk(map_addr, end, prot, + pages + batch_start, prev_shift); + +out: + flush_cache_vmap(addr, end); + return err; +} + +static struct vm_struct *vmap_get_aligned_vm_area(unsigned long size, + unsigned long flags, pgprot_t prot, const void *caller) +{ + struct vm_struct *vm_area; + unsigned int shift; + + if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) { + vm_area = __get_vm_area_node_aligned_caller(size, PMD_SIZE, + flags, caller); + if (vm_area) + return vm_area; + } + + shift = arch_vmap_pte_supported_shift(size); + if (shift > PAGE_SHIFT) { + vm_area = __get_vm_area_node_aligned_caller(size, 1UL << shift, + flags, caller); + if (vm_area) + return vm_area; + } + + return __get_vm_area_node_aligned_caller(size, PAGE_SIZE, flags, caller); +} + /** * vmap - map an array of pages into virtually contiguous space * @pages: array of page pointers @@ -3579,13 +3733,14 @@ void *vmap(struct page **pages, unsigned int count, return NULL; size = (unsigned long)count << PAGE_SHIFT; - area = get_vm_area_caller(size, flags, __builtin_return_address(0)); + area = vmap_get_aligned_vm_area(size, flags, prot, + __builtin_return_address(0)); if (!area) return NULL; addr = (unsigned long)area->addr; - if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), - pages, PAGE_SHIFT) < 0) { + if (vmap_pages_range_batched(addr, addr + size, pgprot_nx(prot), + pages) < 0) { vunmap(area->addr); return NULL; } @@ -4052,11 +4207,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, * supporting them. */ - if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) - shift = PMD_SHIFT; - else - shift = arch_vmap_pte_supported_shift(size); - + shift = vm_shift(prot, size); align = max(original_align, 1UL << shift); } @@ -4067,8 +4218,8 @@ again: if (!area) { bool nofail = gfp_mask & __GFP_NOFAIL; warn_alloc(gfp_mask, NULL, - "vmalloc error: size %lu, vm_struct allocation failed%s", - size, (nofail) ? ". Retrying." : ""); + "vmalloc error: size %lu, align 0x%lx, vm_struct allocation failed%s", + size, align, (nofail) ? ". Retrying." : ""); if (nofail) { schedule_timeout_uninterruptible(1); goto again; @@ -4968,16 +5119,17 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) * @sizes: array containing size of each area * @nr_vms: the number of areas to allocate * @align: alignment, all entries in @offsets and @sizes must be aligned to this + * @gfp: allocation flags passed to the underlying memory allocator * * Returns: kmalloc'd vm_struct pointer array pointing to allocated * vm_structs on success, %NULL on failure * * Percpu allocator wants to use congruent vm areas so that it can * maintain the offsets among percpu areas. This function allocates - * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to - * be scattered pretty far, distance between two areas easily going up - * to gigabytes. To avoid interacting with regular vmallocs, these - * areas are allocated from top. + * congruent vmalloc areas for it. These areas tend to be scattered + * pretty far, distance between two areas easily going up to gigabytes. + * To avoid interacting with regular vmallocs, these areas are allocated + * from top. * * Despite its complicated look, this allocator is rather simple. It * does everything top-down and scans free blocks from the end looking @@ -4988,7 +5140,7 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) */ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); @@ -5026,14 +5178,14 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, return NULL; } - vms = kzalloc_objs(vms[0], nr_vms); - vas = kzalloc_objs(vas[0], nr_vms); + vms = kzalloc_objs(vms[0], nr_vms, gfp); + vas = kzalloc_objs(vas[0], nr_vms, gfp); if (!vas || !vms) goto err_free2; for (area = 0; area < nr_vms; area++) { - vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); - vms[area] = kzalloc_obj(struct vm_struct); + vas[area] = kmem_cache_zalloc(vmap_area_cachep, gfp); + vms[area] = kzalloc_obj(struct vm_struct, gfp); if (!vas[area] || !vms[area]) goto err_free; } @@ -5123,7 +5275,7 @@ retry: /* populate the kasan shadow space */ for (area = 0; area < nr_vms; area++) { - if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], GFP_KERNEL)) + if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], gfp)) goto err_free_shadow; } @@ -5180,7 +5332,7 @@ overflow: continue; vas[area] = kmem_cache_zalloc( - vmap_area_cachep, GFP_KERNEL); + vmap_area_cachep, gfp); if (!vas[area]) goto err_free; } @@ -5220,9 +5372,7 @@ err_free_shadow: kfree(vms[area]); } spin_unlock(&free_vmap_area_lock); - kfree(vas); - kfree(vms); - return NULL; + goto err_free2; } /** diff --git a/mm/vmalloc.h b/mm/vmalloc.h new file mode 100644 index 000000000000..dcfe30eaa80c --- /dev/null +++ b/mm/vmalloc.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal APIs for vmalloc + */ +#ifndef __MM_VMALLOC_H +#define __MM_VMALLOC_H + +#ifdef CONFIG_MMU +void __init vmalloc_init(void); +int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift, gfp_t gfp_mask); +unsigned int get_vm_area_page_order(struct vm_struct *vm); +#else +static inline void vmalloc_init(void) {} + +static inline +int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift, gfp_t gfp_mask) +{ + return -EINVAL; +} + +static inline void vunmap_range_noflush(unsigned long start, unsigned long end) +{ +} +#endif + +struct vm_struct *__get_vm_area_node(unsigned long size, + unsigned long align, unsigned long shift, + unsigned long vm_flags, unsigned long start, + unsigned long end, int node, gfp_t gfp_mask, + const void *caller); + +void clear_vm_uninitialized_flag(struct vm_struct *vm); + +int __must_check __vmap_pages_range_noflush(unsigned long addr, + unsigned long end, pgprot_t prot, + struct page **pages, unsigned int page_shift); + +void vunmap_range_noflush(unsigned long start, unsigned long end); + +void __vunmap_range_noflush(unsigned long start, unsigned long end); + +#endif /* __MM_VMALLOC_H */ diff --git a/mm/vmpressure.c b/mm/vmpressure.c index f053554e5826..9629240d77ad 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -7,16 +7,15 @@ * * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro, * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg. + * + * Tree-mode (cgroup v1 userspace eventfd) bookkeeping lives in + * mm/memcontrol-v1.c; this file holds the shared code and the in-kernel + * (tree=false) socket-pressure path that runs on cgroup v2. */ #include <linux/cgroup.h> -#include <linux/fs.h> #include <linux/log2.h> -#include <linux/sched.h> #include <linux/mm.h> -#include <linux/vmstat.h> -#include <linux/eventfd.h> -#include <linux/slab.h> #include <linux/swap.h> #include <linux/printk.h> #include <linux/vmpressure.h> @@ -35,7 +34,7 @@ * TODO: Make the window size depend on machine size, as we do for vmstat * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). */ -static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; +const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; /* * These thresholds are used when we account memory pressure through @@ -46,68 +45,6 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; static const unsigned int vmpressure_level_med = 60; static const unsigned int vmpressure_level_critical = 95; -/* - * When there are too little pages left to scan, vmpressure() may miss the - * critical pressure as number of pages will be less than "window size". - * However, in that case the vmscan priority will raise fast as the - * reclaimer will try to scan LRUs more deeply. - * - * The vmscan logic considers these special priorities: - * - * prio == DEF_PRIORITY (12): reclaimer starts with that value - * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed - * prio == 0 : close to OOM, kernel scans every page in an lru - * - * Any value in this range is acceptable for this tunable (i.e. from 12 to - * 0). Current value for the vmpressure_level_critical_prio is chosen - * empirically, but the number, in essence, means that we consider - * critical level when scanning depth is ~10% of the lru size (vmscan - * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one - * eights). - */ -static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); - -static struct vmpressure *work_to_vmpressure(struct work_struct *work) -{ - return container_of(work, struct vmpressure, work); -} - -static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) -{ - struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); - - memcg = parent_mem_cgroup(memcg); - if (!memcg) - return NULL; - return memcg_to_vmpressure(memcg); -} - -enum vmpressure_levels { - VMPRESSURE_LOW = 0, - VMPRESSURE_MEDIUM, - VMPRESSURE_CRITICAL, - VMPRESSURE_NUM_LEVELS, -}; - -enum vmpressure_modes { - VMPRESSURE_NO_PASSTHROUGH = 0, - VMPRESSURE_HIERARCHY, - VMPRESSURE_LOCAL, - VMPRESSURE_NUM_MODES, -}; - -static const char * const vmpressure_str_levels[] = { - [VMPRESSURE_LOW] = "low", - [VMPRESSURE_MEDIUM] = "medium", - [VMPRESSURE_CRITICAL] = "critical", -}; - -static const char * const vmpressure_str_modes[] = { - [VMPRESSURE_NO_PASSTHROUGH] = "default", - [VMPRESSURE_HIERARCHY] = "hierarchy", - [VMPRESSURE_LOCAL] = "local", -}; - static enum vmpressure_levels vmpressure_level(unsigned long pressure) { if (pressure >= vmpressure_level_critical) @@ -117,8 +54,8 @@ static enum vmpressure_levels vmpressure_level(unsigned long pressure) return VMPRESSURE_LOW; } -static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, - unsigned long reclaimed) +enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed) { unsigned long scale = scanned + reclaimed; unsigned long pressure = 0; @@ -147,74 +84,6 @@ out: return vmpressure_level(pressure); } -struct vmpressure_event { - struct eventfd_ctx *efd; - enum vmpressure_levels level; - enum vmpressure_modes mode; - struct list_head node; -}; - -static bool vmpressure_event(struct vmpressure *vmpr, - const enum vmpressure_levels level, - bool ancestor, bool signalled) -{ - struct vmpressure_event *ev; - bool ret = false; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ancestor && ev->mode == VMPRESSURE_LOCAL) - continue; - if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) - continue; - if (level < ev->level) - continue; - eventfd_signal(ev->efd); - ret = true; - } - mutex_unlock(&vmpr->events_lock); - - return ret; -} - -static void vmpressure_work_fn(struct work_struct *work) -{ - struct vmpressure *vmpr = work_to_vmpressure(work); - unsigned long scanned; - unsigned long reclaimed; - enum vmpressure_levels level; - bool ancestor = false; - bool signalled = false; - - spin_lock(&vmpr->sr_lock); - /* - * Several contexts might be calling vmpressure(), so it is - * possible that the work was rescheduled again before the old - * work context cleared the counters. In that case we will run - * just after the old work returns, but then scanned might be zero - * here. No need for any locks here since we don't care if - * vmpr->reclaimed is in sync. - */ - scanned = vmpr->tree_scanned; - if (!scanned) { - spin_unlock(&vmpr->sr_lock); - return; - } - - reclaimed = vmpr->tree_reclaimed; - vmpr->tree_scanned = 0; - vmpr->tree_reclaimed = 0; - spin_unlock(&vmpr->sr_lock); - - level = vmpressure_calc_level(scanned, reclaimed); - - do { - if (vmpressure_event(vmpr, level, ancestor, signalled)) - signalled = true; - ancestor = true; - } while ((vmpr = vmpressure_parent(vmpr))); -} - /** * vmpressure() - Account memory pressure through scanned/reclaimed ratio * @gfp: reclaimer's gfp mask @@ -246,11 +115,12 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; /* - * The in-kernel users only care about the reclaim efficiency - * for this @memcg rather than the whole subtree, and there - * isn't and won't be any in-kernel user in a legacy cgroup. + * Only two combinations have a consumer: + * cgroup v2 + tree=false -> in-kernel socket pressure + * cgroup v1 + tree=true -> userspace eventfds (memory.pressure_level) + * Skip the other two: nothing consumes the result. */ - if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !tree) + if (cgroup_subsys_on_dfl(memory_cgrp_subsys) == tree) return; vmpr = memcg_to_vmpressure(memcg); @@ -281,14 +151,7 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; if (tree) { - spin_lock(&vmpr->sr_lock); - scanned = vmpr->tree_scanned += scanned; - vmpr->tree_reclaimed += reclaimed; - spin_unlock(&vmpr->sr_lock); - - if (scanned < vmpressure_win) - return; - schedule_work(&vmpr->work); + vmpressure_v1_account_tree(vmpr, scanned, reclaimed); } else { enum vmpressure_levels level; @@ -331,134 +194,6 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, } /** - * vmpressure_prio() - Account memory pressure through reclaimer priority level - * @gfp: reclaimer's gfp mask - * @memcg: cgroup memory controller handle - * @prio: reclaimer's priority - * - * This function should be called from the reclaim path every time when - * the vmscan's reclaiming priority (scanning depth) changes. - * - * This function does not return any value. - */ -void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) -{ - /* - * We only use prio for accounting critical level. For more info - * see comment for vmpressure_level_critical_prio variable above. - */ - if (prio > vmpressure_level_critical_prio) - return; - - /* - * OK, the prio is below the threshold, updating vmpressure - * information before shrinker dives into long shrinking of long - * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 - * to the vmpressure() basically means that we signal 'critical' - * level. - */ - vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); -} - -#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) - -/** - * vmpressure_register_event() - Bind vmpressure notifications to an eventfd - * @memcg: memcg that is interested in vmpressure notifications - * @eventfd: eventfd context to link notifications with - * @args: event arguments (pressure level threshold, optional mode) - * - * This function associates eventfd context with the vmpressure - * infrastructure, so that the notifications will be delivered to the - * @eventfd. The @args parameter is a comma-delimited string that denotes a - * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", - * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. - * "hierarchy" or "local"). - * - * To be used as memcg event method. - * - * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could - * not be parsed. - */ -int vmpressure_register_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd, const char *args) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; - enum vmpressure_levels level; - char *spec, *spec_orig; - char *token; - int ret = 0; - - spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); - if (!spec) - return -ENOMEM; - - /* Find required level */ - token = strsep(&spec, ","); - ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); - if (ret < 0) - goto out; - level = ret; - - /* Find optional mode */ - token = strsep(&spec, ","); - if (token) { - ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); - if (ret < 0) - goto out; - mode = ret; - } - - ev = kzalloc_obj(*ev); - if (!ev) { - ret = -ENOMEM; - goto out; - } - - ev->efd = eventfd; - ev->level = level; - ev->mode = mode; - - mutex_lock(&vmpr->events_lock); - list_add(&ev->node, &vmpr->events); - mutex_unlock(&vmpr->events_lock); - ret = 0; -out: - kfree(spec_orig); - return ret; -} - -/** - * vmpressure_unregister_event() - Unbind eventfd from vmpressure - * @memcg: memcg handle - * @eventfd: eventfd context that was used to link vmpressure with the @cg - * - * This function does internal manipulations to detach the @eventfd from - * the vmpressure notifications, and then frees internal resources - * associated with the @eventfd (but the @eventfd itself is not freed). - * - * To be used as memcg event method. - */ -void vmpressure_unregister_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ev->efd != eventfd) - continue; - list_del(&ev->node); - kfree(ev); - break; - } - mutex_unlock(&vmpr->events_lock); -} - -/** * vmpressure_init() - Initialize vmpressure control structure * @vmpr: Structure to be initialized * @@ -468,9 +203,7 @@ void vmpressure_unregister_event(struct mem_cgroup *memcg, void vmpressure_init(struct vmpressure *vmpr) { spin_lock_init(&vmpr->sr_lock); - mutex_init(&vmpr->events_lock); - INIT_LIST_HEAD(&vmpr->events); - INIT_WORK(&vmpr->work, vmpressure_work_fn); + vmpressure_v1_init(vmpr); } /** @@ -482,9 +215,5 @@ void vmpressure_init(struct vmpressure *vmpr) */ void vmpressure_cleanup(struct vmpressure *vmpr) { - /* - * Make sure there is no pending work before eventfd infrastructure - * goes away. - */ - flush_work(&vmpr->work); + vmpressure_v1_cleanup(vmpr); } diff --git a/mm/vmscan.c b/mm/vmscan.c index 56708d1d2dfd..9b8ee902f972 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -66,6 +66,7 @@ #include <linux/sched/sysctl.h> #include "internal.h" +#include "page_alloc.h" #include "swap.h" #define CREATE_TRACE_POINTS @@ -79,7 +80,7 @@ struct scan_control { * Nodemask of nodes allowed by the caller. If NULL, all nodes * are scanned. */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* * The memory cgroup that hit its limit and as a result is the @@ -267,6 +268,12 @@ static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) } #endif +static inline bool is_exec_file_folio(const struct folio *folio, + const vma_flags_t *vma_flags) +{ + return vma_flags_test(vma_flags, VMA_EXEC_BIT) && folio_is_file_lru(folio); +} + static void set_task_reclaim_state(struct task_struct *task, struct reclaim_state *rs) { @@ -615,8 +622,8 @@ typedef enum { /* * pageout is called by shrink_folio_list() for each dirty folio. */ -static pageout_t pageout(struct folio *folio, struct address_space *mapping, - struct swap_iocb **plug, struct list_head *folio_list) +static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping, + struct folio *folio, struct list_head *folio_list) { int res; @@ -652,9 +659,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping, * the split out folios get added back to folio_list. */ if (shmem_mapping(mapping)) - res = shmem_writeout(folio, plug, folio_list); + res = shmem_writeout(ctx, folio, folio_list); else - res = swap_writeout(folio, plug); + res = swap_writeout(ctx, folio); if (res < 0) handle_write_error(mapping, folio, res); @@ -823,7 +830,6 @@ void folio_putback_lru(struct folio *folio) enum folio_references { FOLIOREF_RECLAIM, - FOLIOREF_RECLAIM_CLEAN, FOLIOREF_KEEP, FOLIOREF_ACTIVATE, }; @@ -835,10 +841,16 @@ enum folio_references { * with PG_active set. In contrast, the aging (page table walk) path uses * folio_update_gen(). */ -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* Activate file-backed executable folios after first usage. */ + if (is_exec_file_folio(folio, vma_flags)) { + set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + return true; + } + set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return false; } @@ -851,7 +863,7 @@ static bool lru_gen_set_refs(struct folio *folio) return true; } #else -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { return false; } @@ -861,16 +873,16 @@ static enum folio_references folio_check_references(struct folio *folio, struct scan_control *sc) { int referenced_ptes, referenced_folio; - vm_flags_t vm_flags; + vma_flags_t vma_flags; referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, - &vm_flags); + &vma_flags); /* * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. * Let the folio, now marked Mlocked, be moved to the unevictable list. */ - if (vm_flags & VM_LOCKED) + if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) return FOLIOREF_ACTIVATE; /* @@ -886,7 +898,7 @@ static enum folio_references folio_check_references(struct folio *folio, if (!referenced_ptes) return FOLIOREF_RECLAIM; - return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; + return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; } referenced_folio = folio_test_clear_referenced(folio); @@ -914,16 +926,12 @@ static enum folio_references folio_check_references(struct folio *folio, /* * Activate file-backed executable folios after first usage. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) + if (is_exec_file_folio(folio, &vma_flags)) return FOLIOREF_ACTIVATE; return FOLIOREF_KEEP; } - /* Reclaim if clean, defer dirty folios to writeback */ - if (referenced_folio && folio_is_file_lru(folio)) - return FOLIOREF_RECLAIM_CLEAN; - return FOLIOREF_RECLAIM; } @@ -1037,16 +1045,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) { if (gfp_mask & __GFP_FS) return true; - if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) - return false; /* - * We can "enter_fs" for swap-cache with only __GFP_IO - * providing this isn't SWP_FS_OPS. - * ->flags can be updated non-atomically, - * but that will never affect SWP_FS_OPS, so the data_race - * is safe. + * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by + * a swapfile that requires GFP_NOFS I/O. */ - return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); + if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) && + !(__swap_entry_to_info(folio->swap)->ops->flags & + SWAP_OPS_F_REQUIRE_NOFS)) + return true; + return false; } /* @@ -1063,7 +1070,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, unsigned int nr_reclaimed = 0, nr_demoted = 0; unsigned int pgactivate = 0; bool do_demote_pass; - struct swap_iocb *plug = NULL; + struct swap_io_ctx ctx = {}; folio_batch_init(&free_folios); memset(stat, 0, sizeof(*stat)); @@ -1235,7 +1242,6 @@ retry: stat->nr_ref_keep += nr_pages; goto keep_locked; case FOLIOREF_RECLAIM: - case FOLIOREF_RECLAIM_CLEAN: ; /* try to reclaim the folio below */ } @@ -1381,8 +1387,6 @@ retry: goto activate_locked; } - if (references == FOLIOREF_RECLAIM_CLEAN) - goto keep_locked; if (!may_enter_fs(folio, sc->gfp_mask)) goto keep_locked; if (!sc->may_writepage) @@ -1394,7 +1398,7 @@ retry: * starts and then write it out here. */ try_to_unmap_flush_dirty(); - switch (pageout(folio, mapping, &plug, folio_list)) { + switch (pageout(&ctx, mapping, folio, folio_list)) { case PAGE_KEEP: goto keep_locked; case PAGE_ACTIVATE: @@ -1584,8 +1588,7 @@ keep: list_splice(&ret_folios, folio_list); count_vm_events(PGACTIVATE, pgactivate); - if (plug) - swap_write_unplug(plug); + swap_write_submit(&ctx); return nr_reclaimed; } @@ -2072,7 +2075,7 @@ static void shrink_active_list(unsigned long nr_to_scan, { unsigned long nr_taken; unsigned long nr_scanned; - vm_flags_t vm_flags; + vma_flags_t vma_flags; LIST_HEAD(l_hold); /* The folios which were snipped off */ LIST_HEAD(l_active); LIST_HEAD(l_inactive); @@ -2116,7 +2119,7 @@ static void shrink_active_list(unsigned long nr_to_scan, /* Referenced or rmap lock contention: rotate */ if (folio_referenced(folio, 0, sc->target_mem_cgroup, - &vm_flags) != 0) { + &vma_flags) != 0) { /* * Identify referenced, file-backed active folios and * give them one more trip around the active list. So @@ -2126,7 +2129,7 @@ static void shrink_active_list(unsigned long nr_to_scan, * IO, plus JVM can create lots of anon VM_EXEC folios, * so we ignore them here. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { + if (is_exec_file_folio(folio, &vma_flags)) { nr_rotated += folio_nr_pages(folio); list_add(&folio->lru, &l_active); continue; @@ -2501,6 +2504,17 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, enum scan_balance scan_balance; enum lru_list lru; + /* Proactive reclaim initiated by userspace for anonymous memory only */ + if (swappiness == SWAPPINESS_ANON_ONLY) { + WARN_ON_ONCE(!sc->proactive); + if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { + memset(nr, 0, sizeof(*nr) * NR_LRU_LISTS); + return; + } + scan_balance = SCAN_ANON; + goto out; + } + /* If we have no swap space, do not bother scanning anon folios. */ if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { scan_balance = SCAN_FILE; @@ -2519,13 +2533,6 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } - /* Proactive reclaim initiated by userspace for anonymous memory only */ - if (swappiness == SWAPPINESS_ANON_ONLY) { - WARN_ON_ONCE(!sc->proactive); - scan_balance = SCAN_ANON; - goto out; - } - /* * Do not apply any pressure balancing cleverness when the * system is close to OOM, scan both anon and file equally @@ -3195,14 +3202,19 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen) +static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) { unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* + * See the comment on LRU_REFS_FLAGS, and activate file-backed + * executable folios after first usage to avoid typical IO + * thrashing from reclaiming. + */ + if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && + !is_exec_file_folio(folio, vma_flags)) { set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return -1; } @@ -3435,8 +3447,8 @@ static bool suitable_to_scan(int total, int young) return young * n >= total; } -static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, - int new_gen, bool dirty) +static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, + struct folio *folio, int new_gen, bool dirty) { int old_gen; @@ -3449,10 +3461,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen); + old_gen = folio_update_gen(folio, new_gen, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) update_batch_size(walk, folio, old_gen, new_gen); - } else if (lru_gen_set_refs(folio)) { + } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) folio_activate(folio); @@ -3525,7 +3537,7 @@ restart: continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = folio; dirty = false; @@ -3538,7 +3550,7 @@ restart: walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3616,7 +3628,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area goto next; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -3630,7 +3642,7 @@ next: i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -4265,7 +4277,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -4277,7 +4289,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) young += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); @@ -6594,7 +6606,7 @@ static bool allow_direct_reclaim(pg_data_t *pgdat) * happens, the page allocator should not consider triggering the OOM killer. */ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, - nodemask_t *nodemask) + const nodemask_t *nodemask) { struct zoneref *z; struct zone *zone; @@ -6674,7 +6686,7 @@ out: } unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *nodemask) + gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned long nr_reclaimed; struct scan_control sc = { @@ -7654,7 +7666,6 @@ static int __init kswapd_init(void) { int nid; - swap_setup(); for_each_node_state(nid, N_MEMORY) kswapd_run(nid); register_sysctl_init("vm", vmscan_sysctl_table); @@ -7738,7 +7749,7 @@ static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) /* * Try to free up some pages from this node through reclaim. */ -static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, +static unsigned long __node_reclaim(struct pglist_data *pgdat, unsigned long nr_pages, struct scan_control *sc) { @@ -7781,9 +7792,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, return sc->nr_reclaimed; } -int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) +unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) { - int ret; + unsigned long ret; /* Minimum pages needed in order to stay on node */ const unsigned long nr_pages = 1 << order; struct scan_control sc = { @@ -7810,13 +7821,13 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= pgdat->min_slab_pages) - return NODE_RECLAIM_FULL; + return 0; /* * Do not scan if the allocation should not be delayed. */ if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) - return NODE_RECLAIM_NOSCAN; + return 0; /* * Only run node reclaim on the local node or on nodes that do not @@ -7825,15 +7836,15 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) * as wide as possible. */ if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) - return NODE_RECLAIM_NOSCAN; + return 0; if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) - return NODE_RECLAIM_NOSCAN; + return 0; - ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc) >= nr_pages; + ret = __node_reclaim(pgdat, nr_pages, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); - if (ret) + if (ret >= nr_pages) count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS); else count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); @@ -7843,7 +7854,7 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) #else -static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, +static unsigned long __node_reclaim(struct pglist_data *pgdat, unsigned long nr_pages, struct scan_control *sc) { @@ -7919,6 +7930,10 @@ int user_proactive_reclaim(char *buf, if (signal_pending(current)) return -ERESTARTSYS; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg && memcg_is_dying(memcg)) + return -EAGAIN; + /* * This is the final attempt, drain percpu lru caches in the * hope of introducing more evictable pages. @@ -7952,8 +7967,7 @@ int user_proactive_reclaim(char *buf, &pgdat->flags)) return -EBUSY; - reclaimed = __node_reclaim(pgdat, gfp_mask, - batch_size, &sc); + reclaimed = __node_reclaim(pgdat, batch_size, &sc); clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); } @@ -8020,7 +8034,7 @@ static ssize_t reclaim_store(struct device *dev, int ret, nid = dev->id; ret = user_proactive_reclaim((char *)buf, NULL, NODE_DATA(nid)); - return ret ? -EAGAIN : count; + return ret ? ret : count; } static DEVICE_ATTR_WO(reclaim); diff --git a/mm/vmstat.c b/mm/vmstat.c index f534972f517d..4e26e5fd6666 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -30,6 +30,7 @@ #include <linux/sched/isolation.h> #include "internal.h" +#include "page_alloc.h" #ifdef CONFIG_PROC_FS #ifdef CONFIG_NUMA @@ -1488,7 +1489,11 @@ const char * const vmstat_text[] = { #if THREAD_SIZE > 65536 [I(KSTACK_REST)] = "kstack_rest", #endif -#endif +#endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + [I(NRSWPIN)] = "nrswpin", + [I(NRSWPOUT)] = "nrswpout", +#endif /* CONFIG_SWAP */ #undef I #endif /* CONFIG_VM_EVENT_COUNTERS */ }; @@ -1568,7 +1573,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, static int frag_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; - walk_zones_in_node(m, pgdat, true, false, frag_show_print); + walk_zones_in_node(m, pgdat, true, true, frag_show_print); return 0; } diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 83f5820c45f9..3d566a3ee96f 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -21,6 +21,10 @@ * pool->lock * class->lock * zspage->lock + * + * When ZS_OBJ_CLASS_BITS > 0, zs_free() skips pool->lock; it picks + * the size_class from obj's encoded class_idx and serializes against + * page migration via class->lock. */ #include <linux/module.h> @@ -67,8 +71,8 @@ #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS #else /* - * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just - * be PAGE_SHIFT + * If this definition of MAX_PHYSMEM_BITS is used, ZS_OBJ_PFN_SHIFT will + * just be PAGE_SHIFT */ #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG #endif @@ -88,8 +92,23 @@ #define OBJ_TAG_BITS 1 #define OBJ_TAG_MASK OBJ_ALLOCATED_TAG -#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS) -#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1) +/* + * obj is encoded as [PFN | class_idx | obj_idx] within an unsigned long: + * + * |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->| + * +-----------------+-------------------------+-----------------------+ + * | PFN | class_idx | obj_idx | + * +-----------------+-------------------------+-----------------------+ + * MSB ^ LSB + * | + * +-- ZS_OBJ_PFN_SHIFT + * + * Encoding class_idx into obj lets zs_free() locate the size_class + * without holding pool->lock; class_idx is invariant across page + * migration (only PFN changes), so a lockless read of the obj value + * always yields a valid class_idx. + */ +#define ZS_OBJ_PFN_SHIFT (BITS_PER_LONG - _PFN_BITS) #define HUGE_BITS 1 #define FULLNESS_BITS 4 @@ -98,9 +117,61 @@ #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL)) +/* + * Bits to index a page within a zspage = ceil(log2(ZS_MAX_PAGES_PER_ZSPAGE)). + * Computed at preprocessor time, for use in #if below. Kconfig + * restricts ZSMALLOC_CHAIN_SIZE to [4, 16]. + */ +#if ZS_MAX_PAGES_PER_ZSPAGE <= 4 +#define ZS_PAGES_PER_ZSPAGE_BITS 2 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 8 +#define ZS_PAGES_PER_ZSPAGE_BITS 3 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 16 +#define ZS_PAGES_PER_ZSPAGE_BITS 4 +#else +#error "ZSMALLOC_CHAIN_SIZE out of expected range [4,16]" +#endif + +/* + * Bits to index an object within a single PAGE_SIZE at the smallest + * possible object size: log2(PAGE_SIZE / 32) = PAGE_SHIFT - 5. + * 32 is the hard floor of ZS_MIN_ALLOC_SIZE. + */ +#define ZS_OBJS_PER_PAGE_BITS (PAGE_SHIFT - 5) + +/* + * Bits to index any object in the densest possible zspage. Below this, + * ZS_MIN_ALLOC_SIZE is auto-raised by the MAX(32, ...) formula -- still + * correct, but objects are coarser. + */ +#define ZS_OBJS_PER_ZSPAGE_BITS \ + (ZS_PAGES_PER_ZSPAGE_BITS + ZS_OBJS_PER_PAGE_BITS) + +/* + * Encode class_idx only when obj has spare bits; otherwise + * ZS_OBJ_CLASS_BITS folds to 0 (32-bit, or 64-bit UML/fallback). + */ +#if BITS_PER_LONG >= 64 && \ + ZS_OBJ_PFN_SHIFT >= (CLASS_BITS + 1) + ZS_OBJS_PER_ZSPAGE_BITS +#define ZS_OBJ_CLASS_BITS (CLASS_BITS + 1) +#else +#define ZS_OBJ_CLASS_BITS 0 +#endif +#define ZS_OBJ_CLASS_MASK ((_AC(1, UL) << ZS_OBJ_CLASS_BITS) - 1) + +#define ZS_OBJ_IDX_BITS (ZS_OBJ_PFN_SHIFT - ZS_OBJ_CLASS_BITS) +#define ZS_OBJ_IDX_MASK ((_AC(1, UL) << ZS_OBJ_IDX_BITS) - 1) + +/* + * Belt-and-suspenders: the #if above already guarantees this when + * class_idx is enabled. Catches future tweaks that bypass it. + */ +static_assert(ZS_OBJ_IDX_BITS >= ZS_PAGES_PER_ZSPAGE_BITS, + "zsmalloc: ZS_MIN_ALLOC_SIZE would exceed ZS_MAX_ALLOC_SIZE"); + /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */ #define ZS_MIN_ALLOC_SIZE \ - MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS)) + MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> ZS_OBJ_IDX_BITS)) /* each chunk includes extra space to keep handle */ #define ZS_MAX_ALLOC_SIZE PAGE_SIZE @@ -396,10 +467,13 @@ static void cache_free_zspage(struct zspage *zspage) kmem_cache_free(zspage_cachep, zspage); } -/* class->lock(which owns the handle) synchronizes races */ +/* + * Pairs with READ_ONCE() in handle_to_obj(): zs_free() may read the + * handle locklessly, so prevent store tearing here. + */ static void record_obj(unsigned long handle, unsigned long obj) { - *(unsigned long *)handle = obj; + WRITE_ONCE(*(unsigned long *)handle, obj); } static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc) @@ -720,33 +794,36 @@ static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc) static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc, unsigned int *obj_idx) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); - *obj_idx = (obj & OBJ_INDEX_MASK); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); + *obj_idx = (obj & ZS_OBJ_IDX_MASK); } static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); } /** - * location_to_obj - get obj value encoded from (<zpdesc>, <obj_idx>) + * location_to_obj - encode (<zpdesc>, <obj_idx>, <class_idx>) into obj value * @zpdesc: zpdesc object resides in zspage * @obj_idx: object index + * @class_idx: size class index; ignored when ZS_OBJ_CLASS_BITS == 0 */ -static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx) +static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx, + unsigned int class_idx) { unsigned long obj; - obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS; - obj |= obj_idx & OBJ_INDEX_MASK; + obj = zpdesc_pfn(zpdesc) << ZS_OBJ_PFN_SHIFT; + obj |= (unsigned long)(class_idx & ZS_OBJ_CLASS_MASK) << ZS_OBJ_IDX_BITS; + obj |= obj_idx & ZS_OBJ_IDX_MASK; return obj; } static unsigned long handle_to_obj(unsigned long handle) { - return *(unsigned long *)handle; + return READ_ONCE(*(unsigned long *)handle); } static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, @@ -800,13 +877,26 @@ unlock: return 0; } -static void __free_zspage(struct zs_pool *pool, struct size_class *class, - struct zspage *zspage) +/* + * Three free helpers, kept apart here: + * + * __free_zspage_lockless(): bare core; walks zpdescs and returns pages + * to the buddy allocator. Caller owns all zpdesc locks and has + * removed the zspage from its class list. Used by zs_free() outside + * class->lock so the buddy-side work does not stall the class. + * + * __free_zspage(): __free_zspage_lockless() + per-class accounting, + * under class->lock. Used by async_free_zspage(), the worker for + * zspages whose trylock_zspage() failed. + * + * free_zspage(): full wrapper - trylock zpdescs, remove from class + * list, call __free_zspage(); kicks deferred free on contention. + * Used by compaction. + */ +static inline void __free_zspage_lockless(struct zspage *zspage) { struct zpdesc *zpdesc, *next; - assert_spin_locked(&class->lock); - VM_BUG_ON(get_zspage_inuse(zspage)); VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0); @@ -822,7 +912,13 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class, } while (zpdesc != NULL); cache_free_zspage(zspage); +} +static void __free_zspage(struct zs_pool *pool, struct size_class *class, + struct zspage *zspage) +{ + assert_spin_locked(&class->lock); + __free_zspage_lockless(zspage); class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); } @@ -1275,7 +1371,7 @@ static unsigned long obj_malloc(struct zs_pool *pool, kunmap_local(vaddr); mod_zspage_inuse(zspage, 1); - obj = location_to_obj(m_zpdesc, obj); + obj = location_to_obj(m_zpdesc, obj, zspage->class); record_obj(handle, obj); return obj; @@ -1380,37 +1476,97 @@ static void obj_free(int class_size, unsigned long obj) mod_zspage_inuse(zspage, -1); } +#if (ZS_OBJ_CLASS_BITS > 0) || defined(CONFIG_COMPACTION) +/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */ +static unsigned int obj_to_class_idx(unsigned long obj) +{ + return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK; +} +#endif + +/* + * Resolve @handle to its zspage / size_class and acquire class->lock. + * + * When class_idx is encoded in obj (ZS_OBJ_CLASS_BITS > 0), it is + * invariant under page migration, so the handle can be read locklessly + * to pick the size_class. Once class->lock is held migration is + * blocked and the handle is re-read to obtain a stable PFN. + * + * Otherwise (32-bit, or 64-bit fallback paths like UML where the + * encoding is disabled), fall back to pool->lock for the lookup. + */ +#if ZS_OBJ_CLASS_BITS > 0 +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + obj = handle_to_obj(handle); + *classp = pool->size_class[obj_to_class_idx(obj)]; + spin_lock(&(*classp)->lock); + /* Re-read under class->lock: PFN is now stable vs migration. */ + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *objp = obj; +} +#else +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + read_lock(&pool->lock); + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *classp = zspage_class(pool, *zspagep); + spin_lock(&(*classp)->lock); + read_unlock(&pool->lock); + *objp = obj; +} +#endif + void zs_free(struct zs_pool *pool, unsigned long handle) { struct zspage *zspage; - struct zpdesc *f_zpdesc; unsigned long obj; struct size_class *class; int fullness; + struct zspage *zspage_to_free = NULL; if (IS_ERR_OR_NULL((void *)handle)) return; - /* - * The pool->lock protects the race with zpage's migration - * so it's safe to get the page from handle. - */ - read_lock(&pool->lock); - obj = handle_to_obj(handle); - obj_to_zpdesc(obj, &f_zpdesc); - zspage = get_zspage(f_zpdesc); - class = zspage_class(pool, zspage); - spin_lock(&class->lock); - read_unlock(&pool->lock); + obj_class_get_and_lock(pool, handle, &obj, &zspage, &class); class_stat_sub(class, ZS_OBJS_INUSE, 1); obj_free(class->size, obj); fullness = fix_fullness_group(class, zspage); - if (fullness == ZS_INUSE_RATIO_0) - free_zspage(pool, class, zspage); + if (fullness == ZS_INUSE_RATIO_0) { + if (trylock_zspage(zspage)) { + remove_zspage(class, zspage); + class_stat_sub(class, ZS_OBJS_ALLOCATED, + class->objs_per_zspage); + zspage_to_free = zspage; + } else { + kick_deferred_free(pool); + } + } spin_unlock(&class->lock); + + if (zspage_to_free) { + __free_zspage_lockless(zspage_to_free); + atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); + } cache_free_handle(handle); } EXPORT_SYMBOL_GPL(zs_free); @@ -1643,9 +1799,6 @@ static void lock_zspage(struct zspage *zspage) } zspage_read_unlock(zspage); } -#endif /* CONFIG_COMPACTION */ - -#ifdef CONFIG_COMPACTION static void replace_sub_page(struct size_class *class, struct zspage *zspage, struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc) @@ -1761,7 +1914,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page, old_obj = handle_to_obj(handle); obj_to_location(old_obj, &dummy, &obj_idx); - new_obj = (unsigned long)location_to_obj(newzpdesc, obj_idx); + new_obj = location_to_obj(newzpdesc, obj_idx, + obj_to_class_idx(old_obj)); record_obj(handle, new_obj); } } diff --git a/mm/zswap.c b/mm/zswap.c index 761cd699e0a3..4e76a4a87cdc 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, struct folio *folio; struct mempolicy *mpol; struct swap_info_struct *si; + struct swap_io_ctx ctx = {}; int ret = 0; /* try to allocate swap cache folio */ @@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, folio_set_reclaim(folio); /* start writeback */ - __swap_writepage(folio, NULL); + __swap_writepage(&ctx, folio); + swap_write_submit(&ctx); out: if (ret) { @@ -1217,7 +1219,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, * Without memcg, use the zswap pool-wide metrics. */ if (!mem_cgroup_disabled()) { - mem_cgroup_flush_stats(memcg); + mem_cgroup_flush_stats_ratelimited(memcg); nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); } else { diff --git a/samples/damon/Kconfig b/samples/damon/Kconfig index cbf96fd8a8bf..00be3e6bdd65 100644 --- a/samples/damon/Kconfig +++ b/samples/damon/Kconfig @@ -31,7 +31,7 @@ config SAMPLE_DAMON_MTIER bool "DAMON sample module for memory tiering" depends on DAMON && DAMON_PADDR help - Thps builds DAMON sample module for memory tierign. + This builds DAMON sample module for memory tiering. The module assumes the system is constructed with two NUMA nodes, which seems as local and remote nodes to all CPUs. For example, diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index 3785b0c7ffb1..ac9c24b92ead 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -177,6 +177,7 @@ free_out: static int damon_sample_mtier_start(void) { struct damon_ctx *ctx; + int err; ctx = damon_sample_mtier_build_ctx(true); if (!ctx) @@ -188,7 +189,13 @@ static int damon_sample_mtier_start(void) return -ENOMEM; } ctxs[1] = ctx; - return damon_start(ctxs, 2, true); + err = damon_start(ctxs, 2, true); + if (!err) + return 0; + + damon_destroy_ctx(ctxs[0]); + damon_destroy_ctx(ctxs[1]); + return err; } static void damon_sample_mtier_stop(void) diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index b7c50f2656ce..842099bd6228 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -106,11 +106,16 @@ static int damon_sample_prcl_start(void) damon_set_schemes(ctx, &scheme, 1); err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) + damon_destroy_ctx(ctx); + return err; } static void damon_sample_prcl_stop(void) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 799ad4443943..37fd5da20158 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -87,10 +87,15 @@ static int damon_sample_wsse_start(void) target->pid = target_pidp; err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) + damon_destroy_ctx(ctx); + return err; } static void damon_sample_wsse_stop(void) diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py index 8e713a09cfe7..eeabaeed438b 100644 --- a/scripts/gdb/linux/page_owner.py +++ b/scripts/gdb/linux/page_owner.py @@ -34,6 +34,7 @@ class DumpPageOwner(gdb.Command): max_pfn = None p_ops = None migrate_reason_names = None + mr_never = None def __init__(self): super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT) @@ -65,6 +66,7 @@ class DumpPageOwner(gdb.Command): self.max_pfn = int(gdb.parse_and_eval("max_pfn")) self.page_ext_size = int(gdb.parse_and_eval("page_ext_size")) self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names') + self.mr_never = int(gdb.parse_and_eval('MR_NEVER')) def page_ext_invalid(self, page_ext): if page_ext == gdb.Value(0): @@ -138,7 +140,7 @@ class DumpPageOwner(gdb.Command): else: gdb.write('page last free stack trace:\n') stackdepot.stack_depot_print(page_owner["free_handle"]) - if page_owner['last_migrate_reason'] != -1: + if page_owner['last_migrate_reason'] != self.mr_never: gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']]) def read_page_owner(self): diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c index 9320671dfcc8..5bc7e586b430 100644 --- a/sound/core/memalloc.c +++ b/sound/core/memalloc.c @@ -851,7 +851,7 @@ static void snd_dma_noncoherent_free(struct snd_dma_buffer *dmab) static int snd_dma_noncoherent_mmap(struct snd_dma_buffer *dmab, struct vm_area_struct *area) { - area->vm_page_prot = vm_get_page_prot(area->vm_flags); + area->vm_page_prot = vma_get_page_prot(area); return dma_mmap_pages(dmab->dev.dev, area, area->vm_end - area->vm_start, virt_to_page(dmab->area)); diff --git a/tools/include/linux/gfp_types.h b/tools/include/linux/gfp_types.h index 54ca0c88bab6..da47aa089f01 100644 --- a/tools/include/linux/gfp_types.h +++ b/tools/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -137,17 +135,12 @@ enum { * node with no fallbacks or placement policy enforcements. * * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h index 24ddf7bc4f25..f0a26309b6d5 100644 --- a/tools/include/uapi/linux/fs.h +++ b/tools/include/uapi/linux/fs.h @@ -364,6 +364,7 @@ typedef int __bitwise __kernel_rwf_t; #define PAGE_IS_HUGE (1 << 6) #define PAGE_IS_SOFT_DIRTY (1 << 7) #define PAGE_IS_GUARD (1 << 8) +#define PAGE_IS_ACCESSED (1 << 9) /* * struct page_region - Page region with flags diff --git a/tools/include/uapi/linux/userfaultfd.h b/tools/include/uapi/linux/userfaultfd.h index 4283de22d5b6..c8e99df95219 100644 --- a/tools/include/uapi/linux/userfaultfd.h +++ b/tools/include/uapi/linux/userfaultfd.h @@ -25,7 +25,8 @@ #define UFFD_API ((__u64)0xAA) #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ UFFDIO_REGISTER_MODE_WP | \ - UFFDIO_REGISTER_MODE_MINOR) + UFFDIO_REGISTER_MODE_MINOR | \ + UFFDIO_REGISTER_MODE_RWP) #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ @@ -42,10 +43,13 @@ UFFD_FEATURE_WP_UNPOPULATED | \ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ - UFFD_FEATURE_MOVE) + UFFD_FEATURE_MOVE | \ + UFFD_FEATURE_RWP | \ + UFFD_FEATURE_RWP_ASYNC) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_SET_MODE | \ (__u64)1 << _UFFDIO_API) #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ @@ -54,13 +58,15 @@ (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) /* * Valid ioctl command number range with this API is from 0x00 to @@ -79,6 +85,8 @@ #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) +#define _UFFDIO_RWPROTECT (0x09) +#define _UFFDIO_SET_MODE (0x0A) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -103,6 +111,10 @@ struct uffdio_continue) #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ struct uffdio_poison) +#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ + struct uffdio_rwprotect) +#define UFFDIO_SET_MODE _IOW(UFFDIO, _UFFDIO_SET_MODE, \ + struct uffdio_set_mode) /* read() structure */ struct uffd_msg { @@ -158,6 +170,7 @@ struct uffd_msg { #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ +#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */ struct uffdio_api { /* userland asks for an API number and the features to enable */ @@ -230,6 +243,18 @@ struct uffdio_api { * * UFFD_FEATURE_MOVE indicates that the kernel supports moving an * existing page contents from userspace. + * + * UFFD_FEATURE_RWP indicates that the kernel supports + * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. + * Pages are made inaccessible via UFFDIO_RWPROTECT and faults + * are delivered when the pages are re-accessed. + * + * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for + * UFFDIO_REGISTER_MODE_RWP. When set, faults on read-write + * protected pages are auto-resolved by the kernel (PTE + * permissions restored immediately) without delivering a message + * to the userfaultfd handler. Use PAGEMAP_SCAN with inverted + * PAGE_IS_ACCESSED to find pages that were not re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -248,6 +273,8 @@ struct uffdio_api { #define UFFD_FEATURE_POISON (1<<14) #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) +#define UFFD_FEATURE_RWP (1<<17) +#define UFFD_FEATURE_RWP_ASYNC (1<<18) __u64 features; __u64 ioctls; @@ -263,6 +290,7 @@ struct uffdio_register { #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) +#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3) __u64 mode; /* @@ -356,6 +384,14 @@ struct uffdio_poison { __s64 updated; }; +struct uffdio_rwprotect { + struct uffdio_range range; + /* !RWP means undo RWP-protection */ +#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0) +#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + struct uffdio_move { __u64 dst; __u64 src; @@ -374,6 +410,16 @@ struct uffdio_move { __s64 move; }; +struct uffdio_set_mode { + /* + * Toggle async mode for features at runtime. + * Supported: UFFD_FEATURE_RWP_ASYNC. + * Setting a bit in both enable and disable is invalid. + */ + __u64 enable; + __u64 disable; +}; + /* * Flags for the userfaultfd(2) system call itself. */ diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore index 922879f93fc8..1446a659e540 100644 --- a/tools/mm/.gitignore +++ b/tools/mm/.gitignore @@ -2,3 +2,4 @@ slabinfo page-types page_owner_sort +thp_swap_allocator_test diff --git a/tools/mm/Makefile b/tools/mm/Makefile index f5725b5c23aa..858186a6eefd 100644 --- a/tools/mm/Makefile +++ b/tools/mm/Makefile @@ -3,7 +3,7 @@ # include ../scripts/Makefile.include -BUILD_TARGETS=page-types slabinfo page_owner_sort thp_swap_allocator_test +BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps LIB_DIR = ../lib/api @@ -23,7 +23,7 @@ $(LIBS): $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) clean: - $(RM) page-types slabinfo page_owner_sort thp_swap_allocator_test + $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test make -C $(LIB_DIR) clean sbindir ?= /usr/sbin diff --git a/tools/mm/page_owner_filter.c b/tools/mm/page_owner_filter.c new file mode 100644 index 000000000000..1d1f0a38678a --- /dev/null +++ b/tools/mm/page_owner_filter.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * User-space helper to filter page_owner output per-fd + * + * Example use: + * ./page_owner_filter -m handle + * ./page_owner_filter -m stack_handle + * ./page_owner_filter -n 0,1,2 + * + * See Documentation/mm/page_owner.rst + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <ctype.h> +#include <getopt.h> +#include <signal.h> + +#define MAX_CMD_LEN 512 + +static void usage(const char *prog) +{ + fprintf(stderr, "Usage: %s [OPTIONS]\n", prog); + fprintf(stderr, "\nOptions:\n"); + fprintf(stderr, " -m, --mode MODE : print_mode (stack, handle, or stack_handle)\n"); + fprintf(stderr, " -n, --nid NID_LIST : NUMA node IDs (comma-separated or ranges)\n"); + fprintf(stderr, " -o, --output FILE : output file (default: stdout)\n"); + fprintf(stderr, " -h, --help : show this help message\n"); + fprintf(stderr, "\nExamples:\n"); + fprintf(stderr, " %s -m stack\n", prog); + fprintf(stderr, " %s -m handle\n", prog); + fprintf(stderr, " %s -m stack_handle\n", prog); + fprintf(stderr, " %s -m stack -o output.txt\n", prog); + fprintf(stderr, " %s -n 0,1,2\n", prog); + fprintf(stderr, " %s -m stack -n 0\n", prog); +} + +static int validate_mode(const char *mode) +{ + if (strcmp(mode, "stack") == 0 || + strcmp(mode, "handle") == 0 || + strcmp(mode, "stack_handle") == 0) + return 0; + + fprintf(stderr, "Error: Invalid mode '%s'\n", mode); + fprintf(stderr, "Valid modes: stack, handle, stack_handle\n"); + return -1; +} + +static int validate_nid_list(const char *nid_list) +{ + const char *p; + int i = 0; + int has_digit = 0; + int in_range = 0; + int prev_num = 0; + int curr_num = 0; + + if (!nid_list || strlen(nid_list) == 0) + return 0; + + for (p = nid_list; *p; p++) { + if (*p == ',') { + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + i = 0; + has_digit = 0; + in_range = 0; + prev_num = 0; + curr_num = 0; + continue; + } + + if (*p == '-') { + if (!has_digit) { + fprintf(stderr, + "Error: Invalid nid_list format "); + fprintf(stderr, + "(dash without preceding number)\n"); + return -1; + } + if (in_range) { + fprintf(stderr, "Error: Multiple dashes in nid_list\n"); + return -1; + } + prev_num = curr_num; + curr_num = 0; + i = 0; + has_digit = 0; + in_range = 1; + continue; + } + + if (!isdigit((unsigned char)*p)) { + fprintf(stderr, "Error: Invalid character '%c' in nid_list\n", *p); + return -1; + } + + if (i > 5) { + fprintf(stderr, "Error: NID too long (max 65536)\n"); + return -1; + } + curr_num = curr_num * 10 + (*p - '0'); + i++; + has_digit = 1; + } + + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *output_file = NULL; + char filter_cmd[MAX_CMD_LEN]; + FILE *output = NULL; + int fd = -1; + ssize_t ret; + char buf[4096]; + int opt; + size_t cmd_len = 0; + + signal(SIGPIPE, SIG_IGN); + + static struct option long_options[] = { + {"mode", required_argument, 0, 'm'}, + {"nid", required_argument, 0, 'n'}, + {"output", required_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + filter_cmd[0] = '\0'; + + if (argc > 1) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return 0; + } + } + } + + /* Check if page_owner exists and is readable */ + if (access("/sys/kernel/debug/page_owner", F_OK) != 0) { + if (errno == ENOENT) + fprintf(stderr, "Error: /sys/kernel/debug/page_owner does not exist\n"); + else + perror("Error accessing /sys/kernel/debug/page_owner"); + fprintf(stderr, "Make sure page_owner is enabled in kernel\n"); + return 1; + } + + while ((opt = getopt_long(argc, argv, "m:n:o:h", long_options, NULL)) != -1) { + int len; + + switch (opt) { + case 'm': { + const char *mode = optarg; + + if (validate_mode(mode) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%smode=%s", cmd_len > 0 ? " " : "", mode); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'n': { + const char *nid_list = optarg; + + if (validate_nid_list(nid_list) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%snid=%s", cmd_len > 0 ? " " : "", nid_list); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'o': + output_file = optarg; + break; + case 'h': + /* Already handled above */ + break; + default: + usage(argv[0]); + return 1; + } + } + + /* At least one filter must be specified */ + if (cmd_len == 0) { + fprintf(stderr, "Error: At least one filter (-m or -n) must be specified\n\n"); + usage(argv[0]); + return 1; + } + + /* Open page_owner for read-write - this will fail if kernel doesn't support write */ + fd = open("/sys/kernel/debug/page_owner", O_RDWR); + if (fd < 0) { + if (errno == EACCES || errno == EPERM) { + fprintf(stderr, "Error: /sys/kernel/debug/page_owner "); + fprintf(stderr, "does not support write access\n"); + fprintf(stderr, "This kernel does not support "); + fprintf(stderr, "per-fd filtering.\n"); + fprintf(stderr, "Please ensure you have a kernel with "); + fprintf(stderr, "per-fd filtering support.\n"); + } else { + perror("Error opening /sys/kernel/debug/page_owner"); + } + return 1; + } + + if (output_file) { + output = fopen(output_file, "w"); + if (!output) { + perror("open output file"); + close(fd); + return 1; + } + } else { + output = stdout; + } + + ret = write(fd, filter_cmd, strlen(filter_cmd)); + + if (ret < 0) { + if (errno == EINVAL) { + fprintf(stderr, "Error: Kernel rejected the filter command.\n"); + fprintf(stderr, "Possible causes:\n"); + fprintf(stderr, " - Kernel does not support per-fd filtering\n"); + fprintf(stderr, " - NUMA node has no memory\n"); + fprintf(stderr, " - Unknown reason\n"); + } else { + perror("write filter command"); + } + goto out; + } + + if ((size_t)ret != strlen(filter_cmd)) + fprintf(stderr, "Warning: Partial write (%zd/%zu)\n", ret, strlen(filter_cmd)); + + /* Read and display filtered output */ + ret = 0; + while ((ret = read(fd, buf, sizeof(buf))) > 0) { + size_t written = fwrite(buf, 1, ret, output); + + if (written != (size_t)ret) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + goto out; + } + perror("write output"); + ret = -1; + goto out; + } + } + + if (ret < 0) { + perror("read page_owner"); + goto out; + } + + if (fflush(output)) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + } else { + perror("flush output"); + ret = -1; + } + } + +out: + close(fd); + if (output != stdout) + fclose(output); + return ret < 0 ? 1 : 0; +} diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index e6954909401c..35d3d254941c 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -43,6 +43,13 @@ enum FILTER_BIT { FILTER_TGID = 1<<2, FILTER_COMM = 1<<3 }; + +enum FILTER_RESULT { + FILTER_ERROR, + FILTER_SKIP, + FILTER_MATCH +}; + enum CULL_BIT { CULL_PID = 1<<1, CULL_TGID = 1<<2, @@ -230,7 +237,8 @@ static int remove_pattern(regex_t *pattern, char *buf, int len) return len - (pmatch[1].rm_eo - pmatch[1].rm_so); } -static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) +static int search_pattern(regex_t *pattern, char *pattern_str, + size_t pattern_str_size, char *buf) { int err, val_len; regmatch_t pmatch[2]; @@ -242,8 +250,14 @@ static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) return -1; } val_len = pmatch[1].rm_eo - pmatch[1].rm_so; + if ((size_t)val_len >= pattern_str_size) { + if (debug_on) + fprintf(stderr, "pattern too long in %s\n", buf); + return -1; + } memcpy(pattern_str, buf + pmatch[1].rm_so, val_len); + pattern_str[val_len] = '\0'; return 0; } @@ -300,7 +314,8 @@ static int get_page_num(char *buf) char order_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&order_pattern, order_str, buf); + if (search_pattern(&order_pattern, order_str, sizeof(order_str), buf) < 0) + return 0; errno = 0; order_val = strtol(order_str, &endptr, 10); if (order_val > 64 || errno != 0 || endptr == order_str || *endptr != '\0') { @@ -318,7 +333,8 @@ static pid_t get_pid(char *buf) char pid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&pid_pattern, pid_str, buf); + if (search_pattern(&pid_pattern, pid_str, sizeof(pid_str), buf) < 0) + return -1; errno = 0; pid = strtol(pid_str, &endptr, 10); if (errno != 0 || endptr == pid_str || *endptr != '\0') { @@ -337,7 +353,8 @@ static pid_t get_tgid(char *buf) char tgid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&tgid_pattern, tgid_str, buf); + if (search_pattern(&tgid_pattern, tgid_str, sizeof(tgid_str), buf) < 0) + return -1; errno = 0; tgid = strtol(tgid_str, &endptr, 10); if (errno != 0 || endptr == tgid_str || *endptr != '\0') { @@ -356,7 +373,9 @@ static __u64 get_ts_nsec(char *buf) char ts_nsec_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&ts_nsec_pattern, ts_nsec_str, buf); + if (search_pattern(&ts_nsec_pattern, ts_nsec_str, + sizeof(ts_nsec_str), buf) < 0) + return -1; errno = 0; ts_nsec = strtoull(ts_nsec_str, &endptr, 10); if (errno != 0 || endptr == ts_nsec_str || *endptr != '\0') { @@ -372,9 +391,15 @@ static char *get_comm(char *buf) { char *comm_str = malloc(TASK_COMM_LEN); + if (!comm_str) + return NULL; + memset(comm_str, 0, TASK_COMM_LEN); - search_pattern(&comm_pattern, comm_str, buf); + if (search_pattern(&comm_pattern, comm_str, TASK_COMM_LEN, buf) < 0) { + free(comm_str); + return NULL; + } errno = 0; if (errno != 0) { if (debug_on) @@ -386,6 +411,12 @@ static char *get_comm(char *buf) return comm_str; } +static void free_block_list(struct block_list *block) +{ + free(block->comm); + free(block->txt); +} + static int get_arg_type(const char *arg) { if (!strcmp(arg, "pid") || !strcmp(arg, "p")) @@ -450,39 +481,56 @@ static bool match_str_list(const char *str, char **list, int list_size) return false; } -static bool is_need(char *buf) +static enum FILTER_RESULT filter_record(char *buf) { + char *comm; + if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) - return false; + return FILTER_SKIP; if ((filter & FILTER_TGID) && !match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size)) - return false; + return FILTER_SKIP; + if (!(filter & FILTER_COMM)) + return FILTER_MATCH; - char *comm = get_comm(buf); + comm = get_comm(buf); + if (!comm) + return FILTER_ERROR; - if ((filter & FILTER_COMM) && - !match_str_list(comm, fc.comms, fc.comms_size)) { + if (!match_str_list(comm, fc.comms, fc.comms_size)) { free(comm); - return false; + return FILTER_SKIP; } free(comm); - return true; + return FILTER_MATCH; } static bool add_list(char *buf, int len, char *ext_buf) { + enum FILTER_RESULT filter_result; + if (list_size == max_size) { fprintf(stderr, "max_size too small??\n"); return false; } - if (!is_need(buf)) + filter_result = filter_record(buf); + if (filter_result == FILTER_ERROR) { + fprintf(stderr, "Out of memory\n"); + return false; + } + if (filter_result == FILTER_SKIP) return true; list[list_size].pid = get_pid(buf); list[list_size].tgid = get_tgid(buf); list[list_size].comm = get_comm(buf); - list[list_size].txt = malloc(len+1); + if (!list[list_size].comm) { + fprintf(stderr, "Out of memory\n"); + return false; + } + list[list_size].txt = malloc(len + 1); if (!list[list_size].txt) { fprintf(stderr, "Out of memory\n"); + free(list[list_size].comm); return false; } memcpy(list[list_size].txt, buf, len); @@ -841,8 +889,10 @@ int main(int argc, char **argv) } else { list[count-1].num += list[i].num; list[count-1].page_num += list[i].page_num; + free_block_list(&list[i]); } } + list_size = count; qsort(list, count, sizeof(list[0]), compare_sort_condition); @@ -876,8 +926,11 @@ out_free: free(ext_buf); if (buf) free(buf); - if (list) + if (list) { + for (i = 0; i < list_size; i++) + free_block_list(&list[i]); free(list); + } out_ts: regfree(&ts_nsec_pattern); out_comm: diff --git a/tools/testing/memblock/internal.h b/tools/testing/memblock/internal.h index b6b1d147fd75..4168e646af09 100644 --- a/tools/testing/memblock/internal.h +++ b/tools/testing/memblock/internal.h @@ -28,22 +28,6 @@ static inline struct page *virt_to_page(void *virt) return virt; } -void memblock_free_pages(unsigned long pfn, unsigned int order) -{ -} - -static inline void accept_memory(phys_addr_t start, unsigned long size) -{ -} - -unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); -void free_reserved_page(struct page *page); - -static inline bool deferred_pages_enabled(void) -{ - return false; -} - #define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++) @@ -60,10 +44,6 @@ static inline bool __is_kernel(unsigned long addr) #define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++) -static inline void init_deferred_page(unsigned long pfn, int nid) -{ -} - #define __SetPageReserved(p) ((void)(p)) #endif diff --git a/tools/testing/memblock/mm_init.h b/tools/testing/memblock/mm_init.h new file mode 100644 index 000000000000..95bc5f2e8ed8 --- /dev/null +++ b/tools/testing/memblock/mm_init.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __MM_MM_INIT_H +#define __MM_MM_INIT_H + +void memblock_free_pages(unsigned long pfn, unsigned int order) +{ +} + +static inline void accept_memory(phys_addr_t start, unsigned long size) +{ +} + +unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); +void free_reserved_page(struct page *page); + +static inline bool deferred_pages_enabled(void) +{ + return false; +} + +static inline void init_deferred_page(unsigned long pfn, int nid) +{ +} +#endif /* __MM_MM_INIT_H */ diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index 0607913a3022..d967e76a3c06 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -35234,7 +35234,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) mt_set_non_kernel(1); /* Spanning store */ mas_set_range(&mas, 1, 100); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); @@ -35257,7 +35257,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) MT_BUG_ON(mt, mas_allocated(&mas) != 0); mas_set_range(&mas, 0, 200); mt_set_non_kernel(1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 8d4db2241cc2..2cc63e4134fb 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 TARGETS += acct +TARGETS += alloc_tag TARGETS += alsa TARGETS += amd-pstate TARGETS += arm64 @@ -14,6 +15,7 @@ TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug TARGETS += damon +TARGETS += dax TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps diff --git a/tools/testing/selftests/alloc_tag/Makefile b/tools/testing/selftests/alloc_tag/Makefile new file mode 100644 index 000000000000..f41d92d06b18 --- /dev/null +++ b/tools/testing/selftests/alloc_tag/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 + +TEST_GEN_PROGS := allocinfo_ioctl_test + +CFLAGS += -Wall +CFLAGS += $(KHDR_INCLUDES) + +include ../lib.mk + diff --git a/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c new file mode 100644 index 000000000000..7d94abb785a1 --- /dev/null +++ b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* kselftest for allocinfo ioctl + * allocinfo ioctl retrives allocinfo data through ioctl + * Copyright (C) 2026 Google, Inc. + */ + +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdbool.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <linux/types.h> +#include <linux/alloc_tag.h> +#include "../kselftest.h" + +#define MAX_LINE_LEN 512 +#define ALLOCINFO_PROC "/proc/allocinfo" + +enum ioctl_ret { + IOCTL_SUCCESS = 0, + IOCTL_FAILURE = 1, + IOCTL_INVALID_DATA = 2, +}; + +#define VEC_MAX_ENTRIES 32 + +struct allocinfo_tag_data_vec { + struct allocinfo_tag_data tag[VEC_MAX_ENTRIES]; + __u64 count; +}; + +static inline int __allocinfo_get_content_id(int dev_fd, struct allocinfo_content_id *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_CONTENT_ID, params); +} + +static inline int __allocinfo_get_at(int dev_fd, struct allocinfo_get_at *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_GET_AT, params); +} + +static inline int __allocinfo_get_next(int dev_fd, struct allocinfo_tag_data *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_GET_NEXT, params); +} + +static bool match_entry(const struct allocinfo_tag_data *procfs_entry, + const struct allocinfo_tag_data *tag_data, + bool match_bytes, bool match_calls, bool match_lineno, + bool match_function, bool match_filename) +{ + if (match_bytes && tag_data->counter.bytes != procfs_entry->counter.bytes) { + ksft_print_msg("size retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_calls && tag_data->counter.calls != procfs_entry->counter.calls) { + ksft_print_msg("call count retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_lineno && tag_data->tag.lineno != procfs_entry->tag.lineno) { + ksft_print_msg("lineno retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_function && + strncmp(tag_data->tag.function, procfs_entry->tag.function, ALLOCINFO_STR_SIZE)) { + ksft_print_msg("function retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_filename && + strncmp(tag_data->tag.filename, procfs_entry->tag.filename, ALLOCINFO_STR_SIZE)) { + ksft_print_msg("filename retrieved through ioctl does not match procfs\n"); + return false; + } + return true; +} + +static bool match_entries(const struct allocinfo_tag_data_vec *procfs_entries, + const struct allocinfo_tag_data_vec *tags, + bool match_bytes, bool match_calls, bool match_lineno, + bool match_function, bool match_filename) +{ + __u64 i; + + if (procfs_entries->count != tags->count) { + ksft_print_msg("Entry count mismatch. ioctl entries: %llu, proc entries: %llu\n", + tags->count, procfs_entries->count); + return false; + } + for (i = 0; i < procfs_entries->count; i++) { + if (!match_entry(&procfs_entries->tag[i], &tags->tag[i], + match_bytes, match_calls, match_lineno, + match_function, match_filename)) { + ksft_print_msg("%lluth entry does not match.\n", i); + return false; + } + } + return true; +} + +static const char *allocinfo_str(const char *str) +{ + size_t len = strlen(str); + + if (len >= ALLOCINFO_STR_SIZE) + str += (len - ALLOCINFO_STR_SIZE) + 1; + return str; +} + +static void allocinfo_copy_str(char *dest, const char *src) +{ + strncpy(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE - 1); + dest[ALLOCINFO_STR_SIZE - 1] = '\0'; +} + +static int get_filtered_procfs_entries(struct allocinfo_tag_data_vec *procfs_entries, + const struct allocinfo_filter *filter) +{ + FILE *fp = fopen(ALLOCINFO_PROC, "r"); + char line[MAX_LINE_LEN]; + int matches; + struct allocinfo_tag_data procfs_entry; + + if (!fp) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC " for reading\n"); + return 1; + } + memset(procfs_entries, 0, sizeof(*procfs_entries)); + while (fgets(line, sizeof(line), fp) && procfs_entries->count < VEC_MAX_ENTRIES) { + char filename[MAX_LINE_LEN]; + char function[MAX_LINE_LEN]; + + memset(&procfs_entry, 0, sizeof(procfs_entry)); + matches = sscanf(line, "%llu %llu %[^:]:%llu func:%s", + &procfs_entry.counter.bytes, + &procfs_entry.counter.calls, + filename, + &procfs_entry.tag.lineno, + function); + + if (matches != 5) + continue; + + allocinfo_copy_str(procfs_entry.tag.filename, filename); + allocinfo_copy_str(procfs_entry.tag.function, function); + + if (filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) { + if (strncmp(procfs_entry.tag.filename, + filter->fields.filename, ALLOCINFO_STR_SIZE)) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) { + if (strncmp(procfs_entry.tag.function, + filter->fields.function, ALLOCINFO_STR_SIZE)) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_LINENO) { + if (procfs_entry.tag.lineno != filter->fields.lineno) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) { + if (procfs_entry.counter.bytes < filter->min_size) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) { + if (procfs_entry.counter.bytes > filter->max_size) + continue; + } + + memcpy(&procfs_entries->tag[procfs_entries->count++], &procfs_entry, + sizeof(procfs_entry)); + } + fclose(fp); + return 0; +} + +static enum ioctl_ret get_filtered_ioctl_entries(struct allocinfo_tag_data_vec *tags, + const struct allocinfo_filter *filter, + __u64 start_pos) +{ + int fd = open(ALLOCINFO_PROC, O_RDONLY); + + if (fd < 0) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC " for IOCTL\n"); + return IOCTL_FAILURE; + } + + struct allocinfo_content_id start_cont_id, end_cont_id; + struct allocinfo_get_at get_at_params; + const int max_retries = 10; + int retry_count = 0; + int status; + + /* + * __allocinfo_get_content_id may return different values if a kernel module was loaded + * between the two calls. If that happens, the data gathered cannot be considered consistent + * and hence needs to be fetched again to avoid flakiness. + */ + do { + if (__allocinfo_get_content_id(fd, &start_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + status = IOCTL_FAILURE; + break; + } + + memset(tags, 0, sizeof(*tags)); + memset(&get_at_params, 0, sizeof(get_at_params)); + memcpy(&get_at_params.filter, filter, sizeof(*filter)); + get_at_params.pos = start_pos; + if (__allocinfo_get_at(fd, &get_at_params)) { + ksft_print_msg("allocinfo_get_at failed\n"); + status = IOCTL_FAILURE; + break; + } + memcpy(&tags->tag[tags->count++], &get_at_params.data, sizeof(get_at_params.data)); + + while (tags->count < VEC_MAX_ENTRIES && + __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0) + tags->count++; + + if (__allocinfo_get_content_id(fd, &end_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + status = IOCTL_FAILURE; + break; + } + + if (start_cont_id.id == end_cont_id.id) { + status = IOCTL_SUCCESS; + } else { + ksft_print_msg("allocinfo_get_content_id mismatch, retrying...\n"); + status = IOCTL_INVALID_DATA; + } + } while (status == IOCTL_INVALID_DATA && retry_count++ < max_retries); + + close(fd); + return status; +} + +static int run_filter_test(const struct allocinfo_filter *filter) +{ + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + int ioctl_status; + int ret = KSFT_PASS; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (get_filtered_procfs_entries(procfs_entries, filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + + if (procfs_entries->count == 0) { + ksft_print_msg("No entries found in " ALLOCINFO_PROC ", skipping test\n"); + ret = KSFT_SKIP; + goto exit; + } + + ioctl_status = get_filtered_ioctl_entries(tags, filter, 0); + if (ioctl_status == IOCTL_INVALID_DATA) { + ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n"); + ret = KSFT_SKIP; + goto exit; + } + if (ioctl_status == IOCTL_FAILURE) { + ksft_print_msg("Error retrieving IOCTL entries.\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (!match_entries(procfs_entries, tags, false, false, true, true, true)) + ret = KSFT_FAIL; + +exit: + free(tags); + free(procfs_entries); + return ret; +} + +static int test_filename_filter(void) +{ + struct allocinfo_filter filter; + const char *target_filename = "mm/memory.c"; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FILENAME; + strncpy(filter.fields.filename, target_filename, ALLOCINFO_STR_SIZE); + + return run_filter_test(&filter); +} + +static int test_function_filter(void) +{ + struct allocinfo_filter filter; + const char *target_function = "dup_mm"; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION; + strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE); + + return run_filter_test(&filter); +} + +static int test_size_filter(void) +{ + int fd; + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + struct allocinfo_filter filter; + int ret = KSFT_PASS; + __u64 target_size, i, pos; + struct allocinfo_tag_data *found_tag = NULL; + const char *target_function = "do_init_module"; + struct allocinfo_content_id start_cont_id, end_cont_id; + int retry = 0; + const int max_retries = 10; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto freemem; + } + + fd = open(ALLOCINFO_PROC, O_RDONLY); + if (fd < 0) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC ": %s\n", strerror(errno)); + ret = KSFT_SKIP; + goto freemem; + } + + do { + found_tag = NULL; + pos = 0; + + if (__allocinfo_get_content_id(fd, &start_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + ret = KSFT_FAIL; + goto exit; + } + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION; + strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE); + + if (get_filtered_procfs_entries(procfs_entries, &filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + + if (procfs_entries->count == 0) { + ksft_print_msg("Function %s not found in procfs\n", target_function); + ret = KSFT_SKIP; + goto exit; + } + + target_size = procfs_entries->tag[0].counter.bytes; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE; + filter.min_size = target_size; + filter.max_size = target_size; + + while (1) { + struct allocinfo_get_at get_at_params; + + memset(&get_at_params, 0, sizeof(get_at_params)); + memcpy(&get_at_params.filter, &filter, sizeof(filter)); + get_at_params.pos = pos; + + if (__allocinfo_get_at(fd, &get_at_params)) + break; + + tags->count = 0; + memcpy(&tags->tag[tags->count++], &get_at_params.data, + sizeof(get_at_params.data)); + + while (tags->count < VEC_MAX_ENTRIES && + __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0) + tags->count++; + + for (i = 0; i < tags->count; i++) { + if (strcmp(tags->tag[i].tag.function, target_function) == 0) { + found_tag = &tags->tag[i]; + break; + } + } + + if (found_tag || tags->count < VEC_MAX_ENTRIES) + break; + + pos += tags->count; + } + + if (__allocinfo_get_content_id(fd, &end_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (start_cont_id.id == end_cont_id.id) + break; + + ksft_print_msg("Module load detected during size verification, retrying...\n"); + } while (retry++ < max_retries); + + if (start_cont_id.id == end_cont_id.id && !found_tag) { + ksft_print_msg("Entry with function %s not found in IOCTL results\n", + target_function); + ret = KSFT_FAIL; + } else if (start_cont_id.id != end_cont_id.id) { + ksft_print_msg("Failed to match content_ids for procfs and IOCTL, skipping...\n"); + ret = KSFT_SKIP; + } else if (found_tag && found_tag->counter.bytes != target_size) { + ksft_print_msg("IOCTL entry size %llu does not match target size %llu\n", + found_tag->counter.bytes, target_size); + ret = KSFT_FAIL; + } + +exit: + close(fd); +freemem: + free(tags); + free(procfs_entries); + return ret; +} + +static int test_lineno_filter(void) +{ + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + struct allocinfo_filter filter; + enum ioctl_ret ioctl_status; + int ret = KSFT_PASS; + __u64 target_lineno, i; + struct allocinfo_tag_data *target_tag; + bool found = false; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto exit; + } + + memset(&filter, 0, sizeof(filter)); + + if (get_filtered_procfs_entries(procfs_entries, &filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + if (procfs_entries->count == 0) { + ksft_print_msg("Could not retrieve procfs entries\n"); + ret = KSFT_SKIP; + goto exit; + } + /* + * We depend on the procfs results to determine the line number for the filter before + * making the ioctl query. Hence, we cannot reuse run_filter_test here. + */ + target_tag = &procfs_entries->tag[0]; + target_lineno = target_tag->tag.lineno; + + filter.mask |= ALLOCINFO_FILTER_MASK_LINENO; + filter.fields.lineno = target_lineno; + + ioctl_status = get_filtered_ioctl_entries(tags, &filter, 0); + if (ioctl_status == IOCTL_INVALID_DATA) { + ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n"); + ret = KSFT_SKIP; + goto exit; + } + if (ioctl_status == IOCTL_FAILURE) { + ksft_print_msg("Error retrieving IOCTL entries.\n"); + ret = KSFT_FAIL; + goto exit; + } + + for (i = 0; i < tags->count; i++) { + if (tags->tag[i].tag.lineno != target_lineno) { + ksft_print_msg("IOCTL entry %llu has incorrect lineno %llu.\n", + i, tags->tag[i].tag.lineno); + ret = KSFT_FAIL; + goto exit; + } + + if (strncmp(tags->tag[i].tag.function, target_tag->tag.function, + ALLOCINFO_STR_SIZE) == 0 && + strncmp(tags->tag[i].tag.filename, target_tag->tag.filename, + ALLOCINFO_STR_SIZE) == 0) + found = true; + } + + if (!found) { + ksft_print_msg("Original procfs entry not found in IOCTL lineno filter results.\n"); + ret = KSFT_FAIL; + } + +exit: + free(tags); + free(procfs_entries); + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + ksft_set_plan(4); + + ret = test_filename_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_filename_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_filename_filter\n"); + + ret = test_function_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_function_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_function_filter\n"); + + ret = test_size_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_size_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_size_filter\n"); + + ret = test_lineno_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_lineno_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_lineno_filter\n"); + + ksft_finished(); +} diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 2180c328a825..ece244e5c5b9 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -13,6 +13,7 @@ TEST_PROGS += sysfs.py TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py TEST_PROGS += damos_tried_regions.py damon_nr_regions.py +TEST_PROGS += sysfs_refresh.py TEST_PROGS += reclaim.sh lru_sort.sh # regression tests (reproducers of previously found bugs) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 8b12cc048440..e6a2265d721e 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -271,7 +271,7 @@ class DamosFilter: self.type_ = type_ self.matching = matching self.allow = allow - self.memcg_path = memcg_path, + self.memcg_path = memcg_path self.addr_start = addr_start self.addr_end = addr_end self.target_idx = target_idx @@ -624,17 +624,23 @@ class DamonCtx: pause = None idx = None - def __init__(self, ops='paddr', monitoring_attrs=DamonAttrs(), targets=[], - schemes=[], pause=False): + def __init__(self, ops='paddr', monitoring_attrs=None, targets=None, + schemes=None, pause=False): self.ops = ops + if monitoring_attrs is None: + monitoring_attrs = DamonAttrs() self.monitoring_attrs = monitoring_attrs self.monitoring_attrs.context = self + if targets is None: + targets = [] self.targets = targets for idx, target in enumerate(self.targets): target.idx = idx target.context = self + if schemes is None: + schemes = [] self.schemes = schemes for idx, scheme in enumerate(self.schemes): scheme.idx = idx @@ -692,12 +698,14 @@ class DamonCtx: class Kdamond: state = None pid = None + refresh_ms = None contexts = None idx = None # index of this kdamond between siblings kdamonds = None # parent - def __init__(self, contexts=[]): + def __init__(self, contexts=[], refresh_ms=None): self.contexts = contexts + self.refresh_ms = refresh_ms for idx, context in enumerate(self.contexts): context.idx = idx context.kdamond = self @@ -720,6 +728,11 @@ class Kdamond: err = context.stage() if err is not None: return err + if self.refresh_ms is not None: + err = write_file(os.path.join(self.sysfs_dir(), 'refresh_ms'), + '%d' % self.refresh_ms) + if err is not None: + return err err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'on') if err is not None: return err @@ -831,7 +844,7 @@ class Kdamond: for goal in scheme.quota.goals: err = goal.stage() if err is not None: - print('commit_schemes_quota_goals failed stagign: %s'% + print('commit_schemes_quota_goals failed staging: %s'% err) exit(1) return write_file(os.path.join(self.sysfs_dir(), 'state'), diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py index f04d43702481..0f2f36584e48 100755 --- a/tools/testing/selftests/damon/damos_apply_interval.py +++ b/tools/testing/selftests/damon/damos_apply_interval.py @@ -56,7 +56,7 @@ def main(): # Because the second scheme was having the apply interval that is ten times # lower than that of the first scheme, the second scheme should be tried # about ten times more frequently than the first scheme. For possible - # timing errors, check if it was at least nine times more freuqnetly tried. + # timing errors, check if it was at least nine times more frequently tried. ratio = nr_tried_stats[1] / nr_tried_stats[0] if ratio < 9: print('%d / %d = %f (< 9)' % diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py index f76e0412b564..661e4ba4765a 100755 --- a/tools/testing/selftests/damon/damos_quota_goal.py +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -66,7 +66,7 @@ def main(): # effective quota was already minimum that cannot be more reduced if expect_increase is False and last_effective_bytes == 1: continue - print('efective bytes not changed: %d' % goal.effective_bytes) + print('effective bytes not changed: %d' % goal.effective_bytes) exit(1) increased = last_effective_bytes < goal.effective_bytes diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py index 3b347eb28bd2..d6472e6a6e08 100755 --- a/tools/testing/selftests/damon/damos_tried_regions.py +++ b/tools/testing/selftests/damon/damos_tried_regions.py @@ -55,10 +55,10 @@ def main(): collected_nr_regions.sort() sample = collected_nr_regions[4] print('50-th percentile nr_regions: %d' % sample) - print('expectation (>= 14) is %s' % 'met' if sample >= 14 else 'not met') + print('expectation (>= 14) is %s' % ('met' if sample >= 14 else 'not met')) if collected_nr_regions[4] < 14: print('full nr_regions:') - print('\n'.join(collected_nr_regions)) + print('\n'.join(['%d' % x for x in collected_nr_regions])) exit(1) if __name__ == '__main__': diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 972948e6215f..09552e91bc78 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -59,7 +59,6 @@ def region_to_dict(region): ['ar', addr_range_to_dict], ['sampling_addr', int], ['nr_accesses', int], - ['nr_accesses_bp', int], ['age', int], ]) @@ -163,7 +162,7 @@ def damos_filter_to_dict(damos_filter): int(damos_filter.addr_range.end)] elif type_ == 'target': dict_['target_idx'] = int(damos_filter.target_idx) - elif type_ == 'hugeapge_size': + elif type_ == 'hugepage_size': dict_['sz_range'] = [int(damos_filter.sz_range.min), int(damos_filter.sz_range.max)] return dict_ diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index aa03a1187489..3ffa054b6386 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -119,7 +119,7 @@ def assert_access_pattern_committed(pattern, dump): 'max_nr_accesses', dump) assert_true(dump['min_age_region'] == pattern.age[0], 'min_age_region', dump) - assert_true(dump['max_age_region'] == pattern.age[1], 'miaxage_region', + assert_true(dump['max_age_region'] == pattern.age[1], 'max_age_region', dump) def assert_scheme_committed(scheme, dump): @@ -129,7 +129,7 @@ def assert_scheme_committed(scheme, dump): 'cold': 1, 'pageout': 2, 'hugepage': 3, - 'nohugeapge': 4, + 'nohugepage': 4, 'collapse': 5, 'lru_prio': 6, 'lru_deprio': 7, @@ -250,6 +250,35 @@ def assert_ctxs_committed(kdamonds): if ctx in ctxs_paused_for_dump: ctx.pause = False +def test_memcg_filter_memcg_path_staging(): + global kdamonds + memcg_filter = _damon_sysfs.DamosFilter( + type_='memcg', matching=True, allow=True, memcg_path='/') + kdamonds = _damon_sysfs.Kdamonds( + [_damon_sysfs.Kdamond( + contexts=[_damon_sysfs.DamonCtx( + targets=[_damon_sysfs.DamonTarget(pid=-1)], + schemes=[_damon_sysfs.Damos( + ops_filters=[memcg_filter])], + )])]) + kdamonds.start() + + shown, rd_err = _damon_sysfs.read_file( + os.path.join(memcg_filter.sysfs_dir(), 'memcg_path')) + if rd_err is not None: + print('memcg_path staging: sysfs read (%s)' % rd_err) + kdamonds.stop() + exit(1) + if shown.rstrip('\n') != memcg_filter.memcg_path: + print('memcg_path staging: memcg_path readback ' + '(shown=%s, expected=%s)' % + (shown.rstrip('\n'), memcg_filter.memcg_path)) + kdamonds.stop() + exit(1) + + kdamonds.stop() + kdamonds = None + def main(): global kdamonds kdamonds = _damon_sysfs.Kdamonds( @@ -356,5 +385,7 @@ def main(): assert_ctxs_committed(kdamonds) kdamonds.stop() + test_memcg_filter_memcg_path_staging() + if __name__ == '__main__': main() diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index 78f4badb5beb..f7fb94b84e71 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -3,7 +3,7 @@ source _common.sh -# Kselftest frmework requirement - SKIP code is 4. +# Kselftest framework requirement - SKIP code is 4. ksft_skip=4 ensure_write_succ() @@ -28,7 +28,7 @@ ensure_write_fail() if (echo "$content" > "$file") 2> /dev/null then - echo "writing $content to $file succeed ($fail_reason)" + echo "writing $content to $file succeeded ($reason)" echo "expected failure because $reason" exit 1 fi @@ -67,7 +67,7 @@ ensure_file() echo "$file permission: expected $permission but $perm" exit 1 fi - elif [ "$to_ensure" = "not_exist" ] && [ -f "$dir" ] + elif [ "$to_ensure" = "not_exist" ] && [ -f "$file" ] then echo "$file is not expected but found" exit 1 @@ -99,14 +99,55 @@ test_stats() done } +test_dest() +{ + dest_dir=$1 + ensure_file "$dest_dir/id" "exist" "600" + ensure_file "$dest_dir/weight" "exist" "600" +} + +test_dests() +{ + dests_dir=$1 + ensure_file "$dests_dir/nr_dests" "exist" "600" + ensure_write_succ "$dests_dir/nr_dests" "1" "valid input" + test_dest "$dests_dir/0" + + ensure_write_succ "$dests_dir/nr_dests" "2" "valid input" + test_dest "$dests_dir/0" + test_dest "$dests_dir/1" + + ensure_write_succ "$dests_dir/nr_dests" "0" "valid input" + ensure_dir "$dests_dir/0" "not_exist" + ensure_dir "$dests_dir/1" "not_exist" +} + test_filter() { filter_dir=$1 ensure_file "$filter_dir/type" "exist" "600" - ensure_write_succ "$filter_dir/type" "anon" "valid input" - ensure_write_succ "$filter_dir/type" "memcg" "valid input" - ensure_write_succ "$filter_dir/type" "addr" "valid input" - ensure_write_succ "$filter_dir/type" "target" "valid input" + + local dir_name=$(basename "$(dirname "$filter_dir")") + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "ops_filters" ] + then + ensure_write_succ "$filter_dir/type" "anon" "valid input" + ensure_write_succ "$filter_dir/type" "memcg" "valid input" + fi + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "core_filters" ] + then + ensure_write_succ "$filter_dir/type" "addr" "valid input" + ensure_write_succ "$filter_dir/type" "target" "valid input" + fi + if [ "$dir_name" = "core_filters" ] + then + ensure_write_fail "$filter_dir/type" "anon" "ops type" + ensure_write_fail "$filter_dir/type" "memcg" "ops type" + fi + if [ "$dir_name" = "ops_filters" ] + then + ensure_write_fail "$filter_dir/type" "addr" "core type" + ensure_write_fail "$filter_dir/type" "target" "core type" + fi ensure_write_fail "$filter_dir/type" "foo" "invalid input" ensure_file "$filter_dir/matching" "exist" "600" ensure_file "$filter_dir/memcg_path" "exist" "600" @@ -158,6 +199,20 @@ test_goal() ensure_dir "$goal_dir" "exist" ensure_file "$goal_dir/target_value" "exist" "600" ensure_file "$goal_dir/current_value" "exist" "600" + ensure_file "$goal_dir/target_metric" "exist" "600" + local fpath="$goal_dir/target_metric" + ensure_write_succ "$fpath" "user_input" "valid input" + ensure_write_succ "$fpath" "some_mem_psi_us" "valid input" + ensure_write_succ "$fpath" "node_mem_used_bp" "valid input" + ensure_write_succ "$fpath" "node_mem_free_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_used_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_free_bp" "valid input" + ensure_write_succ "$fpath" "active_mem_bp" "valid input" + ensure_write_succ "$fpath" "inactive_mem_bp" "valid input" + ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input" + ensure_write_fail "$fpath" "foo" "invalid input" + ensure_file "$goal_dir/nid" "exist" "600" + ensure_file "$goal_dir/path" "exist" "600" } test_goals() @@ -207,7 +262,10 @@ test_scheme() ensure_file "$scheme_dir/apply_interval_us" "exist" "600" test_quotas "$scheme_dir/quotas" test_watermarks "$scheme_dir/watermarks" + test_dests "$scheme_dir/dests" test_filters "$scheme_dir/filters" + test_filters "$scheme_dir/core_filters" + test_filters "$scheme_dir/ops_filters" test_stats "$scheme_dir/stats" test_tried_regions "$scheme_dir/tried_regions" } @@ -346,8 +404,13 @@ test_probes() ensure_write_succ "$probes_dir/nr_probes" "1" "valid input" test_probe "$probes_dir/0" + ensure_write_succ "$probes_dir/nr_probes" "2" "valid input" + test_probe "$probes_dir/0" + test_probe "$probes_dir/1" + ensure_write_succ "$probes_dir/nr_probes" "0" "valid input" ensure_dir "$probes_dir/0" "not_exist" + ensure_dir "$probes_dir/1" "not_exist" } test_monitoring_attrs() @@ -363,7 +426,7 @@ test_context() { context_dir=$1 ensure_dir "$context_dir" "exist" - ensure_file "$context_dir/avail_operations" "exit" 400 + ensure_file "$context_dir/avail_operations" "exist" 400 ensure_file "$context_dir/operations" "exist" 600 ensure_file "$context_dir/addr_unit" "exist" 600 ensure_file "$context_dir/pause" "exist" 600 diff --git a/tools/testing/selftests/damon/sysfs_refresh.py b/tools/testing/selftests/damon/sysfs_refresh.py new file mode 100755 index 000000000000..012b7e8f509f --- /dev/null +++ b/tools/testing/selftests/damon/sysfs_refresh.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import os +import subprocess +import time + +import _damon_sysfs + +def main(): + # Continuously access a memory region for far longer than the test needs, + # so the kdamond always has a live target to monitor while we poll. + sz_region = 10 * 1024 * 1024 + proc = subprocess.Popen( + ['./access_memory', '1', '%d' % sz_region, '60000', 'repeat']) + + # A 'stat' scheme with the default (maximally wide) access pattern matches + # every monitored region, so its 'nr_tried' stat increases as the kdamond + # runs. refresh_ms should make DAMON update the schemes' stats files under + # sysfs on its own, without a manual 'update_schemes_stats' request. + kdamond = _damon_sysfs.Kdamond( + refresh_ms=100, + contexts=[_damon_sysfs.DamonCtx( + ops='vaddr', + targets=[_damon_sysfs.DamonTarget(pid=proc.pid)], + schemes=[_damon_sysfs.Damos(action='stat')], + )]) + kdamonds = _damon_sysfs.Kdamonds([kdamond]) + + err = kdamonds.start() + if err is not None: + # Kernels older than the refresh_ms feature have no such file; treat + # that as unsupported rather than a failure. + if not os.path.exists(os.path.join(kdamond.sysfs_dir(), 'refresh_ms')): + proc.terminate() + proc.wait() + print('kdamond has no refresh_ms file; skipping') + exit(_damon_sysfs.ksft_skip) + proc.terminate() + proc.wait() + print('kdamond start failed: %s' % err) + exit(1) + + scheme = kdamond.contexts[0].schemes[0] + nr_tried_path = os.path.join(scheme.sysfs_dir(), 'stats', 'nr_tried') + + try: + # Poll the stat file directly. We never request an update (e.g. + # 'update_schemes_stats'), so 'nr_tried' can become non-zero only + # through the periodic refresh that refresh_ms enables. + nr_tried = 0 + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if proc.poll() is not None: + print('the access_memory target exited unexpectedly') + exit(1) + content, err = _damon_sysfs.read_file(nr_tried_path) + if err is not None: + print('reading %s failed: %s' % (nr_tried_path, err)) + exit(1) + nr_tried = int(content) + if nr_tried > 0: + break + time.sleep(0.1) + finally: + kdamonds.stop() + proc.terminate() + proc.wait() + + if nr_tried == 0: + print('refresh_ms did not auto-update the schemes stats') + exit(1) + +if __name__ == '__main__': + main() diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py index 35c724a63f6c..16fdc6e7fc56 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py @@ -7,7 +7,7 @@ import time import _damon_sysfs def pass_wss_estimation(sz_region): - # access two regions of given size, 2 seocnds per each region + # access two regions of given size, 2 seconds per each region proc = subprocess.Popen( ['./access_memory', '2', '%d' % sz_region, '2000', 'repeat']) kdamonds = _damon_sysfs.Kdamonds([_damon_sysfs.Kdamond( diff --git a/tools/testing/selftests/dax/Makefile b/tools/testing/selftests/dax/Makefile new file mode 100644 index 000000000000..25a4f3d73a5b --- /dev/null +++ b/tools/testing/selftests/dax/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +all: + +TEST_PROGS := dax-kmem-hotplug.sh + +include ../lib.mk diff --git a/tools/testing/selftests/dax/config b/tools/testing/selftests/dax/config new file mode 100644 index 000000000000..4c9aaeb6ceb4 --- /dev/null +++ b/tools/testing/selftests/dax/config @@ -0,0 +1,4 @@ +CONFIG_DEV_DAX=m +CONFIG_DEV_DAX_KMEM=m +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y diff --git a/tools/testing/selftests/dax/dax-kmem-hotplug.sh b/tools/testing/selftests/dax/dax-kmem-hotplug.sh new file mode 100755 index 000000000000..9299120ee509 --- /dev/null +++ b/tools/testing/selftests/dax/dax-kmem-hotplug.sh @@ -0,0 +1,317 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Exercise the dax/kmem "state" sysfs attribute: +# /sys/bus/dax/devices/daxX.Y/state -> unplugged | online | online_kernel | online_movable +# +# The test needs a dax device already bound to the kmem driver. +# +# This test mutates a device's memory: online/offline cycles migrate any +# in-use pages, and the optional unbind subtest wedges the device until +# reboot. The tester must identify the target device and opt into the +# destructive unbind tests. +# +# DAX_KMEM_TEST_DEV=daxX.Y test this specific device +# DAX_KMEM_TEST_DEV=auto auto-discover the first kmem-bound dax device +# (best-effort: it may be a device in use!) +# DAX_KMEM_TEST_UNBIND=1 also run the destructive unbind-while-online test +# +# If DAX_KMEM_TEST_DEV is unset the whole test SKIPs. +# +# A dax device can be provisioned with the memmap= boot param, e.g.: +# memmap=2G!4G +# +# then, in the booted system: +# +# ndctl create-namespace -m devdax -e namespace0.0 -f +# daxctl reconfigure-device -N -m system-ram dax0.0 # bind kmem +# DAX_KMEM_TEST_DEV=auto ./dax-kmem-hotplug.sh + +# shellcheck disable=SC1091 +DIR="$(dirname "$(readlink -f "$0")")" +. "$DIR"/../kselftest/ktap_helpers.sh + +DAX_BASE=/sys/bus/dax/devices +MEM_BASE=/sys/devices/system/memory + +memtotal_kb() { awk '/^MemTotal:/ {print $2}' /proc/meminfo; } +get_state() { cat "$HP" 2>/dev/null; } +# set_state STATE -- write a state to the state attribute; returns the +# write's exit status (0 = accepted by the kernel) +set_state() { echo "$1" > "$HP" 2>/dev/null; } + +is_kmem_dax() { + local drv + [ -e "$DAX_BASE/$1/state" ] || return 1 + drv=$(readlink "$DAX_BASE/$1/driver" 2>/dev/null) + [ "$(basename "${drv:-}")" = kmem ] +} + +find_kmem_dax() { + local d + for d in "$DAX_BASE"/dax*; do + is_kmem_dax "$(basename "$d")" || continue + basename "$d" + return 0 + done + return 1 +} + +# find_device_blocks -- print every memoryN block backing this dax device. +# The blocks are derived from the device's own range(s) in /proc/iomem (the +# reserved resource is named after the device), so we act on *its* blocks +# rather than guessing by NUMA node - the target node may also hold unrelated +# (and non-offlineable) memory. +find_device_blocks() { + local bs + bs=$(cat "$MEM_BASE/block_size_bytes" 2>/dev/null) # hex, no leading 0x + [ -n "$bs" ] || return 1 + grep -E " : ${DAX}\$" /proc/iomem | while read -r line; do + local range s e i + range=${line%% :*}; range=${range// /} + s=${range%-*}; e=${range#*-} + for (( i = 0x$s / 0x$bs; i <= 0x$e / 0x$bs; i++ )); do + echo "memory$i" + done + done +} + +# find_device_block -- print the first online block backing this dax device. +find_device_block() { + local b + for b in $(find_device_blocks); do + [ -f "$MEM_BASE/$b/state" ] || continue + [ "$(cat "$MEM_BASE/$b/state")" = online ] || continue + echo "$b" + return 0 + done + return 1 +} + +ktap_print_header + +if [ "$UID" != 0 ]; then + ktap_skip_all "must be run as root" + exit "$KSFT_SKIP" +fi + +# Device selection is opt-in - see the header for why. +DEV_SEL=${DAX_KMEM_TEST_DEV:-} +if [ -z "$DEV_SEL" ]; then + ktap_skip_all "set DAX_KMEM_TEST_DEV=<daxX.Y|auto> to opt in (mutates device memory)" + exit "$KSFT_SKIP" +fi +if [ "$DEV_SEL" = auto ]; then + DAX=$(find_kmem_dax) +else + DAX=$DEV_SEL +fi +if [ -z "$DAX" ] || ! is_kmem_dax "$DAX"; then + ktap_skip_all "no kmem-bound dax device with a state attribute (${DEV_SEL})" + exit "$KSFT_SKIP" +fi +HP=$DAX_BASE/$DAX/state +ORIG=$(get_state) + +# A failure to reach the baseline is environmental (memory in use), not an +# interface failure, so skip rather than fail. +set_state unplugged; rc=$? +if [ "$rc" != 0 ] || [ "$(get_state)" != unplugged ]; then + ktap_skip_all "$DAX: cannot reach 'unplugged' baseline (memory in use?)" + [ -n "$ORIG" ] && set_state "$ORIG" + exit "$KSFT_SKIP" +fi +mt_unplugged=$(memtotal_kb) + +DRV=/sys/bus/dax/drivers/kmem +AOB=$MEM_BASE/auto_online_blocks + +ktap_print_msg "using $DAX (initial state was: $ORIG)" +ktap_set_plan 10 + +# A public (N_MEMORY) kmem node onlined into a kernel zone (online/online_kernel) +# collects unmovable allocations and can then never be offlined, which would +# wedge the device for the rest of this test. So this test only ever +# successfully onlines online_movable, the one mode that is reliably unpluggable. + +set_state online_movable; rc=$? +mt_online=$(memtotal_kb) +if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ] && [ "$mt_online" -gt "$mt_unplugged" ]; then + ktap_test_pass "online_movable: state=online_movable, MemTotal $mt_unplugged -> $mt_online kB" +else + ktap_test_fail "online_movable: rc=$rc state=$(get_state) MemTotal $mt_unplugged -> $mt_online" +fi + +set_state online_movable; rc=$? +if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ]; then + ktap_test_pass "online_movable idempotent" +else + ktap_test_fail "online_movable idempotent: rc=$rc state=$(get_state)" +fi + +# A different online type is rejected without an intervening unplug. The write +# is refused before any hotplug, so this never actually onlines a kernel zone. +set_state online_kernel; rc=$? +if [ "$rc" != 0 ] && [ "$(get_state)" = online_movable ]; then + ktap_test_pass "reject online_kernel without intervening unplug (no kernel-zone online)" +else + ktap_test_fail "online_movable->online_kernel not rejected: rc=$rc state=$(get_state)" +fi + +set_state unplugged; rc=$? +mt=$(memtotal_kb) +if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -lt "$mt_online" ]; then + ktap_test_pass "unplug from online_movable: MemTotal $mt_online -> $mt kB" +else + ktap_test_fail "unplug from online_movable: rc=$rc state=$(get_state) MemTotal $mt_online -> $mt" +fi + +before=$(get_state) +set_state bogus_state; rc=$? +if [ "$rc" != 0 ] && [ "$(get_state)" = "$before" ]; then + ktap_test_pass "reject invalid state string" +else + ktap_test_fail "invalid state not rejected: rc=$rc state=$(get_state)" +fi + +# An online_movable -> unplug cycle must re-acquire the per-range resources on +# each online and release them on each unplug. Assert every iteration grows +# MemTotal past the baseline and returns exactly to it; memory left online after +# unplug (off > baseline) is a partial-free failure. +set_state unplugged +cycle_ok=1; fail_i=0; on=0; off=0 +for i in 1 2 3; do + if ! set_state online_movable; then cycle_ok=0; fail_i=$i; break; fi + on=$(memtotal_kb) + if ! set_state unplugged; then cycle_ok=0; fail_i=$i; break; fi + off=$(memtotal_kb) + # online must grow past baseline, and unplug must return to it - a + # partial free (memory left online) is a failure, not just off == on. + if [ "$on" -le "$mt_unplugged" ] || [ "$off" -gt "$mt_unplugged" ]; then + cycle_ok=0; fail_i=$i; break + fi +done +if [ "$cycle_ok" = 1 ]; then + ktap_test_pass "online_movable/unplug cycle re-acquires resources (3x: added and freed each time)" +else + ktap_test_fail "online_movable/unplug cycle regressed at iteration $fail_i (on=$on off=$off baseline=$mt_unplugged)" +fi + +# Desync: toggle a block through the legacy per-block memoryN/state interface +# behind the driver's back, then unplug the whole device via daxX.Y/state. +# +# The driver only updates daxX.Y/state on its own writes, so it still reports +# online_movable while a block underneath is already offline. +# +# Whole-device unplug must still succeed (within reason, an actor changing a +# device from online_movable to online_kernel can no longer guarantee unplug). +# At the very least, an already-offline block should not produce an error. +set_state unplugged +set_state online_movable +blk=$(find_device_block) +if [ -n "$blk" ] && echo offline > "$MEM_BASE/$blk/state" 2>/dev/null; then + # daxX.Y/state is now stale (still online_movable); unplug the device. + set_state unplugged; rc=$? + mt=$(memtotal_kb) + if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -le "$mt_unplugged" ]; then + ktap_test_pass "unplug tolerates a block pre-offlined via memoryN/state ($blk)" + else + ktap_test_fail "desync unplug: rc=$rc state=$(get_state) MemTotal=$mt baseline=$mt_unplugged" + fi +else + set_state unplugged 2>/dev/null + ktap_test_skip "could not locate a device block to offline for desync test" +fi + +# change system default online policy while the device is unbound, and show +# the new system default policy is utilized across bindings. +set_state unplugged +if [ -w "$AOB" ] && [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then + orig_aob=$(cat "$AOB") + echo "$DAX" > "$DRV/unbind" 2>/dev/null + echo offline > "$AOB" 2>/dev/null + echo "$DAX" > "$DRV/bind" 2>/dev/null + sleep 1 + st=$(get_state) + echo "$orig_aob" > "$AOB" 2>/dev/null # restore system policy + if [ "$st" = offline ]; then + ktap_test_pass "online policy resolved at bind: auto_online_blocks=offline -> state=offline" + else + ktap_test_fail "bind-time policy not honored: state=$st (expected offline)" + fi + set_state unplugged 2>/dev/null +else + ktap_test_skip "auto_online_blocks or driver bind/unbind not writable" +fi + +# Blocks offlined out-of-band (via memoryN/state) leave daxX.Y/state stale +# (still online_movable) while every block is actually offline. A driver unbind +# must still hot-remove the offline memory and free its resources rather than +# trust the stale state and leak until reboot. Unbind uses remove_memory(), +# which never offlines, so removing already-offline blocks is non-destructive and +# the device rebinds cleanly afterwards. +if [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then + set_state unplugged + set_state online_movable + offl_ok=1 + for b in $(find_device_blocks); do + [ -f "$MEM_BASE/$b/state" ] || continue + [ "$(cat "$MEM_BASE/$b/state")" = online ] || continue + echo offline > "$MEM_BASE/$b/state" 2>/dev/null || offl_ok=0 + done + # daxX.Y/state is now stale (still online_movable) while all blocks are + # offline; the unbind must hot-remove them anyway. + if [ "$offl_ok" = 1 ] && [ "$(get_state)" = online_movable ]; then + echo "$DAX" > "$DRV/unbind" 2>/dev/null + mt_after=$(memtotal_kb) + leaked=$(grep -cE " : ${DAX}\$" /proc/iomem) # before rebind + echo "$DAX" > "$DRV/bind" 2>/dev/null # restore for later steps + sleep 1 + if [ "$mt_after" -le "$mt_unplugged" ] && [ "$leaked" = 0 ]; then + ktap_test_pass "unbind with stale online state hot-removes offlined blocks (no leak)" + else + ktap_test_fail "desync unbind leaked: MemTotal=$mt_after baseline=$mt_unplugged iomem_left=$leaked" + fi + set_state unplugged 2>/dev/null + else + ktap_test_skip "could not offline all device blocks for desync-unbind test" + fi +else + ktap_test_skip "driver bind/unbind not writable for desync-unbind test" +fi + +[ -n "$ORIG" ] && set_state "$ORIG" + +# DESTRUCTIVE and opt-in only (DAX_KMEM_TEST_UNBIND=1): +# +# unbinding the driver while memory is online causes the resources to leak - but +# the unbind should not deadlock. Instead the driver leaks it with a warning. + +# This leaves the memory online and the device unbound until reboot, so it runs +# last and only when explicitly requested. online_movable only: this test +# never onlines a public node into a kernel zone. + +if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ -w "$DRV/unbind" ]; then + set_state unplugged; set_state online_movable +fi +if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ "$(get_state)" = online_movable ] && + [ -w "$DRV/unbind" ]; then + mt_on=$(memtotal_kb) + dmesg -C 2>/dev/null + echo "$DAX" > "$DRV/unbind" 2>/dev/null + mt_after=$(memtotal_kb) + # The leaked "System RAM (kmem)" regions stay in the iomem tree; reading + # their names dereferences res_name, which a buggy unbind already freed. + # Walk /proc/iomem to provoke that use-after-free (caught by KASAN). + cat /proc/iomem > /dev/null 2>&1 + splat=$(dmesg 2>/dev/null | grep -ciE "KASAN|BUG:|use-after-free|general protection|Oops|refcount_t") + if [ "$splat" = 0 ] && [ "$mt_after" -ge "$mt_on" ]; then + ktap_test_pass "unbind while online: memory left online, no UAF/oops (MemTotal $mt_on -> $mt_after kB)" + else + ktap_test_fail "unbind while online regressed: splat=$splat MemTotal $mt_on -> $mt_after kB" + fi +else + ktap_test_skip "destructive unbind-while-online test (set DAX_KMEM_TEST_UNBIND=1)" +fi + +ktap_finished diff --git a/tools/testing/selftests/dax/settings b/tools/testing/selftests/dax/settings new file mode 100644 index 000000000000..ba4d85f74cd6 --- /dev/null +++ b/tools/testing/selftests/dax/settings @@ -0,0 +1 @@ +timeout=90 diff --git a/tools/testing/selftests/mincore/mincore_selftest.c b/tools/testing/selftests/mincore/mincore_selftest.c index cdd022c1c497..18dfbf318b1f 100644 --- a/tools/testing/selftests/mincore/mincore_selftest.c +++ b/tools/testing/selftests/mincore/mincore_selftest.c @@ -242,8 +242,10 @@ TEST(check_file_mmap) } /* - * Touch a page in the middle of the mapping. We expect the next - * few pages (the readahead window) to be populated too. + * Touch a page in the middle of the mapping. We expect some + * surrounding pages (the readahead window) to be populated too. + * Depending on the page size and readahead setting, the pages may + * land before the faulted page rather than after it. */ addr[FILE_SIZE / 2] = 1; retval = mincore(addr, FILE_SIZE, vec); @@ -252,6 +254,12 @@ TEST(check_file_mmap) TH_LOG("Page not found in memory after use"); } + i = FILE_SIZE / 2 / page_size - 1; + while (i >= 0 && vec[i]) { + ra_pages++; + i--; + } + i = FILE_SIZE / 2 / page_size + 1; while (i < vec_size && vec[i]) { ra_pages++; diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index e6df968f0971..eae504bd94c8 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -66,7 +66,6 @@ TEST_GEN_FILES += hugetlb-mremap TEST_GEN_FILES += hugetlb-read-hwpoison TEST_GEN_FILES += hugetlb-shm TEST_GEN_FILES += hugetlb-soft-offline -TEST_GEN_FILES += hugetlb-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate TEST_GEN_FILES += map_fixed_noreplace @@ -150,6 +149,7 @@ TEST_PROGS += ksft_gup_test.sh TEST_PROGS += ksft_hmm.sh TEST_PROGS += ksft_hugetlb.sh TEST_PROGS += ksft_hugevm.sh +TEST_PROGS += ksft_kmemleak_confirm.sh TEST_PROGS += ksft_kmemleak_dedup.sh TEST_PROGS += ksft_ksm.sh TEST_PROGS += ksft_ksm_numa.sh @@ -174,6 +174,10 @@ TEST_PROGS += ksft_userfaultfd.sh TEST_PROGS += ksft_vma_merge.sh TEST_PROGS += ksft_vmalloc.sh +# Destructive: every successful run panics the kernel. Installed and +# kept executable, but not run from a default kselftest invocation. +TEST_PROGS_EXTENDED += hwpoison-panic.sh + TEST_FILES := test_vmalloc.sh TEST_FILES += test_hmm.sh TEST_FILES += va_high_addr_switch.sh diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 2f2b9879d100..e2642eca0d02 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -29,6 +29,10 @@ #include <sys/mman.h> #include <sys/ioctl.h> #include <sys/time.h> +#include <sys/syscall.h> +#include <sys/eventfd.h> +#include <linux/userfaultfd.h> +#include <poll.h> /* * This is a private UAPI to the kernel test module so it isn't exported @@ -225,6 +229,53 @@ static void hmm_buffer_free(struct hmm_buffer *buffer) } /* + * Allocate a buffer structure with memory mapping and mirror. + * + * @mmap_size: total size of the mmap region (may differ from @mirror_size + * for alignment padding in THP tests). + * @mirror_size: size of the mirror data buffer (the actual working set). + * @prot: protection flags for the mmap (e.g. PROT_READ | PROT_WRITE). + * @flags: flags for the mmap (e.g. MAP_PRIVATE, MAP_SHARED, + * MAP_ANONYMOUS, MAP_HUGETLB). + * @fd: file descriptor for the mmap; pass -1 for MAP_ANONYMOUS. + * + * All internal allocations are checked; returns NULL and cleans up on any + * failure. Caller must ASSERT_NE or otherwise check the return value. + */ +static struct hmm_buffer *hmm_buffer_alloc(unsigned long mmap_size, + unsigned long mirror_size, + int prot, int flags, + int fd) +{ + struct hmm_buffer *buffer; + + buffer = malloc(sizeof(*buffer)); + if (!buffer) { + perror("malloc buffer"); + return NULL; + } + + buffer->fd = fd; + buffer->size = mmap_size; + buffer->mirror = malloc(mirror_size); + if (!buffer->mirror) { + perror("malloc mirror"); + free(buffer); + return NULL; + } + + buffer->ptr = mmap(NULL, mmap_size, prot, flags, fd, 0); + if (buffer->ptr == MAP_FAILED) { + perror("mmap"); + free(buffer->mirror); + free(buffer); + return NULL; + } + + return buffer; +} + +/* * Create a temporary file that will be deleted on close. */ static int hmm_create_file(unsigned long size) @@ -314,20 +365,11 @@ TEST_F(hmm, anon_read) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* * Initialize buffer in system memory but leave the first two pages * zero (pte_none and pfn_zero). @@ -377,20 +419,11 @@ TEST_F(hmm, anon_read_prot) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -436,20 +469,11 @@ TEST_F(hmm, anon_write) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -484,20 +508,11 @@ TEST_F(hmm, anon_write_prot) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device reading a zero page of memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); ASSERT_EQ(ret, 0); @@ -559,20 +574,10 @@ TEST_F(hmm, anon_write_child) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size * 2, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size * 2; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size * 2, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - old_ptr = buffer->ptr; if (use_thp) { map = (void *)ALIGN((uintptr_t)buffer->ptr, size); @@ -661,20 +666,11 @@ TEST_F(hmm, anon_write_child_shared) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer->ptr so we can tell if it is written. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -736,20 +732,10 @@ TEST_F(hmm, anon_write_huge) size = 2 * read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - size /= 2; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); @@ -795,22 +781,11 @@ TEST_F(hmm, anon_write_hugetlbfs) size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; - buffer = malloc(sizeof(*buffer)); - ASSERT_NE(buffer, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, - -1, 0); - if (buffer->ptr == MAP_FAILED) { - free(buffer); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); + if (!buffer) SKIP(return, "Huge page could not be allocated"); - } - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) @@ -852,14 +827,11 @@ TEST_F(hmm, file_read) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ, + MAP_SHARED, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - /* Write initial contents of the file. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -867,12 +839,6 @@ TEST_F(hmm, file_read) ASSERT_EQ(len, size); memset(buffer->mirror, 0, size); - buffer->ptr = mmap(NULL, size, - PROT_READ, - MAP_SHARED, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); @@ -907,20 +873,11 @@ TEST_F(hmm, file_write) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -960,20 +917,11 @@ TEST_F(hmm, migrate) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1010,20 +958,11 @@ TEST_F(hmm, migrate_file_private) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1058,20 +997,11 @@ TEST_F(hmm, migrate_fault) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1114,18 +1044,11 @@ TEST_F(hmm, migrate_release) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1164,20 +1087,11 @@ TEST_F(hmm, migrate_shared) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, -ENOENT); @@ -1201,20 +1115,10 @@ TEST_F(hmm2, migrate_mixed) npages = 6; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Migrating a protected area should be an error. */ @@ -1295,20 +1199,11 @@ TEST_F(hmm, migrate_multiple) size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1354,20 +1249,11 @@ TEST_F(hmm, anon_read_multiple) size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; @@ -1420,20 +1306,11 @@ TEST_F(hmm, anon_teardown) int *ptr; int rc; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; @@ -1474,22 +1351,11 @@ TEST_F(hmm, mixedmap) npages = 1; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, npages, + PROT_READ | PROT_WRITE, + MAP_PRIVATE, self->fd); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); - - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - self->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device snapshotting CPU pagetables. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); @@ -1519,20 +1385,10 @@ TEST_F(hmm2, snapshot) npages = 7; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, npages, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Punch a hole after the first page address. */ @@ -1626,21 +1482,11 @@ TEST_F(hmm, compound) size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; - buffer = malloc(sizeof(*buffer)); - ASSERT_NE(buffer, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, - -1, 0); - if (buffer->ptr == MAP_FAILED) { - free(buffer); - return; - } - - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); + buffer = hmm_buffer_alloc(size, npages, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); + if (!buffer) + SKIP(return, "Huge page could not be allocated"); /* Initialize the pages the device will snapshot in buffer->ptr. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) @@ -1707,21 +1553,11 @@ TEST_F(hmm2, double_map) npages = 6; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1783,20 +1619,11 @@ TEST_F(hmm, exclusive) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1837,20 +1664,11 @@ TEST_F(hmm, exclusive_mprotect) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1892,20 +1710,11 @@ TEST_F(hmm, exclusive_cow) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1993,20 +1802,11 @@ TEST_F(hmm, hmm_gup_test) npages = 4; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -2079,20 +1879,11 @@ TEST_F(hmm, hmm_cow_in_device) npages = 4; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -2152,21 +1943,12 @@ TEST_F(hmm, migrate_anon_huge_empty) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2204,21 +1986,12 @@ TEST_F(hmm, migrate_anon_huge_zero) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2267,21 +2040,12 @@ TEST_F(hmm, migrate_anon_huge_free) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2329,21 +2093,12 @@ TEST_F(hmm, migrate_anon_huge_fault) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); old_ptr = buffer->ptr; @@ -2425,21 +2180,11 @@ TEST_F(hmm, migrate_partial_unmap_fault) for (use_thp = 0; use_thp < 2; ++use_thp) { for (j = 0; j < ARRAY_SIZE(offsets); ++j) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); if (use_thp) @@ -2505,21 +2250,11 @@ TEST_F(hmm, migrate_remap_fault) if (dont_unmap) flags |= MREMAP_DONTUNMAP; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(8 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 8 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - - buffer->ptr = mmap(NULL, buffer->size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); if (use_thp) @@ -2590,18 +2325,13 @@ TEST_F(hmm, migrate_anon_huge_err) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, 2 * size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(2 * size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, 2 * size); - old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(old_ptr, MAP_FAILED); + old_ptr = buffer->ptr; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)old_ptr, size); @@ -2685,18 +2415,13 @@ TEST_F(hmm, migrate_anon_huge_zero_err) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, 2 * size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(2 * size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, 2 * size); - old_ptr = mmap(NULL, 2 * size, PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(old_ptr, MAP_FAILED); + old_ptr = buffer->ptr; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)old_ptr, size); @@ -2766,19 +2491,6 @@ static double get_time_ms(void) return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); } -static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size) -{ - struct hmm_buffer *buffer; - - buffer = malloc(sizeof(*buffer)); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - memset(buffer->mirror, 0xFF, size); - return buffer; -} - static void print_benchmark_results(const char *test_name, size_t buffer_size, struct benchmark_results *thp, struct benchmark_results *regular) @@ -2823,14 +2535,14 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz int ret, i; int *ptr; - buffer = hmm_buffer_alloc(buffer_size); - - /* Map memory */ - buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - - if (buffer->ptr == MAP_FAILED) - return -1; + buffer = hmm_buffer_alloc(buffer_size, buffer_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); + if (!buffer) { + ret = -1; + goto cleanup; + } + memset(buffer->mirror, 0xFF, buffer_size); /* Apply THP hint if requested */ if (use_thp) @@ -2839,7 +2551,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); if (ret) - return ret; + goto cleanup; /* Initialize memory to make sure pages are allocated */ ptr = (int *)buffer->ptr; @@ -2849,11 +2561,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz /* Warmup iteration */ ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; /* Benchmark iterations */ for (i = 0; i < iterations; i++) { @@ -2862,7 +2574,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); s2d_total += (end - start); @@ -2872,7 +2584,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); d2s_total += (end - start); @@ -2886,9 +2598,9 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / (results->dev_to_sys_time / 1000.0); - /* Cleanup */ +cleanup: hmm_buffer_free(buffer); - return 0; + return ret; } /* @@ -2949,4 +2661,150 @@ TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) &thp_results, ®ular_results); } } +/* + * Test that HMM can fault in pages backed by userfaultfd using the + * hmm_range_fault_unlocked_timeout() path with no timeout. This exercises + * the lock-drop retry logic in the HMM framework. + */ +struct uffd_thread_args { + int uffd; + int stop_fd; + void *page_buffer; + unsigned long page_size; +}; + +static void *uffd_handler_thread(void *arg) +{ + struct uffd_thread_args *args = arg; + struct uffd_msg msg; + struct uffdio_copy copy; + struct pollfd pollfd[2]; + int ret; + + pollfd[0].fd = args->uffd; + pollfd[0].events = POLLIN; + pollfd[1].fd = args->stop_fd; + pollfd[1].events = POLLIN; + + while (1) { + ret = poll(pollfd, 2, -1); + if (ret <= 0) + break; + if (pollfd[1].revents) + break; + if (!(pollfd[0].revents & POLLIN)) + break; + + ret = read(args->uffd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) + break; + + if (msg.event != UFFD_EVENT_PAGEFAULT) + break; + + /* Fill the page with a known pattern */ + memset(args->page_buffer, 0xAB, args->page_size); + + copy.dst = msg.arg.pagefault.address & ~(args->page_size - 1); + copy.src = (unsigned long)args->page_buffer; + copy.len = args->page_size; + copy.mode = 0; + copy.copy = 0; + + ret = ioctl(args->uffd, UFFDIO_COPY, ©); + if (ret < 0) + break; + } + + return NULL; +} + +TEST_F(hmm, userfaultfd_read) +{ + struct hmm_buffer *buffer; + struct uffd_thread_args uffd_args; + unsigned long npages; + unsigned long size; + unsigned long i; + unsigned char *ptr; + pthread_t thread; + int uffd; + int stop_fd; + int ret; + struct uffdio_api api; + struct uffdio_register reg; + uint64_t stop = 1; + ssize_t nwrite; + + npages = 4; + size = npages << self->page_shift; + + /* Create userfaultfd */ + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd < 0) + SKIP(return, "userfaultfd not available"); + + api.api = UFFD_API; + api.features = 0; + ret = ioctl(uffd, UFFDIO_API, &api); + ASSERT_EQ(ret, 0); + + buffer = malloc(sizeof(*buffer)); + ASSERT_NE(buffer, NULL); + + buffer->fd = -1; + buffer->size = size; + buffer->mirror = malloc(size); + ASSERT_NE(buffer->mirror, NULL); + + /* Create anonymous mapping */ + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ASSERT_NE(buffer->ptr, MAP_FAILED); + + /* Register the region with userfaultfd */ + reg.range.start = (unsigned long)buffer->ptr; + reg.range.len = size; + reg.mode = UFFDIO_REGISTER_MODE_MISSING; + ret = ioctl(uffd, UFFDIO_REGISTER, ®); + ASSERT_EQ(ret, 0); + + /* Set up the handler thread */ + uffd_args.uffd = uffd; + stop_fd = eventfd(0, EFD_CLOEXEC); + ASSERT_GE(stop_fd, 0); + uffd_args.stop_fd = stop_fd; + uffd_args.page_buffer = malloc(self->page_size); + ASSERT_NE(uffd_args.page_buffer, NULL); + uffd_args.page_size = self->page_size; + + ret = pthread_create(&thread, NULL, uffd_handler_thread, &uffd_args); + ASSERT_EQ(ret, 0); + + /* + * Use the unlocked read path which allows the mmap lock to be + * dropped during the fault, enabling userfaultfd resolution. + */ + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ_UNLOCKED, + buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + /* Verify the device read the data filled by the uffd handler */ + ptr = buffer->mirror; + for (i = 0; i < size; ++i) + ASSERT_EQ(ptr[i], (unsigned char)0xAB); + + nwrite = write(stop_fd, &stop, sizeof(stop)); + ASSERT_EQ(nwrite, sizeof(stop)); + pthread_join(thread, NULL); + close(stop_fd); + free(uffd_args.page_buffer); + close(uffd); + hmm_buffer_free(buffer); +} + + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index 2eab2110ac6a..d7917dce3aba 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -437,7 +437,7 @@ void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num(path, nr); + write_num_ignore_einval(path, nr); } unsigned long hugetlb_free_pages(unsigned long size) diff --git a/tools/testing/selftests/mm/hugetlb-vmemmap.c b/tools/testing/selftests/mm/hugetlb-vmemmap.c deleted file mode 100644 index 507df78a158d..000000000000 --- a/tools/testing/selftests/mm/hugetlb-vmemmap.c +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * A test case of using hugepage memory in a user application using the - * mmap system call with MAP_HUGETLB flag. Before running this program - * make sure the administrator has allocated enough default sized huge - * pages to cover the 2 MB allocation. - */ -#include <stdlib.h> -#include <stdio.h> -#include <unistd.h> -#include <sys/mman.h> -#include <fcntl.h> -#include "vm_util.h" -#include "hugepage_settings.h" - -#define PAGE_COMPOUND_HEAD (1UL << 15) -#define PAGE_COMPOUND_TAIL (1UL << 16) -#define PAGE_HUGE (1UL << 17) - -#define HEAD_PAGE_FLAGS (PAGE_COMPOUND_HEAD | PAGE_HUGE) -#define TAIL_PAGE_FLAGS (PAGE_COMPOUND_TAIL | PAGE_HUGE) - -#define PM_PFRAME_BITS 55 -#define PM_PFRAME_MASK ~((1UL << PM_PFRAME_BITS) - 1) - -static size_t pagesize; -static size_t maplength; - -static void write_bytes(char *addr, size_t length) -{ - unsigned long i; - - for (i = 0; i < length; i++) - *(addr + i) = (char)i; -} - -static unsigned long virt_to_pfn(void *addr) -{ - int fd; - unsigned long pagemap; - - fd = open("/proc/self/pagemap", O_RDONLY); - if (fd < 0) - return -1UL; - - lseek(fd, (unsigned long)addr / pagesize * sizeof(pagemap), SEEK_SET); - read(fd, &pagemap, sizeof(pagemap)); - close(fd); - - return pagemap & ~PM_PFRAME_MASK; -} - -static int check_page_flags(unsigned long pfn) -{ - int fd, i; - unsigned long pageflags; - - fd = open("/proc/kpageflags", O_RDONLY); - if (fd < 0) - return -1; - - lseek(fd, pfn * sizeof(pageflags), SEEK_SET); - - read(fd, &pageflags, sizeof(pageflags)); - if ((pageflags & HEAD_PAGE_FLAGS) != HEAD_PAGE_FLAGS) { - close(fd); - ksft_print_msg("Head page flags (%lx) is invalid\n", pageflags); - return -1; - } - - /* - * pages other than the first page must be tail and shouldn't be head; - * this also verifies kernel has correctly set the fake page_head to tail - * while hugetlb_free_vmemmap is enabled. - */ - for (i = 1; i < maplength / pagesize; i++) { - read(fd, &pageflags, sizeof(pageflags)); - if ((pageflags & TAIL_PAGE_FLAGS) != TAIL_PAGE_FLAGS || - (pageflags & HEAD_PAGE_FLAGS) == HEAD_PAGE_FLAGS) { - close(fd); - ksft_print_msg("Tail page flags (%lx) is invalid\n", pageflags); - return -1; - } - } - - close(fd); - - return 0; -} - -int main(int argc, char **argv) -{ - void *addr; - unsigned long pfn; - int ret; - - ksft_print_header(); - ksft_set_plan(1); - - if (!hugetlb_setup_default(1)) - ksft_exit_skip("Not enough free huge pages\n"); - - pagesize = psize(); - maplength = default_huge_page_size(); - if (!maplength) - ksft_exit_skip("Unable to determine huge page size\n"); - - addr = mmap(NULL, maplength, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_perror("mmap"); - - /* Trigger allocation of HugeTLB page. */ - write_bytes(addr, maplength); - - pfn = virt_to_pfn(addr); - if (pfn == -1UL) { - ksft_perror("virt_to_pfn"); - munmap(addr, maplength); - ksft_exit_fail(); - } - - ksft_print_msg("Returned address is %p whose pfn is %lx\n", addr, pfn); - - ret = check_page_flags(pfn); - - if (munmap(addr, maplength)) - ksft_exit_fail_perror("munmap"); - - ksft_test_result(!ret, "HugeTLB vmemmap page flags\n"); - ksft_finished(); -} diff --git a/tools/testing/selftests/mm/hwpoison-panic.sh b/tools/testing/selftests/mm/hwpoison-panic.sh new file mode 100755 index 000000000000..d953d1367332 --- /dev/null +++ b/tools/testing/selftests/mm/hwpoison-panic.sh @@ -0,0 +1,255 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Verify vm.panic_on_unrecoverable_memory_failure by injecting a hwpoison +# error on a kernel-owned page and confirming the kernel panics. +# +# Three "kinds" of kernel-owned page can be targeted, selectable via the +# first positional argument (default: rodata): +# +# rodata - a PG_reserved page in the kernel rodata range +# (sourced from /proc/iomem "Kernel rodata"). Exercises +# memory_failure() -> get_any_page() on a PageReserved page. +# +# slab - a slab page found via /proc/kpageflags (KPF_SLAB). +# Exercises memory_failure() -> get_any_page() on a non +# PG_reserved kernel-owned page. This path is what catches +# regressions where get_any_page() collapses kernel-owned +# pages into a transient -EIO instead of -ENOTRECOVERABLE. +# +# pgtable - a page-table page found via /proc/kpageflags (KPF_PGTABLE). +# Same path as slab, different page type. +# +# This test is DESTRUCTIVE: a successful run crashes the kernel. It is +# meant to be executed inside a disposable VM (e.g. virtme-ng) with a +# serial console captured by the harness. It is skipped unless the +# caller opts in via RUN_DESTRUCTIVE=1. +# +# Test passes externally: the kernel must panic with +# "Memory failure: <pfn>: unrecoverable page" +# A return from the inject means no panic fired: that is a failure, +# unless the target PFN raced to a different page type before injection, +# in which case the run is inconclusive and is skipped. +# +# Author: Breno Leitao <leitao@debian.org> + +set -u + +# KTAP output helpers (ktap_print_msg, ktap_skip_all, ktap_exit_fail_msg, ...). +DIR="$(dirname "$(readlink -f "$0")")" +# shellcheck source=../kselftest/ktap_helpers.sh +source "${DIR}"/../kselftest/ktap_helpers.sh + +sysctl_path=/proc/sys/vm/panic_on_unrecoverable_memory_failure +inject_path=/sys/devices/system/memory/hard_offline_page +kpageflags_path=/proc/kpageflags +unpoison_path=/sys/kernel/debug/hwpoison/unpoison-pfn + +# /proc/kpageflags bit positions (see include/uapi/linux/kernel-page-flags.h) +KPF_SLAB=7 +KPF_COMPOUND_TAIL=16 +KPF_HWPOISON=19 +KPF_NOPAGE=20 +KPF_PGTABLE=26 +KPF_RESERVED=32 + +pagesize=$(getconf PAGE_SIZE) + +kind=${1:-rodata} + +if [ "$(id -u)" -ne 0 ]; then + ktap_skip_all "must run as root" + exit "$KSFT_SKIP" +fi + +if [ ! -w "$sysctl_path" ]; then + ktap_skip_all "$sysctl_path not present (kernel without the sysctl?)" + exit "$KSFT_SKIP" +fi + +if [ ! -w "$inject_path" ]; then + ktap_skip_all "$inject_path not present (no MEMORY_HOTPLUG?)" + exit "$KSFT_SKIP" +fi + +if [ "${RUN_DESTRUCTIVE:-0}" != "1" ]; then + ktap_skip_all "destructive test; re-run with RUN_DESTRUCTIVE=1 inside a disposable VM" + exit "$KSFT_SKIP" +fi + +# Pick a PFN inside the kernel image rodata region of /proc/iomem. +# This is preferred over a top-level "Reserved" entry because top-level +# Reserved ranges are often firmware holes that have no backing struct +# page; pfn_to_online_page() returns NULL on those and memory_failure() +# bails out with -ENXIO before reaching the panic path. +# +# "Kernel rodata" is reported as a sub-resource of "System RAM" on every +# major architecture, which guarantees: +# - the PFN is backed by struct page (within an online memory range); +# - PG_reserved is set on the page (kernel image area); +# - the memory is read-only, so setting PG_hwpoison on it does not +# corrupt writable kernel state if the panic somehow does not fire. +# +# /proc/iomem entries look like (indented for sub-resources): +# " 02500000-02ffffff : Kernel rodata" +pick_rodata_phys_addr() { + awk -v pagesize="$(getconf PAGE_SIZE)" ' + # Convert a hex string to a number without relying on the gawk-only + # strtonum(). mawk lacks it and would otherwise spuriously skip + # this test on distros that ship mawk as /usr/bin/awk. + function hex2num(s, n, i, c, v) { + n = 0 + for (i = 1; i <= length(s); i++) { + c = tolower(substr(s, i, 1)) + v = index("0123456789abcdef", c) - 1 + if (v < 0) + return -1 + n = n * 16 + v + } + return n + } + /: Kernel rodata[[:space:]]*$/ { + sub(/^[[:space:]]+/, "") + n = split($0, a, /[- ]/) + start = hex2num(a[1]) + end = hex2num(a[2]) + if (end <= start) + next + # Page-align upward and emit the first byte of that page. + pfn = int((start + pagesize - 1) / pagesize) + printf "0x%x\n", pfn * pagesize + exit 0 + } + ' /proc/iomem +} + +# Walk /proc/kpageflags and return the phys addr of the first PFN that +# has bit $1 set, with KPF_HWPOISON, KPF_NOPAGE and KPF_COMPOUND_TAIL +# all clear (so we attack a real, non-tail, not-already-poisoned page). +# +# We skip the first 16 MiB of PFNs to step past low-memory special +# ranges (BIOS/EFI/ACPI/etc.) that often are PG_reserved and would not +# exhibit the slab/pgtable type we are looking for. +pick_kpageflags_phys_addr() { + local want_bit=$1 + local pagesize skip_pfn + + [ -r "$kpageflags_path" ] || return + + pagesize=$(getconf PAGE_SIZE) + skip_pfn=$(((16 * 1024 * 1024) / pagesize)) + + od -An -tx8 -v -w8 -j "$((skip_pfn * 8))" "$kpageflags_path" 2>/dev/null | \ + awk -v want_bit="$want_bit" \ + -v hwp_bit="$KPF_HWPOISON" \ + -v nopage_bit="$KPF_NOPAGE" \ + -v tail_bit="$KPF_COMPOUND_TAIL" \ + -v base_pfn="$skip_pfn" \ + -v pagesize="$pagesize" ' + # Test whether bit "b" is set in the 16-hex-digit value "hex". + # Done with substring + per-digit lookup so we never rely on awk + # bitwise operators (mawk lacks them), 64-bit FP precision or the + # gawk-only strtonum(). + function bit_set(hex, b, di, bi, c, v) { + di = int(b / 4) + bi = b - di * 4 + c = substr(hex, length(hex) - di, 1) + v = index("0123456789abcdef", tolower(c)) - 1 + if (bi == 0) return (v % 2) == 1 + if (bi == 1) return int(v / 2) % 2 == 1 + if (bi == 2) return int(v / 4) % 2 == 1 + return int(v / 8) % 2 == 1 + } + { + gsub(/^[[:space:]]+/, "") + h = $1 + if (bit_set(h, want_bit) && + !bit_set(h, hwp_bit) && + !bit_set(h, nopage_bit) && + !bit_set(h, tail_bit)) { + pfn = base_pfn + NR - 1 + printf "0x%x\n", pfn * pagesize + exit 0 + } + } + ' +} + +# Return 0 if /proc/kpageflags bit $2 is set for PFN $1, 1 if it is +# clear, or 2 if the word cannot be read. Used to re-confirm the target +# page type after a non-panicking inject. +kpageflags_bit_set() { + local word + + word=$(od -An -tx8 -v -j "$(($1 * 8))" -N 8 "$kpageflags_path" 2>/dev/null | tr -d '[:space:]') + [ -n "$word" ] || return 2 + (( (16#$word >> $2) & 1 )) +} + +# Best-effort: drop the PG_hwpoison marker set by the inject so a failed +# run does not leave a poisoned page behind. hard_offline_page() injects +# with MF_SW_SIMULATED, so the page stays unpoisonable through the +# hwpoison debugfs interface (needs CONFIG_HWPOISON_INJECT + debugfs). +try_unpoison() { + [ -w "$unpoison_path" ] || return 0 + echo "$1" > "$unpoison_path" 2>/dev/null || true +} + +case "$kind" in +rodata) + phys_addr=$(pick_rodata_phys_addr) + recheck_bit=$KPF_RESERVED + missing_msg='no "Kernel rodata" entry in /proc/iomem' + ;; +slab) + phys_addr=$(pick_kpageflags_phys_addr "$KPF_SLAB") + recheck_bit=$KPF_SLAB + missing_msg="no usable slab PFN found in $kpageflags_path" + ;; +pgtable) + phys_addr=$(pick_kpageflags_phys_addr "$KPF_PGTABLE") + recheck_bit=$KPF_PGTABLE + missing_msg="no usable page-table PFN found in $kpageflags_path" + ;; +*) + ktap_exit_fail_msg "unknown kind '$kind' (expected: rodata|slab|pgtable)" + ;; +esac + +if [ -z "$phys_addr" ]; then + ktap_skip_all "$missing_msg" + exit "$KSFT_SKIP" +fi + +ktap_print_msg "enabling $sysctl_path" +prior=$(cat "$sysctl_path") +echo 1 > "$sysctl_path" || ktap_exit_fail_msg "failed to enable sysctl" + +pfn=$((phys_addr / pagesize)) +ktap_print_msg "injecting hwpoison at phys 0x$(printf '%x' "$phys_addr") (pfn 0x$(printf '%x' "$pfn"), kind=$kind)" +ktap_print_msg "expecting kernel panic: 'Memory failure: <pfn>: unrecoverable page'" + +# A successful run never returns from the inject -- it panics the kernel. +# Reaching the code below therefore means no panic fired. Note whether +# the write itself succeeded, then put the machine back: restore the +# sysctl and best-effort unpoison the page we just marked. +if echo "$phys_addr" > "$inject_path"; then + verdict="inject returned without panic; sysctl ineffective" +else + verdict="inject failed before reaching the panic path" +fi + +echo "$prior" > "$sysctl_path" +try_unpoison "$pfn" + +# The page type can change between selection and injection (e.g. a slab +# or page-table page is freed and reused). Only treat a missing panic as +# a failure if the target PFN is still the kernel-owned type we aimed at; +# if it raced to another type the run is inconclusive, so skip instead. +kpageflags_bit_set "$pfn" "$recheck_bit" +case $? in +0) ktap_exit_fail_msg "$verdict (page still $kind)" ;; +1) ktap_skip_all "target PFN no longer $kind; raced before inject, inconclusive" + exit "$KSFT_SKIP" ;; +*) ktap_exit_fail_msg "$verdict (could not reconfirm page type via $kpageflags_path)" ;; +esac diff --git a/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh new file mode 100755 index 000000000000..3a8576e835c8 --- /dev/null +++ b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Functional test for kmemleak's N-consecutive-scan leak confirmation +# (the min_unref_scans module parameter). +# +# kmemleak only reports an object once it has stayed unreferenced for +# min_unref_scans consecutive scans. The default of 1 reports on the first +# scan (historical behaviour); higher values filter transient false +# positives where a live object's only reference is briefly invisible to a +# single scan (e.g. an RCU tree update in flight while the scan runs). The +# test loads samples/kmemleak's helper module to create orphan allocations +# and, counting only those orphans (matched by their [kmemleak_test] +# backtrace so unrelated leaks already present on the system are ignored), +# checks that: +# - a freshly allocated object is greyed on its first scan (its checksum +# settles then), so nothing can be reported before that priming scan; +# each case below primes once first, +# - with the default threshold (min_unref_scans=1) one scan after priming +# reports the orphans, +# - raising the threshold to 2 needs two scans after priming: one is not +# enough, the second reports, +# - the parameter reads back what was written. +# +# The "one post-prime scan is not enough at min_unref_scans=2" check is the +# core regression test: raising min_unref_scans must push the report +# strictly later. Like ksft_kmemleak_dedup.sh, if the module yields no +# detectable orphan at all in the running environment the test skips rather +# than failing. +# +# Author: Breno Leitao <leitao@debian.org> + +# KTAP output helpers (ktap_skip_all, ktap_exit_fail_msg, ktap_test_pass, ...). +DIR="$(dirname "$(readlink -f "$0")")" +# shellcheck source=../kselftest/ktap_helpers.sh +source "${DIR}"/../kselftest/ktap_helpers.sh + +KMEMLEAK=/sys/kernel/debug/kmemleak +PARAM=/sys/module/kmemleak/parameters/min_unref_scans +MODULE=kmemleak-test +AGE=6 # seconds; must exceed kmemleak's 5s minimum object age + +ktap_print_header + +[ "$(id -u)" -eq 0 ] || { ktap_skip_all "must run as root"; exit "$KSFT_SKIP"; } +[ -r "$KMEMLEAK" ] || + { ktap_skip_all "no kmemleak debugfs (CONFIG_DEBUG_KMEMLEAK)"; exit "$KSFT_SKIP"; } +[ -w "$PARAM" ] || + { ktap_skip_all "min_unref_scans module parameter not present"; exit "$KSFT_SKIP"; } +modinfo "$MODULE" >/dev/null 2>&1 || + { ktap_skip_all "$MODULE not built (CONFIG_SAMPLE_KMEMLEAK)"; exit "$KSFT_SKIP"; } + +# kmemleak can be present but disabled at runtime (kmemleak=off boot arg, +# or it self-disabled after an internal error); a "scan" then returns +# EPERM. Probe once and skip if so. +echo scan > "$KMEMLEAK" 2>/dev/null || + { ktap_skip_all "kmemleak is disabled (check dmesg or kmemleak= boot arg)"; exit "$KSFT_SKIP"; } + +prev=$(cat "$PARAM") +# shellcheck disable=SC2317 # invoked indirectly via trap +cleanup() { + echo "$prev" > "$PARAM" 2>/dev/null # restore the parameter + echo scan=on > "$KMEMLEAK" 2>/dev/null # re-enable auto scan + rmmod "$MODULE" 2>/dev/null + echo clear > "$KMEMLEAK" 2>/dev/null +} +trap cleanup EXIT + +# Stop the automatic scan thread: only our manual scans should advance an +# object's consecutive-unreferenced run. An auto scan landing between two +# manual scans would change the result and make the test flaky. +echo scan=off > "$KMEMLEAK" 2>/dev/null + +# Create a fresh, aged set of orphan objects from the helper module's init +# path (its kmalloc/vmalloc/percpu allocations are dropped right away). +# Pre-existing reported leaks are greyed first ("clear") so only our +# orphans are counted. The module is left loaded on purpose: once it is +# unloaded its symbols are gone, so the orphan backtraces no longer resolve +# to [kmemleak_test] and could not be matched below. +gen_orphans() { + rmmod "$MODULE" 2>/dev/null + echo clear > "$KMEMLEAK" + modprobe "$MODULE" || + { ktap_skip_all "failed to load $MODULE"; exit "$KSFT_SKIP"; } + sleep "$AGE" +} + +scan() { echo scan > "$KMEMLEAK"; } + +# Number of helper-module orphans currently reported by kmemleak. Matching +# the module's own backtrace ([kmemleak_test]) keeps the count immune to +# unrelated leaks on the running system. kmemleak only lists an object here +# once it has been reported, so this reflects the confirmation gating. +count_orphans() { + c=$(grep -c '\[kmemleak_test\]' "$KMEMLEAK" 2>/dev/null) + echo "${c:-0}" +} + +# 0) the parameter reads back what was written. +echo 3 > "$PARAM" +[ "$(cat "$PARAM")" = "3" ] || ktap_exit_fail_msg "min_unref_scans did not read back as 3" + +# Priming scan: kmemleak greys a freshly allocated object on its first scan +# (its checksum settles then), so nothing can be reported until a second +# scan. Every case below runs this priming scan before counting. +prime() { scan; } + +# 1) min_unref_scans=1 (default): one scan after priming reports the +# orphans. This also establishes that the helper produces detectable +# orphans here. +echo 1 > "$PARAM" +gen_orphans +prime +scan +first=$(count_orphans) +[ "$first" -gt 0 ] || + { ktap_skip_all "$MODULE produced no detectable orphans (cannot test min_unref_scans)"; exit "$KSFT_SKIP"; } + +# 2) min_unref_scans=2: after priming, one scan is not enough (still +# gated), the second reports. The gated-scan-zero check is the core +# regression. +echo 2 > "$PARAM" +gen_orphans +prime +scan; s1=$(count_orphans) +scan; s2=$(count_orphans) +[ "$s1" -eq 0 ] || ktap_exit_fail_msg "min_unref_scans=2: $s1 orphan(s) after 1 post-prime scan (must be 0)" +[ "$s2" -gt 0 ] || ktap_exit_fail_msg "min_unref_scans=2: no report after 2 post-prime scans (false negative)" + +ktap_set_plan 1 +ktap_test_pass "min_unref_scans=1 reported $first orphan(s) one scan after priming; =2 held them one scan longer ($s1 after one scan, $s2 after two); param read-back ok" +ktap_finished diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a050f4840cfa..5fd7792a0d47 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count) static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) { if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || @@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) static int ksm_restore(struct ksm_sysfs *ksm_sysfs) { if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || @@ -440,9 +440,9 @@ static int get_next_mem_node(int node) mem_node = i % (max_node + 1); node_size = numa_node_size(mem_node, NULL); if (node_size > 0) - break; + return mem_node; } - return mem_node; + return -ENODEV; } static int get_first_mem_node(void) @@ -455,8 +455,8 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo { void *numa1_map_ptr, *numa2_map_ptr; struct timespec start_time; + int first_node, second_node; int page_count = 2; - int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { ksft_perror("clock_gettime"); @@ -467,17 +467,19 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } - if (numa_num_configured_nodes() <= 1) { - ksft_print_msg("At least 2 NUMA nodes must be available\n"); + first_node = get_first_mem_node(); + second_node = get_next_mem_node(first_node); + + if (second_node < 0) { + ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) return KSFT_FAIL; /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ - first_node = get_first_mem_node(); numa1_map_ptr = numa_alloc_onnode(page_size, first_node); - numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); + numa2_map_ptr = numa_alloc_onnode(page_size, second_node); if (!numa1_map_ptr || !numa2_map_ptr) { ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; @@ -844,8 +846,8 @@ int main(int argc, char *argv[]) if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) ksft_exit_fail_msg("Cannot set up KSM tunables\n"); diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index 519e5ac02db7..edfbd39ae58e 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -1305,6 +1305,167 @@ TEST_F(merge, merge_vmas_with_mseal) ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size); } +TEST_F(merge, virt_and_page_offset_mismatch_memfd) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd; + + /* Create a 10 page memfd descriptor. */ + fd = memfd_create("virt_page_offset_test", MFD_CLOEXEC); + ASSERT_NE(fd, -1); + ASSERT_EQ(ftruncate(fd, 10 * page_size), 0); + + /* Map a region using the memfd at page offset 0. */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd, 0); + ASSERT_NE(ptr, MAP_FAILED); + + /* + * Map another separately, and fault in, at page offset 5: + * + * |-----------| |---------| + * | unfaulted | | faulted | + * |-----------| |---------| + */ + ptr2 = mmap(&carveout[10 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd, 5 * page_size); + ASSERT_NE(ptr2, MAP_FAILED); + ptr2[0] = 'x'; + + /* + * Now move it in place: + * + * |----------| + * | | + * v | + * |-----------| |---------| + * | unfaulted | | faulted | + * |-----------| |---------| + * + * Because virtual page offset of the faulted region is now + * &carveout[10 * page_size], despite the two regions being mergeable + * due to file page offset, they are NOT mergeable due to virtual page + * offset. + */ + ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, + MREMAP_MAYMOVE | MREMAP_FIXED, + &carveout[5 * page_size]); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they did not merge. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size); +} + +TEST_F(merge, merge_map_private_dev_zero_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map two MAP_PRIVATE-/dev/zero VMAs next to one another with offset 0 + * each. + * + * With these being made truly anonymous upon mapping, they will + * merge. If they were file-backed VMAs the page offsets would prevent + * merge: + * + * |-----||------| |-------------| + * | ptr || ptr2 | -> | ptr | + * |-----||------| |-------------| + */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + if (ptr == MAP_FAILED) { + close(fd_zero); + ASSERT_TRUE(false); + } + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + if (ptr2 == MAP_FAILED) { + close(fd_zero); + ASSERT_TRUE(false); + } + close(fd_zero); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size); +} + +TEST_F(merge, merge_map_private_dev_zero_faulted_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map a MAP_PRIVATE mapping of /dev/zero with page offset 0, then fault + * it in: + * + * |-------------------------------| + * | faulted | + * |-------------------------------| + */ + ptr = mmap(carveout, 15 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + if (ptr == MAP_FAILED) { + close(fd_zero); + ASSERT_TRUE(false); + } + memset(ptr, 'x', 15 * page_size); + + /* + * Unmap the middle: + * + * |---------| |---------| + * | faulted | | faulted | + * |---------| |---------| + */ + ASSERT_EQ(munmap(&ptr[5 * page_size], 5 * page_size), 0); + + /* + * Map in a new unfaulted mapping in the middle with page offset 0 - + * this should merge and would not if it were treated as a file rather + * than pure anon: + * + * |---------|-----------|---------| + * | faulted | unfaulted | faulted | + * |---------|-----------|---------| + */ + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd_zero, 0); + close(fd_zero); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size); +} + TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev) { struct procmap_fd *procmap = &self->procmap; diff --git a/tools/testing/selftests/mm/mremap_dontunmap.c b/tools/testing/selftests/mm/mremap_dontunmap.c index a4f75d836733..96ba537facf7 100644 --- a/tools/testing/selftests/mm/mremap_dontunmap.c +++ b/tools/testing/selftests/mm/mremap_dontunmap.c @@ -313,7 +313,7 @@ static void mremap_dontunmap_partial_mapping_overwrite(void) mremap(source_mapping, 5 * page_size, 5 * page_size, MREMAP_DONTUNMAP | MREMAP_MAYMOVE | MREMAP_FIXED, dest_mapping); - BUG_ON(dest_mapping == MAP_FAILED, "mremap"); + BUG_ON(remapped_mapping == MAP_FAILED, "mremap"); BUG_ON(dest_mapping != remapped_mapping, "expected to remap to dest_mapping"); BUG_ON(check_region_contains_byte(source_mapping, 5 * page_size, 0) != diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index f9bcff8e78fa..f6ab626a90b4 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -8,6 +8,7 @@ #include <errno.h> #include <malloc.h> #include <linux/types.h> +#include <linux/mman.h> #include <linux/memfd.h> #include <linux/userfaultfd.h> #include <linux/fs.h> @@ -1058,50 +1059,96 @@ static void test_simple(void) * the generic path (reached e.g. via category_anyof_mask) must report every * page written. */ -static void unpopulated_scan_test(void) +/* + * Populate @mem (optionally collapsing it into a THP first), drop it with + * MADV_DONTNEED, then check PAGEMAP_SCAN reports the whole range written via + * both the fast and generic query paths. A dropped THP leaves a pmd_none hole + * with no page table, exercising pagemap_scan_pte_hole(); a base-page range + * leaves pte_none entries. + */ +static void unpopulated_written_test(const char *name, char *mem, long size, + bool use_thp) { - int npages = 16, i; - long mem_size = npages * page_size; + long npages = size / page_size, fast = 0, slow = 0, ret; struct page_region regions[16]; - long fast = 0, slow = 0, ret; - char *mem; - - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("%s mmap failed\n", __func__); + int i; - wp_init(mem, mem_size); + wp_init(mem, size); - /* Populate, then drop: the ptes become pte_none without a marker. */ - memset(mem, 1, mem_size); - if (madvise(mem, mem_size, MADV_DONTNEED)) - ksft_exit_fail_msg("%s MADV_DONTNEED failed\n", __func__); + /* Populate, optionally collapse to a THP, then drop it. */ + memset(mem, 1, size); + if (use_thp && + (madvise(mem, size, MADV_COLLAPSE) || + !check_huge_anon(mem, size / hpage_size, hpage_size))) { + ksft_test_result_skip("%s could not form a THP\n", name); + goto out; + } + if (madvise(mem, size, MADV_DONTNEED)) { + ksft_test_result_fail("%s MADV_DONTNEED failed\n", name); + goto out; + } /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */ - ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, + ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("%s fast scan failed\n", __func__); - for (i = 0; i < ret; i++) + for (i = 0; ret > 0 && i < ret; i++) fast += LEN(regions[i]); /* Generic path: same query expressed via category_anyof_mask. */ - ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, + ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("%s generic scan failed\n", __func__); - for (i = 0; i < ret; i++) + for (i = 0; ret > 0 && i < ret; i++) slow += LEN(regions[i]); ksft_test_result(fast == npages && slow == npages, - "%s unpopulated ptes reported written by both paths (%ld, %ld of %d)\n", - __func__, fast, slow, npages); + "%s unpopulated range reported written by both paths (%ld, %ld of %ld)\n", + name, fast, slow, npages); +out: + wp_free(mem, size); +} - wp_free(mem, mem_size); +static void unpopulated_scan_test(void) +{ + long mem_size = 16 * page_size; + char *mem; + + mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + ksft_test_result_skip("%s mmap failed\n", __func__); + return; + } + + unpopulated_written_test(__func__, mem, mem_size, false); munmap(mem, mem_size); } +/* + * Same as unpopulated_scan_test(), but the range is a THP: a full-PMD + * MADV_DONTNEED leaves a pmd_none hole with no page table. + */ +static void unpopulated_thp_scan_test(void) +{ + char *area, *mem; + + if (!hpage_size) { + ksft_test_result_skip("%s THP not supported\n", __func__); + return; + } + + /* Over-allocate so a PMD-aligned, THP-sized range fits inside. */ + area = mmap(NULL, 2 * hpage_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) { + ksft_test_result_skip("%s mmap failed\n", __func__); + return; + } + mem = (char *)(((unsigned long)area + hpage_size - 1) & ~(hpage_size - 1)); + + unpopulated_written_test(__func__, mem, hpage_size, true); + munmap(area, 2 * hpage_size); +} + int sanity_tests(void) { unsigned long long mem_size, vec_size; @@ -1610,7 +1657,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) if (!hugetlb_setup_default(4)) ksft_print_msg("HugeTLB test will be skipped\n"); - ksft_set_plan(118); + ksft_set_plan(119); page_size = getpagesize(); hpage_size = read_pmd_pagesize(); @@ -1790,6 +1837,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) /* 18. Unpopulated pte scan-path consistency */ unpopulated_scan_test(); + unpopulated_thp_scan_test(); close(pagemap_fd); ksft_finished(); diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 2c377f4e9df1..46a8a1878dc1 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -68,7 +68,9 @@ static inline void sigsafe_printf(const char *format, ...) #define dprintf3(args...) dprintf_level(3, args) #define dprintf4(args...) dprintf_level(4, args) -extern void abort_hooks(void); +void tracing_on(void); +void tracing_off(void); +void abort_hooks(void); #define pkey_assert(condition) do { \ if (!(condition)) { \ dprintf0("# assert() at %s::%d test_nr: %d iteration: %d\n", \ diff --git a/tools/testing/selftests/mm/pkey-powerpc.h b/tools/testing/selftests/mm/pkey-powerpc.h index 17bf2d1b0192..2ce85580b404 100644 --- a/tools/testing/selftests/mm/pkey-powerpc.h +++ b/tools/testing/selftests/mm/pkey-powerpc.h @@ -126,7 +126,7 @@ static inline void *malloc_pkey_with_mprotect_subpage(long size, int prot, u16 p size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = syscall(__NR_subpage_prot, ptr, size, NULL); if (ret) { diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 302fef54049c..c218d0510a2a 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -19,7 +19,6 @@ #include <stdint.h> #include <stdbool.h> #include <signal.h> -#include <assert.h> #include <stdlib.h> #include <sys/mman.h> #include <sys/types.h> @@ -207,22 +206,25 @@ static void test_sigsegv_handler_with_pkey0_disabled(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + ret = pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -247,22 +249,25 @@ static void test_sigsegv_handler_cannot_access_stack(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + ret = pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -285,24 +290,22 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) static stack_t sigstack; void *stack; int pkey; - int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* Allow access to MPK 0 and MPK 1 */ pkey_reg = pkey_reg_restrictive_default(); @@ -317,33 +320,41 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; memset(&siginfo, 0, sizeof(siginfo)); /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_DETACHED, + stack + STACK_SIZE, + NULL, + NULL); if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_segv_maperr_ptr(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); } + child_pid = ret; + pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) pthread_cond_wait(&cond, &mutex); pthread_mutex_unlock(&mutex); + /* Wait for child to exit before returning */ + do { + sched_yield(); + ret = syscall_raw(SYS_tkill, child_pid, 0, 0, 0, 0, 0); + } while (ret != -ESRCH && ret != -EINVAL); + ksft_test_result(siginfo.si_signo == SIGSEGV && siginfo.si_code == SEGV_MAPERR && siginfo.si_addr == NULL, @@ -358,6 +369,7 @@ static void test_pkru_preserved_after_sigusr1(void) { struct sigaction sa; u64 pkey_reg; + int ret; /* Allow access to MPK 0 and an arbitrary set of keys */ pkey_reg = pkey_reg_restrictive_default(); @@ -369,10 +381,8 @@ static void test_pkru_preserved_after_sigusr1(void) sa.sa_sigaction = sigusr1_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR1, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR1, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); @@ -441,9 +451,15 @@ static void test_pkru_sigreturn(void) static stack_t sigstack; void *stack; int pkey; - int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; + + /* + * SIGSEGV handler is reset to SIG_DFL below; turn tracing off first + * so a crash does not leave ftrace enabled. + */ + tracing_off(); sa.sa_handler = SIG_DFL; sa.sa_flags = 0; @@ -453,24 +469,20 @@ static void test_pkru_sigreturn(void) * For this testcase, we do not want to handle SIGSEGV. Reset handler * to default so that the application can crash if it receives SIGSEGV. */ - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigusr2_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR2, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR2, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* * Allow access to MPK 0 and MPK 2. The child thread (to be created @@ -490,21 +502,21 @@ static void test_pkru_sigreturn(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_DETACHED, + stack + STACK_SIZE, + NULL, + NULL); if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_sigusr2_self(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); @@ -530,16 +542,17 @@ static void (*pkey_tests[])(void) = { int main(int argc, char *argv[]) { - int i; - ksft_print_header(); ksft_set_plan(ARRAY_SIZE(pkey_tests)); if (!is_pkeys_supported()) ksft_exit_skip("pkeys not supported\n"); - for (i = 0; i < ARRAY_SIZE(pkey_tests); i++) - (*pkey_tests[i])(); + for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { + tracing_on(); + (*pkey_tests[test_nr])(); + tracing_off(); + } ksft_finished(); return 0; diff --git a/tools/testing/selftests/mm/pkey_util.c b/tools/testing/selftests/mm/pkey_util.c index 255b332f7a08..fbef3cd45447 100644 --- a/tools/testing/selftests/mm/pkey_util.c +++ b/tools/testing/selftests/mm/pkey_util.c @@ -2,9 +2,99 @@ #define __SANE_USERSPACE_TYPES__ #include <sys/syscall.h> #include <unistd.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> #include "pkey-helpers.h" +int iteration_nr = 1; +int test_nr; +int dprint_in_signal; + +#if CONTROL_TRACING > 0 +static void cat_into_file(char *str, char *file) +{ + int fd = open(file, O_RDWR); + int ret; + + dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); + /* + * these need to be raw because they are called under + * pkey_assert() + */ + if (fd < 0) { + fprintf(stderr, "error opening '%s'\n", file); + perror("error: "); + exit(__LINE__); + } + + ret = write(fd, str, strlen(str)); + if (ret != strlen(str)) { + perror("write to file failed"); + fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); + exit(__LINE__); + } + close(fd); +} + +static int warned_tracing; +static int tracing_root_ok(void) +{ + if (geteuid() != 0) { + if (!warned_tracing) + fprintf(stderr, "WARNING: not run as root, " + "can not do tracing control\n"); + warned_tracing = 1; + return 0; + } + return 1; +} +#endif + +void tracing_on(void) +{ +#if CONTROL_TRACING > 0 +#define TRACEDIR "/sys/kernel/tracing" + char pidstr[32]; + + if (!tracing_root_ok()) + return; + + sprintf(pidstr, "%d", getpid()); + cat_into_file("0", TRACEDIR "/tracing_on"); + cat_into_file("\n", TRACEDIR "/trace"); + if (1) { + cat_into_file("function_graph", TRACEDIR "/current_tracer"); + cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); + } else { + cat_into_file("nop", TRACEDIR "/current_tracer"); + } + cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); + cat_into_file("1", TRACEDIR "/tracing_on"); + dprintf1("enabled tracing\n"); +#endif +} + +void tracing_off(void) +{ +#if CONTROL_TRACING > 0 + if (!tracing_root_ok()) + return; + cat_into_file("0", "/sys/kernel/tracing/tracing_on"); +#endif +} + +void abort_hooks(void) +{ + fflush(stdout); + fprintf(stderr, "running %s()...\n", __func__); + tracing_off(); +#ifdef SLEEP_ON_ABORT + sleep(SLEEP_ON_ABORT); +#endif +} + int sys_pkey_alloc(unsigned long flags, unsigned long init_val) { int ret = syscall(SYS_pkey_alloc, flags, init_val); diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index 9a6d954ee371..ae6e1530b354 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -49,11 +49,7 @@ #include "hugepage_settings.h" #include "pkey-helpers.h" -int iteration_nr = 1; -int test_nr; - u64 shadow_pkey_reg; -int dprint_in_signal; noinline int read_ptr(int *ptr) { @@ -62,89 +58,6 @@ noinline int read_ptr(int *ptr) return *ptr; } -#if CONTROL_TRACING > 0 -static void cat_into_file(char *str, char *file) -{ - int fd = open(file, O_RDWR); - int ret; - - dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); - /* - * these need to be raw because they are called under - * pkey_assert() - */ - if (fd < 0) { - fprintf(stderr, "error opening '%s'\n", str); - perror("error: "); - exit(__LINE__); - } - - ret = write(fd, str, strlen(str)); - if (ret != strlen(str)) { - perror("write to file failed"); - fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); - exit(__LINE__); - } - close(fd); -} - -static int warned_tracing; -static int tracing_root_ok(void) -{ - if (geteuid() != 0) { - if (!warned_tracing) - fprintf(stderr, "WARNING: not run as root, " - "can not do tracing control\n"); - warned_tracing = 1; - return 0; - } - return 1; -} -#endif - -static void tracing_on(void) -{ -#if CONTROL_TRACING > 0 -#define TRACEDIR "/sys/kernel/tracing" - char pidstr[32]; - - if (!tracing_root_ok()) - return; - - sprintf(pidstr, "%d", getpid()); - cat_into_file("0", TRACEDIR "/tracing_on"); - cat_into_file("\n", TRACEDIR "/trace"); - if (1) { - cat_into_file("function_graph", TRACEDIR "/current_tracer"); - cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); - } else { - cat_into_file("nop", TRACEDIR "/current_tracer"); - } - cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); - cat_into_file("1", TRACEDIR "/tracing_on"); - dprintf1("enabled tracing\n"); -#endif -} - -static void tracing_off(void) -{ -#if CONTROL_TRACING > 0 - if (!tracing_root_ok()) - return; - cat_into_file("0", "/sys/kernel/tracing/tracing_on"); -#endif -} - -void abort_hooks(void) -{ - fflush(stdout); - fprintf(stderr, "running %s()...\n", __func__); - tracing_off(); -#ifdef SLEEP_ON_ABORT - sleep(SLEEP_ON_ABORT); -#endif -} - /* * This attempts to have roughly a page of instructions followed by a few * instructions that do a write, and another page of instructions. That @@ -669,7 +582,7 @@ static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey) size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey); pkey_assert(!ret); record_pkey_malloc(ptr, size, prot); @@ -692,7 +605,7 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) */ size = ALIGN_UP(size, HPAGE_SIZE * 2); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); record_pkey_malloc(ptr, size, prot); mprotect_pkey(ptr, size, prot, pkey); @@ -750,7 +663,7 @@ static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey) size = ALIGN_UP(size, HPAGE_SIZE * 2); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); mprotect_pkey(ptr, size, prot, pkey); record_pkey_malloc(ptr, size, prot); @@ -779,7 +692,7 @@ static void *malloc_pkey(long size, int prot, u16 pkey) pkey_assert(malloc_type < nr_malloc_types); ret = pkey_malloc[malloc_type](size, prot, pkey); - pkey_assert(ret != (void *)-1); + pkey_assert(ret != MAP_FAILED); malloc_type++; if (malloc_type >= nr_malloc_types) @@ -1197,6 +1110,7 @@ static void arch_force_pkey_reg_init(void) * doing the XSAVE size enumeration dance. */ buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + pkey_assert(buf != MAP_FAILED); /* These __builtins require compiling with -mxsave */ @@ -1763,7 +1677,8 @@ int main(void) ksft_print_msg("running PKEY tests for unsupported CPU/OS\n"); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - assert(ptr != (void *)-1); + if (ptr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); test_mprotect_pkey_on_unsupported_cpu(ptr, 1); ksft_test_result_pass("pkey on unsupported CPU/OS\n"); ksft_finished(); diff --git a/tools/testing/selftests/mm/rmap.c b/tools/testing/selftests/mm/rmap.c index 53f2058b0ef2..1c293ad3f8b8 100644 --- a/tools/testing/selftests/mm/rmap.c +++ b/tools/testing/selftests/mm/rmap.c @@ -430,4 +430,85 @@ TEST_F(migrate, ksm) propagate_children(_metadata, data); } +static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages) +{ + int i; + int retries = 0; + unsigned long first_pfn; + +retry: + if (retries > 10) + return false; + + first_pfn = pagemap_get_pfn(pagemap_fd, region); + for (i = 0; i < nr_pages; i++) { + if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) { + /* + * Retry up to 10 times at most in case of the low chance of page + * compaction migrating the page while we check for pfn. + */ + retries++; + goto retry; + } + } + + return true; +} + +TEST_F(migrate, ksm_and_mremap) +{ + unsigned long old_pfn, new_pfn; + void *region, *mremap_region; + const int nr_pages = 16; + size_t mmap_size; + int pagemap_fd; + + /* Skip if KSM is not available */ + if (ksm_stop() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed"); + if (ksm_get_full_scans() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed"); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + SKIP(return, "opening pagemap failed"); + + /* Allocate and populate twice the anon pages initially. */ + mmap_size = 2 * nr_pages * getpagesize(); + region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + ASSERT_NE(region, MAP_FAILED); + memset(region, 0x77, mmap_size); + + /* mremap the second half over the first half, to stress rmap handling */ + mmap_size /= 2; + mremap_region = mremap(region + mmap_size, mmap_size, mmap_size, + MREMAP_MAYMOVE | MREMAP_FIXED, region); + ASSERT_EQ(mremap_region, region); + + /* Merge all pages into a single KSM page. */ + madvise(region, mmap_size, MADV_MERGEABLE); + ASSERT_EQ(ksm_start(), 0); + + /* The whole range should map the same KSM page. */ + old_pfn = pagemap_get_pfn(pagemap_fd, region); + if (old_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ksm_start(); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); + + /* + * Migrate the KSM page; the whole range should map the new (migrated) + * KSM page. + */ + ASSERT_EQ(try_to_move_page(region), 0); + + new_pfn = pagemap_get_pfn(pagemap_fd, region); + if (new_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ASSERT_NE(new_pfn, old_pfn); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); +} + + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 8c296dedf047..687d115e3bd8 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -262,7 +262,6 @@ echo "TAP version 13" | tap_output CATEGORY="hugetlb" run_test ./hugetlb-mmap CATEGORY="hugetlb" run_test ./hugetlb-shm CATEGORY="hugetlb" run_test ./hugetlb-mremap -CATEGORY="hugetlb" run_test ./hugetlb-vmemmap CATEGORY="hugetlb" run_test ./hugetlb-madvise CATEGORY="hugetlb" run_test ./hugetlb_dio CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv @@ -303,7 +302,7 @@ CATEGORY="compaction" run_test ./compaction_test if command -v sudo &> /dev/null && sudo -u nobody ls ./on-fault-limit >/dev/null; then - CATEGORY="mlock" run_test sudo -u nobody ./on-fault-limit + CATEGORY="mlock" run_test sudo -u nobody sh -c ./on-fault-limit else echo "# SKIP ./on-fault-limit" fi diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 8eb0c5630e7e..a0f9fc900360 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -35,6 +35,7 @@ int main(int argc, char **argv) size_t map_len; int pagemap_fd; int duration = 0; + double elapsed = 0; ksft_print_header(); @@ -60,8 +61,6 @@ int main(int argc, char **argv) len = atoll(argv[i]) << 20; } - ksft_set_plan(1); - if (name) { backing_fd = open(name, O_RDWR); if (backing_fd == -1) @@ -78,6 +77,10 @@ int main(int argc, char **argv) ksft_exit_fail_msg("open pagemap\n"); len -= len % HPAGE_SIZE; + if (len == 0) + ksft_exit_fail_msg("len must be at least %d MiB\n", + HPAGE_SIZE >> 20); + ptr = mmap(NULL, len + HPAGE_SIZE, PROT_RW, mmap_flags, backing_fd, 0); if (ptr == MAP_FAILED) ksft_exit_fail_msg("initial mmap"); @@ -91,6 +94,7 @@ int main(int argc, char **argv) if (!map) ksft_exit_fail_msg("map malloc\n"); + ksft_set_plan(1); clock_gettime(CLOCK_MONOTONIC, &start); while (1) { @@ -125,16 +129,28 @@ int main(int argc, char **argv) /* split transhuge page, keep last page */ if (madvise(p, HPAGE_SIZE - psize(), MADV_DONTNEED)) ksft_exit_fail_msg("MADV_DONTNEED"); + + if (duration > 0) { + clock_gettime(CLOCK_MONOTONIC, &b); + elapsed = b.tv_sec - start.tv_sec + + (b.tv_nsec - start.tv_nsec) / 1000000000.; + if (elapsed >= duration) + break; + } } clock_gettime(CLOCK_MONOTONIC, &b); s = b.tv_sec - a.tv_sec + (b.tv_nsec - a.tv_nsec) / 1000000000.; ksft_print_msg("%.3f s/loop, %.3f ms/page, %10.3f MiB/s\t" - "%4d succeed, %4d failed, %4d different pages\n", - s, s * 1000 / (len >> HPAGE_SHIFT), len / s / (1 << 20), - nr_succeed, nr_failed, nr_pages); - - if (duration > 0 && b.tv_sec - start.tv_sec >= duration) { + "%4d succeed, %4d failed, %4d different pages\n", + s, s * 1000 / (nr_failed + nr_succeed), + s > 0 ? + (double)(nr_failed + nr_succeed) * + (HPAGE_SIZE >> 20) / s : + 0.0, + nr_succeed, nr_failed, nr_pages); + + if (duration > 0 && elapsed >= duration) { ksft_test_result_pass("Completed\n"); ksft_finished(); } diff --git a/tools/testing/selftests/mm/uffd-common.c b/tools/testing/selftests/mm/uffd-common.c index edd02328f77b..f48f5d4594ab 100644 --- a/tools/testing/selftests/mm/uffd-common.c +++ b/tools/testing/selftests/mm/uffd-common.c @@ -639,8 +639,13 @@ int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) { - /* real retval in ufdio_copy.copy */ - if (uffdio_copy.copy != -EEXIST) + /* + * real retval in uffdio_copy.copy + * + * -EEXIST: the page was faulted in concurrently + * -ENOENT: the destination range was concurrently removed + */ + if (uffdio_copy.copy != -EEXIST && uffdio_copy.copy != -ENOENT) err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy); wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size); diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c index a6c14109e818..ef9b3956bdcf 100644 --- a/tools/testing/selftests/mm/uffd-unit-tests.c +++ b/tools/testing/selftests/mm/uffd-unit-tests.c @@ -7,6 +7,8 @@ #include "uffd-common.h" +#include <linux/fs.h> +#include <sys/uio.h> #include "../../../../mm/gup_test.h" #ifdef __NR_userfaultfd @@ -109,6 +111,10 @@ static void uffd_test_skip(const char *message) static void test_uffd_api(bool use_dev) { + const uint64_t expected_ioctls = + BIT_ULL(_UFFDIO_REGISTER) | + BIT_ULL(_UFFDIO_UNREGISTER) | + BIT_ULL(_UFFDIO_API); struct uffdio_api uffdio_api; int uffd; @@ -148,6 +154,15 @@ static void test_uffd_api(bool use_dev) goto out; } + /* Verify returned fd-level ioctls bitmask */ + if ((uffdio_api.ioctls & expected_ioctls) != expected_ioctls) { + uffd_test_fail("UFFDIO_API missing expected ioctls: " + "got=0x%"PRIx64", expected=0x%"PRIx64, + (uint64_t)uffdio_api.ioctls, + expected_ioctls); + goto out; + } + /* Test double requests of UFFDIO_API with a random feature set */ uffdio_api.features = BIT_ULL(0); if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { @@ -602,6 +617,814 @@ void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t * uffd_minor_test_common(gopts, true, false); } +static int uffd_register_rwp(int uffd, void *addr, uint64_t len) +{ + struct uffdio_register reg = { + .range = { .start = (unsigned long)addr, .len = len }, + .mode = UFFDIO_REGISTER_MODE_RWP, + }; + + if (ioctl(uffd, UFFDIO_REGISTER, ®) == -1) + return -errno; + return 0; +} + +static void rwprotect_range(int uffd, __u64 start, __u64 len, bool protect) +{ + struct uffdio_rwprotect rwp = { + .range = { .start = start, .len = len }, + .mode = protect ? UFFDIO_RWPROTECT_MODE_RWP : 0, + }; + + if (ioctl(uffd, UFFDIO_RWPROTECT, &rwp)) + err("UFFDIO_RWPROTECT failed"); +} + +static void set_async_mode(int uffd, bool enable) +{ + struct uffdio_set_mode mode = { }; + + if (enable) + mode.enable = UFFD_FEATURE_RWP_ASYNC; + else + mode.disable = UFFD_FEATURE_RWP_ASYNC; + + if (ioctl(uffd, UFFDIO_SET_MODE, &mode)) + err("UFFDIO_SET_MODE failed"); +} + +/* + * Test async RWP faults on anonymous memory. + * Populate pages, register MODE_RWP with RWP_ASYNC, + * RW-protect, re-access, verify content preserved and no faults delivered. + */ +static void uffd_rwp_async_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + + /* Populate all pages with known content */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + /* Register MODE_RWP */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* RW-protect all pages (sets protnone) */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Access all pages — should auto-resolve, no faults */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + unsigned char expected = p % 255 + 1; + + if (page[0] != expected) { + uffd_test_fail("page %lu content mismatch: %u != %u", + p, page[0], expected); + return; + } + } + + uffd_test_pass(); +} + +/* + * Fault handler for RWP — unprotect the page via UFFDIO_RWPROTECT. + */ +static void uffd_handle_rwp_fault(uffd_global_test_opts_t *gopts, + struct uffd_msg *msg, + struct uffd_args *uargs) +{ + if (!(msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_RWP)) + err("expected RWP fault, got 0x%llx", + msg->arg.pagefault.flags); + + rwprotect_range(gopts->uffd, msg->arg.pagefault.address, + gopts->page_size, false); + uargs->minor_faults++; +} + +/* + * Test sync RWP faults on anonymous memory. + * Populate pages, register MODE_RWP (sync), RW-protect, + * access from worker thread, verify fault delivered, UFFDIO_RWPROTECT resolves. + */ +static void uffd_rwp_sync_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + pthread_t uffd_mon; + struct uffd_args uargs = { }; + bool failed = false; + char c = '\0'; + unsigned long p; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + /* Register MODE_RWP */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* RW-protect all pages */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Start fault handler thread */ + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* Access all pages — triggers sync RWP faults, handler unprotects */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + + if (page[0] != (p % 255 + 1)) { + uffd_test_fail("page %lu content mismatch", p); + failed = true; + goto out; + } + } + +out: + /* + * Stop the handler before reading minor_faults: the last fault + * resolution rwprotect_range()s before incrementing the counter, + * so the main thread can race ahead of the increment. + */ + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + if (failed) + return; + if (uargs.minor_faults == 0) + uffd_test_fail("expected RWP faults, got 0"); + else + uffd_test_pass(); +} + +/* + * Test PAGEMAP_SCAN working-set discovery via the "hot" (accessed) scan. + * + * The working-set primitive is to find pages that were accessed: scan for + * PAGE_IS_ACCESSED, which is set once an access clears the protnone+uffd + * marker. A VMM treats every access fault as "hot" (RWP here; MINOR/MISSING + * for non-resident pages) and reclaims the rest from the backing file. + * + * We deliberately do NOT use an inverted "cold" scan: that only sees + * VMA-resident ptes, so for a file mapping it misses cached-but-unmapped (and + * never-faulted, pre-populated) pages, which are pte_none and thus invisible. + * Hot tracking + file-level reclaim covers them; a cold pte scan cannot. + */ +static void uffd_rwp_pagemap_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + struct page_region regions[16]; + struct pm_scan_arg pm_arg; + int pagemap_fd; + long ret; + + /* Need at least 4 pages */ + if (nr_pages < 4) { + uffd_test_skip("need at least 4 pages"); + return; + } + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, 0xab, page_size); + + /* Register and RW-protect */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Touch first half of pages to re-activate them (async auto-resolve) */ + for (p = 0; p < nr_pages / 2; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + uint64_t start = (uint64_t)gopts->area_dst; + uint64_t boundary = start + (nr_pages / 2) * page_size; + uint64_t end = start + nr_pages * page_size; + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + err("open pagemap"); + + /* + * Hot scan: report the pages that were accessed. PAGE_IS_ACCESSED is + * set once the protnone+uffd marker is cleared (by the access, async + * auto-resolve here). The touched first half must come back as exactly + * one hot region [start, boundary); the untouched second half must not + * appear. + */ + memset(&pm_arg, 0, sizeof(pm_arg)); + pm_arg.size = sizeof(pm_arg); + pm_arg.start = start; + pm_arg.end = end; + pm_arg.vec = (uint64_t)regions; + pm_arg.vec_len = ARRAY_SIZE(regions); + pm_arg.category_mask = PAGE_IS_ACCESSED; + pm_arg.return_mask = PAGE_IS_ACCESSED; + + ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg); + close(pagemap_fd); + + if (ret < 0) { + uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno)); + return; + } + + if (ret != 1 || regions[0].start != start || + regions[0].end != boundary) { + uffd_test_fail("hot set wrong: got %ld regions [0x%lx,0x%lx), expected 1 [0x%lx,0x%lx)", + ret, (unsigned long)regions[0].start, + (unsigned long)regions[0].end, + (unsigned long)start, (unsigned long)boundary); + return; + } + + uffd_test_pass(); +} + +/* + * Test that RWP protection survives a mprotect(PROT_NONE) -> + * mprotect(PROT_READ|PROT_WRITE) round-trip. The uffd-wp bit on a + * VM_UFFD_RWP VMA must continue to carry PROT_NONE semantics after + * mprotect() changes the base protection; otherwise accesses would + * silently succeed and the pagemap bit would stick without a fault + * ever clearing it. + */ +static void uffd_rwp_mprotect_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + struct page_region regions[16]; + struct pm_scan_arg pm_arg; + int pagemap_fd; + uint64_t value; + long ret; + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, 0xab, page_size); + + /* Register and RW-protect the whole range */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Round-trip mprotect(): PROT_NONE -> PROT_READ|PROT_WRITE */ + if (mprotect(gopts->area_dst, nr_pages * page_size, PROT_NONE)) + err("mprotect() PROT_NONE"); + if (mprotect(gopts->area_dst, nr_pages * page_size, + PROT_READ | PROT_WRITE)) + err("mprotect() PROT_READ|PROT_WRITE"); + + /* + * The marker must survive the round-trip; if mprotect() dropped it, + * the touches below would not fault and the scan would pass + * vacuously. + */ + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + close(pagemap_fd); + if (!(value & PM_UFFD_WP)) { + uffd_test_fail("RWP marker lost across mprotect()"); + return; + } + + /* Touch every page. Async RWP must auto-resolve each fault. */ + for (p = 0; p < nr_pages; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + /* + * After touching, no page should remain RW-protected. A stuck + * uffd-wp bit would mean mprotect() silently dropped PROT_NONE and + * the access never faulted. + */ + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + err("open pagemap"); + + memset(&pm_arg, 0, sizeof(pm_arg)); + pm_arg.size = sizeof(pm_arg); + pm_arg.start = (uint64_t)gopts->area_dst; + pm_arg.end = (uint64_t)gopts->area_dst + nr_pages * page_size; + pm_arg.vec = (uint64_t)regions; + pm_arg.vec_len = ARRAY_SIZE(regions); + pm_arg.category_mask = PAGE_IS_ACCESSED; + pm_arg.category_inverted = PAGE_IS_ACCESSED; + pm_arg.return_mask = PAGE_IS_ACCESSED; + + ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg); + close(pagemap_fd); + + if (ret < 0) { + uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno)); + return; + } + if (ret != 0) { + uffd_test_fail("expected no cold pages after mprotect()+touch, got %ld regions", + ret); + return; + } + + uffd_test_pass(); +} + +/* + * Test that GUP resolves through protnone PTEs (async mode). + * vmsplice() into a pipe pins user pages via get_user_pages_fast() -- + * unlike write(), which goes through copy_from_user() and ordinary + * hardware page faults -- so it exercises gup_can_follow_protnone() on + * the RW-protected PTE. In async mode the kernel auto-restores + * permissions and GUP returns the page. + */ +static void uffd_rwp_gup_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + struct iovec iov; + char buf; + int pipefd[2]; + + /* Populate first page with known content */ + memset(gopts->area_dst, 0xCD, gopts->page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, gopts->page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + gopts->page_size, true); + + if (pipe(pipefd)) + err("pipe"); + + /* + * One byte's worth of iov is enough to GUP the containing page and + * keeps the pipe transfer well under any pipe-capacity limit even on + * hugetlb-backed runs. + */ + iov.iov_base = gopts->area_dst; + iov.iov_len = 1; + if (vmsplice(pipefd[1], &iov, 1, 0) != 1) { + uffd_test_fail("vmsplice from RW-protected page failed: %s", + strerror(errno)); + goto out; + } + + if (read(pipefd[0], &buf, 1) != 1) { + uffd_test_fail("read from pipe failed"); + goto out; + } + + if (buf != (char)0xCD) { + uffd_test_fail("content mismatch: got 0x%02x, expected 0xCD", + (unsigned char)buf); + goto out; + } + + uffd_test_pass(); +out: + close(pipefd[0]); + close(pipefd[1]); +} + +/* + * Test runtime toggle between async and sync modes. + * Start in async mode (detection), flip to sync (eviction), verify faults + * block, resolve them, flip back to async. + */ +static void uffd_rwp_async_toggle_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + struct uffd_args uargs = { }; + pthread_t uffd_mon; + char c = '\0'; + unsigned long p; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + /* Populate */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* Phase 1: async detection — RW-protect, access first half */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + for (p = 0; p < nr_pages / 2; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; /* auto-resolves in async mode */ + } + + /* Phase 2: flip to sync for eviction */ + set_async_mode(gopts->uffd, false); + + /* Start handler — will receive faults for cold pages */ + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* Access second half (cold pages) — should trigger sync faults */ + for (p = nr_pages / 2; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + if (page[0] != (p % 255 + 1)) { + uffd_test_fail("page %lu content mismatch", p); + goto out; + } + } + + /* + * Stop the handler before reading minor_faults: the last fault + * resolution rwprotect_range()s before incrementing the counter, + * so the main thread can race ahead of the increment. Stopping + * here also makes Phase 3 a clean async-only test -- with the + * handler still running it would silently resolve any sync fault + * the kernel erroneously delivers, masking a regression. + */ + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + if (uargs.minor_faults == 0) { + uffd_test_fail("expected sync faults, got 0"); + return; + } + + /* Phase 3: flip back to async */ + set_async_mode(gopts->uffd, true); + + /* RW-protect and access again — should auto-resolve */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + for (p = 0; p < nr_pages; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + uffd_test_pass(); + return; +out: + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); +} + +/* + * Test that RW-protected pages become accessible after closing uffd. + */ +static void uffd_rwp_close_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + + /* Populate */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Close uffd — should restore protnone PTEs */ + close(gopts->uffd); + gopts->uffd = -1; + + /* All pages should be accessible with original content */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + unsigned char expected = p % 255 + 1; + + if (page[0] != expected) { + uffd_test_fail("page %lu not accessible after close", p); + return; + } + } + + uffd_test_pass(); +} + +/* + * Test that RWP protection is preserved across fork() when + * UFFD_FEATURE_EVENT_FORK is enabled. Without preservation, the child's + * PTEs would lose the uffd-wp marker and RWP-protected accesses would + * silently fall through to do_numa_page(). + */ +static void uffd_rwp_fork_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + int pagemap_fd; + uint64_t value; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failed"); + + /* Populate + RWP-protect */ + *gopts->area_dst = 1; + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + page_size, true); + + /* Parent: verify uffd-wp bit is set before fork */ + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + pagemap_check_wp(value, true); + + /* + * Fork with EVENT_FORK: child inherits VM_UFFD_RWP. Child reads + * its own pagemap and must still see the uffd-wp bit set. + */ + if (pagemap_test_fork(gopts, true, false)) { + uffd_test_fail("RWP marker lost in child after fork"); + goto out; + } + + uffd_test_pass(); +out: + close(pagemap_fd); +} + +/* + * Test that RWP protection on a pinned anon page is preserved across fork(). + * Pinning forces copy_present_page() in the child path, which must restore + * PAGE_NONE on top of the uffd bit. Using async mode, a read in the child + * auto-resolves if — and only if — the PTE was actually protnone+uffd; the + * cleared uffd bit afterward proves the fault path ran. + */ +static void uffd_rwp_fork_pin_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long page_size = gopts->page_size; + fork_event_args fevent_args = { .gopts = gopts, .child_uffd = -1 }; + pin_args pin_args = {}; + int pagemap_fd, status; + pthread_t fevent_thread; + uint64_t value; + pid_t child; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size)) + err("register failed"); + + /* Populate. */ + *gopts->area_dst = 1; + + /* RO-longterm pin so fork() takes copy_present_page() for this PTE. */ + if (pin_pages(&pin_args, gopts->area_dst, page_size)) { + uffd_test_skip("Possibly CONFIG_GUP_TEST missing or unprivileged"); + uffd_unregister(gopts->uffd, gopts->area_dst, page_size); + return; + } + + /* RWP-protect: PTE is now PAGE_NONE + uffd bit. */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true); + + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + pagemap_check_wp(value, true); + + /* + * UFFD_FEATURE_EVENT_FORK is required so the child inherits + * VM_UFFD_RWP and the marker; without it dup_userfaultfd() resets + * the child VMA and the test would pass for the wrong reason. + * dup_userfaultfd() blocks until the EVENT_FORK message is consumed, + * so spawn a reader before the fork(). + */ + gopts->ready_for_fork = false; + if (pthread_create(&fevent_thread, NULL, fork_event_consumer, + &fevent_args)) + err("pthread_create() for fork event consumer"); + while (!gopts->ready_for_fork) + ; /* Wait for consumer to start polling. */ + + child = fork(); + if (child < 0) + err("fork"); + if (child == 0) { + volatile char c; + int cfd; + + /* + * Precondition: the child must have inherited the marker. + * If copy_present_page() dropped it together with PAGE_NONE, + * the read below would succeed without the fault path and + * the after-read check would pass for the wrong reason. + */ + cfd = pagemap_open(); + value = pagemap_get_entry(cfd, gopts->area_dst); + if (!(value & PM_UFFD_WP)) { + close(cfd); + _exit(2); + } + + /* + * Read the pinned page. Only reaches the fault path if the + * child PTE is protnone + uffd; async mode auto-resolves and + * clears the uffd bit. If copy_present_page() dropped + * PAGE_NONE, the read would silently succeed and the bit + * would still be set. + */ + c = *(volatile char *)gopts->area_dst; + (void)c; + + value = pagemap_get_entry(cfd, gopts->area_dst); + close(cfd); + _exit((value & PM_UFFD_WP) ? 1 : 0); + } + if (waitpid(child, &status, 0) < 0) + err("waitpid"); + if (pthread_join(fevent_thread, NULL)) + err("pthread_join() for fork event consumer"); + if (fevent_args.child_uffd >= 0) + close(fevent_args.child_uffd); + + unpin_pages(&pin_args); + close(pagemap_fd); + if (uffd_unregister(gopts->uffd, gopts->area_dst, page_size)) + err("unregister failed"); + + if (WIFEXITED(status) && WEXITSTATUS(status) == 2) { + uffd_test_fail("RWP marker not inherited by child"); + return; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + uffd_test_fail("RWP not enforced in child after pinned fork"); + return; + } + + uffd_test_pass(); +} + +/* + * A non-exclusive (forked, COW-shared) anon page that is RWP-protected and + * then swapped out must keep tracking across swap-in. On the write that + * swaps it back in, do_swap_page() restores PAGE_NONE and the access retries + * through the RWP fault path, instead of being COWed straight to a fresh + * accessible page -- which would silently drop the marker for a non-exclusive + * folio. Sync mode lets us observe the fault directly: with the bug, the + * write COWs without delivering any RWP fault. + * + * Needs a swap device; skipped if MADV_PAGEOUT cannot evict the page. + */ +static void uffd_rwp_swap_cow_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long page_size = gopts->page_size; + struct uffd_args uargs = { }; + int pagemap_fd, go[2], i; + pthread_t uffd_mon; + char c = '\0'; + pid_t child; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size)) + err("register failed"); + + /* Populate one page (exclusive at this point). */ + *gopts->area_dst = 0x11; + + /* RWP-protect: PTE becomes PAGE_NONE + uffd bit (still exclusive). */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true); + + /* + * Swap the page out while it is still exclusive: a shared (forked) + * folio does not get reclaimed by MADV_PAGEOUT. Retry, since a hot + * page may just be rotated on the first reclaim pass. + */ + pagemap_fd = pagemap_open(); + for (i = 0; i < 100; i++) { + if (madvise(gopts->area_dst, page_size, MADV_PAGEOUT)) + err("MADV_PAGEOUT"); + if (pagemap_is_swapped(pagemap_fd, gopts->area_dst)) + break; + usleep(10000); + } + if (!pagemap_is_swapped(pagemap_fd, gopts->area_dst)) { + uffd_test_skip("MADV_PAGEOUT did not swap the page; is swap enabled?"); + close(pagemap_fd); + uffd_unregister(gopts->uffd, gopts->area_dst, page_size); + return; + } + + /* + * fork() now: the child duplicates the swap entry, so the slot becomes + * non-exclusive. The child parks (keeping the reference) until the + * parent has faulted the page back in. + */ + if (pipe(go)) + err("pipe"); + child = fork(); + if (child < 0) + err("fork"); + if (child == 0) { + close(go[1]); + read(go[0], &c, 1); + _exit(0); + } + close(go[0]); + + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* + * Write the page: swaps it back in (do_swap_page) on a non-exclusive + * folio with FAULT_FLAG_WRITE. The marker must survive and deliver an + * RWP fault rather than COW silently. + */ + *gopts->area_dst = 0x22; + + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join failed"); + + if (uargs.minor_faults == 0) + uffd_test_fail("no RWP fault on swapped-in non-exclusive page"); + else + uffd_test_pass(); + + close(pagemap_fd); + if (write(go[1], &c, 1) != 1) + err("child release"); + close(go[1]); + waitpid(child, NULL, 0); +} + +/* + * WP and RWP share the uffd-wp PTE bit and cannot coexist in the same VMA. + * Registration requesting both modes must be rejected. + */ +static void uffd_rwp_wp_exclusive_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + struct uffdio_register reg = { }; + + reg.range.start = (unsigned long)gopts->area_dst; + reg.range.len = nr_pages * page_size; + reg.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_RWP; + + if (ioctl(gopts->uffd, UFFDIO_REGISTER, ®) == 0) { + uffd_test_fail("register with WP|RWP unexpectedly succeeded"); + return; + } + if (errno != EINVAL) { + uffd_test_fail("register with WP|RWP: expected EINVAL, got %d", + errno); + return; + } + uffd_test_pass(); +} + static sigjmp_buf jbuf, *sigbuf; static void sighndl(int sig, siginfo_t *siginfo, void *ptr) @@ -1605,6 +2428,83 @@ uffd_test_case_t uffd_tests[] = { .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM, }, { + .name = "rwp-async", + .uffd_fn = uffd_rwp_async_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-sync", + .uffd_fn = uffd_rwp_sync_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-pagemap", + .uffd_fn = uffd_rwp_pagemap_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-mprotect", + .uffd_fn = uffd_rwp_mprotect_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-gup", + .uffd_fn = uffd_rwp_gup_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-async-toggle", + .uffd_fn = uffd_rwp_async_toggle_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-close", + .uffd_fn = uffd_rwp_close_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-fork", + .uffd_fn = uffd_rwp_fork_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_EVENT_FORK, + }, + { + .name = "rwp-fork-pin", + .uffd_fn = uffd_rwp_fork_pin_test, + .mem_targets = MEM_ANON, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC | + UFFD_FEATURE_EVENT_FORK, + }, + { + .name = "rwp-swap-cow", + .uffd_fn = uffd_rwp_swap_cow_test, + .mem_targets = MEM_ANON, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-wp-exclusive", + .uffd_fn = uffd_rwp_wp_exclusive_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | + UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_WP_HUGETLBFS_SHMEM, + }, + { .name = "sigbus", .uffd_fn = uffd_sigbus_test, .mem_targets = MEM_ALL, @@ -1814,6 +2714,25 @@ int main(int argc, char *argv[]) uffd_test_skip(errmsg); continue; } + /* + * RWP tracks protection on ptes; a THP-backed shmem/anon + * range (e.g. shmem_enabled=always) would split on + * rwprotect and change behaviour under the test. Keep + * such ranges off THP. hugetlb is huge by definition and + * rejects MADV_NOHUGEPAGE, so skip it. + */ + if ((test->uffd_feature_required & UFFD_FEATURE_RWP) && + !(mem_type->mem_flag & (MEM_HUGETLB | MEM_HUGETLB_PRIVATE))) { + unsigned long len = gopts.nr_pages * gopts.page_size; + + /* + * EINVAL means CONFIG_TRANSPARENT_HUGEPAGE=n: + * nothing to opt out of. + */ + if (madvise(gopts.area_dst, len, MADV_NOHUGEPAGE) && + errno != EINVAL) + err("madvise(MADV_NOHUGEPAGE)"); + } test->uffd_fn(&gopts, &args); uffd_test_ctx_clear(&gopts); } diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 311fc5b4513e..ef1ea11981a7 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -719,7 +719,7 @@ int read_file(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } -void write_file(const char *path, const char *buf, size_t buflen) +static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) { int fd, saved_errno; ssize_t numwritten; @@ -735,14 +735,22 @@ void write_file(const char *path, const char *buf, size_t buflen) saved_errno = errno; close(fd); errno = saved_errno; - if (numwritten < 0) + if (numwritten < 0) { + if (ignore_einval && errno == EINVAL) + return; ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), buf, strerror(errno)); + } if (numwritten != buflen - 1) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } +void write_file(const char *path, const char *buf, size_t buflen) +{ + __write_file(path, buf, buflen, /* ignore_einval = */ false); +} + unsigned long read_num(const char *path) { char buf[21]; @@ -753,12 +761,22 @@ unsigned long read_num(const char *path) return strtoul(buf, NULL, 10); } -void write_num(const char *path, unsigned long num) +static void __write_num(const char *path, unsigned long num, bool ignore_einval) { char buf[21]; sprintf(buf, "%lu", num); - write_file(path, buf, strlen(buf) + 1); + __write_file(path, buf, strlen(buf) + 1, ignore_einval); +} + +void write_num(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ false); +} + +void write_num_ignore_einval(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ true); } static unsigned long shmall, shmmax; diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea8fc8fdf0eb..7799154b67ee 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -168,6 +168,7 @@ void write_file(const char *path, const char *buf, size_t buflen); int read_file(const char *path, char *buf, size_t buflen); unsigned long read_num(const char *path); void write_num(const char *path, unsigned long num); +void write_num_ignore_einval(const char *path, unsigned long num); void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); diff --git a/tools/testing/shared/linux/kmemleak.h b/tools/testing/shared/linux/kmemleak.h index 155f112786c4..65a383090fb5 100644 --- a/tools/testing/shared/linux/kmemleak.h +++ b/tools/testing/shared/linux/kmemleak.h @@ -1 +1,2 @@ static inline void kmemleak_update_trace(const void *ptr) { } +static inline void kmemleak_transient_leak(const void *ptr) { } diff --git a/tools/testing/vma/Makefile b/tools/testing/vma/Makefile index e72b45dedda5..ef6cc558afe1 100644 --- a/tools/testing/vma/Makefile +++ b/tools/testing/vma/Makefile @@ -10,7 +10,7 @@ OFILES = $(SHARED_OFILES) main.o shared.o maple-shim.o TARGETS = vma # These can be varied to test different sizes. -CFLAGS += -DNUM_VMA_FLAG_BITS=128 -DNUM_MM_FLAG_BITS=128 +CFLAGS += -DNUM_VMA_FLAG_BITS=64 -DNUM_MM_FLAG_BITS=64 main.o: main.c shared.c shared.h vma_internal.h tests/merge.c tests/mmap.c tests/vma.c ../../../mm/vma.c ../../../mm/vma_init.c ../../../mm/vma_exec.c ../../../mm/vma.h include/custom.h include/dup.h include/stubs.h diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index bf26b3f48d3a..800e5fa02d78 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -15,6 +15,21 @@ struct task_struct *get_current(void); #define MMF_HAS_MDWE 28 #define current get_current() +#define MINORBITS 20 +#define MINORMASK ((1U << MINORBITS) - 1) + +#define MAJOR(dev) ((unsigned int) ((dev) >> MINORBITS)) +#define MINOR(dev) ((unsigned int) ((dev) & MINORMASK)) +#define MKDEV(ma, mi) (((ma) << MINORBITS) | (mi)) + +#define S_IFMT 00170000 +#define S_IFCHR 0020000 + +#define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) + +#define MEM_MAJOR 1 +#define DEVZERO_MINOR 5 + /* * Define the task command name length as enum, then it can be visible to * BPF programs. @@ -23,6 +38,8 @@ enum { TASK_COMM_LEN = 16, }; +typedef unsigned short umode_t; + /* PARTIALLY implemented types. */ struct mm_struct { struct maple_tree mm_mt; @@ -45,6 +62,10 @@ struct address_space { unsigned long flags; atomic_t i_mmap_writable; }; +struct inode { + umode_t i_mode; + dev_t i_rdev; +}; struct file_operations { int (*mmap)(struct file *, struct vm_area_struct *); int (*mmap_prepare)(struct vm_area_desc *); @@ -52,6 +73,7 @@ struct file_operations { struct file { struct address_space *f_mapping; const struct file_operations *f_op; + struct inode *f_inode; }; struct anon_vma_chain { struct anon_vma *anon_vma; @@ -245,8 +267,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWS_UP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -315,6 +339,8 @@ enum { /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VM_EXEC_BIT : VM_READ_BIT) @@ -573,6 +599,7 @@ struct vm_area_struct { */ unsigned int vm_lock_seq; #endif + unsigned int __vm_virt_pgoff_lo; /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma @@ -609,6 +636,9 @@ struct vm_area_struct { /* Unstable RCU readers are allowed to read this. */ refcount_t vm_refcnt; #endif +#ifdef CONFIG_64BIT + unsigned int __vm_virt_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. @@ -1163,6 +1193,11 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) return mas_find(&vmi->mas, ULONG_MAX); } +static inline bool vma_is_attached(struct vm_area_struct *vma) +{ + return refcount_read(&vma->vm_refcnt); +} + /* * WARNING: to avoid racing with vma_mark_attached()/vma_mark_detached(), these * assertions should be made either under mmap_write_lock or when the object @@ -1170,12 +1205,12 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) */ static inline void vma_assert_attached(struct vm_area_struct *vma) { - WARN_ON_ONCE(!refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(!vma_is_attached(vma)); } static inline void vma_assert_detached(struct vm_area_struct *vma) { - WARN_ON_ONCE(refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(vma_is_attached(vma)); } static inline void vma_assert_write_locked(struct vm_area_struct *); @@ -1301,6 +1336,38 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) +{ + return vma->vm_pgoff; +} + +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + +static inline pgoff_t vma_start_virt_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_virt_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_virt_pgoff_lo; + return pgoff; +} + +static inline pgoff_t vma_end_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_virt_pgoff(vma) + vma_pages(vma); +} + +static inline pgoff_t vma_last_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_virt_pgoff(vma) - 1; +} + static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) { return file->f_op->mmap_prepare(desc); @@ -1372,7 +1439,7 @@ static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) mas_set(&vmi->mas, addr); } -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -1532,9 +1599,73 @@ static inline int get_sysctl_max_map_count(void) #define pgtable_supports_soft_dirty() IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) #endif -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); return vm_get_page_prot(vm_flags); } + +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_pgoff(vma); + return pgoff; +} + +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} + +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} + +static inline pgoff_t __linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_virt_pgoff(vma); + return pgoff; +} + +static inline pgoff_t linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_virt_page_index(vma, address); + + VM_WARN_ON_ONCE(vma_test(vma, VMA_SHARED_BIT)); + if (vma_is_anonymous(vma)) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); + + return pgoff; +} + +static inline struct inode *file_inode(const struct file *f) +{ + return f->f_inode; +} + +static inline unsigned iminor(const struct inode *inode) +{ + return MINOR(inode->i_rdev); +} + +static inline unsigned imajor(const struct inode *inode) +{ + return MAJOR(inode->i_rdev); +} diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 64164e25658f..d6136e19a8af 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -257,13 +257,13 @@ static inline void vm_acct_memory(long pages) { } -static inline void vma_interval_tree_insert(struct vm_area_struct *vma, - struct rb_root_cached *rb) +static inline void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping) { } -static inline void vma_interval_tree_remove(struct vm_area_struct *vma, - struct rb_root_cached *rb) +static inline void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping) { } @@ -271,13 +271,13 @@ static inline void flush_dcache_mmap_unlock(struct address_space *mapping) { } -static inline void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, - struct rb_root_cached *rb) +static inline void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { } -static inline void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, - struct rb_root_cached *rb) +static inline void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { } diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index 2565a5aecb80..f410bb6f858e 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -23,7 +23,8 @@ struct vm_area_struct *alloc_vma(struct mm_struct *mm, vma->vm_start = start; vma->vm_end = end; - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); + vma_set_virt_pgoff(vma, start >> PAGE_SHIFT); vma->flags = vma_flags; vma_assert_detached(vma); @@ -120,12 +121,3 @@ unsigned long rlimit(unsigned int limit) { return (unsigned long)-1; } - -void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h index 8b9e3b11c3cb..97cd7a679dc1 100644 --- a/tools/testing/vma/shared.h +++ b/tools/testing/vma/shared.h @@ -21,19 +21,35 @@ } \ } while (0) -#define ASSERT_TRUE(_expr) \ - do { \ - if (!(_expr)) { \ - fprintf(stderr, \ - "Assert FAILED at %s:%d:%s(): %s is FALSE.\n", \ - __FILE__, __LINE__, __FUNCTION__, #_expr); \ - return false; \ - } \ +#define __ASSERT_TRUE(_expr, _fmt, ...) \ + do { \ + if (!(_expr)) { \ + fprintf(stderr, \ + "Assert FAILED at %s:%d:%s(): %s is FALSE" \ + _fmt ".\n", \ + __FILE__, __LINE__, __FUNCTION__, #_expr \ + __VA_OPT__(,) __VA_ARGS__); \ + return false; \ + } \ } while (0) +#define __TO_SCALAR(x) ((unsigned long long)(uintptr_t)(x)) + +#define ASSERT_TRUE(_expr) __ASSERT_TRUE(_expr, "") #define ASSERT_FALSE(_expr) ASSERT_TRUE(!(_expr)) -#define ASSERT_EQ(_val1, _val2) ASSERT_TRUE((_val1) == (_val2)) -#define ASSERT_NE(_val1, _val2) ASSERT_TRUE((_val1) != (_val2)) +#define ASSERT_EQ(_val1, _val2) do { \ + __typeof__(_val1) __val1 = (_val1); \ + __typeof__(_val2) __val2 = (_val2); \ + __ASSERT_TRUE(__val1 == __val2, " (0x%llx != 0x%llx)", \ + __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ + } while (0) + +#define ASSERT_NE(_val1, _val2) do { \ + __typeof__(_val1) __val1 = (_val1); \ + __typeof__(_val2) __val2 = (_val2); \ + __ASSERT_TRUE(__val1 != __val2, " (0x%llx == 0x%llx)", \ + __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ + } while (0) #define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) \ ASSERT_TRUE(vma_flags_same_mask((_flags), (_flags_other))) @@ -53,8 +69,6 @@ #define ASSERT_FLAGS_NONEMPTY(_flags) \ ASSERT_FALSE(vma_flags_empty(_flags)) -#define IS_SET(_val, _flags) ((_val & _flags) == _flags) - extern bool fail_prealloc; /* Override vma_iter_prealloc() so we can choose to fail it. */ @@ -125,8 +139,3 @@ void __vma_set_dummy_anon_vma(struct vm_area_struct *vma, /* Provide a simple dummy VMA/anon_vma dummy setup for testing. */ void vma_set_dummy_anon_vma(struct vm_area_struct *vma, struct anon_vma_chain *avc); - -/* Helper function to specify a VMA's range. */ -void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff); diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0a..89ba273afe4b 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -45,6 +45,7 @@ void vmg_set_range(struct vma_merge_struct *vmg, unsigned long start, vmg->start = start; vmg->end = end; vmg->pgoff = pgoff; + vmg->anon_pgoff = start >> PAGE_SHIFT; vmg->vma_flags = vma_flags; vmg->just_expand = false; @@ -108,6 +109,7 @@ static bool test_simple_merge(void) .end = 0x2000, .vma_flags = vma_flags, .pgoff = 1, + .anon_pgoff = 1, }; ASSERT_FALSE(attach_vma(&mm, vma_left)); @@ -118,7 +120,8 @@ static bool test_simple_merge(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_FLAGS_SAME_MASK(&vma->flags, vma_flags); detach_free_vma(vma); @@ -150,7 +153,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); - ASSERT_EQ(vma->vm_pgoff, 1); + ASSERT_EQ(vma_start_pgoff(vma), 1); + ASSERT_EQ(vma_start_virt_pgoff(vma), 1); /* * Now walk through the three split VMAs and make sure they are as @@ -162,7 +166,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -171,7 +176,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); - ASSERT_EQ(vma->vm_pgoff, 1); + ASSERT_EQ(vma_start_pgoff(vma), 1); + ASSERT_EQ(vma_start_virt_pgoff(vma), 1); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -180,7 +186,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma), 2); + ASSERT_EQ(vma_start_virt_pgoff(vma), 2); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -209,7 +216,8 @@ static bool test_simple_expand(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -227,11 +235,12 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0x1000)); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -324,7 +333,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -343,7 +352,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -364,7 +374,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 6); + ASSERT_EQ(vma_start_pgoff(vma), 6); + ASSERT_EQ(vma_start_virt_pgoff(vma), 6); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -384,7 +395,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -404,7 +416,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0xa000); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0xa); + ASSERT_EQ(vma_start_pgoff(vma), 0xa); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0xa); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -423,7 +436,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -443,7 +457,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_NE(vma, NULL); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); detach_free_vma(vma); @@ -805,7 +820,8 @@ static bool test_vma_merge_new_with_close(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->vm_ops, &vm_ops); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -861,11 +877,13 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_next->vm_start, 0x3000); ASSERT_EQ(vma_next->vm_end, 0x9000); - ASSERT_EQ(vma_next->vm_pgoff, 3); + ASSERT_EQ(vma_start_pgoff(vma_next), 3); + ASSERT_EQ(vma_start_virt_pgoff(vma_next), 3); ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma), 2); + ASSERT_EQ(vma_start_virt_pgoff(vma), 2); ASSERT_TRUE(vma_write_started(vma)); ASSERT_TRUE(vma_write_started(vma_next)); ASSERT_EQ(mm.map_count, 2); @@ -927,11 +945,13 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x6000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x7000); - ASSERT_EQ(vma->vm_pgoff, 6); + ASSERT_EQ(vma_start_pgoff(vma), 6); + ASSERT_EQ(vma_start_virt_pgoff(vma), 6); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -962,7 +982,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x7000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_EQ(mm.map_count, 1); @@ -994,7 +1015,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x9000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_EQ(mm.map_count, 1); @@ -1416,7 +1438,8 @@ static bool test_merge_extend(void) ASSERT_EQ(vma_merge_extend(&vmi, vma, 0x2000), vma); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -1431,7 +1454,7 @@ static bool test_expand_only_mode(void) struct mm_struct mm = {}; VMA_ITERATOR(vmi, &mm, 0); struct vm_area_struct *vma_prev, *vma; - VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5); + VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5, 5); /* * Place a VMA prior to the one we're expanding so we assert that we do @@ -1456,7 +1479,8 @@ static bool test_expand_only_mode(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma->vm_start, 0x3000); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 3); + ASSERT_EQ(vma_start_pgoff(vma), 3); + ASSERT_EQ(vma_start_virt_pgoff(vma), 3); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(vma_iter_addr(&vmi), 0x3000); vma_assert_attached(vma); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb..b720deb4227d 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -45,7 +45,57 @@ static bool test_mmap_region_basic(void) return true; } +static int dummy_mmap_prepare(struct vm_area_desc *desc) +{ + return 0; +} + +static bool test_pure_anon_dev_zero(void) +{ + const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT); + const struct file_operations f_op = { + .mmap_prepare = dummy_mmap_prepare, + }; + struct inode inode = { + .i_mode = S_IFCHR, + .i_rdev = MKDEV(MEM_MAJOR, DEVZERO_MINOR), + }; + struct file file = { + .f_inode = &inode, + .f_op = &f_op, + }; + struct mm_struct mm = {}; + struct vm_area_struct *vma; + unsigned long addr; + VMA_ITERATOR(vmi, &mm, 0); + + current->mm = &mm; + + /* + * Map a MAP_PRIVATE-/dev/zero mapping at address 0x300000 with a page + * offset of 0x10, which we expect to be reset to the virtual page + * offset. + */ + addr = __mmap_region(&file, 0x300000, 0x3000, vma_flags, 0x10, NULL); + ASSERT_EQ(addr, 0x300000); + + /* Assert that it truly is an anonymous mapping. */ + vma = vma_lookup(&mm, addr); + ASSERT_NE(vma, NULL); + ASSERT_TRUE(vma_is_anonymous(vma)); + ASSERT_EQ(vma->vm_file, NULL); + ASSERT_EQ(vma->vm_private_data, NULL); + /* Expect virtual page offsets. */ + ASSERT_EQ(vma->vm_pgoff, 0x300); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0x300); + + cleanup_mm(&mm, &vmi); + return true; +} + static void run_mmap_tests(int *num_tests, int *num_fail) { TEST(mmap_region_basic); + TEST(pure_anon_dev_zero); } diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da06321..7ca5289e0f95 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index e12ab2c80f95..8a48b231aa7a 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -14,8 +14,8 @@ #include <stdlib.h> -#define CONFIG_MMU -#define CONFIG_PER_VMA_LOCK +#define CONFIG_MMU 1 +#define CONFIG_PER_VMA_LOCK 1 #ifdef __CONCAT #undef __CONCAT @@ -53,6 +53,7 @@ typedef __bitwise unsigned int vm_fault_t; #define VM_WARN_ON(_expr) (WARN_ON(_expr)) #define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr)) +#define VM_WARN_ON_ONCE_VMA(_expr, _vma) (WARN_ON_ONCE(_expr)) #define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr)) #define VM_BUG_ON(_expr) (BUG_ON(_expr)) #define VM_BUG_ON_VMA(_expr, _vma) (BUG_ON(_expr)) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index db57c5766ab6..f0e5da490866 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -440,7 +440,7 @@ static int kvm_gmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpo static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma, unsigned long addr, pgoff_t *ilx) { - pgoff_t pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + pgoff_t pgoff = linear_page_index(vma, addr); struct inode *inode = file_inode(vma->vm_file); *ilx = inode->i_ino; |