From b25e27043b65d8bcc08250a0c4f23c008d6470b3 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Mon, 29 Jun 2026 17:07:51 -0700 Subject: csky: implement flush_cache_vmap() in C To avoid getting an unused-var warning from unsigned long start = something; ... flush_cache_vmap(start, ...); Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202606291606.9h8aGniQ-lkp@intel.com/ Reviewed-by: Guo Ren Reviewed-by: Barry Song Cc: Andrew Donnellan Cc: Anshuman Khandual Cc: Catalin Marinas Cc: David Hildenbrand Cc: Dev Jain Cc: Leo Yan Cc: Mike Rapoport Cc: Ryan Roberts Cc: Uladzislau Rezki Cc: Wen Jiang Cc: Wen Jiang Cc: Will Deacon Cc: Xueyuan Chen Signed-off-by: Andrew Morton --- arch/csky/abiv1/inc/abi/cacheflush.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index d011a81575d2..21cc7965c53d 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -42,7 +42,12 @@ static inline void flush_anon_page(struct vm_area_struct *vma, * Use cache_wbinv_all() here and need to be improved in future. */ extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -#define flush_cache_vmap(start, end) cache_wbinv_all() + +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ + cache_wbinv_all(); +} + #define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) cache_wbinv_all() -- cgit v1.2.3 From 2431873c97784a288461a02b4f06840d93be825b Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:47 +0800 Subject: powerpc/mm: fix wrong addr_pfn tracking in compound vmemmap population vmemmap_populate_compound_pages() uses addr_pfn to determine the PFN offset within a compound page and to decide whether the current vmemmap slot should be populated as a head page mapping or should reuse a tail page mapping. However, addr_pfn is advanced manually in parallel with addr. The loop itself progresses in vmemmap address space, so each PAGE_SIZE step in addr covers PAGE_SIZE / sizeof(struct page) struct page slots. Since addr_pfn is compared against nr_pages in data-PFN units, it should advance by the same number of PFNs. The existing manual increments do not match that and therefore do not reliably track the PFN corresponding to the current addr. As a result, pfn_offset can be computed from the wrong PFN and the code can make the head/tail decision for the wrong compound-page position. Fix this by deriving addr_pfn directly from the current vmemmap address instead of carrying it as loop state. Link: https://lore.kernel.org/20260612035903.2468601-4-songmuchun@bytedance.com Fixes: f2b79c0d7968 ("powerpc/book3s64/radix: add support for vmemmap optimization for radix") Signed-off-by: Muchun Song Acked-by: Oscar Salvador Reviewed-by: Ritesh Harjani (IBM) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Mike Rapoport (Microsoft) Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: Usama Arif Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- arch/powerpc/mm/book3s64/radix_pgtable.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 10aced261cff..cf692b2b5f7b 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1314,7 +1314,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * covering out both edges. */ unsigned long addr; - unsigned long addr_pfn = start_pfn; unsigned long next; pgd_t *pgd; p4d_t *p4d; @@ -1335,7 +1334,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (pmd_leaf(READ_ONCE(*pmd))) { /* existing huge mapping. Skip the range */ - addr_pfn += (PMD_SIZE >> PAGE_SHIFT); next = pmd_addr_end(addr, end); continue; } @@ -1348,11 +1346,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * page whose VMEMMAP_RESERVE_NR pages were mapped and * this request fall in those pages. */ - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } else { unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); pte_t *tail_page_pte; @@ -1376,7 +1374,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte) return -ENOMEM; - addr_pfn += 2; next = addr + 2 * PAGE_SIZE; continue; } @@ -1392,7 +1389,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } @@ -1402,7 +1398,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } -- cgit v1.2.3 From b0190019eafc3aecd04cbfde0c3227bbbfc2675c Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 12 Jun 2026 11:58:57 +0800 Subject: mm/hugetlb: refactor early boot gigantic hugepage allocation The early boot gigantic hugepage allocation helpers currently mix allocation with huge_bootmem_page setup, and leave part of the initialization flow in architecture code. Refactor the interface to return the allocated huge page pointer and move the huge_bootmem_page setup into the generic hugetlb code. This makes the architecture-specific paths focus only on finding memory, while the common code handles node placement and early page metadata setup in one place. This also lets powerpc benefit from memblock_reserved_mark_noinit(), which it did not enable before. In addition, upcoming cross-zone validation for boot-time gigantic hugetlb reservation is common logic. With this refactoring, that logic can stay in the generic code instead of being duplicated in architecture-specific paths. Link: https://lore.kernel.org/20260612035903.2468601-14-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Oscar Salvador (SUSE) Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Frank van der Linden Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Oscar Salvador (SUSE) Cc: "Ritesh Harjani (IBM)" Cc: Usama Arif Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/powerpc/mm/hugetlbpage.c | 13 +++--- include/linux/hugetlb.h | 18 +++----- mm/hugetlb.c | 95 +++++++++++++++++-------------------------- mm/hugetlb_cma.c | 13 +++--- mm/hugetlb_cma.h | 8 ++-- mm/internal.h | 9 ++++ 6 files changed, 64 insertions(+), 92 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 558fafb82b8a..a298746dc143 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -104,17 +104,14 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p } } -static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) +static __init void *pseries_alloc_bootmem_huge_page(struct hstate *hstate) { - struct huge_bootmem_page *m; + void *m; if (nr_gpages == 0) - return 0; + return NULL; m = phys_to_virt(gpage_freearray[--nr_gpages]); gpage_freearray[nr_gpages] = 0; - list_add(&m->list, &huge_boot_pages[0]); - m->hstate = hstate; - m->flags = 0; - return 1; + return m; } bool __init hugetlb_node_alloc_supported(void) @@ -124,7 +121,7 @@ bool __init hugetlb_node_alloc_supported(void) #endif -int __init alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) { #ifdef CONFIG_PPC_BOOK3S_64 diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 431848873370..cf39ac68e9bf 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -675,19 +675,11 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -struct cma; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; - struct cma *cma; -}; - #define HUGE_BOOTMEM_HVO 0x0001 #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 +struct huge_bootmem_page; bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); @@ -707,8 +699,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct folio *folio); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); -int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid); bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_add_hstate(unsigned order); @@ -1139,9 +1131,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, return NULL; } -static inline int __alloc_bootmem_huge_page(struct hstate *h) +static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid) { - return 0; + return NULL; } static inline struct hstate *hstate_file(struct file *f) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d2537247a877..2e5db9808193 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3039,79 +3039,58 @@ out_end_reservation: static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) { - struct huge_bootmem_page *m; - int listnode = nid; - if (hugetlb_early_cma(h)) - m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); - else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), + return hugetlb_cma_alloc_bootmem(h, nid, node_exact); + + if (node_exact) + return memblock_alloc_exact_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), + + return memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - /* - * For pre-HVO to work correctly, pages need to be on - * the list for the node they were actually allocated - * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, - * extract the actual node first. - */ - if (m) - listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); - } - - if (m) { - m->flags = 0; - m->cma = NULL; - } - } - - if (m) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - * - * Put them into a private list first because mem_map - * is not up yet. - */ - INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[listnode]); - m->hstate = h; - } - - return m; } -int alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node = nid; /* do node specific alloc */ - if (nid != NUMA_NO_NODE) { - m = alloc_bootmem(h, node, true); - if (!m) - return 0; - goto found; - } + if (nid != NUMA_NO_NODE) + return alloc_bootmem(h, node, true); /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, - &hugetlb_bootmem_nodes) { - m = alloc_bootmem(h, node, false); - if (!m) - return 0; - goto found; - } + &hugetlb_bootmem_nodes) + return alloc_bootmem(h, node, false); -found: + return NULL; +} + +static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) +{ + struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); + + if (!m) + return false; + + nid = early_pfn_to_nid(PHYS_PFN(__pa(m))); + /* + * Use the beginning of the huge page to store the huge_bootmem_page + * struct (until gather_bootmem puts them into the mem_map). + * + * Put them into a private list first because mem_map is not up yet. + */ + INIT_LIST_HEAD(&m->list); + list_add(&m->list, &huge_boot_pages[nid]); + m->hstate = h; + if (!hugetlb_early_cma(h)) { + m->cma = NULL; + m->flags = 0; + } /* * Only initialize the head struct page in memmap_init_reserved_pages, @@ -3123,7 +3102,7 @@ found: memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), huge_page_size(h) - PAGE_SIZE); - return 1; + return true; } /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */ diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 665d910b0f13..ae21b24e45b0 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -56,14 +56,13 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, return folio; } -struct huge_bootmem_page * __init -hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) +void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; struct huge_bootmem_page *m; - int node = *nid; + int node; - cma = hugetlb_cma[*nid]; + cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); if (!m) { if (node_exact) @@ -71,13 +70,11 @@ hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) for_each_node_mask(node, hugetlb_bootmem_nodes) { cma = hugetlb_cma[node]; - if (!cma || node == *nid) + if (!cma || node == nid) continue; m = cma_reserve_early(cma, huge_page_size(h)); - if (m) { - *nid = node; + if (m) break; - } } } diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index c619c394b1ae..3aa483573d17 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -6,8 +6,7 @@ void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int nid, nodemask_t *nodemask); -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact); +void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact); bool hugetlb_cma_exclusive_alloc(void); unsigned long hugetlb_cma_total_size(void); void hugetlb_cma_validate_params(void); @@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, return NULL; } -static inline -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact) +static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, + bool node_exact) { return NULL; } diff --git a/mm/internal.h b/mm/internal.h index 09efb9f4d126..3401759924d9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,15 @@ #include "vma.h" struct folio_batch; +struct hstate; +struct cma; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; + struct cma *cma; +}; /* * Maintains state across a page table move. The operation assumes both source -- cgit v1.2.3 From 70b40217e4a2afc1ce2e68705c3a90971db08ce1 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Mon, 29 Jun 2026 15:49:47 +0200 Subject: sparc/mm: drop custom pte_clear_not_present_full() Patch series "mm: cleanup clear_not_present_full_ptes()", v2. While doing some review, I stumbled over clear_not_present_full_ptes() and concluded that it needs some love. Let's remove pte_clear_not_present_full() and cleanup clear_not_present_full_ptes(), renaming it to clear_non_present_ptes(). This patch (of 3): On sparc64, pte_clear_not_present_full() nowadays does a simple __set_pte_at(). In __set_pte_at() -> maybe_tlb_batch_add(), we check pte_accessible() to see whether to call tlb_batch_add(). However, non-present PTEs are surely not accessible, so tlb_batch_add() is never called and the "full" parameter is irrelevant. Let's drop the helper and just let common code do a pte_clear(). pte_clear() on sparc64 maps to set_pte_at()->set_ptes()->__set_pte_at() ... so it ends up calling the same function, just with "full=0". Given that "full" is irrelevant, there is no change. We added pte_clear_not_present_full() for sparc64 in commit 90f08e399d05 ("sparc: mmu_gather rework"), and I suspect that it was already not required back then. Link: https://lore.kernel.org/20260629-clear_not_present_full_ptes-v2-0-96089871a1e7@kernel.org Link: https://lore.kernel.org/20260629-clear_not_present_full_ptes-v2-1-96089871a1e7@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Cc: Peter Zijlstra Cc: Andreas Larsson Cc: David S. Miller Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Oscar Salvador (SUSE) Signed-off-by: Andrew Morton --- arch/sparc/include/asm/pgtable_64.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'arch') diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 74ede706fb32..0837ebbc5dce 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -945,10 +945,6 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, #define pte_clear(mm,addr,ptep) \ set_pte_at((mm), (addr), (ptep), __pte(0UL)) -#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \ - __set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm)) - #ifdef DCACHE_ALIASING_POSSIBLE #define __HAVE_ARCH_MOVE_PTE #define move_pte(pte, old_addr, new_addr) \ -- cgit v1.2.3 From 47fe2c009446fa4c491e5d4f4ae1be80af6c96f9 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 6 Jul 2026 04:42:08 -0700 Subject: mm: rename ARCH_ENABLE_THP_MIGRATION to ARCH_HAS_PMD_SOFTLEAVES CONFIG_ARCH_ENABLE_THP_MIGRATION gates PMD-level migration entries. PMD-level device-private entries use the same migration mechanism and therefore require the same architecture support. Upcoming PMD-level swap entries can use the same PMD softleaf helpers without depending on page migration, so rename the architecture gate to CONFIG_ARCH_HAS_PMD_SOFTLEAVES. This describes the PMD entry capability rather than one current user of it. This is a pure rename: the set of selecting architectures (x86, arm64, s390, riscv, loongarch, and powerpc on PPC_BOOK3S_64) and the gating semantics are unchanged. No functional change intended. Link: https://lore.kernel.org/20260706114320.1643046-7-usama.arif@linux.dev Signed-off-by: Usama Arif Reviewed-by: Zi Yan Acked-by: David Hildenbrand (Arm) Cc: Alexandre Ghiti Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Dev Jain Cc: "Huang, Ying" Cc: Johannes Weiner Cc: Kairui Song Cc: Kemeng Shi Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Nhat Pham Cc: Nico Pache Cc: Rik van Riel Cc: Ryan Roberts Cc: Shakeel Butt Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/pgtable.h | 4 ++-- arch/loongarch/Kconfig | 2 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 +- arch/powerpc/platforms/Kconfig.cputype | 2 +- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/pgtable.h | 8 ++++---- arch/s390/Kconfig | 2 +- arch/s390/include/asm/pgtable.h | 2 +- arch/x86/Kconfig | 2 +- arch/x86/include/asm/pgtable.h | 2 +- include/linux/huge_mm.h | 2 +- include/linux/leafops.h | 8 ++++---- include/linux/pgtable.h | 2 +- include/linux/swapops.h | 6 +++--- mm/Kconfig | 2 +- mm/debug_vm_pgtable.c | 8 ++++---- mm/hmm.c | 4 ++-- mm/huge_memory.c | 2 +- mm/migrate.c | 4 ++-- mm/migrate_device.c | 6 +++--- mm/rmap.c | 2 +- 22 files changed, 38 insertions(+), 38 deletions(-) (limited to 'arch') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..e136e5abdb33 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -17,7 +17,7 @@ config ARM64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..ed1cc6dba780 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1534,10 +1534,10 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * Ensure that there are not more swap files than can be encoded in the kernel diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d8d252325017..c6c611f44f3f 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -12,7 +12,7 @@ config LOONGARCH select ARCH_NEEDS_DEFER_KASAN select ARCH_DISABLE_KASAN_INLINE select ARCH_ENABLE_MEMORY_HOTPLUG - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index e67e64ac6e8c..800ff3cc5f71 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1060,7 +1060,7 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) #define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))) #define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))) #define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd)) #define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))) diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index bac02c83bb3e..9f6847b53e37 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -112,7 +112,7 @@ config PPC_THP depends on PPC_RADIX_MMU || (PPC_64S_HASH_MMU && PAGE_SIZE_64KB) select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE choice prompt "CPU selection" diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index f7028caaeae0..760d0e0ae1f0 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -22,7 +22,7 @@ config RISCV select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5d5756bda82e..23eae82c26a6 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -946,7 +946,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline bool pmd_swp_soft_dirty(pmd_t pmd) { return pte_swp_soft_dirty(pmd_pte(pmd)); @@ -961,7 +961,7 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, @@ -1208,10 +1208,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /* * In the RV64 Linux scheme, we give the user half of the virtual-address space diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..6b317717bf48 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -82,7 +82,7 @@ config S390 select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 859ce7c7d454..645fa2d99c6a 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -903,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES #define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) #define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) #define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..50acb2830570 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -70,7 +70,7 @@ config X86 select ARCH_ENABLE_HUGEPAGE_MIGRATION if X86_64 && HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE) - select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE + select ARCH_HAS_PMD_SOFTLEAVES if X86_64 && TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_ATTACK_VECTORS if CPU_MITIGATIONS select ARCH_HAS_CACHE_LINE_SIZE diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f..f69b02c1d3d3 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1545,7 +1545,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ad20f7f8c179..b34e8e5e5f9a 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -567,7 +567,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 803d312437df..ef84ac1cce19 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -587,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_PMD_SOFTLEAVES) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -606,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index d52d2a976e5a..3df6db1e9e39 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1839,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..1125401b787e 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); @@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca4824905..a360861c74fa 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -703,7 +703,7 @@ config DEVICE_MIGRATION config ARCH_ENABLE_HUGEPAGE_MIGRATION bool -config ARCH_ENABLE_THP_MIGRATION +config ARCH_HAS_PMD_SOFTLEAVES bool config HUGETLB_PAGE_SIZE_VARIABLE diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 18411fb09aab..7987962950c1 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -751,7 +751,7 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) pmd_t pmd; if (!pgtable_supports_soft_dirty() || - !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) + !IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES)) return; if (!has_transparent_hugepage()) @@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args) WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { swp_entry_t arch_entry; @@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) pmd2 = __swp_entry_to_pmd(arch_entry); WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1))); } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static void __init swap_migration_tests(struct pgtable_debug_args *args) { diff --git a/mm/hmm.c b/mm/hmm.c index c72c9ddfdb2f..e5c1f4deed24 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -331,7 +331,7 @@ fault: return hmm_vma_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -391,7 +391,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index f43b94a40f58..b8e3d8ad928f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4882,7 +4882,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { diff --git a/mm/migrate.c b/mm/migrate.c index 0092613552e2..d54e5b7d244b 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -362,7 +362,7 @@ static bool remove_migration_pte(struct folio *folio, idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; new = folio_page(folio, idx); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || @@ -545,7 +545,7 @@ fail: } #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 39a6f5230a42..e74fee90489a 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -772,7 +772,7 @@ int migrate_vma_setup(struct migrate_vma *args) } EXPORT_SYMBOL(migrate_vma_setup); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm * at @addr. folio is already allocated as a part of the migration process with @@ -927,7 +927,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; return ret; } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, unsigned long addr, struct page *page, @@ -948,7 +948,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, static unsigned long migrate_vma_nr_pages(unsigned long *src) { unsigned long nr = 1; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES if (*src & MIGRATE_PFN_COMPOUND) nr = HPAGE_PMD_NR; #else diff --git a/mm/rmap.c b/mm/rmap.c index b93caabd186f..408e542649b5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2472,7 +2472,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_restart(&pvmw); continue; } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES pmdval = pmdp_get(pvmw.pmd); if (likely(pmd_present(pmdval))) pfn = pmd_pfn(pmdval); -- cgit v1.2.3 From bd48a270e8c17fef9de414dbee8b712fa51b8d45 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:48 +0000 Subject: perf/x86/intel: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-8-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Namhyung Kim Cc: Mark Rutland Cc: Alexander Shishkin Cc: Jiri Olsa Cc: Ian Rogers Cc: Adrian Hunter Cc: James Clark Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- arch/x86/events/intel/ds.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 91a093d8cf2e..70be80211d82 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -832,7 +832,7 @@ static void *dsalloc_pages(size_t size, gfp_t flags, int cpu) int node = cpu_to_node(cpu); struct page *page; - page = __alloc_pages_node(node, flags | __GFP_ZERO, order); + page = alloc_pages_node(node, flags | __GFP_ZERO, order); return page ? page_address(page) : NULL; } @@ -1088,9 +1088,9 @@ void init_arch_pebs_on_cpu(int cpu) /* * 4KB-aligned pointer of the output buffer - * (__alloc_pages_node() return page aligned address) + * (alloc_pages_node() returns page aligned address) * Buffer Size = 4KB * 2^SIZE - * contiguous physical buffer (__alloc_pages_node() with order) + * contiguous physical buffer (alloc_pages_node() with order) */ arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT; wrmsrq_on_cpu(cpu, MSR_IA32_PEBS_BASE, arch_pebs_base); -- cgit v1.2.3 From bd0eaf35cbd2696ba711935fc2bafb929c319721 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:49 +0000 Subject: KVM: VMX: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-9-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Sean Christopherson Cc: Paolo Bonzini Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cc75feec05da..6eaa337bd276 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3027,7 +3027,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) struct page *pages; struct vmcs *vmcs; - pages = __alloc_pages_node(node, flags, 0); + pages = alloc_pages_node(node, flags, 0); if (!pages) return NULL; vmcs = page_address(pages); -- cgit v1.2.3 From 1eabba2519f776a034c7b204fb9da7551cf4cc8a Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Fri, 3 Jul 2026 12:31:50 +0000 Subject: x86/virt: use higher-level allocator API The difference between __alloc_pages_node() and alloc_pages_node() is that the latter allows you to pass NUMA_NO_NODE. The former is going away and the latter works fine here so switch over. No functional change intended. Link: https://lore.kernel.org/20260703-alloc-trylock-v5-10-c87b714e19d3@google.com Signed-off-by: Brendan Jackman Reviewed-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Assisted-by: Gemini:unknown-version Signed-off-by: Andrew Morton --- arch/x86/virt/hw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 7e9091c640be..a236447ac7a2 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -196,7 +196,7 @@ static __init int __x86_vmx_init(void) struct page *page; struct vmcs *vmcs; - page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); if (WARN_ON_ONCE(!page)) { x86_vmx_exit(); return -ENOMEM; -- cgit v1.2.3 From d1e4da4ebee8ddebd07f9bb4fb094c7a78ba70b2 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 3 Jul 2026 11:41:54 +0000 Subject: arm64: make huge_ptep_get handled unaligned addresses Patch series "Fix incorrect access of hugetlb pte entries", v3. There are various places which use ptep_get() to get the pte entry corresponding to a hugetlb folio. Some arches (like s390) have special handling to compute the pteval, so they provide huge_ptep_get(). Use this helper consistently. Additionally, some code paths may provide huge_ptep_get with an unaligned address. This is a problem on arm64 (I checked other arches and it looks fine for them), which is fixed in patch 1. The fix is made to be backport-friendly: the cleaner fix would be to perhaps pass the hstate to huge_ptep_get() - that is wider churn and we can do that later. This patch (of 6): huge_ptep_get() can be handed a virtual address pointing to the middle of a contpmd/contpte mapped hugetlb folio (examples of callers are pagemap_hugetlb_range, page_mapped_in_vma). The arm64 helper rewalks the pgtables in find_num_contig to answer whether the huge pte we have maps a contpmd or a contpte hugetlb folio, and returns CONT_PMDS or CONT_PTES, so that it can collect a/d bits over the contiguous ptes. We can falsely return CONT_PTES instead of CONT_PMDS if the addr is not aligned. THis mean that on systems where CONT_PTES != CONT_PMDS (meaning page size is 16K) we could collect excess a/d bit state, meaning extra work for the kernel. Fix this by aligning the pmdp pointer down to a contpmd base before checking equality with the passed huge pte pointer, to correctly answer whether the huge pte is the base of a contpmd block. Link: https://lore.kernel.org/20260703114202.365553-1-dev.jain@arm.com Link: https://lore.kernel.org/20260703114202.365553-2-dev.jain@arm.com Fixes: 29cb80519689 ("arm64: hugetlb: Cleanup huge_pte size discovery mechanisms") Signed-off-by: Dev Jain Acked-by: David Hildenbrand (Arm) Acked-by: Muchun Song Cc: Alistair Popple Cc: Andi Kleen Cc: Anshuman Khandual Cc: Byungchul Park Cc: Catalin Marinas Cc: Dave Hansen Cc: Gregory Price Cc: Harry Yoo Cc: "Huang, Ying" Cc: Jann Horn Cc: Josh Poimboeuf Cc: Joshua Hahn Cc: Jun'ichi "Nick" Nomura Cc: Kiryl Shutsemau Cc: Lance Yang Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Brost Cc: Mel Gorman Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Pedro Falcato Cc: Rakie Kim Cc: Ralph Campbell Cc: Rik van Riel Cc: Ryan Roberts Cc: Vlastimil Babka Cc: Will Deacon Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- arch/arm64/mm/hugetlbpage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 30772a909aea..8e799c1fe0aa 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -87,7 +87,7 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, p4dp = p4d_offset(pgdp, addr); pudp = pud_offset(p4dp, addr); pmdp = pmd_offset(pudp, addr); - if ((pte_t *)pmdp == ptep) { + if ((pte_t *)PTR_ALIGN_DOWN(pmdp, sizeof(*pmdp) * CONT_PMDS) == ptep) { *pgsize = PMD_SIZE; return CONT_PMDS; } -- cgit v1.2.3 From bef7a4aa0f6b9e901ef767c4ec718d994cfc301d Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:02 +0100 Subject: mm: decouple protnone helpers from CONFIG_NUMA_BALANCING Patch series "userfaultfd: working set tracking for VM guest memory", v10. This series adds userfaultfd support for tracking the working set of VM guest memory, so a VMM can identify hot pages and reclaim cold ones to tiered or remote storage. This patch (of 15): pte_protnone() and pmd_protnone() detect present-but-inaccessible page table entries. This capability is useful beyond NUMA balancing -- for example, userfaultfd working set tracking uses protnone PTEs to track page access without unmapping pages. Introduce CONFIG_ARCH_HAS_PTE_PROTNONE to decouple the protnone PTE infrastructure from CONFIG_NUMA_BALANCING. The six architectures that support protnone PTEs (x86_64, arm64, powerpc, s390, riscv, loongarch) now select this option, and CONFIG_NUMA_BALANCING depends on it. No functional change -- the same set of architectures continues to have working protnone support, but the infrastructure is now available independently of NUMA balancing. Link: https://lore.kernel.org/20260708111417.173443-1-kirill@shutemov.name Link: https://lore.kernel.org/20260708111417.173443-2-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau (Meta) Assisted-by: Claude:claude-opus-4-6 Acked-by: SeongJae Park Acked-by: Mike Rapoport (Microsoft) Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/pgtable.h | 7 ++---- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/pgtable.h | 4 ++-- arch/powerpc/include/asm/book3s/64/pgtable.h | 8 +++---- arch/powerpc/platforms/Kconfig.cputype | 1 + arch/riscv/Kconfig | 1 + arch/riscv/include/asm/pgtable.h | 7 ++---- arch/s390/Kconfig | 1 + arch/s390/include/asm/pgtable.h | 4 ++-- arch/x86/Kconfig | 1 + arch/x86/include/asm/pgtable.h | 8 ++----- include/linux/pgtable.h | 32 +++++++++++++++++++--------- init/Kconfig | 8 +++++++ mm/debug_vm_pgtable.c | 4 ++-- 15 files changed, 52 insertions(+), 36 deletions(-) (limited to 'arch') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index e136e5abdb33..11a9c534b7b4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -78,6 +78,7 @@ config ARM64 select ARCH_SUPPORTS_CFI select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index ed1cc6dba780..313f75af1207 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -553,10 +553,7 @@ static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/linux/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { /* @@ -575,7 +572,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_present(pmd) pte_present(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index c6c611f44f3f..e20acbe5fe7b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -67,6 +67,7 @@ config LOONGARCH select ARCH_SUPPORTS_LTO_CLANG select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE if 64BIT select ARCH_SUPPORTS_NUMA_BALANCING if NUMA select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_SUPPORTS_RT diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 223528c04d73..1952e34bc8ee 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -623,7 +623,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline long pte_protnone(pte_t pte) { return (pte_val(pte) & _PAGE_PROTNONE); @@ -633,7 +633,7 @@ static inline long pmd_protnone(pmd_t pmd) { return (pmd_val(pmd) & _PAGE_PROTNONE); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0) #define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0) diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 800ff3cc5f71..f4db7d7fbd5c 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -490,13 +490,13 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte) } #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) == cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_hw_valid(pte_t pte) { @@ -1067,12 +1067,12 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) #endif #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index 9f6847b53e37..f0adbf7ecb7f 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -87,6 +87,7 @@ config PPC_BOOK3S_64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK select ARCH_SUPPORTS_HUGETLBFS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select HAVE_MOVE_PMD select HAVE_MOVE_PUD diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 760d0e0ae1f0..7b9c373d82fa 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -71,6 +71,7 @@ config RISCV select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU select ARCH_SUPPORTS_PER_VMA_LOCK if MMU + select ARCH_HAS_PTE_PROTNONE if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_SCHED_MC if SMP diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 23eae82c26a6..cbc58ec42752 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -534,10 +534,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) PAGE_SIZE) #endif -#ifdef CONFIG_NUMA_BALANCING -/* - * See the comment in include/asm-generic/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; @@ -547,7 +544,7 @@ static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 6b317717bf48..ab8fccc2cc4e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -148,6 +148,7 @@ config S390 select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS + select ARCH_HAS_PTE_PROTNONE select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_SUPPORTS_PAGE_TABLE_CHECK select ARCH_SUPPORTS_PER_VMA_LOCK diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 645fa2d99c6a..e882663a58e7 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -842,7 +842,7 @@ static inline int pte_same(pte_t a, pte_t b) return pte_val(a) == pte_val(b); } -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return pte_present(pte) && !(pte_val(pte) & _PAGE_READ); @@ -853,7 +853,7 @@ static inline int pmd_protnone(pmd_t pmd) /* pmd_leaf(pmd) implies pmd_present(pmd) */ return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ); } -#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline bool pte_swp_exclusive(pte_t pte) { diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 50acb2830570..fb298e219179 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -124,6 +124,7 @@ config X86 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_HUGETLBFS select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64 + select ARCH_HAS_PTE_PROTNONE if X86_64 select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 select ARCH_SUPPORTS_CFI if X86_64 diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index f69b02c1d3d3..ef61a75cd98b 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -985,11 +985,7 @@ static inline int pmd_present(pmd_t pmd) return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE); } -#ifdef CONFIG_NUMA_BALANCING -/* - * These work without NUMA balancing but the kernel does not care. See the - * comment in include/linux/pgtable.h - */ +#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE static inline int pte_protnone(pte_t pte) { return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT)) @@ -1001,7 +997,7 @@ static inline int pmd_protnone(pmd_t pmd) return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT)) == _PAGE_PROTNONE; } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ static inline int pmd_none(pmd_t pmd) { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 3df6db1e9e39..8c093c119e5a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2110,18 +2110,26 @@ static inline int pud_trans_unstable(pud_t *pud) return 0; } -#ifndef CONFIG_NUMA_BALANCING +#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE /* - * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is - * perfectly valid to indicate "no" in that case, which is why our default - * implementation defaults to "always no". + * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It + * is perfectly valid to indicate "no" in that case, which is why our + * default implementation defaults to "always no". * - * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE - * page protection due to NUMA hinting. NUMA hinting faults only apply in - * accessible VMAs. + * In an accessible VMA, pte_protnone() reliably indicates a present + * PROT_NONE page protection. Today the kernel uses such PTEs for two + * purposes: NUMA hinting faults, and userfaultfd RWP tracking on + * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and + * the VMA flag; see include/linux/userfaultfd_k.h. * - * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, - * looking at the VMA accessibility is sufficient. + * So, to reliably identify PROT_NONE PTEs that require kernel handling, + * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is + * sufficient. + * + * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero + * stubs below; PAGE_NONE references that survive to runtime fire the + * BUILD_BUG() fallback, since callers should have folded such paths to + * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). */ static inline int pte_protnone(pte_t pte) { @@ -2132,7 +2140,11 @@ static inline int pmd_protnone(pmd_t pmd) { return 0; } -#endif /* CONFIG_NUMA_BALANCING */ + +#ifndef PAGE_NONE +#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; }) +#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #endif /* CONFIG_MMU */ diff --git a/init/Kconfig b/init/Kconfig index 5230d4879b1c..9d91074c5d0a 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -944,6 +944,13 @@ config SCHED_PROXY_EXEC endmenu +# +# For architectures that support present-but-inaccessible (PROT_NONE) page +# table entries detectable via pte_protnone() / pmd_protnone(): +# +config ARCH_HAS_PTE_PROTNONE + bool + # # For architectures that want to enable the support for NUMA-affine scheduler # balancing logic: @@ -1010,6 +1017,7 @@ config ARCH_WANT_NUMA_VARIABLE_LOCALITY config NUMA_BALANCING bool "Memory placement aware NUMA scheduler" depends on ARCH_SUPPORTS_NUMA_BALANCING + depends on ARCH_HAS_PTE_PROTNONE depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY depends on SMP && NUMA_MIGRATION && !PREEMPT_RT help diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 7987962950c1..2875fd22d7bb 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -672,7 +672,7 @@ static void __init pte_protnone_tests(struct pgtable_debug_args *args) { pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; pr_debug("Validating PTE protnone\n"); @@ -685,7 +685,7 @@ static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { pmd_t pmd; - if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return; if (!has_transparent_hugepage()) -- cgit v1.2.3 From f5414a5d20c5d31c1bc8e47b9309b58cbca40be3 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:03 +0100 Subject: mm: rename uffd-wp PTE bit macros to uffd The uffd-wp PTE bit is about to gain a second consumer: userfaultfd RWP will use the same bit to mark access-tracking PTEs, distinct from mprotect(PROT_NONE) or NUMA-hinting PTEs. WP vs RWP semantics come from the VMA flag; the bit is just "uffd has claimed this entry." Drop the "_wp" suffix from the arch-private bit macros so they reflect that. x86: _PAGE_BIT_UFFD_WP -> _PAGE_BIT_UFFD _PAGE_UFFD_WP -> _PAGE_UFFD _PAGE_SWP_UFFD_WP -> _PAGE_SWP_UFFD arm64: PTE_UFFD_WP -> PTE_UFFD PTE_SWP_UFFD_WP -> PTE_SWP_UFFD riscv: _PAGE_UFFD_WP -> _PAGE_UFFD _PAGE_SWP_UFFD_WP -> _PAGE_SWP_UFFD Pure mechanical rename -- no behavior change. Link: https://lore.kernel.org/20260708111417.173443-3-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: SeongJae Park Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable-prot.h | 8 ++++---- arch/arm64/include/asm/pgtable.h | 12 ++++++------ arch/riscv/include/asm/pgtable-bits.h | 12 ++++++------ arch/riscv/include/asm/pgtable.h | 14 +++++++------- arch/x86/include/asm/pgtable.h | 24 ++++++++++++------------ arch/x86/include/asm/pgtable_types.h | 16 ++++++++-------- 6 files changed, 43 insertions(+), 43 deletions(-) (limited to 'arch') diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 212ce1b02e15..09d7c00cf405 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -28,11 +28,11 @@ #define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */ -#define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */ +#define PTE_UFFD (_AT(pteval_t, 1) << 58) /* userfaultfd tracking */ +#define PTE_SWP_UFFD (_AT(pteval_t, 1) << 3) /* only for swp ptes */ #else -#define PTE_UFFD_WP (_AT(pteval_t, 0)) -#define PTE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#define PTE_UFFD (_AT(pteval_t, 0)) +#define PTE_SWP_UFFD (_AT(pteval_t, 0)) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 313f75af1207..0e490a2cf665 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -343,17 +343,17 @@ static inline pmd_t pmd_mknoncont(pmd_t pmd) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & PTE_UFFD_WP); + return !!(pte_val(pte) & PTE_UFFD); } static inline pte_t pte_mkuffd_wp(pte_t pte) { - return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP))); + return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD))); } static inline pte_t pte_clear_uffd_wp(pte_t pte) { - return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP)); + return clear_pte_bit(pte, __pgprot(PTE_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -539,17 +539,17 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); + return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } static inline int pte_swp_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & PTE_SWP_UFFD_WP); + return !!(pte_val(pte) & PTE_SWP_UFFD); } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { - return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP)); + return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index b422d9691e60..d5a86b4df3ce 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -40,20 +40,20 @@ #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */ -#define _PAGE_UFFD_WP \ +/* ext_svrsw60t59b: Bit(60) for userfaultfd tracking */ +#define _PAGE_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ (1UL << 60) : 0) /* * Bit 4 is not involved into swap entry computation, so we - * can borrow it for swap page uffd-wp tracking. + * can borrow it for swap page userfaultfd tracking. */ -#define _PAGE_SWP_UFFD_WP \ +#define _PAGE_SWP_UFFD \ ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ _PAGE_USER : 0) #else -#define _PAGE_UFFD_WP 0 -#define _PAGE_SWP_UFFD_WP 0 +#define _PAGE_UFFD 0 +#define _PAGE_SWP_UFFD 0 #endif #define _PAGE_TABLE _PAGE_PRESENT diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index cbc58ec42752..ae4307cbaed5 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -415,32 +415,32 @@ static inline pte_t pte_wrprotect(pte_t pte) static inline bool pte_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & _PAGE_UFFD_WP); + return !!(pte_val(pte) & _PAGE_UFFD); } static inline pte_t pte_mkuffd_wp(pte_t pte) { - return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP)); + return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } static inline pte_t pte_clear_uffd_wp(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } static inline bool pte_swp_uffd_wp(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP); + return !!(pte_val(pte) & _PAGE_SWP_UFFD); } static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP); + return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP)); + return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -1167,7 +1167,7 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot) * bit 0: _PAGE_PRESENT (zero) * bit 1 to 2: (zero) * bit 3: _PAGE_SWP_SOFT_DIRTY - * bit 4: _PAGE_SWP_UFFD_WP + * bit 4: _PAGE_SWP_UFFD * bit 5: _PAGE_PROT_NONE (zero) * bit 6: exclusive marker * bits 7 to 11: swap type diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ef61a75cd98b..54432c308f26 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -413,17 +413,17 @@ static inline pte_t pte_wrprotect(pte_t pte) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd_wp(pte_t pte) { - return pte_flags(pte) & _PAGE_UFFD_WP; + return pte_flags(pte) & _PAGE_UFFD; } static inline pte_t pte_mkuffd_wp(pte_t pte) { - return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD_WP)); + return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD)); } static inline pte_t pte_clear_uffd_wp(pte_t pte) { - return pte_clear_flags(pte, _PAGE_UFFD_WP); + return pte_clear_flags(pte, _PAGE_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -528,17 +528,17 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pmd_uffd_wp(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_UFFD_WP; + return pmd_flags(pmd) & _PAGE_UFFD; } static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) { - return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD_WP)); + return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD)); } static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_UFFD_WP); + return pmd_clear_flags(pmd, _PAGE_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ @@ -1562,32 +1562,32 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline pte_t pte_swp_mkuffd_wp(pte_t pte) { - return pte_set_flags(pte, _PAGE_SWP_UFFD_WP); + return pte_set_flags(pte, _PAGE_SWP_UFFD); } static inline int pte_swp_uffd_wp(pte_t pte) { - return pte_flags(pte) & _PAGE_SWP_UFFD_WP; + return pte_flags(pte) & _PAGE_SWP_UFFD; } static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) { - return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP); + return pte_clear_flags(pte, _PAGE_SWP_UFFD); } static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) { - return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP); + return pmd_set_flags(pmd, _PAGE_SWP_UFFD); } static inline int pmd_swp_uffd_wp(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP; + return pmd_flags(pmd) & _PAGE_SWP_UFFD; } static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP); + return pmd_clear_flags(pmd, _PAGE_SWP_UFFD); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 2ec250ba467e..af08d98be930 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -31,7 +31,7 @@ #define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 -#define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */ +#define _PAGE_BIT_UFFD _PAGE_BIT_SOFTW2 /* userfaultfd tracking */ #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_KERNEL_4K _PAGE_BIT_SOFTW3 /* page must not be converted to large */ @@ -39,7 +39,7 @@ #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */ #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */ #else -/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */ +/* Shared with _PAGE_BIT_UFFD which is not supported on 32 bit */ #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */ #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */ #endif @@ -111,11 +111,11 @@ #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP) -#define _PAGE_SWP_UFFD_WP _PAGE_USER +#define _PAGE_UFFD (_AT(pteval_t, 1) << _PAGE_BIT_UFFD) +#define _PAGE_SWP_UFFD _PAGE_USER #else -#define _PAGE_UFFD_WP (_AT(pteval_t, 0)) -#define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0)) +#define _PAGE_UFFD (_AT(pteval_t, 0)) +#define _PAGE_SWP_UFFD (_AT(pteval_t, 0)) #endif #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) @@ -129,7 +129,7 @@ /* * The hardware requires shadow stack to be Write=0,Dirty=1. However, * there are valid cases where the kernel might create read-only PTEs that - * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In + * are dirty (e.g., fork(), mprotect(), userfaultfd, soft-dirty tracking). In * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit, * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have * (Write=0,SavedDirty=1,Dirty=0) set. @@ -151,7 +151,7 @@ #define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | \ _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \ - _PAGE_CC | _PAGE_UFFD_WP) + _PAGE_CC | _PAGE_UFFD) #define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT) #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE) -- cgit v1.2.3 From bc965ee22ee91bf6d85d847c9e3fea69fa38e142 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Wed, 8 Jul 2026 12:14:04 +0100 Subject: mm: rename uffd-wp PTE accessors to uffd Userfaultfd RWP will reuse the uffd-wp PTE bit to mark access-tracking PTEs, alongside the write-protected ones it already marks. The bit's meaning now depends on the VMA flag (WP or RWP), not on its name. Rename the kernel-internal names that describe the bit: - pte/pmd/huge_pte accessors (and swap variants) - pgtable_supports_uffd() capability query - SCAN_PTE_UFFD khugepaged enum The ftrace string emitted by mm_khugepaged_scan_pmd for this enum is kept as "pte_uffd_wp" so existing trace-based tooling keeps matching. Pure mechanical rename -- no behavior change. Link: https://lore.kernel.org/20260708111417.173443-4-kirill@shutemov.name Signed-off-by: Kiryl Shutsemau Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: SeongJae Park Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: James Houghton Cc: Jonathan Corbet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Paolo Bonzini Cc: Peter Xu Cc: Sean Christopherson Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable.h | 28 +++++++++--------- arch/riscv/include/asm/pgtable.h | 38 ++++++++++++------------- arch/s390/include/asm/hugetlb.h | 12 ++++---- arch/x86/include/asm/pgtable.h | 24 ++++++++-------- fs/proc/task_mmu.c | 44 ++++++++++++++--------------- include/asm-generic/hugetlb.h | 18 ++++++------ include/asm-generic/pgtable_uffd.h | 32 ++++++++++----------- include/linux/leafops.h | 4 +-- include/linux/mm_inline.h | 4 +-- include/linux/swapops.h | 4 +-- include/linux/userfaultfd_k.h | 14 ++++----- include/trace/events/huge_memory.h | 2 +- mm/huge_memory.c | 58 +++++++++++++++++++------------------- mm/hugetlb.c | 44 ++++++++++++++--------------- mm/internal.h | 4 +-- mm/khugepaged.c | 22 +++++++-------- mm/memory.c | 34 +++++++++++----------- mm/migrate.c | 12 ++++---- mm/migrate_device.c | 8 +++--- mm/mprotect.c | 12 ++++---- mm/mremap.c | 4 +-- mm/page_table_check.c | 8 +++--- mm/rmap.c | 16 +++++------ mm/swapfile.c | 4 +-- mm/userfaultfd.c | 6 ++-- 25 files changed, 228 insertions(+), 228 deletions(-) (limited to 'arch') diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0e490a2cf665..a2681d755358 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -341,17 +341,17 @@ static inline pmd_t pmd_mknoncont(pmd_t pmd) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pte_uffd_wp(pte_t pte) +static inline int pte_uffd(pte_t pte) { return !!(pte_val(pte) & PTE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD))); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_UFFD)); } @@ -537,17 +537,17 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } -static inline int pte_swp_uffd_wp(pte_t pte) +static inline int pte_swp_uffd(pte_t pte) { return !!(pte_val(pte) & PTE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD)); } @@ -590,13 +590,13 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd))) #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd)) -#define pmd_mkuffd_wp(pmd) pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))) -#define pmd_clear_uffd_wp(pmd) pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))) -#define pmd_swp_uffd_wp(pmd) pte_swp_uffd_wp(pmd_pte(pmd)) -#define pmd_swp_mkuffd_wp(pmd) pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))) -#define pmd_swp_clear_uffd_wp(pmd) \ - pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))) +#define pmd_uffd(pmd) pte_uffd(pmd_pte(pmd)) +#define pmd_mkuffd(pmd) pte_pmd(pte_mkuffd(pmd_pte(pmd))) +#define pmd_clear_uffd(pmd) pte_pmd(pte_clear_uffd(pmd_pte(pmd))) +#define pmd_swp_uffd(pmd) pte_swp_uffd(pmd_pte(pmd)) +#define pmd_swp_mkuffd(pmd) pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))) +#define pmd_swp_clear_uffd(pmd) \ + pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))) #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) @@ -1512,7 +1512,7 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * Encode and decode a swap entry: * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive - * bit 3: remember uffd-wp state + * bit 3: remember uffd state * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index ae4307cbaed5..7f21742b2b6a 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -410,35 +410,35 @@ static inline pte_t pte_wrprotect(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -#define pgtable_supports_uffd_wp() \ +#define pgtable_supports_uffd() \ riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B) -static inline bool pte_uffd_wp(pte_t pte) +static inline bool pte_uffd(pte_t pte) { return !!(pte_val(pte) & _PAGE_UFFD); } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { return __pte(pte_val(pte) & ~(_PAGE_UFFD)); } -static inline bool pte_swp_uffd_wp(pte_t pte) +static inline bool pte_swp_uffd(pte_t pte) { return !!(pte_val(pte) & _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { return __pte(pte_val(pte) | _PAGE_SWP_UFFD); } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD)); } @@ -896,34 +896,34 @@ static inline pud_t pud_mkspecial(pud_t pud) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline bool pmd_uffd_wp(pmd_t pmd) +static inline bool pmd_uffd(pmd_t pmd) { - return pte_uffd_wp(pmd_pte(pmd)); + return pte_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { - return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_clear_uffd(pmd_pte(pmd))); } -static inline bool pmd_swp_uffd_wp(pmd_t pmd) +static inline bool pmd_swp_uffd(pmd_t pmd) { - return pte_swp_uffd_wp(pmd_pte(pmd)); + return pte_swp_uffd(pmd_pte(pmd)); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { - return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd))); } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { - return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))); + return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd))); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index e33a5b587ee4..02821a7d9d91 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -77,20 +77,20 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte)); } -#define __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { return pte; } -#define __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#define __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { return 0; } diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 54432c308f26..8e0018fadd14 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -411,17 +411,17 @@ static inline pte_t pte_wrprotect(pte_t pte) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pte_uffd_wp(pte_t pte) +static inline int pte_uffd(pte_t pte) { return pte_flags(pte) & _PAGE_UFFD; } -static inline pte_t pte_mkuffd_wp(pte_t pte) +static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD)); } -static inline pte_t pte_clear_uffd_wp(pte_t pte) +static inline pte_t pte_clear_uffd(pte_t pte) { return pte_clear_flags(pte, _PAGE_UFFD); } @@ -526,17 +526,17 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline int pmd_uffd_wp(pmd_t pmd) +static inline int pmd_uffd(pmd_t pmd) { return pmd_flags(pmd) & _PAGE_UFFD; } -static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_mkuffd(pmd_t pmd) { return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD)); } -static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_clear_uffd(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_UFFD); } @@ -1560,32 +1560,32 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) #endif #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static inline pte_t pte_swp_mkuffd(pte_t pte) { return pte_set_flags(pte, _PAGE_SWP_UFFD); } -static inline int pte_swp_uffd_wp(pte_t pte) +static inline int pte_swp_uffd(pte_t pte) { return pte_flags(pte) & _PAGE_SWP_UFFD; } -static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static inline pte_t pte_swp_clear_uffd(pte_t pte) { return pte_clear_flags(pte, _PAGE_SWP_UFFD); } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_SWP_UFFD); } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { return pmd_flags(pmd) & _PAGE_SWP_UFFD; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_SWP_UFFD); } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 95012243b9b3..75cfd9befb4d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2035,14 +2035,14 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) flags |= PM_SOFT_DIRTY; - if (pte_uffd_wp(pte)) + if (pte_uffd(pte)) flags |= PM_UFFD_WP; } else { softleaf_t entry; if (pte_swp_soft_dirty(pte)) flags |= PM_SOFT_DIRTY; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) flags |= PM_UFFD_WP; entry = softleaf_from_pte(pte); if (pm->show_pfn) { @@ -2108,7 +2108,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) flags |= PM_SOFT_DIRTY; - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) flags |= PM_UFFD_WP; if (pm->show_pfn) frame = pmd_pfn(pmd) + idx; @@ -2127,7 +2127,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SWAP; if (pmd_swp_soft_dirty(pmd)) flags |= PM_SOFT_DIRTY; - if (pmd_swp_uffd_wp(pmd)) + if (pmd_swp_uffd(pmd)) flags |= PM_UFFD_WP; if (softleaf_has_pfn(entry)) page = softleaf_to_page(entry); @@ -2233,14 +2233,14 @@ static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, !hugetlb_pmd_shared(ptep)) flags |= PM_MMAP_EXCLUSIVE; - if (huge_pte_uffd_wp(pte)) + if (huge_pte_uffd(pte)) flags |= PM_UFFD_WP; flags |= PM_PRESENT; if (pm->show_pfn) frame = pte_pfn(pte) + ((addr & ~hmask) >> PAGE_SHIFT); - } else if (pte_swp_uffd_wp_any(pte)) { + } else if (pte_swp_uffd_any(pte)) { flags |= PM_UFFD_WP; } @@ -2451,7 +2451,7 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_PRESENT; - if (!pte_uffd_wp(pte)) + if (!pte_uffd(pte)) categories |= PAGE_IS_WRITTEN; if (p->masks_of_interest & PAGE_IS_FILE) { @@ -2469,7 +2469,7 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, categories = PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) + if (!pte_swp_uffd_any(pte)) categories |= PAGE_IS_WRITTEN; entry = softleaf_from_pte(pte); @@ -2494,13 +2494,13 @@ static void make_uffd_wp_pte(struct vm_area_struct *vma, pte_t old_pte; old_pte = ptep_modify_prot_start(vma, addr, pte); - ptent = pte_mkuffd_wp(old_pte); + ptent = pte_mkuffd(old_pte); ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); } else if (pte_none(ptent)) { set_pte_at(vma->vm_mm, addr, pte, make_pte_marker(PTE_MARKER_UFFD_WP)); } else { - ptent = pte_swp_mkuffd_wp(ptent); + ptent = pte_swp_mkuffd(ptent); set_pte_at(vma->vm_mm, addr, pte, ptent); } } @@ -2519,7 +2519,7 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, struct page *page; categories |= PAGE_IS_PRESENT; - if (!pmd_uffd_wp(pmd)) + if (!pmd_uffd(pmd)) categories |= PAGE_IS_WRITTEN; if (p->masks_of_interest & PAGE_IS_FILE) { @@ -2534,7 +2534,7 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, categories |= PAGE_IS_SOFT_DIRTY; } else { categories |= PAGE_IS_SWAPPED; - if (!pmd_swp_uffd_wp(pmd)) + if (!pmd_swp_uffd(pmd)) categories |= PAGE_IS_WRITTEN; if (pmd_swp_soft_dirty(pmd)) categories |= PAGE_IS_SOFT_DIRTY; @@ -2558,10 +2558,10 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma, if (pmd_present(pmd)) { old = pmdp_invalidate_ad(vma, addr, pmdp); - pmd = pmd_mkuffd_wp(old); + pmd = pmd_mkuffd(old); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } else if (pmd_is_migration_entry(pmd)) { - pmd = pmd_swp_mkuffd_wp(pmd); + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } } @@ -2583,7 +2583,7 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) if (pte_present(pte)) { categories |= PAGE_IS_PRESENT; - if (!huge_pte_uffd_wp(pte)) + if (!huge_pte_uffd(pte)) categories |= PAGE_IS_WRITTEN; if (!PageAnon(pte_page(pte))) categories |= PAGE_IS_FILE; @@ -2594,7 +2594,7 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) } else { categories |= PAGE_IS_SWAPPED; - if (!pte_swp_uffd_wp_any(pte)) + if (!pte_swp_uffd_any(pte)) categories |= PAGE_IS_WRITTEN; if (pte_swp_soft_dirty(pte)) categories |= PAGE_IS_SOFT_DIRTY; @@ -2622,12 +2622,12 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, if (softleaf_is_migration(entry)) { set_huge_pte_at(vma->vm_mm, addr, ptep, - pte_swp_mkuffd_wp(ptent), psize); + pte_swp_mkuffd(ptent), psize); } else { pte_t old_pte, new_pte; old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); - new_pte = huge_pte_mkuffd_wp(old_pte); + new_pte = huge_pte_mkuffd(old_pte); huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); } } @@ -2860,8 +2860,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { pte_t ptent = ptep_get(pte); - if ((pte_present(ptent) && pte_uffd_wp(ptent)) || - pte_swp_uffd_wp_any(ptent)) + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) continue; make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) @@ -2878,8 +2878,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long next = addr + PAGE_SIZE; pte_t ptent = ptep_get(pte); - if ((pte_present(ptent) && pte_uffd_wp(ptent)) || - pte_swp_uffd_wp_any(ptent)) + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) continue; ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, p, addr, &next); diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h index e1a2e1b7c8e7..635c41cc3479 100644 --- a/include/asm-generic/hugetlb.h +++ b/include/asm-generic/hugetlb.h @@ -37,24 +37,24 @@ static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) return pte_modify(pte, newprot); } -#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { - return huge_pte_wrprotect(pte_mkuffd_wp(pte)); + return huge_pte_wrprotect(pte_mkuffd(pte)); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { - return pte_clear_uffd_wp(pte); + return pte_clear_uffd(pte); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { - return pte_uffd_wp(pte); + return pte_uffd(pte); } #endif diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h index 0d85791efdf7..30e88fc1de2f 100644 --- a/include/asm-generic/pgtable_uffd.h +++ b/include/asm-generic/pgtable_uffd.h @@ -2,79 +2,79 @@ #define _ASM_GENERIC_PGTABLE_UFFD_H /* - * Some platforms can customize the uffd-wp bit, making it unavailable + * Some platforms can customize the uffd PTE bit, making it unavailable * even if the architecture provides the resource. * Adding this API allows architectures to add their own checks for the * devices on which the kernel is running. * Note: When overriding it, please make sure the * CONFIG_HAVE_ARCH_USERFAULTFD_WP is part of this macro. */ -#ifndef pgtable_supports_uffd_wp -#define pgtable_supports_uffd_wp() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) +#ifndef pgtable_supports_uffd +#define pgtable_supports_uffd() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) #endif static inline bool uffd_supports_wp_marker(void) { - return pgtable_supports_uffd_wp() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); + return pgtable_supports_uffd() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); } #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static __always_inline int pte_uffd_wp(pte_t pte) +static __always_inline int pte_uffd(pte_t pte) { return 0; } -static __always_inline int pmd_uffd_wp(pmd_t pmd) +static __always_inline int pmd_uffd(pmd_t pmd) { return 0; } -static __always_inline pte_t pte_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_mkuffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_mkuffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_clear_uffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_clear_uffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_mkuffd(pte_t pte) { return pte; } -static __always_inline int pte_swp_uffd_wp(pte_t pte) +static __always_inline int pte_swp_uffd(pte_t pte) { return 0; } -static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_clear_uffd(pte_t pte) { return pte; } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { return pmd; } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { return 0; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { return pmd; } diff --git a/include/linux/leafops.h b/include/linux/leafops.h index ef84ac1cce19..4c1476ae3234 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -100,8 +100,8 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) if (pmd_swp_soft_dirty(pmd)) pmd = pmd_swp_clear_soft_dirty(pmd); - if (pmd_swp_uffd_wp(pmd)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (pmd_swp_uffd(pmd)) + pmd = pmd_swp_clear_uffd(pmd); arch_entry = __pmd_to_swp_entry(pmd); /* Temporary until swp_entry_t eliminated. */ diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index efcddb9925ad..b5c4dc0f3fe3 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -600,14 +600,14 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, return false; /* A uffd-wp wr-protected normal pte */ - if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval))) + if (unlikely(pte_present(pteval) && pte_uffd(pteval))) arm_uffd_pte = true; /* * A uffd-wp wr-protected swap pte. Note: this should even cover an * existing pte marker with uffd-wp bit set. */ - if (unlikely(pte_swp_uffd_wp_any(pteval))) + if (unlikely(pte_swp_uffd_any(pteval))) arm_uffd_pte = true; if (unlikely(arm_uffd_pte)) { diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 1125401b787e..c956bc445ee0 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -73,8 +73,8 @@ static inline pte_t pte_swp_clear_flags(pte_t pte) pte = pte_swp_clear_exclusive(pte); if (pte_swp_soft_dirty(pte)) pte = pte_swp_clear_soft_dirty(pte); - if (pte_swp_uffd_wp(pte)) - pte = pte_swp_clear_uffd_wp(pte); + if (pte_swp_uffd(pte)) + pte = pte_swp_clear_uffd(pte); return pte; } diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 68edac4dcd78..658740df2978 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -211,13 +211,13 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { - return userfaultfd_wp(vma) && pte_uffd_wp(pte); + return userfaultfd_wp(vma) && pte_uffd(pte); } static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd) { - return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); + return userfaultfd_wp(vma) && pmd_uffd(pmd); } static inline bool userfaultfd_armed(struct vm_area_struct *vma) @@ -272,10 +272,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { if (!uffd_supports_wp_marker()) return false; @@ -283,7 +283,7 @@ static inline bool pte_swp_uffd_wp_any(pte_t pte) if (pte_present(pte)) return false; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) return true; if (pte_is_uffd_wp_marker(pte)) @@ -424,7 +424,7 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) * Returns true if this is a swap pte and was uffd-wp wr-protected in either * forms (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { return false; } diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 291fae364c62..5a48c5406cce 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -16,7 +16,7 @@ EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \ - EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \ + EM( SCAN_PTE_UFFD, "pte_uffd_wp") \ EM( SCAN_PTE_MAPPED_HUGEPAGE, "pte_mapped_hugepage") \ EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \ EM( SCAN_PAGE_NULL, "page_null") \ diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8860cbf6c1d5..6ec8ecaf3255 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1822,8 +1822,8 @@ static void copy_huge_non_present_pmd( pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } else if (softleaf_is_device_private(entry)) { /* @@ -1836,8 +1836,8 @@ static void copy_huge_non_present_pmd( if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } @@ -1857,7 +1857,7 @@ static void copy_huge_non_present_pmd( mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); if (!userfaultfd_wp(dst_vma)) - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); } @@ -1953,7 +1953,7 @@ out_zero_page: pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); pmdp_set_wrprotect(src_mm, addr, src_pmd); if (!userfaultfd_wp(dst_vma)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); pmd = pmd_wrprotect(pmd); set_pmd: pmd = pmd_mkold(pmd); @@ -2494,9 +2494,9 @@ static pmd_t clear_uffd_wp_pmd(pmd_t pmd) if (pmd_none(pmd)) return pmd; if (pmd_present(pmd)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); else - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); return pmd; } @@ -2577,16 +2577,16 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); newpmd = softleaf_to_pmd(entry); - if (pmd_swp_uffd_wp(*pmd)) - newpmd = pmd_swp_mkuffd_wp(newpmd); + if (pmd_swp_uffd(*pmd)) + newpmd = pmd_swp_mkuffd(newpmd); } else { newpmd = *pmd; } if (uffd_wp) - newpmd = pmd_swp_mkuffd_wp(newpmd); + newpmd = pmd_swp_mkuffd(newpmd); else if (uffd_wp_resolve) - newpmd = pmd_swp_clear_uffd_wp(newpmd); + newpmd = pmd_swp_clear_uffd(newpmd); if (!pmd_same(*pmd, newpmd)) set_pmd_at(mm, addr, pmd, newpmd); } @@ -2667,14 +2667,14 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, entry = pmd_modify(oldpmd, newprot); if (uffd_wp) - entry = pmd_mkuffd_wp(entry); + entry = pmd_mkuffd(entry); else if (uffd_wp_resolve) /* * Leave the write bit to be handled by PF interrupt * handler, then things like COW could be properly * handled. */ - entry = pmd_clear_uffd_wp(entry); + entry = pmd_clear_uffd(entry); /* See change_pte_range(). */ if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && @@ -3017,8 +3017,8 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, entry = pfn_pte(zero_pfn(addr), vma->vm_page_prot); entry = pte_mkspecial(entry); - if (pmd_uffd_wp(old_pmd)) - entry = pte_mkuffd_wp(entry); + if (pmd_uffd(old_pmd)) + entry = pte_mkuffd(entry); VM_BUG_ON(!pte_none(ptep_get(pte))); set_pte_at(mm, addr, pte, entry); pte++; @@ -3104,7 +3104,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_migration_write(entry); if (PageAnon(page)) @@ -3120,7 +3120,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_device_private_write(entry); anon_exclusive = PageAnonExclusive(page); @@ -3177,7 +3177,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); - uffd_wp = pmd_uffd_wp(old_pmd); + uffd_wp = pmd_uffd(old_pmd); VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); @@ -3248,7 +3248,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3275,7 +3275,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3293,7 +3293,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); for (i = 0; i < HPAGE_PMD_NR; i++) VM_WARN_ON(!pte_none(ptep_get(pte + i))); @@ -4913,7 +4913,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = pmd_write(pmdval); softdirty = pmd_soft_dirty(pmdval); - uffd_wp = pmd_uffd_wp(pmdval); + uffd_wp = pmd_uffd(pmdval); } else { softleaf_t old_entry; @@ -4922,7 +4922,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, writable = softleaf_is_device_private_write(old_entry); softdirty = pmd_swp_soft_dirty(pmdval); - uffd_wp = pmd_swp_uffd_wp(pmdval); + uffd_wp = pmd_swp_uffd(pmdval); } /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ @@ -4953,7 +4953,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) - pmdswp = pmd_swp_mkuffd_wp(pmdswp); + pmdswp = pmd_swp_mkuffd(pmdswp); set_pmd_at(mm, address, pvmw->pmd, pmdswp); /* Migration entry installed: cleanup rmap, folio. */ @@ -4985,8 +4985,8 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) pmde = pmd_mksoft_dirty(pmde); if (softleaf_is_migration_write(entry)) pmde = pmd_mkwrite(pmde, vma); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_mkuffd(pmde); if (!softleaf_is_migration_young(entry)) pmde = pmd_mkold(pmde); /* NOTE: this may contain setting soft-dirty on some archs */ @@ -5006,8 +5006,8 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_swp_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_swp_mkuffd(pmde); } if (folio_test_anon(folio)) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ccd82741d76e..989156c0de83 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4883,8 +4883,8 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add __folio_mark_uptodate(new_folio); hugetlb_add_new_anon_rmap(new_folio, vma, addr); - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old)) - newpte = huge_pte_mkuffd_wp(newpte); + if (userfaultfd_wp(vma) && huge_pte_uffd(old)) + newpte = huge_pte_mkuffd(newpte); set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz); hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm); folio_set_hugetlb_migratable(new_folio); @@ -4965,7 +4965,7 @@ again: */ set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(softleaf_is_migration(softleaf))) { - bool uffd_wp = pte_swp_uffd_wp(entry); + bool uffd = pte_swp_uffd(entry); if (!softleaf_is_migration_read(softleaf) && cow) { /* @@ -4975,12 +4975,12 @@ again: softleaf = make_readable_migration_entry( swp_offset(softleaf)); entry = swp_entry_to_pte(softleaf); - if (userfaultfd_wp(src_vma) && uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + if (userfaultfd_wp(src_vma) && uffd) + entry = pte_swp_mkuffd(entry); set_huge_pte_at(src, addr, src_pte, entry, sz); } if (!userfaultfd_wp(dst_vma)) - entry = pte_swp_clear_uffd_wp(entry); + entry = pte_swp_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(pte_is_marker(entry))) { const pte_marker marker = copy_pte_marker(softleaf, dst_vma); @@ -5057,7 +5057,7 @@ again: } if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); + entry = huge_pte_clear_uffd(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); hugetlb_count_add(npages, dst); @@ -5105,9 +5105,9 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, } else { if (need_clear_uffd_wp) { if (pte_present(pte)) - pte = huge_pte_clear_uffd_wp(pte); + pte = huge_pte_clear_uffd(pte); else - pte = pte_swp_clear_uffd_wp(pte); + pte = pte_swp_clear_uffd(pte); } set_huge_pte_at(mm, new_addr, dst_pte, pte, sz); } @@ -5241,7 +5241,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, * drop the uffd-wp bit in this zap, then replace the * pte with a marker. */ - if (pte_swp_uffd_wp_any(pte) && + if (pte_swp_uffd_any(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5277,7 +5277,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, if (huge_pte_dirty(pte)) folio_mark_dirty(folio); /* Leave a uffd-wp pte marker if needed */ - if (huge_pte_uffd_wp(pte) && + if (huge_pte_uffd(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), @@ -5481,7 +5481,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf) * can trigger this, because hugetlb_fault() will always resolve * uffd-wp bit first. */ - if (!unshare && huge_pte_uffd_wp(pte)) + if (!unshare && huge_pte_uffd(pte)) return 0; /* Let's take out MAP_SHARED mappings first. */ @@ -5625,8 +5625,8 @@ retry_avoidcopy: huge_ptep_clear_flush(vma, vmf->address, vmf->pte); hugetlb_remove_rmap(old_folio); hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address); - if (huge_pte_uffd_wp(pte)) - newpte = huge_pte_mkuffd_wp(newpte); + if (huge_pte_uffd(pte)) + newpte = huge_pte_mkuffd(newpte); set_huge_pte_at(mm, vmf->address, vmf->pte, newpte, huge_page_size(h)); folio_set_hugetlb_migratable(new_folio); @@ -5904,7 +5904,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, * if populated. */ if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte))) - new_pte = huge_pte_mkuffd_wp(new_pte); + new_pte = huge_pte_mkuffd(new_pte); set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h)); hugetlb_count_add(pages_per_huge_page(h), mm); @@ -6102,7 +6102,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, goto out_ptl; /* Handle userfault-wp first, before trying to lock more pages */ - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(mm, vmf.address, vmf.pte)) && + if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) && (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) { if (!userfaultfd_wp_async(vma)) { spin_unlock(vmf.ptl); @@ -6111,7 +6111,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, return handle_userfault(&vmf, VM_UFFD_WP); } - vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte); + vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte); set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte, huge_page_size(hstate_vma(vma))); /* Fallthrough to CoW */ @@ -6396,7 +6396,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, _dst_pte = pte_mkyoung(_dst_pte); if (wp_enabled) - _dst_pte = huge_pte_mkuffd_wp(_dst_pte); + _dst_pte = huge_pte_mkuffd(_dst_pte); set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); @@ -6520,9 +6520,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, } if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); + newpte = pte_swp_mkuffd(newpte); else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(pte, newpte)) set_huge_pte_at(mm, address, ptep, newpte, psize); } else if (unlikely(pte_is_marker(pte))) { @@ -6543,9 +6543,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte = huge_pte_modify(old_pte, newprot); pte = arch_make_huge_pte(pte, shift, vma->vm_flags); if (uffd_wp) - pte = huge_pte_mkuffd_wp(pte); + pte = huge_pte_mkuffd(pte); else if (uffd_wp_resolve) - pte = huge_pte_clear_uffd_wp(pte); + pte = huge_pte_clear_uffd(pte); huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); diff --git a/mm/internal.h b/mm/internal.h index c289d167ffac..3d30650a8440 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -486,8 +486,8 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) new = pte_swp_mksoft_dirty(new); if (pte_swp_exclusive(pte)) new = pte_swp_mkexclusive(new); - if (pte_swp_uffd_wp(pte)) - new = pte_swp_mkuffd_wp(new); + if (pte_swp_uffd(pte)) + new = pte_swp_mkuffd(new); return new; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 58e14d1543ec..0b5fb0d0a0c1 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -38,7 +38,7 @@ enum scan_result { SCAN_EXCEED_SWAP_PTE, SCAN_EXCEED_SHARED_PTE, SCAN_PTE_NON_PRESENT, - SCAN_PTE_UFFD_WP, + SCAN_PTE_UFFD, SCAN_PTE_MAPPED_HUGEPAGE, SCAN_LACK_REFERENCED_PAGE, SCAN_PAGE_NULL, @@ -696,8 +696,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, result = SCAN_PTE_NON_PRESENT; goto out; } - if (pte_uffd_wp(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_uffd(pteval)) { + result = SCAN_PTE_UFFD; goto out; } page = vm_normal_page(vma, addr, pteval); @@ -1544,7 +1544,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, case SCAN_PAGE_NULL: case SCAN_DEL_PAGE_LRU: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_PAGE_LAZYFREE: last_result = ret; goto next_order; @@ -1665,15 +1665,15 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, /* * Always be strict with uffd-wp * enabled swap entries. Please see - * comment below for pte_uffd_wp(). + * comment below for pte_uffd(). */ - if (pte_swp_uffd_wp_any(pteval)) { - result = SCAN_PTE_UFFD_WP; + if (pte_swp_uffd_any(pteval)) { + result = SCAN_PTE_UFFD; goto out_unmap; } continue; } - if (pte_uffd_wp(pteval)) { + if (pte_uffd(pteval)) { /* * Don't collapse the page if any of the small * PTEs are armed with uffd write protection. @@ -1683,7 +1683,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, * userfault messages that falls outside of * the registered range. So, just be simple. */ - result = SCAN_PTE_UFFD_WP; + result = SCAN_PTE_UFFD; goto out_unmap; } @@ -1895,7 +1895,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ if (userfaultfd_wp(vma)) - return SCAN_PTE_UFFD_WP; + return SCAN_PTE_UFFD; folio = filemap_lock_folio(vma->vm_file->f_mapping, linear_page_index(vma, haddr)); @@ -3242,7 +3242,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, /* Whitelisted set of results where continuing OK */ case SCAN_NO_PTE_TABLE: case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD_WP: + case SCAN_PTE_UFFD: case SCAN_LACK_REFERENCED_PAGE: case SCAN_PAGE_NULL: case SCAN_PAGE_COUNT: diff --git a/mm/memory.c b/mm/memory.c index 631c170f812a..f4fb52e9e731 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -893,8 +893,8 @@ static void restore_exclusive_pte(struct vm_area_struct *vma, if (pte_swp_soft_dirty(orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_mkuffd(pte); if ((vma->vm_flags & VM_WRITE) && can_change_pte_writable(vma, address, pte)) { @@ -977,8 +977,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_private(entry)) { @@ -1011,8 +1011,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_device_private_entry( swp_offset(entry)); pte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(orig_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(orig_pte)) + pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_exclusive(entry)) { @@ -1035,7 +1035,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, return 0; } if (!userfaultfd_wp(dst_vma)) - pte = pte_swp_clear_uffd_wp(pte); + pte = pte_swp_clear_uffd(pte); set_pte_at(dst_mm, addr, dst_pte, pte); return 0; } @@ -1083,7 +1083,7 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte))) /* Uffd-wp needs to be delivered to dest pte as well */ - pte = pte_mkuffd_wp(pte); + pte = pte_mkuffd(pte); set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); return 0; } @@ -1106,7 +1106,7 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, pte = pte_mkold(pte); if (!userfaultfd_wp(dst_vma)) - pte = pte_clear_uffd_wp(pte); + pte = pte_clear_uffd(pte); set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); } @@ -3918,8 +3918,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (unlikely(unshare)) { if (pte_soft_dirty(vmf->orig_pte)) entry = pte_mksoft_dirty(entry); - if (pte_uffd_wp(vmf->orig_pte)) - entry = pte_mkuffd_wp(entry); + if (pte_uffd(vmf->orig_pte)) + entry = pte_mkuffd(entry); } else { entry = maybe_mkwrite(pte_mkdirty(entry), vma); } @@ -4258,7 +4258,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) * etc.) because we're only removing the uffd-wp bit, * which is completely invisible to the user. */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + pte = pte_clear_uffd(ptep_get(vmf->pte)); set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); /* @@ -5023,8 +5023,8 @@ check_folio: pte = mk_pte(page, vma->vm_page_prot); if (pte_swp_soft_dirty(vmf->orig_pte)) pte = pte_mksoft_dirty(pte); - if (pte_swp_uffd_wp(vmf->orig_pte)) - pte = pte_mkuffd_wp(pte); + if (pte_swp_uffd(vmf->orig_pte)) + pte = pte_mkuffd(pte); /* * Similar logic as in do_wp_page(); however, optimize for pages that @@ -5239,7 +5239,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); @@ -5306,7 +5306,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) return handle_userfault(vmf, VM_UFFD_MISSING); } if (vmf_orig_pte_uffd_wp(vmf)) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); set_pte_at(vma->vm_mm, addr, vmf->pte, entry); /* No need to invalidate - it was non-present before */ @@ -5556,7 +5556,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, else if (pte_write(entry) && folio_test_dirty(folio)) entry = pte_mkdirty(entry); if (unlikely(vmf_orig_pte_uffd_wp(vmf))) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { VM_BUG_ON_FOLIO(nr != 1, folio); diff --git a/mm/migrate.c b/mm/migrate.c index d3bd5cc8f552..0ac0d8f51b0c 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -327,8 +327,8 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, if (pte_swp_soft_dirty(old_pte)) newpte = pte_mksoft_dirty(newpte); - if (pte_swp_uffd_wp(old_pte)) - newpte = pte_mkuffd_wp(newpte); + if (pte_swp_uffd(old_pte)) + newpte = pte_mkuffd(newpte); set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); @@ -396,8 +396,8 @@ static bool remove_migration_pte(struct folio *folio, if (softleaf_is_migration_write(entry)) pte = pte_mkwrite(pte, vma); - else if (pte_swp_uffd_wp(old_pte)) - pte = pte_mkuffd_wp(pte); + else if (pte_swp_uffd(old_pte)) + pte = pte_mkuffd(pte); if (folio_test_anon(folio) && !softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; @@ -412,8 +412,8 @@ static bool remove_migration_pte(struct folio *folio, pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(old_pte)) pte = pte_swp_mksoft_dirty(pte); - if (pte_swp_uffd_wp(old_pte)) - pte = pte_swp_mkuffd_wp(pte); + if (pte_swp_uffd(old_pte)) + pte = pte_swp_mkuffd(pte); } #ifdef CONFIG_HUGETLB_PAGE diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 2fffeb1f9969..18d097c38853 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -456,13 +456,13 @@ again: if (pte_present(pte)) { if (pte_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pte)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pte)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pte)) + swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, addr, ptep, swp_pte); diff --git a/mm/mprotect.c b/mm/mprotect.c index cec2a1eed539..a971d86c06ca 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -240,8 +240,8 @@ static long change_softleaf_pte(struct vm_area_struct *vma, */ entry = make_readable_device_private_entry(swp_offset(entry)); newpte = swp_entry_to_pte(entry); - if (pte_swp_uffd_wp(oldpte)) - newpte = pte_swp_mkuffd_wp(newpte); + if (pte_swp_uffd(oldpte)) + newpte = pte_swp_mkuffd(newpte); } else if (softleaf_is_marker(entry)) { /* * Ignore error swap entries unconditionally, @@ -266,9 +266,9 @@ static long change_softleaf_pte(struct vm_area_struct *vma, } if (uffd_wp) - newpte = pte_swp_mkuffd_wp(newpte); + newpte = pte_swp_mkuffd(newpte); else if (uffd_wp_resolve) - newpte = pte_swp_clear_uffd_wp(newpte); + newpte = pte_swp_clear_uffd(newpte); if (!pte_same(oldpte, newpte)) { set_pte_at(vma->vm_mm, addr, pte, newpte); @@ -290,9 +290,9 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb, ptent = pte_modify(oldpte, newprot); if (uffd_wp) - ptent = pte_mkuffd_wp(ptent); + ptent = pte_mkuffd(ptent); else if (uffd_wp_resolve) - ptent = pte_clear_uffd_wp(ptent); + ptent = pte_clear_uffd(ptent); /* * In some writable, shared mappings, we might want diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832..12732a5c547e 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -297,9 +297,9 @@ static int move_ptes(struct pagetable_move_control *pmc, else { if (need_clear_uffd_wp) { if (pte_present(pte)) - pte = pte_clear_uffd_wp(pte); + pte = pte_clear_uffd(pte); else - pte = pte_swp_clear_uffd_wp(pte); + pte = pte_swp_clear_uffd(pte); } set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); } diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a11410..6ffc536359cd 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -199,8 +199,8 @@ static inline bool softleaf_cached_writable(softleaf_t entry) static void page_table_check_pte_flags(pte_t pte) { if (pte_present(pte)) { - WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte)); - } else if (pte_swp_uffd_wp(pte)) { + WARN_ON_ONCE(pte_uffd(pte) && pte_write(pte)); + } else if (pte_swp_uffd(pte)) { const softleaf_t entry = softleaf_from_pte(pte); WARN_ON_ONCE(softleaf_cached_writable(entry)); @@ -227,9 +227,9 @@ EXPORT_SYMBOL(__page_table_check_ptes_set); static inline void page_table_check_pmd_flags(pmd_t pmd) { if (pmd_present(pmd)) { - if (pmd_uffd_wp(pmd)) + if (pmd_uffd(pmd)) WARN_ON_ONCE(pmd_write(pmd)); - } else if (pmd_swp_uffd_wp(pmd)) { + } else if (pmd_swp_uffd(pmd)) { const softleaf_t entry = softleaf_from_pmd(pmd); WARN_ON_ONCE(softleaf_cached_writable(entry)); diff --git a/mm/rmap.c b/mm/rmap.c index 9939400e77c7..50f973b63f1c 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2317,13 +2317,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, if (likely(pte_present(pteval))) { if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { if (pte_swp_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } set_pte_at(mm, address, pvmw.pte, swp_pte); } else { @@ -2694,14 +2694,14 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } else { swp_pte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd_wp(pteval)) - swp_pte = pte_swp_mkuffd_wp(swp_pte); + if (pte_swp_uffd(pteval)) + swp_pte = pte_swp_mkuffd(swp_pte); } if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, diff --git a/mm/swapfile.c b/mm/swapfile.c index a602e5820513..baa3462ce517 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2496,8 +2496,8 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot)); if (pte_swp_soft_dirty(old_pte)) new_pte = pte_mksoft_dirty(new_pte); - if (pte_swp_uffd_wp(old_pte)) - new_pte = pte_mkuffd_wp(new_pte); + if (pte_swp_uffd(old_pte)) + new_pte = pte_mkuffd(new_pte); setpte: set_pte_at(vma->vm_mm, addr, pte, new_pte); folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry))); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 33ef42d5a04e..7b2adfc50e47 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -376,7 +376,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (writable) _dst_pte = pte_mkwrite(_dst_pte, dst_vma); if (flags & MFILL_ATOMIC_WP) - _dst_pte = pte_mkuffd_wp(_dst_pte); + _dst_pte = pte_mkuffd(_dst_pte); ret = -EAGAIN; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); @@ -3609,7 +3609,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) vm_flags |= VM_UFFD_MISSING; if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { - if (!pgtable_supports_uffd_wp()) + if (!pgtable_supports_uffd()) goto out; vm_flags |= VM_UFFD_WP; @@ -4319,7 +4319,7 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); #endif - if (!pgtable_supports_uffd_wp()) + if (!pgtable_supports_uffd()) uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; if (!uffd_supports_wp_marker()) { -- cgit v1.2.3 From c6664c4fab60030ae089622d5e69f2af8dca5a7c Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:00 +0200 Subject: x86/mm: drop order parameter from free_pagetable() Patch series "mm: remove CONFIG_HAVE_BOOTMEM_INFO_NODE (Part 2)", v2. Let's remove the remaining pieces of CONFIG_HAVE_BOOTMEM_INFO_NODE, performing some smaller cleanups around freeing of reserved vmemmap pages on the way. This patch (of 10): All callers pass 0, so let's drop the parameter. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-0-4afc76c73d61@kernel.org Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-1-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index d57f29ca23a5..69e36f02a663 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1006,22 +1006,19 @@ static void free_reserved_pages(struct page *page, unsigned long nr_pages) free_reserved_page(page++); } -static void __meminit free_pagetable(struct page *page, int order) +static void __meminit free_pagetable(struct page *page) { /* bootmem page has reserved flag */ if (PageReserved(page)) { - unsigned long nr_pages = 1 << order; #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE enum bootmem_type type = bootmem_type(page); - if (type == MIX_SECTION_INFO) { - while (nr_pages--) - put_page_bootmem(page++); - } else { - free_reserved_pages(page, nr_pages); - } + if (type == MIX_SECTION_INFO) + put_page_bootmem(page); + else + free_reserved_page(page); #else - free_reserved_pages(page, nr_pages); + free_reserved_page(page); #endif } else { pagetable_free(page_ptdesc(page)); @@ -1060,7 +1057,7 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) } /* free a pte table */ - free_pagetable(pmd_page(*pmd), 0); + free_pagetable(pmd_page(*pmd)); spin_lock(&init_mm.page_table_lock); pmd_clear(pmd); spin_unlock(&init_mm.page_table_lock); @@ -1078,7 +1075,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) } /* free a pmd table */ - free_pagetable(pud_page(*pud), 0); + free_pagetable(pud_page(*pud)); spin_lock(&init_mm.page_table_lock); pud_clear(pud); spin_unlock(&init_mm.page_table_lock); @@ -1096,7 +1093,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) } /* free a pud table */ - free_pagetable(p4d_page(*p4d), 0); + free_pagetable(p4d_page(*p4d)); spin_lock(&init_mm.page_table_lock); p4d_clear(p4d); spin_unlock(&init_mm.page_table_lock); -- cgit v1.2.3 From 0a762b96790ec816cdb2007c653cd72eff75d0c7 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:01 +0200 Subject: mm: provide free_reserved_pages(), removing x86 variant Let's extend free_reserved_page() in page_alloc.c to free_reserved_pages(), dropping the custom x86 variant. The common-code variant will consume an order, so adjust the x86 callers accordingly. Make free_reserved_pages() assume that we are freeing ordinary high-order pages, just with the special "reserved" flavor. The target use case for now is freeing vmemmap PMD pages. Set the refcount directly to 0 (instead of 1) and call __free_frozen_pages(). Set the page count to 0 before clearing PG_reserved, so someone checking PG_reserved (and not finding it set) to then try grabbing a ref would not suddenly have that ref be dropped. That is arguably cleaner and safer than the old way of doing it. Add some kerneldoc. Use a single adjust_managed_page_count() call. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-2-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Muchun Song Reviewed-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 8 +------- include/linux/mm.h | 8 ++++++-- mm/page_alloc.c | 33 ++++++++++++++++++++++++++------- 3 files changed, 33 insertions(+), 16 deletions(-) (limited to 'arch') diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 69e36f02a663..320e742b0bb3 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1000,12 +1000,6 @@ int arch_add_memory(int nid, u64 start, u64 size, return add_pages(nid, start_pfn, nr_pages, params); } -static void free_reserved_pages(struct page *page, unsigned long nr_pages) -{ - while (nr_pages--) - free_reserved_page(page++); -} - static void __meminit free_pagetable(struct page *page) { /* bootmem page has reserved flag */ @@ -1038,7 +1032,7 @@ static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, while (nr_pages--) put_page_bootmem(page++); } else { - free_reserved_pages(page, nr_pages); + free_reserved_pages(page, order); } } else { __free_pages(page, order); diff --git a/include/linux/mm.h b/include/linux/mm.h index 25e669632d2c..e1e1154dafd7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3996,8 +3996,12 @@ extern unsigned long free_reserved_area(void *start, void *end, extern void adjust_managed_page_count(struct page *page, long count); -/* Free the reserved page into the buddy system, so it gets managed. */ -void free_reserved_page(struct page *page); +void free_reserved_pages(struct page *page, unsigned int order); + +static inline void free_reserved_page(struct page *page) +{ + free_reserved_pages(page, 0); +} static inline void mark_page_reserved(struct page *page) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cc45dfe89c76..f61e4203eced 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6389,15 +6389,34 @@ void adjust_managed_page_count(struct page *page, long count) } EXPORT_SYMBOL(adjust_managed_page_count); -void free_reserved_page(struct page *page) +/** + * free_reserved_pages - free reserved pages + * @page: First page to free. + * @order: The page order to free. + * + * Free pages allocated through memblock during boot, letting the buddy + * manage them from now on. + * + * @page must be naturally aligned to the order and the order must not + * exceed MAX_PAGE_ORDER. All pages must be reserved. + */ +void free_reserved_pages(struct page *page, unsigned int order) { - clear_page_tag_ref(page); - ClearPageReserved(page); - init_page_count(page); - __free_page(page); - adjust_managed_page_count(page, 1); + const unsigned long nr_pages = 1UL << order; + int i; + + VM_WARN_ON_ONCE(!IS_ALIGNED(page_to_pfn(page), nr_pages)); + VM_WARN_ON_ONCE(order > MAX_PAGE_ORDER); + + for (i = 0; i < nr_pages; i++) { + clear_page_tag_ref(page + i); + set_page_count(page + i, 0); + ClearPageReserved(page + i); + } + adjust_managed_page_count(page, nr_pages); + __free_frozen_pages(page, order, FPI_NONE); } -EXPORT_SYMBOL(free_reserved_page); +EXPORT_SYMBOL(free_reserved_pages); static int page_alloc_cpu_dead(unsigned int cpu) { -- cgit v1.2.3 From 536a195aa42b693b23db9695eac1852e005fa25b Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:02 +0200 Subject: s390/mm: use free_reserved_pages() in vmem_free_pages() Let's use our new generic helper. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-3-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Acked-by: Heiko Carstens Reviewed-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Zi Yan Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/s390/mm/vmem.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 94ee06ae8913..d2879ce860a1 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -39,7 +39,6 @@ static void __ref *vmem_alloc_pages(unsigned int order) static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap) { - unsigned int nr_pages = 1 << order; struct page *page; if (altmap) { @@ -49,8 +48,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a page = virt_to_page((void *)addr); if (PageReserved(page)) { /* allocated from memblock */ - while (nr_pages--) - free_reserved_page(page++); + free_reserved_pages(page, order); } else { free_pages(addr, order); } -- cgit v1.2.3 From 4a0b91cc8a0481cfbc5519b892d3a754746c7c5b Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:04 +0200 Subject: x86/mm: stop marking vmemmap as SECTION_INFO We added the whole bootmem registration machinery in commit 04753278769f ("memory hotplug: register section/node id to free"). The main use case was to remember to which memory section memmap pages belonged, so the memmap could be handled accordingly when freeing memory. However, all that machinery is not required anymore: a memory section can only get offlined if *all* pages can get offlined; and it can only get unplugged once offline. If some of these pages are unmovable memmap pages: bad luck, doesn't work. Offlining will fail. Further, a lot of this machinery was required for pre-vmemmap support. Now we only support the vmemmap with memory hotplug. So the whole machinery is useless today. Let's start by removing the last pieces by first stopping to mark vmemmap pages as SECTION_INFO. In free_vmemmap_pages(), we can now always just free the reserved pages directly. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-5-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) (limited to 'arch') diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 320e742b0bb3..82bb0384b8ab 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1024,19 +1024,12 @@ static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, { unsigned long nr_pages = 1u << order; - if (altmap) { + if (altmap) vmem_altmap_free(altmap, nr_pages); - } else if (PageReserved(page)) { - if (IS_ENABLED(CONFIG_HAVE_BOOTMEM_INFO_NODE) && - bootmem_type(page) == SECTION_INFO) { - while (nr_pages--) - put_page_bootmem(page++); - } else { - free_reserved_pages(page, order); - } - } else { + else if (PageReserved(page)) + free_reserved_pages(page, order); + else __free_pages(page, order); - } } static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) @@ -1593,12 +1586,8 @@ void register_page_bootmem_memmap(unsigned long section_nr, p4d_t *p4d; pud_t *pud; pmd_t *pmd; - unsigned int nr_pmd_pages; - struct page *page; for (; addr < end; addr = next) { - pte_t *pte = NULL; - pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { next = (addr + PAGE_SIZE) & PAGE_MASK; @@ -1630,19 +1619,8 @@ void register_page_bootmem_memmap(unsigned long section_nr, next = (addr + PAGE_SIZE) & PAGE_MASK; get_page_bootmem(section_nr, pmd_page(*pmd), MIX_SECTION_INFO); - - pte = pte_offset_kernel(pmd, addr); - if (pte_none(*pte)) - continue; - get_page_bootmem(section_nr, pte_page(*pte), - SECTION_INFO); } else { next = pmd_addr_end(addr, end); - nr_pmd_pages = (next - addr) >> PAGE_SHIFT; - page = pmd_page(*pmd); - while (nr_pmd_pages--) - get_page_bootmem(section_nr, page++, - SECTION_INFO); } } } -- cgit v1.2.3 From e425a1004b9a1b50227073798178eef3c342162f Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:05 +0200 Subject: x86/mm: stop marking page tables as MIX_SECTION_INFO There is no good reason to mark boot page tables as MIX_SECTION_INFO: we only free boot page tables when they are completely empty, and memory offlining/hotunplug doesn't benefit from it in any way. So just stop marking page tables as MIX_SECTION_INFO. In free_pagetable(), we can now simply free reserved pages directly. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-6-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 59 ++------------------------------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) (limited to 'arch') diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 82bb0384b8ab..1b17dcf781e6 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1002,21 +1002,10 @@ int arch_add_memory(int nid, u64 start, u64 size, static void __meminit free_pagetable(struct page *page) { - /* bootmem page has reserved flag */ - if (PageReserved(page)) { -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE - enum bootmem_type type = bootmem_type(page); - - if (type == MIX_SECTION_INFO) - put_page_bootmem(page); - else - free_reserved_page(page); -#else + if (PageReserved(page)) free_reserved_page(page); -#endif - } else { + else pagetable_free(page_ptdesc(page)); - } } static void __meminit free_vmemmap_pages(struct page *page, unsigned int order, @@ -1579,50 +1568,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, void register_page_bootmem_memmap(unsigned long section_nr, struct page *start_page, unsigned long nr_pages) { - unsigned long addr = (unsigned long)start_page; - unsigned long end = (unsigned long)(start_page + nr_pages); - unsigned long next; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - for (; addr < end; addr = next) { - pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO); - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO); - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO); - - pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - continue; - } - - if (!boot_cpu_has(X86_FEATURE_PSE) || !pmd_leaf(*pmd)) { - next = (addr + PAGE_SIZE) & PAGE_MASK; - get_page_bootmem(section_nr, pmd_page(*pmd), - MIX_SECTION_INFO); - } else { - next = pmd_addr_end(addr, end); - } - } } #endif -- cgit v1.2.3 From 246924680514f38879af5b3843bd6d86a286daec Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Thu, 16 Jul 2026 16:51:06 +0200 Subject: x86/mm: remove CONFIG_HAVE_BOOTMEM_INFO_NODE CONFIG_HAVE_BOOTMEM_INFO_NODE now essentially doesn't do anything. So let's remove support for CONFIG_HAVE_BOOTMEM_INFO_NODE. Link: https://lore.kernel.org/20260716-bootmem_info_part2-v2-7-4afc76c73d61@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Dave Hansen Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Christian Borntraeger Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Johannes Weiner Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Muchun Song Cc: Oscar Salvador Cc: Peter Zijlstra Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/x86/mm/init_64.c | 26 -------------------------- mm/Kconfig | 1 - 2 files changed, 27 deletions(-) (limited to 'arch') diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 1b17dcf781e6..ab4c5a02326f 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -1286,16 +1285,6 @@ void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap, static struct kcore_list kcore_vsyscall; -static void __init register_page_bootmem_info(void) -{ -#if defined(CONFIG_NUMA) || defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) - int i; - - for_each_online_node(i) - register_page_bootmem_info_node(NODE_DATA(i)); -#endif -} - /* * Pre-allocates page-table pages for the vmalloc area in the kernel page-table. * Only the level which needs to be synchronized between all page-tables is @@ -1358,14 +1347,6 @@ void __init mem_init(void) after_bootmem = 1; x86_init.hyper.init_after_bootmem(); - /* - * Must be done after boot memory is put on freelist, because here we - * might set fields in deferred struct pages that have not yet been - * initialized, and memblock_free_all() initializes all the reserved - * deferred pages for us. - */ - register_page_bootmem_info(); - /* Register memory areas for /proc/kcore */ if (get_gate_vma(&init_mm)) kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); @@ -1564,13 +1545,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return err; } -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void register_page_bootmem_memmap(unsigned long section_nr, - struct page *start_page, unsigned long nr_pages) -{ -} -#endif - void __meminit vmemmap_populate_print_last(void) { if (p_start) { diff --git a/mm/Kconfig b/mm/Kconfig index cf13771585aa..27e05f08f0b4 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -590,7 +590,6 @@ endchoice config MEMORY_HOTREMOVE bool "Allow for memory hot remove" - select HAVE_BOOTMEM_INFO_NODE if X86_64 depends on MEMORY_HOTPLUG select MIGRATION -- cgit v1.2.3 From f10d3e08fb640caed6e21410a39d2faae2bed0f0 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 15 Jul 2026 20:08:07 +0800 Subject: arm64/hugetlb: extend batching of multiple CONT_PTE in a single PTE setup Patch series "mm/vmalloc: Speed up ioremap, vmalloc and vmap with contiguous memory", v7. This patchset accelerates ioremap, vmalloc, and vmap when the memory is physically fully or partially contiguous. Two techniques are used: 1. Avoid page table rewalk when setting PTEs/PMDs for multiple memory segments 2. Use batched mappings wherever possible in both vmalloc and ARM64 layers Besides accelerating the mapping path, this also enables large mappings (PMD and cont-PTE) for vmap, which are currently not supported. Patches 1-2 extend ARM64 vmalloc CONT-PTE mapping to support multiple CONT-PTE regions instead of just one. Patch 3 extracts a common helper vmap_set_ptes() that consolidates PTE mapping logic for the ioremap and vmalloc/vmap paths, handling both CONT_PTE and regular PTE mappings. This prepares for the next patch. Patch 4 extends the page table walk path to support page shifts other than PAGE_SHIFT and eliminates the page table rewalk for huge vmalloc mappings. The function is renamed from vmap_small_pages_range_noflush() to vmap_pages_range_noflush_walk(). Patch 5 extracts vm_shift() to consolidate vmalloc mapping shift selection for reuse in the batching path. Patches 6-7 add huge vmap support for contiguous pages, including support for non-compound pages with pfn alignment verification. On the RK3588 8-core ARM64 SoC, with tasks pinned to a little core and the performance CPUfreq policy enabled, benchmark results: * ioremap(1 MB): 1.35x faster (3407 ns -> 2526 ns) * vmalloc(1 MB) mapping time (excluding allocation) with VM_ALLOW_HUGE_VMAP: 1.42x faster (5.00 us -> 3.53 us) * vmap(100MB) with order-8 pages: 8.3x faster (1235 us -> 149 us) Many thanks to Xueyuan Chen for his testing efforts on RK3588 boards. Large vmap() mappings were also tested by Leo Yan with ARM trace buffer units, including TRBE and SPE. These units use the CPU page tables for address translation when writing trace data to DRAM, so using larger vmap() mapping granules can reduce TLB pressure on the trace writer. The TRBE test used a 1G CoreSight ETM AUX buffer. Across five runs on an isolated CPU, the average results were: * dtlb_walk: 68.4 -> 59.4 (-13.16%) * l1d_tlb_refill: 155.8 -> 119.6 (-23.23%) * l2d_tlb_refill: 161435.8 -> 495.0 (-99.69%) The SPE test used a 512M ARM SPE AUX buffer. Across five runs on an isolated CPU, the average results were: * dtlb_walk: 1710.4 -> 1315.6 (-23.08%) * l1d_tlb_refill: 16000.0 -> 15950.2 (-0.31%) * l2d_tlb_refill: 4796.0 -> 2931.2 (-38.88%) These results show that enabling larger vmap() mappings can materially reduce page table walks and TLB refills for large trace buffers. Many thanks to Leo Yan for his testing efforts on ARM trace buffers. This patch (of 7): For sizes aligned to CONT_PTE_SIZE and smaller than PMD_SIZE, we can handle CONT_PTE_SIZE groups together. These additional sizes are mapping spans used by non-hugetlbfs(vmalloc) mm code, not new HugeTLB hstate sizes. Link: https://lore.kernel.org/20260715120813.3609949-1-jiangwen6@xiaomi.com Link: https://lore.kernel.org/20260715120813.3609949-2-jiangwen6@xiaomi.com Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Wen Jiang Tested-by: Xueyuan Chen Tested-by: Leo Yan Reviewed-by: Dev Jain Cc: Andrew Donnellan Cc: Anshuman Khandual Cc: Catalin Marinas Cc: David Hildenbrand Cc: Mike Rapoport Cc: Ryan Roberts Cc: "Uladzislau Rezki (Sony)" Cc: Will Deacon Cc: Wen Jiang Signed-off-by: Andrew Morton --- arch/arm64/mm/hugetlbpage.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'arch') diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 8e799c1fe0aa..8429d220660b 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -94,6 +94,11 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, return CONT_PTES; } +/* + * num_contig_ptes(), set_huge_pte_at() and arch_make_huge_pte() can be + * used by non-hugetlbfs(vmalloc) mm code to set multiple huge mappings + * at the PTE level. + */ static inline int num_contig_ptes(unsigned long size, size_t *pgsize) { int contig_ptes = 1; @@ -110,6 +115,12 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize) contig_ptes = CONT_PTES; break; default: + if (size > 0 && size < PMD_SIZE && + IS_ALIGNED(size, CONT_PTE_SIZE)) { + *pgsize = PAGE_SIZE; + contig_ptes = size >> PAGE_SHIFT; + break; + } WARN_ON(!__hugetlb_valid_size(size)); } @@ -359,6 +370,10 @@ pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) case CONT_PTE_SIZE: return pte_mkcont(entry); default: + if (pagesize > 0 && pagesize < PMD_SIZE && + IS_ALIGNED(pagesize, CONT_PTE_SIZE)) + return pte_mkcont(entry); + break; } pr_warn("%s: unrecognized huge page size 0x%lx\n", -- cgit v1.2.3 From 683ab482493acbb3727584572c10435a89c75c78 Mon Sep 17 00:00:00 2001 From: "Barry Song (Xiaomi)" Date: Wed, 15 Jul 2026 20:08:08 +0800 Subject: arm64/vmalloc: allow arch_vmap_pte_range_map_size to batch multiple CONT_PTE Allow arch_vmap_pte_range_map_size to batch across multiple CONT_PTE blocks, reducing both PTE setup and TLB flush iterations. For CONT_PTE_SIZE-aligned ranges, return a power-of-two mapping size that may cover multiple CONT_PTE blocks, capped below PMD_SIZE. These sizes are vmalloc mapping spans, not HugeTLB hstate sizes. Link: https://lore.kernel.org/20260715120813.3609949-3-jiangwen6@xiaomi.com Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Wen Jiang Tested-by: Xueyuan Chen Tested-by: Leo Yan Reviewed-by: Dev Jain Cc: Andrew Donnellan Cc: Anshuman Khandual Cc: Catalin Marinas Cc: David Hildenbrand Cc: Mike Rapoport Cc: Ryan Roberts Cc: "Uladzislau Rezki (Sony)" Cc: Wen Jiang Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/include/asm/vmalloc.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm64/include/asm/vmalloc.h b/arch/arm64/include/asm/vmalloc.h index 4ec1acd3c1b3..d665f9d68742 100644 --- a/arch/arm64/include/asm/vmalloc.h +++ b/arch/arm64/include/asm/vmalloc.h @@ -23,10 +23,14 @@ static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr, unsigned long end, u64 pfn, unsigned int max_page_shift) { + unsigned long size; + /* * If the block is at least CONT_PTE_SIZE in size, and is naturally * aligned in both virtual and physical space, then we can pte-map the * block using the PTE_CONT bit for more efficient use of the TLB. + * The returned mapping size may cover multiple CONT_PTE_SIZE blocks, + * capped below PMD_SIZE. */ if (max_page_shift < CONT_PTE_SHIFT) return PAGE_SIZE; @@ -40,7 +44,9 @@ static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr, if (!IS_ALIGNED(PFN_PHYS(pfn), CONT_PTE_SIZE)) return PAGE_SIZE; - return CONT_PTE_SIZE; + size = min3(end - addr, 1UL << max_page_shift, PMD_SIZE >> 1); + size = rounddown_pow_of_two(size); + return size; } #define arch_vmap_pte_range_unmap_size arch_vmap_pte_range_unmap_size -- cgit v1.2.3 From 82c74ad2d9d779efe47cb25006fc2ce0a66ef09c Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:47 +0100 Subject: mm/rmap: parameterise vma_interval_tree_*() by address_space The file-backed mapping interval tree functions vma_interval_tree_*() accept a raw rb_root_cached pointer to determine the tree in which they are operating. However, in each case, this is always associated with an address_space data type. So simply pass a pointer to that instead to simplify the code, and more clearly differentiate between these operations and those concerning anonymous mappings. While we're here, make the generated interval tree functions static as they do not need to be used externally (any previously existing external users have now been removed). We also rename VMA parameters from 'node' to 'vma' as calling this a node is simply confusing, update the input index types to pgoff_t since they reference page offsets and rename the parameters to pgoff_start and pgoff_last. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-6-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Pedro Falcato Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- arch/arm/mm/fault-armv.c | 2 +- arch/arm/mm/flush.c | 2 +- arch/nios2/mm/cacheflush.c | 2 +- arch/parisc/kernel/cache.c | 2 +- fs/dax.c | 2 +- fs/hugetlbfs/inode.c | 15 +++++----- include/linux/mm.h | 34 ++++++++++++----------- kernel/events/uprobes.c | 2 +- mm/hugetlb.c | 4 +-- mm/interval_tree.c | 58 +++++++++++++++++++++++++++++---------- mm/khugepaged.c | 4 +-- mm/memory-failure.c | 7 ++--- mm/memory.c | 8 +++--- mm/mmap.c | 3 +- mm/nommu.c | 8 +++--- mm/pagewalk.c | 2 +- mm/rmap.c | 3 +- mm/vma.c | 14 ++++------ tools/testing/vma/include/stubs.h | 4 +-- 19 files changed, 100 insertions(+), 76 deletions(-) (limited to 'arch') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 91e488767783..cd52cf7f8874 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * cache coherency. */ flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(mpnt, mapping, pgoff, pgoff) { /* * If we are using split PTE locks, then we need to take the pte * lock. Otherwise we are using shared mm->page_table_lock which diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 4d7ef5cc36b6..8c593e9898ee 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio * pgoff_end = pgoff + folio_nr_pages(folio) - 1; flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff_end) { unsigned long start, offset, pfn; unsigned int nr; diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c index 8321182eb927..42e3bf892316 100644 --- a/arch/nios2/mm/cacheflush.c +++ b/arch/nios2/mm/cacheflush.c @@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio) pgoff = folio->index; flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long start; if (vma->vm_mm != mm) diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 0170b69a21d3..f28aa7884cbf 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio) * on machines that support equivalent aliasing */ flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long offset = pgoff - vma->vm_pgoff; unsigned long pfn = folio_pfn(folio); diff --git a/fs/dax.c b/fs/dax.c index 6d175cd47a99..2f0818a68a7f 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, /* Walk all mappings of a given index of a file and writeprotect them */ i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, index, end) { + vma_interval_tree_foreach(vma, mapping, index, end) { pfn_mkclean_range(pfn, count, index, vma); cond_resched(); } diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b2..4ea1798f1ffb 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -380,7 +380,6 @@ static void hugetlb_unmap_file_folio(struct hstate *h, struct address_space *mapping, struct folio *folio, pgoff_t index) { - struct rb_root_cached *root = &mapping->i_mmap; struct hugetlb_vma_lock *vma_lock; unsigned long pfn = folio_pfn(folio); struct vm_area_struct *vma; @@ -394,7 +393,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h, i_mmap_lock_write(mapping); retry: vma_lock = NULL; - vma_interval_tree_foreach(vma, root, start, end - 1) { + vma_interval_tree_foreach(vma, mapping, start, end - 1) { v_start = vma_offset_start(vma, start); v_end = vma_offset_end(vma, end); @@ -460,8 +459,8 @@ retry: } static void -hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, - zap_flags_t zap_flags) +hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start, + pgoff_t end, zap_flags_t zap_flags) { struct vm_area_struct *vma; @@ -470,7 +469,8 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, * unmapped. Note, end is exclusive, whereas the interval tree takes * an inclusive "last". */ - vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) { + vma_interval_tree_foreach(vma, mapping, start, + end ? end - 1 : ULONG_MAX) { unsigned long v_start; unsigned long v_end; @@ -615,8 +615,7 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) i_size_write(inode, offset); i_mmap_lock_write(mapping); if (mapping_mapped(mapping)) - hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0, - ZAP_FLAG_DROP_MARKER); + hugetlb_vmdelete_list(mapping, pgoff, 0, ZAP_FLAG_DROP_MARKER); i_mmap_unlock_write(mapping); remove_inode_hugepages(inode, offset, LLONG_MAX); } @@ -676,7 +675,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) /* Unmap users of full pages in the hole. */ if (hole_end > hole_start) { if (mapping_mapped(mapping)) - hugetlb_vmdelete_list(&mapping->i_mmap, + hugetlb_vmdelete_list(mapping, hole_start >> PAGE_SHIFT, hole_end >> PAGE_SHIFT, 0); } diff --git a/include/linux/mm.h b/include/linux/mm.h index 3e45978812e2..349d59a737eb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4094,23 +4094,25 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *node, - struct rb_root_cached *root); -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void vma_interval_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping); +void vma_interval_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root); -void vma_interval_tree_remove(struct vm_area_struct *node, - struct rb_root_cached *root); -struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, - unsigned long start, unsigned long last); - -#define vma_interval_tree_foreach(vma, root, start, last) \ - for (vma = vma_interval_tree_iter_first(root, start, last); \ - vma; vma = vma_interval_tree_iter_next(vma, start, last)) + struct address_space *mapping); +void vma_interval_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping); +struct vm_area_struct * +vma_interval_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last); +struct vm_area_struct * +vma_interval_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); + +#define vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = vma_interval_tree_iter_first(mapping, pgoff_start, \ + pgoff_last); \ + vma; vma = vma_interval_tree_iter_next(vma, pgoff_start, \ + pgoff_last)) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, struct rb_root_cached *root); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e284..50a96a4d812d 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1210,7 +1210,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { if (!valid_vma(vma, is_register)) continue; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a7aba6fdcaf5..01bb45f9c7eb 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5455,7 +5455,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * __unmap_hugepage_range() is called as the lock is already held */ i_mmap_lock_write(mapping); - vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(iter_vma, mapping, pgoff, pgoff) { /* Do not unmap the current VMA */ if (iter_vma == vma) continue; @@ -6998,7 +6998,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *pte; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { + vma_interval_tree_foreach(svma, mapping, idx, idx) { if (svma == vma) continue; diff --git a/mm/interval_tree.c b/mm/interval_tree.c index 2d50bc6228c4..ff36fd14ef37 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -14,19 +14,26 @@ /* File-backed interval tree (address_space->i_mmap) */ INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, - unsigned long, shared.rb_subtree_last, - vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree) + pgoff_t, shared.rb_subtree_last, + vma_start_pgoff, vma_last_pgoff, static, + __vma_interval_tree) -/* Insert node immediately after prev in the interval tree */ -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void vma_interval_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __vma_interval_tree_insert(vma, &mapping->i_mmap); +} + +/* Insert vma immediately after prev in the interval tree */ +void vma_interval_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root) + struct address_space *mapping) { struct rb_node **link; struct vm_area_struct *parent; - unsigned long last = vma_last_pgoff(node); + const pgoff_t pgoff_last = vma_last_pgoff(vma); - VM_WARN_ON_ONCE_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node); + VM_WARN_ON_ONCE_VMA(vma_start_pgoff(vma) != vma_start_pgoff(prev), vma); if (!prev->shared.rb.rb_right) { parent = prev; @@ -34,21 +41,42 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node, } else { parent = rb_entry(prev->shared.rb.rb_right, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; while (parent->shared.rb.rb_left) { parent = rb_entry(parent->shared.rb.rb_left, struct vm_area_struct, shared.rb); - if (parent->shared.rb_subtree_last < last) - parent->shared.rb_subtree_last = last; + if (parent->shared.rb_subtree_last < pgoff_last) + parent->shared.rb_subtree_last = pgoff_last; } link = &parent->shared.rb.rb_left; } - node->shared.rb_subtree_last = last; - rb_link_node(&node->shared.rb, &parent->shared.rb, link); - rb_insert_augmented(&node->shared.rb, &root->rb_root, - &vma_interval_tree_augment); + vma->shared.rb_subtree_last = pgoff_last; + rb_link_node(&vma->shared.rb, &parent->shared.rb, link); + rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root, + &__vma_interval_tree_augment); +} + +void vma_interval_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping) +{ + __vma_interval_tree_remove(vma, &mapping->i_mmap); +} + +struct vm_area_struct * +vma_interval_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __vma_interval_tree_iter_first(&mapping->i_mmap, + pgoff_start, pgoff_last); +} + +struct vm_area_struct * +vma_interval_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last) +{ + return __vma_interval_tree_iter_next(vma, pgoff_start, pgoff_last); } /* Anonymous interval tree (anon_vma->rb_root) */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 55157567dc4c..e5462e026642 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2141,7 +2141,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct vm_area_struct *vma; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; @@ -2573,7 +2573,7 @@ xa_unlocked: * not be able to observe any missing pages due to the * previously inserted retry entries. */ - vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { + vma_interval_tree_foreach(vma, mapping, start, end) { if (userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index bf717ec59508..4358856e764f 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -598,8 +598,7 @@ static void collect_procs_file(const struct folio *folio, if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, - pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { /* * Send early kill signal to tasks where a vma covers * the page but the corrupted page is not necessarily @@ -650,7 +649,7 @@ static void collect_procs_fsdax(const struct page *page, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { if (vma->vm_mm == t->mm) add_to_kill_fsdax(t, page, vma, to_kill, pgoff); } @@ -2297,7 +2296,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) { + vma_interval_tree_foreach(vma, mapping, 0, ULONG_MAX) { pgoff_t pgoff; if (vma->vm_mm == t->mm && diff --git a/mm/memory.c b/mm/memory.c index edbbcae337cd..1aac6e4ea221 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4416,7 +4416,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_copy(vmf); } -static inline void unmap_mapping_range_tree(struct rb_root_cached *root, +static inline void unmap_mapping_range_tree(struct address_space *mapping, pgoff_t first_index, pgoff_t last_index, struct zap_details *details) @@ -4425,7 +4425,7 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, unsigned long start, size; struct mmu_gather tlb; - vma_interval_tree_foreach(vma, root, first_index, last_index) { + vma_interval_tree_foreach(vma, mapping, first_index, last_index) { const pgoff_t start_idx = max(first_index, vma->vm_pgoff); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; @@ -4467,7 +4467,7 @@ void unmap_mapping_folio(struct folio *folio) i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } @@ -4497,7 +4497,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) - unmap_mapping_range_tree(&mapping->i_mmap, first_index, + unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping); } diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff4..2f22fb0d068d 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1830,8 +1830,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, - &mapping->i_mmap); + vma_interval_tree_insert_after(tmp, mpnt, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } diff --git a/mm/nommu.c b/mm/nommu.c index 852ec9bd0505..13023d73d1e4 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + vma_interval_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + vma_interval_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -1843,7 +1843,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, i_mmap_lock_read(inode->i_mapping); /* search for VMAs that fall within the dead zone */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) { + vma_interval_tree_foreach(vma, inode->i_mapping, low, high) { /* found one - only interested if it's shared out of the page * cache */ if (vma->vm_flags & VM_SHARED) { @@ -1859,7 +1859,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, * we don't check for any regions that start beyond the EOF as there * shouldn't be any */ - vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) { + vma_interval_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { if (!(vma->vm_flags & VM_SHARED)) continue; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a255..bdede741c9e6 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -816,7 +816,7 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); - vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, + vma_interval_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ vba = vma->vm_pgoff; diff --git a/mm/rmap.c b/mm/rmap.c index 26166a6b8cb9..73673cd3961e 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -3053,8 +3053,7 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, i_mmap_lock_read(mapping); } lookup: - vma_interval_tree_foreach(vma, &mapping->i_mmap, - pgoff_start, pgoff_end) { + vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { unsigned long address = vma_address(vma, pgoff_start, nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); diff --git a/mm/vma.c b/mm/vma.c index 9eea2850818a..ce4ec4b71138 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -234,7 +234,7 @@ static void __vma_link_file(struct vm_area_struct *vma, mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); + vma_interval_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -248,7 +248,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, &mapping->i_mmap); + vma_interval_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -319,10 +319,9 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->file) { flush_dcache_mmap_lock(vp->mapping); - vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap); + vma_interval_tree_remove(vp->vma, vp->mapping); if (vp->adj_next) - vma_interval_tree_remove(vp->adj_next, - &vp->mapping->i_mmap); + vma_interval_tree_remove(vp->adj_next, vp->mapping); } } @@ -340,9 +339,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, { if (vp->file) { if (vp->adj_next) - vma_interval_tree_insert(vp->adj_next, - &vp->mapping->i_mmap); - vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap); + vma_interval_tree_insert(vp->adj_next, vp->mapping); + vma_interval_tree_insert(vp->vma, vp->mapping); flush_dcache_mmap_unlock(vp->mapping); } diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 64164e25658f..94442b29458d 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -258,12 +258,12 @@ static inline void vm_acct_memory(long pages) } static inline void vma_interval_tree_insert(struct vm_area_struct *vma, - struct rb_root_cached *rb) + struct address_space *mapping) { } static inline void vma_interval_tree_remove(struct vm_area_struct *vma, - struct rb_root_cached *rb) + struct address_space *mapping) { } -- cgit v1.2.3 From eed6128ee35fdb8f5ecb6c38fbba7a9e27de01e6 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:16:49 +0100 Subject: mm/rmap: rename vma_interval_tree_*() to mapping_rmap_tree_*() The family of vma_interval_tree_() functions manipulate the address_space (which, of course, is generally referred to as 'mapping') reverse mapping, but are named the 'VMA' interval tree. VMAs may be mapped by an anon_vma, an address_space, or both. Therefore calling the mapping interval tree a 'VMA' interval tree is rather confusing. This is also inconsistent with the anon_vma_interval_tree_*() functions which explicitly reference the rmap object to which they pertain. Rename the vma_interval_tree_*() functions to mapping_rmap_tree_*() to correct this. We will rename the anon rmap functions similarly in a subsequent patch. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-8-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Gregory Price Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Zi Yan Cc: Ackerley Tng Cc: David Hildenbrand (Arm) Cc: Kai Huang Cc: Marek Szyprowski Cc: Pedro Falcato Cc: SJ Park Cc: Thomas Zimmermann Cc: Liam R. Howlett (Oracle) Signed-off-by: Andrew Morton --- arch/arm/mm/fault-armv.c | 2 +- arch/arm/mm/flush.c | 2 +- arch/nios2/mm/cacheflush.c | 2 +- arch/parisc/kernel/cache.c | 2 +- fs/dax.c | 2 +- fs/hugetlbfs/inode.c | 4 ++-- include/linux/mm.h | 16 ++++++++-------- kernel/events/uprobes.c | 2 +- mm/hugetlb.c | 4 ++-- mm/interval_tree.c | 22 +++++++++++----------- mm/khugepaged.c | 4 ++-- mm/memory-failure.c | 6 +++--- mm/memory.c | 2 +- mm/mmap.c | 2 +- mm/nommu.c | 8 ++++---- mm/pagewalk.c | 2 +- mm/rmap.c | 2 +- mm/vma.c | 12 ++++++------ tools/testing/vma/include/stubs.h | 4 ++-- 19 files changed, 50 insertions(+), 50 deletions(-) (limited to 'arch') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index cd52cf7f8874..b505f1fd7984 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * cache coherency. */ flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(mpnt, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(mpnt, mapping, pgoff, pgoff) { /* * If we are using split PTE locks, then we need to take the pte * lock. Otherwise we are using shared mm->page_table_lock which diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 8c593e9898ee..66270904f874 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio * pgoff_end = pgoff + folio_nr_pages(folio) - 1; flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff_end) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff_end) { unsigned long start, offset, pfn; unsigned int nr; diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c index 42e3bf892316..10232476031f 100644 --- a/arch/nios2/mm/cacheflush.c +++ b/arch/nios2/mm/cacheflush.c @@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio) pgoff = folio->index; flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long start; if (vma->vm_mm != mm) diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index f28aa7884cbf..98200b6b5b82 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio) * on machines that support equivalent aliasing */ flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) { unsigned long offset = pgoff - vma->vm_pgoff; unsigned long pfn = folio_pfn(folio); diff --git a/fs/dax.c b/fs/dax.c index 2f0818a68a7f..4ea3384ecefd 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, /* Walk all mappings of a given index of a file and writeprotect them */ i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, mapping, index, end) { + mapping_rmap_tree_foreach(vma, mapping, index, end) { pfn_mkclean_range(pfn, count, index, vma); cond_resched(); } diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 4ea1798f1ffb..9fddd1e0b813 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -393,7 +393,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h, i_mmap_lock_write(mapping); retry: vma_lock = NULL; - vma_interval_tree_foreach(vma, mapping, start, end - 1) { + mapping_rmap_tree_foreach(vma, mapping, start, end - 1) { v_start = vma_offset_start(vma, start); v_end = vma_offset_end(vma, end); @@ -469,7 +469,7 @@ hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start, * unmapped. Note, end is exclusive, whereas the interval tree takes * an inclusive "last". */ - vma_interval_tree_foreach(vma, mapping, start, + mapping_rmap_tree_foreach(vma, mapping, start, end ? end - 1 : ULONG_MAX) { unsigned long v_start; unsigned long v_end; diff --git a/include/linux/mm.h b/include/linux/mm.h index 349d59a737eb..38132060ac58 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4094,24 +4094,24 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *vma, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping); -void vma_interval_tree_insert_after(struct vm_area_struct *vma, +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, struct address_space *mapping); -void vma_interval_tree_remove(struct vm_area_struct *vma, +void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping); struct vm_area_struct * -vma_interval_tree_iter_first(struct address_space *mapping, +mapping_rmap_tree_iter_first(struct address_space *mapping, pgoff_t pgoff_start, pgoff_t pgoff_last); struct vm_area_struct * -vma_interval_tree_iter_next(struct vm_area_struct *vma, +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, pgoff_t pgoff_start, pgoff_t pgoff_last); -#define vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ - for (vma = vma_interval_tree_iter_first(mapping, pgoff_start, \ +#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ pgoff_last); \ - vma; vma = vma_interval_tree_iter_next(vma, pgoff_start, \ + vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ pgoff_last)) void anon_vma_interval_tree_insert(struct anon_vma_chain *node, diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 50a96a4d812d..ac340a703d06 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1210,7 +1210,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (!valid_vma(vma, is_register)) continue; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 01bb45f9c7eb..35fb79784192 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5455,7 +5455,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * __unmap_hugepage_range() is called as the lock is already held */ i_mmap_lock_write(mapping); - vma_interval_tree_foreach(iter_vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) { /* Do not unmap the current VMA */ if (iter_vma == vma) continue; @@ -6998,7 +6998,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *pte; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(svma, mapping, idx, idx) { + mapping_rmap_tree_foreach(svma, mapping, idx, idx) { if (svma == vma) continue; diff --git a/mm/interval_tree.c b/mm/interval_tree.c index b387d39e0547..39e020555910 100644 --- a/mm/interval_tree.c +++ b/mm/interval_tree.c @@ -16,16 +16,16 @@ INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb, pgoff_t, shared.rb_subtree_last, vma_start_pgoff, vma_last_pgoff, static, - __vma_interval_tree) + __mapping_rmap_tree) -void vma_interval_tree_insert(struct vm_area_struct *vma, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping) { - __vma_interval_tree_insert(vma, &mapping->i_mmap); + __mapping_rmap_tree_insert(vma, &mapping->i_mmap); } /* Insert vma immediately after prev in the interval tree */ -void vma_interval_tree_insert_after(struct vm_area_struct *vma, +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, struct address_space *mapping) { @@ -55,28 +55,28 @@ void vma_interval_tree_insert_after(struct vm_area_struct *vma, vma->shared.rb_subtree_last = pgoff_last; rb_link_node(&vma->shared.rb, &parent->shared.rb, link); rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root, - &__vma_interval_tree_augment); + &__mapping_rmap_tree_augment); } -void vma_interval_tree_remove(struct vm_area_struct *vma, +void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping) { - __vma_interval_tree_remove(vma, &mapping->i_mmap); + __mapping_rmap_tree_remove(vma, &mapping->i_mmap); } struct vm_area_struct * -vma_interval_tree_iter_first(struct address_space *mapping, +mapping_rmap_tree_iter_first(struct address_space *mapping, pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __vma_interval_tree_iter_first(&mapping->i_mmap, + return __mapping_rmap_tree_iter_first(&mapping->i_mmap, pgoff_start, pgoff_last); } struct vm_area_struct * -vma_interval_tree_iter_next(struct vm_area_struct *vma, +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, pgoff_t pgoff_start, pgoff_t pgoff_last) { - return __vma_interval_tree_iter_next(vma, pgoff_start, pgoff_last); + return __mapping_rmap_tree_iter_next(vma, pgoff_start, pgoff_last); } /* Anonymous interval tree (anon_vma->rb_root) */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index e5462e026642..912be3daf37e 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2141,7 +2141,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct vm_area_struct *vma; i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; @@ -2573,7 +2573,7 @@ xa_unlocked: * not be able to observe any missing pages due to the * previously inserted retry entries. */ - vma_interval_tree_foreach(vma, mapping, start, end) { + mapping_rmap_tree_foreach(vma, mapping, start, end) { if (userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 4358856e764f..66ae93c6d4b2 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -598,7 +598,7 @@ static void collect_procs_file(const struct folio *folio, if (!t) continue; - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { /* * Send early kill signal to tasks where a vma covers * the page but the corrupted page is not necessarily @@ -649,7 +649,7 @@ static void collect_procs_fsdax(const struct page *page, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, mapping, pgoff, pgoff) { + mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) { if (vma->vm_mm == t->mm) add_to_kill_fsdax(t, page, vma, to_kill, pgoff); } @@ -2296,7 +2296,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space, t = task_early_kill(tsk, true); if (!t) continue; - vma_interval_tree_foreach(vma, mapping, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, mapping, 0, ULONG_MAX) { pgoff_t pgoff; if (vma->vm_mm == t->mm && diff --git a/mm/memory.c b/mm/memory.c index 1aac6e4ea221..921b935ffb58 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4425,7 +4425,7 @@ static inline void unmap_mapping_range_tree(struct address_space *mapping, unsigned long start, size; struct mmu_gather tlb; - vma_interval_tree_foreach(vma, mapping, first_index, last_index) { + mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) { const pgoff_t start_idx = max(first_index, vma->vm_pgoff); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; diff --git a/mm/mmap.c b/mm/mmap.c index 2f22fb0d068d..91eaa0c9e7ee 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1830,7 +1830,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ - vma_interval_tree_insert_after(tmp, mpnt, mapping); + mapping_rmap_tree_insert_after(tmp, mpnt, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } diff --git a/mm/nommu.c b/mm/nommu.c index 13023d73d1e4..f51a4a375c8c 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, mapping); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma) i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, mapping); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } @@ -1843,7 +1843,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, i_mmap_lock_read(inode->i_mapping); /* search for VMAs that fall within the dead zone */ - vma_interval_tree_foreach(vma, inode->i_mapping, low, high) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, low, high) { /* found one - only interested if it's shared out of the page * cache */ if (vma->vm_flags & VM_SHARED) { @@ -1859,7 +1859,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size, * we don't check for any regions that start beyond the EOF as there * shouldn't be any */ - vma_interval_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { + mapping_rmap_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) { if (!(vma->vm_flags & VM_SHARED)) continue; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index bdede741c9e6..2745e9d015ed 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -816,7 +816,7 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); - vma_interval_tree_foreach(vma, mapping, first_index, + mapping_rmap_tree_foreach(vma, mapping, first_index, first_index + nr - 1) { /* Clip to the vma */ vba = vma->vm_pgoff; diff --git a/mm/rmap.c b/mm/rmap.c index 73673cd3961e..d3d3d6cb17b8 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -3053,7 +3053,7 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, i_mmap_lock_read(mapping); } lookup: - vma_interval_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { + mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) { unsigned long address = vma_address(vma, pgoff_start, nr_pages); VM_BUG_ON_VMA(address == -EFAULT, vma); diff --git a/mm/vma.c b/mm/vma.c index ce4ec4b71138..d99e2088de4e 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -234,7 +234,7 @@ static void __vma_link_file(struct vm_area_struct *vma, mapping_allow_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, mapping); + mapping_rmap_tree_insert(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -248,7 +248,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma, mapping_unmap_writable(mapping); flush_dcache_mmap_lock(mapping); - vma_interval_tree_remove(vma, mapping); + mapping_rmap_tree_remove(vma, mapping); flush_dcache_mmap_unlock(mapping); } @@ -319,9 +319,9 @@ static void vma_prepare(struct vma_prepare *vp) if (vp->file) { flush_dcache_mmap_lock(vp->mapping); - vma_interval_tree_remove(vp->vma, vp->mapping); + mapping_rmap_tree_remove(vp->vma, vp->mapping); if (vp->adj_next) - vma_interval_tree_remove(vp->adj_next, vp->mapping); + mapping_rmap_tree_remove(vp->adj_next, vp->mapping); } } @@ -339,8 +339,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, { if (vp->file) { if (vp->adj_next) - vma_interval_tree_insert(vp->adj_next, vp->mapping); - vma_interval_tree_insert(vp->vma, vp->mapping); + mapping_rmap_tree_insert(vp->adj_next, vp->mapping); + mapping_rmap_tree_insert(vp->vma, vp->mapping); flush_dcache_mmap_unlock(vp->mapping); } diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 94442b29458d..37f81b11e8b7 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -257,12 +257,12 @@ static inline void vm_acct_memory(long pages) { } -static inline void vma_interval_tree_insert(struct vm_area_struct *vma, +static inline void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping) { } -static inline void vma_interval_tree_remove(struct vm_area_struct *vma, +static inline void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping) { } -- cgit v1.2.3 From 935ec96c02912124797c1cdf26ce1026175d4ec4 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Fri, 10 Jul 2026 21:17:02 +0100 Subject: mm: use linear_page_[index, delta]() consistently There are a number of places where we open code what linear_page_index() and linear_page_delta() calculate. Replace this code with the appropriate functions for consistency. No functional change intended. Link: https://lore.kernel.org/20260710-b4-pre-scalable-cow-v2-21-2a5aa403d977@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Thomas Zimmermann [DRM] Acked-by: Kai Huang # for sgx Reviewed-by: Gregory Price Reviewed-by: Pedro Falcato # for mm Reviewed-by: Ackerley Tng [guest_memfd] Reviewed-by: Vlastimil Babka (SUSE) Cc: David Hildenbrand (Arm) Cc: Marek Szyprowski Cc: SJ Park Cc: Liam R. Howlett (Oracle) Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm/mm/fault-armv.c | 2 +- arch/x86/kernel/cpu/sgx/virt.c | 3 ++- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 3 ++- drivers/gpu/drm/gma500/gem.c | 2 +- drivers/gpu/drm/msm/msm_gem.c | 3 ++- drivers/gpu/drm/omapdrm/omap_gem.c | 5 +++-- drivers/gpu/drm/tegra/gem.c | 3 ++- drivers/gpu/drm/ttm/ttm_bo_vm.c | 7 ++++--- drivers/vfio/pci/nvgrace-gpu/main.c | 3 ++- drivers/vfio/pci/vfio_pci_core.c | 3 ++- mm/nommu.c | 2 +- mm/vma.c | 2 +- virt/kvm/guest_memfd.c | 2 +- 13 files changed, 24 insertions(+), 16 deletions(-) (limited to 'arch') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index b505f1fd7984..2769b0f3325f 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -132,7 +132,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, pgoff_t pgoff; int aliases = 0; - pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + pgoff = linear_page_index(vma, addr); /* * If we have any shared mappings that are in the same mm diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c index db6806c40483..6a1933ddc6fc 100644 --- a/arch/x86/kernel/cpu/sgx/virt.c +++ b/arch/x86/kernel/cpu/sgx/virt.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -41,7 +42,7 @@ static int __sgx_vepc_fault(struct sgx_vepc *vepc, WARN_ON(!mutex_is_locked(&vepc->lock)); /* Calculate index of EPC page in virtual EPC's page_array */ - index = vma->vm_pgoff + PFN_DOWN(addr - vma->vm_start); + index = linear_page_index(vma, addr); epc_page = xa_load(&vepc->page_array, index); if (epc_page) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index b0436a1e103f..2e4d6d117ee2 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -188,7 +189,7 @@ static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) } /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); pfn = page_to_pfn(pages[pgoff]); diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c index 88f1e86c8903..2708e8c68f4c 100644 --- a/drivers/gpu/drm/gma500/gem.c +++ b/drivers/gpu/drm/gma500/gem.c @@ -288,7 +288,7 @@ static vm_fault_t psb_gem_fault(struct vm_fault *vmf) /* Page relative to the VMA start - we must calculate this ourselves because vmf->pgoff is the fake GEM offset */ - page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + page_offset = linear_page_delta(vma, vmf->address); /* CPU view of the page, don't go via the GART for CPU writes */ if (pobj->stolen) diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index efd3d3c9a449..cbf723a5d86f 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -360,7 +361,7 @@ static vm_fault_t msm_gem_fault(struct vm_fault *vmf) } /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); pfn = page_to_pfn(pages[pgoff]); diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 8e013e4f2c6b..00404fb6c29a 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -359,7 +360,7 @@ static vm_fault_t omap_gem_fault_1d(struct drm_gem_object *obj, pgoff_t pgoff; /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); if (omap_obj->pages) { omap_gem_cpu_sync_page(obj, pgoff); @@ -407,7 +408,7 @@ static vm_fault_t omap_gem_fault_2d(struct drm_gem_object *obj, const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE); /* We don't use vmf->pgoff since that has the fake offset: */ - pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + pgoff = linear_page_delta(vma, vmf->address); /* * Actual address we start mapping at is rounded down to previous slot diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 436394e04812..1d8d27a5ea89 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -564,7 +565,7 @@ static vm_fault_t tegra_bo_fault(struct vm_fault *vmf) if (!bo->pages) return VM_FAULT_SIGBUS; - offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + offset = linear_page_delta(vma, vmf->address); page = bo->pages[offset]; return vmf_insert_page(vma, vmf->address, page); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index a80510489c45..88babf435ac2 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -32,6 +32,7 @@ #define pr_fmt(fmt) "[TTM] " fmt #include +#include #include #include @@ -208,9 +209,9 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, if (unlikely(err != 0)) return VM_FAULT_SIGBUS; - page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); - page_last = vma_pages(vma) + vma->vm_pgoff - + page_offset = linear_page_index(vma, address) - + drm_vma_node_start(&bo->base.vma_node); + page_last = vma_end_pgoff(vma) - drm_vma_node_start(&bo->base.vma_node); if (unlikely(page_offset >= PFN_UP(bo->base.size))) diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index d07dcacb76bd..963fd8ded20d 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -385,7 +386,7 @@ static unsigned long addr_to_pgoff(struct vm_area_struct *vma, u64 pgoff = vma->vm_pgoff & ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); - return ((addr - vma->vm_start) >> PAGE_SHIFT) + pgoff; + return linear_page_delta(vma, addr) + pgoff; } static vm_fault_t nvgrace_gpu_vfio_pci_huge_fault(struct vm_fault *vmf, diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3f11a9624b9c..a113c55845e1 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1780,7 +1781,7 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf, struct vm_area_struct *vma = vmf->vma; struct vfio_pci_core_device *vdev = vma->vm_private_data; unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1); - unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; + unsigned long pgoff = linear_page_delta(vma, addr); unsigned long pfn = vma_to_pfn(vma) + pgoff; vm_fault_t ret = VM_FAULT_FALLBACK; diff --git a/mm/nommu.c b/mm/nommu.c index 52074ea90acb..c0a0869cd0d6 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1355,7 +1355,7 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, *region = *vma->vm_region; new->vm_region = region; - npages = (addr - vma->vm_start) >> PAGE_SHIFT; + npages = linear_page_delta(vma, addr); if (new_below) { region->vm_top = region->vm_end = new->vm_end = addr; diff --git a/mm/vma.c b/mm/vma.c index 4a4d096ba86a..7aa0149f076c 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -517,7 +517,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + new->vm_pgoff += linear_page_delta(vma, addr); } err = -ENOMEM; diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index db57c5766ab6..f0e5da490866 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -440,7 +440,7 @@ static int kvm_gmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpo static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma, unsigned long addr, pgoff_t *ilx) { - pgoff_t pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + pgoff_t pgoff = linear_page_index(vma, addr); struct inode *inode = file_inode(vma->vm_file); *ilx = inode->i_ino; -- cgit v1.2.3 From c1edc161f98eba4363b0faa2bafed00ff98dfa13 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:44:59 +0100 Subject: mm/vma: update do_mmap() to use vma_flags_t The core do_mmap() function accepts a vm_flags_t parameter which it then manipulates before passing to mmap_region() to do the heavy lifting of the memory mapping. Update do_mmap() to instead accept a vma_flags_t parameter, and adjust all the logic within do_mmap() to manipulate this instead. This is as part of the ongoing effort to convert VMA flags from a system word size to a bitmap type which allows us to unrestrict the number of VMA flags, as well as gain control over how VMA flag manipulation occurs. We do not cascade these changes to all functions which accept vm_flags_t, but rather use vma_flags_to_legacy() where necessary, specifically deferring converting calc_vm_prot_bits(), calc_vm_flag_bits() and __get_unmapped_area() to vma_flags_t. Also utilise the new vma_flags_can_grow() predicate which correctly handles the case of architectures without upward growing stacks. As part of this change, introduce VMA_SHADOW_STACK so we can correctly handle the case of the shadow stack not being defined. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-2-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Lance Yang Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- arch/mips/kernel/vdso.c | 4 +-- fs/aio.c | 2 +- include/linux/memfd.h | 6 ++--- include/linux/mm.h | 6 +++-- ipc/shm.c | 3 ++- mm/memfd.c | 15 +++++------ mm/mmap.c | 67 +++++++++++++++++++++++++++++-------------------- mm/nommu.c | 3 ++- mm/util.c | 10 ++++---- mm/vma.c | 7 +++--- mm/vma.h | 2 +- 11 files changed, 69 insertions(+), 56 deletions(-) (limited to 'arch') diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index bd1fc17d3975..94873775fc0f 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -91,8 +91,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) /* Map delay slot emulation page */ base = do_mmap(NULL, STACK_TOP, PAGE_SIZE, PROT_READ | PROT_EXEC, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, 0, 0, &unused, - NULL); + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, + EMPTY_VMA_FLAGS, 0, &unused, NULL); if (IS_ERR_VALUE(base)) { ret = base; goto out; diff --git a/fs/aio.c b/fs/aio.c index f57fa21a2503..3de586da197b 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -604,7 +604,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, PROT_READ | PROT_WRITE, - MAP_SHARED, 0, 0, &unused, NULL); + MAP_SHARED, EMPTY_VMA_FLAGS, 0, &unused, NULL); mmap_write_unlock(mm); if (IS_ERR((void *)ctx->mmap_base)) { ctx->mmap_size = 0; diff --git a/include/linux/memfd.h b/include/linux/memfd.h index b4fda09dab9f..c159e40e3f34 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -14,9 +14,9 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); * to sealing, or 0 otherwise. * * We also update VMA flags if appropriate by manipulating the VMA flags pointed - * to by vm_flags_ptr. + * to by vma_flags_ptr. */ -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr); +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr); struct file *memfd_alloc_file(const char *name, unsigned int flags); int memfd_get_seals(struct file *file); int memfd_add_seals(struct file *file, unsigned int seals); @@ -30,7 +30,7 @@ static inline struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) return ERR_PTR(-EINVAL); } static inline int memfd_check_seals_mmap(struct file *file, - vm_flags_t *vm_flags_ptr) + vma_flags_t *vma_flags_ptr) { return 0; } diff --git a/include/linux/mm.h b/include/linux/mm.h index 7a7f559b3df0..d20aa2f80472 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -465,9 +465,11 @@ enum { #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI) #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) +#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) #else #define VM_SHADOW_STACK VM_NONE +#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) #endif #if defined(CONFIG_PPC64) @@ -4198,9 +4200,9 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, return __get_unmapped_area(file, addr, len, pgoff, flags, 0); } -extern unsigned long do_mmap(struct file *file, unsigned long addr, +unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d..bb1a721a3e74 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1661,7 +1661,8 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto invalid; } - addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL); + addr = do_mmap(file, addr, size, prot, flags, EMPTY_VMA_FLAGS, 0, + &populate, NULL); *raddr = addr; err = 0; if (IS_ERR_VALUE(addr)) diff --git a/mm/memfd.c b/mm/memfd.c index 6c72fe6caef7..c708d92533f4 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -370,39 +370,36 @@ static inline bool is_write_sealed(unsigned int seals) return seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE); } -static int check_write_seal(vm_flags_t *vm_flags_ptr) +static int check_write_seal(vma_flags_t *vma_flags_ptr) { - vm_flags_t vm_flags = *vm_flags_ptr; - vm_flags_t mask = vm_flags & (VM_SHARED | VM_WRITE); - /* If a private mapping then writability is irrelevant. */ - if (!(mask & VM_SHARED)) + if (!vma_flags_test(vma_flags_ptr, VMA_SHARED_BIT)) return 0; /* * New PROT_WRITE and MAP_SHARED mmaps are not allowed when * write seals are active. */ - if (mask & VM_WRITE) + if (vma_flags_test(vma_flags_ptr, VMA_WRITE_BIT)) return -EPERM; /* * This is a read-only mapping, disallow mprotect() from making a * write-sealed mapping writable in future. */ - *vm_flags_ptr &= ~VM_MAYWRITE; + vma_flags_clear(vma_flags_ptr, VMA_MAYWRITE_BIT); return 0; } -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr) +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr) { int err = 0; unsigned int *seals_ptr = memfd_file_seals_ptr(file); unsigned int seals = seals_ptr ? *seals_ptr : 0; if (is_write_sealed(seals)) - err = check_write_seal(vm_flags_ptr); + err = check_write_seal(vma_flags_ptr); return err; } diff --git a/mm/mmap.c b/mm/mmap.c index 308a43eabd83..bcc3793e532d 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -280,7 +280,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * do_mmap() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to - * apply @vm_flags. If this is a file-backed mapping then the file is specified + * apply @vma_flags. If this is a file-backed mapping then the file is specified * in @file and page offset into the file via @pgoff. * * This function does not perform security checks on the file and assumes, if @@ -320,7 +320,8 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * (2) for details. * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. - * @vm_flags: VMA flags which should be set by default, or 0 otherwise. + * @vma_flags: VMA flags which should be set by default, or EMPTY_VMA_FLAGS + * otherwise. * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be @@ -335,7 +336,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, */ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flags, vm_flags_t vm_flags, + unsigned long flags, vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { @@ -399,13 +400,19 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * to. we assume access permissions have been handled by the open * of the memory object, so we don't do any here. */ - vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) | - mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_prot_bits(prot, pkey))); + vma_flags_set_mask(&vma_flags, + legacy_to_vma_flags(calc_vm_flag_bits(file, flags))); + vma_flags_set_mask(&vma_flags, mm->def_vma_flags); + vma_flags_set(&vma_flags, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, + VMA_MAYEXEC_BIT); /* Obtain the address to map to. we verify (or select) it and ensure * that it represents a valid section of the address space. */ - addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags); + addr = __get_unmapped_area(file, addr, len, pgoff, flags, + vma_flags_to_legacy(vma_flags)); if (IS_ERR_VALUE(addr)) return addr; @@ -418,7 +425,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (!can_do_mlock()) return -EPERM; - if (!mlock_future_ok(mm, vm_flags & VM_LOCKED, len)) + if (!mlock_future_ok(mm, vma_flags_test(&vma_flags, VMA_LOCKED_BIT), len)) return -EAGAIN; if (file) { @@ -461,22 +468,23 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) return -EACCES; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); if (!(file->f_mode & FMODE_WRITE)) - vm_flags &= ~(VM_MAYWRITE | VM_SHARED); + vma_flags_clear(&vma_flags, VMA_MAYWRITE_BIT, + VMA_SHARED_BIT); fallthrough; case MAP_PRIVATE: if (!(file->f_mode & FMODE_READ)) return -EACCES; if (path_noexec(&file->f_path)) { - if (vm_flags & VM_EXEC) + if (vma_flags_test(&vma_flags, VMA_EXEC_BIT)) return -EPERM; - vm_flags &= ~VM_MAYEXEC; + vma_flags_clear(&vma_flags, VMA_MAYEXEC_BIT); } if (!can_mmap_file(file)) return -ENODEV; - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; break; @@ -488,23 +496,27 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Check to see if we are violating any seals and update VMA * flags if necessary to avoid future seal violations. */ - err = memfd_check_seals_mmap(file, &vm_flags); + err = memfd_check_seals_mmap(file, &vma_flags); if (err) return (unsigned long)err; } else { switch (flags & MAP_TYPE) { case MAP_SHARED: - if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; /* * Ignore pgoff. */ pgoff = 0; - vm_flags |= VM_SHARED | VM_MAYSHARE; + vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT); break; - case MAP_DROPPABLE: - if (VM_DROPPABLE == VM_NONE) + case MAP_DROPPABLE: { + vma_flags_t droppable = VMA_DROPPABLE; + + if (vma_flags_empty(&droppable)) return -EOPNOTSUPP; + vma_flags_set_mask(&vma_flags, droppable); + /* * A locked or stack area makes no sense to be droppable. * @@ -515,23 +527,24 @@ unsigned long do_mmap(struct file *file, unsigned long addr, */ if (flags & (MAP_LOCKED | MAP_HUGETLB)) return -EINVAL; - if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP)) + if (vma_flags_can_grow(&vma_flags)) return -EINVAL; - vm_flags |= VM_DROPPABLE; - /* * If the pages can be dropped, then it doesn't make * sense to reserve them. */ - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* * Likewise, they're volatile enough that they * shouldn't survive forks or coredumps. */ - vm_flags |= VM_WIPEONFORK | VM_DONTDUMP; + vma_flags_set(&vma_flags, VMA_WIPEONFORK_BIT, + VMA_DONTDUMP_BIT); + fallthrough; + } case MAP_PRIVATE: /* * Set pgoff according to addr for anon_vma. @@ -550,16 +563,16 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (flags & MAP_NORESERVE) { /* We honor MAP_NORESERVE if allowed to overcommit */ if (sysctl_overcommit_memory != OVERCOMMIT_NEVER) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); /* hugetlb applies strict overcommit unless MAP_NORESERVE */ if (file && is_file_hugepages(file)) - vm_flags |= VM_NORESERVE; + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vma_flags, pgoff, uf); if (!IS_ERR_VALUE(addr) && - ((vm_flags & VM_LOCKED) || + (vma_flags_test(&vma_flags, VMA_LOCKED_BIT) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) *populate = len; return addr; @@ -1191,7 +1204,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } ret = do_mmap(vma->vm_file, start, size, - prot, flags, 0, pgoff, &populate, NULL); + prot, flags, EMPTY_VMA_FLAGS, pgoff, &populate, NULL); out: mmap_write_unlock(mm); fput(file); diff --git a/mm/nommu.c b/mm/nommu.c index adc0b0ca906b..498e01ee40b0 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1014,11 +1014,12 @@ unsigned long do_mmap(struct file *file, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { + vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); struct vm_area_struct *vma; struct vm_region *region; struct rb_node *rb; diff --git a/mm/util.c b/mm/util.c index 5090457542ca..bf0513d1d3d0 100644 --- a/mm/util.c +++ b/mm/util.c @@ -578,8 +578,8 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); + ret = do_mmap(file, addr, len, prot, flag, EMPTY_VMA_FLAGS, pgoff, + &populate, &uf); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -627,20 +627,20 @@ EXPORT_SYMBOL(vm_mmap); unsigned long vm_mmap_shadow_stack(unsigned long addr, unsigned long len, unsigned long flags) { + vma_flags_t vma_flags = VMA_SHADOW_STACK; struct mm_struct *mm = current->mm; unsigned long ret, unused; - vm_flags_t vm_flags = VM_SHADOW_STACK; flags |= MAP_ANONYMOUS | MAP_PRIVATE; if (addr) flags |= MAP_FIXED_NOREPLACE; if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) - vm_flags |= VM_NOHUGEPAGE; + vma_flags_set(&vma_flags, VMA_NOHUGEPAGE_BIT); mmap_write_lock(mm); ret = do_mmap(NULL, addr, len, PROT_READ | PROT_WRITE, flags, - vm_flags, 0, &unused, NULL); + vma_flags, 0, &unused, NULL); mmap_write_unlock(mm); return ret; diff --git a/mm/vma.c b/mm/vma.c index 12a684c36b6c..dcf613b5e695 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2883,7 +2883,7 @@ abort_munmap: * file to be mapped, otherwise NULL. * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. - * @vm_flags: The VMA flags which should be applied to the mapping. + * @vma_flags: The VMA flags which should be applied to the mapping. * @pgoff: If @file is specified, the page offset into the file, if not then * the virtual page offset in memory of the anonymous mapping. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap @@ -2893,12 +2893,11 @@ abort_munmap: * been performed. */ unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; - const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); @@ -2907,7 +2906,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return -EACCES; /* Allow architectures to sanity-check the vm_flags. */ - if (!arch_validate_flags(vm_flags)) + if (!arch_validate_flags(vma_flags_to_legacy(vma_flags))) return -EINVAL; /* Map writable and ensure this isn't a sealed memfd. */ diff --git a/mm/vma.h b/mm/vma.h index adb7a0ba1192..4eb85b4a6adc 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -517,7 +517,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vma_flags_t vma_flags, unsigned long pgoff, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, -- cgit v1.2.3 From cafb3c1e4367c0258eb66f1b6028b1e8c26e94b0 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sat, 11 Jul 2026 19:45:01 +0100 Subject: mm: update generic_get_unmapped_area[_topdown]() to use vma_flags_t As part of the changes converting VMA flags from a system word size to a bitmap, extend this change to generic_get_unmapped_area() and generic_get_unmapped_area_topdown(), which also allows us to convert stack_guard_placement() as well. We retain arch_get_unmapped_area() and arch_get_unmapped_area_topdown() as-is for now, using legacy_to_vma_flags() as necessary to do so. No functional change intended. Link: https://lore.kernel.org/20260711-b4-vma-flags-mm-v2-4-0fa2357d5431@kernel.org Signed-off-by: Lorenzo Stoakes Reviewed-by: Zi Yan Reviewed-by: Lance Yang Reviewed-by: Vlastimil Babka (SUSE) Cc: Baolin Wang Cc: Barry Song Cc: Christian Brauner Cc: Dave Airlie Cc: David Hildenbrand Cc: Dev Jain Cc: Jani Nikula Cc: Jan Kara Cc: Jann Horn Cc: Mike Rapoport Cc: Muchun Song Cc: Nico Pache Cc: Oscar Salvador Cc: Pedro Falcato Cc: Suren Baghdasaryan Cc: Thomas Zimmermann Signed-off-by: Andrew Morton --- arch/powerpc/mm/book3s64/slice.c | 6 ++++-- include/linux/sched/mm.h | 4 ++-- mm/mmap.c | 16 ++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c index 28bec5bc7879..82127e31dca6 100644 --- a/arch/powerpc/mm/book3s64/slice.c +++ b/arch/powerpc/mm/book3s64/slice.c @@ -659,7 +659,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned int psize; if (radix_enabled()) - return generic_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags); + return generic_get_unmapped_area(filp, addr, len, pgoff, flags, + legacy_to_vma_flags(vm_flags)); if (filp && is_file_hugepages(filp)) psize = file_to_psize(filp); @@ -679,7 +680,8 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned int psize; if (radix_enabled()) - return generic_get_unmapped_area_topdown(filp, addr0, len, pgoff, flags, vm_flags); + return generic_get_unmapped_area_topdown(filp, addr0, len, + pgoff, flags, legacy_to_vma_flags(vm_flags)); if (filp && is_file_hugepages(filp)) psize = file_to_psize(filp); diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 10d723bee95c..10be8a54b416 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -200,11 +200,11 @@ unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); #else static inline void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) {} diff --git a/mm/mmap.c b/mm/mmap.c index 4a88b2a233f7..39d33f3a55c2 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -657,9 +657,9 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) * Determine if the allocation needs to ensure that there is no * existing mapping within it's guard gaps, for use as start_gap. */ -static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) +static inline unsigned long stack_guard_placement(vma_flags_t vma_flags) { - if (vm_flags & VM_SHADOW_STACK) + if (vma_flags_test_single_mask(&vma_flags, VMA_SHADOW_STACK)) return PAGE_SIZE; return 0; @@ -701,7 +701,7 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev; @@ -726,7 +726,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = mmap_end; - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); return vm_unmapped_area(&info); @@ -739,7 +739,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif @@ -750,7 +750,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) + unsigned long flags, vma_flags_t vma_flags) { struct vm_area_struct *vma, *prev; struct mm_struct *mm = current->mm; @@ -778,7 +778,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); - info.start_gap = stack_guard_placement(vm_flags); + info.start_gap = stack_guard_placement(vma_flags); if (filp && is_file_hugepages(filp)) info.align_mask = huge_page_mask_align(filp); addr = vm_unmapped_area(&info); @@ -807,7 +807,7 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long flags, vm_flags_t vm_flags) { return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags, - vm_flags); + legacy_to_vma_flags(vm_flags)); } #endif -- cgit v1.2.3 From 93190aa9450b47ac6f0335b5f5c013d1a164e5f5 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Thu, 9 Jul 2026 02:06:46 +0900 Subject: riscv: remove unused __late_set_fixmap() and __late_clear_fixmap() __late_set_fixmap() and __late_clear_fixmap() are only used after early_ioremap_reset() has been called. riscv never calls it because __set_fixmap() works before and after paging_init(). So remove them. No functional change. Link: https://lore.kernel.org/20260708170647.362562-3-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Cc: Albert Ou Cc: Alexandre Ghiti Cc: Catalin Marinas Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Palmer Dabbelt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/riscv/include/asm/fixmap.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'arch') diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 0a55099bb734..a81823c5ee4a 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -55,9 +55,6 @@ enum fixed_addresses { #define __early_set_fixmap __set_fixmap -#define __late_set_fixmap __set_fixmap -#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) - extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); -- cgit v1.2.3 From 1f2e3e7920a76a6a26844f7601fa51919701b050 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Thu, 9 Jul 2026 02:06:47 +0900 Subject: arm64: remove early_ioremap_reset() call and __late_* macros On arm64, __early_set_fixmap(), __late_set_fixmap() and __late_clear_fixmap() are all __set_fixmap(). Calling early_ioremap_reset() changes nothing. So remove the call and the macros. No functional change. Link: https://lore.kernel.org/20260708170647.362562-4-ekffu200098@gmail.com Signed-off-by: Sang-Heon Jeon Acked-by: Will Deacon Cc: Albert Ou Cc: Alexandre Ghiti Cc: Catalin Marinas Cc: David Hildenbrand Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Palmer Dabbelt Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- arch/arm64/include/asm/fixmap.h | 3 --- arch/arm64/kernel/setup.c | 2 -- 2 files changed, 5 deletions(-) (limited to 'arch') diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 65555284446e..abd9800aabf7 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -110,9 +110,6 @@ void __init early_fixmap_init(void); #define __early_set_fixmap __set_fixmap -#define __late_set_fixmap __set_fixmap -#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) - extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); #include diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2..29c6100f0c50 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -347,8 +347,6 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) request_standard_resources(); - early_ioremap_reset(); - if (acpi_disabled) psci_dt_init(); else -- cgit v1.2.3