diff options
Diffstat (limited to 'include')
65 files changed, 1095 insertions, 654 deletions
diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h index 29cab7947980..3ff832ebcea5 100644 --- a/include/asm-generic/fixmap.h +++ b/include/asm-generic/fixmap.h @@ -21,7 +21,7 @@ #define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) #define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * 'index to address' translation. If anyone tries to use the idx * directly without translation, we catch the bug with a NULL-deference @@ -97,5 +97,5 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) #define set_fixmap_io(idx, phys) \ __set_fixmap(idx, phys, FIXMAP_PAGE_IO) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_FIXMAP_H */ diff --git a/include/asm-generic/getorder.h b/include/asm-generic/getorder.h index f2979e3a96b6..875ccae19683 100644 --- a/include/asm-generic/getorder.h +++ b/include/asm-generic/getorder.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_GETORDER_H #define __ASM_GENERIC_GETORDER_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/compiler.h> #include <linux/log2.h> @@ -47,6 +47,6 @@ static __always_inline __attribute_const__ int get_order(unsigned long size) #endif } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_GETORDER_H */ diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h index e1a2e1b7c8e7..635c41cc3479 100644 --- a/include/asm-generic/hugetlb.h +++ b/include/asm-generic/hugetlb.h @@ -37,24 +37,24 @@ static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) return pte_modify(pte, newprot); } -#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD_WP -static inline pte_t huge_pte_mkuffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD +static inline pte_t huge_pte_mkuffd(pte_t pte) { - return huge_pte_wrprotect(pte_mkuffd_wp(pte)); + return huge_pte_wrprotect(pte_mkuffd(pte)); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP -static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD +static inline pte_t huge_pte_clear_uffd(pte_t pte) { - return pte_clear_uffd_wp(pte); + return pte_clear_uffd(pte); } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_UFFD_WP -static inline int huge_pte_uffd_wp(pte_t pte) +#ifndef __HAVE_ARCH_HUGE_PTE_UFFD +static inline int huge_pte_uffd(pte_t pte) { - return pte_uffd_wp(pte); + return pte_uffd(pte); } #endif diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h index efa6610acbc7..c6b4eafaf4cb 100644 --- a/include/asm-generic/memory_model.h +++ b/include/asm-generic/memory_model.h @@ -4,7 +4,7 @@ #include <linux/pfn.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * supports 3 memory models. @@ -53,7 +53,7 @@ static inline int pfn_valid(unsigned long pfn) */ #define __page_to_pfn(pg) \ ({ const struct page *__pg = (pg); \ - int __sec = memdesc_section(__pg->flags); \ + int __sec = memdesc_section(&__pg->flags); \ (unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec))); \ }) @@ -86,6 +86,6 @@ static inline int pfn_valid(unsigned long pfn) #endif /* CONFIG_DEBUG_VIRTUAL */ #define phys_to_page(phys) pfn_to_page(PHYS_PFN(phys)) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/include/asm-generic/mmu.h b/include/asm-generic/mmu.h index 061838037542..5f78971e3ac2 100644 --- a/include/asm-generic/mmu.h +++ b/include/asm-generic/mmu.h @@ -6,7 +6,7 @@ * This is the mmu.h header for nommu implementations. * Architectures with an MMU need something more complex. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef struct { unsigned long end_brk; diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index 03b7dae47dd4..89c21f84cffb 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOP4D_H #define _PGTABLE_NOP4D_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __PAGETABLE_P4D_FOLDED 1 @@ -54,5 +54,5 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOP4D_H */ diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 8ffd64e7a24c..36b6490ed180 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPMD_H #define _PGTABLE_NOPMD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm-generic/pgtable-nopud.h> @@ -68,6 +68,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPMD_H */ diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index eb70c6d7ceff..356cbfbaab24 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPUD_H #define _PGTABLE_NOPUD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm-generic/pgtable-nop4d.h> @@ -62,5 +62,5 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) #undef pud_addr_end #define pud_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPUD_H */ diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h index 0d85791efdf7..30e88fc1de2f 100644 --- a/include/asm-generic/pgtable_uffd.h +++ b/include/asm-generic/pgtable_uffd.h @@ -2,79 +2,79 @@ #define _ASM_GENERIC_PGTABLE_UFFD_H /* - * Some platforms can customize the uffd-wp bit, making it unavailable + * Some platforms can customize the uffd PTE bit, making it unavailable * even if the architecture provides the resource. * Adding this API allows architectures to add their own checks for the * devices on which the kernel is running. * Note: When overriding it, please make sure the * CONFIG_HAVE_ARCH_USERFAULTFD_WP is part of this macro. */ -#ifndef pgtable_supports_uffd_wp -#define pgtable_supports_uffd_wp() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) +#ifndef pgtable_supports_uffd +#define pgtable_supports_uffd() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) #endif static inline bool uffd_supports_wp_marker(void) { - return pgtable_supports_uffd_wp() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); + return pgtable_supports_uffd() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); } #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP -static __always_inline int pte_uffd_wp(pte_t pte) +static __always_inline int pte_uffd(pte_t pte) { return 0; } -static __always_inline int pmd_uffd_wp(pmd_t pmd) +static __always_inline int pmd_uffd(pmd_t pmd) { return 0; } -static __always_inline pte_t pte_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_mkuffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_mkuffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_clear_uffd(pte_t pte) { return pte; } -static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +static __always_inline pmd_t pmd_clear_uffd(pmd_t pmd) { return pmd; } -static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_mkuffd(pte_t pte) { return pte; } -static __always_inline int pte_swp_uffd_wp(pte_t pte) +static __always_inline int pte_swp_uffd(pte_t pte) { return 0; } -static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +static __always_inline pte_t pte_swp_clear_uffd(pte_t pte) { return pte; } -static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_mkuffd(pmd_t pmd) { return pmd; } -static inline int pmd_swp_uffd_wp(pmd_t pmd) +static inline int pmd_swp_uffd(pmd_t pmd) { return 0; } -static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd) { return pmd; } diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 6ed9f82e639f..7f2d80a59792 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -165,11 +165,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) { WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } -void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags); +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags); #else static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} static inline void alloc_tag_sub_check(union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {} +static inline void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) {} #endif /* Caller should verify both ref and tag to be valid */ diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index 5b7d12b40d5e..c2284466e7aa 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -76,8 +76,6 @@ static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item) return percpu_counter_sum_positive(&wb->stat[item]); } -extern void wb_writeout_inc(struct bdi_writeback *wb); - /* * maximal error of a stat counter. */ diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h deleted file mode 100644 index f724340755e5..000000000000 --- a/include/linux/bootmem_info.h +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_BOOTMEM_INFO_H -#define __LINUX_BOOTMEM_INFO_H - -#include <linux/mm.h> -#include <linux/kmemleak.h> - -/* - * Types for free bootmem stored in the low bits of page->private. - */ -enum bootmem_type { - MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 1, - SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE, - MIX_SECTION_INFO, - NODE_INFO, - MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO, -}; - -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void __init register_page_bootmem_info_node(struct pglist_data *pgdat); -void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, - unsigned long nr_pages); - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type); -void put_page_bootmem(struct page *page); - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return (unsigned long)page->private & 0xf; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return (unsigned long)page->private >> 4; -} - -/* - * Any memory allocated via the memblock allocator and not via the - * buddy will be marked reserved already in the memmap. For those - * pages, we can call this function to free it to buddy allocator. - */ -static inline void free_bootmem_page(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - VM_BUG_ON_PAGE(page_ref_count(page) != 2, page); - - if (type == SECTION_INFO || type == MIX_SECTION_INFO) - put_page_bootmem(page); - else - VM_BUG_ON_PAGE(1, page); -} -#else -static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) -{ -} - -static inline void register_page_bootmem_memmap(unsigned long section_nr, - struct page *map, unsigned long nr_pages) -{ -} - -static inline void put_page_bootmem(struct page *page) -{ -} - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return SECTION_INFO; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return 0; -} - -static inline void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ -} - -static inline void free_bootmem_page(struct page *page) -{ - free_reserved_page(page); -} -#endif - -#endif /* __LINUX_BOOTMEM_INFO_H */ diff --git a/include/linux/codetag.h b/include/linux/codetag.h index ddae7484ca45..a25a085c2df1 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -77,6 +77,8 @@ struct codetag_iterator { void codetag_lock_module_list(struct codetag_type *cttype); bool codetag_trylock_module_list(struct codetag_type *cttype); void codetag_unlock_module_list(struct codetag_type *cttype); +unsigned long codetag_get_content_id(struct codetag_type *cttype); +unsigned int codetag_get_count(struct codetag_type *cttype); struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); struct codetag *codetag_next_ct(struct codetag_iterator *iter); diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..a80d38e752d2 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -83,7 +83,7 @@ extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask); extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask); @@ -224,7 +224,7 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) #define cpuset_current_mems_allowed (node_states[N_MEMORY]) static inline void cpuset_init_current_mems_allowed(void) {} -static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +static inline int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return 1; } diff --git a/include/linux/damon.h b/include/linux/damon.h index cfbbf8ba28f6..327f534696aa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -45,13 +45,11 @@ struct damon_size_range { * @ar: The address range of the region. * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. - * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for - * each sampling interval. * @probe_hits: Number of probe-positive region samples. - * @list: List head for siblings. * @age: Age of this region. * - * For any use case, @ar should be non-zero positive size. + * For any use case, @ar should be non-zero positive size. damon_set_regions() + * does the validation. * * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be * increased for every &damon_attrs->sample_interval if an access to the region @@ -59,13 +57,6 @@ struct damon_size_range { * not be done with direct access but with the helper function, * damon_update_region_access_rate(). * - * @nr_accesses_bp is another representation of @nr_accesses in basis point - * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a - * manner similar to moving sum. By the algorithm, this value becomes - * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can - * be used when the aggregation interval is too huge and therefore cannot wait - * for it before getting the access monitoring results. - * * @age is initially zero, increased for each aggregation interval, and reset * to zero again if the access frequency is significantly changed. If two * regions are merged into a new region, both @nr_accesses and @age of the new @@ -75,21 +66,19 @@ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; - unsigned int nr_accesses_bp; unsigned char probe_hits[DAMON_MAX_PROBES]; - struct list_head list; - unsigned int age; -/* private: Internal value for age calculation. */ +/* private: internal use only. */ + /* List head for siblings. */ + struct list_head list; + /* for age calculation. */ unsigned int last_nr_accesses; + unsigned char last_probe_hits[DAMON_MAX_PROBES]; }; /** * struct damon_target - Represents a monitoring target. * @pid: The PID of the virtual address space to monitor. - * @nr_regions: Number of monitoring target regions of this target. - * @regions_list: Head of the monitoring target regions of this target. - * @list: List head for siblings. * @obsolete: Whether the commit destination target is obsolete. * * Each monitoring context could have multiple targets. For example, a context @@ -103,10 +92,14 @@ struct damon_region { */ struct damon_target { struct pid *pid; + bool obsolete; +/* private: */ + /* Number of monitoring target regions of this target. */ unsigned int nr_regions; + /* Head of the monitoring target regions of this target. */ struct list_head regions_list; + /* List head for siblings. */ struct list_head list; - bool obsolete; }; /** @@ -182,10 +175,8 @@ enum damos_quota_goal_metric { * @metric: Metric to be used for representing the goal. * @target_value: Target value of @metric to achieve with the tuning. * @current_value: Current value of @metric. - * @last_psi_total: Last measured total PSI * @nid: Node id. * @memcg_id: Memcg id. - * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. The * score is calculated by how close @current_value and @target_value are. Then @@ -208,12 +199,17 @@ struct damos_quota_goal { unsigned long current_value; /* metric-dependent fields */ union { - u64 last_psi_total; struct { int nid; u64 memcg_id; }; +/* private: */ + /* Last measured total PSI */ + u64 last_psi_total; + }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; @@ -232,7 +228,6 @@ enum damos_quota_goal_tuner { * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @goals: Head of quota tuning goals (&damos_quota_goal) list. * @goal_tuner: Goal-based @esz tuning algorithm to use. * @esz: Effective size quota in bytes. * @fail_charge_num: Failed regions charge rate numerator. @@ -259,7 +254,7 @@ enum damos_quota_goal_tuner { * the scheme's action. DAMON then compares it against &sz and uses smaller * one as the effective quota. * - * If @goals is not empty, DAMON calculates yet another size quota based on the + * If goals is not empty, DAMON calculates yet another size quota based on the * goals using its internal feedback loop algorithm, for every @reset_interval. * Then, if the new size quota is smaller than the effective quota, it uses the * new size quota as the effective quota. @@ -280,7 +275,7 @@ struct damos_quota { unsigned long reset_interval; unsigned long ms; unsigned long sz; - struct list_head goals; + enum damos_quota_goal_tuner goal_tuner; unsigned long esz; @@ -292,6 +287,8 @@ struct damos_quota { unsigned int weight_age; /* private: */ + /* Head of quota tuning goals (&damos_quota_goal) list. */ + struct list_head goals; /* For throughput estimation */ unsigned long total_charged_sz; unsigned long total_charged_ns; @@ -435,7 +432,6 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. - * @list: List head for siblings. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -453,6 +449,8 @@ struct damos_filter { int target_idx; struct damon_size_range sz_range; }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; @@ -524,12 +522,8 @@ struct damos_migrate_dests { * @wmarks: Watermarks for automated (in)activation of this scheme. * @migrate_dests: Destination nodes if @action is "migrate_{hot,cold}". * @target_nid: Destination node if @action is "migrate_{hot,cold}". - * @core_filters: Additional set of &struct damos_filter for &action. - * @ops_filters: ops layer handling &struct damos_filter objects list. - * @last_applied: Last @action applied ops-managing entity. * @stat: Statistics of this scheme. * @max_nr_snapshots: Upper limit of nr_snapshots stat. - * @list: List head for siblings. * * For each @apply_interval_us, DAMON finds regions which fit in the * &pattern and applies &action to those. To avoid consuming too much @@ -551,16 +545,7 @@ struct damos_migrate_dests { * * Before applying the &action to a memory region, &struct damon_operations * implementation could check pages of the region and skip &action to respect - * &core_filters - * - * The minimum entity that @action can be applied depends on the underlying - * &struct damon_operations. Since it may not be aligned with the core layer - * abstract, namely &struct damon_region, &struct damon_operations could apply - * @action to same entity multiple times. Large folios that underlying on - * multiple &struct damon region objects could be such examples. The &struct - * damon_operations can use @last_applied to avoid that. DAMOS core logic - * unsets @last_applied when each regions walking for applying the scheme is - * finished. + * &struct damos_filter. * * After applying the &action to each region, &stat is updated. * @@ -571,6 +556,16 @@ struct damos { struct damos_access_pattern pattern; enum damos_action action; unsigned long apply_interval_us; + struct damos_quota quota; + struct damos_watermarks wmarks; + union { + struct { + int target_nid; + struct damos_migrate_dests migrate_dests; + }; + }; + struct damos_stat stat; + unsigned long max_nr_snapshots; /* private: internal use only */ /* * number of sample intervals that should be passed before applying @@ -587,20 +582,25 @@ struct damos { /* whether to reject core/ops filters umatched regions */ bool core_filters_default_reject; bool ops_filters_default_reject; -/* public: */ - struct damos_quota quota; - struct damos_watermarks wmarks; - union { - struct { - int target_nid; - struct damos_migrate_dests migrate_dests; - }; - }; + /* Additional set of &struct damos_filter for &action. */ struct list_head core_filters; + /* ops layer handling &struct damos_filter objects list. */ struct list_head ops_filters; + /* + * Last @action applied ops-managing entity. + * + * The minimum entity that @action can be applied depends on the + * underlying &struct damon_operations. Since it may not be aligned + * with the core layer abstract, namely &struct damon_region, &struct + * damon_operations could apply @action to same entity multiple times. + * Large folios that underlying on multiple &struct damon region + * objects could be such examples. The &struct damon_operations can + * use @last_applied to avoid that. DAMOS core logic unsets + * @last_applied when each regions walking for applying the scheme is + * finished. + */ void *last_applied; - struct damos_stat stat; - unsigned long max_nr_snapshots; + /* List head for siblings. */ struct list_head list; }; @@ -656,7 +656,10 @@ enum damon_ops_id { * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. * @apply_probes should apply the data attribute probes to each region and - * accordingly update the probe hits counter of the region. + * accordingly update the probe hits counter of the region. It should also + * set &damon_region->sampling_addr of each region if ``set_samples`` is true. + * It should also return maximum probe hits weighted sum of regions if + * ``return_max_wsum`` is true. * @get_scheme_score should return the priority score of a region for a scheme * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided @@ -674,7 +677,8 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); - void (*apply_probes)(struct damon_ctx *context); + unsigned int (*apply_probes)(struct damon_ctx *context, + bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, struct damon_region *r, struct damos *scheme); unsigned long (*apply_scheme)(struct damon_ctx *context, @@ -754,7 +758,6 @@ enum damon_filter_type { * @matching: Whether this filter is for the type-matching ones. * @allow: Whether the @type-@matching ones should pass this filter. * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. - * @list: Siblings list. */ struct damon_filter { enum damon_filter_type type; @@ -763,17 +766,22 @@ struct damon_filter { union { u64 memcg_id; }; +/* private: */ + /* Siblings list. */ struct list_head list; }; /** * struct damon_probe - Data region attribute probe. * - * @filters: Filters for assessing if a given region is for this probe. - * @list: Siblings list. + * @weight: Relative priority of the attribute for this probe. */ struct damon_probe { + unsigned int weight; +/* private: */ + /* Filters for assessing if a given region is for this probe. */ struct list_head filters; + /* Siblings list. */ struct list_head list; }; @@ -840,14 +848,9 @@ struct damon_attrs { * thread other than the kdamond should be made using safe DAMON APIs, * including damon_call() and damos_walk(). * - * @ops: Set of monitoring operations for given use cases. - * @probes: Head of probes (&damon_probe) list. * @addr_unit: Scale factor for core to ops address conversion. * @min_region_sz: Minimum region size. * @pause: Pause kdamond main loop. - * @adaptive_targets: Head of monitoring targets (&damon_target) list. - * @schemes: Head of schemes (&damos) list. - * @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct damon_ctx { struct damon_attrs attrs; @@ -896,15 +899,21 @@ struct damon_ctx { struct mutex kdamond_lock; /* public: */ - struct damon_operations ops; - struct list_head probes; unsigned long addr_unit; unsigned long min_region_sz; bool pause; +/* private: */ + /* Set of monitoring operations for given use cases. */ + struct damon_operations ops; + /* Head of monitoring targets (&damon_target) list. */ struct list_head adaptive_targets; + /* Head of probes (&damon_probe) list. */ + struct list_head probes; + /* Head of schemes (&damos) list. */ struct list_head schemes; + /* @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct rnd_state rnd_state; }; @@ -1011,11 +1020,16 @@ struct damon_probe *damon_new_probe(void); void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe); struct damon_region *damon_new_region(unsigned long start, unsigned long end); +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx); +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx); +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs); +void damon_update_region_access_rate(struct damon_region *r, bool accessed); struct damos_filter *damos_new_filter(enum damos_filter_type type, bool matching, bool allow); @@ -1062,21 +1076,23 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; } -static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) +/* Returns number of samples per aggregation interval */ +static inline unsigned int damon_nr_samples_per_aggr( + const struct damon_attrs *attrs) { unsigned long sample_interval; - unsigned long max_nr_accesses; + unsigned long nr_samples; sample_interval = attrs->sample_interval ? : 1; - max_nr_accesses = min(attrs->aggr_interval / sample_interval, + nr_samples = min(attrs->aggr_interval / sample_interval, (unsigned long)UINT_MAX); - return max_nr_accesses ? : 1; + return nr_samples ? : 1; } bool damon_initialized(void); int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs); bool damon_is_running(struct damon_ctx *ctx); int damon_kdamond_pid(struct damon_ctx *ctx); diff --git a/include/linux/gfp.h b/include/linux/gfp.h index cdf95a9f0b87..872bc53f32ec 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -17,28 +17,6 @@ struct mempolicy; #define __default_gfp(a,b,...) b #define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL) -/* Convert GFP flags to their corresponding migrate type */ -#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) -#define GFP_MOVABLE_SHIFT 3 - -static inline int gfp_migratetype(const gfp_t gfp_flags) -{ - VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); - BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); - BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); - BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); - BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> - GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); - - if (unlikely(page_group_by_mobility_disabled)) - return MIGRATE_UNMOVABLE; - - /* Group based on mobility */ - return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; -} -#undef GFP_MOVABLE_MASK -#undef GFP_MOVABLE_SHIFT - static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) { return !!(gfp_flags & __GFP_DIRECT_RECLAIM); @@ -226,10 +204,6 @@ static inline void arch_free_page(struct page *page, int order) { } static inline void arch_alloc_page(struct page *page, int order) { } #endif -struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask); -#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) - struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask); #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__)) @@ -278,25 +252,9 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) dump_stack(); } -/* - * Allocate pages, preferring the node given as nid. The node must be valid and - * online. For more general interface, see alloc_pages_node(). - */ -static inline struct page * -__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) -{ - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - warn_if_node_offline(nid, gfp_mask); - - return __alloc_pages_noprof(gfp_mask, order, nid, NULL); -} - -#define __alloc_pages_node(...) alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__)) - static inline struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp); return __folio_alloc_noprof(gfp, order, nid, NULL); @@ -309,14 +267,7 @@ struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) * prefer the current CPU's closest node. Otherwise node must be valid and * online. */ -static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, - unsigned int order) -{ - if (nid == NUMA_NO_NODE) - nid = numa_mem_id(); - - return __alloc_pages_node_noprof(nid, gfp_mask, order); -} +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order); #define alloc_pages_node(...) alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__)) @@ -395,10 +346,6 @@ extern void free_pages(unsigned long addr, unsigned int order); #define __free_page(page) __free_pages((page), 0) #define free_page(addr) free_pages((addr), 0) -void page_alloc_init_cpuhp(void); -bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); -void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); -void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); void page_alloc_init_late(void); diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 54ca0c88bab6..190191411009 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -136,18 +134,14 @@ enum { * %__GFP_THISNODE forces the allocation to be satisfied from the requested * node with no fallbacks or placement policy enforcements. * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. + * %__GFP_ACCOUNT causes the allocation to be accounted to the active + * cgroup context. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers @@ -320,7 +314,7 @@ enum { * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. * * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. + * accounted to the active cgroup context. * * %GFP_NOWAIT is for kernel allocations that should not stall for direct * reclaim, start physical IO or use any filesystem callback. It is very diff --git a/include/linux/hmm.h b/include/linux/hmm.h index db75ffc949a7..6f04e3932f5b 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -123,6 +123,8 @@ struct hmm_range { * Please see Documentation/mm/hmm.rst for how to use the range API. */ int hmm_range_fault(struct hmm_range *range); +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout); /* * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ad20f7f8c179..c745f7ad2298 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -230,6 +230,7 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, /* Don't have to check pgoff for anonymous vma */ if (!vma_is_anonymous(vma)) { + /* vma_start_pgoff() in mm.h so not available. */ if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, hpage_size >> PAGE_SHIFT)) return false; @@ -390,9 +391,9 @@ static inline bool thp_disabled_by_hw(void) unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags); + vma_flags_t vma_flags); enum split_type { SPLIT_TYPE_UNIFORM, @@ -471,6 +472,24 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, unsigned long address); +/** + * pud_is_huge() - Is this PUD either a huge PUD entry or a software leaf entry? + * @pud: The PUD to check. + * + * This is similar to pmd_is_huge(), but it checks at the PUD level. + * + * Returns: true if this PUD is huge, false otherwise. + */ +static inline bool pud_is_huge(pud_t pud) +{ + if (pud_present(pud)) + return pud_trans_huge(pud); + else if (!pud_none(pud)) + return true; + + return false; +} + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, @@ -529,6 +548,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf); + vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf); extern struct folio *huge_zero_folio; @@ -567,7 +588,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, @@ -614,18 +635,13 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, #define thp_get_unmapped_area NULL static inline unsigned long -thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags) { return 0; } -static inline bool -can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) -{ - return false; -} static inline int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order) @@ -723,6 +739,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, return NULL; } +static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + return 0; +} + static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { return 0; @@ -790,22 +811,21 @@ static inline bool pmd_is_huge(pmd_t pmd) { return false; } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline bool is_pmd_order(unsigned int order) +static inline bool pud_is_huge(pud_t pud) { - return order == HPAGE_PMD_ORDER; + return false; } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline int split_folio_to_list_to_order(struct folio *folio, - struct list_head *list, int new_order) +static inline bool is_pmd_order(unsigned int order) { - return split_huge_page_to_list_to_order(&folio->page, list, new_order); + return order == HPAGE_PMD_ORDER; } static inline int split_folio_to_order(struct folio *folio, int new_order) { - return split_folio_to_list_to_order(folio, NULL, new_order); + return split_huge_page_to_list_to_order(&folio->page, NULL, new_order); } /** diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e9..16c4c4caa126 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -2,6 +2,7 @@ #ifndef _LINUX_HUGETLB_H #define _LINUX_HUGETLB_H +#include <linux/mempolicy.h> #include <linux/mm.h> #include <linux/mm_types.h> #include <linux/mmdebug.h> @@ -154,7 +155,8 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end, bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); void folio_putback_hugetlb(struct folio *folio); -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason); void hugetlb_fix_reserve_counts(struct inode *inode); extern struct mutex *hugetlb_fault_mutex_table; u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); @@ -171,6 +173,7 @@ extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); extern nodemask_t hugetlb_bootmem_nodes; void hugetlb_bootmem_set_nodes(void); @@ -424,7 +427,7 @@ static inline void folio_putback_hugetlb(struct folio *folio) } static inline void move_hugetlb_state(struct folio *old_folio, - struct folio *new_folio, int reason) + struct folio *new_folio, enum migrate_reason reason) { } @@ -673,24 +676,30 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -struct cma; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; - struct cma *cma; -}; - #define HUGE_BOOTMEM_HVO 0x0001 #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 -bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); + +struct mempolicy_interpreted { + int nid; + nodemask_t *nodemask; + enum mempolicy_mode mode; +}; + +enum hugetlb_alloc_flag { + HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT = 0, + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT, +}; + +#define HUGETLB_ALLOC_CHARG_CGROUP_RSVD BIT(HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT) +#define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) + +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, @@ -705,8 +714,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct folio *folio); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); -int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid); bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_add_hstate(unsigned order); @@ -792,8 +801,7 @@ static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, { struct hstate *h = hstate_vma(vma); - return ((address - vma->vm_start) >> huge_page_shift(h)) + - (vma->vm_pgoff >> huge_page_order(h)); + return linear_page_index(vma, address) >> huge_page_order(h); } static inline bool order_is_gigantic(unsigned int order) @@ -956,7 +964,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return modified_mask; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { bool allowed_fallback = false; @@ -1137,9 +1145,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, return NULL; } -static inline int __alloc_bootmem_huge_page(struct hstate *h) +static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid) { - return 0; + return NULL; } static inline struct hstate *hstate_file(struct file *f) @@ -1238,7 +1246,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return 0; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { return false; } @@ -1261,6 +1269,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); +unsigned long huge_pte_dirty(pte_t pte); + static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -1293,6 +1304,10 @@ static inline bool hugetlbfs_pagecache_present( static inline void hugetlb_bootmem_alloc(void) { } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index e5d64b8b59c2..16d72c8c71f6 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -267,5 +267,5 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio, { } -#endif /* CONFIG_MEM_RES_CTLR_HUGETLB */ +#endif /* CONFIG_CGROUP_HUGETLB */ #endif diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 992cd8bd8ed0..4c1476ae3234 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -100,14 +100,29 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) if (pmd_swp_soft_dirty(pmd)) pmd = pmd_swp_clear_soft_dirty(pmd); - if (pmd_swp_uffd_wp(pmd)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (pmd_swp_uffd(pmd)) + pmd = pmd_swp_clear_uffd(pmd); arch_entry = __pmd_to_swp_entry(pmd); /* Temporary until swp_entry_t eliminated. */ return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +/** + * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry. + * @entry: Leaf entry. + * + * This generates an architecture-specific PMD entry that can be utilised to + * encode the metadata the leaf entry encodes. + * + * Returns: Architecture-specific PMD entry encoding leaf entry. + */ +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + /* Temporary until swp_entry_t eliminated. */ + return swp_entry_to_pmd(entry); +} + #else static inline softleaf_t softleaf_from_pmd(pmd_t pmd) @@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return softleaf_mk_none(); } +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + return __pmd(0); +} + #endif /** @@ -567,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_PMD_SOFTLEAVES) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -586,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 621566345406..a6451ecbbe9a 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie); extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); +extern u32 lock_sequence(struct lockdep_map *lock); +#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map) + #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) #define lockdep_assert(cond) \ diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index eae115a26488..55c4b152fedf 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -253,7 +253,8 @@ struct held_lock { unsigned int hardirqs_off:1; unsigned int sync:1; unsigned int references:11; /* 32 bits */ - unsigned int pin_count; + unsigned int pin_count:24; + unsigned int seq_count:8; }; #else /* !CONFIG_LOCKDEP */ diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 4a5631906aff..b723384b3bf4 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -11,7 +11,6 @@ #include <linux/kernel.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> -/* #define CONFIG_MAPLE_RCU_DISABLED */ /* * Allocated nodes are mutable until they have been inserted into the tree, @@ -29,13 +28,11 @@ #define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) #else /* 32bit sizes */ #define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ #define MAPLE_NODE_MASK 255UL @@ -485,6 +482,12 @@ struct ma_state { unsigned char mas_flags; unsigned char end; /* The end of the node */ enum store_type store_type; /* The type of store needed for this operation */ +#ifdef CONFIG_LOCKDEP + u32 ld_seq; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + unsigned long rcu_gp; +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ +#endif /* CONFIG_LOCKDEP */ }; struct ma_wr_state { @@ -572,7 +575,7 @@ void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); @@ -864,9 +867,6 @@ static inline void mt_init(struct maple_tree *mt) static inline bool mt_in_rcu(struct maple_tree *mt) { -#ifdef CONFIG_MAPLE_RCU_DISABLED - return false; -#endif return mt->ma_flags & MT_FLAGS_USE_RCU; } diff --git a/include/linux/memfd.h b/include/linux/memfd.h index b4fda09dab9f..c159e40e3f34 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -14,9 +14,9 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); * to sealing, or 0 otherwise. * * We also update VMA flags if appropriate by manipulating the VMA flags pointed - * to by vm_flags_ptr. + * to by vma_flags_ptr. */ -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr); +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr); struct file *memfd_alloc_file(const char *name, unsigned int flags); int memfd_get_seals(struct file *file); int memfd_add_seals(struct file *file, unsigned int seals); @@ -30,7 +30,7 @@ static inline struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) return ERR_PTR(-EINVAL); } static inline int memfd_check_seals_mmap(struct file *file, - vm_flags_t *vm_flags_ptr) + vma_flags_t *vma_flags_ptr) { return 0; } diff --git a/include/linux/memory.h b/include/linux/memory.h index 463dc02f6cff..1783299073e4 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -20,6 +20,7 @@ #include <linux/compiler.h> #include <linux/mutex.h> #include <linux/memory_hotplug.h> +#include <linux/range.h> #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS) @@ -100,6 +101,32 @@ int arch_get_memory_phys_device(unsigned long start_pfn); unsigned long memory_block_size_bytes(void); int set_memory_block_size_order(unsigned int order); +/** + * memory_block_aligned_range - align a physical address range to memory blocks + * @range: the input range to align + * + * Aligns the start address up and the end address down to memory block + * boundaries. This is required for memory hotplug operations which must + * operate on memory-block aligned ranges. + * + * Returns the aligned range. Callers should check that the returned + * range is valid (aligned.start < aligned.end) before using it. + */ +static inline struct range memory_block_aligned_range(const struct range *range) +{ + struct range aligned; + + aligned.start = ALIGN(range->start, memory_block_size_bytes()); + aligned.end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()); + /* No whole block fits (e.g. range below the first boundary): empty. */ + if (aligned.end <= aligned.start) + aligned.start = aligned.end; + else + aligned.end -= 1; + + return aligned; +} + struct memory_notify { unsigned long start_pfn; unsigned long nr_pages; diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 06c58cb05779..b39605d30896 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -127,6 +127,7 @@ extern int arch_add_memory(int nid, u64 start, u64 size, extern u64 max_mem_size; extern int mhp_online_type_from_str(const char *str); +const char *mhp_online_type_to_str(int online_type); /* If movable_node boot option specified */ extern bool movable_node_enabled; @@ -267,6 +268,8 @@ extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages, extern int remove_memory(u64 start, u64 size); extern void __remove_memory(u64 start, u64 size); extern int offline_and_remove_memory(u64 start, u64 size); +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges); #else static inline void try_offline_node(int nid) {} @@ -283,6 +286,12 @@ static inline int remove_memory(u64 start, u64 size) } static inline void __remove_memory(u64 start, u64 size) {} + +static inline int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + return -EBUSY; +} #endif /* CONFIG_MEMORY_HOTREMOVE */ #ifdef CONFIG_MEMORY_HOTPLUG @@ -294,6 +303,9 @@ extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, mhp_t mhp_flags); +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type); extern int add_memory_driver_managed(int nid, u64 start, u64 size, const char *resource_name, mhp_t mhp_flags); @@ -316,6 +328,8 @@ extern struct zone *zone_for_pfn_range(enum mmop online_type, extern int arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params); void arch_remove_linear_mapping(u64 start, u64 size); +#else +static inline enum mmop mhp_get_default_online_type(void) { return MMOP_OFFLINE; } #endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/include/linux/migrate.h b/include/linux/migrate.h index d5af2b7f577b..78424b3824c2 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -57,8 +57,9 @@ void putback_movable_pages(struct list_head *l); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, - unsigned long private, enum migrate_mode mode, int reason, - unsigned int *ret_succeeded); + unsigned long private, enum migrate_mode mode, + enum migrate_reason reason, + unsigned int *ret_succeeded); struct folio *alloc_migration_target(struct folio *src, unsigned long private); bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode); bool isolate_folio_to_list(struct folio *folio, struct list_head *list); @@ -77,7 +78,8 @@ int set_movable_ops(const struct movable_operations *ops, enum pagetype type); static inline void putback_movable_pages(struct list_head *l) {} static inline int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, + unsigned int *ret_succeeded) { return -ENOSYS; } static inline struct folio *alloc_migration_target(struct folio *src, unsigned long private) diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 265c4328b36a..05102d4d2490 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -25,6 +25,7 @@ enum migrate_reason { MR_LONGTERM_PIN, MR_DEMOTION, MR_DAMON, + MR_NEVER, /* page has never been migrated */ MR_TYPES }; diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..7fabe6c66b4b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -37,6 +37,7 @@ #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/iommu-debug-pagealloc.h> +#include <linux/kcsan-checks.h> struct mempolicy; struct anon_vma; @@ -353,6 +354,7 @@ enum { #endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), + DECLARE_VMA_BIT(UFFD_RWP, 43), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), @@ -440,8 +442,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWSUP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -463,15 +467,18 @@ enum { #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI) #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) +#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) #else #define VM_SHADOW_STACK VM_NONE +#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) #endif #if defined(CONFIG_PPC64) #define VM_SAO INIT_VM_FLAG(SAO) #elif defined(CONFIG_PARISC) #define VM_GROWSUP INIT_VM_FLAG(GROWSUP) +#define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT) #elif defined(CONFIG_SPARC64) #define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI) #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) @@ -483,6 +490,7 @@ enum { #endif #ifndef VM_GROWSUP #define VM_GROWSUP VM_NONE +#define VMA_GROWSUP EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARM64_MTE #define VM_MTE INIT_VM_FLAG(MTE) @@ -496,12 +504,17 @@ enum { #else #define VM_UFFD_MINOR VM_NONE #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) +#else +#define VM_UFFD_RWP VM_NONE +#endif /* - * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on - * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits - * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose - * bitmap cannot hold it. + * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are + * gated on the same configs as their VM_* flags above -- both of which imply + * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build + * whose bitmap cannot hold it. */ #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) @@ -510,6 +523,11 @@ enum { #else #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) +#else +#define VMA_UFFD_RWP EMPTY_VMA_FLAGS +#endif #ifdef CONFIG_64BIT #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) @@ -528,6 +546,8 @@ enum { /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VMA_EXEC_BIT : VMA_READ_BIT) @@ -648,29 +668,32 @@ enum { * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the - * source, as VM_UFFD_WP may not be propagated to destination, while all other - * flags will be. + * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination + * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did + * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd - * write protect handler, which cannot be - * reconstructed on page fault. + * VM_UFFD_RWP write- or read-write-protect handler, which + * cannot be reconstructed on page fault. * - * We always copy pgtables when dst_vma has uffd-wp - * enabled even if it's file-backed - * (e.g. shmem). Because when uffd-wp is enabled, - * pgtable contains uffd-wp protection information, - * that's something we can't retrieve from page cache, - * and skip copying will lose those info. + * We always copy pgtables when dst_vma has the + * uffd PTE bit in use even if it's file-backed + * (e.g. shmem). Because when the uffd bit is + * in use, the pgtable contains the protection + * information, that's something we can't + * retrieve from page cache, and skip copying + * will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables * only and thus cannot be reconstructed on page * fault. */ -#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD) +#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ + VM_MAYBE_GUARD) /* * mapping from the currently active vm_flags protection bits (the @@ -717,6 +740,9 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags) { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } +/* /dev/zero minor device number. Special due to MAP_PRIVATE semantics. */ +#define DEVZERO_MINOR 5 + /* * vm_fault is filled by the pagefault handler and passed to the vma's * ->fault function. The vma's ->fault is responsible for returning a bitmask @@ -1528,12 +1554,7 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) -{ - desc->vm_ops = NULL; -} - -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -1563,11 +1584,24 @@ static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) vma->vm_end >= vma->vm_mm->start_stack; } -static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) +static inline bool vma_flags_can_grow(const vma_flags_t *flags) +{ + if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) + return true; + if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) + return true; + + return false; +} + +static inline bool vma_can_grow(const struct vm_area_struct *vma) { - int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); + return vma_flags_can_grow(&vma->flags); +} - if (!maybe_stack) +static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) +{ + if (!vma_can_grow(vma)) return false; if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == @@ -2286,22 +2320,30 @@ static inline int page_zone_id(struct page *page) } #ifdef NODE_NOT_IN_PAGE_FLAGS -int memdesc_nid(memdesc_flags_t mdf); +int memdesc_nid(const memdesc_flags_t *mdf); +#else +#ifdef CONFIG_NUMA +static inline int memdesc_nid(const memdesc_flags_t *mdf) +{ + ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); + return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; +} #else -static inline int memdesc_nid(memdesc_flags_t mdf) +static inline int memdesc_nid(const memdesc_flags_t *mdf) { - return (mdf.f >> NODES_PGSHIFT) & NODES_MASK; + return 0; } #endif +#endif static inline int page_to_nid(const struct page *page) { - return memdesc_nid(PF_POISONED_CHECK(page)->flags); + return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); } static inline int folio_nid(const struct folio *folio) { - return memdesc_nid(folio->flags); + return memdesc_nid(&folio->flags); } #ifdef CONFIG_NUMA_BALANCING @@ -2541,12 +2583,13 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { - return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); + return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; } @@ -3317,6 +3360,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen); #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE) +/* Whether this change is for uffd RWP */ +#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ +#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ +#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ + MM_CP_UFFD_RWP_RESOLVE) bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte); @@ -3967,8 +4015,12 @@ extern unsigned long free_reserved_area(void *start, void *end, extern void adjust_managed_page_count(struct page *page, long count); -/* Free the reserved page into the buddy system, so it gets managed. */ -void free_reserved_page(struct page *page); +void free_reserved_pages(struct page *page, unsigned int order); + +static inline void free_reserved_page(struct page *page) +{ + free_reserved_pages(page, 0); +} static inline void mark_page_reserved(struct page *page) { @@ -4042,7 +4094,7 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); extern void mem_init(void); extern void __init mmap_init(void); -extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); static inline void show_mem(void) { __show_mem(0, NULL, MAX_NR_ZONES - 1); @@ -4052,7 +4104,7 @@ extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); extern __printf(3, 4) -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); extern void setup_per_cpu_pageset(void); @@ -4061,44 +4113,46 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *node, - struct rb_root_cached *root); -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping); +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root); -void vma_interval_tree_remove(struct vm_area_struct *node, - struct rb_root_cached *root); -struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, - unsigned long start, unsigned long last); - -#define vma_interval_tree_foreach(vma, root, start, last) \ - for (vma = vma_interval_tree_iter_first(root, start, last); \ - vma; vma = vma_interval_tree_iter_next(vma, start, last)) - -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root); -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root); + struct address_space *mapping); +void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping); +struct vm_area_struct * +mapping_rmap_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last); +struct vm_area_struct * +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); + +#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ + pgoff_last); \ + vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ + pgoff_last)) + +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +struct anon_vma_chain * +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct anon_vma_chain *anon_vma_interval_tree_iter_next( - struct anon_vma_chain *node, unsigned long start, unsigned long last); +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last); #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node); +void anon_rmap_tree_verify(struct anon_vma_chain *avc); #endif -#define anon_vma_interval_tree_foreach(avc, root, start, last) \ - for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ - avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) +#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ + for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ + avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last)) /* mmap.c */ extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); -extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void exit_mmap(struct mm_struct *); bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, bool write); @@ -4139,18 +4193,20 @@ unsigned long randomize_page(unsigned long start, unsigned long range); unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); + unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags); static inline unsigned long get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return __get_unmapped_area(file, addr, len, pgoff, flags, 0); + return __get_unmapped_area(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } -extern unsigned long do_mmap(struct file *file, unsigned long addr, +unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, @@ -4278,9 +4334,119 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } -static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) +/** + * vma_start_pgoff() - Get the page offset of the start of @vma + * @vma: The VMA whose page offset is required. + * + * If the VMA is file-backed, this is the page offset into the file. + * + * If @vma is anonymous, this is the virtual page offset of the start of the + * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the + * virtual page offset at the time of first fault. + * + * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the + * page offset within the file. + * + * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions + * have their page offset set to the first PFN in the range. + * + * Returns: The page offset of the start of @vma. + */ +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) { - return vma->vm_pgoff + vma_pages(vma) - 1; + return vma->vm_pgoff; +} + +/** + * vma_end_pgoff() - Get the page offset of the exclusive end of @vma + * @vma: The VMA whose end page offset is required. + * + * This returns the exclusive end page offset of @vma, which is useful for + * expressing page offset ranges. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The exclusive end page offset of @vma. + */ +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_pgoff() - Get the page offset of the last page in @vma + * @vma: The VMA whose last page offset is required. + * + * This returns the last page offset contained within @vma. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The last page offset of @vma. + */ +static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_pgoff(vma) - 1; +} + +/** + * vma_start_virt_pgoff() - Get the virtual page offset of the start of @vma + * @vma: The VMA whose virtual page offset is required. + * + * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the + * virtual page offset at the time of first fault. + * + * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). + * + * This value is used for tracking MAP_PRIVATE file-backed mappings by their + * virtual page offset. + * + * Returns: The virtual page offset of the start of @vma. + */ +static inline pgoff_t vma_start_virt_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_virt_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_virt_pgoff_lo; + return pgoff; +} + +/** + * vma_end_virt_pgoff() - Get the virtual page offset of the exclusive end of + * @vma. + * @vma: The VMA whose end virtual page offset is required. + * + * This returns the virtual exclusive end page offset of @vma, which is useful + * for expressing page offset ranges. + * + * See the description of vma_start_virt_pgoff() for a description of VMA + * virtual page offsets. + * + * Returns: The exclusive end virtual page offset of @vma. + */ +static inline pgoff_t vma_end_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_virt_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_virt_pgoff() - Get the virtual page offset of the last page in + * @vma. + * @vma: The VMA whose last virtual page offset is required. + * + * See the description of vma_start_virt_pgoff() for a description of VMA + * virtual page offsets. + * + * Returns: The last virtual page offset of @vma. + */ +static inline pgoff_t vma_last_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_virt_pgoff(vma) - 1; } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4497,26 +4663,35 @@ static inline bool range_in_vma_desc(const struct vm_area_desc *desc, #ifdef CONFIG_MMU pgprot_t vm_get_page_prot(vm_flags_t vm_flags); -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); return vm_get_page_prot(vm_flags); } +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} + void vma_set_page_prot(struct vm_area_struct *vma); #else static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { return __pgprot(0); } -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) +{ + return __pgprot(0); +} +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) { return __pgprot(0); } static inline void vma_set_page_prot(struct vm_area_struct *vma) { - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); } #endif @@ -4625,12 +4800,26 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) /* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether - * a (NUMA hinting) fault is required. + * a (NUMA hinting or userfaultfd RWP) fault is required. */ static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags) { /* + * VM_UFFD_RWP uses protnone as an access-tracking marker, not for + * NUMA hinting. GUP must always take a fault so the access is + * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. + * + * Only do so while the VMA is accessible. If it has been made + * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard + * below: forcing a fault there would loop, as handle_mm_fault() makes + * no progress on protnone in an inaccessible VMA, and the access is + * denied regardless of RWP anyway. + */ + if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) + return false; + + /* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. */ @@ -5158,13 +5347,9 @@ int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t * int reserve_mem_release_by_name(const char *name); #ifdef CONFIG_64BIT -int do_mseal(unsigned long start, size_t len_in, unsigned long flags); +void mseal_mmap_page_zero(void); #else -static inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags) -{ - /* noop on 32 bit */ - return 0; -} +static inline void mseal_mmap_page_zero(void) {} #endif /* diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index a8430a7ae054..b5c4dc0f3fe3 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -600,14 +600,14 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, return false; /* A uffd-wp wr-protected normal pte */ - if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval))) + if (unlikely(pte_present(pteval) && pte_uffd(pteval))) arm_uffd_pte = true; /* * A uffd-wp wr-protected swap pte. Note: this should even cover an * existing pte marker with uffd-wp bit set. */ - if (unlikely(pte_swp_uffd_wp_any(pteval))) + if (unlikely(pte_swp_uffd_any(pteval))) arm_uffd_pte = true; if (unlikely(arm_uffd_pte)) { @@ -650,7 +650,7 @@ static inline bool vma_has_recency(const struct vm_area_struct *vma) static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) { struct page *cur_page = pages[0]; - unsigned long section = memdesc_section(cur_page->flags); + unsigned long section = memdesc_section(&cur_page->flags); size_t i; for (i = 1; i < nr_pages; i++) { @@ -660,7 +660,7 @@ static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) * In unproblematic kernel configs, page_to_section() == 0 and * the whole check will get optimized out. */ - if (memdesc_section(cur_page->flags) != section) + if (memdesc_section(&cur_page->flags) != section) break; } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b18c2b2e7d2c..b5d4cd3b067b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -368,6 +368,8 @@ typedef unsigned short mm_id_t; * dax_associate_entry. * @private: Filesystem per-folio data (see folio_attach_private()). * @swap: Used for swp_entry_t if folio_test_swapcache(). + * @migrate_info: Stores migration state (anon_vma pointer and + * FOLIO_WAS_* markers). * @_mapcount: Do not access this member directly. Use folio_mapcount() to * find out how many times this folio is mapped by userspace. * @_refcount: Do not access this member directly. Use folio_ref_count() @@ -427,6 +429,7 @@ struct folio { union { void *private; swp_entry_t swap; + unsigned long migrate_info; }; atomic_t _mapcount; atomic_t _refcount; @@ -965,6 +968,11 @@ struct vm_area_struct { unsigned int vm_lock_seq; #endif /* + * Low 32-bits of virtual page offset. + * See vma_start_virt_pgoff() comment for details. + */ + unsigned int __vm_virt_pgoff_lo; + /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma * list, after a COW of one of the file pages. A MAP_SHARED vma * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack @@ -1039,6 +1047,13 @@ struct vm_area_struct { struct lockdep_map vmlock_dep_map; #endif #endif +#ifdef CONFIG_64BIT + /* + * High 32-bits of virtual page offset. + * See vma_start_virt_pgoff() comment for details. + */ + unsigned int __vm_virt_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 047f5f5e2c34..87f77e3da77f 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -506,6 +506,8 @@ static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } static inline void vma_assert_write_locked(struct vm_area_struct *vma) { mmap_assert_write_locked(vma->vm_mm); } +static inline bool vma_is_attached(struct vm_area_struct *vma) + { return true; } static inline void vma_assert_attached(struct vm_area_struct *vma) {} static inline void vma_assert_detached(struct vm_area_struct *vma) {} static inline void vma_mark_attached(struct vm_area_struct *vma) {} @@ -530,6 +532,12 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma) #endif /* CONFIG_PER_VMA_LOCK */ +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} + static inline void mmap_write_lock(struct mm_struct *mm) { __mmap_lock_trace_start_locking(mm, true); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ca2712187147..0507193b3ae3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2,7 +2,7 @@ #ifndef _LINUX_MMZONE_H #define _LINUX_MMZONE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __GENERATING_BOUNDS_H #include <linux/spinlock.h> @@ -1272,31 +1272,33 @@ static inline bool zone_is_empty(const struct zone *zone) #define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) -static inline enum zone_type memdesc_zonenum(memdesc_flags_t flags) +static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags) { - ASSERT_EXCLUSIVE_BITS(flags.f, ZONES_MASK << ZONES_PGSHIFT); - return (flags.f >> ZONES_PGSHIFT) & ZONES_MASK; +#if ZONES_WIDTH != 0 + ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT); +#endif + return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK; } static inline enum zone_type page_zonenum(const struct page *page) { - return memdesc_zonenum(page->flags); + return memdesc_zonenum(&page->flags); } static inline enum zone_type folio_zonenum(const struct folio *folio) { - return memdesc_zonenum(folio->flags); + return memdesc_zonenum(&folio->flags); } #ifdef CONFIG_ZONE_DEVICE -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return memdesc_zonenum(mdf) == ZONE_DEVICE; } static inline struct dev_pagemap *page_pgmap(const struct page *page) { - VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(page->flags), page); + VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap; } @@ -1311,9 +1313,9 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b) { - if (memdesc_is_zone_device(a->flags) != memdesc_is_zone_device(b->flags)) + if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; - if (!memdesc_is_zone_device(a->flags)) + if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b); } @@ -1321,7 +1323,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a, extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *); #else -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return false; } @@ -1338,12 +1340,12 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool is_zone_device_page(const struct page *page) { - return memdesc_is_zone_device(page->flags); + return memdesc_is_zone_device(&page->flags); } static inline bool folio_is_zone_device(const struct folio *folio) { - return memdesc_is_zone_device(folio->flags); + return memdesc_is_zone_device(&folio->flags); } static inline bool is_zone_movable_page(const struct page *page) @@ -1815,7 +1817,7 @@ static inline int zonelist_node_idx(const struct zoneref *zoneref) struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes); + const nodemask_t *nodes); /** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point @@ -1834,7 +1836,7 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z, */ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; @@ -1860,7 +1862,7 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, */ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes); @@ -2156,8 +2158,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) } void sparse_vmemmap_init_nid_early(int nid); -void sparse_vmemmap_init_nid_late(int nid); - #else static inline int preinited_vmemmap_section(const struct mem_section *section) { @@ -2166,10 +2166,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) static inline void sparse_vmemmap_init_nid_early(int nid) { } - -static inline void sparse_vmemmap_init_nid_late(int nid) -{ -} #endif static inline int online_section_nr(unsigned long nr) @@ -2374,7 +2370,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #else #define sparse_vmemmap_init_nid_early(_nid) do {} while (0) -#define sparse_vmemmap_init_nid_late(_nid) do {} while (0) #define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ @@ -2389,5 +2384,5 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #endif /* !__GENERATING_BOUNDS.H */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _LINUX_MMZONE_H */ diff --git a/include/linux/oom.h b/include/linux/oom.h index 7b02bc1d0a7e..00da05d227e6 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -30,7 +30,7 @@ struct oom_control { struct zonelist *zonelist; /* Used to determine mempolicy */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* Memory cgroup in which oom is invoked, or NULL for global oom */ struct mem_cgroup *memcg; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7223f6f4e2b4..7a863572adce 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -879,20 +879,6 @@ FOLIO_FLAG_FALSE(partially_mapped) #define PG_head_mask ((1UL << PG_head)) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * PageTransCompound returns true for both transparent huge pages - * and hugetlbfs pages, so it should only be called when it's known - * that hugetlbfs pages aren't involved. - */ -static inline int PageTransCompound(const struct page *page) -{ - return PageCompound(page); -} -#else -TESTPAGEFLAG_FALSE(TransCompound, transcompound) -#endif - #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE) /* * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 3328357f6dba..8188ddc5c412 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -3,6 +3,7 @@ #define __LINUX_PAGE_OWNER_H #include <linux/jump_label.h> +#include <linux/migrate_mode.h> #ifdef CONFIG_PAGE_OWNER extern struct static_key_false page_owner_inited; @@ -14,7 +15,7 @@ extern void __set_page_owner(struct page *page, extern void __split_page_owner(struct page *page, int old_order, int new_order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); -extern void __folio_set_owner_migrate_reason(struct folio *folio, int reason); +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason); extern void __dump_page_owner(const struct page *page); extern void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone); @@ -43,7 +44,7 @@ static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) if (static_branch_unlikely(&page_owner_inited)) __folio_copy_owner(newfolio, old); } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { if (static_branch_unlikely(&page_owner_inited)) __folio_set_owner_migrate_reason(folio, reason); @@ -68,7 +69,7 @@ static inline void split_page_owner(struct page *page, int old_order, static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) { } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { } static inline void dump_page_owner(const struct page *page) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2c3718d592d6..4e8b2b29f6d3 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1063,15 +1063,110 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) return folio->index; } +/** + * linear_page_delta() - Determine the relative page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose relative page offset is required. + * + * The result is identical for both file-backed and anonymous mappings and + * simply determines how many pages @address lies from @vma->vm_start. + * + * Returns: The number of pages @address is offset by within @vma. + */ +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +/** + * linear_page_index() - Determine the absolute page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * See the comment for vma_start_pgoff() for a description of what the page + * offset signifies. + * + * Returns: The absolute page offset of @address within @vma. + */ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_pgoff(vma); + return pgoff; +} + +static inline pgoff_t __linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_virt_pgoff(vma); + return pgoff; +} + +/** + * linear_virt_page_index() - Determine the absolute virtual page offset of + * @address within @vma. + * @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * This returns the virtual page offset of @address, which is the page offset + * the address possessed at the time the VMA was first faulted. + * + * For anonymous mappings, this returns the same value as linear_page_index(). + * + * For MAP_PRIVATE file-backed mappings, this returns the virtual page offset of + * @address, which is the page offset the address possessed at the time the VMA + * was first faulted. + * + * It is not valid to call this function for shared file-backed mappings. + * + * Returns: The absolute virtual page offset of @address within @vma. + */ +static inline pgoff_t linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_virt_page_index(vma, address); + + VM_WARN_ON_ONCE(vma_test(vma, VMA_SHARED_BIT)); + if (vma_is_anonymous(vma)) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); + return pgoff; } +/** + * linear_folio_page_index() - Determine the absolute page offset of + * @address within @vma from @folio. + * @folio: The folio whose linear page index is sought. + * @vma: The VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * Determines whether to obtain the virtual linear page index based on whether + * @folio is anonymous or not. + * + * See the descriptions of linear_virt_page_index() and linear_page_index() for + * details of each. + * + * Returns: The absolute page offset of @address within @vma. + */ +static inline pgoff_t linear_folio_page_index(const struct folio *folio, + const struct vm_area_struct *vma, + const unsigned long address) +{ + if (folio_test_anon(folio)) + return linear_virt_page_index(vma, address); + + return linear_page_index(vma, address); +} + struct wait_page_key { struct folio *folio; int bit_nr; @@ -1219,7 +1314,7 @@ static inline vm_fault_t folio_lock_or_retry(struct folio *folio, void folio_wait_bit(struct folio *folio, int bit_nr); int folio_wait_bit_killable(struct folio *folio, int bit_nr); -/* +/* * Wait for a folio to be unlocked. * * This must be called with the caller "holding" the folio, diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 43c854a273c3..2cba7cc2b01f 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -203,7 +203,7 @@ /* * Accessors and operations. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating @@ -514,5 +514,5 @@ do { \ #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_PERCPU_DEFS_H */ diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index d73a1c08c3e3..1e3212e2c827 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -12,8 +12,8 @@ * start shutting down you call percpu_ref_kill() _before_ dropping the initial * refcount. * - * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less - * than an atomic_t - this is because of the way shutdown works, see + * The refcount will have a range of 0 to LONG_MAX, i.e. one bit less + * than an atomic_long_t - this is because of the way shutdown works, see * percpu_ref_kill()/PERCPU_COUNT_BIAS. * * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the @@ -269,6 +269,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) /** * percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the * caller is responsible for taking RCU. + * @ref: percpu_ref to try-get * * This function is safe to call as long as @ref is between init and exit. */ diff --git a/include/linux/pfn.h b/include/linux/pfn.h index b90ca0b6c331..cfedf0f61bb3 100644 --- a/include/linux/pfn.h +++ b/include/linux/pfn.h @@ -2,7 +2,7 @@ #ifndef _LINUX_PFN_H_ #define _LINUX_PFN_H_ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #endif diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2981e386da7b..8c093c119e5a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -8,7 +8,7 @@ #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT) #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_MMU #include <linux/mm_types.h> @@ -988,49 +988,30 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma, update_mmu_tlb_range(vma, address, ptep, 1); } -/* - * Some architectures may be able to avoid expensive synchronization - * primitives when modifications are made to PTE's which are already - * not present, or in the process of an address space destruction. - */ -#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -static inline void pte_clear_not_present_full(struct mm_struct *mm, - unsigned long address, - pte_t *ptep, - int full) -{ - pte_clear(mm, address, ptep); -} -#endif - -#ifndef clear_not_present_full_ptes /** - * clear_not_present_full_ptes - Clear multiple not present PTEs which are - * consecutive in the pgtable. + * clear_nonpresent_ptes - Clear multiple non-present PTEs which are + * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. - * @full: Whether we are clearing a full mm. - * - * May be overridden by the architecture; otherwise, implemented as a simple - * loop over pte_clear_not_present_full(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */ -static inline void clear_not_present_full_ptes(struct mm_struct *mm, - unsigned long addr, pte_t *ptep, unsigned int nr, int full) +static inline void clear_nonpresent_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) { + (void)addr; + for (;;) { - pte_clear_not_present_full(mm, addr, ptep, full); + pte_clear(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; } } -#endif #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, @@ -1858,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; @@ -2129,18 +2110,26 @@ static inline int pud_trans_unstable(pud_t *pud) return 0; } -#ifndef CONFIG_NUMA_BALANCING +#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE /* - * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is - * perfectly valid to indicate "no" in that case, which is why our default - * implementation defaults to "always no". + * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It + * is perfectly valid to indicate "no" in that case, which is why our + * default implementation defaults to "always no". + * + * In an accessible VMA, pte_protnone() reliably indicates a present + * PROT_NONE page protection. Today the kernel uses such PTEs for two + * purposes: NUMA hinting faults, and userfaultfd RWP tracking on + * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and + * the VMA flag; see include/linux/userfaultfd_k.h. * - * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE - * page protection due to NUMA hinting. NUMA hinting faults only apply in - * accessible VMAs. + * So, to reliably identify PROT_NONE PTEs that require kernel handling, + * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is + * sufficient. * - * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, - * looking at the VMA accessibility is sufficient. + * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero + * stubs below; PAGE_NONE references that survive to runtime fire the + * BUILD_BUG() fallback, since callers should have folded such paths to + * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). */ static inline int pte_protnone(pte_t pte) { @@ -2151,7 +2140,11 @@ static inline int pmd_protnone(pmd_t pmd) { return 0; } -#endif /* CONFIG_NUMA_BALANCING */ + +#ifndef PAGE_NONE +#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; }) +#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #endif /* CONFIG_MMU */ @@ -2320,7 +2313,7 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) #ifdef CONFIG_PHYS_ADDR_T_64BIT diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index eae67015ce51..057edc4cbb6e 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -111,7 +111,7 @@ struct radix_tree_iter { }; /** - * Radix-tree synchronization + * DOC: Radix-tree synchronization * * The radix-tree API requires that users provide all synchronisation (with * specific exceptions, noted below). @@ -182,6 +182,7 @@ static inline void *radix_tree_deref_slot(void __rcu **slot) /** * radix_tree_deref_slot_protected - dereference a slot with tree lock held * @slot: slot pointer, returned by radix_tree_lookup_slot + * @treelock: caller must hold this spinlock * * Similar to radix_tree_deref_slot. The caller does not hold the RCU read * lock but it must hold the tree lock to prevent parallel updates. @@ -306,7 +307,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) * Also it fills @iter with data about chunk: position in the tree (index), * its end (next_index), and constructs a bit mask for tagged iterating (tags). */ -void __rcu **radix_tree_next_chunk(const struct radix_tree_root *, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned flags); /** diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f00..a174758f7777 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -843,7 +843,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio, * Called from mm/vmscan.c to handle paging out */ int folio_referenced(struct folio *, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags); + struct mem_cgroup *memcg, vma_flags_t *vma_flags); void try_to_migrate(struct folio *folio, enum ttu_flags flags); void try_to_unmap(struct folio *, enum ttu_flags flags); @@ -871,6 +871,7 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; + bool is_anon_walk; }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -881,6 +882,7 @@ struct page_vma_mapped_walk { .vma = _vma, \ .address = _address, \ .flags = _flags, \ + .is_anon_walk = folio_test_anon(_folio), \ } static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) @@ -975,10 +977,9 @@ struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, #define anon_vma_prepare(vma) (0) static inline int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, - vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { - *vm_flags = 0; + vma_flags_clear_all(vma_flags); return 0; } diff --git a/include/linux/sched.h b/include/linux/sched.h index 373bcc0598d1..14d5ce8dd613 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1288,6 +1288,7 @@ struct task_struct { u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; + unsigned int lockdep_seq; struct held_lock held_locks[MAX_LOCK_DEPTH]; #endif diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df584..10be8a54b416 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -193,21 +193,18 @@ unsigned long mm_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, - unsigned long addr, - unsigned long len, - unsigned long pgoff, - unsigned long flags, - vm_flags_t vm_flags); +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); #else static inline void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) {} diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index e729b9b0e38d..5663dff53186 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -12,8 +12,6 @@ #include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb; - /* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping) void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, - struct list_head *folio_list); +int shmem_write_folio(struct folio *folio); void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end); int shmem_unuse(unsigned int type); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 22eda1d54a0e..4431b026e429 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3573,7 +3573,7 @@ static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask, * 3. If requesting a order 0 page it will not be compound * due to the check to see if order has a value in prep_new_page * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to - * code in gfp_to_alloc_flags that should be enforcing this. + * code in alloc_flags_slowpath() that should be enforcing this. */ gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; diff --git a/include/linux/swap.h b/include/linux/swap.h index 8f0f68e245ba..0544b2ec4c56 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -16,10 +16,6 @@ #include <uapi/linux/mempolicy.h> #include <asm/page.h> -struct notifier_block; - -struct bio; - #define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */ #define SWAP_FLAG_PRIO_MASK 0x7fff #define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */ @@ -29,7 +25,6 @@ struct bio; #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) -#define SWAP_BATCH 64 static inline int current_is_kswapd(void) { @@ -175,7 +170,6 @@ static inline void mm_account_reclaimed_pages(unsigned long pages) struct address_space; struct sysinfo; -struct writeback_control; struct zone; /* @@ -208,7 +202,6 @@ enum { SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */ SWP_BLKDEV = (1 << 6), /* its a block device */ SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */ - SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */ SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */ SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */ @@ -282,6 +275,7 @@ struct swap_info_struct { struct work_struct reclaim_work; /* reclaim worker */ struct list_head discard_clusters; /* discard clusters list */ struct plist_node avail_list; /* entry in swap_avail_head */ + const struct swap_ops *ops; }; static inline swp_entry_t page_swap_entry(struct page *page) @@ -293,39 +287,19 @@ static inline swp_entry_t page_swap_entry(struct page *page) return entry; } -/* linux/mm/workingset.c */ -bool workingset_test_recent(void *shadow, bool file, bool *workingset, - bool flush); -void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); -void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); -void workingset_refault(struct folio *folio, void *shadow); -void workingset_activation(struct folio *folio); - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; /* Definition of global_zone_page_state not available yet */ #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES) +/* linux/mm/folio.c */ +void folio_add_lru(struct folio *folio); +void folio_mark_accessed(struct folio *folio); +void lru_add_drain_all(void); -/* linux/mm/swap.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated); -void lru_note_cost_refault(struct folio *); -void folio_add_lru(struct folio *); -void folio_add_lru_vma(struct folio *, struct vm_area_struct *); -void mark_page_accessed(struct page *); -void folio_mark_accessed(struct folio *); - -static inline bool folio_may_be_lru_cached(struct folio *folio) -{ - /* - * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. - * Holding small numbers of low-order mTHP folios in per-CPU LRU cache - * will be sensible, but nobody has implemented and tested that yet. - */ - return !folio_test_large(folio); -} +/* linux/mm/folio-compat.c */ +void mark_page_accessed(struct page *page); extern atomic_t lru_disable_count; @@ -334,43 +308,6 @@ static inline bool lru_cache_disabled(void) return atomic_read(&lru_disable_count); } -static inline void lru_cache_enable(void) -{ - atomic_dec(&lru_disable_count); -} - -extern void lru_cache_disable(void); -extern void lru_add_drain(void); -extern void lru_add_drain_cpu(int cpu); -extern void lru_add_drain_cpu_zone(struct zone *zone); -extern void lru_add_drain_all(void); -void folio_deactivate(struct folio *folio); -void folio_mark_lazyfree(struct folio *folio); -extern void swap_setup(void); - -/* linux/mm/vmscan.c */ -extern unsigned long zone_reclaimable_pages(struct zone *zone); -extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *mask); -unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); - -#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) -#define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 - -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) - -extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - unsigned long nr_pages, - gfp_t gfp_mask, - unsigned int reclaim_options, - int *swappiness); -extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); extern unsigned long shrink_all_memory(unsigned long nr_pages); extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); @@ -391,11 +328,6 @@ static inline void reclaim_unregister_node(struct node *node) } #endif /* CONFIG_SYSFS && CONFIG_NUMA */ -#ifdef CONFIG_NUMA -extern int sysctl_min_unmapped_ratio; -extern int sysctl_min_slab_ratio; -#endif - void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); @@ -403,6 +335,7 @@ extern void __meminit kswapd_stop(int nid); #ifdef CONFIG_SWAP +int swap_fs_activate(struct swap_info_struct *sis); int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block); int generic_swapfile_activate(struct swap_info_struct *, struct file *, @@ -442,7 +375,6 @@ extern sector_t swapdev_block(int, pgoff_t); extern int __swap_count(swp_entry_t entry); extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); -struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); @@ -529,6 +461,10 @@ static inline bool folio_free_swap(struct folio *folio) return false; } +static inline int swap_fs_activate(struct swap_info_struct *sis) +{ + return -EINVAL; +} static inline int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block) @@ -552,7 +488,7 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); #else -static inline int mem_cgroup_swappiness(struct mem_cgroup *mem) +static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { return READ_ONCE(vm_swappiness); } diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..c956bc445ee0 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -73,8 +73,8 @@ static inline pte_t pte_swp_clear_flags(pte_t pte) pte = pte_swp_clear_exclusive(pte); if (pte_swp_soft_dirty(pte)) pte = pte_swp_clear_soft_dirty(pte); - if (pte_swp_uffd_wp(pte)) - pte = pte_swp_clear_uffd_wp(pte); + if (pte_swp_uffd(pte)) + pte = pte_swp_clear_uffd(pte); return pte; } @@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); @@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 68edac4dcd78..bfbd6a59909f 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -21,10 +21,11 @@ #include <linux/hugetlb_inline.h> /* The set of all possible UFFD-related VM flags. */ -#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) +#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \ + VM_UFFD_WP | VM_UFFD_RWP) #define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \ - VMA_UFFD_MINOR) + VMA_UFFD_MINOR, VMA_UFFD_RWP) /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining @@ -149,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); +extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp); /* move_pages */ void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); @@ -168,7 +171,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, /* * Never enable huge pmd sharing on some uffd registered vmas: * - * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. + * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access + * tracking information is per pgtable entry. * * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for * VMAs which share huge pmds. (If you have two mappings to the same @@ -178,51 +182,80 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, */ static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } /* - * Don't do fault around for either WP or MINOR registered uffd range. For + * Don't do fault around for WP, RWP or MINOR registered uffd range. For * MINOR registered range, fault around will be a total disaster and ptes can * be installed without notifications; for WP it should mostly be fine as long * as the fault around checks for pte_none() before the installation, however - * to be super safe we just forbid it. + * to be super safe we just forbid it; for RWP, pre-faulted neighbours would + * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED) + * and pollute the tracked working set, so each page must be populated by its + * own fault. */ static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } static inline bool userfaultfd_missing(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MISSING; + return vma_test_any_mask(vma, VMA_UFFD_MISSING); } static inline bool userfaultfd_wp(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_WP; + return vma_test_any_mask(vma, VMA_UFFD_WP); } static inline bool userfaultfd_minor(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MINOR; + return vma_test_any_mask(vma, VMA_UFFD_MINOR); +} + +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return vma_test_single_mask(vma, VMA_UFFD_RWP); +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return userfaultfd_wp(vma) || userfaultfd_rwp(vma); } static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { - return userfaultfd_wp(vma) && pte_uffd_wp(pte); + return userfaultfd_wp(vma) && pte_uffd(pte); } static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd) { - return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); + return userfaultfd_wp(vma) && pmd_uffd(pmd); +} + +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return userfaultfd_rwp(vma) && pte_uffd(pte); +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return userfaultfd_rwp(vma) && pmd_uffd(pmd); } static inline bool userfaultfd_armed(struct vm_area_struct *vma) { - return vma->vm_flags & __VM_UFFD_FLAGS; + return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); } static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) @@ -253,6 +286,7 @@ extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); extern bool userfaultfd_wp_async(struct vm_area_struct *vma); +extern bool userfaultfd_rwp_async(struct vm_area_struct *vma); static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) { @@ -272,10 +306,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { if (!uffd_supports_wp_marker()) return false; @@ -283,7 +317,7 @@ static inline bool pte_swp_uffd_wp_any(pte_t pte) if (pte_present(pte)) return false; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) return true; if (pte_is_uffd_wp_marker(pte)) @@ -328,6 +362,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return false; +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return false; +} + static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { @@ -340,6 +384,17 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, return false; } +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return false; +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return false; +} static inline bool userfaultfd_armed(struct vm_area_struct *vma) { @@ -410,6 +465,11 @@ static inline bool userfaultfd_wp_async(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return false; +} + static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) { return false; @@ -421,10 +481,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { return false; } diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 03fe95f5a020..2628ccda076a 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST, #endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */ NR_VM_EVENT_ITEMS }; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index d87dc7f77f4e..e4d8d0a9f30f 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -310,14 +310,14 @@ static inline void set_vm_flush_reset_perms(void *addr) {} #if defined(CONFIG_MMU) && defined(CONFIG_SMP) struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align); + size_t align, gfp_t gfp); void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms); # else static inline struct vm_struct ** pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { return NULL; } diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h index faecd5522401..b4d13457bc2a 100644 --- a/include/linux/vmpressure.h +++ b/include/linux/vmpressure.h @@ -13,18 +13,31 @@ struct vmpressure { unsigned long scanned; unsigned long reclaimed; + /* The lock is used to keep the scanned/reclaimed in sync. */ + spinlock_t sr_lock; +#ifdef CONFIG_MEMCG_V1 + /* + * tree=true accumulators feed the v1 userspace eventfd interface + * (memory.pressure_level). Drained by @work. v2 has no equivalent + * interface, so this state is omitted on CONFIG_MEMCG_V1=n builds. + */ unsigned long tree_scanned; unsigned long tree_reclaimed; - /* The lock is used to keep the scanned/reclaimed above in sync. */ - spinlock_t sr_lock; - /* The list of vmpressure_event structs. */ struct list_head events; /* Have to grab the lock on events traversal or modifications. */ struct mutex events_lock; struct work_struct work; +#endif +}; + +enum vmpressure_levels { + VMPRESSURE_LOW = 0, + VMPRESSURE_MEDIUM, + VMPRESSURE_CRITICAL, + VMPRESSURE_NUM_LEVELS, }; struct mem_cgroup; @@ -32,18 +45,41 @@ struct mem_cgroup; #ifdef CONFIG_MEMCG void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed); -extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); - extern void vmpressure_init(struct vmpressure *vmpr); extern void vmpressure_cleanup(struct vmpressure *vmpr); extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg); extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr); + +/* Shared with the v1 vmpressure block in mm/memcontrol-v1.c. */ +extern const unsigned long vmpressure_win; +extern enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed); + +#ifdef CONFIG_MEMCG_V1 +extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); extern int vmpressure_register_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); extern void vmpressure_unregister_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); + +/* v1 hooks called from mm/vmpressure.c; no-ops below when !MEMCG_V1. */ +extern void vmpressure_v1_init(struct vmpressure *vmpr); +extern void vmpressure_v1_cleanup(struct vmpressure *vmpr); +extern void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed); #else +static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, + int prio) {} +static inline void vmpressure_v1_init(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_cleanup(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) {} +#endif /* CONFIG_MEMCG_V1 */ + +#else /* !CONFIG_MEMCG */ static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed) {} diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 62552a2ce5b9..b749a9a5a5ee 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -244,7 +244,7 @@ void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, struct inode *inode); /** - * wbc_init_bio - writeback specific initializtion of bio + * wbc_init_bio - writeback specific initialization of bio * @wbc: writeback_control for the writeback in progress * @bio: bio to be initialized * diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index 78388538acf4..8851727ae162 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -78,9 +78,11 @@ TRACE_EVENT_CONDITION(damos_before_apply, TP_PROTO(unsigned int context_idx, unsigned int scheme_idx, unsigned int target_idx, struct damon_region *r, - unsigned int nr_regions, bool do_trace), + unsigned int nr_accesses, unsigned int nr_regions, + bool do_trace), - TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_regions, do_trace), + TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_accesses, + nr_regions, do_trace), TP_CONDITION(do_trace), @@ -101,7 +103,7 @@ TRACE_EVENT_CONDITION(damos_before_apply, __entry->target_idx = target_idx; __entry->start = r->ar.start; __entry->end = r->ar.end; - __entry->nr_accesses = r->nr_accesses_bp / 10000; + __entry->nr_accesses = nr_accesses; __entry->age = r->age; __entry->nr_regions = nr_regions; ), diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 291fae364c62..5a48c5406cce 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -16,7 +16,7 @@ EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \ - EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \ + EM( SCAN_PTE_UFFD, "pte_uffd_wp") \ EM( SCAN_PTE_MAPPED_HUGEPAGE, "pte_mapped_hugepage") \ EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \ EM( SCAN_PAGE_NULL, "page_null") \ diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index cd01dd7b3640..15ee2ef201b5 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h @@ -23,7 +23,8 @@ EM( MR_CONTIG_RANGE, "contig_range") \ EM( MR_LONGTERM_PIN, "longterm_pin") \ EM( MR_DEMOTION, "demotion") \ - EMe(MR_DAMON, "damon") + EM( MR_DAMON, "damon") \ + EMe(MR_NEVER, "never_migrated") /* * First define the enums in the above macros to be exported to userspace @@ -51,7 +52,7 @@ TRACE_EVENT(mm_migrate_pages, TP_PROTO(unsigned long succeeded, unsigned long failed, unsigned long thp_succeeded, unsigned long thp_failed, unsigned long thp_split, unsigned long large_folio_split, - enum migrate_mode mode, int reason), + enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(succeeded, failed, thp_succeeded, thp_failed, thp_split, large_folio_split, mode, reason), @@ -64,7 +65,7 @@ TRACE_EVENT(mm_migrate_pages, __field( unsigned long, thp_split) __field( unsigned long, large_folio_split) __field( enum migrate_mode, mode) - __field( int, reason) + __field( enum migrate_reason, reason) ), TP_fast_assign( @@ -91,13 +92,13 @@ TRACE_EVENT(mm_migrate_pages, TRACE_EVENT(mm_migrate_pages_start, - TP_PROTO(enum migrate_mode mode, int reason), + TP_PROTO(enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(mode, reason), TP_STRUCT__entry( __field(enum migrate_mode, mode) - __field(int, reason) + __field(enum migrate_reason, reason) ), TP_fast_assign( diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index a6e5a44c9b42..935893e5ea53 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -54,18 +54,10 @@ # define TRACE_GFP_FLAGS_LOCKDEP #endif -#ifdef CONFIG_SLAB_OBJ_EXT -# define TRACE_GFP_FLAGS_SLAB \ - TRACE_GFP_EM(NO_OBJ_EXT) -#else -# define TRACE_GFP_FLAGS_SLAB -#endif - #define TRACE_GFP_FLAGS \ TRACE_GFP_FLAGS_GENERAL \ TRACE_GFP_FLAGS_KASAN \ - TRACE_GFP_FLAGS_LOCKDEP \ - TRACE_GFP_FLAGS_SLAB + TRACE_GFP_FLAGS_LOCKDEP #undef TRACE_GFP_EM #define TRACE_GFP_EM(a) TRACE_DEFINE_ENUM(___GFP_##a##_BIT); @@ -194,6 +186,12 @@ IF_HAVE_PG_ARCH_3(arch_3) # define IF_HAVE_UFFD_MINOR(flag, name) #endif +#ifdef CONFIG_USERFAULTFD_RWP +# define IF_HAVE_UFFD_RWP(flag, name) {flag, name}, +#else +# define IF_HAVE_UFFD_RWP(flag, name) +#endif + #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) # define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name}, #else @@ -215,6 +213,7 @@ IF_HAVE_UFFD_MINOR(VM_UFFD_MINOR, "uffd_minor" ) \ {VM_PFNMAP, "pfnmap" }, \ {VM_MAYBE_GUARD, "maybe_guard" }, \ {VM_UFFD_WP, "uffd_wp" }, \ +IF_HAVE_UFFD_RWP(VM_UFFD_RWP, "uffd_rwp" ) \ {VM_LOCKED, "locked" }, \ {VM_IO, "io" }, \ {VM_SEQ_READ, "seqread" }, \ diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 171524d3526d..36c3a90f0acc 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -77,6 +77,14 @@ TRACE_EVENT(mm_lru_activate, TP_printk("folio=%p pfn=0x%lx", __entry->folio, __entry->pfn) ); +DECLARE_TRACE(mm_lru_add_drain, + TP_PROTO(int cpu, unsigned int nr_folios), + TP_ARGS(cpu, nr_folios)); + +DECLARE_TRACE(mm_lru_add_drain_all, + TP_PROTO(bool force_all_cpus), + TP_ARGS(force_all_cpus)); + #endif /* _TRACE_PAGEMAP_H */ /* This part must be outside protection */ diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h new file mode 100644 index 000000000000..270f693b1822 --- /dev/null +++ b/include/uapi/linux/alloc_tag.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * alloc_tag IOCTL API definition + * + * Copyright (C) 2026 Google, LLC. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _UAPI_ALLOC_TAG_H +#define _UAPI_ALLOC_TAG_H + +#include <linux/types.h> + +/* + * Function, file and module names often have the same prefixes, therefore + * when filtering by these criteria, we compare the last 64 characters to + * minimize the chances of name collisions + */ +#define ALLOCINFO_STR_SIZE 64 + +struct allocinfo_content_id { + __u64 id; +}; + +struct allocinfo_tag { + /* Longer names are trimmed */ + char modname[ALLOCINFO_STR_SIZE]; + char function[ALLOCINFO_STR_SIZE]; + char filename[ALLOCINFO_STR_SIZE]; + __u64 lineno; + /* filter criteria only; see allocinfo_counter.accurate for actual accuracy */ + __u64 inaccurate; +}; + +/* The alignment ensures 32-bit compatible interfaces are not broken */ +struct allocinfo_counter { + __u64 bytes; + __u64 calls; + __u8 accurate; +} __attribute__((aligned(8))); + +struct allocinfo_tag_data { + struct allocinfo_tag tag; + struct allocinfo_counter counter; +}; + +enum { + ALLOCINFO_FILTER_MODNAME, + ALLOCINFO_FILTER_FUNCTION, + ALLOCINFO_FILTER_FILENAME, + ALLOCINFO_FILTER_LINENO, + ALLOCINFO_FILTER_INACCURATE, + ALLOCINFO_FILTER_MIN_SIZE, + ALLOCINFO_FILTER_MAX_SIZE, + __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE +}; + +#define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME) +#define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION) +#define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME) +#define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO) +#define ALLOCINFO_FILTER_MASK_INACCURATE (1 << ALLOCINFO_FILTER_INACCURATE) +#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE) +#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE) + +#define ALLOCINFO_FILTER_MASKS \ + ((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1) + +struct allocinfo_filter { + __u64 mask; /* bitmask of the filter fields used */ + struct allocinfo_tag fields; + __u64 min_size; + __u64 max_size; +}; + +struct allocinfo_get_at { + /* inputs */ + __u64 pos; + struct allocinfo_filter filter; + /* output */ + struct allocinfo_tag_data data; +}; + +#define _ALLOCINFO_IOC_CONTENT_ID 0 +#define _ALLOCINFO_IOC_GET_AT 1 +#define _ALLOCINFO_IOC_GET_NEXT 2 + +#define ALLOCINFO_IOC_BASE 0xA6 +#define ALLOCINFO_IOC_CONTENT_ID _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID, \ + struct allocinfo_content_id) +#define ALLOCINFO_IOC_GET_AT _IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT, \ + struct allocinfo_get_at) +#define ALLOCINFO_IOC_GET_NEXT _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT, \ + struct allocinfo_tag_data) + +#endif /* _UAPI_ALLOC_TAG_H */ diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index bd87262f2e34..34c6f219462a 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -471,6 +471,7 @@ typedef int __bitwise __kernel_rwf_t; #define PAGE_IS_HUGE (1 << 6) #define PAGE_IS_SOFT_DIRTY (1 << 7) #define PAGE_IS_GUARD (1 << 8) +#define PAGE_IS_ACCESSED (1 << 9) /* * struct page_region - Page region with flags diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 6c962d866e86..7f6fc9599693 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -16,7 +16,7 @@ */ /* Policies */ -enum { +enum mempolicy_mode { MPOL_DEFAULT, MPOL_PREFERRED, MPOL_BIND, diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 2841e4ea8f2c..cea11aad6b54 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -25,7 +25,8 @@ #define UFFD_API ((__u64)0xAA) #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ UFFDIO_REGISTER_MODE_WP | \ - UFFDIO_REGISTER_MODE_MINOR) + UFFDIO_REGISTER_MODE_MINOR | \ + UFFDIO_REGISTER_MODE_RWP) #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ @@ -42,10 +43,13 @@ UFFD_FEATURE_WP_UNPOPULATED | \ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ - UFFD_FEATURE_MOVE) + UFFD_FEATURE_MOVE | \ + UFFD_FEATURE_RWP | \ + UFFD_FEATURE_RWP_ASYNC) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_SET_MODE | \ (__u64)1 << _UFFDIO_API) #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ @@ -54,13 +58,15 @@ (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) /* * Valid ioctl command number range with this API is from 0x00 to @@ -79,6 +85,8 @@ #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) +#define _UFFDIO_RWPROTECT (0x09) +#define _UFFDIO_SET_MODE (0x0A) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -103,6 +111,10 @@ struct uffdio_continue) #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ struct uffdio_poison) +#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ + struct uffdio_rwprotect) +#define UFFDIO_SET_MODE _IOW(UFFDIO, _UFFDIO_SET_MODE, \ + struct uffdio_set_mode) /* read() structure */ struct uffd_msg { @@ -158,6 +170,7 @@ struct uffd_msg { #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ +#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */ struct uffdio_api { /* userland asks for an API number and the features to enable */ @@ -230,6 +243,18 @@ struct uffdio_api { * * UFFD_FEATURE_MOVE indicates that the kernel supports moving an * existing page contents from userspace. + * + * UFFD_FEATURE_RWP indicates that the kernel supports + * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. + * Pages are made inaccessible via UFFDIO_RWPROTECT and faults + * are delivered when the pages are re-accessed. + * + * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for + * UFFDIO_REGISTER_MODE_RWP. When set, faults on read-write + * protected pages are auto-resolved by the kernel (PTE + * permissions restored immediately) without delivering a message + * to the userfaultfd handler. Use PAGEMAP_SCAN with inverted + * PAGE_IS_ACCESSED to find pages that were not re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -248,6 +273,8 @@ struct uffdio_api { #define UFFD_FEATURE_POISON (1<<14) #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) +#define UFFD_FEATURE_RWP (1<<17) +#define UFFD_FEATURE_RWP_ASYNC (1<<18) __u64 features; __u64 ioctls; @@ -263,6 +290,7 @@ struct uffdio_register { #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) +#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3) __u64 mode; /* @@ -356,6 +384,14 @@ struct uffdio_poison { __s64 updated; }; +struct uffdio_rwprotect { + struct uffdio_range range; + /* !RWP means undo RWP-protection */ +#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0) +#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + struct uffdio_move { __u64 dst; __u64 src; @@ -374,6 +410,16 @@ struct uffdio_move { __s64 move; }; +struct uffdio_set_mode { + /* + * Toggle async mode for features at runtime. + * Supported: UFFD_FEATURE_RWP_ASYNC. + * Setting a bit in both enable and disable is invalid. + */ + __u64 enable; + __u64 disable; +}; + /* * Flags for the userfaultfd(2) system call itself. */ |