diff options
Diffstat (limited to 'mm/internal.h')
| -rw-r--r-- | mm/internal.h | 668 |
1 files changed, 184 insertions, 484 deletions
diff --git a/mm/internal.h b/mm/internal.h index 181e79f1d6a2..41561fdeb56d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,80 @@ #include "vma.h" struct folio_batch; +struct hstate; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; +}; + +/* mm/workingset.c */ +bool workingset_test_recent(void *shadow, bool file, bool *workingset, + bool flush); +void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); +void *workingset_eviction(struct folio *folio, + struct mem_cgroup *target_memcg); +void workingset_refault(struct folio *folio, void *shadow); +void workingset_activation(struct folio *folio); + +/* mm/folio.c */ +void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, + unsigned int nr_io, unsigned int nr_rotated); +void lru_note_cost_refault(struct folio *folio); +void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); + +static inline bool folio_may_be_lru_cached(struct folio *folio) +{ + /* + * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. + * Holding small numbers of low-order mTHP folios in per-CPU LRU cache + * will be sensible, but nobody has implemented and tested that yet. + */ + return !folio_test_large(folio); +} + +static inline void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} + +void lru_cache_disable(void); +void lru_add_drain(void); +void lru_add_drain_cpu(int cpu); +void lru_add_drain_cpu_zone(struct zone *zone); +void folio_deactivate(struct folio *folio); +void folio_mark_lazyfree(struct folio *folio); + +/* mm/vmscan.c */ +unsigned long zone_reclaimable_pages(struct zone *zone); +unsigned long try_to_free_pages(struct zonelist *zonelist, int order, + gfp_t gfp_mask, const nodemask_t *mask); +unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx); + +#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) +#define MEMCG_RECLAIM_PROACTIVE (1 << 2) +#define MIN_SWAPPINESS 0 +#define MAX_SWAPPINESS 200 + +/* Just reclaim from anon folios in proactive memory reclaim */ +#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + +unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness); +unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, + gfp_t gfp_mask, bool noswap, + pg_data_t *pgdat, + unsigned long *nr_scanned); + +#ifdef CONFIG_NUMA +extern int sysctl_min_unmapped_ratio; +extern int sysctl_min_slab_ratio; +#endif /* * Maintains state across a page table move. The operation assumes both source @@ -166,6 +240,10 @@ static inline int mmap_file(struct file *file, struct vm_area_struct *vma) { int err = vfs_mmap(file, vma); + /* Hooks cannot mark themselves anonymous. */ + if (WARN_ON_ONCE(vma_is_anonymous(vma))) + err = -EINVAL; + if (likely(!err)) return 0; @@ -412,8 +490,8 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) new = pte_swp_mksoft_dirty(new); if (pte_swp_exclusive(pte)) new = pte_swp_mkexclusive(new); - if (pte_swp_uffd_wp(pte)) - new = pte_swp_mkuffd_wp(new); + if (pte_swp_uffd(pte)) + new = pte_swp_mkuffd(new); return new; } @@ -642,190 +720,16 @@ void set_recommended_min_free_kbytes(void); extern char * const zone_names[MAX_NR_ZONES]; -/* perform sanity checks on struct pages being allocated or freed */ -DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); - extern int min_free_kbytes; extern int defrag_mode; void setup_per_zone_wmarks(void); void calculate_min_free_kbytes(void); int __meminit init_per_zone_wmark_min(void); -void page_alloc_sysctl_init(void); - -/* - * Structure for holding the mostly immutable allocation parameters passed - * between functions involved in allocations, including the alloc_pages* - * family of functions. - * - * nodemask, migratetype and highest_zoneidx are initialized only once in - * __alloc_pages() and then never change. - * - * zonelist, preferred_zone and highest_zoneidx are set first in - * __alloc_pages() for the fast path, and might be later changed - * in __alloc_pages_slowpath(). All other functions pass the whole structure - * by a const pointer. - */ -struct alloc_context { - struct zonelist *zonelist; - nodemask_t *nodemask; - struct zoneref *preferred_zoneref; - int migratetype; - - /* - * highest_zoneidx represents highest usable zone index of - * the allocation request. Due to the nature of the zone, - * memory on lower zone than the highest_zoneidx will be - * protected by lowmem_reserve[highest_zoneidx]. - * - * highest_zoneidx is also used by reclaim/compaction to limit - * the target zone since higher zone than this index cannot be - * usable for this allocation request. - */ - enum zone_type highest_zoneidx; - bool spread_dirty_pages; -}; - -/* - * This function returns the order of a free page in the buddy system. In - * general, page_zone(page)->lock must be held by the caller to prevent the - * page from being allocated in parallel and returning garbage as the order. - * If a caller does not hold page_zone(page)->lock, it must guarantee that the - * page cannot be allocated or merged in parallel. Alternatively, it must - * handle invalid values gracefully, and use buddy_order_unsafe() below. - */ -static inline unsigned int buddy_order(struct page *page) -{ - /* PageBuddy() must be checked by the caller */ - return page_private(page); -} - -/* - * Like buddy_order(), but for callers who cannot afford to hold the zone lock. - * PageBuddy() should be checked first by the caller to minimize race window, - * and invalid values must be handled gracefully. - * - * READ_ONCE is used so that if the caller assigns the result into a local - * variable and e.g. tests it for valid range before using, the compiler cannot - * decide to remove the variable and inline the page_private(page) multiple - * times, potentially observing different values in the tests and the actual - * use of the result. - */ -#define buddy_order_unsafe(page) READ_ONCE(page_private(page)) - -/* - * This function checks whether a page is free && is the buddy - * we can coalesce a page and its buddy if - * (a) the buddy is not in a hole (check before calling!) && - * (b) the buddy is in the buddy system && - * (c) a page and its buddy have the same order && - * (d) a page and its buddy are in the same zone. - * - * For recording whether a page is in the buddy system, we set PageBuddy. - * Setting, clearing, and testing PageBuddy is serialized by zone->lock. - * - * For recording page's order, we use page_private(page). - */ -static inline bool page_is_buddy(struct page *page, struct page *buddy, - unsigned int order) -{ - if (!page_is_guard(buddy) && !PageBuddy(buddy)) - return false; - - if (buddy_order(buddy) != order) - return false; - - /* - * zone check is done late to avoid uselessly calculating - * zone/node ids for pages that could never merge. - */ - if (page_zone_id(page) != page_zone_id(buddy)) - return false; - - VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); - - return true; -} - -/* - * Locate the struct page for both the matching buddy in our - * pair (buddy1) and the combined O(n+1) page they form (page). - * - * 1) Any buddy B1 will have an order O twin B2 which satisfies - * the following equation: - * B2 = B1 ^ (1 << O) - * For example, if the starting buddy (buddy2) is #8 its order - * 1 buddy is #10: - * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 - * - * 2) Any buddy B will have an order O+1 parent P which - * satisfies the following equation: - * P = B & ~(1 << O) - * - * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER - */ -static inline unsigned long -__find_buddy_pfn(unsigned long page_pfn, unsigned int order) -{ - return page_pfn ^ (1 << order); -} - -/* - * Find the buddy of @page and validate it. - * @page: The input page - * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the - * function is used in the performance-critical __free_one_page(). - * @order: The order of the page - * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to - * page_to_pfn(). - * - * The found buddy can be a non PageBuddy, out of @page's zone, or its order is - * not the same as @page. The validation is necessary before use it. - * - * Return: the found buddy page or NULL if not found. - */ -static inline struct page *find_buddy_page_pfn(struct page *page, - unsigned long pfn, unsigned int order, unsigned long *buddy_pfn) -{ - unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order); - struct page *buddy; - - buddy = page + (__buddy_pfn - pfn); - if (buddy_pfn) - *buddy_pfn = __buddy_pfn; - - if (page_is_buddy(page, buddy, order)) - return buddy; - return NULL; -} - -extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone); - -static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn, - unsigned long end_pfn, struct zone *zone) -{ - if (zone->contiguous) - return pfn_to_page(start_pfn); - - return __pageblock_pfn_to_page(start_pfn, end_pfn, zone); -} - -void set_zone_contiguous(struct zone *zone); -bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, - unsigned long nr_pages); - -static inline void clear_zone_contiguous(struct zone *zone) -{ - zone->contiguous = false; -} extern int __isolate_free_page(struct page *page, unsigned int order); extern void __putback_isolated_page(struct page *page, unsigned int order, int mt); -extern void memblock_free_pages(unsigned long pfn, unsigned int order); -extern void __free_pages_core(struct page *page, unsigned int order, - enum meminit_context context); /* * This will have no effect, other than possibly generating a warning, if the @@ -895,7 +799,7 @@ static inline void prep_compound_tail(struct page *tail, { tail->mapping = TAIL_MAPPING; set_compound_head(tail, head, order); - set_page_private(tail, 0); + VM_WARN_ON_ONCE(tail->private); } static inline void init_compound_tail(struct page *tail, @@ -907,101 +811,6 @@ static inline void init_compound_tail(struct page *tail, prep_compound_tail(tail, head, order); } -void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); -extern bool free_pages_prepare(struct page *page, unsigned int order); - -extern int user_min_free_kbytes; - -struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid, - nodemask_t *); -#define __alloc_frozen_pages(...) \ - alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__)) -void free_frozen_pages(struct page *page, unsigned int order); -void free_unref_folios(struct folio_batch *fbatch); - -#ifdef CONFIG_NUMA -struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order); -#else -static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order) -{ - return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL); -} -#endif - -#define alloc_frozen_pages(...) \ - alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__)) - -struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order); -#define alloc_frozen_pages_nolock(...) \ - alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__)) -void free_frozen_pages_nolock(struct page *page, unsigned int order); - -extern void zone_pcp_reset(struct zone *zone); -extern void zone_pcp_disable(struct zone *zone); -extern void zone_pcp_enable(struct zone *zone); -extern void zone_pcp_init(struct zone *zone); - -extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, - int nid, bool exact_nid); - -void memmap_init_range(unsigned long, int, unsigned long, unsigned long, - unsigned long, enum meminit_context, struct vmem_altmap *, int, - bool); - -/* - * mm/sparse.c - */ -#ifdef CONFIG_SPARSEMEM -void sparse_init(void); -int sparse_index_init(unsigned long section_nr, int nid); - -static inline void sparse_init_one_section(struct mem_section *ms, - unsigned long pnum, struct page *mem_map, - struct mem_section_usage *usage, unsigned long flags) -{ - unsigned long coded_mem_map; - - BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); - - /* - * We encode the start PFN of the section into the mem_map such that - * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it - * from the page pointer to obtain the PFN. - */ - coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum)); - VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK); - - ms->section_mem_map &= ~SECTION_MAP_MASK; - ms->section_mem_map |= coded_mem_map; - ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP; - ms->usage = usage; -} - -static inline void __section_mark_present(struct mem_section *ms, - unsigned long section_nr) -{ - if (section_nr > __highest_present_section_nr) - __highest_present_section_nr = section_nr; - - ms->section_mem_map |= SECTION_MARKED_PRESENT; -} -#else -static inline void sparse_init(void) {} -#endif /* CONFIG_SPARSEMEM */ - -/* - * mm/sparse-vmemmap.c - */ -#ifdef CONFIG_SPARSEMEM_VMEMMAP -void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages); -#else -static inline void sparse_init_subsection_map(unsigned long pfn, - unsigned long nr_pages) -{ -} -#endif /* CONFIG_SPARSEMEM_VMEMMAP */ - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* @@ -1070,9 +879,6 @@ int isolate_migratepages_range(struct compact_control *cc, unsigned long low_pfn, unsigned long end_pfn); -/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ -void init_cma_reserved_pageblock(struct page *page); - #endif /* CONFIG_COMPACTION || CONFIG_CMA */ struct cma; @@ -1080,7 +886,6 @@ struct cma; #ifdef CONFIG_CMA bool cma_validate_zones(struct cma *cma); void *cma_reserve_early(struct cma *cma, unsigned long size); -void init_cma_pageblock(struct page *page); #else static inline bool cma_validate_zones(struct cma *cma) { @@ -1090,28 +895,8 @@ static inline void *cma_reserve_early(struct cma *cma, unsigned long size) { return NULL; } -static inline void init_cma_pageblock(struct page *page) -{ -} #endif -enum fallback_result { - /* Found suitable migratetype, *mt_out is valid. */ - FALLBACK_FOUND, - /* No fallback found in requested order. */ - FALLBACK_EMPTY, - /* Passed @claimable, but claiming whole block is a bad idea. */ - FALLBACK_NOCLAIM, -}; -enum fallback_result -find_suitable_fallback(struct free_area *area, unsigned int order, - int migratetype, bool claimable, int *mt_out); - -static inline bool free_area_empty(struct free_area *area, int migratetype) -{ - return list_empty(&area->free_list[migratetype]); -} - /* mm/util.c */ struct anon_vma *folio_anon_vma(const struct folio *folio); @@ -1143,26 +928,29 @@ static inline bool folio_within_range(struct folio *folio, struct vm_area_struct *vma, unsigned long start, unsigned long end) { - pgoff_t pgoff, addr; - unsigned long vma_pglen = vma_pages(vma); + const unsigned long vma_pglen = vma_pages(vma); + pgoff_t pgoff_folio, pgoff_vma_start; + unsigned long addr; VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio); if (start > end) return false; + pgoff_folio = folio_pgoff(folio); + pgoff_vma_start = folio_test_anon(folio) ? + vma_start_virt_pgoff(vma) : vma_start_pgoff(vma); + if (start < vma->vm_start) start = vma->vm_start; if (end > vma->vm_end) end = vma->vm_end; - pgoff = folio_pgoff(folio); - /* if folio start address is not in vma range */ - if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) + if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen)) return false; - addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT); return !(addr < start || end - addr < folio_size(folio)); } @@ -1222,27 +1010,18 @@ void mlock_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); -/** - * vma_address - Find the virtual address a page range is mapped at - * @vma: The vma which maps this object. - * @pgoff: The page offset within its object. - * @nr_pages: The number of pages to consider. - * - * If any page in this range is mapped by this VMA, return the first address - * where any of these pages appear. Otherwise, return -EFAULT. - */ -static inline unsigned long vma_address(const struct vm_area_struct *vma, - pgoff_t pgoff, unsigned long nr_pages) +static inline unsigned long __vma_address(const struct vm_area_struct *vma, + pgoff_t pgoff, pgoff_t pgoff_start, unsigned long nr_pages) { unsigned long address; - if (pgoff >= vma->vm_pgoff) { + if (pgoff >= pgoff_start) { address = vma->vm_start + - ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + ((pgoff - pgoff_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; - } else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) { + } else if (pgoff + nr_pages - 1 >= pgoff_start) { /* Test above avoids possibility of wrap to 0 on 32-bit */ address = vma->vm_start; } else { @@ -1251,22 +1030,68 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, return address; } +/** + * vma_filebacked_address - Find the virtual address a file-backed page range is + * mapped at. + * @vma: The vma which maps this object. + * @pgoff: The page offset within its object. + * @nr_pages: The number of pages to consider. + * + * Returns: If any page in this range is mapped by this VMA, return the first + * address where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma, + pgoff_t pgoff, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(vma_is_anonymous(vma)); + + return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages); +} + +/** + * vma_anon_address - Find the virtual address an anonymous page range is mapped + * at. + * @vma: The vma which maps this object. + * @pgoff_virt: The virtual page index belonging to the folio. + * @nr_pages: The number of pages to consider. + * + * This is only valid for anonymous or MAP_PRIVATE-mapped file-backed VMAs. + * + * Returns: If any page in this range is mapped by this VMA, return the first address + * where any of these pages appear. Otherwise, return -EFAULT. + */ +static inline unsigned long vma_anon_address(const struct vm_area_struct *vma, + pgoff_t pgoff_virt, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(!vma_is_anonymous(vma) && vma_test(vma, VMA_SHARED_BIT)); + + return __vma_address(vma, pgoff_virt, vma_start_virt_pgoff(vma), nr_pages); +} + /* - * Then at what user virtual address will none of the range be found in vma? + * At what user virtual address will none of the range be found in vma? * Assumes that vma_address() already returned a good starting address. */ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) { - struct vm_area_struct *vma = pvmw->vma; - pgoff_t pgoff; + const struct vm_area_struct *vma = pvmw->vma; + const pgoff_t pgoff = pvmw->pgoff; + pgoff_t pgoff_vma_start; unsigned long address; + pgoff_t pgoff_end; /* Common case, plus ->pgoff is invalid for KSM */ if (pvmw->nr_pages == 1) return pvmw->address + PAGE_SIZE; - pgoff = pvmw->pgoff + pvmw->nr_pages; - address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pvmw->is_anon_walk) + pgoff_vma_start = vma_start_virt_pgoff(vma); + else + pgoff_vma_start = vma_start_pgoff(vma); + + pgoff_end = pgoff + pvmw->nr_pages; + address = vma->vm_start + + ((pgoff_end - pgoff_vma_start) << PAGE_SHIFT); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; @@ -1311,85 +1136,21 @@ static inline void mlock_new_folio(struct folio *folio) { } static inline bool need_mlock_drain(int cpu) { return false; } static inline void mlock_drain_local(void) { } static inline void mlock_drain_remote(int cpu) { } -static inline void vunmap_range_noflush(unsigned long start, unsigned long end) -{ -} #endif /* !CONFIG_MMU */ -/* Memory initialisation debug and verification */ -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -DECLARE_STATIC_KEY_TRUE(deferred_pages); - -static inline bool deferred_pages_enabled(void) -{ - return static_branch_unlikely(&deferred_pages); -} - -bool __init deferred_grow_zone(struct zone *zone, unsigned int order); -#else -static inline bool deferred_pages_enabled(void) -{ - return false; -} -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - -void init_deferred_page(unsigned long pfn, int nid); - -enum mminit_level { - MMINIT_WARNING, - MMINIT_VERIFY, - MMINIT_TRACE -}; - -#ifdef CONFIG_DEBUG_MEMORY_INIT - -extern int mminit_loglevel; - -#define mminit_dprintk(level, prefix, fmt, arg...) \ -do { \ - if (level < mminit_loglevel) { \ - if (level <= MMINIT_WARNING) \ - pr_warn("mminit::" prefix " " fmt, ##arg); \ - else \ - printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \ - } \ -} while (0) - -extern void mminit_verify_pageflags_layout(void); -extern void mminit_verify_zonelist(void); -#else - -static inline void mminit_dprintk(enum mminit_level level, - const char *prefix, const char *fmt, ...) -{ -} - -static inline void mminit_verify_pageflags_layout(void) -{ -} - -static inline void mminit_verify_zonelist(void) -{ -} -#endif /* CONFIG_DEBUG_MEMORY_INIT */ - -#define NODE_RECLAIM_NOSCAN -2 -#define NODE_RECLAIM_FULL -1 -#define NODE_RECLAIM_SOME 0 -#define NODE_RECLAIM_SUCCESS 1 - #ifdef CONFIG_NUMA extern int node_reclaim_mode; -extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int); +extern unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t gfp_mask, unsigned int order); extern int find_next_best_node(int node, nodemask_t *used_node_mask); #else #define node_reclaim_mode 0 -static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask, - unsigned int order) +static inline unsigned long node_reclaim(struct pglist_data *pgdat, + gfp_t mask, unsigned int order) { - return NODE_RECLAIM_NOSCAN; + return 0; } static inline int find_next_best_node(int node, nodemask_t *used_node_mask) { @@ -1436,50 +1197,9 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -extern void set_pageblock_order(void); unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); -/* The ALLOC_WMARK bits are used as an index to zone->watermark */ -#define ALLOC_WMARK_MIN WMARK_MIN -#define ALLOC_WMARK_LOW WMARK_LOW -#define ALLOC_WMARK_HIGH WMARK_HIGH -#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ - -/* Mask to get the watermark bits */ -#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) - -/* - * Only MMU archs have async oom victim reclaim - aka oom_reaper so we - * cannot assume a reduced access to memory reserves is sufficient for - * !MMU - */ -#ifdef CONFIG_MMU -#define ALLOC_OOM 0x08 -#else -#define ALLOC_OOM ALLOC_NO_WATERMARKS -#endif - -#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access - * to 25% of the min watermark or - * 62.5% if __GFP_HIGH is set. - */ -#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% - * of the min watermark. - */ -#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ -#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ -#ifdef CONFIG_ZONE_DMA32 -#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */ -#else -#define ALLOC_NOFRAGMENT 0x0 -#endif -#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ -#define ALLOC_TRYLOCK 0x400 /* Only use spin_trylock in allocation path */ -#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ - -/* Flags that allow allocations below the min watermark. */ -#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) enum ttu_flags; struct tlbflush_unmap_batch; @@ -1526,37 +1246,6 @@ struct migration_target_control { size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, struct folio *folio, loff_t fpos, size_t size); -/* - * mm/vmalloc.c - */ -#ifdef CONFIG_MMU -void __init vmalloc_init(void); -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask); -unsigned int get_vm_area_page_order(struct vm_struct *vm); -#else -static inline void vmalloc_init(void) -{ -} - -static inline -int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, - pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask) -{ - return -EINVAL; -} -#endif - -void clear_vm_uninitialized_flag(struct vm_struct *vm); - -int __must_check __vmap_pages_range_noflush(unsigned long addr, - unsigned long end, pgprot_t prot, - struct page **pages, unsigned int page_shift); - -void vunmap_range_noflush(unsigned long start, unsigned long end); - -void __vunmap_range_noflush(unsigned long start, unsigned long end); - static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) { if (vma->vm_flags & VM_SHARED) @@ -1584,12 +1273,6 @@ int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, void free_zone_device_folio(struct folio *folio); int migrate_device_coherent_folio(struct folio *folio); -struct vm_struct *__get_vm_area_node(unsigned long size, - unsigned long align, unsigned long shift, - unsigned long vm_flags, unsigned long start, - unsigned long end, int node, gfp_t gfp_mask, - const void *caller); - /* * mm/gup.c */ @@ -1712,18 +1395,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, return !PageAnonExclusive(page); } -extern bool mirrored_kernelcore; -bool memblock_has_mirror(void); -void memblock_free_all(void); - -static __always_inline void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) { @@ -1753,10 +1424,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte); } -void __meminit __init_single_page(struct page *page, unsigned long pfn, - unsigned long zone, int nid); -void __meminit __init_page_from_nid(unsigned long pfn, int nid); - /* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, int priority); @@ -1951,4 +1618,37 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) +{ + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } +} + +static inline bool can_spin_trylock(void) +{ + /* + * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is + * unsafe in NMI. If spin_trylock() is called from hard IRQ the current + * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will + * mark the task as the owner of another rt_spin_lock which will + * confuse PI logic, so return immediately if called from hard IRQ or + * NMI. + * + * Note, irqs_disabled() case is ok. spin_trylock() can be called + * from raw_spin_lock_irqsave region. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) + return false; + + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return false; + + return true; +} + #endif /* __MM_INTERNAL_H */ |