1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __MM_COLLAPSE_H
#define __MM_COLLAPSE_H
#include <linux/mm.h>
#include <linux/nodemask.h>
#include <linux/pgtable.h>
#include <linux/types.h>
#define COLLAPSE_MAX_PTES_LIMIT (HPAGE_PMD_NR - 1)
#define COLLAPSE_MIN_MTHP_ORDER 2
enum scan_result {
SCAN_FAIL,
SCAN_SUCCEED,
SCAN_NO_PTE_TABLE,
SCAN_PMD_MAPPED,
SCAN_EXCEED_NONE_PTE,
SCAN_EXCEED_SWAP_PTE,
SCAN_EXCEED_SHARED_PTE,
SCAN_PTE_NON_PRESENT,
SCAN_PTE_UFFD,
SCAN_PTE_MAPPED_HUGEPAGE,
SCAN_LACK_REFERENCED_PAGE,
SCAN_PAGE_NULL,
SCAN_SCAN_ABORT,
SCAN_PAGE_COUNT,
SCAN_PAGE_LRU,
SCAN_PAGE_LOCK,
SCAN_PAGE_ANON,
SCAN_PAGE_LAZYFREE,
SCAN_PAGE_COMPOUND,
SCAN_ANY_PROCESS,
SCAN_VMA_NULL,
SCAN_VMA_CHECK,
SCAN_ADDRESS_RANGE,
SCAN_DEL_PAGE_LRU,
SCAN_ALLOC_HUGE_PAGE_FAIL,
SCAN_CGROUP_CHARGE_FAIL,
SCAN_TRUNCATED,
SCAN_PAGE_HAS_PRIVATE,
SCAN_STORE_FAILED,
SCAN_COPY_MC,
SCAN_PAGE_FILLED,
SCAN_PAGE_DIRTY_OR_WRITEBACK,
};
/* How many PTEs of a window may be missing, swapped out or shared */
struct collapse_limits {
/* Counted over a PMD-sized window; HPAGE_PMD_NR means "no limit" */
unsigned int max_ptes_none;
unsigned int max_ptes_swap;
unsigned int max_ptes_shared;
};
/* What a collapse is allowed to do, decided by the caller that asks for it */
struct collapse_policy {
/* Limits for a PMD-sized window */
struct collapse_limits pmd;
/*
* Limits for a smaller window. Its max_ptes_none is either 0 or
* COLLAPSE_MAX_PTES_LIMIT, the latter meaning all but one PTE of the
* window whatever its order; any other value counts as 0.
*/
struct collapse_limits sub_pmd;
/* Leave clean lazyfree folios to reclaim rather than collapse them */
bool anon_skip_lazyfree;
/* Refuse an anonymous range with no sign of use */
bool anon_require_referenced;
/* Map the PMD over a file collapse instead of leaving it to a fault */
bool file_install_pmd;
/* Write dirty pages back and retry once instead of refusing them */
bool file_writeback_dirty;
/* How hard to try for a destination folio */
gfp_t gfp;
/* Which VMAs are eligible, as thp_vma_allowable_orders() spells it */
enum tva_type tva_type;
};
struct collapse_control {
struct collapse_policy policy;
/* Num pages scanned per node */
u32 node_load[MAX_NUMNODES];
/* Num pages scanned (see khugepaged_pages_to_scan) */
unsigned int progress;
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
/* Each bit marks a PTE the scan accepted as a collapse source */
DECLARE_BITMAP(eligible_ptes, MAX_PTRS_PER_PTE);
/*
* What a scan found and the collapse after it needs. Live only between
* the two, and read by nobody else.
*/
/* Anonymous only: orders to try, referenced and swap PTE counts */
unsigned long scan_orders;
int scan_referenced;
int scan_unmapped;
/*
* File only. A file collapse works on the page cache and never sees a
* VMA, so the scan takes a reference to the file while it still has
* one, and the collapse gives it back.
*/
struct file *scan_file;
pgoff_t scan_pgoff;
};
#endif /* __MM_COLLAPSE_H */
|
