summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--fs/proc/page.c1
-rw-r--r--include/linux/kernel-page-flags.h1
-rw-r--r--include/linux/mm.h19
-rw-r--r--include/linux/page-flags.h57
-rw-r--r--include/trace/events/pagemap.h2
-rw-r--r--mm/huge_memory.c2
-rw-r--r--mm/migrate.c2
-rw-r--r--mm/page-writeback.c2
-rw-r--r--mm/vmscan.c2
-rw-r--r--tools/mm/page-types.c2
10 files changed, 68 insertions, 22 deletions
diff --git a/fs/proc/page.c b/fs/proc/page.c
index 260772b20bd9..f90e1030825e 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -232,7 +232,6 @@ u64 stable_page_flags(const struct page *page)
u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved);
u |= kpf_copy_bit(k, KPF_OWNER_2, PG_owner_2);
- u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private);
u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
diff --git a/include/linux/kernel-page-flags.h b/include/linux/kernel-page-flags.h
index 196778a087c4..fe5ab6e50bd7 100644
--- a/include/linux/kernel-page-flags.h
+++ b/include/linux/kernel-page-flags.h
@@ -11,7 +11,6 @@
#define KPF_RESERVED 32
#define KPF_MLOCKED 33
#define KPF_OWNER_2 34
-#define KPF_PRIVATE 35
#define KPF_PRIVATE_2 36
#define KPF_OWNER_PRIVATE 37
#define KPF_ARCH 38
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 30a3365bca82..6c8df7715eb2 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3005,9 +3005,9 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio)
* @folio: the folio
*
* Calculate the expected folio refcount, taking references from the pagecache,
- * swapcache, PG_private and page table mappings into account. Useful in
- * combination with folio_ref_count() to detect unexpected references (e.g.,
- * GUP or other temporary references).
+ * swapcache, private data (folio->private != NULL) and page table mappings into
+ * account. Useful in combination with folio_ref_count() to detect unexpected
+ * references (e.g., GUP or other temporary references).
*
* Does currently not consider references from the LRU cache. If the folio
* was isolated from the LRU (which is the case during migration or split),
@@ -3045,10 +3045,15 @@ static inline int folio_expected_ref_count(const struct folio *folio)
ref_count += folio_test_swapcache(folio) << order;
if (!folio_test_anon(folio)) {
- /* One reference per page from the pagecache. */
- ref_count += !!folio->mapping << order;
- /* One reference from PG_private. */
- ref_count += folio_test_private(folio);
+ /*
+ * One reference per page from the pagecache.
+ * Use data_race() since folio might not be locked.
+ */
+ ref_count += !!data_race(folio->mapping) << order;
+ /*
+ * One reference from filesystem private data.
+ */
+ ref_count += folio_has_attached_private(folio);
}
/* One reference per page table mapping. */
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 7080a6a1a79e..6d839f50bdcb 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -575,9 +575,31 @@ FOLIO_FLAG(swapbacked, FOLIO_HEAD_PAGE)
/*
* Private page markings that may be used by the filesystem that owns the page
* for its own purposes.
- * - PG_private and PG_private_2 cause release_folio() and co to be invoked
+ * - folio->private and PG_private_2 cause release_folio() and co to be invoked
*/
-PAGEFLAG(Private, private, PF_ANY)
+
+static __always_inline bool folio_test_private(const struct folio *folio)
+{
+ /*
+ * data_race() is added for readers without holding the folio lock.
+ * Only the NULL/non-NULL answer is used and both are valid while
+ * private is being attached or detached, so the race is benign.
+ */
+ return data_race(folio->private);
+}
+
+static __always_inline int PagePrivate(const struct page *page)
+{
+ /* See folio_test_private() for data_race() use */
+ return !!data_race(page->private);
+}
+
+/* no-ops during transition */
+static __always_inline void folio_set_private(struct folio *folio) { }
+static __always_inline void folio_clear_private(struct folio *folio) { }
+static __always_inline void SetPagePrivate(struct page *page) { }
+static __always_inline void ClearPagePrivate(struct page *page) { }
+
FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE)
/* owner_2 can be set on tail pages for anon memory */
@@ -1169,7 +1191,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
*/
#define PAGE_FLAGS_CHECK_AT_FREE \
(1UL << PG_lru | 1UL << PG_locked | \
- 1UL << PG_private | 1UL << PG_private_2 | \
+ 1UL << PG_private_2 | \
1UL << PG_writeback | 1UL << PG_reserved | \
1UL << PG_active | \
1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK)
@@ -1193,8 +1215,31 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
(0xffUL /* order */ | 1UL << PG_has_hwpoisoned | \
1UL << PG_large_rmappable | 1UL << PG_partially_mapped)
-#define PAGE_FLAGS_PRIVATE \
- (1UL << PG_private | 1UL << PG_private_2)
+/**
+ * folio_has_attached_private - check if the folio has private data attached
+ * @folio: The folio to check.
+ *
+ * Use this in code that may encounter swapcache or hugetlb folios but only
+ * wants to detect attached private data.
+ *
+ * Return: true if the folio has private data attached.
+ */
+static inline bool folio_has_attached_private(const struct folio *folio)
+{
+ /*
+ * Swapcache stores swp_entry_t in folio->swap, a union with
+ * folio->private, and hugetlb stores its own flags in folio->private;
+ * both are excluded.
+ *
+ * NOTE: For swapcache, folio->swap.val PG_swapcache are not set as
+ * a whole, so folio_test_swapcache() is not reliable to exclude
+ * swapcache. Use folio_test_swapbacked() instead, since it remains set
+ * when a folio is added to/removed from swapcache.
+ */
+
+ return folio_test_private(folio) && !folio_test_swapbacked(folio) &&
+ !folio_test_hugetlb(folio);
+}
/**
* folio_has_private - Determine if folio has private stuff
* @folio: The folio to be checked
@@ -1204,7 +1249,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
*/
static inline int folio_has_private(const struct folio *folio)
{
- return !!(folio->flags.f & PAGE_FLAGS_PRIVATE);
+ return folio_has_attached_private(folio) || folio_test_private_2(folio);
}
#undef PF_ANY
diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h
index 36c3a90f0acc..5d47b774633a 100644
--- a/include/trace/events/pagemap.h
+++ b/include/trace/events/pagemap.h
@@ -22,7 +22,7 @@
(folio_test_swapcache(folio) ? PAGEMAP_SWAPCACHE : 0) | \
(folio_test_swapbacked(folio) ? PAGEMAP_SWAPBACKED : 0) | \
(folio_test_mappedtodisk(folio) ? PAGEMAP_MAPPEDDISK : 0) | \
- (folio_test_private(folio) ? PAGEMAP_BUFFERS : 0) \
+ (folio_has_attached_private(folio) ? PAGEMAP_BUFFERS : 0) \
)
TRACE_EVENT(mm_lru_insertion,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 194188c292af..c822e831554b 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -4845,7 +4845,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
* will try to drop it before split and then check if the folio
* can be split or not. So skip the check here.
*/
- if (!folio_test_private(folio) &&
+ if (!folio_has_attached_private(folio) &&
folio_expected_ref_count(folio) != folio_ref_count(folio))
goto next;
diff --git a/mm/migrate.c b/mm/migrate.c
index a369d0c95c38..b7b92925a28c 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1327,7 +1327,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
* free the metadata, so the page can be freed.
*/
if (!src->mapping) {
- if (folio_test_private(src)) {
+ if (folio_has_attached_private(src)) {
try_to_free_buffers(src);
goto out;
}
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index eeab25d6ce36..499a35473e4f 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -2705,7 +2705,7 @@ bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio)
if (folio_test_set_dirty(folio))
return false;
- __folio_mark_dirty(folio, mapping, !folio_test_private(folio));
+ __folio_mark_dirty(folio, mapping, !folio_has_attached_private(folio));
if (mapping->host) {
/* !PageAnon && !swapper_space */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 80041e2b8049..dd6261c86279 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1029,7 +1029,7 @@ static void folio_check_dirty_writeback(struct folio *folio,
*writeback = folio_test_writeback(folio);
/* Verify dirty/writeback state if the filesystem supports it */
- if (!folio_test_private(folio))
+ if (!folio_has_attached_private(folio))
return;
mapping = folio_mapping(folio);
diff --git a/tools/mm/page-types.c b/tools/mm/page-types.c
index 7fc5a8be5997..47e4781c5fc3 100644
--- a/tools/mm/page-types.c
+++ b/tools/mm/page-types.c
@@ -73,7 +73,6 @@
#define KPF_RESERVED 32
#define KPF_MLOCKED 33
#define KPF_OWNER_2 34
-#define KPF_PRIVATE 35
#define KPF_PRIVATE_2 36
#define KPF_OWNER_PRIVATE 37
#define KPF_ARCH 38
@@ -131,7 +130,6 @@ static const char * const page_flag_names[] = {
[KPF_RESERVED] = "r:reserved",
[KPF_MLOCKED] = "m:mlocked",
[KPF_OWNER_2] = "d:owner_2",
- [KPF_PRIVATE] = "P:private",
[KPF_PRIVATE_2] = "p:private_2",
[KPF_OWNER_PRIVATE] = "O:owner_private",
[KPF_ARCH] = "h:arch",