diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2022-05-26 12:32:41 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2022-05-26 12:32:41 -0700 |
commit | 98931dd95fd489fcbfa97da563505a6f071d7c77 (patch) | |
tree | 44683fc4a92efa614acdca2742a7ff19d26da1e3 /mm/z3fold.c | |
parent | df202b452fe6c6d6f1351bad485e2367ef1e644e (diff) | |
parent | f403f22f8ccb12860b2b62fec3173c6ccd45938b (diff) | |
download | lwn-98931dd95fd489fcbfa97da563505a6f071d7c77.tar.gz lwn-98931dd95fd489fcbfa97da563505a6f071d7c77.zip |
Merge tag 'mm-stable-2022-05-25' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull MM updates from Andrew Morton:
"Almost all of MM here. A few things are still getting finished off,
reviewed, etc.
- Yang Shi has improved the behaviour of khugepaged collapsing of
readonly file-backed transparent hugepages.
- Johannes Weiner has arranged for zswap memory use to be tracked and
managed on a per-cgroup basis.
- Munchun Song adds a /proc knob ("hugetlb_optimize_vmemmap") for
runtime enablement of the recent huge page vmemmap optimization
feature.
- Baolin Wang contributes a series to fix some issues around hugetlb
pagetable invalidation.
- Zhenwei Pi has fixed some interactions between hwpoisoned pages and
virtualization.
- Tong Tiangen has enabled the use of the presently x86-only
page_table_check debugging feature on arm64 and riscv.
- David Vernet has done some fixup work on the memcg selftests.
- Peter Xu has taught userfaultfd to handle write protection faults
against shmem- and hugetlbfs-backed files.
- More DAMON development from SeongJae Park - adding online tuning of
the feature and support for monitoring of fixed virtual address
ranges. Also easier discovery of which monitoring operations are
available.
- Nadav Amit has done some optimization of TLB flushing during
mprotect().
- Neil Brown continues to labor away at improving our swap-over-NFS
support.
- David Hildenbrand has some fixes to anon page COWing versus
get_user_pages().
- Peng Liu fixed some errors in the core hugetlb code.
- Joao Martins has reduced the amount of memory consumed by
device-dax's compound devmaps.
- Some cleanups of the arch-specific pagemap code from Anshuman
Khandual.
- Muchun Song has found and fixed some errors in the TLB flushing of
transparent hugepages.
- Roman Gushchin has done more work on the memcg selftests.
... and, of course, many smaller fixes and cleanups. Notably, the
customary million cleanup serieses from Miaohe Lin"
* tag 'mm-stable-2022-05-25' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (381 commits)
mm: kfence: use PAGE_ALIGNED helper
selftests: vm: add the "settings" file with timeout variable
selftests: vm: add "test_hmm.sh" to TEST_FILES
selftests: vm: check numa_available() before operating "merge_across_nodes" in ksm_tests
selftests: vm: add migration to the .gitignore
selftests/vm/pkeys: fix typo in comment
ksm: fix typo in comment
selftests: vm: add process_mrelease tests
Revert "mm/vmscan: never demote for memcg reclaim"
mm/kfence: print disabling or re-enabling message
include/trace/events/percpu.h: cleanup for "percpu: improve percpu_alloc_percpu event trace"
include/trace/events/mmflags.h: cleanup for "tracing: incorrect gfp_t conversion"
mm: fix a potential infinite loop in start_isolate_page_range()
MAINTAINERS: add Muchun as co-maintainer for HugeTLB
zram: fix Kconfig dependency warning
mm/shmem: fix shmem folio swapoff hang
cgroup: fix an error handling path in alloc_pagecache_max_30M()
mm: damon: use HPAGE_PMD_SIZE
tracing: incorrect isolate_mote_t cast in mm_vmscan_lru_isolate
nodemask.h: fix compilation error with GCC12
...
Diffstat (limited to 'mm/z3fold.c')
-rw-r--r-- | mm/z3fold.c | 64 |
1 files changed, 17 insertions, 47 deletions
diff --git a/mm/z3fold.c b/mm/z3fold.c index b3c0577b8095..83b5a3514427 100644 --- a/mm/z3fold.c +++ b/mm/z3fold.c @@ -297,9 +297,6 @@ static inline void free_handle(unsigned long handle, struct z3fold_header *zhdr) int i; bool is_free; - if (handle & (1 << PAGE_HEADLESS)) - return; - if (WARN_ON(*(unsigned long *)handle == 0)) return; @@ -345,7 +342,7 @@ static struct file_system_type z3fold_fs = { }; static struct vfsmount *z3fold_mnt; -static int z3fold_mount(void) +static int __init z3fold_mount(void) { int ret = 0; @@ -420,7 +417,6 @@ static void free_z3fold_page(struct page *page, bool headless) __ClearPageMovable(page); unlock_page(page); } - ClearPagePrivate(page); __free_page(page); } @@ -521,6 +517,8 @@ static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) list_add(&zhdr->buddy, &pool->stale); queue_work(pool->release_wq, &pool->work); spin_unlock(&pool->stale_lock); + + atomic64_dec(&pool->pages_nr); } static void release_z3fold_page(struct kref *ref) @@ -738,13 +736,9 @@ static struct z3fold_header *compact_single_buddy(struct z3fold_header *zhdr) return new_zhdr; out_fail: - if (new_zhdr) { - if (kref_put(&new_zhdr->refcount, release_z3fold_page_locked)) - atomic64_dec(&pool->pages_nr); - else { - add_to_unbuddied(pool, new_zhdr); - z3fold_page_unlock(new_zhdr); - } + if (new_zhdr && !kref_put(&new_zhdr->refcount, release_z3fold_page_locked)) { + add_to_unbuddied(pool, new_zhdr); + z3fold_page_unlock(new_zhdr); } return NULL; @@ -817,10 +811,8 @@ static void do_compact_page(struct z3fold_header *zhdr, bool locked) list_del_init(&zhdr->buddy); spin_unlock(&pool->lock); - if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { - atomic64_dec(&pool->pages_nr); + if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) return; - } if (test_bit(PAGE_STALE, &page->private) || test_and_set_bit(PAGE_CLAIMED, &page->private)) { @@ -830,9 +822,7 @@ static void do_compact_page(struct z3fold_header *zhdr, bool locked) if (!zhdr->foreign_handles && buddy_single(zhdr) && zhdr->mapped_count == 0 && compact_single_buddy(zhdr)) { - if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) - atomic64_dec(&pool->pages_nr); - else { + if (!kref_put(&zhdr->refcount, release_z3fold_page_locked)) { clear_bit(PAGE_CLAIMED, &page->private); z3fold_page_unlock(zhdr); } @@ -877,7 +867,6 @@ lookup: /* Re-check under lock. */ spin_lock(&pool->lock); - l = &unbuddied[i]; if (unlikely(zhdr != list_first_entry(READ_ONCE(l), struct z3fold_header, buddy)) || !z3fold_page_trylock(zhdr)) { @@ -1064,9 +1053,6 @@ static void z3fold_destroy_pool(struct z3fold_pool *pool) * performed first. If no suitable free region is found, then a new page is * allocated and added to the pool to satisfy the request. * - * gfp should not set __GFP_HIGHMEM as highmem pages cannot be used - * as z3fold pool pages. - * * Return: 0 if success and handle is set, otherwise -EINVAL if the size or * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate * a new page. @@ -1094,10 +1080,8 @@ retry: if (zhdr) { bud = get_free_buddy(zhdr, chunks); if (bud == HEADLESS) { - if (kref_put(&zhdr->refcount, + if (!kref_put(&zhdr->refcount, release_z3fold_page_locked)) - atomic64_dec(&pool->pages_nr); - else z3fold_page_unlock(zhdr); pr_err("No free chunks in unbuddied\n"); WARN_ON(1); @@ -1190,9 +1174,9 @@ headless: * @handle: handle associated with the allocation returned by z3fold_alloc() * * In the case that the z3fold page in which the allocation resides is under - * reclaim, as indicated by the PG_reclaim flag being set, this function - * only sets the first|last_chunks to 0. The page is actually freed - * once both buddies are evicted (see z3fold_reclaim_page() below). + * reclaim, as indicated by the PAGE_CLAIMED flag being set, this function + * only sets the first|middle|last_chunks to 0. The page is actually freed + * once all buddies are evicted (see z3fold_reclaim_page() below). */ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) { @@ -1244,13 +1228,11 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) if (!page_claimed) free_handle(handle, zhdr); - if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) { - atomic64_dec(&pool->pages_nr); + if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) return; - } if (page_claimed) { /* the page has not been claimed by us */ - z3fold_page_unlock(zhdr); + put_z3fold_header(zhdr); return; } if (test_and_set_bit(NEEDS_COMPACTING, &page->private)) { @@ -1259,9 +1241,6 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) return; } if (zhdr->cpu < 0 || !cpu_online(zhdr->cpu)) { - spin_lock(&pool->lock); - list_del_init(&zhdr->buddy); - spin_unlock(&pool->lock); zhdr->cpu = -1; kref_get(&zhdr->refcount); clear_bit(PAGE_CLAIMED, &page->private); @@ -1358,9 +1337,7 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) break; } if (!z3fold_page_trylock(zhdr)) { - if (kref_put(&zhdr->refcount, - release_z3fold_page)) - atomic64_dec(&pool->pages_nr); + kref_put(&zhdr->refcount, release_z3fold_page); zhdr = NULL; continue; /* can't evict at this point */ } @@ -1371,10 +1348,8 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) */ if (zhdr->foreign_handles || test_and_set_bit(PAGE_CLAIMED, &page->private)) { - if (kref_put(&zhdr->refcount, + if (!kref_put(&zhdr->refcount, release_z3fold_page_locked)) - atomic64_dec(&pool->pages_nr); - else z3fold_page_unlock(zhdr); zhdr = NULL; continue; /* can't evict such page */ @@ -1452,7 +1427,6 @@ next: if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { kmem_cache_free(pool->c_handle, slots); - atomic64_dec(&pool->pages_nr); return 0; } /* @@ -1638,7 +1612,6 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa INIT_LIST_HEAD(&new_zhdr->buddy); new_mapping = page_mapping(page); __ClearPageMovable(page); - ClearPagePrivate(page); get_page(newpage); z3fold_page_lock(new_zhdr); @@ -1658,7 +1631,6 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work); - page_mapcount_reset(page); clear_bit(PAGE_CLAIMED, &page->private); put_page(page); return 0; @@ -1676,10 +1648,8 @@ static void z3fold_page_putback(struct page *page) if (!list_empty(&zhdr->buddy)) list_del_init(&zhdr->buddy); INIT_LIST_HEAD(&page->lru); - if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { - atomic64_dec(&pool->pages_nr); + if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) return; - } spin_lock(&pool->lock); list_add(&page->lru, &pool->lru); spin_unlock(&pool->lock); |