summaryrefslogtreecommitdiff
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/damon/sysfs-schemes.c9
-rw-r--r--mm/hugetlb.c71
-rw-r--r--mm/memblock.c6
-rw-r--r--mm/memcontrol.c12
-rw-r--r--mm/memfd.c12
-rw-r--r--mm/memfd_luo.c25
-rw-r--r--mm/memory.c24
-rw-r--r--mm/memory_hotplug.c2
-rw-r--r--mm/migrate_device.c6
-rw-r--r--mm/page_alloc.c21
-rw-r--r--mm/rmap.c2
-rw-r--r--mm/slab_common.c2
-rw-r--r--mm/slub.c17
-rw-r--r--mm/vmalloc.c2
14 files changed, 128 insertions, 83 deletions
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index 245d63808411..a8014780edae 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -88,7 +88,6 @@ static void damon_sysfs_scheme_region_release(struct kobject *kobj)
struct damon_sysfs_scheme_region *region = container_of(kobj,
struct damon_sysfs_scheme_region, kobj);
- list_del(&region->list);
kfree(region);
}
@@ -164,7 +163,7 @@ static void damon_sysfs_scheme_regions_rm_dirs(
struct damon_sysfs_scheme_region *r, *next;
list_for_each_entry_safe(r, next, &regions->regions_list, list) {
- /* release function deletes it from the list */
+ list_del(&r->list);
kobject_put(&r->kobj);
regions->nr_regions--;
}
@@ -2594,6 +2593,7 @@ static int damon_sysfs_memcg_path_to_id(char *memcg_path, u64 *id)
if (damon_sysfs_memcg_path_eq(memcg, path, memcg_path)) {
*id = mem_cgroup_id(memcg);
found = true;
+ mem_cgroup_iter_break(NULL, memcg);
break;
}
}
@@ -2927,14 +2927,15 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes,
if (!region)
return;
region->sz_filter_passed = sz_filter_passed;
- list_add_tail(&region->list, &sysfs_regions->regions_list);
- sysfs_regions->nr_regions++;
if (kobject_init_and_add(&region->kobj,
&damon_sysfs_scheme_region_ktype,
&sysfs_regions->kobj, "%d",
sysfs_regions->nr_regions++)) {
kobject_put(&region->kobj);
+ return;
}
+ list_add_tail(&region->list, &sysfs_regions->regions_list);
+ sysfs_regions->nr_regions++;
}
int damon_sysfs_schemes_clear_regions(
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index f24bf49be047..4b80b167cc9c 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -116,6 +116,7 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init;
/* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta);
static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
unsigned long start, unsigned long end, bool take_locks);
@@ -413,21 +414,17 @@ static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
}
}
-/*
- * vma specific semaphore used for pmd sharing and fault/truncation
- * synchronization
- */
-int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
{
struct hugetlb_vma_lock *vma_lock;
/* Only establish in (flags) sharable vmas */
if (!vma || !(vma->vm_flags & VM_MAYSHARE))
- return 0;
+ return;
/* Should never get here with non-NULL vm_private_data */
if (vma->vm_private_data)
- return -EINVAL;
+ return;
vma_lock = kmalloc_obj(*vma_lock);
if (!vma_lock) {
@@ -442,15 +439,13 @@ int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
* allocation failure.
*/
pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
- return -EINVAL;
+ return;
}
kref_init(&vma_lock->refs);
init_rwsem(&vma_lock->rw_sema);
vma_lock->vma = vma;
vma->vm_private_data = vma_lock;
-
- return 0;
}
/* Helper that removes a struct file_region from the resv_map cache and returns
@@ -1147,28 +1142,20 @@ static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
}
}
-static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
+static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
{
VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
- VM_WARN_ON_ONCE_VMA(vma->vm_flags & VM_MAYSHARE, vma);
+ VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
- set_vma_private_data(vma, get_vma_private_data(vma) | flags);
+ set_vma_private_data(vma, (unsigned long)map);
}
-static void set_vma_desc_resv_map(struct vm_area_desc *desc, struct resv_map *map)
-{
- VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
- VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_MAYSHARE_BIT));
-
- desc->private_data = map;
-}
-
-static void set_vma_desc_resv_flags(struct vm_area_desc *desc, unsigned long flags)
+static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
{
- VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
- VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_MAYSHARE_BIT));
+ VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
+ VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
- desc->private_data = (void *)((unsigned long)desc->private_data | flags);
+ set_vma_private_data(vma, get_vma_private_data(vma) | flags);
}
static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
@@ -1178,13 +1165,6 @@ static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
return (get_vma_private_data(vma) & flag) != 0;
}
-static bool is_vma_desc_resv_set(struct vm_area_desc *desc, unsigned long flag)
-{
- VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
-
- return ((unsigned long)desc->private_data) & flag;
-}
-
bool __vma_private_lock(struct vm_area_struct *vma)
{
return !(vma->vm_flags & VM_MAYSHARE) &&
@@ -6553,7 +6533,7 @@ next:
long hugetlb_reserve_pages(struct inode *inode,
long from, long to,
- struct vm_area_desc *desc,
+ struct vm_area_struct *vma,
vma_flags_t vma_flags)
{
long chg = -1, add = -1, spool_resv, gbl_resv;
@@ -6571,6 +6551,12 @@ long hugetlb_reserve_pages(struct inode *inode,
}
/*
+ * vma specific semaphore used for pmd sharing and fault/truncation
+ * synchronization
+ */
+ hugetlb_vma_lock_alloc(vma);
+
+ /*
* Only apply hugepage reservation if asked. At fault time, an
* attempt will be made for VM_NORESERVE to allocate a page
* without using reserves
@@ -6582,9 +6568,9 @@ long hugetlb_reserve_pages(struct inode *inode,
* Shared mappings base their reservation on the number of pages that
* are already allocated on behalf of the file. Private mappings need
* to reserve the full area even if read-only as mprotect() may be
- * called to make the mapping read-write. Assume !desc is a shm mapping
+ * called to make the mapping read-write. Assume !vma is a shm mapping
*/
- if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
+ if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
/*
* resv_map can not be NULL as hugetlb_reserve_pages is only
* called for inodes for which resv_maps were created (see
@@ -6603,8 +6589,8 @@ long hugetlb_reserve_pages(struct inode *inode,
chg = to - from;
- set_vma_desc_resv_map(desc, resv_map);
- set_vma_desc_resv_flags(desc, HPAGE_RESV_OWNER);
+ set_vma_resv_map(vma, resv_map);
+ set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
}
if (chg < 0) {
@@ -6618,7 +6604,7 @@ long hugetlb_reserve_pages(struct inode *inode,
if (err < 0)
goto out_err;
- if (desc && !vma_desc_test(desc, VMA_MAYSHARE_BIT) && h_cg) {
+ if (vma && !vma_test(vma, VMA_MAYSHARE_BIT) && h_cg) {
/* For private mappings, the hugetlb_cgroup uncharge info hangs
* of the resv_map.
*/
@@ -6655,7 +6641,7 @@ long hugetlb_reserve_pages(struct inode *inode,
* consumed reservations are stored in the map. Hence, nothing
* else has to be done for private mappings here
*/
- if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
+ if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
add = region_add(resv_map, from, to, regions_needed, h, h_cg);
if (unlikely(add < 0)) {
@@ -6719,15 +6705,16 @@ out_uncharge_cgroup:
hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
chg * pages_per_huge_page(h), h_cg);
out_err:
- if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT))
+ hugetlb_vma_lock_free(vma);
+ if (!vma || vma_test(vma, VMA_MAYSHARE_BIT))
/* Only call region_abort if the region_chg succeeded but the
* region_add failed or didn't run.
*/
if (chg >= 0 && add < 0)
region_abort(resv_map, from, to, regions_needed);
- if (desc && is_vma_desc_resv_set(desc, HPAGE_RESV_OWNER)) {
+ if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
kref_put(&resv_map->refs, resv_map_release);
- set_vma_desc_resv_map(desc, NULL);
+ set_vma_resv_map(vma, NULL);
}
return err;
}
diff --git a/mm/memblock.c b/mm/memblock.c
index a6a1c91e276d..ccd43f3abb82 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -989,13 +989,15 @@ void __init_memblock memblock_free(void *ptr, size_t size)
int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size)
{
phys_addr_t end = base + size - 1;
- int ret;
+ int ret = 0;
memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
&base, &end, (void *)_RET_IP_);
kmemleak_free_part_phys(base, size);
- ret = memblock_remove_range(&memblock.reserved, base, size);
+
+ if (!slab_is_available() || IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
+ ret = memblock_remove_range(&memblock.reserved, base, size);
if (slab_is_available())
__free_reserved_area(base, base + size, -1);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c03d4787d466..177732fef010 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -679,7 +679,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, long val,
if (!val)
return;
- css_rstat_updated(&memcg->css, cpu);
+ __css_rstat_updated(&memcg->css, cpu);
statc_pcpu = memcg->vmstats_percpu;
for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
statc = this_cpu_ptr(statc_pcpu);
@@ -2796,7 +2796,7 @@ static inline void account_slab_nmi_safe(struct mem_cgroup *memcg,
struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
/* preemption is disabled in_nmi(). */
- css_rstat_updated(&memcg->css, smp_processor_id());
+ __css_rstat_updated(&memcg->css, smp_processor_id());
if (idx == NR_SLAB_RECLAIMABLE_B)
atomic_add(nr, &pn->slab_reclaimable);
else
@@ -3019,7 +3019,7 @@ static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val)
mod_memcg_state(memcg, MEMCG_KMEM, val);
} else {
/* preemption is disabled in_nmi(). */
- css_rstat_updated(&memcg->css, smp_processor_id());
+ __css_rstat_updated(&memcg->css, smp_processor_id());
atomic_add(val, &memcg->kmem_stat);
}
}
@@ -4352,6 +4352,9 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
lstats->state[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
+ memcg->vmstats->state[index] += slab;
+ if (parent)
+ parent->vmstats->state_pending[index] += slab;
}
if (atomic_read(&pn->slab_unreclaimable)) {
int slab = atomic_xchg(&pn->slab_unreclaimable, 0);
@@ -4360,6 +4363,9 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
lstats->state[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
+ memcg->vmstats->state[index] += slab;
+ if (parent)
+ parent->vmstats->state_pending[index] += slab;
}
}
}
diff --git a/mm/memfd.c b/mm/memfd.c
index fb425f4e315f..abe13b291ddc 100644
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -283,6 +283,12 @@ int memfd_add_seals(struct file *file, unsigned int seals)
goto unlock;
}
+ /*
+ * SEAL_EXEC implies SEAL_WRITE, making W^X from the start.
+ */
+ if (seals & F_SEAL_EXEC && inode->i_mode & 0111)
+ seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE;
+
if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
error = mapping_deny_writable(file->f_mapping);
if (error)
@@ -295,12 +301,6 @@ int memfd_add_seals(struct file *file, unsigned int seals)
}
}
- /*
- * SEAL_EXEC implies SEAL_WRITE, making W^X from the start.
- */
- if (seals & F_SEAL_EXEC && inode->i_mode & 0111)
- seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE;
-
*file_seals |= seals;
error = 0;
diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c
index b02b503c750d..59de210bee5f 100644
--- a/mm/memfd_luo.c
+++ b/mm/memfd_luo.c
@@ -50,6 +50,11 @@
* memfds are always opened with ``O_RDWR`` and ``O_LARGEFILE``. This property
* is maintained.
*
+ * Seals
+ * File seals set on the memfd are preserved and re-applied on restore.
+ * Only seals known to this LUO version (see ``MEMFD_LUO_ALL_SEALS``) may
+ * be present; preservation fails with ``-EOPNOTSUPP`` otherwise.
+ *
* Non-Preserved Properties
* ========================
*
@@ -61,10 +66,6 @@
* A memfd can be created with the ``MFD_CLOEXEC`` flag that sets the
* ``FD_CLOEXEC`` on the file. This flag is not preserved and must be set
* again after restore via ``fcntl()``.
- *
- * Seals
- * File seals are not preserved. The file is unsealed on restore and if
- * needed, must be sealed again via ``fcntl()``.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -259,7 +260,7 @@ static int memfd_luo_preserve(struct liveupdate_file_op_args *args)
struct inode *inode = file_inode(args->file);
struct memfd_luo_folio_ser *folios_ser;
struct memfd_luo_ser *ser;
- u64 nr_folios;
+ u64 nr_folios, inode_size;
int err = 0, seals;
inode_lock(inode);
@@ -285,7 +286,18 @@ static int memfd_luo_preserve(struct liveupdate_file_op_args *args)
}
ser->pos = args->file->f_pos;
- ser->size = i_size_read(inode);
+ inode_size = i_size_read(inode);
+
+ /*
+ * memfd_pin_folios() caps at UINT_MAX folios; refuse larger
+ * files to avoid silently preserving only a prefix.
+ */
+ if (DIV_ROUND_UP_ULL(inode_size, PAGE_SIZE) > UINT_MAX) {
+ err = -EFBIG;
+ goto err_free_ser;
+ }
+
+ ser->size = inode_size;
ser->seals = seals;
err = memfd_luo_preserve_folios(args->file, &ser->folios,
@@ -427,6 +439,7 @@ static int memfd_luo_retrieve_folios(struct file *file,
if (!folio) {
pr_err("Unable to restore folio at physical address: %llx\n",
phys);
+ err = -EIO;
goto put_folios;
}
index = pfolio->index;
diff --git a/mm/memory.c b/mm/memory.c
index ea6568571131..86a973119bd4 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -612,6 +612,21 @@ static void print_bad_page_map(struct vm_area_struct *vma,
dump_stack();
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
}
+
+static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
+{
+ switch (level) {
+ case PGTABLE_LEVEL_PTE:
+ return IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL);
+ case PGTABLE_LEVEL_PMD:
+ return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP);
+ case PGTABLE_LEVEL_PUD:
+ return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP);
+ default:
+ return false;
+ }
+}
+
#define print_bad_pte(vma, addr, pte, page) \
print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE)
@@ -684,7 +699,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
unsigned long addr, unsigned long pfn, bool special,
unsigned long long entry, enum pgtable_level level)
{
- if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
+ if (pgtable_level_has_pxx_special(level)) {
if (unlikely(special)) {
#ifdef CONFIG_FIND_NORMAL_PAGE
if (vma->vm_ops && vma->vm_ops->find_normal_page)
@@ -699,8 +714,9 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
return NULL;
}
/*
- * With CONFIG_ARCH_HAS_PTE_SPECIAL, any special page table
- * mappings (incl. shared zero folios) are marked accordingly.
+ * With working pte_special()/pmd_special()..., any special page
+ * table mappings (incl. shared zero folios) are marked
+ * accordingly.
*/
} else {
if (unlikely(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))) {
@@ -1739,7 +1755,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,
* consider uffd-wp bit when zap. For more information,
* see zap_install_uffd_wp_if_needed().
*/
- WARN_ON_ONCE(!vma_is_anonymous(vma));
+ WARN_ON_ONCE(!folio_test_anon(folio));
rss[mm_counter(folio)]--;
folio_remove_rmap_pte(folio, page, vma);
folio_put(folio);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 2a943ec57c85..40c7915dabe0 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1422,6 +1422,8 @@ static void remove_memory_blocks_and_altmaps(u64 start, u64 size)
altmap = mem->altmap;
mem->altmap = NULL;
+ /* drop the ref. we got via find_memory_block() */
+ put_device(&mem->dev);
remove_memory_block_devices(cur_start, memblock_size);
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index fbfe5715f635..19cd14b34114 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -840,7 +840,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
} else {
if (folio_is_zone_device(folio) &&
!folio_is_device_coherent(folio)) {
- goto abort;
+ goto free_abort;
}
entry = folio_mk_pmd(folio, vma->vm_page_prot);
if (vma->vm_flags & VM_WRITE)
@@ -850,7 +850,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
ptl = pmd_lock(vma->vm_mm, pmdp);
csa_ret = check_stable_address_space(vma->vm_mm);
if (csa_ret)
- goto abort;
+ goto unlock_abort;
/*
* Check for userfaultfd but do not deliver the fault. Instead,
@@ -893,6 +893,8 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
unlock_abort:
spin_unlock(ptl);
+free_abort:
+ pte_free(vma->vm_mm, pgtable);
abort:
for (i = 0; i < HPAGE_PMD_NR; i++)
src[i] &= ~MIGRATE_PFN_MIGRATE;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 227d58dc3de6..d49c254174da 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1035,6 +1035,7 @@ static inline bool page_expected_state(struct page *page,
#ifdef CONFIG_MEMCG
page->memcg_data |
#endif
+ page_pool_page_is_pp(page) |
(page->flags.f & check_flags)))
return false;
@@ -1061,6 +1062,8 @@ static const char *page_bad_reason(struct page *page, unsigned long flags)
if (unlikely(page->memcg_data))
bad_reason = "page still charged to cgroup";
#endif
+ if (unlikely(page_pool_page_is_pp(page)))
+ bad_reason = "page_pool leak";
return bad_reason;
}
@@ -1377,17 +1380,9 @@ __always_inline bool __free_pages_prepare(struct page *page,
mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
folio->mapping = NULL;
}
- if (unlikely(page_has_type(page))) {
- /* networking expects to clear its page type before releasing */
- if (is_check_pages_enabled()) {
- if (unlikely(PageNetpp(page))) {
- bad_page(page, "page_pool leak");
- return false;
- }
- }
+ if (unlikely(page_has_type(page)))
/* Reset the page_type (which overlays _mapcount) */
page->page_type = UINT_MAX;
- }
if (is_check_pages_enabled()) {
if (free_page_is_bad(page))
@@ -1808,9 +1803,9 @@ static inline bool should_skip_init(gfp_t flags)
inline void post_alloc_hook(struct page *page, unsigned int order,
gfp_t gfp_flags)
{
+ const bool zero_tags = gfp_flags & __GFP_ZEROTAGS;
bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
!should_skip_init(gfp_flags);
- bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS);
int i;
set_page_private(page, 0);
@@ -1832,11 +1827,11 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
*/
/*
- * If memory tags should be zeroed
- * (which happens only when memory should be initialized as well).
+ * Clearing tags can efficiently clear the memory for us as well, if
+ * required.
*/
if (zero_tags)
- init = !tag_clear_highpages(page, 1 << order);
+ init = tag_clear_highpages(page, 1 << order, /* clear_pages= */init);
if (!should_skip_kasan_unpoison(gfp_flags) &&
kasan_unpoison_pages(page, order, init)) {
diff --git a/mm/rmap.c b/mm/rmap.c
index 78b7fb5f367c..99e1b3dc390b 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2030,6 +2030,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(&pvmw)) {
+ nr_pages = 1;
+
/*
* If the folio is in an mlock()d vma, we must not swap it out.
*/
diff --git a/mm/slab_common.c b/mm/slab_common.c
index d5a70a831a2a..8b661fff5eed 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -2110,7 +2110,9 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
{
if (cache_has_sheaves(s)) {
+ cpus_read_lock();
flush_rcu_sheaves_on_cache(s);
+ cpus_read_unlock();
rcu_barrier();
}
diff --git a/mm/slub.c b/mm/slub.c
index 0baa906f39ab..a2bf3756ca7d 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4024,6 +4024,7 @@ void flush_rcu_sheaves_on_cache(struct kmem_cache *s)
struct slub_flush_work *sfw;
unsigned int cpu;
+ lockdep_assert_cpus_held();
mutex_lock(&flush_lock);
for_each_online_cpu(cpu) {
@@ -6883,6 +6884,22 @@ void kvfree(const void *addr)
EXPORT_SYMBOL(kvfree);
/**
+ * kvfree_atomic() - Free memory.
+ * @addr: Pointer to allocated memory.
+ *
+ * Same as kvfree(), but uses vfree_atomic() for vmalloc
+ * backed memory. Must not be called from NMI context.
+ */
+void kvfree_atomic(const void *addr)
+{
+ if (is_vmalloc_addr(addr))
+ vfree_atomic(addr);
+ else
+ kfree(addr);
+}
+EXPORT_SYMBOL(kvfree_atomic);
+
+/**
* kvfree_sensitive - Free a data object containing sensitive information.
* @addr: address of the data object to be freed.
* @len: length of the data object.
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index c31a8615a832..bb6ae08d18f5 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3203,7 +3203,7 @@ struct vm_struct *__get_vm_area_node(unsigned long size,
struct vm_struct *area;
unsigned long requested_size = size;
- BUG_ON(in_interrupt());
+ BUG_ON(in_nmi() || in_hardirq());
size = ALIGN(size, 1ul << shift);
if (unlikely(!size))
return NULL;