From 9eed9653a35fa2ae9191d0025d279ef7694a4cdf Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Wed, 1 Jul 2026 18:16:13 -0400 Subject: mm/mm_init: handle alloc_percpu failure in free_area_init_core_hotplug We miss a failed allocation check for pgdat->per_cpu_nodestats, which results in a NULL deref when we offset into the per-cpu area. Propagate -ENOMEM up the stack and leave per_cpu_nodestats pointing at boot_nodestats so a later online can retry the allocation. hotadd_init_pgdat() returns NULL on failure, which __try_online_node() already maps to -ENOMEM. On failure nothing needs to be unwound: - the node is never marked online - per_cpu_nodestats is left pointing at boot_nodestats - __add_memory_resource() cleans up pending memblock resources - later online attempts retry the per_cpu_nodestats allocation Link: https://lore.kernel.org/20260701221613.2818148-1-gourry@gourry.net Fixes: 75ef71840539 ("mm, vmstat: add infrastructure for per-node vmstats") Signed-off-by: Gregory Price Reported-by: Sashiko Link: https://sashiko.dev/#/patchset/20260627202243.758289-1-gourry%40gourry.net Acked-by: David Hildenbrand (Arm) Cc: Johannes Weiner Cc: Mel Gorman Cc: Mike Rapoport Cc: Oscar Salvador Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/memory_hotplug.h') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7c9d66729c60..06c58cb05779 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -289,7 +289,7 @@ static inline void __remove_memory(u64 start, u64 size) {} /* Default online_type (MMOP_*) when new memory blocks are added. */ extern enum mmop mhp_get_default_online_type(void); extern void mhp_set_default_online_type(enum mmop online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, -- cgit v1.2.3 From 5459c5be28c22c29afc1115c14911f7c5f8aff20 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:56 -0400 Subject: mm/memory_hotplug: add mhp_online_type_to_str() and export string helpers Add mhp_online_type_to_str() as the inverse of mhp_online_type_from_str(), and export both so a driver can render and parse the memory online type through its own sysfs interface. Link: https://lore.kernel.org/20260712154505.3564379-3-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Dave Jiang Acked-by: David Hildenbrand (Arm) Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- drivers/base/memory.c | 9 +++++++++ include/linux/memory_hotplug.h | 1 + 2 files changed, 10 insertions(+) (limited to 'include/linux/memory_hotplug.h') diff --git a/drivers/base/memory.c b/drivers/base/memory.c index bcfe2d9f4adb..5eead3346f1e 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -46,6 +46,15 @@ int mhp_online_type_from_str(const char *str) } return -EINVAL; } +EXPORT_SYMBOL_GPL(mhp_online_type_from_str); + +const char *mhp_online_type_to_str(int online_type) +{ + if (online_type < 0 || online_type >= (int)ARRAY_SIZE(online_type_to_str)) + return NULL; + return online_type_to_str[online_type]; +} +EXPORT_SYMBOL_GPL(mhp_online_type_to_str); #define to_memory_block(dev) container_of(dev, struct memory_block, dev) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 06c58cb05779..3f5d63892d90 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -127,6 +127,7 @@ extern int arch_add_memory(int nid, u64 start, u64 size, extern u64 max_mem_size; extern int mhp_online_type_from_str(const char *str); +const char *mhp_online_type_to_str(int online_type); /* If movable_node boot option specified */ extern bool movable_node_enabled; -- cgit v1.2.3 From 7e00a85f783e33ce6106891d6e3ad3c0fd77668f Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:58 -0400 Subject: mm/memory_hotplug: export mhp_get_default_online_type Drivers which may pass hotplug policy down to DAX need MMOP_ symbols and the mhp_get_default_online_type function for hotplug use cases. Some drivers (cxl) co-mingle their hotplug and devdax use-cases into the same driver code, and chose the dax_kmem path as the default driver path - making it difficult to require hotplug as a predicate to building the overall driver (it may break other non-hotplug use-cases). Export mhp_get_default_online_type function to allow these drivers to build when hotplug is disabled and still use the DAX use case. In the built-out case we simply return MMOP_OFFLINE as it's non-destructive. The internal function can never return -1 either, so we choose this to allow for defining the function with 'enum mmop'. Link: https://lore.kernel.org/20260712154505.3564379-5-gourry@gourry.net Signed-off-by: Gregory Price Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Acked-by: David Hildenbrand (Arm) Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 2 ++ mm/memory_hotplug.c | 1 + 2 files changed, 3 insertions(+) (limited to 'include/linux/memory_hotplug.h') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 3f5d63892d90..5240aa28bc90 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -317,6 +317,8 @@ extern struct zone *zone_for_pfn_range(enum mmop online_type, extern int arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params); void arch_remove_linear_mapping(u64 start, u64 size); +#else +static inline enum mmop mhp_get_default_online_type(void) { return MMOP_OFFLINE; } #endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 1dc0a1bb3a75..73c4a1b9895d 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -241,6 +241,7 @@ enum mmop mhp_get_default_online_type(void) return mhp_default_online_type; } +EXPORT_SYMBOL_GPL(mhp_get_default_online_type); void mhp_set_default_online_type(enum mmop online_type) { -- cgit v1.2.3 From 2b047c8a8c2769319a6ed17c47773c285b047080 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:44:59 -0400 Subject: mm/memory_hotplug: add __add_memory_driver_managed() with online_type arg Existing callers of add_memory_driver_managed cannot select the preferred online type (ZONE_NORMAL vs ZONE_MOVABLE), requiring it to hot-add memory as offline blocks, and then follow up by onlining each memory block individually. Most drivers prefer the system default, but the CXL driver wants to plumb a preferred policy through the dax kmem driver. Refactor APIs to add a new interface which allows the dax kmem module to select a preferred policy. Overriding the configured auto-online policy is only safe for known in-tree modules, where we know the override reflects a different, user-requested policy. We do not want arbitrary out-of-tree drivers silently overriding the system-wide onlining policy, so restrict the new interface to the kmem module using EXPORT_SYMBOL_FOR_MODULES() rather than a plain EXPORT_SYMBOL_GPL(). Other in-tree modules (e.g. cxl_core) can be added to the allowed list as the need arises. Refactor add_memory_driver_managed, extract __add_memory_driver_managed - Add proper kernel-doc for add_memory_driver_managed while refactoring - New helper accepts an explicit online_type. - New helper validates online_type is between OFFLINE and ONLINE_MOVABLE Refactor: add_memory_resource, extract __add_memory_resource - new helper accepts an explicit online_type Original APIs now explicitly pass the system-default to new helpers. No functional change for existing users. Link: https://lore.kernel.org/20260712154505.3564379-6-gourry@gourry.net Signed-off-by: Gregory Price Acked-by: David Hildenbrand (Arm) Reviewed-by: Pankaj Gupta Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 3 +++ mm/memory_hotplug.c | 61 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 8 deletions(-) (limited to 'include/linux/memory_hotplug.h') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 5240aa28bc90..d69b273ed304 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -295,6 +295,9 @@ extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, mhp_t mhp_flags); +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type); extern int add_memory_driver_managed(int nid, u64 start, u64 size, const char *resource_name, mhp_t mhp_flags); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 73c4a1b9895d..fe6b38ef9aac 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1497,10 +1497,10 @@ out: * * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ -int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +static int __add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags, + enum mmop online_type) { struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) }; - enum mmop online_type = mhp_get_default_online_type(); enum memblock_flags memblock_flags = MEMBLOCK_NONE; struct memory_group *group = NULL; u64 start, size; @@ -1588,7 +1588,7 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) merge_system_ram_resource(res); /* online pages if requested */ - if (mhp_get_default_online_type() != MMOP_OFFLINE) + if (online_type != MMOP_OFFLINE) walk_memory_blocks(start, size, &online_type, online_memory_block); @@ -1606,7 +1606,13 @@ error_mem_hotplug_end: return ret; } -/* requires device_hotplug_lock, see add_memory_resource() */ +int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) +{ + return __add_memory_resource(nid, res, mhp_flags, + mhp_get_default_online_type()); +} + +/* requires device_hotplug_lock, see __add_memory_resource() */ int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) { struct resource *res; @@ -1634,7 +1640,15 @@ int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) } EXPORT_SYMBOL_GPL(add_memory); -/* +/** + * __add_memory_driver_managed - add driver-managed memory with explicit online_type + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * @online_type: Auto-Online behavior (offline, online, kernel, movable) + * * Add special, driver-managed memory to the system as system RAM. Such * memory is not exposed via the raw firmware-provided memmap as system * RAM, instead, it is detected and added by a driver - during cold boot, @@ -1642,6 +1656,7 @@ EXPORT_SYMBOL_GPL(add_memory); * * Reasons why this memory should not be used for the initial memmap of a * kexec kernel or for placing kexec images: + * * - The booting kernel is in charge of determining how this memory will be * used (e.g., use persistent memory as system RAM) * - Coordination with a hypervisor is required before this memory @@ -1654,9 +1669,12 @@ EXPORT_SYMBOL_GPL(add_memory); * * The resource_name (visible via /proc/iomem) has to have the format * "System RAM ($DRIVER)". + * + * Return: 0 on success, negative error code on failure. */ -int add_memory_driver_managed(int nid, u64 start, u64 size, - const char *resource_name, mhp_t mhp_flags) +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type) { struct resource *res; int rc; @@ -1666,6 +1684,9 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, resource_name[strlen(resource_name) - 1] != ')') return -EINVAL; + if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE) + return -EINVAL; + lock_device_hotplug(); res = register_memory_resource(start, size, resource_name); @@ -1674,7 +1695,7 @@ int add_memory_driver_managed(int nid, u64 start, u64 size, goto out_unlock; } - rc = add_memory_resource(nid, res, mhp_flags); + rc = __add_memory_resource(nid, res, mhp_flags, online_type); if (rc < 0) release_memory_resource(res); @@ -1682,6 +1703,30 @@ out_unlock: unlock_device_hotplug(); return rc; } +EXPORT_SYMBOL_FOR_MODULES(__add_memory_driver_managed, "kmem"); + +/** + * add_memory_driver_managed - add driver-managed memory + * @nid: NUMA node ID where the memory will be added + * @start: Start physical address of the memory range + * @size: Size of the memory range in bytes + * @resource_name: Resource name in format "System RAM ($DRIVER)" + * @mhp_flags: Memory hotplug flags + * + * Add driver-managed memory with the system default online type set by + * build config or kernel boot parameter. + * + * See __add_memory_driver_managed for more details. + * + * Return: 0 on success, negative error code on failure. + */ +int add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags) +{ + return __add_memory_driver_managed(nid, start, size, resource_name, + mhp_flags, + mhp_get_default_online_type()); +} EXPORT_SYMBOL_GPL(add_memory_driver_managed); /* -- cgit v1.2.3 From c17ba40c37e890987dce846cf6e1ec6a2bf79ea3 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Sun, 12 Jul 2026 11:45:00 -0400 Subject: mm/memory_hotplug: add offline_and_remove_memory_ranges() offline_and_remove_memory() handles a single contiguous range. Callers that manage a device composed of several ranges (dax/kmem) currently have to call it in a loop, which gives up atomicity. In addition to pushing rollback logic into the driver, the lack of atomicity creates a race condition between system daemons trying to manage the same resource: - Manager 1: Offlines memory blocks. Removes device. ^^^^ - Manager 2: Detects offline memory blocks, re-onlines them. Add offline_and_remove_memory_ranges(), which takes an array of ranges and processes them as one operation under a single lock_device_hotplug(): - Phase 1 offlines every block of every range. - Phase 2 removes the ranges only if all ranges are offline. - If any offline fails, the whole operation is reverted. This gives callers all-or-nothing semantics for the offline step, so a failed or interrupted unplug leaves the device in a consistent state. This also resolves the battling managers race - the second manager's operation simply fails when the block is destroyed / cannot be onlined. offline_and_remove_memory() becomes a thin wrapper that passes its single range to the new helper, so the offline/rollback logic lives in one place. Link: https://lore.kernel.org/20260712154505.3564379-7-gourry@gourry.net Signed-off-by: Gregory Price Suggested-by: David Hildenbrand (Arm) Reviewed-by: Dave Jiang Acked-by: David Hildenbrand (Arm) Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Danilo Krummrich Cc: Greg Kroah-Hartman Cc: Hannes Reinecke Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: Mike Rapoport Cc: Oscar Salvador Cc: Pankaj Gupta Cc: "Rafael J. Wysocki" Cc: Shuah Khan Cc: Suren Baghdasaryan Cc: Vishal Verma Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 8 ++++ mm/memory_hotplug.c | 94 ++++++++++++++++++++++++++++++------------ 2 files changed, 75 insertions(+), 27 deletions(-) (limited to 'include/linux/memory_hotplug.h') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index d69b273ed304..b39605d30896 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -268,6 +268,8 @@ extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages, extern int remove_memory(u64 start, u64 size); extern void __remove_memory(u64 start, u64 size); extern int offline_and_remove_memory(u64 start, u64 size); +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges); #else static inline void try_offline_node(int nid) {} @@ -284,6 +286,12 @@ static inline int remove_memory(u64 start, u64 size) } static inline void __remove_memory(u64 start, u64 size) {} + +static inline int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + return -EBUSY; +} #endif /* CONFIG_MEMORY_HOTREMOVE */ #ifdef CONFIG_MEMORY_HOTPLUG diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index fe6b38ef9aac..226ab9cb078a 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -2432,58 +2432,98 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg) */ int offline_and_remove_memory(u64 start, u64 size) { - const unsigned long mb_count = size / memory_block_size_bytes(); + struct range range = { + .start = start, + .end = start + size - 1, + }; + + return offline_and_remove_memory_ranges(&range, 1); +} +EXPORT_SYMBOL_GPL(offline_and_remove_memory); + +/** + * offline_and_remove_memory_ranges - offline and remove multiple memory ranges + * @ranges: array of physical address ranges to offline and remove + * @nr_ranges: number of entries in @ranges + * + * Offline and remove several memory ranges as one operation, serialized + * against other hotplug operations by a single lock_device_hotplug(). + * + * This offlines all ranges before removing any of them. If offlining any + * range fails, the entire process is reverted and nothing is removed. + * This provides a fully atomic semantic for unplugging an entire device. + * + * Each range must be memory-block aligned in start and size. + * + * Return: 0 on success, negative errno on failure (never positive). On + * failure no range has been removed. + */ +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + unsigned long mb_count = 0; uint8_t *online_types, *tmp; - int rc; + unsigned int i; + int rc = 0; - if (!IS_ALIGNED(start, memory_block_size_bytes()) || - !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + if (!ranges || !nr_ranges) return -EINVAL; + for (i = 0; i < nr_ranges; i++) { + const u64 start = ranges[i].start; + const u64 size = range_len(&ranges[i]); + + if (!IS_ALIGNED(start, memory_block_size_bytes()) || + !IS_ALIGNED(size, memory_block_size_bytes()) || !size) + return -EINVAL; + mb_count += size / memory_block_size_bytes(); + } + /* - * We'll remember the old online type of each memory block, so we can - * try to revert whatever we did when offlining one memory block fails - * after offlining some others succeeded. + * Remember the old online type of every memory block across all ranges, + * so we can revert if offlining a later block fails. All entries start + * as MMOP_OFFLINE so blocks we never touched are skipped on rollback. */ online_types = kmalloc_array(mb_count, sizeof(*online_types), GFP_KERNEL); if (!online_types) return -ENOMEM; - /* - * Initialize all states to MMOP_OFFLINE, so when we abort processing in - * try_offline_memory_block(), we'll skip all unprocessed blocks in - * try_reonline_memory_block(). - */ memset(online_types, MMOP_OFFLINE, mb_count); lock_device_hotplug(); - tmp = online_types; - rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block); - /* - * In case we succeeded to offline all memory, remove it. - * This cannot fail as it cannot get onlined in the meantime. + * Phase 1: offline every block in every range. An already-offline + * block folds to success, so out-of-band offlining never blocks unplug. */ - if (!rc) { - rc = try_remove_memory(start, size); + tmp = online_types; + for (i = 0; i < nr_ranges; i++) { + rc = walk_memory_blocks(ranges[i].start, range_len(&ranges[i]), + &tmp, try_offline_memory_block); if (rc) - pr_err("%s: Failed to remove memory: %d", __func__, rc); + break; } - /* - * Rollback what we did. While memory onlining might theoretically fail - * (nacked by a notifier), it barely ever happens. - */ + /* If any failure occurred at all, rollback any changes and bail */ if (rc) { tmp = online_types; - walk_memory_blocks(start, size, &tmp, - try_reonline_memory_block); + for (i = 0; i < nr_ranges; i++) + walk_memory_blocks(ranges[i].start, + range_len(&ranges[i]), &tmp, + try_reonline_memory_block); + goto out_unlock; } + + /* Phase 2: Remove. This should never fail holding the hotplug lock */ + for (i = 0; i < nr_ranges; i++) + WARN_ON_ONCE(try_remove_memory(ranges[i].start, + range_len(&ranges[i]))); + +out_unlock: unlock_device_hotplug(); kfree(online_types); return rc; } -EXPORT_SYMBOL_GPL(offline_and_remove_memory); +EXPORT_SYMBOL_GPL(offline_and_remove_memory_ranges); #endif /* CONFIG_MEMORY_HOTREMOVE */ -- cgit v1.2.3