summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-06-21 09:46:14 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-06-21 09:46:14 -0700
commitd639d9fa162aadec1ae9980c4dcf6e50bd2f8290 (patch)
treef9bc42aecac73a391e96e08d944a8fe962821356 /kernel
parent1e762b53a86d7ed19016c66cc1883e4b9f8b2c1b (diff)
parente98a9c61721c14bcd29f11f4802e52e908701f7a (diff)
downloadlwn-d639d9fa162aadec1ae9980c4dcf6e50bd2f8290.tar.gz
lwn-d639d9fa162aadec1ae9980c4dcf6e50bd2f8290.zip
Merge tag 'liveupdate-v7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux
Pull liveupdate updates from Mike Rapoport: "Kexec Handover (KHO): - make memory preservation compatible with deferred initialization of the memory map Live Update Orchestrator (LUO): - add LIVEUPDATE_SESSION_GET_NAME ioctl and parameter verification for LIVEUPDATE_IOCTL_CREATE_SESSION ioctl - documentation updates for liveupdate=on command line option, systemd support and the current compatibility status - remove the fixed limits on the number of files that can be preserved within a single session, and the total number of sessions managed by the LUO Misc fixes: - reference count incoming File-Lifecycle-Bound (FLB) data so it cannot be freed while a subsystem is still using it - fixes for a TOCTOU race in luo_session_retrieve(), a use- after-free in the file finish and unpreserve paths, concurrent session mutations during reboot and serialization on preserve_context kexec - make sure ioctls for incoming LUO sessions are blocked for outgoing sessions and vice versa - make sure KHO scratch size is always aligned by CMA_MIN_ALIGNMENT_BYTES - fix memblock tests build issue introduced by KHO changes" * tag 'liveupdate-v7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux: (36 commits) liveupdate: Document that retrieve failure is permanent docs: memfd_preservation: fix rendering of ABI documentation selftests/liveupdate: Add stress-files kexec test selftests/liveupdate: Add stress-sessions kexec test selftests/liveupdate: Test session and file limit removal liveupdate: Remove limit on the number of files per session liveupdate: Remove limit on the number of sessions liveupdate: defer session block allocation and physical address setting kho: add support for linked-block serialization liveupdate: Extract luo_session_deserialize_one helper liveupdate: Extract luo_file_deserialize_one helper liveupdate: register luo_ser as KHO subtree liveupdate: centralize state management into struct luo_ser liveupdate: avoid mixing cleanup guards with goto in luo_session_retrieve_fd liveupdate: change file_set->count type to u64 for type safety liveupdate: Remove unused ser field from struct luo_session liveupdate: fix u-a-f in luo_file_unpreserve_files() and luo_file_finish() liveupdate: block session mutations during reboot liveupdate: fix TOCTOU race in luo_session_retrieve() liveupdate: skip serialization for context-preserving kexec ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/kexec_core.c8
-rw-r--r--kernel/liveupdate/Kconfig2
-rw-r--r--kernel/liveupdate/Makefile1
-rw-r--r--kernel/liveupdate/kexec_handover.c84
-rw-r--r--kernel/liveupdate/kho_block.c416
-rw-r--r--kernel/liveupdate/luo_core.c103
-rw-r--r--kernel/liveupdate/luo_file.c210
-rw-r--r--kernel/liveupdate/luo_flb.c112
-rw-r--r--kernel/liveupdate/luo_internal.h18
-rw-r--r--kernel/liveupdate/luo_session.c336
10 files changed, 865 insertions, 425 deletions
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index a43d2da0fe3e..dc770b9a6d05 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -1146,9 +1146,11 @@ int kernel_kexec(void)
goto Unlock;
}
- error = liveupdate_reboot();
- if (error)
- goto Unlock;
+ if (!kexec_image->preserve_context) {
+ error = liveupdate_reboot();
+ if (error)
+ goto Unlock;
+ }
#ifdef CONFIG_KEXEC_JUMP
if (kexec_image->preserve_context) {
diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig
index 1a8513f16ef7..c13af38ba23a 100644
--- a/kernel/liveupdate/Kconfig
+++ b/kernel/liveupdate/Kconfig
@@ -1,12 +1,10 @@
# SPDX-License-Identifier: GPL-2.0-only
menu "Live Update and Kexec HandOver"
- depends on !DEFERRED_STRUCT_PAGE_INIT
config KEXEC_HANDOVER
bool "kexec handover"
depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE
- depends on !DEFERRED_STRUCT_PAGE_INIT
select MEMBLOCK_KHO_SCRATCH
select KEXEC_FILE
select LIBFDT
diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile
index d2f779cbe279..eec9d3ae07eb 100644
--- a/kernel/liveupdate/Makefile
+++ b/kernel/liveupdate/Makefile
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
luo-y := \
+ kho_block.o \
luo_core.o \
luo_file.o \
luo_flb.o \
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index 1b592d86dc48..4834a809985a 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -459,6 +459,31 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages)
}
EXPORT_SYMBOL_GPL(kho_restore_pages);
+/*
+ * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, struct pages in higher memory regions
+ * may not be initialized yet at the time KHO deserializes preserved memory.
+ * KHO uses the struct page to store metadata and a later initialization would
+ * overwrite it.
+ * Ensure all the struct pages in the preservation are
+ * initialized. kho_preserved_memory_reserve() marks the reservation as noinit
+ * to make sure they don't get re-initialized later.
+ */
+static struct page *__init kho_get_preserved_page(phys_addr_t phys,
+ unsigned int order)
+{
+ unsigned long pfn = PHYS_PFN(phys);
+ int nid;
+
+ if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT))
+ return pfn_to_page(pfn);
+
+ nid = early_pfn_to_nid(pfn);
+ for (unsigned long i = 0; i < (1UL << order); i++)
+ init_deferred_page(pfn + i, nid);
+
+ return pfn_to_page(pfn);
+}
+
static int __init kho_preserved_memory_reserve(phys_addr_t phys,
unsigned int order)
{
@@ -467,7 +492,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys,
u64 sz;
sz = 1 << (order + PAGE_SHIFT);
- page = phys_to_page(phys);
+ page = kho_get_preserved_page(phys, order);
/* Reserve the memory preserved in KHO in memblock */
memblock_reserve(phys, sz);
@@ -593,20 +618,30 @@ early_param("kho_scratch", kho_parse_scratch_size);
static void __init scratch_size_update(void)
{
- phys_addr_t size;
+ /*
+ * If fixed sizes are not provided via command line, calculate them
+ * now.
+ */
+ if (scratch_scale) {
+ phys_addr_t size;
- if (!scratch_scale)
- return;
+ size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT,
+ NUMA_NO_NODE);
+ size = size * scratch_scale / 100;
+ scratch_size_lowmem = size;
- size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT,
- NUMA_NO_NODE);
- size = size * scratch_scale / 100;
- scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES);
+ size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
+ NUMA_NO_NODE);
+ size = size * scratch_scale / 100 - scratch_size_lowmem;
+ scratch_size_global = size;
+ }
- size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
- NUMA_NO_NODE);
- size = size * scratch_scale / 100 - scratch_size_lowmem;
- scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES);
+ /*
+ * Scratch areas are released as MIGRATE_CMA. Round them up to the right
+ * size.
+ */
+ scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES);
+ scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES);
}
static phys_addr_t __init scratch_size_node(int nid)
@@ -1584,35 +1619,10 @@ err_free_scratch:
}
fs_initcall(kho_init);
-static void __init kho_release_scratch(void)
-{
- phys_addr_t start, end;
- u64 i;
-
- memmap_init_kho_scratch_pages();
-
- /*
- * Mark scratch mem as CMA before we return it. That way we
- * ensure that no kernel allocations happen on it. That means
- * we can reuse it as scratch memory again later.
- */
- __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE,
- MEMBLOCK_KHO_SCRATCH, &start, &end, NULL) {
- ulong start_pfn = pageblock_start_pfn(PFN_DOWN(start));
- ulong end_pfn = pageblock_align(PFN_UP(end));
- ulong pfn;
-
- for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages)
- init_pageblock_migratetype(pfn_to_page(pfn),
- MIGRATE_CMA, false);
- }
-}
-
void __init kho_memory_init(void)
{
if (kho_in.scratch_phys) {
kho_scratch = phys_to_virt(kho_in.scratch_phys);
- kho_release_scratch();
if (kho_mem_retrieve(kho_get_fdt()))
kho_in.fdt_phys = 0;
diff --git a/kernel/liveupdate/kho_block.c b/kernel/liveupdate/kho_block.c
new file mode 100644
index 000000000000..0d2a342ef422
--- /dev/null
+++ b/kernel/liveupdate/kho_block.c
@@ -0,0 +1,416 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@soleen.com>
+ */
+
+/**
+ * DOC: KHO Serialization Blocks
+ *
+ * KHO provides a mechanism to preserve stateful data across a kexec handover
+ * by serializing it into memory blocks, and provides the common
+ * infrastructure for managing these blocks.
+ *
+ * Each block consists of a header (struct kho_block_header_ser) followed by an
+ * array of serialized entries. Multiple blocks are linked together via a
+ * physical pointer in the header, forming a linked list that can be easily
+ * traversed in both the current and the next kernel.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/io.h>
+#include <linux/kexec_handover.h>
+#include <linux/kho/abi/block.h>
+#include <linux/kho_block.h>
+#include <linux/slab.h>
+
+/*
+ * Safeguard limit for the number of serialization blocks. This is used to
+ * prevent infinite loops and excessive memory allocation in case of memory
+ * corruption in the preserved state.
+ *
+ * With a 4KB page size, 10k blocks is about 40MB. For 32-byte entries
+ * (e.g. 4 u64s), each block holds up to 127 entries (accounting for the
+ * 16-byte header), allowing the block set to hold up to 1.27M entries.
+ */
+#define KHO_MAX_BLOCKS 10000
+
+/**
+ * kho_block_set_init - Initialize a block set.
+ * @bs: The block set to initialize.
+ * @entry_size: The size of each entry in the blocks.
+ */
+void kho_block_set_init(struct kho_block_set *bs, size_t entry_size)
+{
+ *bs = (struct kho_block_set)KHO_BLOCK_SET_INIT(*bs, entry_size);
+ WARN_ON_ONCE(!bs->count_per_block);
+}
+
+/* Serialized entries start immediately after the block header */
+static void *kho_block_entries(struct kho_block *block)
+{
+ return (void *)(block->ser + 1);
+}
+
+/* Get the address of the serialized entry at the specified index */
+static void *kho_block_entry(struct kho_block_set_it *it, u64 index)
+{
+ return kho_block_entries(it->block) + (index * it->bs->entry_size);
+}
+
+/* Free serialized data */
+static void kho_block_free_ser(struct kho_block_set *bs,
+ struct kho_block_header_ser *ser)
+{
+ if (bs->incoming)
+ kho_restore_free(ser);
+ else
+ kho_unpreserve_free(ser);
+}
+
+static struct kho_block_header_ser *kho_block_alloc_ser(struct kho_block_set *bs)
+{
+ WARN_ON_ONCE(bs->incoming);
+ return kho_alloc_preserve(KHO_BLOCK_SIZE);
+}
+
+static int kho_block_add(struct kho_block_set *bs,
+ struct kho_block_header_ser *ser)
+{
+ struct kho_block *block, *last;
+
+ if (bs->nblocks >= KHO_MAX_BLOCKS)
+ return -ENOSPC;
+
+ block = kzalloc_obj(*block);
+ if (!block)
+ return -ENOMEM;
+
+ block->ser = ser;
+ last = list_last_entry_or_null(&bs->blocks, struct kho_block, list);
+ list_add_tail(&block->list, &bs->blocks);
+ bs->nblocks++;
+
+ if (last)
+ last->ser->next = virt_to_phys(ser);
+ else
+ bs->head_pa = virt_to_phys(ser);
+
+ return 0;
+}
+
+static int kho_block_set_grow_one(struct kho_block_set *bs)
+{
+ struct kho_block_header_ser *ser;
+ int err;
+
+ ser = kho_block_alloc_ser(bs);
+ if (IS_ERR(ser))
+ return PTR_ERR(ser);
+
+ err = kho_block_add(bs, ser);
+ if (err) {
+ kho_block_free_ser(bs, ser);
+ return err;
+ }
+
+ return 0;
+}
+
+static void kho_block_set_shrink_one(struct kho_block_set *bs)
+{
+ struct kho_block *last, *new_last;
+
+ if (list_empty(&bs->blocks))
+ return;
+
+ last = list_last_entry(&bs->blocks, struct kho_block, list);
+ list_del(&last->list);
+ bs->nblocks--;
+ kho_block_free_ser(bs, last->ser);
+ kfree(last);
+
+ new_last = list_last_entry_or_null(&bs->blocks, struct kho_block, list);
+ if (new_last)
+ new_last->ser->next = 0;
+ else
+ bs->head_pa = 0;
+}
+
+/**
+ * kho_block_set_grow - Expand the block set to accommodate the target count.
+ * @bs: The block set.
+ * @count: The target number of valid entries to accommodate.
+ *
+ * Dynamically preallocates and links preserved memory blocks if the target
+ * entry count exceeds the current total capacity of the set, ensuring they
+ * are available during serialization/deserialization.
+ *
+ * Context: Caller must hold a lock protecting the block set.
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int kho_block_set_grow(struct kho_block_set *bs, u64 count)
+{
+ long orig_nblocks = bs->nblocks;
+ int err;
+
+ if (WARN_ON_ONCE(bs->incoming))
+ return -EINVAL;
+
+ while (count > bs->nblocks * bs->count_per_block) {
+ err = kho_block_set_grow_one(bs);
+ if (err)
+ goto err_shrink;
+ }
+
+ return 0;
+
+err_shrink:
+ while (bs->nblocks > orig_nblocks)
+ kho_block_set_shrink_one(bs);
+ return err;
+}
+
+/**
+ * kho_block_set_shrink - Shrink the block set to accommodate the target count.
+ * @bs: The block set.
+ * @count: The target number of valid entries to accommodate.
+ *
+ * Releases and unallocates redundant preserved memory blocks. Checks if the
+ * last block in the set can be removed because the remaining entry count is
+ * fully accommodated by the preceding blocks.
+ *
+ * Note: It is the caller's responsibility to ensure that entries are removed
+ * in the reverse order of their insertion. Because shrinking destroys the last
+ * block in the set, removing entries in any other order would corrupt active
+ * data.
+ *
+ * Context: Caller must hold a lock protecting the block set.
+ */
+void kho_block_set_shrink(struct kho_block_set *bs, u64 count)
+{
+ while (bs->nblocks > 0 && count <= (bs->nblocks - 1) * bs->count_per_block)
+ kho_block_set_shrink_one(bs);
+}
+
+/*
+ * kho_block_set_is_cyclic - Check for cycles in a linked list of blocks.
+ * Uses Floyd's cycle-finding algorithm to ensure sanity of the incoming list.
+ *
+ * Return: true if a cycle or corruption is detected, false otherwise.
+ */
+static bool kho_block_set_is_cyclic(struct kho_block_set *bs)
+{
+ struct kho_block_header_ser *fast;
+ struct kho_block_header_ser *slow;
+ int count = 0;
+
+ fast = phys_to_virt(bs->head_pa);
+ slow = fast;
+
+ while (fast) {
+ if (count++ >= KHO_MAX_BLOCKS) {
+ pr_err("Block set is corrupted\n");
+ return true;
+ }
+
+ if (!fast->next)
+ break;
+
+ fast = phys_to_virt(fast->next);
+ if (!fast->next)
+ break;
+
+ fast = phys_to_virt(fast->next);
+ slow = phys_to_virt(slow->next);
+
+ if (slow == fast) {
+ pr_err("Block set is corrupted\n");
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * kho_block_set_restore - Restore a block set from a physical address.
+ * @bs: The block set to restore.
+ * @head_pa: Physical address of the first block header.
+ *
+ * Restores a serialized block set from a given physical address. The caller is
+ * responsible for ensuring that the block set @bs has been allocated and
+ * initialized prior to calling this function.
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa)
+{
+ struct kho_block_header_ser *ser;
+ u64 next_pa = head_pa;
+ int err;
+
+ /* Restored block sets use size from the previous kernel */
+ bs->incoming = true;
+ if (!head_pa)
+ return 0;
+
+ bs->head_pa = head_pa;
+ if (kho_block_set_is_cyclic(bs)) {
+ bs->head_pa = 0;
+ return -EINVAL;
+ }
+
+ while (next_pa) {
+ ser = phys_to_virt(next_pa);
+ if (!ser->count || ser->count > bs->count_per_block) {
+ pr_warn("Block contains invalid entry count: %llu\n",
+ ser->count);
+ err = -EINVAL;
+ goto err_destroy;
+ }
+ err = kho_block_add(bs, ser);
+ if (err)
+ goto err_destroy;
+ next_pa = ser->next;
+ }
+
+ return 0;
+
+err_destroy:
+ kho_block_set_destroy(bs);
+
+ /* Free the remaining un-restored blocks in the physical chain */
+ while (next_pa) {
+ struct kho_block_header_ser *next_ser = phys_to_virt(next_pa);
+
+ next_pa = next_ser->next;
+ kho_block_free_ser(bs, next_ser);
+ }
+ return err;
+}
+
+/**
+ * kho_block_set_destroy - Destroy all blocks in a block set.
+ * @bs: The block set.
+ */
+void kho_block_set_destroy(struct kho_block_set *bs)
+{
+ struct kho_block *block, *tmp;
+
+ list_for_each_entry_safe(block, tmp, &bs->blocks, list) {
+ list_del(&block->list);
+ kho_block_free_ser(bs, block->ser);
+ kfree(block);
+ }
+ bs->nblocks = 0;
+ bs->head_pa = 0;
+}
+
+/**
+ * kho_block_set_clear - Clear all serialized data in a block set.
+ * @bs: The block set to clear.
+ */
+void kho_block_set_clear(struct kho_block_set *bs)
+{
+ struct kho_block *block;
+
+ list_for_each_entry(block, &bs->blocks, list) {
+ block->ser->count = 0;
+ memset(block->ser + 1, 0, KHO_BLOCK_SIZE - sizeof(*block->ser));
+ }
+}
+
+/**
+ * kho_block_set_it_init - Initialize a block set iterator.
+ * @it: The iterator to initialize.
+ * @bs: The block set to iterate over.
+ */
+void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs)
+{
+ it->bs = bs;
+ it->block = list_first_entry_or_null(&bs->blocks, struct kho_block, list);
+ it->i = 0;
+}
+
+/**
+ * kho_block_set_it_reserve_entry - Reserve and return the next available slot for writing.
+ * @it: The block iterator.
+ *
+ * Reserves a slot in the current block during state serialization to add a new
+ * entry, advancing the internal index. If the current block is full, it
+ * automatically moves to the next block in the set.
+ *
+ * Return: A pointer to the reserved entry slot, or NULL if the block set's
+ * capacity is fully exhausted.
+ */
+void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it)
+{
+ void *entry;
+
+ if (!it->block)
+ return NULL;
+
+ if (it->i == it->bs->count_per_block) {
+ if (list_is_last(&it->block->list, &it->bs->blocks))
+ return NULL;
+ it->block = list_next_entry(it->block, list);
+ it->i = 0;
+ }
+
+ entry = kho_block_entry(it, it->i++);
+ it->block->ser->count = it->i;
+ return entry;
+}
+
+/**
+ * kho_block_set_it_read_entry - Read the next serialized entry from the block set.
+ * @it: The block iterator.
+ *
+ * Iterates through previously written entries during state deserialization,
+ * respecting the actual count stored in each block's header.
+ *
+ * Return: A pointer to the next serialized entry, or NULL if all serialized
+ * entries have been read.
+ */
+void *kho_block_set_it_read_entry(struct kho_block_set_it *it)
+{
+ if (!it->block)
+ return NULL;
+
+ if (it->i == it->block->ser->count) {
+ if (list_is_last(&it->block->list, &it->bs->blocks))
+ return NULL;
+ it->block = list_next_entry(it->block, list);
+ it->i = 0;
+ }
+
+ return kho_block_entry(it, it->i++);
+}
+
+/**
+ * kho_block_set_it_prev - Return the previous entry slot in the block set.
+ * @it: The block iterator.
+ *
+ * If the current index is at the start of a block, it automatically moves to
+ * the end of the previous block.
+ *
+ * Return: A pointer to the previous entry slot, or NULL if at the very
+ * beginning of the block set.
+ */
+void *kho_block_set_it_prev(struct kho_block_set_it *it)
+{
+ if (!it->block)
+ return NULL;
+
+ if (it->i == 0) {
+ if (list_is_first(&it->block->list, &it->bs->blocks))
+ return NULL;
+ it->block = list_prev_entry(it->block, list);
+ it->i = it->bs->count_per_block;
+ }
+
+ return kho_block_entry(it, --it->i);
+}
diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c
index 803f51c84275..1b2bda22902d 100644
--- a/kernel/liveupdate/luo_core.c
+++ b/kernel/liveupdate/luo_core.c
@@ -36,6 +36,10 @@
*
* LUO uses Kexec Handover to transfer memory state from the current kernel to
* the next kernel. For more details see Documentation/core-api/kho/index.rst.
+ *
+ * .. note::
+ * To enable LUO, boot the kernel with the ``liveupdate=on`` command line
+ * parameter.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -50,22 +54,19 @@
#include <linux/kexec_handover.h>
#include <linux/kho/abi/luo.h>
#include <linux/kobject.h>
-#include <linux/libfdt.h>
#include <linux/liveupdate.h>
#include <linux/miscdevice.h>
#include <linux/mm.h>
#include <linux/rwsem.h>
#include <linux/sizes.h>
#include <linux/string.h>
-#include <linux/unaligned.h>
#include "kexec_handover_internal.h"
#include "luo_internal.h"
static struct {
bool enabled;
- void *fdt_out;
- void *fdt_in;
+ struct luo_ser *luo_ser_out;
u64 liveupdate_num;
} luo_global;
@@ -82,9 +83,10 @@ early_param("liveupdate", early_liveupdate_param);
static int __init luo_early_startup(void)
{
- phys_addr_t fdt_phys;
- int err, ln_size;
- const void *ptr;
+ phys_addr_t luo_ser_phys;
+ struct luo_ser *luo_ser;
+ size_t len;
+ int err;
if (!kho_is_enabled()) {
if (liveupdate_enabled())
@@ -93,48 +95,43 @@ static int __init luo_early_startup(void)
return 0;
}
- /* Retrieve LUO subtree, and verify its format. */
- err = kho_retrieve_subtree(LUO_FDT_KHO_ENTRY_NAME, &fdt_phys, NULL);
+ /* Retrieve LUO state from KHO. */
+ err = kho_retrieve_subtree(LUO_KHO_ENTRY_NAME, &luo_ser_phys, &len);
if (err) {
if (err != -ENOENT) {
- pr_err("failed to retrieve FDT '%s' from KHO: %pe\n",
- LUO_FDT_KHO_ENTRY_NAME, ERR_PTR(err));
+ pr_err("failed to retrieve LUO state '%s' from KHO: %pe\n",
+ LUO_KHO_ENTRY_NAME, ERR_PTR(err));
return err;
}
return 0;
}
- luo_global.fdt_in = phys_to_virt(fdt_phys);
- err = fdt_node_check_compatible(luo_global.fdt_in, 0,
- LUO_FDT_COMPATIBLE);
- if (err) {
- pr_err("FDT '%s' is incompatible with '%s' [%d]\n",
- LUO_FDT_KHO_ENTRY_NAME, LUO_FDT_COMPATIBLE, err);
-
+ if (len < sizeof(*luo_ser)) {
+ pr_err("LUO state is too small (%zu < %zu)\n", len, sizeof(*luo_ser));
return -EINVAL;
}
- ln_size = 0;
- ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_LIVEUPDATE_NUM,
- &ln_size);
- if (!ptr || ln_size != sizeof(luo_global.liveupdate_num)) {
- pr_err("Unable to get live update number '%s' [%d]\n",
- LUO_FDT_LIVEUPDATE_NUM, ln_size);
-
+ luo_ser = phys_to_virt(luo_ser_phys);
+ if (strncmp(luo_ser->compatible, LUO_ABI_COMPATIBLE, LUO_ABI_COMPAT_LEN)) {
+ pr_err("LUO state is incompatible with '%s'\n", LUO_ABI_COMPATIBLE);
return -EINVAL;
}
- luo_global.liveupdate_num = get_unaligned((u64 *)ptr);
+ luo_global.liveupdate_num = luo_ser->liveupdate_num;
pr_info("Retrieved live update data, liveupdate number: %lld\n",
luo_global.liveupdate_num);
- err = luo_session_setup_incoming(luo_global.fdt_in);
+ err = luo_session_setup_incoming(luo_ser->sessions_pa);
if (err)
- return err;
+ goto out_free_ser;
+
+ luo_flb_setup_incoming(luo_ser->flbs_pa);
- err = luo_flb_setup_incoming(luo_global.fdt_in);
+ err = 0;
+out_free_ser:
+ kho_restore_free(luo_ser);
return err;
}
@@ -153,42 +150,38 @@ static int __init liveupdate_early_init(void)
}
early_initcall(liveupdate_early_init);
-/* Called during boot to create outgoing LUO fdt tree */
-static int __init luo_fdt_setup(void)
+/* Called during boot to create outgoing LUO state */
+static int __init luo_state_setup(void)
{
- const u64 ln = luo_global.liveupdate_num + 1;
- void *fdt_out;
+ struct luo_ser *luo_ser;
int err;
- fdt_out = kho_alloc_preserve(LUO_FDT_SIZE);
- if (IS_ERR(fdt_out)) {
- pr_err("failed to allocate/preserve FDT memory\n");
- return PTR_ERR(fdt_out);
+ luo_ser = kho_alloc_preserve(sizeof(*luo_ser));
+ if (IS_ERR(luo_ser)) {
+ pr_err("failed to allocate/preserve LUO state memory\n");
+ return PTR_ERR(luo_ser);
}
- err = fdt_create(fdt_out, LUO_FDT_SIZE);
- err |= fdt_finish_reservemap(fdt_out);
- err |= fdt_begin_node(fdt_out, "");
- err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE);
- err |= fdt_property(fdt_out, LUO_FDT_LIVEUPDATE_NUM, &ln, sizeof(ln));
- err |= luo_session_setup_outgoing(fdt_out);
- err |= luo_flb_setup_outgoing(fdt_out);
- err |= fdt_end_node(fdt_out);
- err |= fdt_finish(fdt_out);
+ strscpy(luo_ser->compatible, LUO_ABI_COMPATIBLE, sizeof(luo_ser->compatible));
+ luo_ser->liveupdate_num = luo_global.liveupdate_num + 1;
+
+ luo_session_setup_outgoing(&luo_ser->sessions_pa);
+
+ err = luo_flb_setup_outgoing(&luo_ser->flbs_pa);
if (err)
- goto exit_free;
+ goto exit_free_luo_ser;
- err = kho_add_subtree(LUO_FDT_KHO_ENTRY_NAME, fdt_out,
- fdt_totalsize(fdt_out));
+ err = kho_add_subtree(LUO_KHO_ENTRY_NAME, luo_ser, sizeof(*luo_ser));
if (err)
- goto exit_free;
- luo_global.fdt_out = fdt_out;
+ goto exit_free_luo_ser;
+
+ luo_global.luo_ser_out = luo_ser;
return 0;
-exit_free:
- kho_unpreserve_free(fdt_out);
- pr_err("failed to prepare LUO FDT: %d\n", err);
+exit_free_luo_ser:
+ kho_unpreserve_free(luo_ser);
+ pr_err("failed to prepare LUO state: %d\n", err);
return err;
}
@@ -204,7 +197,7 @@ static int __init luo_late_startup(void)
if (!liveupdate_enabled())
return 0;
- err = luo_fdt_setup();
+ err = luo_state_setup();
if (err)
luo_global.enabled = false;
diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c
index a0a419085e28..c39f96961a85 100644
--- a/kernel/liveupdate/luo_file.c
+++ b/kernel/liveupdate/luo_file.c
@@ -118,11 +118,6 @@ static LIST_HEAD(luo_file_handler_list);
/* Keep track of files being preserved by LUO */
static DEFINE_XARRAY(luo_preserved_files);
-/* 2 4K pages, give space for 128 files per file_set */
-#define LUO_FILE_PGCNT 2ul
-#define LUO_FILE_MAX \
- ((LUO_FILE_PGCNT << PAGE_SHIFT) / sizeof(struct luo_file_ser))
-
/**
* struct luo_file - Represents a single preserved file instance.
* @fh: Pointer to the &struct liveupdate_file_handler that manages
@@ -174,39 +169,6 @@ struct luo_file {
u64 token;
};
-static int luo_alloc_files_mem(struct luo_file_set *file_set)
-{
- size_t size;
- void *mem;
-
- if (file_set->files)
- return 0;
-
- WARN_ON_ONCE(file_set->count);
-
- size = LUO_FILE_PGCNT << PAGE_SHIFT;
- mem = kho_alloc_preserve(size);
- if (IS_ERR(mem))
- return PTR_ERR(mem);
-
- file_set->files = mem;
-
- return 0;
-}
-
-static void luo_free_files_mem(struct luo_file_set *file_set)
-{
- /* If file_set has files, no need to free preservation memory */
- if (file_set->count)
- return;
-
- if (!file_set->files)
- return;
-
- kho_unpreserve_free(file_set->files);
- file_set->files = NULL;
-}
-
static unsigned long luo_get_id(struct liveupdate_file_handler *fh,
struct file *file)
{
@@ -276,16 +238,15 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
if (luo_token_is_used(file_set, token))
return -EEXIST;
- if (file_set->count == LUO_FILE_MAX)
- return -ENOSPC;
+ err = kho_block_set_grow(&file_set->block_set, file_set->count + 1);
+ if (err)
+ return err;
file = fget(fd);
- if (!file)
- return -EBADF;
-
- err = luo_alloc_files_mem(file_set);
- if (err)
- goto err_fput;
+ if (!file) {
+ err = -EBADF;
+ goto err_shrink;
+ }
err = -ENOENT;
down_read(&luo_register_rwlock);
@@ -300,7 +261,7 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
/* err is still -ENOENT if no handler was found */
if (err)
- goto err_free_files_mem;
+ goto err_fput;
err = xa_insert(&luo_preserved_files, luo_get_id(fh, file),
file, GFP_KERNEL);
@@ -343,10 +304,10 @@ err_erase_xa:
xa_erase(&luo_preserved_files, luo_get_id(fh, file));
err_module_put:
module_put(fh->ops->owner);
-err_free_files_mem:
- luo_free_files_mem(file_set);
err_fput:
fput(file);
+err_shrink:
+ kho_block_set_shrink(&file_set->block_set, file_set->count);
return err;
}
@@ -385,19 +346,21 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set)
args.private_data = luo_file->private_data;
luo_file->fh->ops->unpreserve(&args);
luo_flb_file_unpreserve(luo_file->fh);
- module_put(luo_file->fh->ops->owner);
xa_erase(&luo_preserved_files,
luo_get_id(luo_file->fh, luo_file->file));
+ module_put(luo_file->fh->ops->owner);
+
list_del(&luo_file->list);
file_set->count--;
+ kho_block_set_shrink(&file_set->block_set, file_set->count);
fput(luo_file->file);
mutex_destroy(&luo_file->mutex);
kfree(luo_file);
}
- luo_free_files_mem(file_set);
+ kho_block_set_destroy(&file_set->block_set);
}
static int luo_file_freeze_one(struct luo_file_set *file_set,
@@ -453,7 +416,7 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set,
luo_file_unfreeze_one(file_set, luo_file);
}
- memset(file_set->files, 0, LUO_FILE_PGCNT << PAGE_SHIFT);
+ kho_block_set_clear(&file_set->block_set);
}
/**
@@ -492,19 +455,24 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set,
int luo_file_freeze(struct luo_file_set *file_set,
struct luo_file_set_ser *file_set_ser)
{
- struct luo_file_ser *file_ser = file_set->files;
struct luo_file *luo_file;
+ struct kho_block_set_it it;
int err;
- int i;
if (!file_set->count)
return 0;
- if (WARN_ON(!file_ser))
- return -EINVAL;
+ kho_block_set_it_init(&it, &file_set->block_set);
- i = 0;
list_for_each_entry(luo_file, &file_set->files_list, list) {
+ struct luo_file_ser *file_ser = kho_block_set_it_reserve_entry(&it);
+
+ /* This should not fail normally as blocks were pre-allocated */
+ if (WARN_ON_ONCE(!file_ser)) {
+ err = -ENOSPC;
+ goto err_unfreeze;
+ }
+
err = luo_file_freeze_one(file_set, luo_file);
if (err < 0) {
pr_warn("Freeze failed for token[%#0llx] handler[%s] err[%pe]\n",
@@ -513,16 +481,14 @@ int luo_file_freeze(struct luo_file_set *file_set,
goto err_unfreeze;
}
- strscpy(file_ser[i].compatible, luo_file->fh->compatible,
- sizeof(file_ser[i].compatible));
- file_ser[i].data = luo_file->serialized_data;
- file_ser[i].token = luo_file->token;
- i++;
+ strscpy(file_ser->compatible, luo_file->fh->compatible,
+ sizeof(file_ser->compatible));
+ file_ser->data = luo_file->serialized_data;
+ file_ser->token = luo_file->token;
}
file_set_ser->count = file_set->count;
- if (file_set->files)
- file_set_ser->files = virt_to_phys(file_set->files);
+ file_set_ser->files = kho_block_set_head_pa(&file_set->block_set);
return 0;
@@ -677,7 +643,6 @@ static void luo_file_finish_one(struct luo_file_set *file_set,
luo_file->fh->ops->finish(&args);
luo_flb_file_finish(luo_file->fh);
- module_put(luo_file->fh->ops->owner);
}
/**
@@ -738,17 +703,56 @@ int luo_file_finish(struct luo_file_set *file_set)
luo_get_id(luo_file->fh, luo_file->file));
fput(luo_file->file);
}
+ module_put(luo_file->fh->ops->owner);
list_del(&luo_file->list);
file_set->count--;
+ kho_block_set_shrink(&file_set->block_set, file_set->count);
mutex_destroy(&luo_file->mutex);
kfree(luo_file);
}
- if (file_set->files) {
- kho_restore_free(file_set->files);
- file_set->files = NULL;
+ kho_block_set_destroy(&file_set->block_set);
+
+ return 0;
+}
+
+static int luo_file_deserialize_one(struct luo_file_set *file_set,
+ struct luo_file_ser *ser)
+{
+ struct liveupdate_file_handler *fh;
+ bool handler_found = false;
+ struct luo_file *luo_file;
+
+ down_read(&luo_register_rwlock);
+ list_private_for_each_entry(fh, &luo_file_handler_list, list) {
+ if (!strcmp(fh->compatible, ser->compatible)) {
+ if (try_module_get(fh->ops->owner))
+ handler_found = true;
+ break;
+ }
+ }
+ up_read(&luo_register_rwlock);
+
+ if (!handler_found) {
+ pr_warn("No registered handler for compatible '%.*s'\n",
+ (int)sizeof(ser->compatible),
+ ser->compatible);
+ return -ENOENT;
}
+ luo_file = kzalloc_obj(*luo_file);
+ if (!luo_file) {
+ module_put(fh->ops->owner);
+ return -ENOMEM;
+ }
+
+ luo_file->fh = fh;
+ luo_file->file = NULL;
+ luo_file->serialized_data = ser->data;
+ luo_file->token = ser->token;
+ mutex_init(&luo_file->mutex);
+ list_add_tail(&luo_file->list, &file_set->files_list);
+
return 0;
}
@@ -781,15 +785,18 @@ int luo_file_deserialize(struct luo_file_set *file_set,
struct luo_file_set_ser *file_set_ser)
{
struct luo_file_ser *file_ser;
- u64 i;
+ struct kho_block_set_it it;
+ int err;
if (!file_set_ser->files) {
WARN_ON(file_set_ser->count);
return 0;
}
- file_set->count = file_set_ser->count;
- file_set->files = phys_to_virt(file_set_ser->files);
+ file_set->count = 0;
+ err = kho_block_set_restore(&file_set->block_set, file_set_ser->files);
+ if (err)
+ return err;
/*
* Note on error handling:
@@ -806,55 +813,50 @@ int luo_file_deserialize(struct luo_file_set *file_set,
* userspace to detect the failure and trigger a reboot, which will
* reliably reset devices and reclaim memory.
*/
- file_ser = file_set->files;
- for (i = 0; i < file_set->count; i++) {
- struct liveupdate_file_handler *fh;
- bool handler_found = false;
- struct luo_file *luo_file;
+ kho_block_set_it_init(&it, &file_set->block_set);
+ while ((file_ser = kho_block_set_it_read_entry(&it))) {
+ err = luo_file_deserialize_one(file_set, file_ser);
+ if (err)
+ goto err_destroy_blocks;
+ file_set->count++;
+ }
- down_read(&luo_register_rwlock);
- list_private_for_each_entry(fh, &luo_file_handler_list, list) {
- if (!strcmp(fh->compatible, file_ser[i].compatible)) {
- if (try_module_get(fh->ops->owner))
- handler_found = true;
- break;
- }
- }
- up_read(&luo_register_rwlock);
+ if (file_set->count != file_set_ser->count) {
+ pr_warn("File count mismatch: expected %llu, found %llu\n",
+ file_set_ser->count, file_set->count);
+ err = -EINVAL;
+ goto err_destroy_blocks;
+ }
- if (!handler_found) {
- pr_warn("No registered handler for compatible '%.*s'\n",
- (int)sizeof(file_ser[i].compatible),
- file_ser[i].compatible);
- return -ENOENT;
- }
+ return 0;
- luo_file = kzalloc_obj(*luo_file);
- if (!luo_file) {
- module_put(fh->ops->owner);
- return -ENOMEM;
- }
+err_destroy_blocks:
+ while (!list_empty(&file_set->files_list)) {
+ struct luo_file *luo_file;
- luo_file->fh = fh;
- luo_file->file = NULL;
- luo_file->serialized_data = file_ser[i].data;
- luo_file->token = file_ser[i].token;
- mutex_init(&luo_file->mutex);
- list_add_tail(&luo_file->list, &file_set->files_list);
+ luo_file = list_first_entry(&file_set->files_list,
+ struct luo_file, list);
+ list_del(&luo_file->list);
+ module_put(luo_file->fh->ops->owner);
+ mutex_destroy(&luo_file->mutex);
+ kfree(luo_file);
}
-
- return 0;
+ file_set->count = 0;
+ kho_block_set_destroy(&file_set->block_set);
+ return err;
}
void luo_file_set_init(struct luo_file_set *file_set)
{
INIT_LIST_HEAD(&file_set->files_list);
+ kho_block_set_init(&file_set->block_set, sizeof(struct luo_file_ser));
}
void luo_file_set_destroy(struct luo_file_set *file_set)
{
WARN_ON(file_set->count);
WARN_ON(!list_empty(&file_set->files_list));
+ WARN_ON(!kho_block_set_is_empty(&file_set->block_set));
}
/**
diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c
index 00f5494812c4..5c27134ce7ba 100644
--- a/kernel/liveupdate/luo_flb.c
+++ b/kernel/liveupdate/luo_flb.c
@@ -44,13 +44,11 @@
#include <linux/io.h>
#include <linux/kexec_handover.h>
#include <linux/kho/abi/luo.h>
-#include <linux/libfdt.h>
#include <linux/list_private.h>
#include <linux/liveupdate.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
-#include <linux/unaligned.h>
#include "luo_internal.h"
#define LUO_FLB_PGCNT 1ul
@@ -111,7 +109,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
struct luo_flb_private *private = luo_flb_get_private(flb);
scoped_guard(mutex, &private->outgoing.lock) {
- if (!private->outgoing.count) {
+ if (!refcount_read(&private->outgoing.count)) {
struct liveupdate_flb_op_args args = {0};
int err;
@@ -126,8 +124,10 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
}
private->outgoing.data = args.data;
private->outgoing.obj = args.obj;
+ refcount_set(&private->outgoing.count, 1);
+ } else {
+ refcount_inc(&private->outgoing.count);
}
- private->outgoing.count++;
}
return 0;
@@ -138,8 +138,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb)
struct luo_flb_private *private = luo_flb_get_private(flb);
scoped_guard(mutex, &private->outgoing.lock) {
- private->outgoing.count--;
- if (!private->outgoing.count) {
+ if (refcount_dec_and_test(&private->outgoing.count)) {
struct liveupdate_flb_op_args args = {0};
args.flb = flb;
@@ -164,7 +163,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
bool found = false;
int err;
- guard(mutex)(&private->incoming.lock);
+ lockdep_assert_held(&private->incoming.lock);
if (private->incoming.finished)
return -ENODATA;
@@ -178,7 +177,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
for (int i = 0; i < fh->header_ser->count; i++) {
if (!strcmp(fh->ser[i].name, flb->compatible)) {
private->incoming.data = fh->ser[i].data;
- private->incoming.count = fh->ser[i].count;
+ refcount_set(&private->incoming.count, fh->ser[i].count);
found = true;
break;
}
@@ -205,16 +204,14 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
return 0;
}
-static void luo_flb_file_finish_one(struct liveupdate_flb *flb)
+void liveupdate_flb_put_incoming(struct liveupdate_flb *flb)
{
struct luo_flb_private *private = luo_flb_get_private(flb);
- u64 count;
-
- scoped_guard(mutex, &private->incoming.lock)
- count = --private->incoming.count;
+ struct liveupdate_flb_op_args args = {0};
- if (!count) {
- struct liveupdate_flb_op_args args = {0};
+ scoped_guard(mutex, &private->incoming.lock) {
+ if (!refcount_dec_and_test(&private->incoming.count))
+ return;
if (!private->incoming.retrieved) {
int err = luo_flb_retrieve_one(flb);
@@ -223,16 +220,14 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb)
return;
}
- scoped_guard(mutex, &private->incoming.lock) {
- args.flb = flb;
- args.obj = private->incoming.obj;
- flb->ops->finish(&args);
+ args.flb = flb;
+ args.obj = private->incoming.obj;
+ flb->ops->finish(&args);
- private->incoming.data = 0;
- private->incoming.obj = NULL;
- private->incoming.finished = true;
- module_put(flb->ops->owner);
- }
+ private->incoming.data = 0;
+ private->incoming.obj = NULL;
+ private->incoming.finished = true;
+ module_put(flb->ops->owner);
}
}
@@ -315,7 +310,7 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh)
guard(rwsem_read)(&luo_register_rwlock);
list_for_each_entry_reverse(iter, flb_list, list)
- luo_flb_file_finish_one(iter->flb);
+ liveupdate_flb_put_incoming(iter->flb);
}
static void luo_flb_unregister_one(struct liveupdate_file_handler *fh,
@@ -512,6 +507,8 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
if (!liveupdate_enabled())
return -EOPNOTSUPP;
+ guard(mutex)(&private->incoming.lock);
+
if (!private->incoming.obj) {
int err = luo_flb_retrieve_one(flb);
@@ -519,7 +516,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
return err;
}
- guard(mutex)(&private->incoming.lock);
+ refcount_inc(&private->incoming.count);
*objp = private->incoming.obj;
return 0;
@@ -552,27 +549,15 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp)
return 0;
}
-int __init luo_flb_setup_outgoing(void *fdt_out)
+int __init luo_flb_setup_outgoing(u64 *flbs_pa)
{
struct luo_flb_header_ser *header_ser;
- u64 header_ser_pa;
- int err;
header_ser = kho_alloc_preserve(LUO_FLB_PGCNT << PAGE_SHIFT);
if (IS_ERR(header_ser))
return PTR_ERR(header_ser);
- header_ser_pa = virt_to_phys(header_ser);
-
- err = fdt_begin_node(fdt_out, LUO_FDT_FLB_NODE_NAME);
- err |= fdt_property_string(fdt_out, "compatible",
- LUO_FDT_FLB_COMPATIBLE);
- err |= fdt_property(fdt_out, LUO_FDT_FLB_HEADER, &header_ser_pa,
- sizeof(header_ser_pa));
- err |= fdt_end_node(fdt_out);
-
- if (err)
- goto err_unpreserve;
+ *flbs_pa = virt_to_phys(header_ser);
header_ser->pgcnt = LUO_FLB_PGCNT;
luo_flb_global.outgoing.header_ser = header_ser;
@@ -580,53 +565,19 @@ int __init luo_flb_setup_outgoing(void *fdt_out)
luo_flb_global.outgoing.active = true;
return 0;
-
-err_unpreserve:
- kho_unpreserve_free(header_ser);
-
- return err;
}
-int __init luo_flb_setup_incoming(void *fdt_in)
+void __init luo_flb_setup_incoming(u64 flbs_pa)
{
struct luo_flb_header_ser *header_ser;
- int err, header_size, offset;
- const void *ptr;
- u64 header_ser_pa;
- offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_FLB_NODE_NAME);
- if (offset < 0) {
- pr_err("Unable to get FLB node [%s]\n", LUO_FDT_FLB_NODE_NAME);
-
- return -ENOENT;
- }
-
- err = fdt_node_check_compatible(fdt_in, offset,
- LUO_FDT_FLB_COMPATIBLE);
- if (err) {
- pr_err("FLB node is incompatible with '%s' [%d]\n",
- LUO_FDT_FLB_COMPATIBLE, err);
-
- return -EINVAL;
- }
-
- header_size = 0;
- ptr = fdt_getprop(fdt_in, offset, LUO_FDT_FLB_HEADER, &header_size);
- if (!ptr || header_size != sizeof(u64)) {
- pr_err("Unable to get FLB header property '%s' [%d]\n",
- LUO_FDT_FLB_HEADER, header_size);
-
- return -EINVAL;
- }
-
- header_ser_pa = get_unaligned((u64 *)ptr);
- header_ser = phys_to_virt(header_ser_pa);
+ if (!flbs_pa)
+ return;
+ header_ser = phys_to_virt(flbs_pa);
luo_flb_global.incoming.header_ser = header_ser;
luo_flb_global.incoming.ser = (void *)(header_ser + 1);
luo_flb_global.incoming.active = true;
-
- return 0;
}
/**
@@ -652,12 +603,13 @@ void luo_flb_serialize(void)
guard(rwsem_read)(&luo_register_rwlock);
list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) {
struct luo_flb_private *private = luo_flb_get_private(gflb);
+ long count = refcount_read(&private->outgoing.count);
- if (private->outgoing.count > 0) {
+ if (count > 0) {
strscpy(fh->ser[i].name, gflb->compatible,
sizeof(fh->ser[i].name));
fh->ser[i].data = private->outgoing.data;
- fh->ser[i].count = private->outgoing.count;
+ fh->ser[i].count = count;
i++;
}
}
diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h
index 875844d7a41d..64879ffe7378 100644
--- a/kernel/liveupdate/luo_internal.h
+++ b/kernel/liveupdate/luo_internal.h
@@ -10,6 +10,7 @@
#include <linux/liveupdate.h>
#include <linux/uaccess.h>
+#include <linux/kho_block.h>
struct luo_ucmd {
void __user *ubuffer;
@@ -44,22 +45,20 @@ static inline int luo_ucmd_respond(struct luo_ucmd *ucmd,
* struct luo_file_set - A set of files that belong to the same sessions.
* @files_list: An ordered list of files associated with this session, it is
* ordered by preservation time.
- * @files: The physically contiguous memory block that holds the serialized
- * state of files.
+ * @block_set: The set of serialization blocks.
* @count: A counter tracking the number of files currently stored in the
* @files_list for this session.
*/
struct luo_file_set {
struct list_head files_list;
- struct luo_file_ser *files;
- long count;
+ struct kho_block_set block_set;
+ u64 count;
};
/**
* struct luo_session - Represents an active or incoming Live Update session.
* @name: A unique name for this session, used for identification and
* retrieval.
- * @ser: Pointer to the serialized data for this session.
* @list: A list_head member used to link this session into a global list
* of either outgoing (to be preserved) or incoming (restored from
* previous kernel) sessions.
@@ -70,7 +69,6 @@ struct luo_file_set {
*/
struct luo_session {
char name[LIVEUPDATE_SESSION_NAME_LENGTH];
- struct luo_session_ser *ser;
struct list_head list;
bool retrieved;
struct luo_file_set file_set;
@@ -81,8 +79,8 @@ extern struct rw_semaphore luo_register_rwlock;
int luo_session_create(const char *name, struct file **filep);
int luo_session_retrieve(const char *name, struct file **filep);
-int __init luo_session_setup_outgoing(void *fdt);
-int __init luo_session_setup_incoming(void *fdt);
+void __init luo_session_setup_outgoing(u64 *sessions_pa);
+int __init luo_session_setup_incoming(u64 sessions_pa);
int luo_session_serialize(void);
int luo_session_deserialize(void);
@@ -104,8 +102,8 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh);
void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh);
void luo_flb_file_finish(struct liveupdate_file_handler *fh);
void luo_flb_unregister_all(struct liveupdate_file_handler *fh);
-int __init luo_flb_setup_outgoing(void *fdt);
-int __init luo_flb_setup_incoming(void *fdt);
+int __init luo_flb_setup_outgoing(u64 *flbs_pa);
+void __init luo_flb_setup_incoming(u64 flbs_pa);
void luo_flb_serialize(void);
#ifdef CONFIG_LIVEUPDATE_TEST
diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c
index 7a42385dabe2..b79b2a488974 100644
--- a/kernel/liveupdate/luo_session.c
+++ b/kernel/liveupdate/luo_session.c
@@ -24,10 +24,10 @@
* ioctls on /dev/liveupdate.
*
* - Serialization: Session metadata is preserved using the KHO framework. When
- * a live update is triggered via kexec, an array of `struct luo_session_ser`
- * is populated and placed in a preserved memory region. An FDT node is also
- * created, containing the count of sessions and the physical address of this
- * array.
+ * a live update is triggered via kexec, session metadata is serialized into
+ * a chain of linked-blocks and placed in a preserved memory region. The
+ * physical address of the first block header is stored in the centralized
+ * `struct luo_ser` structure.
*
* Session Lifecycle:
*
@@ -46,6 +46,38 @@
* 4. Retrieval: A userspace agent in the new kernel can then call
* `luo_session_retrieve()` with a session name to get a new file
* descriptor and access the preserved state.
+ *
+ * Locking:
+ *
+ * The LUO session subsystem uses a three-tier locking hierarchy to ensure thread
+ * safety and prevent deadlocks during concurrent session mutations and kexec
+ * serialization:
+ *
+ * 1. `luo_session_serialize_rwsem` (global rwsem):
+ * Protects session mutations (creation, retrieval, release, and ioctls)
+ * against the serialization process during reboot.
+ *
+ * - Readers: Taken by any path modifying or accessing session state (e.g.,
+ * `luo_session_create()`, `luo_session_retrieve()`, `luo_session_release()`,
+ * and `luo_session_ioctl()`).
+ * - Writer: Taken by the serialization process (`luo_session_serialize()`)
+ * during reboot. On success, the write lock is held indefinitely to freeze
+ * the subsystem. On failure, it is released to allow recovery.
+ *
+ * 2. `luo_session_header->rwsem` (per-list rwsem):
+ * Synchronizes list-level operations for the incoming and outgoing session headers.
+ *
+ * - Writer: Taken during list mutation operations (inserting or removing a
+ * session from the list).
+ * - Reader: Taken when traversing the list (e.g., retrieving a session by name).
+ *
+ * 3. `luo_session->mutex` (per-session mutex):
+ * Protects the internal state and file sets of an individual session. It is
+ * acquired during per-session operations such as preserving, retrieving,
+ * or freezing files.
+ *
+ * Lock Hierarchy:
+ * `luo_session_serialize_rwsem` -> `luo_session_header->rwsem` -> `luo_session->mutex`
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -58,41 +90,34 @@
#include <linux/fs.h>
#include <linux/io.h>
#include <linux/kexec_handover.h>
+#include <linux/kho_block.h>
#include <linux/kho/abi/luo.h>
-#include <linux/libfdt.h>
#include <linux/list.h>
#include <linux/liveupdate.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
-#include <linux/unaligned.h>
#include <uapi/linux/liveupdate.h>
#include "luo_internal.h"
-/* 16 4K pages, give space for 744 sessions */
-#define LUO_SESSION_PGCNT 16ul
-#define LUO_SESSION_MAX (((LUO_SESSION_PGCNT << PAGE_SHIFT) - \
- sizeof(struct luo_session_header_ser)) / \
- sizeof(struct luo_session_ser))
-
+static DECLARE_RWSEM(luo_session_serialize_rwsem);
/**
* struct luo_session_header - Header struct for managing LUO sessions.
- * @count: The number of sessions currently tracked in the @list.
- * @list: The head of the linked list of `struct luo_session` instances.
- * @rwsem: A read-write semaphore providing synchronized access to the
- * session list and other fields in this structure.
- * @header_ser: The header data of serialization array.
- * @ser: The serialized session data (an array of
- * `struct luo_session_ser`).
- * @active: Set to true when first initialized. If previous kernel did not
- * send session data, active stays false for incoming.
+ * @count: The number of sessions currently tracked in the @list.
+ * @list: The head of the linked list of `struct luo_session` instances.
+ * @rwsem: A read-write semaphore providing synchronized access to the
+ * session list and other fields in this structure.
+ * @block_set: The set of serialization blocks.
+ * @sessions_pa: Points to the location of sessions_pa within struct luo_ser.
+ * @active: Set to true when first initialized. If previous kernel did not
+ * send session data, active stays false for incoming.
*/
struct luo_session_header {
long count;
struct list_head list;
struct rw_semaphore rwsem;
- struct luo_session_header_ser *header_ser;
- struct luo_session_ser *ser;
+ struct kho_block_set block_set;
+ u64 *sessions_pa;
bool active;
};
@@ -110,10 +135,14 @@ static struct luo_session_global luo_session_global = {
.incoming = {
.list = LIST_HEAD_INIT(luo_session_global.incoming.list),
.rwsem = __RWSEM_INITIALIZER(luo_session_global.incoming.rwsem),
+ .block_set = KHO_BLOCK_SET_INIT(luo_session_global.incoming.block_set,
+ sizeof(struct luo_session_ser)),
},
.outgoing = {
.list = LIST_HEAD_INIT(luo_session_global.outgoing.list),
.rwsem = __RWSEM_INITIALIZER(luo_session_global.outgoing.rwsem),
+ .block_set = KHO_BLOCK_SET_INIT(luo_session_global.outgoing.block_set,
+ sizeof(struct luo_session_ser)),
},
};
@@ -144,6 +173,7 @@ static int luo_session_insert(struct luo_session_header *sh,
struct luo_session *session)
{
struct luo_session *it;
+ int err;
guard(rwsem_write)(&sh->rwsem);
@@ -152,8 +182,9 @@ static int luo_session_insert(struct luo_session_header *sh,
* for new session.
*/
if (sh == &luo_session_global.outgoing) {
- if (sh->count == LUO_SESSION_MAX)
- return -ENOMEM;
+ err = kho_block_set_grow(&sh->block_set, sh->count + 1);
+ if (err)
+ return err;
}
/*
@@ -178,6 +209,8 @@ static void luo_session_remove(struct luo_session_header *sh,
guard(rwsem_write)(&sh->rwsem);
list_del(&session->list);
sh->count--;
+ if (sh == &luo_session_global.outgoing)
+ kho_block_set_shrink(&sh->block_set, sh->count);
}
static int luo_session_finish_one(struct luo_session *session)
@@ -205,6 +238,7 @@ static int luo_session_release(struct inode *inodep, struct file *filep)
struct luo_session *session = filep->private_data;
struct luo_session_header *sh;
+ guard(rwsem_read)(&luo_session_serialize_rwsem);
/* If retrieved is set, it means this session is from incoming list */
if (session->retrieved) {
int err = luo_session_finish_one(session);
@@ -256,10 +290,11 @@ static int luo_session_retrieve_fd(struct luo_session *session,
if (argp->fd < 0)
return argp->fd;
- guard(mutex)(&session->mutex);
+ mutex_lock(&session->mutex);
err = luo_retrieve_file(&session->file_set, argp->token, &file);
+ mutex_unlock(&session->mutex);
if (err < 0)
- goto err_put_fd;
+ goto err_put_fd;
err = luo_ucmd_respond(ucmd, sizeof(*argp));
if (err)
@@ -289,38 +324,80 @@ static int luo_session_finish(struct luo_session *session,
return luo_ucmd_respond(ucmd, sizeof(*argp));
}
+static int luo_session_get_name(struct luo_session *session,
+ struct luo_ucmd *ucmd)
+{
+ struct liveupdate_session_get_name *argp = ucmd->cmd;
+
+ if (argp->reserved != 0)
+ return -EINVAL;
+
+ strscpy((char *)argp->name, session->name, sizeof(argp->name));
+
+ return luo_ucmd_respond(ucmd, sizeof(*argp));
+}
+
union ucmd_buffer {
struct liveupdate_session_finish finish;
struct liveupdate_session_preserve_fd preserve;
struct liveupdate_session_retrieve_fd retrieve;
+ struct liveupdate_session_get_name get_name;
+};
+
+/* Type of sessions the ioctl applies to. */
+enum luo_ioctl_type {
+ LUO_IOCTL_INCOMING,
+ LUO_IOCTL_OUTGOING,
+ LUO_IOCTL_ALL,
};
struct luo_ioctl_op {
unsigned int size;
unsigned int min_size;
unsigned int ioctl_num;
+ enum luo_ioctl_type type;
int (*execute)(struct luo_session *session, struct luo_ucmd *ucmd);
};
-#define IOCTL_OP(_ioctl, _fn, _struct, _last) \
+#define IOCTL_OP(_ioctl, _fn, _struct, _last, _type) \
[_IOC_NR(_ioctl) - LIVEUPDATE_CMD_SESSION_BASE] = { \
.size = sizeof(_struct) + \
BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \
sizeof(_struct)), \
.min_size = offsetofend(_struct, _last), \
.ioctl_num = _ioctl, \
+ .type = _type, \
.execute = _fn, \
}
static const struct luo_ioctl_op luo_session_ioctl_ops[] = {
IOCTL_OP(LIVEUPDATE_SESSION_FINISH, luo_session_finish,
- struct liveupdate_session_finish, reserved),
+ struct liveupdate_session_finish, reserved, LUO_IOCTL_INCOMING),
IOCTL_OP(LIVEUPDATE_SESSION_PRESERVE_FD, luo_session_preserve_fd,
- struct liveupdate_session_preserve_fd, token),
+ struct liveupdate_session_preserve_fd, token, LUO_IOCTL_OUTGOING),
IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd,
- struct liveupdate_session_retrieve_fd, token),
+ struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_INCOMING),
+ IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name,
+ struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_ALL),
};
+static bool luo_ioctl_type_valid(struct luo_session *session,
+ const struct luo_ioctl_op *op)
+{
+ switch (op->type) {
+ case LUO_IOCTL_INCOMING:
+ /* Retrieved is only set on incoming sessions */
+ return session->retrieved;
+ case LUO_IOCTL_OUTGOING:
+ return !session->retrieved;
+ case LUO_IOCTL_ALL:
+ return true;
+ }
+
+ /* Catch-all. */
+ return false;
+}
+
static long luo_session_ioctl(struct file *filep, unsigned int cmd,
unsigned long arg)
{
@@ -345,6 +422,8 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd,
op = &luo_session_ioctl_ops[nr - LIVEUPDATE_CMD_SESSION_BASE];
if (op->ioctl_num != cmd)
return -ENOIOCTLCMD;
+ if (!luo_ioctl_type_valid(session, op))
+ return -EINVAL;
if (ucmd.user_size < op->min_size)
return -EINVAL;
@@ -354,6 +433,7 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd,
if (ret)
return ret;
+ guard(rwsem_read)(&luo_session_serialize_rwsem);
return op->execute(session, &ucmd);
}
@@ -382,21 +462,28 @@ static int luo_session_getfile(struct luo_session *session, struct file **filep)
int luo_session_create(const char *name, struct file **filep)
{
+ size_t len = strnlen(name, LIVEUPDATE_SESSION_NAME_LENGTH);
struct luo_session *session;
int err;
+ if (len == 0 || len > LIVEUPDATE_SESSION_NAME_LENGTH - 1)
+ return -EINVAL;
+
session = luo_session_alloc(name);
if (IS_ERR(session))
return PTR_ERR(session);
+ down_read(&luo_session_serialize_rwsem);
err = luo_session_insert(&luo_session_global.outgoing, session);
if (err)
goto err_free;
- scoped_guard(mutex, &session->mutex)
- err = luo_session_getfile(session, filep);
+ mutex_lock(&session->mutex);
+ err = luo_session_getfile(session, filep);
+ mutex_unlock(&session->mutex);
if (err)
goto err_remove;
+ up_read(&luo_session_serialize_rwsem);
return 0;
@@ -404,6 +491,7 @@ err_remove:
luo_session_remove(&luo_session_global.outgoing, session);
err_free:
luo_session_free(session);
+ up_read(&luo_session_serialize_rwsem);
return err;
}
@@ -415,12 +503,12 @@ int luo_session_retrieve(const char *name, struct file **filep)
struct luo_session *it;
int err;
- scoped_guard(rwsem_read, &sh->rwsem) {
- list_for_each_entry(it, &sh->list, list) {
- if (!strncmp(it->name, name, sizeof(it->name))) {
- session = it;
- break;
- }
+ guard(rwsem_read)(&luo_session_serialize_rwsem);
+ guard(rwsem_read)(&sh->rwsem);
+ list_for_each_entry(it, &sh->list, list) {
+ if (!strncmp(it->name, name, sizeof(it->name))) {
+ session = it;
+ break;
}
}
@@ -438,74 +526,58 @@ int luo_session_retrieve(const char *name, struct file **filep)
return err;
}
-int __init luo_session_setup_outgoing(void *fdt_out)
+void __init luo_session_setup_outgoing(u64 *sessions_pa)
{
- struct luo_session_header_ser *header_ser;
- u64 header_ser_pa;
- int err;
+ luo_session_global.outgoing.sessions_pa = sessions_pa;
+ luo_session_global.outgoing.active = true;
+}
- header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT);
- if (IS_ERR(header_ser))
- return PTR_ERR(header_ser);
- header_ser_pa = virt_to_phys(header_ser);
+int __init luo_session_setup_incoming(u64 sessions_pa)
+{
+ struct luo_session_header *sh = &luo_session_global.incoming;
+ int err;
- err = fdt_begin_node(fdt_out, LUO_FDT_SESSION_NODE_NAME);
- err |= fdt_property_string(fdt_out, "compatible",
- LUO_FDT_SESSION_COMPATIBLE);
- err |= fdt_property(fdt_out, LUO_FDT_SESSION_HEADER, &header_ser_pa,
- sizeof(header_ser_pa));
- err |= fdt_end_node(fdt_out);
+ if (!sessions_pa)
+ return 0;
+ err = kho_block_set_restore(&sh->block_set, sessions_pa);
if (err)
- goto err_unpreserve;
-
- luo_session_global.outgoing.header_ser = header_ser;
- luo_session_global.outgoing.ser = (void *)(header_ser + 1);
- luo_session_global.outgoing.active = true;
+ return err;
+ sh->active = true;
return 0;
-
-err_unpreserve:
- kho_unpreserve_free(header_ser);
- return err;
}
-int __init luo_session_setup_incoming(void *fdt_in)
+static int luo_session_deserialize_one(struct luo_session_header *sh,
+ struct luo_session_ser *ser)
{
- struct luo_session_header_ser *header_ser;
- int err, header_size, offset;
- u64 header_ser_pa;
- const void *ptr;
-
- offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_SESSION_NODE_NAME);
- if (offset < 0) {
- pr_err("Unable to get session node: [%s]\n",
- LUO_FDT_SESSION_NODE_NAME);
- return -EINVAL;
+ struct luo_session *session;
+ int err;
+
+ session = luo_session_alloc(ser->name);
+ if (IS_ERR(session)) {
+ pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n",
+ (int)sizeof(ser->name), ser->name, session);
+ return PTR_ERR(session);
}
- err = fdt_node_check_compatible(fdt_in, offset,
- LUO_FDT_SESSION_COMPATIBLE);
+ err = luo_session_insert(sh, session);
if (err) {
- pr_err("Session node incompatible [%s]\n",
- LUO_FDT_SESSION_COMPATIBLE);
- return -EINVAL;
+ pr_warn("Failed to insert session [%s] %pe\n",
+ session->name, ERR_PTR(err));
+ luo_session_free(session);
+ return err;
}
- header_size = 0;
- ptr = fdt_getprop(fdt_in, offset, LUO_FDT_SESSION_HEADER, &header_size);
- if (!ptr || header_size != sizeof(u64)) {
- pr_err("Unable to get session header '%s' [%d]\n",
- LUO_FDT_SESSION_HEADER, header_size);
- return -EINVAL;
+ scoped_guard(mutex, &session->mutex) {
+ err = luo_file_deserialize(&session->file_set,
+ &ser->file_set_ser);
+ }
+ if (err) {
+ pr_warn("Failed to deserialize files for session [%s] %pe\n",
+ session->name, ERR_PTR(err));
+ return err;
}
-
- header_ser_pa = get_unaligned((u64 *)ptr);
- header_ser = phys_to_virt(header_ser_pa);
-
- luo_session_global.incoming.header_ser = header_ser;
- luo_session_global.incoming.ser = (void *)(header_ser + 1);
- luo_session_global.incoming.active = true;
return 0;
}
@@ -514,6 +586,8 @@ int luo_session_deserialize(void)
{
struct luo_session_header *sh = &luo_session_global.incoming;
static bool is_deserialized;
+ struct luo_session_ser *ser;
+ struct kho_block_set_it it;
static int saved_err;
int err;
@@ -540,43 +614,19 @@ int luo_session_deserialize(void)
* userspace to detect the failure and trigger a reboot, which will
* reliably reset devices and reclaim memory.
*/
- for (int i = 0; i < sh->header_ser->count; i++) {
- struct luo_session *session;
-
- session = luo_session_alloc(sh->ser[i].name);
- if (IS_ERR(session)) {
- pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n",
- (int)sizeof(sh->ser[i].name),
- sh->ser[i].name, session);
- err = PTR_ERR(session);
- goto save_err;
- }
-
- err = luo_session_insert(sh, session);
- if (err) {
- pr_warn("Failed to insert session [%s] %pe\n",
- session->name, ERR_PTR(err));
- luo_session_free(session);
- goto save_err;
- }
-
- scoped_guard(mutex, &session->mutex) {
- err = luo_file_deserialize(&session->file_set,
- &sh->ser[i].file_set_ser);
- }
- if (err) {
- pr_warn("Failed to deserialize files for session [%s] %pe\n",
- session->name, ERR_PTR(err));
+ kho_block_set_it_init(&it, &sh->block_set);
+ while ((ser = kho_block_set_it_read_entry(&it))) {
+ err = luo_session_deserialize_one(sh, ser);
+ if (err)
goto save_err;
- }
}
- kho_restore_free(sh->header_ser);
- sh->header_ser = NULL;
- sh->ser = NULL;
+ kho_block_set_destroy(&sh->block_set);
return 0;
+
save_err:
+ kho_block_set_destroy(&sh->block_set);
saved_err = err;
return err;
}
@@ -585,30 +635,48 @@ int luo_session_serialize(void)
{
struct luo_session_header *sh = &luo_session_global.outgoing;
struct luo_session *session;
- int i = 0;
+ struct kho_block_set_it it;
int err;
- guard(rwsem_write)(&sh->rwsem);
+ down_write(&luo_session_serialize_rwsem);
+ down_write(&sh->rwsem);
+ *sh->sessions_pa = 0;
+
+ kho_block_set_it_init(&it, &sh->block_set);
+
list_for_each_entry(session, &sh->list, list) {
- err = luo_session_freeze_one(session, &sh->ser[i]);
- if (err)
+ struct luo_session_ser *ser = kho_block_set_it_reserve_entry(&it);
+
+ /* This should not fail normally as blocks were pre-allocated */
+ if (WARN_ON_ONCE(!ser)) {
+ err = -ENOSPC;
+ goto err_undo;
+ }
+
+ err = luo_session_freeze_one(session, ser);
+ if (err) {
+ kho_block_set_it_prev(&it);
goto err_undo;
+ }
- strscpy(sh->ser[i].name, session->name,
- sizeof(sh->ser[i].name));
- i++;
+ strscpy(ser->name, session->name, sizeof(ser->name));
}
- sh->header_ser->count = sh->count;
+
+ if (sh->count > 0)
+ *sh->sessions_pa = kho_block_set_head_pa(&sh->block_set);
+ up_write(&sh->rwsem);
return 0;
err_undo:
list_for_each_entry_continue_reverse(session, &sh->list, list) {
- i--;
- luo_session_unfreeze_one(session, &sh->ser[i]);
- memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name));
+ struct luo_session_ser *ser = kho_block_set_it_prev(&it);
+
+ luo_session_unfreeze_one(session, ser);
+ memset(ser->name, 0, sizeof(ser->name));
}
+ up_write(&sh->rwsem);
+ up_write(&luo_session_serialize_rwsem);
return err;
}
-