// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2023-2026 ARM Ltd. */ #include #include #include #include #include #include #include #include /* RMM v2.0 defines RmiFeatureRegister0 to RmiFeatureRegister4. */ static unsigned long rmi_feat_reg_cache[5] __ro_after_init; static bool arm64_rmm_active; static bool arm64_rmi_is_available; /* is_rmm_active: Returns if the RMM is ACTIVE */ bool is_rmm_active(void) { return READ_ONCE(arm64_rmm_active); } EXPORT_SYMBOL_GPL(is_rmm_active); /* * is_rmi_available: Returns if the RMM is ACTIVE and is configured with static * carveout for metadata nad L1GPTs. */ bool is_rmi_available(void) { return READ_ONCE(arm64_rmi_is_available); } EXPORT_SYMBOL_GPL(is_rmi_available); /* * rmi_granule_range_delegate() - Delegate granules * @base: PA of the first granule of the range * @top: PA of the first granule after the range * @out_top: PA of the first granule not delegated * * Delegate a range of granule for use by the realm world. If the entire range * was delegated then @out_top == @top, otherwise the function should be called * again with @base == @out_top. * * Return: 0 on success, positive RMI result code or negative Linux error code */ static long rmi_granule_range_delegate(unsigned long base, unsigned long top, unsigned long *out_top) { struct arm_smccc_1_2_regs regs = { SMC_RMI_GRANULE_RANGE_DELEGATE, base, top }; long ret = rmi_sro_execute(®s); if (ret == RMI_SUCCESS && out_top) *out_top = regs.a1; return ret; } /* * rmi_delegate_range: Delegate a physically contiguous range. * We iterate over the range until we hit an error. So we may * return an error, but with a partially delegated range. The * caller must always look at the @out_phys to figure out, how * much progress was made. * * @phys: Base of the physical address range * @size: Size of the physical address range * @out_phys: Top of the range that was completed. This is always * valid, irrespective of the result. * * Returns RMI_SUCCESS on successful completion. Otherwise, returns * the Linux error number or the RMI status code as described * by the RMM spec for RMI_GRANULE_DELEGATE_RANGE or RMI_BLOCKED. */ int rmi_delegate_range(phys_addr_t phys, unsigned long size, phys_addr_t *out_phys) { long ret = 0; unsigned long top = phys + size; unsigned long out_top; while (phys < top) { ret = rmi_granule_range_delegate(phys, top, &out_top); if (ret != RMI_SUCCESS) break; /* * Buggy RMM ? Let the caller handle the failure. * We can't know how far the RMM delegated in this * iteration, so we return the best known good limit. * RMM can deal with granules already in "undelegated" * in a given range. So, it is fine for the caller to * try the range we return. */ if (WARN_ON(out_top <= phys)) { ret = -ENXIO; break; } phys = out_top; } if (out_phys) *out_phys = phys; return ret; } EXPORT_SYMBOL_GPL(rmi_delegate_range); /* * rmi_granule_range_undelegate() - Undelegate a range of granules * @base: Base PA of the target range * @top: Top PA of the target range * @out_top: Returns the top PA of range whose state is undelegated * * Undelegate a range of granules to allow use by the normal world. Will fail * if the granules are in use by RMM. RMM can ignore granules that are already * undelegated and thus is safe to be called on a range with a mix of delegated * and undelegated granules. * * Return: 0 on success, positive RMI result code or negative Linux error code */ static long rmi_granule_range_undelegate(unsigned long base, unsigned long top, unsigned long *out_top) { struct arm_smccc_1_2_regs regs = { SMC_RMI_GRANULE_RANGE_UNDELEGATE, base, top }; long ret = rmi_sro_execute(®s); if (ret == RMI_SUCCESS && out_top) *out_top = regs.a1; return ret; } int rmi_undelegate_range(phys_addr_t phys, unsigned long size) { long ret = 0; unsigned long top = phys + size; unsigned long out_top; while (phys < top) { ret = rmi_granule_range_undelegate(phys, top, &out_top); if (ret != RMI_SUCCESS) break; /* Buggy RMM ? Let the caller leak the pages */ if (WARN_ON(out_top <= phys)) return -ENXIO; phys = out_top; } return ret; } EXPORT_SYMBOL_GPL(rmi_undelegate_range); /* * rmi_free_delegated_page: Undelegate and free a page that has been previously * delegated to the Realm world. If we are unable to undelegate it, the page is * leaked. * NOTE: Do not use this helper if the page could be concurrently operated by * another thread, as it may get leaked if the undelegation fails due to RMI_BLOCKED */ int rmi_free_delegated_page(phys_addr_t phys) { if (WARN_ON_ONCE(rmi_undelegate_page(phys))) { /* Undelegate failed: leak the page */ return -EBUSY; } free_page((unsigned long)phys_to_virt(phys)); return 0; } EXPORT_SYMBOL_GPL(rmi_free_delegated_page); /* * Convert the RmiAddrBlockSize to actual size. This is used in RmiDonateReq * and RmiAddrRangeDesc*. */ static unsigned long rmi_addr_block_size_to_bytes(unsigned long block_size_fld) { return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(3 - block_size_fld)); } /* * free_addr_range: Free memory described by the address range entry, which may * be partially consumed by RMM. * * @entry: RMI_ADDR_RANGE descriptor * @consumed_size: Page aligned size consumed by the RMM from the address range. * * If the state of the address is DELEGATED, undelegate it back, before freeing. * Leaks the memory if we cannot undelegate the range. */ static void free_addr_range(unsigned long entry, unsigned long consumed_size) { unsigned long phys = RMI_ADDR_RANGE_ADDR(entry); unsigned long block_size_fld = RMI_ADDR_RANGE_BLOCK_SIZE(entry); unsigned long count = RMI_ADDR_RANGE_COUNT(entry); unsigned long state = RMI_ADDR_RANGE_STATE(entry); unsigned long size = rmi_addr_block_size_to_bytes(block_size_fld) * count; WARN_ON(!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(consumed_size)); /* We shouldn't see this in reclaim path, leak it for now */ if (WARN_ON((state != RMI_OP_MEM_DELEGATED) && (state != RMI_OP_MEM_UNDELEGATED))) return; /* Adjust the address and size for partially consumed entry */ phys += consumed_size; size -= consumed_size; /* * Undelegate the pages back if required. If we can't * change them back, leak the pages. */ if (state == RMI_OP_MEM_DELEGATED && WARN_ON(rmi_undelegate_range(phys, size))) return; free_pages_exact(phys_to_virt(phys), size); } static void rmi_op_continue(unsigned long sro_handle, unsigned long flags, struct arm_smccc_1_2_regs *out_regs) { *out_regs = (struct arm_smccc_1_2_regs) { SMC_RMI_OP_CONTINUE, sro_handle, flags }; rmi_smccc_invoke(out_regs); } static void rmi_op_cancel(unsigned long sro_handle, struct arm_smccc_1_2_regs *out_regs) { *out_regs = (struct arm_smccc_1_2_regs) { SMC_RMI_OP_CANCEL, sro_handle }; rmi_smccc_invoke(out_regs); } static void rmi_op_mem_donate(unsigned long sro_handle, unsigned long list_addr, unsigned long list_count, unsigned long flags, struct arm_smccc_1_2_regs *out_regs) { *out_regs = (struct arm_smccc_1_2_regs) { SMC_RMI_OP_MEM_DONATE, sro_handle, list_addr, list_count, flags }; /* * The output donated count (a1) is always valid, irrespective * of the return result. i.e., 0 if there was an error */ rmi_smccc_invoke(out_regs); } static void rmi_op_mem_reclaim(unsigned long sro_handle, unsigned long list_addr, unsigned long list_count, struct arm_smccc_1_2_regs *out_regs) { *out_regs = (struct arm_smccc_1_2_regs) { SMC_RMI_OP_MEM_RECLAIM, sro_handle, list_addr, list_count }; rmi_smccc_invoke(out_regs); } static int rmi_sro_ensure_capacity(struct rmi_sro_state *sro, unsigned long count) { if (WARN_ON_ONCE(sro->addr_count > RMI_MAX_ADDR_LIST)) return -EOVERFLOW; if (count > RMI_MAX_ADDR_LIST - sro->addr_count) return -ENOSPC; return 0; } static int rmi_sro_donate_contig(struct rmi_sro_state *sro, unsigned long sro_handle, unsigned long donatereq, struct arm_smccc_1_2_regs *out_regs, gfp_t gfp) { unsigned long block_size_fld = RMI_DONATE_BLOCK_SIZE(donatereq); unsigned long block_size = rmi_addr_block_size_to_bytes(block_size_fld); unsigned long count = RMI_DONATE_COUNT(donatereq); unsigned long state = RMI_DONATE_STATE(donatereq); unsigned long size = block_size * count; unsigned long addr_range; unsigned long donated_size; int ret; void *virt; phys_addr_t phys; /* * The RMM specification requires contiguous allocations are always a * power of 2 */ if (WARN_ON_ONCE(!is_power_of_2(size))) return -EINVAL; /* * RMM clamps the Maximum value of RmiOpMemDonateReq:count to prevent * overflow in the RMI_ADDR_RANGE_COUNT field. */ if (WARN_ON_ONCE(count > FIELD_MAX(RMI_ADDR_RANGE_COUNT_MASK))) return -EINVAL; /* * We can't allocate blocks more than what the buddy allocator can * satisfy. TODO: Add support for larger allocations. */ if (get_order(size) > MAX_PAGE_ORDER) return -ENOMEM; /* Reuse the cached address range if we have one */ for (int i = 0; i < sro->addr_count; i++) { unsigned long entry = sro->addr_list[i]; if (RMI_ADDR_RANGE_BLOCK_SIZE(entry) == block_size_fld && RMI_ADDR_RANGE_COUNT(entry) == count && RMI_ADDR_RANGE_STATE(entry) == state && IS_ALIGNED(RMI_ADDR_RANGE_ADDR(entry), size)) { sro->addr_count--; swap(sro->addr_list[sro->addr_count], sro->addr_list[i]); goto mem_donate; } } ret = rmi_sro_ensure_capacity(sro, 1); /* In the odd case we have filled up the cache, free up the entire list */ if (ret) rmi_sro_free(sro); virt = alloc_pages_exact(size, gfp); if (!virt) return -ENOMEM; phys = virt_to_phys(virt); if (state == RMI_OP_MEM_DELEGATED) { phys_addr_t delegated_phys; if (rmi_delegate_range(phys, size, &delegated_phys)) { if (!rmi_undelegate_range(phys, delegated_phys - phys)) free_pages_exact(virt, size); return -ENXIO; } } addr_range = phys & RMI_ADDR_RANGE_ADDR_MASK; FIELD_MODIFY(RMI_ADDR_RANGE_BLOCK_SIZE_MASK, &addr_range, block_size_fld); FIELD_MODIFY(RMI_ADDR_RANGE_COUNT_MASK, &addr_range, count); FIELD_MODIFY(RMI_ADDR_RANGE_STATE_MASK, &addr_range, state); sro->addr_list[sro->addr_count] = addr_range; mem_donate: rmi_op_mem_donate(sro_handle, virt_to_phys(&sro->addr_list[sro->addr_count]), 1, 0, out_regs); donated_size = out_regs->a1 << PAGE_SHIFT; if (WARN_ON(out_regs->a1 > (size >> PAGE_SHIFT))) donated_size = size; /* All granules consumed by the RMM */ if (donated_size == size) return 0; /* * No granules were consumed by the RMM, cache them. This is a valid * case, where the RMM was donated memory by another competing thread. */ if (donated_size == 0) { sro->addr_count++; return 0; } /* The granules were partially consumed, reclaim the unused ones. */ free_addr_range(sro->addr_list[sro->addr_count], donated_size); return 0; } static int rmi_sro_donate_noncontig(struct rmi_sro_state *sro, unsigned long sro_handle, unsigned long donatereq, struct arm_smccc_1_2_regs *out_regs, gfp_t gfp) { unsigned long block_size_fld = RMI_DONATE_BLOCK_SIZE(donatereq); unsigned long block_size = rmi_addr_block_size_to_bytes(block_size_fld); unsigned long count = RMI_DONATE_COUNT(donatereq); unsigned long state = RMI_DONATE_STATE(donatereq); unsigned long found = 0; unsigned long donated_granules; unsigned long granules_per_block = block_size >> PAGE_SHIFT; unsigned long consumed_blocks; int addr_list_start = sro->addr_count; int ret, i, src; /* * We can't allocate blocks more than what the buddy allocator can * satisfy. TODO: Add support for larger allocations. */ if (get_order(block_size) > MAX_PAGE_ORDER) return -ENOMEM; /* * Maximum value of RmiOpMemDonateReq:count is 2^(PAGE_SHIFT-2) - 1. * But we further clamp it down by the number of entries we can do * in one go. The RMM can request the remaining in the next iteration. */ if (count > RMI_MAX_ADDR_LIST) count = RMI_MAX_ADDR_LIST; /* Gather the suitable entries to the end of the list */ i = 0; while (i < addr_list_start && found < count) { unsigned long entry = sro->addr_list[i]; if (RMI_ADDR_RANGE_BLOCK_SIZE(entry) == block_size_fld && RMI_ADDR_RANGE_COUNT(entry) == 1 && RMI_ADDR_RANGE_STATE(entry) == state) { addr_list_start--; swap(sro->addr_list[addr_list_start], sro->addr_list[i]); found++; /* Continue from the swapped in entry */ continue; } /* Skip past the entry */ i++; } ret = rmi_sro_ensure_capacity(sro, count - found); if (ret) { /* If we have found some entries, donate them and try again */ if (found) goto mem_donate; /* Otherwise free up the list and start again */ rmi_sro_free(sro); /* Reset the addr_list_start to match sro->addr_count */ addr_list_start = 0; } for (; found < count; found++) { unsigned long addr_range; void *virt = alloc_pages_exact(block_size, gfp); phys_addr_t phys; if (!virt) return -ENOMEM; phys = virt_to_phys(virt); if (state == RMI_OP_MEM_DELEGATED) { phys_addr_t delegated_phys; if (rmi_delegate_range(phys, block_size, &delegated_phys)) { if (!rmi_undelegate_range(phys, delegated_phys - phys)) free_pages_exact(virt, block_size); return -ENXIO; } } addr_range = phys & RMI_ADDR_RANGE_ADDR_MASK; FIELD_MODIFY(RMI_ADDR_RANGE_BLOCK_SIZE_MASK, &addr_range, block_size_fld); FIELD_MODIFY(RMI_ADDR_RANGE_COUNT_MASK, &addr_range, 1); FIELD_MODIFY(RMI_ADDR_RANGE_STATE_MASK, &addr_range, state); sro->addr_list[sro->addr_count++] = addr_range; } mem_donate: rmi_op_mem_donate(sro_handle, virt_to_phys(&sro->addr_list[addr_list_start]), found, 0, out_regs); donated_granules = out_regs->a1; /* * The RMM shouldn't report more granules than we provided, but clamp * just in case. */ if (WARN_ON_ONCE(donated_granules > found * granules_per_block)) donated_granules = found * granules_per_block; /* * The RMM reports the consumed memory in terms of granules, but we * track in the address lists in block-sized ranges. So divide to get * the number of (complete) consumed blocks. */ consumed_blocks = donated_granules / granules_per_block; if (donated_granules % granules_per_block) { /* * A block has been partially consumed, the start is owned by * the RMM, the tail is owned by the host */ unsigned long entry = sro->addr_list[addr_list_start + consumed_blocks]; unsigned long donated_size = (donated_granules % granules_per_block) << PAGE_SHIFT; free_addr_range(entry, donated_size); /* * This block is now fully 'consumed' (either held by the RMM or * freed) */ consumed_blocks++; } /* * Keep just the blocks the RMM didn't use in addr_list * RMM claimed consumed_blocks entries from addr_list_start. * Move the entries left out at the end i.e., * [ addr_list_start + consumed_blocks, addr_list_start + found) * to the rest of the valid entries and adjust the addr_count to * reflect the available entries. */ for (i = 0, src = addr_list_start + consumed_blocks; i < found - consumed_blocks; i++) sro->addr_list[addr_list_start + i] = sro->addr_list[src + i]; sro->addr_count -= consumed_blocks; return 0; } static int rmi_sro_donate(struct rmi_sro_state *sro, unsigned long sro_handle, unsigned long donatereq, struct arm_smccc_1_2_regs *regs, gfp_t gfp) { unsigned long state = RMI_DONATE_STATE(donatereq); if (WARN_ON_ONCE(!RMI_DONATE_COUNT(donatereq))) return -EINVAL; /* * We do not support RMI_OP_MEM_CONDITIONAL yet. This is only required * for use in RMI_GRANULE_TRACKING_SET, which we don't support yet. */ if (WARN_ON_ONCE((state != RMI_OP_MEM_DELEGATED) && (state != RMI_OP_MEM_UNDELEGATED))) return -EINVAL; if (RMI_DONATE_CONTIG(donatereq) == RMI_OP_MEM_CONTIG) return rmi_sro_donate_contig(sro, sro_handle, donatereq, regs, gfp); else return rmi_sro_donate_noncontig(sro, sro_handle, donatereq, regs, gfp); } static int rmi_sro_reclaim(struct rmi_sro_state *sro, unsigned long sro_handle, struct arm_smccc_1_2_regs *out_regs) { unsigned long capacity; /* * We don't do a partial free of the entries. So for now free the * entire address list as we prepare to reclaim more from the RMM. */ if (rmi_sro_ensure_capacity(sro, 1)) rmi_sro_free(sro); capacity = RMI_MAX_ADDR_LIST - sro->addr_count; rmi_op_mem_reclaim(sro_handle, virt_to_phys(&sro->addr_list[sro->addr_count]), capacity, out_regs); /* * RMI_OP_MEM_RECLAIM always return RMI_INCOMPLETE, except when the * input parameters were invalid. */ if (WARN_ON_ONCE(RMI_RESULT_STATUS(out_regs->a0) != RMI_INCOMPLETE)) return -EINVAL; if (WARN_ON_ONCE(out_regs->a1 > capacity)) out_regs->a1 = capacity; sro->addr_count += out_regs->a1; return 0; } void rmi_sro_free(struct rmi_sro_state *sro) { /* Handle the worse */ if (WARN_ON(sro->addr_count < 0)) return; if (WARN_ON(sro->addr_count > RMI_MAX_ADDR_LIST)) sro->addr_count = RMI_MAX_ADDR_LIST; for (int i = 0; i < sro->addr_count; i++) free_addr_range(sro->addr_list[i], 0); sro->addr_count = 0; } EXPORT_SYMBOL_GPL(rmi_sro_free); long rmi_sro_memxfer_execute(struct rmi_sro_state *sro, gfp_t gfp) { struct arm_smccc_1_2_regs *regs = &sro->regs; bool cancelled = false; unsigned long sro_handle; rmi_smccc_invoke(regs); sro_handle = regs->a1; while (RMI_RESULT_STATUS(regs->a0) == RMI_INCOMPLETE) { bool can_cancel = RMI_RESULT_CAN_CANCEL(regs->a0) == RMI_OP_CAN_CANCEL; int ret = 0; switch (RMI_RESULT_MEMREQ(regs->a0)) { case RMI_OP_MEM_REQ_NONE: rmi_op_continue(sro_handle, RMI_CONTINUE_KEEP_GOING, regs); break; case RMI_OP_MEM_REQ_DONATE: ret = rmi_sro_donate(sro, sro_handle, regs->a2, regs, gfp); break; case RMI_OP_MEM_REQ_RECLAIM: ret = rmi_sro_reclaim(sro, sro_handle, regs); break; default: WARN_ON_ONCE(1); ret = -ENXIO; } if (ret) { /* * All memory donating SROs must be cancellable. So a * failure in memory allocation shouldn't be an issue. * However, if we encounter a random failure (e.g., * buggy RMM), don't loop forever, just give up. */ if (WARN_ON_ONCE(!can_cancel)) return ret; /* * If we have already cancelled, and came back here due * to an error in MEMREQ, then there is no point * in going in loops. */ if (WARN_ON_ONCE(cancelled)) break; rmi_op_cancel(sro_handle, regs); cancelled = true; if (WARN_ON_ONCE(RMI_RESULT_STATUS(regs->a0) != RMI_INCOMPLETE)) return ret; } } if (cancelled) return -ECANCELED; return regs->a0; } EXPORT_SYMBOL_GPL(rmi_sro_memxfer_execute); /* * rmi_sro_execute: Execute an RMI command that is stateful but not memory * transferring. Takes regs, filled with the FIDs and the arguments in place. * * Returns : * -ECANCELED - If the operation had to be aborted and SRO was cancellable. * Otherwise, returns the result of the RMI command. */ long rmi_sro_execute(struct arm_smccc_1_2_regs *regs) { bool cancelled = false; unsigned long sro_handle; rmi_smccc_invoke(regs); sro_handle = regs->a1; while (RMI_RESULT_STATUS(regs->a0) == RMI_INCOMPLETE) { bool can_cancel = RMI_RESULT_CAN_CANCEL(regs->a0) == RMI_OP_CAN_CANCEL; switch (RMI_RESULT_MEMREQ(regs->a0)) { case RMI_OP_MEM_REQ_NONE: rmi_op_continue(sro_handle, RMI_CONTINUE_KEEP_GOING, regs); break; default: WARN_ON_ONCE(1); if (!can_cancel) return regs->a0; /* * We can't get here normally, but handle this anyway * for a buggy RMM implementation. */ if (cancelled) return -ECANCELED; rmi_op_cancel(sro_handle, regs); cancelled = true; } } if (cancelled) return -ECANCELED; return regs->a0; } EXPORT_SYMBOL_GPL(rmi_sro_execute); static int rmi_check_version(void) { unsigned short version_major, version_minor; unsigned long host_version = RMI_ABI_VERSION(RMI_ABI_MAJOR_VERSION, RMI_ABI_MINOR_VERSION); unsigned long aa64pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); struct arm_smccc_1_2_regs res = { SMC_RMI_VERSION, host_version, }; /* If RME isn't supported, then RMI can't be */ if (cpuid_feature_extract_unsigned_field(aa64pfr0, ID_AA64PFR0_EL1_RME_SHIFT) == 0) return -ENXIO; if (arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_SMC) return -ENXIO; rmi_smccc_invoke(&res); if (res.a0 == SMCCC_RET_NOT_SUPPORTED) return -ENXIO; version_major = RMI_ABI_VERSION_GET_MAJOR(res.a1); version_minor = RMI_ABI_VERSION_GET_MINOR(res.a1); if (res.a0 != RMI_SUCCESS) { unsigned short high_version_major, high_version_minor; high_version_major = RMI_ABI_VERSION_GET_MAJOR(res.a2); high_version_minor = RMI_ABI_VERSION_GET_MINOR(res.a2); pr_err("Unsupported RMI ABI (v%d.%d - v%d.%d) we want v%d.%d\n", version_major, version_minor, high_version_major, high_version_minor, RMI_ABI_MAJOR_VERSION, RMI_ABI_MINOR_VERSION); return -ENXIO; } pr_info("RMI ABI version %d.%d\n", version_major, version_minor); return 0; } static int rmi_read_features(void) { /* * Since we've negotiated a compatible version these feature registers * should always be available */ for (int i = 0; i < ARRAY_SIZE(rmi_feat_reg_cache); i++) { struct arm_smccc_1_2_regs args = { SMC_RMI_FEATURES, i, }; rmi_smccc_invoke(&args); if (WARN_ON(args.a0 != RMI_SUCCESS)) return -EINVAL; rmi_feat_reg_cache[i] = args.a1; } return 0; } unsigned long rmi_feat_reg(unsigned int index) { if (WARN_ON(index >= ARRAY_SIZE(rmi_feat_reg_cache))) return 0; return rmi_feat_reg_cache[index]; } EXPORT_SYMBOL_GPL(rmi_feat_reg); static int rmi_rmm_config_set(unsigned long cfg_ptr) { struct arm_smccc_1_2_regs regs = { SMC_RMI_RMM_CONFIG_SET, cfg_ptr, }; rmi_smccc_invoke(®s); return regs.a0; } static int rmi_configure(void) { unsigned long granule_feature; unsigned long granule_size; int ret; switch (PAGE_SIZE) { case SZ_4K: granule_size = RMI_GRANULE_SIZE_4KB; granule_feature = RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_4KB; break; case SZ_16K: granule_size = RMI_GRANULE_SIZE_16KB; granule_feature = RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_16KB; break; case SZ_64K: granule_size = RMI_GRANULE_SIZE_64KB; granule_feature = RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_64KB; break; default: BUILD_BUG(); } if (!(rmi_feat_reg(1) & granule_feature)) { pr_err("RMM does not support %luKB granules\n", PAGE_SIZE >> 10); return -ENXIO; } struct rmm_config *config __free(free_page) = (struct rmm_config *)get_zeroed_page(GFP_KERNEL); if (!config) { pr_err("Unable to allocate memory for RMM config\n"); return -ENOMEM; } config->rmi_granule_size = granule_size; /* * For now we set the tracking_region_size to 0 which is the only option * for 4KB PAGE_SIZE (1GB for 4KB PAGE_SIZE, 32MB/512MB for 16KB/64KB). * TODO: Support other tracking sizes via Kconfig option for other * PAGE_SIZES */ config->tracking_region_size = 0; ret = rmi_rmm_config_set(virt_to_phys(config)); if (ret) { pr_err("RMM config set failed (%d)\n", ret); return -EINVAL; } return 0; } /** * rmi_granule_tracking_get() - Get configuration of a Granule tracking region * @start: Base PA of the tracking region * @end: End of the PA region * @out_category: Memory category * @out_state: Tracking region state * @out_top: Top of the memory region * * Return: RMI return code */ static int rmi_granule_tracking_get(unsigned long start, unsigned long end, unsigned long *out_category, unsigned long *out_state, unsigned long *out_top) { struct arm_smccc_1_2_regs regs = { SMC_RMI_GRANULE_TRACKING_GET, start, end, }; rmi_smccc_invoke(®s); if (regs.a0 != RMI_SUCCESS) return regs.a0; if (out_category) *out_category = regs.a1; if (out_state) *out_state = regs.a2; if (out_top) *out_top = regs.a3; return RMI_SUCCESS; } /* * Make sure the area is tracked by RMM at FINE granularity. * We do not support changing the tracking yet. */ static int rmi_verify_memory_tracking(phys_addr_t start, phys_addr_t end) { while (start < end) { unsigned long ret, category, state, next; ret = rmi_granule_tracking_get(start, end, &category, &state, &next); if (ret != RMI_SUCCESS) { pr_err("RMI_GRANULE_TRACKING_GET failed for %llx-%llx: (%lu)\n", start, end, ret); return -ENXIO; } if (WARN_ON(next <= start)) return -ENXIO; if (state != RMI_TRACKING_FINE || category != RMI_MEM_CATEGORY_CONVENTIONAL) { /* TODO: Set granule tracking in this case */ pr_err("Granule tracking for region isn't fine/conventional: %llx-%lx\n", start, next); return -ENODEV; } start = next; } return 0; } /* * rmi_gpt_info - Query the GPT info for the given PAR. * @start: Base of the physical address region * @end: Top of the physical address region * @out_top: Top of the physical address region for which the GPT @out_gpt_par_state * is valid * @out_gpt_par_state: State of the GPT covered by [start, out_top) */ static long rmi_gpt_info(unsigned long start, unsigned long end, unsigned long *out_top, unsigned long *out_gpt_par_state) { struct arm_smccc_1_2_regs regs = { SMC_RMI_GPT_INFO, start, end, }; rmi_smccc_invoke(®s); if (regs.a0 != RMI_SUCCESS) return regs.a0; if (out_top) *out_top = regs.a1; if (out_gpt_par_state) *out_gpt_par_state = regs.a2; return RMI_SUCCESS; } /* * We do not support creating L1 GPTs yet. So, make sure that all the regions * are managed by the firmware. */ static int rmi_verify_gpt_firmware_managed(phys_addr_t start, phys_addr_t end) { unsigned long l0gpt_sz; unsigned long next, par_state; l0gpt_sz = 1UL << (30 + FIELD_GET(RMI_FEATURE_REGISTER_1_L0GPTSZ, rmi_feat_reg(1))); start = ALIGN_DOWN(start, l0gpt_sz); end = ALIGN(end, l0gpt_sz); while (start < end) { long ret = rmi_gpt_info(start, end, &next, &par_state); if (ret != RMI_SUCCESS) { pr_err("RMI_GPT_INFO failed for %llx-%llx: (%ld)\n", start, end, ret); return -ENXIO; } if (WARN_ON(next <= start)) return -ENXIO; if (par_state != RMI_GPT_PAR_PLAT) { pr_err("GPT for the region is not managed by firmware %llx-%lx\n", start, next); return -ENODEV; } start = next; } return 0; } static int rmi_prepare_memory(phys_addr_t start, phys_addr_t end) { int ret; if (start >= end) return -EINVAL; ret = rmi_verify_memory_tracking(start, end); if (ret) return ret; return rmi_verify_gpt_firmware_managed(start, end); } static int rmi_init_metadata(void) { phys_addr_t start, end; struct memblock_region *r; for_each_mem_region(r) { int ret; /* Firmware-reserved NOMAP regions are not usable system RAM */ if (memblock_is_nomap(r)) continue; start = PAGE_ALIGN(r->base); /* * We always deal with PAGE_SIZE and if in the odd case this * region boundary is not PAGE aligned, we stick to the page * that we can use from the region. */ end = PAGE_ALIGN_DOWN(r->base + r->size); /* Too small ? */ if (start >= end) continue; ret = rmi_prepare_memory(start, end); if (ret) return ret; } return 0; } static int rmi_memory_notifier(struct notifier_block *nb, unsigned long action, void *data) { struct memory_notify *arg = data; phys_addr_t start, end; int ret; if (action != MEM_GOING_ONLINE) return NOTIFY_DONE; start = PFN_PHYS(arg->start_pfn); end = PFN_PHYS(arg->start_pfn + arg->nr_pages); /* * If the hotplugged memory region is in ZONE_MOVABLE, neither the * guest-memfd nor the kernel allocations can come from there. So we are * fine to skip this. * For boot time memory, we don't allow this discount, as the * decision to move boot memory pages to ZONE_MOVABLE is purely * a kernel choice and the firmware must have decided what it covers. */ if (is_zone_movable_page(pfn_to_page(arg->start_pfn))) return NOTIFY_DONE; ret = rmi_prepare_memory(start, end); return notifier_from_errno(ret); } static struct notifier_block rmi_memory_nb = { .notifier_call = rmi_memory_notifier, }; static int rmi_init_memory(void) { int ret; ret = rmi_init_metadata(); if (ret) return ret; return register_memory_notifier(&rmi_memory_nb); } static int __init arm64_init_rmi(void) { int ret; /* If we can't agree on the RMI ABI version, don't proceed further */ ret = rmi_check_version(); if (ret) return ret; ret = rmi_read_features(); if (ret) return ret; ret = rmi_configure(); if (ret) return ret; /* Activate the RMM */ struct rmi_sro_state *sro __free(kfree) = kmalloc_obj(*sro); if (!sro) return -ENOMEM; ret = rmi_sro_memxfer_cmd(sro, GFP_KERNEL, SMC_RMI_RMM_ACTIVATE); if (ret) { pr_err("RMM activate failed (%d)\n", ret); ret = ret < 0 ? ret : -ENXIO; return ret; } WRITE_ONCE(arm64_rmm_active, true); ret = rmi_init_memory(); if (ret) { /* Deactivate the RMM */ if (!WARN_ON(rmi_sro_memxfer_cmd(sro, GFP_KERNEL, SMC_RMI_RMM_DEACTIVATE))) WRITE_ONCE(arm64_rmm_active, false); return ret; } /* Mark the RMM is usable */ WRITE_ONCE(arm64_rmi_is_available, true); pr_info("RMI configured\n"); return 0; } /* * Note arm64_init_rmi() must be called before kvm_init_rmi() otherwise KVM * will not support realm guests. subsys_initcall() is called before * module_init() (used for KVM) so this is OK. */ subsys_initcall(arm64_init_rmi);