diff options
Diffstat (limited to 'drivers/firmware/imx/se_ctrl.c')
| -rw-r--r-- | drivers/firmware/imx/se_ctrl.c | 2434 |
1 files changed, 2434 insertions, 0 deletions
diff --git a/drivers/firmware/imx/se_ctrl.c b/drivers/firmware/imx/se_ctrl.c new file mode 100644 index 000000000000..519f24d24a85 --- /dev/null +++ b/drivers/firmware/imx/se_ctrl.c @@ -0,0 +1,2434 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2026 NXP + */ + +#include <linux/bitfield.h> +#include <linux/cleanup.h> +#include <linux/completion.h> +#include <linux/delay.h> +#include <linux/dev_printk.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/firmware.h> +#include <linux/firmware/imx/se_api.h> +#include <linux/genalloc.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kref.h> +#include <linux/miscdevice.h> +#include <linux/module.h> +#include <linux/of_platform.h> +#include <linux/of_reserved_mem.h> +#include <linux/platform_device.h> +#include <linux/sched/mm.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/sys_soc.h> +#include <uapi/linux/se_ioctl.h> + +#include "ele_base_msg.h" +#include "ele_common.h" +#include "ele_fw_api.h" +#include "se_ctrl.h" + +/* Maximum response buffer size in bytes for debug-dump replies. */ +#define MAX_ALLOWED_TX_MSG_SZ SZ_4K + +#define MAX_SOC_INFO_DATA_SZ 256 + +struct se_soc_dev_regn { + bool soc_dev_registered; + struct soc_device *soc_dev; + struct soc_device_attribute *soc_dev_attr; +}; + +struct se_var_info { + u16 soc_rev; + struct se_soc_dev_regn soc_dev_regn; + /* To serialize populating common SoC level info. */ + struct mutex se_var_info_lock; +}; + +/* contains fixed information */ +struct se_soc_info { + const u16 soc_id; + const char *soc_name; + const struct se_fw_img_name se_fw_img_nm; + bool imem_state_mgmt; +}; + +struct se_if_node { + struct se_soc_info *se_info; + u8 *pool_name; + bool reserved_dma_ranges; + struct se_if_defines if_defs; +}; + +/* common for all the SoC. */ +static struct se_var_info var_se_info = { + .soc_rev = 0, + .se_var_info_lock = __MUTEX_INITIALIZER(var_se_info.se_var_info_lock) +}; + +static struct se_soc_info se_imx8ulp_info = { + .soc_id = SOC_ID_OF_IMX8ULP, + .soc_name = "i.MX8ULP", + .se_fw_img_nm = { + .prim_fw_nm_in_rfs = IMX_ELE_FW_DIR + "mx8ulpa2-ahab-container.img", + .seco_fw_nm_in_rfs = IMX_ELE_FW_DIR + "mx8ulpa2ext-ahab-container.img", + }, + .imem_state_mgmt = true, +}; + +static struct se_if_node imx8ulp_se_ele_hsm = { + .se_info = &se_imx8ulp_info, + .pool_name = "sram", + .reserved_dma_ranges = true, + .if_defs = { + .se_if_type = SE_TYPE_ID_HSM, + .cmd_tag = 0x17, + .rsp_tag = 0xe1, + .success_tag = ELE_SUCCESS_IND, + .base_api_ver = MESSAGING_VERSION_6, + .fw_api_ver = MESSAGING_VERSION_7, + }, +}; + +static struct se_soc_info se_imx93_info = { + .soc_id = SOC_ID_OF_IMX93, +}; + +static struct se_if_node imx93_se_ele_hsm = { + .se_info = &se_imx93_info, + .reserved_dma_ranges = true, + .if_defs = { + .se_if_type = SE_TYPE_ID_HSM, + .cmd_tag = 0x17, + .rsp_tag = 0xe1, + .success_tag = ELE_SUCCESS_IND, + .base_api_ver = MESSAGING_VERSION_6, + .fw_api_ver = MESSAGING_VERSION_7, + }, +}; + +static const struct of_device_id se_match[] = { + { .compatible = "fsl,imx8ulp-se-ele-hsm", .data = &imx8ulp_se_ele_hsm }, + { .compatible = "fsl,imx93-se-ele-hsm", .data = &imx93_se_ele_hsm }, + { } +}; +MODULE_DEVICE_TABLE(of, se_match); + +/** + * get_se_if_name() - return a human-readable string for a SE interface type. + * @se_if_id: SE interface type identifier (e.g. SE_TYPE_ID_HSM). + * + * Return: pointer to a constant string naming the interface type, or "unknown" + * if @se_if_id does not match any known type. + */ +char *get_se_if_name(u8 se_if_id) +{ + switch (se_if_id) { + case SE_TYPE_ID_DBG: return SE_TYPE_STR_DBG; + case SE_TYPE_ID_HSM: return SE_TYPE_STR_HSM; + } + + return "unknown"; +} + +static u32 get_se_soc_id(struct se_if_priv *priv) +{ + const struct se_if_node *if_node = device_get_match_data(priv->dev); + + return if_node->se_info->soc_id; +} + +static struct se_fw_load_info *get_load_fw_instance(struct se_if_priv *priv) +{ + return &priv->load_fw; +} + +static void se_soc_device_unregister(struct se_soc_dev_regn *soc_dev_regn) +{ + guard(mutex)(&var_se_info.se_var_info_lock); + + if (soc_dev_regn->soc_dev) { + soc_device_unregister(soc_dev_regn->soc_dev); + soc_dev_regn->soc_dev = NULL; + } + + if (soc_dev_regn->soc_dev_attr) { + /* + * revision and serial_number are the only kasprintf()-allocated + * strings. machine points into the DT, and soc_id/family are + * constants, so they must not be freed. + */ + kfree(soc_dev_regn->soc_dev_attr->revision); + kfree(soc_dev_regn->soc_dev_attr->serial_number); + kfree(soc_dev_regn->soc_dev_attr); + soc_dev_regn->soc_dev_attr = NULL; + } + + soc_dev_regn->soc_dev_registered = false; +} + +/* + * Build and register a soc_device entry for this SoC. Separated from + * get_se_soc_info() so that the firmware-fetch path and the sysfs + * registration path can be reasoned about independently. + */ +static int se_soc_dev_register(struct se_if_priv *priv, u16 soc_rev, + const char *soc_name, const u8 *uid) +{ + struct soc_device_attribute *attr; + struct soc_device *sdev; + int err; + + if (!soc_rev || !soc_name || !uid) + return -EINVAL; + + attr = kzalloc_obj(*attr); + if (!attr) + return -ENOMEM; + + if (FIELD_GET(DEV_GETINFO_MIN_VER_MASK, soc_rev)) + attr->revision = kasprintf(GFP_KERNEL, "%x.%x", + FIELD_GET(DEV_GETINFO_MAJ_VER_MASK, soc_rev), + FIELD_GET(DEV_GETINFO_MIN_VER_MASK, soc_rev)); + else + attr->revision = kasprintf(GFP_KERNEL, "%x", + FIELD_GET(DEV_GETINFO_MAJ_VER_MASK, soc_rev)); + + if (!attr->revision) { + err = -ENOMEM; + goto err_free_attr; + } + + attr->soc_id = soc_name; + + err = of_property_read_string(of_root, "model", &attr->machine); + if (err) { + err = -EINVAL; + goto err_free_rev; + } + + attr->family = "Freescale i.MX"; + + attr->serial_number = kasprintf(GFP_KERNEL, "%016llX", + GET_SERIAL_NUM_FROM_UID(uid, MAX_UID_SIZE >> 2)); + if (!attr->serial_number) { + err = -ENOMEM; + goto err_free_rev; + } + + sdev = soc_device_register(attr); + if (IS_ERR(sdev)) { + err = PTR_ERR(sdev); + goto err_free_serial; + } + + /* + * Publish the singleton. Freed once, at module unload, by + * se_soc_device_unregister(). Caller holds se_var_info_lock. + */ + var_se_info.soc_dev_regn.soc_dev = sdev; + var_se_info.soc_dev_regn.soc_dev_attr = attr; + + /* Mark registration complete so get_se_soc_info() skips this path on retry. */ + var_se_info.soc_dev_regn.soc_dev_registered = true; + + return 0; + +err_free_serial: + kfree(attr->serial_number); +err_free_rev: + kfree(attr->revision); +err_free_attr: + kfree(attr); + + return err; +} + +static int get_se_soc_info(struct se_if_priv *priv, const struct se_soc_info *se_info) +{ + struct se_fw_load_info *load_fw = get_load_fw_instance(priv); + u8 data[MAX_SOC_INFO_DATA_SZ]; + struct ele_dev_info *s_info; + int err; + + guard(mutex)(&var_se_info.se_var_info_lock); + + /* + * Early exit: both objectives already complete, nothing to do. + * Do not exit early when imem_mgmt is active: load_fw is per-probe + * (embedded in priv) and starts zeroed on every probe, so imem.state + * must be refreshed from firmware on each probe even when soc_rev is + * already cached in the module-lifetime var_se_info. + */ + if (var_se_info.soc_rev && + (!se_info->soc_name || var_se_info.soc_dev_regn.soc_dev_registered) && + !load_fw->imem_mgmt) + return 0; + + err = ele_fetch_soc_info(priv, &data); + if (err < 0) + return dev_err_probe(priv->dev, err, "Failed to fetch SoC Info.\n"); + + s_info = (struct ele_dev_info *)data; + + if (!var_se_info.soc_rev) + var_se_info.soc_rev = s_info->d_info.soc_rev; + + /* + * imem.state is per-probe state (lives in priv->load_fw which is + * zeroed on every probe). Update it unconditionally whenever the + * IMEM management path is active, regardless of whether soc_rev was + * already cached from a previous probe or a sibling interface. + */ + if (load_fw->imem_mgmt) + load_fw->imem.state = s_info->d_addn_info.imem_state; + + if (se_info->soc_name && !var_se_info.soc_dev_regn.soc_dev_registered) { + err = se_soc_dev_register(priv, var_se_info.soc_rev, + se_info->soc_name, s_info->d_info.uid); + if (err < 0) + return dev_err_probe(priv->dev, err, + "Failed to register SE SoC device.\n"); + } + + return 0; +} + +static int load_firmware(struct se_if_priv *priv, const u8 *se_img_file_to_load) +{ + const struct firmware *fw = NULL; + dma_addr_t se_fw_dma_addr; + u32 se_fw_buf_len; + void *se_fw_buf; + int ret; + + if (!se_img_file_to_load) { + dev_err(priv->dev, "FW image is not provided.\n"); + return -EINVAL; + } + ret = request_firmware(&fw, se_img_file_to_load, priv->dev); + if (ret) + return ret; + + if (fw->size > U32_MAX) { + ret = -EFBIG; + release_firmware(fw); + return ret; + } + dev_info(priv->dev, "loading firmware %s.\n", se_img_file_to_load); + + /* + * Serialize access to priv_dev_ctx shared memory to prevent pos + * corruption if two driver-internal callers run concurrently (e.g. + * ele_get_info() racing with load_firmware()). + */ + scoped_guard(mutex, &priv->priv_dev_ctx->fops_lock) { + se_fw_buf_len = fw->size; + ret = get_shared_mem_slot(priv->priv_dev_ctx, + &se_fw_buf_len, &se_fw_dma_addr, + &se_fw_buf); + if (ret) { + dev_err(priv->dev, "Failed to allocate firmware shared buffer: %d\n", + ret); + release_firmware(fw); + return ret; + } + + memcpy(se_fw_buf, fw->data, fw->size); + ret = ele_fw_authenticate(priv, se_fw_dma_addr, se_fw_dma_addr); + if (ret < 0) { + dev_err(priv->dev, + "Error %pe: Authenticate & load SE firmware %s.", + ERR_PTR(ret), se_img_file_to_load); + ret = -EPERM; + } + if (!se_is_fw_busy_ctx(priv->priv_dev_ctx)) + se_dev_ctx_shared_mem_cleanup(priv->priv_dev_ctx); + } + + release_firmware(fw); + + return ret; +} + +static int se_load_firmware(struct se_if_priv *priv) +{ + struct se_fw_load_info *load_fw = get_load_fw_instance(priv); + int ret = 0; + + guard(mutex)(&load_fw->load_fw_lock); + if (!load_fw->is_fw_tobe_loaded) + return 0; + + if (load_fw->imem.state == ELE_IMEM_STATE_BAD) { + ret = load_firmware(priv, load_fw->se_fw_img_nm->prim_fw_nm_in_rfs); + if (ret) { + dev_err(priv->dev, "Failed to load boot firmware.\n"); + return -EPERM; + } + } + + ret = load_firmware(priv, load_fw->se_fw_img_nm->seco_fw_nm_in_rfs); + if (ret) { + dev_err(priv->dev, "Failed to load runtime firmware.\n"); + return -EPERM; + } + + load_fw->is_fw_tobe_loaded = false; + + return ret; +} + +static int init_se_shared_mem(struct se_if_device_ctx *dev_ctx) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct se_if_priv *priv = dev_ctx->priv; + + INIT_LIST_HEAD(&se_shared_mem_mgmt->pending_out); + INIT_LIST_HEAD(&se_shared_mem_mgmt->pending_in); + + if (priv->mem_pool) + INIT_LIST_HEAD(&se_shared_mem_mgmt->mem_pool_buf_list); + + se_shared_mem_mgmt->non_secure_mem.ptr = + dma_alloc_coherent(priv->dev, MAX_DATA_SIZE_PER_USER, + &se_shared_mem_mgmt->non_secure_mem.dma_addr, + GFP_KERNEL); + if (!se_shared_mem_mgmt->non_secure_mem.ptr) + return -ENOMEM; + + se_shared_mem_mgmt->non_secure_mem.size = MAX_DATA_SIZE_PER_USER; + se_shared_mem_mgmt->non_secure_mem.pos = 0; + + return 0; +} + +static void cleanup_se_shared_mem(struct se_if_device_ctx *dev_ctx, bool reclaim) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct se_if_priv *priv = dev_ctx->priv; + bool free_dma_buf; + + /* + * mem_pool_buf_list is only initialised for interfaces that own a + * gen_pool (priv->mem_pool != NULL). On interfaces without a pool + * (e.g. imx93, which has no pool_name) the list head is left + * zero-filled, so se_cleanup_mem_pool_buf() must not walk it here or + * list_for_each_entry_safe() would dereference a NULL head and panic + * the kernel on close/teardown. Skip the pool cleanup entirely when + * there is no pool; there is nothing to reclaim in that case. + */ + if (priv->mem_pool) + se_cleanup_mem_pool_buf(dev_ctx, reclaim); + + /* Guard against being called before shared memory was ever allocated + * (e.g. probe failure before dma_alloc_coherent succeeded). + */ + if (!se_shared_mem_mgmt->non_secure_mem.ptr) + return; + + /* + * Decide whether the DMA buffer can be released before touching the + * pending lists. se_dev_ctx_shared_mem_cleanup() resets + * non_secure_mem.pos, so the "nothing staged" test must be sampled + * here first. When reclaim is false the buffer is released only if no + * data is still staged for the firmware; otherwise the enclave may + * still be DMA-ing into it and the buffer is deliberately leaked to + * avoid a DMA-after-free. + */ + free_dma_buf = reclaim || !se_shared_mem_mgmt->non_secure_mem.pos; + + /* + * Free any se_buf_desc items that were never consumed (e.g. when the + * fd is closed while pending I/O buffers are still listed). This must + * happen before the DMA backing memory is released to avoid a leak. + */ + se_dev_ctx_shared_mem_cleanup(dev_ctx); + + if (free_dma_buf) { + dma_free_coherent(priv->dev, MAX_DATA_SIZE_PER_USER, + se_shared_mem_mgmt->non_secure_mem.ptr, + se_shared_mem_mgmt->non_secure_mem.dma_addr); + } + + /* + * Drop the host-side tracking unconditionally. On the reclaim path the + * buffer has been freed. On the deliberate-leak path the buffer is + * abandoned on purpose, so clearing the pointer here guarantees a later + * cleanup pass (e.g. se_if_priv_release()) cannot double-free it. + */ + se_shared_mem_mgmt->non_secure_mem.ptr = NULL; + se_shared_mem_mgmt->non_secure_mem.dma_addr = 0; + se_shared_mem_mgmt->non_secure_mem.size = 0; + se_shared_mem_mgmt->non_secure_mem.pos = 0; +} + +static int se_dev_ctx_cpy_out_data(struct se_if_device_ctx *dev_ctx) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct se_if_priv *priv = dev_ctx->priv; + struct se_buf_desc *b_desc, *temp; + bool do_cpy = true; + + list_for_each_entry_safe(b_desc, temp, &se_shared_mem_mgmt->pending_out, link) { + if (b_desc->usr_buf_ptr && b_desc->shared_buf_ptr && do_cpy) { + dev_dbg(priv->dev, "Copying output data to user.\n"); + if (do_cpy && copy_to_user(b_desc->usr_buf_ptr, + b_desc->shared_buf_ptr, + b_desc->size)) { + dev_err(priv->dev, "Failure copying output data to user.\n"); + do_cpy = false; + } + } + + if (b_desc->shared_buf_ptr) + memset(b_desc->shared_buf_ptr, 0, b_desc->size); + + list_del(&b_desc->link); + kfree(b_desc); + } + + return do_cpy ? 0 : -EFAULT; +} + +/* + * Clean the used Shared Memory space, + * whether its Input Data copied from user buffers, or + * Data received from FW. + */ +void se_dev_ctx_shared_mem_cleanup(struct se_if_device_ctx *dev_ctx) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct list_head *pending_lists[] = {&se_shared_mem_mgmt->pending_in, + &se_shared_mem_mgmt->pending_out}; + struct se_buf_desc *b_desc, *temp; + bool is_fw_busy_dev_ctx; + int i; + + /* + * If this context is the one that caused a firmware timeout the shared + * DMA buffers may still be actively read/written by the firmware. + */ + is_fw_busy_dev_ctx = se_is_fw_busy_ctx(dev_ctx); + + for (i = 0; i < ARRAY_SIZE(pending_lists); i++) { + list_for_each_entry_safe(b_desc, temp, pending_lists[i], link) { + if (!is_fw_busy_dev_ctx && b_desc->shared_buf_ptr) + memset(b_desc->shared_buf_ptr, 0, b_desc->size); + + list_del(&b_desc->link); + kfree(b_desc); + } + } + + /* + * Keep non_secure_mem.pos non-zero while this context still owns an + * outstanding firmware transaction. A non-zero pos is the marker that + * data is still staged for the enclave, which cleanup_se_shared_mem() + * uses to decide the buffer must be leaked rather than freed. Resetting + * it here would let a later teardown pass free a buffer the enclave may + * still be DMA-ing into. + */ + if (!is_fw_busy_dev_ctx) + se_shared_mem_mgmt->non_secure_mem.pos = 0; +} + +static struct se_buf_desc *add_b_desc_to_pending_list(void *shared_ptr_with_pos, + struct se_ioctl_setup_iobuf *io, + struct se_if_device_ctx *dev_ctx) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct se_buf_desc *b_desc = NULL; + + b_desc = kzalloc_obj(*b_desc); + if (!b_desc) + return ERR_PTR(-ENOMEM); + + b_desc->shared_buf_ptr = shared_ptr_with_pos; + b_desc->usr_buf_ptr = u64_to_user_ptr(io->user_buf); + b_desc->size = io->length; + + if (io->flags & SE_IO_BUF_FLAGS_IS_INPUT) { + /* + * buffer is input: + * add an entry in the "pending input buffers" list so + * that copied data can be cleaned from shared memory + * later. + */ + list_add_tail(&b_desc->link, &se_shared_mem_mgmt->pending_in); + } else { + /* + * buffer is output: + * add an entry in the "pending out buffers" list so data + * can be copied to user space when receiving Secure-Enclave + * response. + */ + list_add_tail(&b_desc->link, &se_shared_mem_mgmt->pending_out); + } + + return b_desc; +} + +static void se_if_open_gate_release(struct kref *kref) +{ + struct se_if_open_gate *gate = + container_of(kref, struct se_if_open_gate, refcount); + + kfree(gate); +} + +static bool se_if_open_gate_get(struct se_if_open_gate *gate) +{ + if (!gate) + return false; + + return kref_get_unless_zero(&gate->refcount); +} + +static void se_if_open_gate_put(struct se_if_open_gate *gate) +{ + if (gate) + kref_put(&gate->refcount, se_if_open_gate_release); +} + +/* + * Distinct lockdep class for the internal priv_dev_ctx fops_lock. Taking it + * while an open context's fops_lock is held (for example a firmware load + * triggered from an ioctl) is valid hierarchical locking, but shares the same + * class as the per-open fops_lock and would otherwise be misreported as + * recursive locking by lockdep. + */ +static struct lock_class_key se_priv_ctx_fops_key; + +static int init_misc_device_context(struct se_if_priv *priv, int ch_id, + struct se_if_device_ctx **new_dev_ctx, + const struct file_operations *se_if_fops) +{ + struct se_if_open_gate *gate = NULL; + struct se_if_device_ctx *dev_ctx; + int ret = -ENOMEM; + + dev_ctx = kzalloc_obj(*dev_ctx); + if (!dev_ctx) + return -ENOMEM; + + dev_ctx->priv = priv; + dev_ctx->devname = kasprintf(GFP_KERNEL, "%s0_ch%d", + get_se_if_name(priv->if_defs->se_if_type), + ch_id); + if (!dev_ctx->devname) { + kfree(dev_ctx); + return -ENOMEM; + } + + mutex_init(&dev_ctx->fops_lock); + lockdep_set_class(&dev_ctx->fops_lock, &se_priv_ctx_fops_key); + + kref_init(&dev_ctx->refcount); + dev_ctx->cleanup_done = false; + *new_dev_ctx = dev_ctx; + set_se_rcv_msg_timeout(dev_ctx, SE_RCV_MSG_DEFAULT_TIMEOUT_MS); + + ret = init_se_shared_mem(dev_ctx); + if (ret < 0) + goto exit; + + gate = kzalloc_obj(*gate); + if (!gate) { + ret = -ENOMEM; + goto exit; + } + + mutex_init(&gate->lock); + kref_init(&gate->refcount); /* device-owned reference */ + gate->priv = priv; + gate->dying = false; + priv->open_gate = gate; + + /* + * The miscdevice storage is now owned by the open gate object. + * priv->priv_dev_ctx still keeps a pointer to that miscdevice. + */ + dev_ctx->miscdev = &gate->miscdev; + + dev_ctx->miscdev->name = dev_ctx->devname; + dev_ctx->miscdev->minor = MISC_DYNAMIC_MINOR; + dev_ctx->miscdev->fops = se_if_fops; + dev_ctx->miscdev->parent = priv->dev; + + return 0; +exit: + *new_dev_ctx = NULL; + + if (gate) { + priv->open_gate = NULL; + se_if_open_gate_put(gate); + } + cleanup_se_shared_mem(dev_ctx, true); + kfree(dev_ctx->devname); + kfree(dev_ctx); + return ret; +} + +static int se_if_request_channel(struct device *dev, struct mbox_chan **chan, + struct mbox_client *cl, const char *name) +{ + struct mbox_chan *t_chan; + + t_chan = mbox_request_channel_byname(cl, name); + if (IS_ERR(t_chan)) + return dev_err_probe(dev, PTR_ERR(t_chan), + "Failed to request %s channel.\n", name); + + *chan = t_chan; + + return 0; +} + +/* + * Forward declarations. se_if_probe_cleanup() and se_if_probe() are kept + * together as the teardown/probe pair, but several helpers, the file + * operations table and the firmware-busy work handler they reference are + * defined further down in this file. + */ +static void dlink_dev_ctx(struct se_if_device_ctx *dev_ctx); +static void cleanup_dev_ctx(struct se_if_device_ctx *dev_ctx, bool is_fclose); +static void se_clear_fw_busy(struct se_if_priv *priv); +static void se_if_dev_ctx_release(struct kref *kref); +static void se_if_priv_release(struct kref *kref); +static int se_if_misc_register(struct se_if_priv *priv); +static void se_fw_busy_work(struct work_struct *work); +static const struct file_operations se_if_fops; + +static void se_if_probe_cleanup(void *plat_dev) +{ + struct platform_device *pdev = plat_dev; + struct se_if_device_ctx *dev_ctx; + struct device *dev = &pdev->dev; + struct fw_busy_info *fbusy_info; + struct se_if_priv *priv; + + priv = dev_get_drvdata(dev); + if (!priv) + return; + + fbusy_info = &priv->fw_busy_info; + + /* + * Announce teardown, then wake any in-flight waiter. going_away makes + * ele_msg_send_rcv() bail out instead of arming a new transaction and + * lets ele_msg_rcv() tell a teardown-forced completion apart from a + * real response; it must be set before complete_all(). + * + * Set it under clbk_rx_lock, not se_if_cmd_lock: se_if_cmd_lock is held + * across the whole blocking transaction, so taking it here would stall + * unbind for a full receive-timeout. clbk_rx_lock is the short spinlock + * ele_msg_send_rcv() holds while arming, so this closes the lost-wakeup + * window - the sender either sees going_away and bails before arming, or + * armed first and this store (and complete_all()) is ordered after its + * reinit_completion() - and supplies the ordering the relaxed atomics do + * not. + */ + scoped_guard(spinlock_irqsave, &priv->waiting_rsp_clbk_hdl.clbk_rx_lock) + atomic_set(&priv->going_away, 1); + /* + * Wake the waiter before iterating the device-context list. It sleeps on + * this completion holding dev_ctx->fops_lock, which cleanup_dev_ctx() + * below also takes, so completing first avoids an unbind hang. Runs + * outside clbk_rx_lock; the going_away store above already orders it + * against the arming path. + */ + complete_all(&priv->waiting_rsp_clbk_hdl.done); + + /* + * Only now reserve the messaging interface for this teardown flow. + * + * se_reserve_msg_if() blocks on msg_excl_flow_lock, and the fw_busy + * recovery worker (se_fw_busy_work() -> se_clear_fw_busy()) may already + * hold that reservation while parked uninterruptibly in ele_msg_rcv() + * waiting on a possibly hung firmware for up to the full receive + * timeout. The only thing that cuts that wait short is the complete_all() + * above, so it MUST run before this reserve: otherwise teardown would + * sleep on the reservation the worker holds, the worker would stay + * blocked on firmware, and unbind would stall for the entire multi- + * thousand-second timeout (an unbind hang / hung-task). + * + * With going_away already set and the in-flight waiter already forced to + * unwind, the worker returns promptly (its send is failed with -ENODEV), + * drops the reservation, and this call acquires it without waiting on + * anything firmware-related. From here on teardown is the exclusive + * owner: ele_msg_send_rcv() lets only this task's priv_dev_ctx close + * traffic through and rejects every other caller. + */ + se_reserve_msg_if(priv); + + /* + * Mark the private device context as cleanup_done first. + * This prevents new device contexts from being created in open(). + */ + if (priv->priv_dev_ctx) { + /* + * Mark cleanup_done under fops_lock so that se_if_fops_open(), + * which checks cleanup_done while holding fops_lock, cannot + * race past this and add a new device context after teardown. + */ + scoped_guard(mutex, &priv->priv_dev_ctx->fops_lock) + priv->priv_dev_ctx->cleanup_done = true; + + if (priv->open_gate) { + scoped_guard(mutex, &priv->open_gate->lock) { + priv->open_gate->dying = true; + priv->open_gate->priv = NULL; + } + } + + /* + * misc_register() is deferred to the end of probe, so the + * device may have a miscdev set up but never registered if + * probe failed before se_if_misc_register(). Only deregister + * when registration actually succeeded. + */ + if (priv->open_gate && priv->open_gate->registered && + priv->priv_dev_ctx->miscdev) + misc_deregister(priv->priv_dev_ctx->miscdev); + } + + while (true) { + bool list_was_empty = false; + + dev_ctx = NULL; + + scoped_guard(mutex, &priv->modify_lock) { + if (list_empty(&priv->dev_ctx_list)) { + list_was_empty = true; + } else { + dev_ctx = list_first_entry(&priv->dev_ctx_list, + struct se_if_device_ctx, link); + + /* pin this context so close() cannot free it under us */ + kref_get(&dev_ctx->refcount); + dlink_dev_ctx(dev_ctx); + } + } + + if (list_was_empty) + break; + + /* + * Local cleanup outside the global lock avoids ABBA deadlock + * with paths that already take dev_ctx->fops_lock first. + */ + cleanup_dev_ctx(dev_ctx, false); + kref_put(&dev_ctx->refcount, se_if_dev_ctx_release); + } + + se_release_msg_if(priv); + /* + * Release any dev_ctx retained by the firmware-busy circuit breaker. + * A synchronous command that timed out parks its dev_ctx in + * fbusy_info->fw_busy_dev_ctx so that a late firmware response can still be + * routed back. If no such response arrived before teardown, that + * retained reference must be dropped here to avoid a leak. + * se_clear_fw_busy() is safe to call unconditionally: it checks + * fbusy_info->fw_busy_dev_ctx under fw_busy_lock and is a no-op when + * nothing is parked. + */ + se_clear_fw_busy(priv); + + /* + * Free the mailbox channels under se_if_cmd_lock. ele_msg_send_rcv() + * holds se_if_cmd_lock for the full duration of a synchronous + * transaction, including the mbox_send_message() call on priv->tx_chan. + * going_away was set above and complete_all() has already woken any + * in-flight waiter, so any transaction in progress will unwind to + * -ENODEV and release the lock quickly. Acquiring se_if_cmd_lock here + * guarantees no caller is still touching the channels when they are + * freed, and nulling the pointers under the lock prevents any sender + * that races past the going_away check from accessing a freed channel. + */ + scoped_guard(mutex, &priv->se_if_cmd_lock) { + if (priv->rx_chan) { + mbox_free_channel(priv->rx_chan); + priv->rx_chan = NULL; + } + if (priv->tx_chan) { + mbox_free_channel(priv->tx_chan); + priv->tx_chan = NULL; + } + } + + /* + * Cancel any pending fw_busy_work before dropping the initial priv + * reference. going_away was set above, so no new work can be scheduled + * after this point. Canceling here while probe_cleanup still holds its + * own priv reference prevents two races: + * + * 1. UAF: if fw_busy_work has dev_ctx == priv_dev_ctx, letting it run + * past this point while se_if_priv_release() frees priv_dev_ctx + * causes a use-after-free of dev_ctx->fops_lock in se_clear_fw_busy(). + * + * 2. Deadlock: if fw_busy_work drops the last priv reference, + * se_if_dev_ctx_release() -> se_if_priv_release() would call + * cancel_work_sync() from inside the worker, causing the worker to + * wait for its own completion. + * + * Both are avoided by canceling here: probe_cleanup still holds a priv + * reference so the worker cannot invoke se_if_priv_release(), and the + * cancel runs from a non-worker context. + */ + cancel_work_sync(&fbusy_info->fw_busy_work); + + /* + * Reclaim priv_dev_ctx shared memory before of_reserved_mem_device_release(): + * cleanup_se_shared_mem() calls dma_free_coherent(), which must run while + * the DMA config is still active. fw_busy_work was canceled above, so no + * concurrent caller holds priv_dev_ctx->fops_lock. + * + * reclaim=true is safe even if FW hung at teardown (command timed out, pos + * still non-zero): the ELE region is no-map/shared-dma-pool, so freeing only + * drops the kernel VA/bitmap while the physical pages stay reserved (no + * DMA-after-free). The next probe also sends ELE_GET_INFO into a fresh + * buffer before accepting commands, so a stale FW write to the old buffer is + * never observed by the new driver instance. + */ + if (priv->priv_dev_ctx) { + scoped_guard(mutex, &priv->priv_dev_ctx->fops_lock) + cleanup_se_shared_mem(priv->priv_dev_ctx, true); + } + + /* + * Release the reserved DMA memory configuration at unbind time, paired + * with of_reserved_mem_device_init() in se_if_probe(). This must not be + * deferred to se_if_priv_release(): that runs when the last file + * descriptor closes, which may be after a new driver instance has already + * called of_reserved_mem_device_init() on the same struct device. Calling + * the release at that point would corrupt the new instance's DMA setup. + */ + of_reserved_mem_device_release(dev); + + /* + * Being device managed buffer, no need to free the buffer allocated + * in se probe to store encrypted IMEM. + */ + + dev_set_drvdata(dev, NULL); + + /* Drop the initial reference - priv will be freed when last fd closes */ + kref_put(&priv->refcount, se_if_priv_release); +} + +static int se_if_probe(struct platform_device *pdev) +{ + const struct se_soc_info *se_info; + const struct se_if_node *if_node; + struct device *dev = &pdev->dev; + struct se_fw_load_info *load_fw; + struct se_if_priv *priv; + int ret; + + if_node = device_get_match_data(dev); + if (!if_node) + return -EINVAL; + + se_info = if_node->se_info; + + priv = kzalloc_obj(*priv); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + /* + * Pin the parent device for the lifetime of priv. A file descriptor may + * stay open after the device is unbound; close() then still passes + * priv->dev to dma_free_coherent()/dev_warn(). Without this reference + * the struct device could be freed while priv->dev still points at it, + * so the reference is dropped in se_if_priv_release() via put_device(). + */ + get_device(priv->dev); + kref_init(&priv->refcount); + priv->if_defs = &if_node->if_defs; + dev_set_drvdata(dev, priv); + + spin_lock_init(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock); + spin_lock_init(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock); + priv->msg_excl_flow.msg_excl_owner = NULL; + spin_lock_init(&priv->msg_excl_flow.msg_excl_lock); + mutex_init(&priv->msg_excl_flow.msg_excl_flow_lock); + struct fw_busy_info *fbusy_info = &priv->fw_busy_info; + + atomic_set(&fbusy_info->fw_busy, 0); + + spin_lock_init(&fbusy_info->fw_busy_lock); + fbusy_info->fw_busy_dev_ctx = NULL; + INIT_WORK(&fbusy_info->fw_busy_work, se_fw_busy_work); + + init_completion(&priv->waiting_rsp_clbk_hdl.done); + init_completion(&priv->cmd_receiver_clbk_hdl.done); + INIT_LIST_HEAD(&priv->dev_ctx_list); + + mutex_init(&priv->se_if_cmd_lock); + mutex_init(&priv->modify_lock); + + load_fw = get_load_fw_instance(priv); + mutex_init(&load_fw->load_fw_lock); + if (se_info->se_fw_img_nm.seco_fw_nm_in_rfs) { + load_fw->se_fw_img_nm = &se_info->se_fw_img_nm; + load_fw->is_fw_tobe_loaded = true; + } + ret = devm_add_action_or_reset(dev, se_if_probe_cleanup, pdev); + if (ret) + return ret; + + /* Mailbox client configuration */ + priv->se_mb_cl.dev = dev; + priv->se_mb_cl.tx_block = false; + priv->se_mb_cl.knows_txdone = false; + priv->se_mb_cl.rx_callback = se_if_rx_callback; + + ret = se_if_request_channel(dev, &priv->tx_chan, &priv->se_mb_cl, "tx"); + if (ret) + return ret; + + ret = se_if_request_channel(dev, &priv->rx_chan, &priv->se_mb_cl, "rx"); + if (ret) + return ret; + + if (if_node->pool_name) { + priv->mem_pool = of_gen_pool_get(dev->of_node, if_node->pool_name, 0); + if (!priv->mem_pool) + return dev_err_probe(dev, -ENOMEM, + "Unable to get sram pool = %s.\n", + if_node->pool_name); + } + + if (if_node->reserved_dma_ranges) { + ret = of_reserved_mem_device_init(dev); + if (ret) + return dev_err_probe(dev, ret, + "Failed to init reserved memory region.\n"); + } + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + + /* By default, there is no pending FW to be loaded.*/ + if (se_info->imem_state_mgmt) { + /* allocate buffer where SE store encrypted IMEM */ + load_fw->imem.buf = dmam_alloc_coherent(priv->dev, ELE_IMEM_SIZE, + &load_fw->imem.daddr, + GFP_KERNEL); + if (!load_fw->imem.buf) + return dev_err_probe(dev, -ENOMEM, + "dmam-alloc-failed: To store encr-IMEM.\n"); + load_fw->imem_mgmt = true; + } + + ret = init_misc_device_context(priv, 0, &priv->priv_dev_ctx, &se_if_fops); + if (ret) + return dev_err_probe(dev, ret, + "Failed to create device contexts.\n"); + + if (if_node->if_defs.se_if_type == SE_TYPE_ID_HSM) { + ret = get_se_soc_info(priv, se_info); + if (ret) + return dev_err_probe(dev, ret, "Failed to fetch SoC Info.\n"); + } + + /* + * All probe-time initialization is complete; expose the + * interface to userspace last so that an open()/ioctl cannot + * race against a not-yet-initialized device. + */ + ret = se_if_misc_register(priv); + if (ret) + return ret; + + dev_info(dev, "i.MX secure-enclave: %s0 interface to firmware, configured.\n", + get_se_if_name(priv->if_defs->se_if_type)); + + return ret; +} + +/* + * Expose the interface to userspace. Deferred until the end of probe so + * the device node only becomes openable after SoC info has been fetched + * and, on SoCs with IMEM management, the encrypted-IMEM buffer has been + * allocated. This prevents userspace from opening the node and issuing + * commands against a partially initialized interface. + */ +static int se_if_misc_register(struct se_if_priv *priv) +{ + int ret; + + ret = misc_register(priv->priv_dev_ctx->miscdev); + if (ret) + return dev_err_probe(priv->dev, ret, + "Failed to register misc device."); + + priv->open_gate->registered = true; + + return 0; +} + +static void se_if_priv_release(struct kref *kref) +{ + struct se_if_priv *priv = container_of(kref, struct se_if_priv, refcount); + + /* + * Free priv_dev_ctx if it still exists. se_if_priv_release() always + * runs after se_if_probe_cleanup() has completed: the initial kref + * held by probe_cleanup is the last one dropped by probe_cleanup + * itself, so no other kref_put() can reach zero -- and therefore + * trigger se_if_priv_release() -- until probe_cleanup's own + * kref_put() fires. By that time cleanup_se_shared_mem() and + * of_reserved_mem_device_release() have already run in + * probe_cleanup, so only the struct itself and its devname string + * need to be freed here. Calling cleanup_se_shared_mem() again + * would be a use-after-free of already-freed DMA memory. + */ + if (priv->priv_dev_ctx) { + kfree(priv->priv_dev_ctx->devname); + kfree(priv->priv_dev_ctx); + priv->priv_dev_ctx = NULL; + } + /* + * Be defensive: if teardown did not already drop the device-owned + * gate reference for some reason, release it here. + */ + if (priv->open_gate) { + se_if_open_gate_put(priv->open_gate); + priv->open_gate = NULL; + } + + /* + * Drop the reference on priv->dev taken in se_if_probe(). The device was + * pinned so that a file descriptor closed after device unbind can still + * safely pass priv->dev to dma_free_coherent()/dev_warn(). + */ + put_device(priv->dev); + mutex_destroy(&priv->load_fw.load_fw_lock); + mutex_destroy(&priv->modify_lock); + mutex_destroy(&priv->se_if_cmd_lock); + mutex_destroy(&priv->msg_excl_flow.msg_excl_flow_lock); + + /* Free any remaining resources that weren't devm-managed */ + kfree(priv); +} + +static void se_if_dev_ctx_release(struct kref *kref) +{ + struct se_if_device_ctx *dev_ctx = + container_of(kref, struct se_if_device_ctx, refcount); + struct se_if_priv *priv = dev_ctx->priv; + + kfree(dev_ctx); + + /* drop the priv reference owned by this device context */ + kref_put(&priv->refcount, se_if_priv_release); +} + +/* + * se_reserve_msg_if() - reserve the SE messaging interface for the current task. + * + * Blocks on msg_excl_flow_lock until this task owns the reservation, then + * publishes current as msg_excl_owner under msg_excl_lock. While a reservation + * is held, ele_msg_send_rcv() lets only the owning task issue transactions and + * rejects every other caller with -EBUSY. Used by the fw_busy recovery flow in + * se_clear_fw_busy() to drive its teardown-close messages through the otherwise + * closed circuit breaker. + * + * If a second flow tries to reserve while the interface is already reserved, + * it sleeps on msg_excl_flow_lock until the current owner calls + * se_release_msg_if(). Must be called from process/workqueue context (it may + * sleep) and every successful call must be balanced by se_release_msg_if(). + * + * Return: 0 (the reservation is always acquired once this returns). + */ +int se_reserve_msg_if(struct se_if_priv *priv) +{ + unsigned long flags; + + mutex_lock(&priv->msg_excl_flow.msg_excl_flow_lock); + /* + * The mutex guarantees this task is now the sole reserver, so + * msg_excl_owner is either NULL or already current. Publish current + * under msg_excl_lock so the lockless READ_ONCE in ele_msg_send_rcv() + * observes a consistent pointer. + */ + spin_lock_irqsave(&priv->msg_excl_flow.msg_excl_lock, flags); + priv->msg_excl_flow.msg_excl_owner = current; + spin_unlock_irqrestore(&priv->msg_excl_flow.msg_excl_lock, flags); + + return 0; +} + +/* se_release_msg_if() - release a reservation taken by se_reserve_msg_if(). */ +void se_release_msg_if(struct se_if_priv *priv) +{ + unsigned long flags; + + spin_lock_irqsave(&priv->msg_excl_flow.msg_excl_lock, flags); + priv->msg_excl_flow.msg_excl_owner = NULL; + spin_unlock_irqrestore(&priv->msg_excl_flow.msg_excl_lock, flags); + mutex_unlock(&priv->msg_excl_flow.msg_excl_flow_lock); +} + +/* se_clear_fw_busy() - atomically clear fw_busy and reclaim the parked dev_ctx. */ +static void se_clear_fw_busy(struct se_if_priv *priv) +{ + struct fw_busy_info *fbusy_info = &priv->fw_busy_info; + struct se_if_device_ctx *dev_ctx = NULL; + unsigned long flags; + + scoped_guard(spinlock_irqsave, &fbusy_info->fw_busy_lock) { + dev_ctx = fbusy_info->fw_busy_dev_ctx; + fbusy_info->fw_busy_dev_ctx = NULL; + + if (!dev_ctx) { + /* + * No parked context: nothing to recover. Clear fw_busy + * and return without reserving the interface, so the + * no-op path never leaves a dangling recovery + * reservation. The scoped_guard releases fw_busy_lock + * on this return. + */ + atomic_set(&fbusy_info->fw_busy, 0); + return; + } + } + + /* + * A context is parked and its handles must be recovered. Keep + * fw_busy set (breaker stays closed to all third parties) and + * reserve the SE interface exclusively for this recovery flow by + * publishing the current task as msg_excl_owner. ele_msg_send_rcv() + * then lets only this task's teardown-close messages through and + * rejects everyone else with -EBUSY. The reservation is assigned here, + * from outside ele_msg_send_rcv(), and released with se_release_msg_if() + * at the end, after which the interface is available for general + * se_if_cmd_lock message exchange. + * + * The scoped_guard above has already dropped fw_busy_lock before this + * se_reserve_msg_if() call: the reserve helper takes msg_excl_lock, and + * taking it while still holding fw_busy_lock would introduce a new + * fw_busy_lock -> msg_excl_lock nesting. The brief fw_busy == 1 / + * owner == NULL window that this opens is harmless - the breaker is + * fully closed, so every caller (including a would-be re-arm) is + * rejected with -EBUSY. + */ + se_reserve_msg_if(priv); + + scoped_guard(mutex, &dev_ctx->fops_lock) { + /* + * Snapshot any orphaned late FW response. On the teardown + * path se_if_probe_cleanup calls se_clear_fw_busy before + * cancel_work_sync, so fw_busy is still 1 here and a late + * IRQ can write orphan_fw_rx_msg concurrently - take + * clbk_rx_lock. On the workqueue path the IRQ writer has + * already finished; the lock is a no-contention formality. + * Call fw_api_specific_ops() outside the spinlock since it + * may sleep. + */ + u8 late_rx_snap[MAX_ALLOWED_RX_MSG_SZ]; + bool have_snap; + + spin_lock_irqsave(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags); + have_snap = fbusy_info->orphan_fw_rx_msg[0] != 0; + if (have_snap) { + memcpy(late_rx_snap, fbusy_info->orphan_fw_rx_msg, + sizeof(late_rx_snap)); + memset(fbusy_info->orphan_fw_rx_msg, 0, + sizeof(fbusy_info->orphan_fw_rx_msg)); + } + spin_unlock_irqrestore(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags); + + if (have_snap) { + /* + * FW responded late. DMA staging buffer is no longer + * being written - safe to reclaim. Close any firmware + * resource handle carried in the response. + */ + fw_api_specific_ops(priv->priv_dev_ctx, + (struct se_api_msg *)late_rx_snap, true); + + if (dev_ctx == priv->priv_dev_ctx) { + /* + * Internal context: probe-time static DMA buf; + * se_if_probe_cleanup reclaims it explicitly. + * Just reset logical pos and return gen_pool + * loan buffers for reuse. + */ + if (priv->mem_pool) + se_cleanup_mem_pool_buf(dev_ctx, true); + se_dev_ctx_shared_mem_cleanup(dev_ctx); + } else if (dev_ctx->cleanup_done) { + /* + * Userspace fd already closed while fw_busy was + * armed (e.g. SIGKILL). FW has now responded; + * close deferred handles and free the DMA buf. + * + * cleanup_dev_ctx() already freed dev_ctx->devname + * and set it to NULL, so use the stable snapshot + * captured at arm time (fbusy_info->devname) for + * these diagnostics rather than dev_ctx->devname. + */ + if (dev_ctx->strg_hdl && + se_close_storage(priv->priv_dev_ctx, + dev_ctx->strg_hdl)) + dev_err(priv->dev, + "%s: failed to close deferred storage handle\n", + fbusy_info->devname); + if (dev_ctx->sess_hdl && + se_close_session(priv->priv_dev_ctx, + dev_ctx->sess_hdl)) + dev_err(priv->dev, + "%s: failed to close deferred session handle\n", + fbusy_info->devname); + dev_ctx->strg_hdl = 0; + dev_ctx->sess_hdl = 0; + cleanup_se_shared_mem(dev_ctx, true); + } else { + /* Pure timeout, fd still open: reset pos only. */ + se_dev_ctx_shared_mem_cleanup(dev_ctx); + } + } else { + /* + * have_snap=false only on teardown (FW never responded, + * or teardown beat the late IRQ, which going_away then + * drops). priv_dev_ctx is handled by probe_cleanup's + * single cleanup_se_shared_mem(reclaim=true) after + * cancel_work_sync, so nothing to do here. For a + * userspace dev_ctx (cleanup_done already true), use + * reclaim=false: the pos gate leaks the buffer if FW may + * still be writing, else frees it. reclaim=true would + * also be safe here since the region is no-map. + */ + if (dev_ctx != priv->priv_dev_ctx && dev_ctx->cleanup_done) + cleanup_se_shared_mem(dev_ctx, false); + } + } + + /* + * Recovery flow is done: release the exclusive reservation, then clear + * the breaker. se_release_msg_if() drops msg_excl_owner; the fw_busy + * clear below reopens the interface. Ordering is safe either way: while + * fw_busy is still 1 a third party is rejected regardless of owner, and + * once fw_busy is 0 the owner is no longer consulted. + */ + se_release_msg_if(priv); + + spin_lock_irqsave(&fbusy_info->fw_busy_lock, flags); + atomic_set(&fbusy_info->fw_busy, 0); + spin_unlock_irqrestore(&fbusy_info->fw_busy_lock, flags); + kref_put(&dev_ctx->refcount, se_if_dev_ctx_release); +} + +void unset_dev_ctx_as_command_receiver(struct se_if_device_ctx *dev_ctx) +{ + struct se_if_priv *priv = dev_ctx->priv; + struct se_api_msg *old_rx_msg = NULL; + struct se_clbk_handle *se_clbk_hdl; + unsigned long flags; + + lockdep_assert_held(&priv->modify_lock); + + se_clbk_hdl = &priv->cmd_receiver_clbk_hdl; + + if (se_clbk_hdl->dev_ctx == dev_ctx) { + spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags); + old_rx_msg = se_clbk_hdl->rx_msg; + se_clbk_hdl->dev_ctx = NULL; + se_clbk_hdl->rx_msg = NULL; + se_clbk_hdl->rx_msg_sz = 0; + spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags); + + kfree(old_rx_msg); + complete_all(&se_clbk_hdl->done); + } +} + +/* + * check_cmd_rcvr_status() - check whether dev_ctx can become the command + * receiver or is already become the command receiver. + * + * Returns: + * 0 - dev_ctx is already the registered receiver + * -EBUSY - another context is already the receiver + * -EINVAL - dev_ctx has no storage handle + * -ENXIO - ready to proceed: no receiver set, strg_hdl present + * + * Caller must hold priv->modify_lock. + */ +static int check_cmd_rcvr_status(struct se_if_device_ctx *dev_ctx) +{ + struct se_if_priv *priv = dev_ctx->priv; + struct se_clbk_handle *se_clbk_hdl = &priv->cmd_receiver_clbk_hdl; + + lockdep_assert_held(&priv->modify_lock); + + if (se_clbk_hdl->dev_ctx == dev_ctx) + return 0; + + if (se_clbk_hdl->dev_ctx) + return -EBUSY; + + if (!dev_ctx->strg_hdl) + return -EINVAL; + + /* Reaching here means, with a valid storage handle and command-receiver as NULL, + * either the registration process is to be done or failed. + */ + return -ENXIO; +} + +int set_dev_ctx_as_command_receiver(struct se_if_device_ctx *dev_ctx) +{ + struct se_if_priv *priv = dev_ctx->priv; + struct se_clbk_handle *se_clbk_hdl = &priv->cmd_receiver_clbk_hdl; + struct se_api_msg *new_rx_msg = NULL; + unsigned long flags; + int ret; + + guard(mutex)(&priv->modify_lock); + + /* + * All state checks happen inside modify_lock so the result cannot + * go stale between the check and the arming below. + */ + ret = check_cmd_rcvr_status(dev_ctx); + if (ret != -ENXIO) + return ret; + + if (!se_clbk_hdl->rx_msg) { + new_rx_msg = kzalloc(MAX_NVM_MSG_LEN, GFP_KERNEL); + if (!new_rx_msg) + return -ENOMEM; + } + spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags); + if (new_rx_msg) + se_clbk_hdl->rx_msg = new_rx_msg; + reinit_completion(&se_clbk_hdl->done); + se_clbk_hdl->rx_msg_sz = MAX_NVM_MSG_LEN; + se_clbk_hdl->dev_ctx = dev_ctx; + dev_ctx->rcv_msg_timeout_jiffies = MAX_SCHEDULE_TIMEOUT; + spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags); + + return 0; +} + +static void dlink_dev_ctx(struct se_if_device_ctx *dev_ctx) +{ + struct se_if_priv *priv = dev_ctx->priv; + + unset_dev_ctx_as_command_receiver(dev_ctx); + + if (!list_empty(&dev_ctx->link)) { + list_del_init(&dev_ctx->link); + priv->active_devctx_count--; + } +} + +bool se_is_fw_busy_ctx(struct se_if_device_ctx *dev_ctx) +{ + struct se_if_priv *priv = dev_ctx->priv; + struct fw_busy_info *fbusy_info = &priv->fw_busy_info; + unsigned long flags; + bool match; + + spin_lock_irqsave(&fbusy_info->fw_busy_lock, flags); + match = fbusy_info->fw_busy_dev_ctx == dev_ctx; + spin_unlock_irqrestore(&fbusy_info->fw_busy_lock, flags); + + return match; +} + +static void cleanup_dev_ctx(struct se_if_device_ctx *dev_ctx, bool is_fclose) +{ + struct fw_busy_info *fbusy_info = &dev_ctx->priv->fw_busy_info; + bool already_done; + + scoped_guard(mutex, &dev_ctx->fops_lock) { + already_done = dev_ctx->cleanup_done; + if (!already_done) { + /* + * Ask FW to drop this context's session and storage so + * the kernel and FW stay in sync. Done here, under this + * context's fops_lock only (not the global modify_lock), + * because both close requests block on a firmware + * round-trip; issuing them while modify_lock was held + * would stall every other context for the FW timeout. + * + * Skip the round-trips once the FW path is marked busy. + * fw_busy is armed when a synchronous transaction times + * out; while it is set ele_msg_send_rcv() rejects further + * commands with -EBUSY without waiting. It is only cleared + * by se_clear_fw_busy(), which during unbind runs once + * after this loop (or earlier from fw_busy_work only if a + * genuine late FW response arrives). On a hung FW no late + * response comes, so the breaker stays set for the rest of + * the loop and the remaining closes would just return + * -EBUSY and log spurious "failed to close" errors. Skip + * them and emit a single warning instead. + */ + if (atomic_read(&fbusy_info->fw_busy)) { + if (dev_ctx->strg_hdl || dev_ctx->sess_hdl) + dev_warn(dev_ctx->priv->dev, + "%s: skipping session/storage close, FW is busy\n", + dev_ctx->devname); + } else { + /* + * Choose which dev_ctx sends the close messages. + * fclose: use the caller's own dev_ctx so a race with + * unbind is rejected with -ENODEV instead of hitting a + * freed tx_chan. Teardown: use priv_dev_ctx; going_away + * is set but the reservation (msg_excl_owner == current) + * lets these closes through while tx_chan is still live. + */ + struct se_if_device_ctx *tx_ctx = is_fclose ? dev_ctx : + dev_ctx->priv->priv_dev_ctx; + + if (dev_ctx->strg_hdl && + se_close_storage(tx_ctx, dev_ctx->strg_hdl)) + dev_err(dev_ctx->priv->dev, "failed to close storage.\n"); + if (dev_ctx->sess_hdl && + se_close_session(tx_ctx, dev_ctx->sess_hdl)) + dev_err(dev_ctx->priv->dev, "failed to close session.\n"); + } + /* + * fw_busy is caused by one timed-out synchronous transaction. + * Only that transaction's dev_ctx may still have coherent + * memory referenced by FW. Do not skip cleanup for unrelated + * contexts while fw_busy is set. + */ + if (se_is_fw_busy_ctx(dev_ctx)) + dev_warn(dev_ctx->priv->dev, + "%s: deferring shared memory cleanup while FW is busy\n", + dev_ctx->devname); + else + cleanup_se_shared_mem(dev_ctx, true); + + kfree(dev_ctx->devname); + dev_ctx->devname = NULL; + dev_ctx->cleanup_done = true; + } + } + + if (is_fclose) + kref_put(&dev_ctx->refcount, se_if_dev_ctx_release); +} + +static void dlink_n_cleanup_dev_ctx(struct se_if_device_ctx *dev_ctx, bool is_fclose) +{ + struct se_if_priv *priv = dev_ctx->priv; + + if (is_fclose) { + scoped_guard(mutex, &priv->modify_lock) + dlink_dev_ctx(dev_ctx); + } + + cleanup_dev_ctx(dev_ctx, is_fclose); +} + +static int init_device_context(struct se_if_priv *priv, int ch_id, + struct se_if_device_ctx **new_dev_ctx) +{ + struct se_if_device_ctx *dev_ctx; + int ret = 0; + + dev_ctx = kzalloc_obj(*dev_ctx); + + if (!dev_ctx) + return -ENOMEM; + + dev_ctx->devname = kasprintf(GFP_KERNEL, "%s0_ch%d", + get_se_if_name(priv->if_defs->se_if_type), + ch_id); + if (!dev_ctx->devname) { + kfree(dev_ctx); + return -ENOMEM; + } + + mutex_init(&dev_ctx->fops_lock); + kref_init(&dev_ctx->refcount); + dev_ctx->priv = priv; + dev_ctx->cleanup_done = false; + INIT_LIST_HEAD(&dev_ctx->link); + set_se_rcv_msg_timeout(dev_ctx, SE_RCV_MSG_LONG_TIMEOUT_MS); + *new_dev_ctx = dev_ctx; + + ret = init_se_shared_mem(dev_ctx); + if (ret < 0) { + kfree(dev_ctx->devname); + kfree(dev_ctx); + *new_dev_ctx = NULL; + + return ret; + } + + /* Take a reference to priv for this device context */ + kref_get(&priv->refcount); + + scoped_guard(mutex, &priv->modify_lock) { + list_add_tail(&dev_ctx->link, &priv->dev_ctx_list); + priv->active_devctx_count++; + } + + return ret; +} + +static int se_ioctl_cmd_snd_rcv_cleanup(struct se_if_device_ctx *dev_ctx, void __user *uarg, + struct se_ioctl_cmd_snd_rcv_rsp_info *cmd_snd_rcv_rsp_info) +{ + /* shared memory is allocated before this IOCTL */ + se_dev_ctx_shared_mem_cleanup(dev_ctx); + + if (cmd_snd_rcv_rsp_info->rx_buf_sz && + copy_to_user(uarg, cmd_snd_rcv_rsp_info, sizeof(*cmd_snd_rcv_rsp_info))) { + dev_err(dev_ctx->priv->dev, "%s: Failed to copy cmd_snd_rcv_rsp_info to user.\n", + dev_ctx->devname); + return -EFAULT; + } + + return 0; +} + +static int se_ioctl_cmd_snd_rcv_rsp_handler(struct se_if_device_ctx *dev_ctx, + void __user *uarg) +{ + struct se_ioctl_cmd_snd_rcv_rsp_info cmd_snd_rcv_rsp_info = {0}; + struct se_if_priv *priv = dev_ctx->priv; + int rsp_status_err = 0; + int act_rx_msg_sz = 0; + int cleanup_err = 0; + int err = 0; + + if (copy_from_user(&cmd_snd_rcv_rsp_info, uarg, + sizeof(cmd_snd_rcv_rsp_info))) { + dev_err(priv->dev, + "%s: Failed to copy cmd_snd_rcv_rsp_info from user.", + dev_ctx->devname); + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return -EFAULT; + } + + if (cmd_snd_rcv_rsp_info.tx_buf_sz < SE_MU_HDR_SZ || + cmd_snd_rcv_rsp_info.tx_buf_sz > MAX_ALLOWED_TX_MSG_SZ) { + dev_err(priv->dev, "%s: User buffer too small/large(%d < %d)\n", + dev_ctx->devname, cmd_snd_rcv_rsp_info.tx_buf_sz, + cmd_snd_rcv_rsp_info.tx_buf_sz < SE_MU_HDR_SZ ? SE_MU_HDR_SZ : + MAX_ALLOWED_TX_MSG_SZ); + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return -ENOSPC; + } + + struct se_api_msg *tx_msg __free(kfree) = + memdup_user(u64_to_user_ptr(cmd_snd_rcv_rsp_info.tx_buf), + cmd_snd_rcv_rsp_info.tx_buf_sz); + if (IS_ERR(tx_msg)) { + err = PTR_ERR(tx_msg); + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return err; + } + + err = se_chk_tx_cmd_msg_hdr(dev_ctx, &tx_msg->header, + cmd_snd_rcv_rsp_info.tx_buf_sz, + cmd_snd_rcv_rsp_info.rx_buf_sz); + if (err) { + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return err; + } + + if (cmd_snd_rcv_rsp_info.rx_buf_sz < SE_MU_HDR_SZ || + cmd_snd_rcv_rsp_info.rx_buf_sz > MAX_ALLOWED_RX_MSG_SZ) { + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return -EINVAL; + } + + if (tx_msg->header.tag != priv->if_defs->cmd_tag) { + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return -EINVAL; + } + + if (tx_msg->header.ver == priv->if_defs->fw_api_ver && + get_load_fw_instance(priv)->is_fw_tobe_loaded) { + err = se_load_firmware(priv); + if (err) { + dev_err(priv->dev, "Could not send msg as FW is not loaded.\n"); + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return -EPERM; + } + } + + struct se_api_msg *rx_msg __free(kfree) = + kzalloc(cmd_snd_rcv_rsp_info.rx_buf_sz, GFP_KERNEL); + if (!rx_msg) { + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return -ENOMEM; + } + + err = ele_msg_send_rcv(dev_ctx, tx_msg, cmd_snd_rcv_rsp_info.tx_buf_sz, + rx_msg, cmd_snd_rcv_rsp_info.rx_buf_sz, &act_rx_msg_sz); + if (err < 0) { + /* + * -ERESTARTSYS here means the wait was interrupted by a signal + * after the command had already been handed to - and executed + * by - the firmware, with its response delivered into rx_msg + * (ele_msg_send_rcv() converts only a positive, i.e. successfully + * received, result to -ERESTARTSYS). If that response carried a + * freshly allocated session/storage handle, record it now via + * fw_api_specific_ops(): the handle is already live in firmware, + * so leaving it untracked would stop cleanup_dev_ctx() from ever + * closing it and leak the firmware resource. Validate the + * delivered response first, using its own declared length bounded + * by the caller's buffer, so a truncated or malformed reply is + * not acted upon. + */ + if (err == -ERESTARTSYS) { + u32 rsp_sz = rx_msg->header.size << 2; + + if (rsp_sz && rsp_sz <= cmd_snd_rcv_rsp_info.rx_buf_sz && + !se_val_rsp_hdr_n_status(dev_ctx, rx_msg, + tx_msg->header.command, act_rx_msg_sz, + tx_msg->header.ver)) { + se_dev_ctx_cpy_out_data(dev_ctx); + fw_api_specific_ops(dev_ctx, rx_msg, true); + } + /* + * NOTE: se_dev_ctx_cpy_out_data() above has already + * copied the firmware response payload to userspace before + * this point. Returning -EINTR here is intentional, not + * -ERESTARTSYS: the VFS would transparently restart the + * ioctl on -ERESTARTSYS, re-issuing the command with + * already-zeroed shared input buffers. -EINTR prevents + * auto-restart and lets userspace enter its signal handler + * to decide whether to reissue the command. + * See Documentation/driver-api/firmware/other_interfaces.rst, + * section "Signal handling after a completed hardware + * operation". + */ + err = -EINTR; + } + + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + + return err; + } + + /* + * ele_msg_send_rcv() returns a positive received-message size on + * success. Returning that raw size as the ioctl result would make a + * successful transaction look like a positive (non-zero) return value + * to userspace. Record the actual received size in rx_buf_sz for the + * response copied back to userspace, then normalise err to 0 so the + * ioctl reports plain success; the firmware status is conveyed to + * userspace inside the response buffer itself. + */ + cmd_snd_rcv_rsp_info.rx_buf_sz = act_rx_msg_sz; + err = 0; + + dev_dbg(priv->dev, "%s: %s %s.\n", dev_ctx->devname, __func__, + "message received, start transmit to user"); + + rsp_status_err = + se_val_rsp_hdr_n_status(dev_ctx, rx_msg, tx_msg->header.command, + act_rx_msg_sz, tx_msg->header.ver); + + if (!rsp_status_err) { + /* + * For msg IDs handled by fw_api_specific_ops(), the exact + * response size was already ensured in ele_uapi_allowed_fw_cmd(). + */ + err = fw_api_specific_ops(dev_ctx, rx_msg, false); + if (err) { + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return err; + } + + err = se_dev_ctx_cpy_out_data(dev_ctx); + if (err < 0) { + se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + return err; + } + } + + /* Copy data from the buffer */ + print_hex_dump_debug("to user ", DUMP_PREFIX_OFFSET, 4, 4, rx_msg, + cmd_snd_rcv_rsp_info.rx_buf_sz, false); + + if (copy_to_user(u64_to_user_ptr(cmd_snd_rcv_rsp_info.rx_buf), rx_msg, + cmd_snd_rcv_rsp_info.rx_buf_sz)) { + dev_err(priv->dev, "%s: Failed to copy to user.\n", dev_ctx->devname); + err = -EFAULT; + } + + cleanup_err = se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info); + + if (cleanup_err && !err) + err = cleanup_err; + + return err; +} + +static int se_ioctl_get_mu_info(struct se_if_device_ctx *dev_ctx, + void __user *uarg) +{ + struct se_if_priv *priv = dev_ctx->priv; + struct se_ioctl_get_if_info if_info; + struct se_if_node *if_node; + int err = 0; + + if_node = container_of(priv->if_defs, typeof(*if_node), if_defs); + + if_info.se_if_id = 0; + if_info.interrupt_idx = 0; + if_info.tz = 0; + if_info.did = 0; + if_info.cmd_tag = priv->if_defs->cmd_tag; + if_info.rsp_tag = priv->if_defs->rsp_tag; + if_info.success_tag = priv->if_defs->success_tag; + if_info.base_api_ver = priv->if_defs->base_api_ver; + if_info.fw_api_ver = priv->if_defs->fw_api_ver; + + dev_dbg(priv->dev, "%s: info [se_if_id: %d, irq_idx: %d, tz: 0x%x, did: 0x%x].\n", + dev_ctx->devname, if_info.se_if_id, if_info.interrupt_idx, if_info.tz, + if_info.did); + + if (copy_to_user(uarg, &if_info, sizeof(if_info))) { + dev_err(priv->dev, "%s: Failed to copy mu info to user.\n", + dev_ctx->devname); + err = -EFAULT; + } + + return err; +} + +static void rollback_shared_mem_pos(struct se_if_device_ctx *dev_ctx, u32 length) +{ + struct se_shared_mem *shared_mem = NULL; + + shared_mem = &dev_ctx->se_shared_mem_mgmt.non_secure_mem; + + if (WARN_ON_ONCE(length > shared_mem->pos)) { + shared_mem->pos = 0; + return; + } + + shared_mem->pos -= length; +} + +int get_shared_mem_slot(struct se_if_device_ctx *dev_ctx, + u32 *length, dma_addr_t *ele_dma_addr, void **ptr) +{ + struct se_shared_mem *shared_mem = NULL; + bool is_fw_busy_dev_ctx; + size_t aligned_len = 0; + u32 pos; + + /* + * If this context is the one that caused a firmware timeout the shared + * DMA buffers may still be actively read/written by the firmware. + */ + is_fw_busy_dev_ctx = se_is_fw_busy_ctx(dev_ctx); + if (is_fw_busy_dev_ctx) + return -EBUSY; + + aligned_len = round_up((size_t)*length, 8); + if (aligned_len < *length) { + dev_err(dev_ctx->priv->dev, "%s: Invalid buffer length.\n", + dev_ctx->devname); + return -EINVAL; + } + + /* No specific requirement for this buffer. */ + shared_mem = &dev_ctx->se_shared_mem_mgmt.non_secure_mem; + + /* Check there is enough space in the shared memory. */ + dev_dbg(dev_ctx->priv->dev, "%s: req_size = %zd, max_size= %d, curr_pos = %d\n", + dev_ctx->devname, aligned_len, shared_mem->size, + shared_mem->pos); + + if (shared_mem->size < shared_mem->pos || + aligned_len > (shared_mem->size - shared_mem->pos)) { + dev_err(dev_ctx->priv->dev, "%s: Not enough space in shared memory.\n", + dev_ctx->devname); + return -ENOMEM; + } + + /* Allocate space in shared memory. 8 bytes aligned. */ + pos = shared_mem->pos; + shared_mem->pos += aligned_len; + *ele_dma_addr = (u64)shared_mem->dma_addr + pos; + *ptr = shared_mem->ptr + pos; + *length = aligned_len; + + memset(shared_mem->ptr + pos, 0, aligned_len); + + return 0; +} + +/* + * Copy a buffer of data to/from the user and return the address to use in + * messages + */ +static int se_ioctl_setup_iobuf_handler(struct se_if_device_ctx *dev_ctx, + void __user *uarg) +{ + struct se_ioctl_setup_iobuf io = {0}; + struct se_buf_desc *b_desc = NULL; + void *dma_buf_ptr = NULL; + dma_addr_t ele_dma_addr; + u32 aligned_len = 0; + int err = 0; + + if (copy_from_user(&io, uarg, sizeof(io))) { + dev_err(dev_ctx->priv->dev, "%s: Failed copy iobuf config from user.\n", + dev_ctx->devname); + return -EFAULT; + } + + dev_dbg(dev_ctx->priv->dev, "%s: io [buf: %p(%d) flag: %x].\n", dev_ctx->devname, + u64_to_user_ptr(io.user_buf), io.length, io.flags); + + if (io.length == 0 || !io.user_buf) { + /* + * Accept NULL pointers since some buffers are optional + * in FW commands. In this case we should return 0 as + * pointer to be embedded into the message. + * Skip all data copy part of code below. + */ + io.ele_addr = 0; + goto copy; + } + + aligned_len = io.length; + err = get_shared_mem_slot(dev_ctx, &aligned_len, &ele_dma_addr, &dma_buf_ptr); + if (err) + return err; + + io.ele_addr = ele_dma_addr; + if ((io.flags & SE_IO_BUF_FLAGS_IS_INPUT) || + (io.flags & SE_IO_BUF_FLAGS_IS_IN_OUT)) { + /* + * buffer is input: + * copy data from user space to this allocated buffer. + */ + if (copy_from_user(dma_buf_ptr, u64_to_user_ptr(io.user_buf), + io.length)) { + dev_err(dev_ctx->priv->dev, + "%s: Failed copy data to shared memory.", + dev_ctx->devname); + err = -EFAULT; + goto rollback; + } + } + + b_desc = add_b_desc_to_pending_list(dma_buf_ptr, &io, dev_ctx); + if (IS_ERR(b_desc)) { + err = PTR_ERR(b_desc); + dev_err(dev_ctx->priv->dev, "%s: Failed to allocate/link b_desc.\n", + dev_ctx->devname); + goto rollback; + } + +copy: + /* Provide the EdgeLock Enclave address to user space only if success.*/ + if (copy_to_user(uarg, &io, sizeof(io))) { + dev_err(dev_ctx->priv->dev, "%s: Failed to copy iobuff setup to user.\n", + dev_ctx->devname); + err = -EFAULT; + goto rollback; + } + return err; + +rollback: + if (!IS_ERR_OR_NULL(b_desc)) { + list_del(&b_desc->link); + kfree(b_desc); + } + + if (dma_buf_ptr && aligned_len) { + memset(dma_buf_ptr, 0, aligned_len); + rollback_shared_mem_pos(dev_ctx, aligned_len); + } + + return err; +} + +/* IOCTL to provide SoC information */ +static int se_ioctl_get_se_soc_info_handler(struct se_if_device_ctx *dev_ctx, + void __user *uarg) +{ + struct se_ioctl_get_soc_info soc_info; + int err = -EINVAL; + + soc_info.soc_id = get_se_soc_id(dev_ctx->priv); + soc_info.soc_rev = var_se_info.soc_rev; + + err = copy_to_user(uarg, (u8 *)(&soc_info), sizeof(soc_info)); + if (err) { + dev_err(dev_ctx->priv->dev, "%s: Failed to copy soc info to user.\n", + dev_ctx->devname); + err = -EFAULT; + } + + return err; +} + +/* + * File operations for user-space + */ + +/* Write a message to the MU. */ +static ssize_t se_if_fops_write(struct file *fp, const char __user *buf, + size_t size, loff_t *ppos) +{ + struct se_if_device_ctx *dev_ctx = fp->private_data; + struct se_if_priv *priv; + int err; + + scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &dev_ctx->fops_lock) { + if (dev_ctx->cleanup_done) + return -ENODEV; + + priv = dev_ctx->priv; + + dev_dbg(priv->dev, "%s: write from buf (%p)%zu, ppos=%lld.\n", dev_ctx->devname, + buf, size, ((ppos) ? *ppos : 0)); + + if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx) { + se_dev_ctx_shared_mem_cleanup(dev_ctx); + return -EINVAL; + } + + if (size < SE_MU_HDR_SZ || size > MAX_ALLOWED_TX_MSG_SZ) { + dev_err(priv->dev, "%s: User buffer too small/large(%zu < %d)\n", + dev_ctx->devname, size, + size < SE_MU_HDR_SZ ? SE_MU_HDR_SZ : + MAX_ALLOWED_TX_MSG_SZ); + return -ENOSPC; + } + + struct se_api_msg *tx_msg __free(kfree) = memdup_user(buf, size); + if (IS_ERR(tx_msg)) + return PTR_ERR(tx_msg); + + err = se_chk_tx_rsp_msg_hdr(dev_ctx, &tx_msg->header, size); + if (err) + return err; + + print_hex_dump_debug("from user ", DUMP_PREFIX_OFFSET, 4, 4, + tx_msg, size, false); + + err = ele_msg_send(dev_ctx, tx_msg, size); + + return err; + } +} + +/* + * Read a message from the MU. + * Blocking until a message is available. + */ +static ssize_t se_if_fops_read(struct file *fp, char __user *buf, size_t size, + loff_t *ppos) +{ + struct se_if_device_ctx *dev_ctx = fp->private_data; + u8 rx_msg_snap[MAX_NVM_MSG_LEN] = {}; + char devname_snap[32] = {}; + struct se_if_priv *priv; + unsigned long flags; + size_t copy_len; + int err; + + scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &dev_ctx->fops_lock) { + priv = dev_ctx->priv; + + if (dev_ctx->cleanup_done) + return -ENODEV; + + /* + * Snapshot devname once while fops_lock is held. After the + * scoped guard releases the lock, a concurrent cleanup_dev_ctx() + * could free dev_ctx->devname before the error path below runs. + */ + strscpy(devname_snap, dev_ctx->devname, sizeof(devname_snap)); + + dev_dbg(priv->dev, "%s: read to buf %p(%zu), ppos=%lld.\n", devname_snap, + buf, size, ((ppos) ? *ppos : 0)); + + mutex_lock(&priv->modify_lock); + if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx) { + mutex_unlock(&priv->modify_lock); + se_dev_ctx_shared_mem_cleanup(dev_ctx); + return -EINVAL; + } + mutex_unlock(&priv->modify_lock); + } + + err = ele_msg_rcv(dev_ctx, &priv->cmd_receiver_clbk_hdl); + if (err < 0) { + if (err != -ERESTARTSYS) + dev_err(priv->dev, + "%s: Er[0x%x]: Signal Interrupted. Current act-dev-ctx count: %d.", + devname_snap, err, dev_ctx->priv->active_devctx_count); + return err; + } + + /* + * Reacquire fops_lock before touching any dev_ctx state (pending lists, + * rx_msg) after the blocking wait. fops_lock was dropped before calling + * ele_msg_rcv(). If cleanup_dev_ctx() ran concurrently it could have + * freed the DMA buffers and the pending lists, leading to UAF and list + * corruption. Re-checking cleanup_done under fops_lock prevents that. + */ + mutex_lock(&dev_ctx->fops_lock); + + if (dev_ctx->cleanup_done) { + mutex_unlock(&dev_ctx->fops_lock); + return -ENODEV; + } + + /* + * Snapshot the whole rx_msg under modify_lock + clbk_rx_lock, not just + * copy_len bytes: fw_api_specific_ops() reads data words (e.g. strg_hdl + * at data[1]) beyond the userspace read size; truncating would record a + * zero handle. Run fw_api_specific_ops() OUTSIDE modify_lock + * (ELE_STORAGE_OPEN_REQ re-takes it, else deadlock). + */ + scoped_guard(mutex, &priv->modify_lock) { + spin_lock_irqsave(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock, flags); + if (priv->cmd_receiver_clbk_hdl.dev_ctx != dev_ctx || + !priv->cmd_receiver_clbk_hdl.rx_msg || + !priv->cmd_receiver_clbk_hdl.rx_msg_sz) { + spin_unlock_irqrestore(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock, flags); + mutex_unlock(&dev_ctx->fops_lock); + return -ENODEV; + } + copy_len = min(size, (size_t)priv->cmd_receiver_clbk_hdl.rx_msg_sz); + memcpy(rx_msg_snap, priv->cmd_receiver_clbk_hdl.rx_msg, + priv->cmd_receiver_clbk_hdl.rx_msg_sz); + priv->cmd_receiver_clbk_hdl.rx_msg_sz = 0; + spin_unlock_irqrestore(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock, flags); + + /* We may need to copy the output data to user before + * delivering the completion message. + */ + err = se_dev_ctx_cpy_out_data(dev_ctx); + if (err < 0) { + se_dev_ctx_shared_mem_cleanup(dev_ctx); + mutex_unlock(&dev_ctx->fops_lock); + return err; + } + } + + /* fw_api_specific_ops() runs outside modify_lock; see comment above. */ + print_hex_dump_debug("to user ", DUMP_PREFIX_OFFSET, 4, 4, + rx_msg_snap, copy_len, false); + + cmd_receiver_specific_ops(dev_ctx, (struct se_api_msg *)rx_msg_snap); + err = copy_len; + if (copy_to_user(buf, rx_msg_snap, copy_len)) + err = -EFAULT; + + se_dev_ctx_shared_mem_cleanup(dev_ctx); + mutex_unlock(&dev_ctx->fops_lock); + + return err; +} + +/* Open a character device. */ +static int se_if_fops_open(struct inode *nd, struct file *fp) +{ + struct miscdevice *miscdev = fp->private_data; + struct se_if_open_gate *gate; + struct se_if_device_ctx *misc_dev_ctx; + struct se_if_device_ctx *dev_ctx; + struct se_if_priv *priv; + int err = 0; + + gate = container_of(miscdev, struct se_if_open_gate, miscdev); + + if (!se_if_open_gate_get(gate)) + return -ENODEV; + + if (mutex_lock_interruptible(&gate->lock)) { + se_if_open_gate_put(gate); + return -ERESTARTSYS; + } + + if (gate->dying || !gate->priv || + !kref_get_unless_zero(&gate->priv->refcount)) { + mutex_unlock(&gate->lock); + se_if_open_gate_put(gate); + return -ENODEV; + } + + priv = gate->priv; + mutex_unlock(&gate->lock); + + misc_dev_ctx = priv->priv_dev_ctx; + + if (mutex_lock_interruptible(&misc_dev_ctx->fops_lock)) { + err = -ERESTARTSYS; + goto out_put_priv; + } + + if (misc_dev_ctx->cleanup_done) { + err = -ENODEV; + goto out_unlock_misc; + } + + priv->dev_ctx_mono_count++; + err = init_device_context(priv, priv->dev_ctx_mono_count, &dev_ctx); + if (err) { + dev_err(priv->dev, "Failed to create dev-ctx.\n"); + goto out_unlock_misc; + } + + fp->private_data = dev_ctx; + +out_unlock_misc: + mutex_unlock(&misc_dev_ctx->fops_lock); +out_put_priv: + kref_put(&priv->refcount, se_if_priv_release); + se_if_open_gate_put(gate); + return err; +} + +/* Close a character device. */ +static int se_if_fops_close(struct inode *nd, struct file *fp) +{ + struct se_if_device_ctx *dev_ctx = fp->private_data; + + dlink_n_cleanup_dev_ctx(dev_ctx, true); + + return 0; +} + +/* IOCTL entry point of a character device */ +static long se_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) +{ + struct se_if_device_ctx *dev_ctx = fp->private_data; + struct se_if_priv *priv; + void __user *uarg = (void __user *)arg; + long err; + + /* Prevent race during change of device context */ + scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &dev_ctx->fops_lock) { + if (dev_ctx->cleanup_done) + return -ENODEV; + + priv = dev_ctx->priv; + + switch (cmd) { + case SE_IOCTL_CHECK_CMD_RCV_REG_STATUS: { + guard(mutex)(&priv->modify_lock); + err = check_cmd_rcvr_status(dev_ctx); + break; + } + case SE_IOCTL_GET_MU_INFO: + err = se_ioctl_get_mu_info(dev_ctx, uarg); + break; + case SE_IOCTL_SETUP_IOBUF: + err = se_ioctl_setup_iobuf_handler(dev_ctx, uarg); + break; + case SE_IOCTL_GET_SOC_INFO: + err = se_ioctl_get_se_soc_info_handler(dev_ctx, uarg); + break; + case SE_IOCTL_CMD_SEND_RCV_RSP: + err = se_ioctl_cmd_snd_rcv_rsp_handler(dev_ctx, uarg); + break; + default: + err = -ENOTTY; + dev_dbg(priv->dev, "%s: IOCTL %.8x not supported.\n", + dev_ctx->devname, cmd); + } + } + + return err; +} + +/* Char driver setup */ +static const struct file_operations se_if_fops = { + .open = se_if_fops_open, + .owner = THIS_MODULE, + .release = se_if_fops_close, + .unlocked_ioctl = se_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .read = se_if_fops_read, + .write = se_if_fops_write, +}; + +int se_get_mem_pool_buf(struct se_if_device_ctx *dev_ctx, void **buf, + dma_addr_t *daddr, u32 len) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct se_if_priv *priv = dev_ctx->priv; + struct se_buf_desc *b_desc = NULL; + + lockdep_assert_held(&dev_ctx->fops_lock); + + if (se_is_fw_busy_ctx(dev_ctx)) + return -EBUSY; + + b_desc = kzalloc_obj(*b_desc); + if (!b_desc) + return -ENOMEM; + + /* + * gen_pool is internally thread-safe, so contexts may allocate + * concurrently. The buffer is tracked on this context's own + * mem_pool_buf_list and released on its cleanup path. + */ + *buf = gen_pool_dma_alloc(priv->mem_pool, len, daddr); + if (!*buf) { + dev_err(priv->dev, "Failed to alloc from gen_pool.\n"); + kfree(b_desc); + return -ENOMEM; + } + + /* gen_pool_dma_alloc() does not zero the buffer. */ + memset(*buf, 0, len); + b_desc->shared_buf_ptr = *buf; + b_desc->size = len; + + list_add_tail(&b_desc->link, &se_shared_mem_mgmt->mem_pool_buf_list); + + return 0; +} + +void se_cleanup_mem_pool_buf(struct se_if_device_ctx *dev_ctx, bool reclaim) +{ + struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt; + struct se_if_priv *priv = dev_ctx->priv; + struct se_buf_desc *b_desc, *temp; + + /* + * Free only the buffers this context allocated. A context that never + * used the pool has an empty list, so this is a no-op for it. + * + * Unlike the coherent staging buffer, the pool path needs no + * "nothing staged" (pos) gate on the reclaim=false leg. Pool buffers + * are ephemeral, per-transaction allocations: se_get_mem_pool_buf() + * refuses to allocate once the context is fw_busy, ele_msg_send_rcv() + * refuses to start a new command while fw_busy, and the success path + * frees the whole list via se_cleanup_mem_pool_buf(reclaim=true) + * before returning. se_if_cmd_lock serialises synchronous commands, so + * at most one transaction is outstanding. The only way to reach here + * with reclaim=false and a non-empty list is the single fw_busy + * context still owning the buffer(s) from the one timed-out + * transaction. Those buffers are exactly the in-flight ones the + * enclave may still be DMA-ing into, so leaving them on the list (no + * gen_pool_free) deliberately leaks them to avoid a DMA-after-free - + * there are no already-consumed pool buffers to reclaim on this leg. + */ + list_for_each_entry_safe(b_desc, temp, &se_shared_mem_mgmt->mem_pool_buf_list, link) { + if (reclaim) + gen_pool_free(priv->mem_pool, + (unsigned long)b_desc->shared_buf_ptr, + b_desc->size); + list_del(&b_desc->link); + kfree(b_desc); + } +} + +static void se_fw_busy_work(struct work_struct *work) +{ + struct fw_busy_info *fbusy_info = + container_of(work, struct fw_busy_info, fw_busy_work); + struct se_if_priv *priv = + container_of(fbusy_info, struct se_if_priv, fw_busy_info); + + se_clear_fw_busy(priv); +} + +static int se_suspend(struct device *dev) +{ + struct se_if_priv *priv = dev_get_drvdata(dev); + struct se_fw_load_info *load_fw; + unsigned int noio_flag; + int ret = 0; + + load_fw = get_load_fw_instance(priv); + + if (load_fw->imem_mgmt) { + /* + * Set PF_MEMALLOC_NOIO for the duration of the suspend + * callbacks. This covers all allocations in the call chain + * (ele_get_info, se_service_swap, se_get_mem_pool_buf) without + * requiring each site to pass GFP_NOIO explicitly. Without this, + * GFP_KERNEL allocations in those paths could trigger direct + * reclaim and attempt I/O to a storage device that is already + * suspended, causing a deadlock. + */ + noio_flag = memalloc_noio_save(); + ret = se_save_imem_state(priv, &load_fw->imem); + memalloc_noio_restore(noio_flag); + if (ret) + dev_err(dev, "Failure saving IMEM state[0x%x]\n", ret); + } + + return ret; +} + +static int se_resume(struct device *dev) +{ + struct se_if_priv *priv = dev_get_drvdata(dev); + struct se_fw_load_info *load_fw; + unsigned int noio_flag; + int ret = 0; + + load_fw = get_load_fw_instance(priv); + + if (load_fw->imem_mgmt) { + noio_flag = memalloc_noio_save(); + ret = se_restore_imem_state(priv, &load_fw->imem); + memalloc_noio_restore(noio_flag); + if (ret) + dev_err(dev, "Failure restoring IMEM state[0x%x]\n", ret); + } + + return ret; +} + +DEFINE_SIMPLE_DEV_PM_OPS(se_pm, se_suspend, se_resume); + +static struct platform_driver se_driver = { + .driver = { + .name = "fsl-se", + .of_match_table = se_match, + .pm = pm_sleep_ptr(&se_pm), + }, + .probe = se_if_probe, +}; + +static int __init se_init(void) +{ + return platform_driver_register(&se_driver); +} +module_init(se_init); + +static void __exit se_exit(void) +{ + platform_driver_unregister(&se_driver); + + /* + * The soc_device is a module-scoped singleton that outlives any single + * MU interface bind/unbind. Release it here, once, after every interface + * has been unbound, so its lifetime is tied to the module rather than to + * the first-probed interface. + */ + se_soc_device_unregister(&var_se_info.soc_dev_regn); +} +module_exit(se_exit); + +MODULE_AUTHOR("Pankaj Gupta <pankaj.gupta@nxp.com>"); +MODULE_DESCRIPTION("iMX Secure Enclave Driver."); +MODULE_LICENSE("GPL"); |