diff options
Diffstat (limited to 'drivers/ufs/core/ufs-mcq.c')
-rw-r--r-- | drivers/ufs/core/ufs-mcq.c | 296 |
1 files changed, 271 insertions, 25 deletions
diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 51b3c6ae781d..6fb0e007af63 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -12,21 +12,26 @@ #include <linux/module.h> #include <linux/platform_device.h> #include "ufshcd-priv.h" +#include <linux/delay.h> +#include <scsi/scsi_cmnd.h> +#include <linux/bitfield.h> +#include <linux/iopoll.h> #define MAX_QUEUE_SUP GENMASK(7, 0) #define UFS_MCQ_MIN_RW_QUEUES 2 #define UFS_MCQ_MIN_READ_QUEUES 0 -#define UFS_MCQ_NUM_DEV_CMD_QUEUES 1 #define UFS_MCQ_MIN_POLL_QUEUES 0 #define QUEUE_EN_OFFSET 31 #define QUEUE_ID_OFFSET 16 -#define MAX_DEV_CMD_ENTRIES 2 #define MCQ_CFG_MAC_MASK GENMASK(16, 8) #define MCQ_QCFG_SIZE 0x40 #define MCQ_ENTRY_SIZE_IN_DWORD 8 #define CQE_UCD_BA GENMASK_ULL(63, 7) +/* Max mcq register polling time in microseconds */ +#define MCQ_POLL_US 500000 + static int rw_queue_count_set(const char *val, const struct kernel_param *kp) { return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES, @@ -108,8 +113,7 @@ struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, u32 utag = blk_mq_unique_tag(req); u32 hwq = blk_mq_unique_tag_to_hwq(utag); - /* uhq[0] is used to serve device commands */ - return &hba->uhq[hwq + UFSHCD_MCQ_IO_QUEUE_OFFSET]; + return &hba->uhq[hwq]; } /** @@ -153,8 +157,7 @@ static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba) /* maxq is 0 based value */ hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1; - tot_queues = UFS_MCQ_NUM_DEV_CMD_QUEUES + read_queues + poll_queues + - rw_queues; + tot_queues = read_queues + poll_queues + rw_queues; if (hba_maxq < tot_queues) { dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n", @@ -162,7 +165,7 @@ static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba) return -EOPNOTSUPP; } - rem = hba_maxq - UFS_MCQ_NUM_DEV_CMD_QUEUES; + rem = hba_maxq; if (rw_queues) { hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues; @@ -188,7 +191,7 @@ static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba) for (i = 0; i < HCTX_MAX_TYPES; i++) host->nr_hw_queues += hba->nr_queues[i]; - hba->nr_hw_queues = host->nr_hw_queues + UFS_MCQ_NUM_DEV_CMD_QUEUES; + hba->nr_hw_queues = host->nr_hw_queues; return 0; } @@ -270,44 +273,57 @@ static int ufshcd_mcq_get_tag(struct ufs_hba *hba, } static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, - struct ufs_hw_queue *hwq) + struct ufs_hw_queue *hwq) { struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); int tag = ufshcd_mcq_get_tag(hba, hwq, cqe); - ufshcd_compl_one_cqe(hba, tag, cqe); + if (cqe->command_desc_base_addr) { + ufshcd_compl_one_cqe(hba, tag, cqe); + /* After processed the cqe, mark it empty (invalid) entry */ + cqe->command_desc_base_addr = 0; + } } -unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba, - struct ufs_hw_queue *hwq) +void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, + struct ufs_hw_queue *hwq) { - unsigned long completed_reqs = 0; + unsigned long flags; + u32 entries = hwq->max_entries; - ufshcd_mcq_update_cq_tail_slot(hwq); - while (!ufshcd_mcq_is_cq_empty(hwq)) { + spin_lock_irqsave(&hwq->cq_lock, flags); + while (entries > 0) { ufshcd_mcq_process_cqe(hba, hwq); ufshcd_mcq_inc_cq_head_slot(hwq); - completed_reqs++; + entries--; } - if (completed_reqs) - ufshcd_mcq_update_cq_head(hwq); - - return completed_reqs; + ufshcd_mcq_update_cq_tail_slot(hwq); + hwq->cq_head_slot = hwq->cq_tail_slot; + spin_unlock_irqrestore(&hwq->cq_lock, flags); } -EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_nolock); unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba, struct ufs_hw_queue *hwq) { - unsigned long completed_reqs, flags; + unsigned long completed_reqs = 0; + unsigned long flags; spin_lock_irqsave(&hwq->cq_lock, flags); - completed_reqs = ufshcd_mcq_poll_cqe_nolock(hba, hwq); + ufshcd_mcq_update_cq_tail_slot(hwq); + while (!ufshcd_mcq_is_cq_empty(hwq)) { + ufshcd_mcq_process_cqe(hba, hwq); + ufshcd_mcq_inc_cq_head_slot(hwq); + completed_reqs++; + } + + if (completed_reqs) + ufshcd_mcq_update_cq_head(hwq); spin_unlock_irqrestore(&hwq->cq_lock, flags); return completed_reqs; } +EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock); void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) { @@ -420,13 +436,243 @@ int ufshcd_mcq_init(struct ufs_hba *hba) hwq->max_entries = hba->nutrs; spin_lock_init(&hwq->sq_lock); spin_lock_init(&hwq->cq_lock); + mutex_init(&hwq->sq_mutex); } /* The very first HW queue serves device commands */ hba->dev_cmd_queue = &hba->uhq[0]; - /* Give dev_cmd_queue the minimal number of entries */ - hba->dev_cmd_queue->max_entries = MAX_DEV_CMD_ENTRIES; host->host_tagset = 1; return 0; } + +static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq) +{ + void __iomem *reg; + u32 id = hwq->id, val; + int err; + + if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) + return -ETIMEDOUT; + + writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); + reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; + err = read_poll_timeout(readl, val, val & SQ_STS, 20, + MCQ_POLL_US, false, reg); + if (err) + dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", + __func__, id, err); + return err; +} + +static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) +{ + void __iomem *reg; + u32 id = hwq->id, val; + int err; + + if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) + return -ETIMEDOUT; + + writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); + reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; + err = read_poll_timeout(readl, val, !(val & SQ_STS), 20, + MCQ_POLL_US, false, reg); + if (err) + dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", + __func__, id, err); + return err; +} + +/** + * ufshcd_mcq_sq_cleanup - Clean up submission queue resources + * associated with the pending command. + * @hba - per adapter instance. + * @task_tag - The command's task tag. + * + * Returns 0 for success; error code otherwise. + */ +int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) +{ + struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + struct scsi_cmnd *cmd = lrbp->cmd; + struct ufs_hw_queue *hwq; + void __iomem *reg, *opr_sqd_base; + u32 nexus, id, val; + int err; + + if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) + return -ETIMEDOUT; + + if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) { + if (!cmd) + return -EINVAL; + hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); + } else { + hwq = hba->dev_cmd_queue; + } + + id = hwq->id; + + mutex_lock(&hwq->sq_mutex); + + /* stop the SQ fetching before working on it */ + err = ufshcd_mcq_sq_stop(hba, hwq); + if (err) + goto unlock; + + /* SQCTI = EXT_IID, IID, LUN, Task Tag */ + nexus = lrbp->lun << 8 | task_tag; + opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id); + writel(nexus, opr_sqd_base + REG_SQCTI); + + /* SQRTCy.ICU = 1 */ + writel(SQ_ICU, opr_sqd_base + REG_SQRTC); + + /* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */ + reg = opr_sqd_base + REG_SQRTS; + err = read_poll_timeout(readl, val, val & SQ_CUS, 20, + MCQ_POLL_US, false, reg); + if (err) + dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n", + __func__, id, task_tag, + FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg))); + + if (ufshcd_mcq_sq_start(hba, hwq)) + err = -ETIMEDOUT; + +unlock: + mutex_unlock(&hwq->sq_mutex); + return err; +} + +/** + * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry. + * Write the sqe's Command Type to 0xF. The host controller will not + * fetch any sqe with Command Type = 0xF. + * + * @utrd - UTP Transfer Request Descriptor to be nullified. + */ +static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) +{ + u32 dword_0; + + dword_0 = le32_to_cpu(utrd->header.dword_0); + dword_0 &= ~UPIU_COMMAND_TYPE_MASK; + dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF); + utrd->header.dword_0 = cpu_to_le32(dword_0); +} + +/** + * ufshcd_mcq_sqe_search - Search for the command in the submission queue + * If the command is in the submission queue and not issued to the device yet, + * nullify the sqe so the host controller will skip fetching the sqe. + * + * @hba - per adapter instance. + * @hwq - Hardware Queue to be searched. + * @task_tag - The command's task tag. + * + * Returns true if the SQE containing the command is present in the SQ + * (not fetched by the controller); returns false if the SQE is not in the SQ. + */ +static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, + struct ufs_hw_queue *hwq, int task_tag) +{ + struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + struct utp_transfer_req_desc *utrd; + u32 mask = hwq->max_entries - 1; + __le64 cmd_desc_base_addr; + bool ret = false; + u64 addr, match; + u32 sq_head_slot; + + if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) + return true; + + mutex_lock(&hwq->sq_mutex); + + ufshcd_mcq_sq_stop(hba, hwq); + sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq); + if (sq_head_slot == hwq->sq_tail_slot) + goto out; + + cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr; + addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA; + + while (sq_head_slot != hwq->sq_tail_slot) { + utrd = hwq->sqe_base_addr + + sq_head_slot * sizeof(struct utp_transfer_req_desc); + match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA; + if (addr == match) { + ufshcd_mcq_nullify_sqe(utrd); + ret = true; + goto out; + } + sq_head_slot = (sq_head_slot + 1) & mask; + } + +out: + ufshcd_mcq_sq_start(hba, hwq); + mutex_unlock(&hwq->sq_mutex); + return ret; +} + +/** + * ufshcd_mcq_abort - Abort the command in MCQ. + * @cmd - The command to be aborted. + * + * Returns SUCCESS or FAILED error codes + */ +int ufshcd_mcq_abort(struct scsi_cmnd *cmd) +{ + struct Scsi_Host *host = cmd->device->host; + struct ufs_hba *hba = shost_priv(host); + int tag = scsi_cmd_to_rq(cmd)->tag; + struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct ufs_hw_queue *hwq; + int err = FAILED; + + if (!ufshcd_cmd_inflight(lrbp->cmd)) { + dev_err(hba->dev, + "%s: skip abort. cmd at tag %d already completed.\n", + __func__, tag); + goto out; + } + + /* Skip task abort in case previous aborts failed and report failure */ + if (lrbp->req_abort_skip) { + dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n", + __func__, tag); + goto out; + } + + hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); + + if (ufshcd_mcq_sqe_search(hba, hwq, tag)) { + /* + * Failure. The command should not be "stuck" in SQ for + * a long time which resulted in command being aborted. + */ + dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n", + __func__, hwq->id, tag); + goto out; + } + + /* + * The command is not in the submission queue, and it is not + * in the completion queue either. Query the device to see if + * the command is being processed in the device. + */ + if (ufshcd_try_to_abort_task(hba, tag)) { + dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err); + lrbp->req_abort_skip = true; + goto out; + } + + err = SUCCESS; + if (ufshcd_cmd_inflight(lrbp->cmd)) + ufshcd_release_scsi_cmd(hba, lrbp); + +out: + return err; +} |