diff options
Diffstat (limited to 'drivers/ufs/core/ufshcd.c')
| -rw-r--r-- | drivers/ufs/core/ufshcd.c | 652 |
1 files changed, 364 insertions, 288 deletions
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 8a01e4393159..5cb6132b8147 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -258,10 +258,15 @@ ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, return UFS_PM_LVL_0; } +static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) +{ + return hba->outstanding_tasks || hba->active_uic_cmd || + hba->uic_async_done; +} + static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) { - return (hba->clk_gating.active_reqs || hba->outstanding_reqs || hba->outstanding_tasks || - hba->active_uic_cmd || hba->uic_async_done); + return scsi_host_busy(hba->host) || ufshcd_has_pending_tasks(hba); } static const struct ufs_dev_quirk ufs_fixups[] = { @@ -273,6 +278,7 @@ static const struct ufs_dev_quirk ufs_fixups[] = { .model = UFS_ANY_MODEL, .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | + UFS_DEVICE_QUIRK_PA_HIBER8TIME | UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, { .wmanufacturerid = UFS_VENDOR_SKHYNIX, .model = UFS_ANY_MODEL, @@ -364,7 +370,7 @@ static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, else header = &hba->lrb[tag].ucd_rsp_ptr->header; - trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb, + trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, UFS_TSF_CDB); } @@ -375,7 +381,7 @@ static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, if (!trace_ufshcd_upiu_enabled()) return; - trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header, + trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, &rq_rsp->qr, UFS_TSF_OSF); } @@ -388,12 +394,12 @@ static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, return; if (str_t == UFS_TM_SEND) - trace_ufshcd_upiu(dev_name(hba->dev), str_t, + trace_ufshcd_upiu(hba, str_t, &descp->upiu_req.req_header, &descp->upiu_req.input_param1, UFS_TSF_TM_INPUT); else - trace_ufshcd_upiu(dev_name(hba->dev), str_t, + trace_ufshcd_upiu(hba, str_t, &descp->upiu_rsp.rsp_header, &descp->upiu_rsp.output_param1, UFS_TSF_TM_OUTPUT); @@ -413,7 +419,7 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, else cmd = ufshcd_readl(hba, REG_UIC_COMMAND); - trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd, + trace_ufshcd_uic_command(hba, str_t, cmd, ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); @@ -468,7 +474,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, } else { doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); } - trace_ufshcd_command(cmd->device, str_t, tag, doorbell, hwq_id, + trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, transfer_len, intr, lba, opcode, group_id); } @@ -623,8 +629,8 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); - dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n", - hba->outstanding_reqs, hba->outstanding_tasks); + dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", + scsi_host_busy(hba->host), hba->outstanding_tasks); dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", hba->saved_err, hba->saved_uic_err); dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", @@ -1058,7 +1064,7 @@ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) clki->max_freq, ret); break; } - trace_ufshcd_clk_scaling(dev_name(hba->dev), + trace_ufshcd_clk_scaling(hba, "scaled up", clki->name, clki->curr_freq, clki->max_freq); @@ -1076,7 +1082,7 @@ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) clki->min_freq, ret); break; } - trace_ufshcd_clk_scaling(dev_name(hba->dev), + trace_ufshcd_clk_scaling(hba, "scaled down", clki->name, clki->curr_freq, clki->min_freq); @@ -1117,7 +1123,7 @@ int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, return ret; } - trace_ufshcd_clk_scaling(dev_name(dev), + trace_ufshcd_clk_scaling(hba, (scaling_down ? "scaled down" : "scaled up"), clki->name, hba->clk_scaling.target_freq, freq); } @@ -1157,7 +1163,7 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, int ret = 0; ktime_t start = ktime_get(); - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE); + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); if (ret) goto out; @@ -1168,7 +1174,7 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, if (ret) goto out; - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); if (ret) { if (hba->use_pm_opp) ufshcd_opp_set_rate(hba, @@ -1181,7 +1187,7 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, ufshcd_pm_qos_update(hba, scale_up); out: - trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + trace_ufshcd_profile_clk_scaling(hba, (scale_up ? "up" : "down"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; @@ -1308,16 +1314,26 @@ out: /** * ufshcd_scale_gear - scale up/down UFS gear * @hba: per adapter instance + * @target_gear: target gear to scale to * @scale_up: True for scaling up gear and false for scaling down * * Return: 0 for success; -EBUSY if scaling can't happen at this time; * non-zero for any other errors. */ -static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) +static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) { int ret = 0; struct ufs_pa_layer_attr new_pwr_info; + if (target_gear) { + new_pwr_info = hba->pwr_info; + new_pwr_info.gear_tx = target_gear; + new_pwr_info.gear_rx = target_gear; + + goto config_pwr_mode; + } + + /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ if (scale_up) { memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, sizeof(struct ufs_pa_layer_attr)); @@ -1338,6 +1354,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) } } +config_pwr_mode: /* check if the power mode needs to be changed or not? */ ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); if (ret) @@ -1382,13 +1399,13 @@ out: return ret; } -static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up) +static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) { up_write(&hba->clk_scaling_lock); - /* Enable Write Booster if we have scaled up else disable it */ + /* Enable Write Booster if current gear requires it else disable it */ if (ufshcd_enable_wb_if_scaling_up(hba) && !err) - ufshcd_wb_toggle(hba, scale_up); + ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); mutex_unlock(&hba->wb_mutex); @@ -1408,15 +1425,19 @@ static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool sc static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, bool scale_up) { + u32 old_gear = hba->pwr_info.gear_rx; + u32 new_gear = 0; int ret = 0; + new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); + ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); if (ret) return ret; /* scale down the gear before scaling down clocks */ if (!scale_up) { - ret = ufshcd_scale_gear(hba, false); + ret = ufshcd_scale_gear(hba, new_gear, false); if (ret) goto out_unprepare; } @@ -1424,13 +1445,13 @@ static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, ret = ufshcd_scale_clks(hba, freq, scale_up); if (ret) { if (!scale_up) - ufshcd_scale_gear(hba, true); + ufshcd_scale_gear(hba, old_gear, true); goto out_unprepare; } /* scale up the gear after scaling up clocks */ if (scale_up) { - ret = ufshcd_scale_gear(hba, true); + ret = ufshcd_scale_gear(hba, new_gear, true); if (ret) { ufshcd_scale_clks(hba, hba->devfreq->previous_freq, false); @@ -1439,7 +1460,7 @@ static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, } out_unprepare: - ufshcd_clock_scaling_unprepare(hba, ret, scale_up); + ufshcd_clock_scaling_unprepare(hba, ret); return ret; } @@ -1447,16 +1468,16 @@ static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_scaling.suspend_work); - unsigned long irq_flags; - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (hba->clk_scaling.active_reqs || + hba->clk_scaling.is_suspended) + return; + + hba->clk_scaling.is_suspended = true; + hba->clk_scaling.window_start_t = 0; } - hba->clk_scaling.is_suspended = true; - hba->clk_scaling.window_start_t = 0; - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); devfreq_suspend_device(hba->devfreq); } @@ -1465,15 +1486,13 @@ static void ufshcd_clk_scaling_resume_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_scaling.resume_work); - unsigned long irq_flags; - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (!hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (!hba->clk_scaling.is_suspended) + return; + hba->clk_scaling.is_suspended = false; } - hba->clk_scaling.is_suspended = false; - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); devfreq_resume_device(hba->devfreq); } @@ -1487,7 +1506,6 @@ static int ufshcd_devfreq_target(struct device *dev, bool scale_up = false, sched_clk_scaling_suspend_work = false; struct list_head *clk_list = &hba->clk_list_head; struct ufs_clk_info *clki; - unsigned long irq_flags; if (!ufshcd_is_clkscaling_supported(hba)) return -EINVAL; @@ -1508,50 +1526,45 @@ static int ufshcd_devfreq_target(struct device *dev, *freq = (unsigned long) clk_round_rate(clki->clk, *freq); } - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (ufshcd_eh_in_progress(hba)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return 0; - } + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (ufshcd_eh_in_progress(hba)) + return 0; - /* Skip scaling clock when clock scaling is suspended */ - if (hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - dev_warn(hba->dev, "clock scaling is suspended, skip"); - return 0; - } + /* Skip scaling clock when clock scaling is suspended */ + if (hba->clk_scaling.is_suspended) { + dev_warn(hba->dev, "clock scaling is suspended, skip"); + return 0; + } - if (!hba->clk_scaling.active_reqs) - sched_clk_scaling_suspend_work = true; + if (!hba->clk_scaling.active_reqs) + sched_clk_scaling_suspend_work = true; - if (list_empty(clk_list)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - goto out; - } + if (list_empty(clk_list)) + goto out; - /* Decide based on the target or rounded-off frequency and update */ - if (hba->use_pm_opp) - scale_up = *freq > hba->clk_scaling.target_freq; - else - scale_up = *freq == clki->max_freq; + /* Decide based on the target or rounded-off frequency and update */ + if (hba->use_pm_opp) + scale_up = *freq > hba->clk_scaling.target_freq; + else + scale_up = *freq == clki->max_freq; - if (!hba->use_pm_opp && !scale_up) - *freq = clki->min_freq; + if (!hba->use_pm_opp && !scale_up) + *freq = clki->min_freq; - /* Update the frequency */ - if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - ret = 0; - goto out; /* no state change required */ + /* Update the frequency */ + if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { + ret = 0; + goto out; /* no state change required */ + } } - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); start = ktime_get(); ret = ufshcd_devfreq_scale(hba, *freq, scale_up); if (!ret) hba->clk_scaling.target_freq = *freq; - trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + trace_ufshcd_profile_clk_scaling(hba, (scale_up ? "up" : "down"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); @@ -1569,7 +1582,6 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) @@ -1577,7 +1589,8 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, memset(stat, 0, sizeof(*stat)); - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + curr_t = ktime_get(); if (!scaling->window_start_t) goto start_window; @@ -1613,7 +1626,7 @@ start_window: scaling->busy_start_t = 0; scaling->is_busy_started = false; } - spin_unlock_irqrestore(hba->host->host_lock, flags); + return 0; } @@ -1677,19 +1690,19 @@ static void ufshcd_devfreq_remove(struct ufs_hba *hba) static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) { - unsigned long flags; bool suspend = false; cancel_work_sync(&hba->clk_scaling.suspend_work); cancel_work_sync(&hba->clk_scaling.resume_work); - spin_lock_irqsave(hba->host->host_lock, flags); - if (!hba->clk_scaling.is_suspended) { - suspend = true; - hba->clk_scaling.is_suspended = true; - hba->clk_scaling.window_start_t = 0; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (!hba->clk_scaling.is_suspended) { + suspend = true; + hba->clk_scaling.is_suspended = true; + hba->clk_scaling.window_start_t = 0; + } } - spin_unlock_irqrestore(hba->host->host_lock, flags); if (suspend) devfreq_suspend_device(hba->devfreq); @@ -1697,15 +1710,15 @@ static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) static void ufshcd_resume_clkscaling(struct ufs_hba *hba) { - unsigned long flags; bool resume = false; - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_scaling.is_suspended) { - resume = true; - hba->clk_scaling.is_suspended = false; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (hba->clk_scaling.is_suspended) { + resume = true; + hba->clk_scaling.is_suspended = false; + } } - spin_unlock_irqrestore(hba->host->host_lock, flags); if (resume) devfreq_resume_device(hba->devfreq); @@ -1723,6 +1736,8 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_clk_info *clki; + unsigned long freq; u32 value; int err = 0; @@ -1746,14 +1761,25 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev, if (value) { ufshcd_resume_clkscaling(hba); - } else { - ufshcd_suspend_clkscaling(hba); - err = ufshcd_devfreq_scale(hba, ULONG_MAX, true); - if (err) - dev_err(hba->dev, "%s: failed to scale clocks up %d\n", - __func__, err); + goto out_rel; } + clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); + freq = clki->max_freq; + + ufshcd_suspend_clkscaling(hba); + + if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) + goto out_rel; + + err = ufshcd_devfreq_scale(hba, freq, true); + if (err) + dev_err(hba->dev, "%s: failed to scale clocks up %d\n", + __func__, err); + else + hba->clk_scaling.target_freq = freq; + +out_rel: ufshcd_release(hba); ufshcd_rpm_put_sync(hba); out: @@ -1786,11 +1812,17 @@ static void ufshcd_init_clk_scaling(struct ufs_hba *hba) if (!hba->clk_scaling.min_gear) hba->clk_scaling.min_gear = UFS_HS_G1; + if (!hba->clk_scaling.wb_gear) + /* Use intermediate gear speed HS_G3 as the default wb_gear */ + hba->clk_scaling.wb_gear = UFS_HS_G3; + INIT_WORK(&hba->clk_scaling.suspend_work, ufshcd_clk_scaling_suspend_work); INIT_WORK(&hba->clk_scaling.resume_work, ufshcd_clk_scaling_resume_work); + spin_lock_init(&hba->clk_scaling.lock); + hba->clk_scaling.workq = alloc_ordered_workqueue( "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); @@ -1811,19 +1843,16 @@ static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) static void ufshcd_ungate_work(struct work_struct *work) { int ret; - unsigned long flags; struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_gating.ungate_work); cancel_delayed_work_sync(&hba->clk_gating.gate_work); - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_gating.state == CLKS_ON) { - spin_unlock_irqrestore(hba->host->host_lock, flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { + if (hba->clk_gating.state == CLKS_ON) + return; } - spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_hba_vreg_set_hpm(hba); ufshcd_setup_clocks(hba, true); @@ -1858,7 +1887,7 @@ void ufshcd_hold(struct ufs_hba *hba) if (!ufshcd_is_clkgating_allowed(hba) || !hba->clk_gating.is_initialized) return; - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); hba->clk_gating.active_reqs++; start: @@ -1874,18 +1903,18 @@ start: */ if (ufshcd_can_hibern8_during_gating(hba) && ufshcd_is_link_hibern8(hba)) { - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); flush_result = flush_work(&hba->clk_gating.ungate_work); if (hba->clk_gating.is_suspended && !flush_result) return; - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); goto start; } break; case REQ_CLKS_OFF: if (cancel_delayed_work(&hba->clk_gating.gate_work)) { hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); break; } @@ -1897,7 +1926,7 @@ start: fallthrough; case CLKS_OFF: hba->clk_gating.state = REQ_CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); queue_work(hba->clk_gating.clk_gating_workq, &hba->clk_gating.ungate_work); @@ -1907,17 +1936,17 @@ start: */ fallthrough; case REQ_CLKS_ON: - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); flush_work(&hba->clk_gating.ungate_work); /* Make sure state is CLKS_ON before returning */ - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); goto start; default: dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", __func__, hba->clk_gating.state); break; } - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_hold); @@ -1925,28 +1954,32 @@ static void ufshcd_gate_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_gating.gate_work.work); - unsigned long flags; int ret; - spin_lock_irqsave(hba->host->host_lock, flags); - /* - * In case you are here to cancel this work the gating state - * would be marked as REQ_CLKS_ON. In this case save time by - * skipping the gating work and exit after changing the clock - * state to CLKS_ON. - */ - if (hba->clk_gating.is_suspended || - (hba->clk_gating.state != REQ_CLKS_OFF)) { - hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), - hba->clk_gating.state); - goto rel_lock; - } + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { + /* + * In case you are here to cancel this work the gating state + * would be marked as REQ_CLKS_ON. In this case save time by + * skipping the gating work and exit after changing the clock + * state to CLKS_ON. + */ + if (hba->clk_gating.is_suspended || + hba->clk_gating.state != REQ_CLKS_OFF) { + hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(hba, + hba->clk_gating.state); + return; + } - if (ufshcd_is_ufs_dev_busy(hba) || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) - goto rel_lock; + if (hba->clk_gating.active_reqs) + return; + } - spin_unlock_irqrestore(hba->host->host_lock, flags); + scoped_guard(spinlock_irqsave, hba->host->host_lock) { + if (ufshcd_is_ufs_dev_busy(hba) || + hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + return; + } /* put the link into hibern8 mode before turning off clocks */ if (ufshcd_can_hibern8_during_gating(hba)) { @@ -1955,9 +1988,9 @@ static void ufshcd_gate_work(struct work_struct *work) hba->clk_gating.state = CLKS_ON; dev_err(hba->dev, "%s: hibern8 enter failed %d\n", __func__, ret); - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - goto out; + return; } ufshcd_set_link_hibern8(hba); } @@ -1977,35 +2010,36 @@ static void ufshcd_gate_work(struct work_struct *work) * prevent from doing cancel work multiple times when there are * new requests arriving before the current cancel work is done. */ - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); if (hba->clk_gating.state == REQ_CLKS_OFF) { hba->clk_gating.state = CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); } -rel_lock: - spin_unlock_irqrestore(hba->host->host_lock, flags); -out: - return; } -/* host lock must be held before calling this variant */ static void __ufshcd_release(struct ufs_hba *hba) { + lockdep_assert_held(&hba->clk_gating.lock); + if (!ufshcd_is_clkgating_allowed(hba)) return; hba->clk_gating.active_reqs--; if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || - hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || - hba->outstanding_tasks || !hba->clk_gating.is_initialized || - hba->active_uic_cmd || hba->uic_async_done || + !hba->clk_gating.is_initialized || hba->clk_gating.state == CLKS_OFF) return; + scoped_guard(spinlock_irqsave, hba->host->host_lock) { + if (ufshcd_has_pending_tasks(hba) || + hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + return; + } + hba->clk_gating.state = REQ_CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state); + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); queue_delayed_work(hba->clk_gating.clk_gating_workq, &hba->clk_gating.gate_work, msecs_to_jiffies(hba->clk_gating.delay_ms)); @@ -2013,11 +2047,8 @@ static void __ufshcd_release(struct ufs_hba *hba) void ufshcd_release(struct ufs_hba *hba) { - unsigned long flags; - - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); __ufshcd_release(hba); - spin_unlock_irqrestore(hba->host->host_lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_release); @@ -2032,11 +2063,9 @@ static ssize_t ufshcd_clkgate_delay_show(struct device *dev, void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) { struct ufs_hba *hba = dev_get_drvdata(dev); - unsigned long flags; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); hba->clk_gating.delay_ms = value; - spin_unlock_irqrestore(hba->host->host_lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); @@ -2064,7 +2093,6 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); - unsigned long flags; u32 value; if (kstrtou32(buf, 0, &value)) @@ -2072,9 +2100,10 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, value = !!value; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); + if (value == hba->clk_gating.is_enabled) - goto out; + return count; if (value) __ufshcd_release(hba); @@ -2082,8 +2111,7 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, hba->clk_gating.active_reqs++; hba->clk_gating.is_enabled = value; -out: - spin_unlock_irqrestore(hba->host->host_lock, flags); + return count; } @@ -2154,19 +2182,17 @@ static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) { bool queue_resume_work = false; ktime_t curr_t = ktime_get(); - unsigned long flags; if (!ufshcd_is_clkscaling_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + if (!hba->clk_scaling.active_reqs++) queue_resume_work = true; - if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) { - spin_unlock_irqrestore(hba->host->host_lock, flags); + if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) return; - } if (queue_resume_work) queue_work(hba->clk_scaling.workq, @@ -2182,18 +2208,17 @@ static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) hba->clk_scaling.busy_start_t = curr_t; hba->clk_scaling.is_busy_started = true; } - spin_unlock_irqrestore(hba->host->host_lock, flags); } static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) { struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; if (!ufshcd_is_clkscaling_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + hba->clk_scaling.active_reqs--; if (!scaling->active_reqs && scaling->is_busy_started) { scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), @@ -2201,7 +2226,6 @@ static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) scaling->busy_start_t = 0; scaling->is_busy_started = false; } - spin_unlock_irqrestore(hba->host->host_lock, flags); } static inline int ufshcd_monitor_opcode2dir(u8 opcode) @@ -2418,12 +2442,7 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) else hba->lsdb_sup = true; - if (!hba->mcq_sup) - return 0; - hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); - hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT, - hba->mcq_capabilities); return 0; } @@ -3118,8 +3137,13 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) case UPIU_TRANSACTION_QUERY_RSP: { u8 response = lrbp->ucd_rsp_ptr->header.response; - if (response == 0) + if (response == 0) { err = ufshcd_copy_query_response(hba, lrbp); + } else { + err = -EINVAL; + dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", + __func__, response); + } break; } case UPIU_TRANSACTION_REJECT_UPIU: @@ -3153,16 +3177,10 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, int err; retry: - time_left = wait_for_completion_timeout(hba->dev_cmd.complete, + time_left = wait_for_completion_timeout(&hba->dev_cmd.complete, time_left); if (likely(time_left)) { - /* - * The completion handler called complete() and the caller of - * this function still owns the @lrbp tag so the code below does - * not trigger any race conditions. - */ - hba->dev_cmd.complete = NULL; err = ufshcd_get_tr_ocs(lrbp, NULL); if (!err) err = ufshcd_dev_cmd_completion(hba, lrbp); @@ -3176,7 +3194,6 @@ retry: /* successfully cleared the command, retry if needed */ if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) err = -EAGAIN; - hba->dev_cmd.complete = NULL; return err; } @@ -3192,11 +3209,9 @@ retry: spin_lock_irqsave(&hba->outstanding_lock, flags); pending = test_bit(lrbp->task_tag, &hba->outstanding_reqs); - if (pending) { - hba->dev_cmd.complete = NULL; + if (pending) __clear_bit(lrbp->task_tag, &hba->outstanding_reqs); - } spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (!pending) { @@ -3214,8 +3229,6 @@ retry: spin_lock_irqsave(&hba->outstanding_lock, flags); pending = test_bit(lrbp->task_tag, &hba->outstanding_reqs); - if (pending) - hba->dev_cmd.complete = NULL; spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (!pending) { @@ -3249,13 +3262,9 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba) static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, const u32 tag, int timeout) { - DECLARE_COMPLETION_ONSTACK(wait); int err; - hba->dev_cmd.complete = &wait; - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - ufshcd_send_command(hba, tag, hba->dev_cmd_queue); err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); @@ -4014,7 +4023,7 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) * * Return: 0 on success, non-zero value on failure. */ -static int ufshcd_dme_reset(struct ufs_hba *hba) +int ufshcd_dme_reset(struct ufs_hba *hba) { struct uic_command uic_cmd = { .command = UIC_CMD_DME_RESET, @@ -4028,6 +4037,7 @@ static int ufshcd_dme_reset(struct ufs_hba *hba) return ret; } +EXPORT_SYMBOL_GPL(ufshcd_dme_reset); int ufshcd_dme_configure_adapt(struct ufs_hba *hba, int agreed_gear, @@ -4053,7 +4063,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); * * Return: 0 on success, non-zero value on failure. */ -static int ufshcd_dme_enable(struct ufs_hba *hba) +int ufshcd_dme_enable(struct ufs_hba *hba) { struct uic_command uic_cmd = { .command = UIC_CMD_DME_ENABLE, @@ -4067,6 +4077,7 @@ static int ufshcd_dme_enable(struct ufs_hba *hba) return ret; } +EXPORT_SYMBOL_GPL(ufshcd_dme_enable); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) { @@ -4431,7 +4442,7 @@ int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); - trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter", + trace_ufshcd_profile_hibern8(hba, "enter", ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) @@ -4456,7 +4467,7 @@ int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); - trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit", + trace_ufshcd_profile_hibern8(hba, "exit", ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) { @@ -4812,20 +4823,14 @@ EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); */ void ufshcd_hba_stop(struct ufs_hba *hba) { - unsigned long flags; int err; - /* - * Obtain the host lock to prevent that the controller is disabled - * while the UFS interrupt handler is active on another CPU. - */ - spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_disable_irq(hba); ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); - spin_unlock_irqrestore(hba->host->host_lock, flags); - err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, CONTROLLER_DISABLE, 10, 1); + ufshcd_enable_irq(hba); if (err) dev_err(hba->dev, "%s: Controller disable failed\n", __func__); } @@ -5195,12 +5200,12 @@ set_qdepth: } /** - * ufshcd_slave_alloc - handle initial SCSI device configurations + * ufshcd_sdev_init - handle initial SCSI device configurations * @sdev: pointer to SCSI device * * Return: success. */ -static int ufshcd_slave_alloc(struct scsi_device *sdev) +static int ufshcd_sdev_init(struct scsi_device *sdev) { struct ufs_hba *hba; @@ -5243,14 +5248,14 @@ static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) } /** - * ufshcd_device_configure - adjust SCSI device configurations + * ufshcd_sdev_configure - adjust SCSI device configurations * @sdev: pointer to SCSI device * @lim: queue limits * * Return: 0 (success). */ -static int ufshcd_device_configure(struct scsi_device *sdev, - struct queue_limits *lim) +static int ufshcd_sdev_configure(struct scsi_device *sdev, + struct queue_limits *lim) { struct ufs_hba *hba = shost_priv(sdev->host); struct request_queue *q = sdev->request_queue; @@ -5281,10 +5286,10 @@ static int ufshcd_device_configure(struct scsi_device *sdev, } /** - * ufshcd_slave_destroy - remove SCSI device configurations + * ufshcd_sdev_destroy - remove SCSI device configurations * @sdev: pointer to SCSI device */ -static void ufshcd_slave_destroy(struct scsi_device *sdev) +static void ufshcd_sdev_destroy(struct scsi_device *sdev) { struct ufs_hba *hba; unsigned long flags; @@ -5566,12 +5571,12 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, ufshcd_release_scsi_cmd(hba, lrbp); /* Do not touch lrbp after scsi done */ scsi_done(cmd); - } else if (hba->dev_cmd.complete) { + } else { if (cqe) { ocs = le32_to_cpu(cqe->status) & MASK_OCS; lrbp->utr_descriptor_ptr->header.ocs = ocs; } - complete(hba->dev_cmd.complete); + complete(&hba->dev_cmd.complete); } } @@ -5673,6 +5678,8 @@ static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, continue; hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); + if (!hwq) + continue; if (force_compl) { ufshcd_mcq_compl_all_cqes_lock(hba, hwq); @@ -5823,7 +5830,7 @@ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = true; - trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled"); + trace_ufshcd_auto_bkops_state(hba, "Enabled"); /* No need of URGENT_BKOPS exception from the device */ err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); @@ -5874,7 +5881,7 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = false; - trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled"); + trace_ufshcd_auto_bkops_state(hba, "Disabled"); hba->is_urgent_bkops_lvl_checked = false; out: return err; @@ -5994,22 +6001,40 @@ out: __func__, err); } -static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status) +int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) { - u32 value; + struct utp_upiu_query_v4_0 *upiu_resp; + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + int err; - if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, - QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value)) - return; + if (hba->dev_info.wspecversion < 0x410) + return -EOPNOTSUPP; - dev_info(hba->dev, "exception Tcase %d\n", value - 80); + ufshcd_hold(hba); + mutex_lock(&hba->dev_cmd.lock); - ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); + ufshcd_init_query(hba, &request, &response, + UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); - /* - * A placeholder for the platform vendors to add whatever additional - * steps required - */ + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + + if (err) { + dev_err(hba->dev, "%s: failed to read device level exception %d\n", + __func__, err); + goto out; + } + + upiu_resp = (struct utp_upiu_query_v4_0 *)response; + *exception_id = get_unaligned_be64(&upiu_resp->osf3); +out: + mutex_unlock(&hba->dev_cmd.lock); + ufshcd_release(hba); + + return err; } static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) @@ -6082,7 +6107,7 @@ int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) return ret; } -static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, +static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, u32 avail_buf) { u32 cur_buf; @@ -6164,15 +6189,13 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) } /* - * The ufs device needs the vcc to be ON to flush. * With user-space reduction enabled, it's enough to enable flush * by checking only the available buffer. The threshold * defined here is > 90% full. * With user-space preserved enabled, the current-buffer * should be checked too because the wb buffer size can reduce * when disk tends to be full. This info is provided by current - * buffer (dCurrentWriteBoosterBufferSize). There's no point in - * keeping vcc on when current buffer is empty. + * buffer (dCurrentWriteBoosterBufferSize). */ index = ufshcd_wb_get_query_index(hba); ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, @@ -6187,7 +6210,7 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) if (!hba->dev_info.b_presrv_uspc_en) return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); - return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf); + return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); } static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) @@ -6226,13 +6249,23 @@ static void ufshcd_exception_event_handler(struct work_struct *work) return; } - trace_ufshcd_exception_event(dev_name(hba->dev), status); + trace_ufshcd_exception_event(hba, status); if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) ufshcd_bkops_exception_event_handler(hba); if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) - ufshcd_temp_exception_event_handler(hba, status); + ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); + + if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { + hba->critical_health_count++; + sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); + } + + if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { + atomic_inc(&hba->dev_lvl_exception_count); + sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); + } ufs_debugfs_exception_event(hba, status); } @@ -7685,7 +7718,7 @@ static void ufshcd_process_probe_result(struct ufs_hba *hba, hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; spin_unlock_irqrestore(hba->host->host_lock, flags); - trace_ufshcd_init(dev_name(hba->dev), ret, + trace_ufshcd_init(hba, ret, ktime_to_us(ktime_sub(ktime_get(), probe_start)), hba->curr_dev_pwr_mode, hba->uic_link_state); } @@ -8133,29 +8166,20 @@ static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) } } -static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf) +static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) { - struct ufs_dev_info *dev_info = &hba->dev_info; u32 ext_ufs_feature; - u32 ext_iid_en = 0; - int err; - /* Only UFS-4.0 and above may support EXT_IID */ - if (dev_info->wspecversion < 0x400) - goto out; + if (hba->dev_info.wspecversion < 0x410) + return; ext_ufs_feature = get_unaligned_be32(desc_buf + - DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); - if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP)) - goto out; - - err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, - QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en); - if (err) - dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err); + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); + if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) + return; -out: - dev_info->b_ext_iid_en = ext_iid_en; + atomic_set(&hba->dev_lvl_exception_count, 0); + ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); } static void ufshcd_set_rtt(struct ufs_hba *hba) @@ -8259,7 +8283,9 @@ static void ufshcd_rtc_work(struct work_struct *work) hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); /* Update RTC only when there are no requests in progress and UFSHCI is operational */ - if (!ufshcd_is_ufs_dev_busy(hba) && hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) + if (!ufshcd_is_ufs_dev_busy(hba) && + hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && + !hba->clk_gating.active_reqs) ufshcd_update_rtc(hba); if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) @@ -8349,10 +8375,14 @@ static int ufs_get_device_desc(struct ufs_hba *hba) ufshcd_temp_notif_probe(hba, desc_buf); + if (dev_info->wspecversion >= 0x410) { + hba->critical_health_count = 0; + ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); + } + ufs_init_rtc(hba, desc_buf); - if (hba->ext_iid_sup) - ufshcd_ext_iid_probe(hba, desc_buf); + ufshcd_device_lvl_exception_probe(hba, desc_buf); /* * ufshcd_read_string_desc returns size of the string @@ -8443,6 +8473,31 @@ out: return ret; } +/** + * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. + * @hba: per-adapter instance + * + * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter + * to ensure proper hibernation timing. This function retrieves the current + * PA_HIBERN8TIME value and increments it by 100us. + */ +static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) +{ + u32 pa_h8time; + int ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); + if (ret) { + dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); + return; + } + + /* Increment by 1 to increase hibernation time by 100 µs */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); + if (ret) + dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); +} + static void ufshcd_tune_unipro_params(struct ufs_hba *hba) { ufshcd_vops_apply_dev_quirks(hba); @@ -8453,6 +8508,9 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) ufshcd_quirk_tune_host_pa_tactivate(hba); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) + ufshcd_quirk_override_pa_h8time(hba); } static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) @@ -8858,7 +8916,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) ufshcd_device_reset(hba); ufs_put_device_desc(hba); ufshcd_hba_stop(hba); - ufshcd_vops_reinit_notify(hba); ret = ufshcd_hba_enable(hba); if (ret) { dev_err(hba->dev, "Host controller enable failed\n"); @@ -8942,7 +8999,7 @@ static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", __func__, hba->outstanding_tasks); - return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; + return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; } static const struct attribute_group *ufshcd_driver_groups[] = { @@ -8968,9 +9025,9 @@ static const struct scsi_host_template ufshcd_driver_template = { .map_queues = ufshcd_map_queues, .queuecommand = ufshcd_queuecommand, .mq_poll = ufshcd_poll, - .slave_alloc = ufshcd_slave_alloc, - .device_configure = ufshcd_device_configure, - .slave_destroy = ufshcd_slave_destroy, + .sdev_init = ufshcd_sdev_init, + .sdev_configure = ufshcd_sdev_configure, + .sdev_destroy = ufshcd_sdev_destroy, .change_queue_depth = ufshcd_change_queue_depth, .eh_abort_handler = ufshcd_abort, .eh_device_reset_handler = ufshcd_eh_device_reset_handler, @@ -9156,7 +9213,6 @@ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) int ret = 0; struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; - unsigned long flags; ktime_t start = ktime_get(); bool clk_state_changed = false; @@ -9206,16 +9262,15 @@ out: if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) clk_disable_unprepare(clki->clk); } - } else if (!ret && on) { - spin_lock_irqsave(hba->host->host_lock, flags); - hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + } else if (!ret && on && hba->clk_gating.is_initialized) { + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) + hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - spin_unlock_irqrestore(hba->host->host_lock, flags); } if (clk_state_changed) - trace_ufshcd_profile_clk_gating(dev_name(hba->dev), + trace_ufshcd_profile_clk_gating(hba, (on ? "on" : "off"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; @@ -9915,7 +9970,7 @@ static int ufshcd_wl_runtime_suspend(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); - trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret, + trace_ufshcd_wl_runtime_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9935,7 +9990,7 @@ static int ufshcd_wl_runtime_resume(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); - trace_ufshcd_wl_runtime_resume(dev_name(dev), ret, + trace_ufshcd_wl_runtime_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9967,7 +10022,7 @@ static int ufshcd_wl_suspend(struct device *dev) out: if (!ret) hba->is_sys_suspended = true; - trace_ufshcd_wl_suspend(dev_name(dev), ret, + trace_ufshcd_wl_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9990,7 +10045,7 @@ static int ufshcd_wl_resume(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); out: - trace_ufshcd_wl_resume(dev_name(dev), ret, + trace_ufshcd_wl_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) @@ -10028,7 +10083,7 @@ static int ufshcd_suspend(struct ufs_hba *hba) } if (ufshcd_is_clkgating_allowed(hba)) { hba->clk_gating.state = CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); } @@ -10101,7 +10156,7 @@ int ufshcd_system_suspend(struct device *dev) ret = ufshcd_suspend(hba); out: - trace_ufshcd_system_suspend(dev_name(hba->dev), ret, + trace_ufshcd_system_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -10129,7 +10184,7 @@ int ufshcd_system_resume(struct device *dev) ret = ufshcd_resume(hba); out: - trace_ufshcd_system_resume(dev_name(hba->dev), ret, + trace_ufshcd_system_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -10155,7 +10210,7 @@ int ufshcd_runtime_suspend(struct device *dev) ret = ufshcd_suspend(hba); - trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret, + trace_ufshcd_runtime_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -10182,7 +10237,7 @@ int ufshcd_runtime_resume(struct device *dev) ret = ufshcd_resume(hba); - trace_ufshcd_runtime_resume(dev_name(hba->dev), ret, + trace_ufshcd_runtime_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -10294,16 +10349,6 @@ EXPORT_SYMBOL_GPL(ufshcd_system_thaw); #endif /* CONFIG_PM_SLEEP */ /** - * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA) - * @hba: pointer to Host Bus Adapter (HBA) - */ -void ufshcd_dealloc_host(struct ufs_hba *hba) -{ - scsi_host_put(hba->host); -} -EXPORT_SYMBOL_GPL(ufshcd_dealloc_host); - -/** * ufshcd_set_dma_mask - Set dma mask based on the controller * addressing capability * @hba: per adapter instance @@ -10322,11 +10367,25 @@ static int ufshcd_set_dma_mask(struct ufs_hba *hba) } /** + * ufshcd_devres_release - devres cleanup handler, invoked during release of + * hba->dev + * @host: pointer to SCSI host + */ +static void ufshcd_devres_release(void *host) +{ + scsi_host_put(host); +} + +/** * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) * @dev: pointer to device handle * @hba_handle: driver private handle * * Return: 0 on success, non-zero value on failure. + * + * NOTE: There is no corresponding ufshcd_dealloc_host() because this function + * keeps track of its allocations using devres and deallocates everything on + * device removal automatically. */ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) { @@ -10348,6 +10407,13 @@ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) err = -ENOMEM; goto out_error; } + + err = devm_add_action_or_reset(dev, ufshcd_devres_release, + host); + if (err) + return dev_err_probe(dev, err, + "failed to add ufshcd dealloc action\n"); + host->nr_maps = HCTX_TYPE_POLL + 1; hba = shost_priv(host); hba->host = host; @@ -10412,7 +10478,6 @@ static int ufshcd_add_scsi_host(struct ufs_hba *hba) .nr_hw_queues = 1, .queue_depth = hba->nutmrs, .ops = &ufshcd_tmf_ops, - .flags = BLK_MQ_F_NO_SCHED, }; err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); if (err < 0) @@ -10477,6 +10542,29 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) hba->irq = irq; hba->vps = &ufs_hba_vps; + /* + * Initialize clk_gating.lock early since it is being used in + * ufshcd_setup_clocks() + */ + spin_lock_init(&hba->clk_gating.lock); + + /* + * Set the default power management level for runtime and system PM. + * Host controller drivers can override them in their + * 'ufs_hba_variant_ops::init' callback. + * + * Default power saving mode is to keep UFS link in Hibern8 state + * and UFS device in sleep state. + */ + hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + + init_completion(&hba->dev_cmd.complete); + err = ufshcd_hba_init(hba); if (err) goto out_error; @@ -10590,18 +10678,6 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) goto out_disable; } - /* - * Set the default power management level for runtime and system PM. - * Default power saving mode is to keep UFS link in Hibern8 state - * and UFS device in sleep state. - */ - hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( - UFS_SLEEP_PWR_MODE, - UIC_LINK_HIBERN8_STATE); - hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( - UFS_SLEEP_PWR_MODE, - UIC_LINK_HIBERN8_STATE); - INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); |