projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: hisi_sas: Move debugfs code to v3 hw driver
[linux-2.6-block.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
67351119 1// SPDX-License-Identifier: GPL-2.0-or-later
7a3e97b0 2/*
e0eca63e 3 * Universal Flash Storage Host controller driver Core
3b1d0580 4 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 6 *
3b1d0580
VH		 7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 10 */
11
6ccf44fe 12#include <linux/async.h>
856b3483 13#include <linux/devfreq.h>
b573d484 14#include <linux/nls.h>
54b879b7 15#include <linux/of.h>
ad448378 16#include <linux/bitfield.h>
fb276f77 17#include <linux/blk-pm.h>
c72e79c0 18#include <linux/blkdev.h>
e0eca63e 19#include "ufshcd.h"
c58ab7aa 20#include "ufs_quirks.h"
53b3d9c3 21#include "unipro.h"
cbb6813e 22#include "ufs-sysfs.h"
df032bf2 23#include "ufs_bsg.h"
df043c74 24#include "ufshcd-crypto.h"
3d17b9b5
AD		 25#include <asm/unaligned.h>
26#include <linux/blkdev.h>
7a3e97b0 27
7ff5ab47 28#define CREATE_TRACE_POINTS
29#include <trace/events/ufs.h>
30
2fbd009b
SJ		 31#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
32 UTP_TASK_REQ_COMPL |\
33 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 34/* UIC command timeout, unit: ms */
35#define UIC_CMD_TIMEOUT 500
2fbd009b 36
5a0b0cb9
SRT		 37/* NOP OUT retries waiting for NOP IN response */
38#define NOP_OUT_RETRIES 10
782e2efb
DP		 39/* Timeout after 50 msecs if NOP OUT hangs without response */
40#define NOP_OUT_TIMEOUT 50 /* msecs */
5a0b0cb9 41
68078d5c 42/* Query request retries */
10fe5888 43#define QUERY_REQ_RETRIES 3
68078d5c 44/* Query request timeout */
10fe5888 45#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 46
e2933132
SRT		 47/* Task management command timeout */
48#define TM_CMD_TIMEOUT 100 /* msecs */
49
64238fbd
YG		 50/* maximum number of retries for a general UIC command */
51#define UFS_UIC_COMMAND_RETRIES 3
52
1d337ec2
SRT		 53/* maximum number of link-startup retries */
54#define DME_LINKSTARTUP_RETRIES 3
55
87d0b4a6
YG		 56/* Maximum retries for Hibern8 enter */
57#define UIC_HIBERN8_ENTER_RETRIES 3
58
1d337ec2
SRT		 59/* maximum number of reset retries before giving up */
60#define MAX_HOST_RESET_RETRIES 5
61
68078d5c
DR		 62/* Expose the flag value from utp_upiu_query.value */
63#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
64
7d568652
SJ		 65/* Interrupt aggregation default timeout, unit: 40us */
66#define INT_AGGR_DEF_TO 0x02
67
49615ba1
SC		 68/* default delay of autosuspend: 2000 ms */
69#define RPM_AUTOSUSPEND_DELAY_MS 2000
70
51dd905b
SC		 71/* Default delay of RPM device flush delayed work */
72#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
73
09f17791
CG		 74/* Default value of wait time before gating device ref clock */
75#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
76
29707fab
KK		 77/* Polling time to wait for fDeviceInit */
78#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
79
aa497613
SRT		 80#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
81 ({ \
82 int _ret; \
83 if (_on) \
84 _ret = ufshcd_enable_vreg(_dev, _vreg); \
85 else \
86 _ret = ufshcd_disable_vreg(_dev, _vreg); \
87 _ret; \
88 })
89
ba80917d
TW		 90#define ufshcd_hex_dump(prefix_str, buf, len) do { \
91 size_t __len = (len); \
92 print_hex_dump(KERN_ERR, prefix_str, \
93 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
94 16, 4, buf, __len, false); \
95} while (0)
96
97int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
98 const char *prefix)
99{
d6724756
MG		 100 u32 *regs;
101 size_t pos;
102
103 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
104 return -EINVAL;
ba80917d 105
cddaebaf 106 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 107 if (!regs)
108 return -ENOMEM;
109
d6724756
MG		 110 for (pos = 0; pos < len; pos += 4)
111 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
112
ba80917d
TW		 113 ufshcd_hex_dump(prefix, regs, len);
114 kfree(regs);
115
116 return 0;
117}
118EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 119
7a3e97b0
SY		 120enum {
121 UFSHCD_MAX_CHANNEL = 0,
122 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 123 UFSHCD_CMD_PER_LUN = 32,
124 UFSHCD_CAN_QUEUE = 32,
125};
126
127/* UFSHCD states */
128enum {
7a3e97b0
SY		 129 UFSHCD_STATE_RESET,
130 UFSHCD_STATE_ERROR,
3441da7d 131 UFSHCD_STATE_OPERATIONAL,
5586dd8e
CG		 132 UFSHCD_STATE_EH_SCHEDULED_FATAL,
133 UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
3441da7d
SRT		 134};
135
136/* UFSHCD error handling flags */
137enum {
138 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 139};
140
e8e7f271
SRT		 141/* UFSHCD UIC layer error flags */
142enum {
143 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 144 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
145 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
146 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
147 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
148 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
2355b66e 149 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
e8e7f271
SRT		 150};
151
3441da7d 152#define ufshcd_set_eh_in_progress(h) \
9c490d2d 153 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 154#define ufshcd_eh_in_progress(h) \
9c490d2d 155 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 156#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 157 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 158
cbb6813e 159struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 160 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
161 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
162 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
163 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
164 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
165 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
fe1d4c2e
AH		 166 /*
167 * For DeepSleep, the link is first put in hibern8 and then off.
168 * Leaving the link in hibern8 is not supported.
169 */
170 {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
57d104c1
SJ		 171};
172
173static inline enum ufs_dev_pwr_mode
174ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
175{
176 return ufs_pm_lvl_states[lvl].dev_state;
177}
178
179static inline enum uic_link_state
180ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
181{
182 return ufs_pm_lvl_states[lvl].link_state;
183}
184
0c8f7586 185static inline enum ufs_pm_level
186ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
187 enum uic_link_state link_state)
188{
189 enum ufs_pm_level lvl;
190
191 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
192 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
193 (ufs_pm_lvl_states[lvl].link_state == link_state))
194 return lvl;
195 }
196
197 /* if no match found, return the level 0 */
198 return UFS_PM_LVL_0;
199}
200
56d4a186
SJ		 201static struct ufs_dev_fix ufs_fixups[] = {
202 /* UFS cards deviations table */
c0a18ee0
SC		 203 UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
204 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186 205 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
ed0b40ff
SC		 206 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
207 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
56d4a186 208 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
ed0b40ff
SC		 209 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
210 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
211 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
212 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 213 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
214 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
215 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
216 UFS_DEVICE_QUIRK_PA_TACTIVATE),
217 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
218 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 219 END_FIX
220};
221
9333d775 222static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 223static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 224static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 225static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 226static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2 227static void ufshcd_hba_exit(struct ufs_hba *hba);
1b9e2141 228static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
1ab27c9c
ST		 229static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
230 bool skip_ref_clk);
231static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
1ab27c9c 232static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 233static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 234static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 235static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
236static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 237static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 238static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 239static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 240static int ufshcd_change_power_mode(struct ufs_hba *hba,
241 struct ufs_pa_layer_attr *pwr_mode);
4db7a236 242static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
c72e79c0
CG		 243static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
244static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
245static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
246 struct ufs_vreg *vreg);
307348f6 247static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
3d17b9b5
AD		 248static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba);
249static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba);
250static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable);
251static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
252static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
dd7143e2
CG		 253static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
254static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
3d17b9b5 255
14497328
YG		 256static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
257{
258 return tag >= 0 && tag < hba->nutrs;
259}
57d104c1 260
5231d38c 261static inline void ufshcd_enable_irq(struct ufs_hba *hba)
57d104c1 262{
57d104c1 263 if (!hba->is_irq_enabled) {
5231d38c 264 enable_irq(hba->irq);
57d104c1
SJ		 265 hba->is_irq_enabled = true;
266 }
57d104c1
SJ		 267}
268
269static inline void ufshcd_disable_irq(struct ufs_hba *hba)
270{
271 if (hba->is_irq_enabled) {
5231d38c 272 disable_irq(hba->irq);
57d104c1
SJ		 273 hba->is_irq_enabled = false;
274 }
275}
3441da7d 276
3d17b9b5
AD		 277static inline void ufshcd_wb_config(struct ufs_hba *hba)
278{
279 int ret;
280
79e3520f 281 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 282 return;
283
284 ret = ufshcd_wb_ctrl(hba, true);
285 if (ret)
286 dev_err(hba->dev, "%s: Enable WB failed: %d\n", __func__, ret);
287 else
288 dev_info(hba->dev, "%s: Write Booster Configured\n", __func__);
289 ret = ufshcd_wb_toggle_flush_during_h8(hba, true);
290 if (ret)
291 dev_err(hba->dev, "%s: En WB flush during H8: failed: %d\n",
292 __func__, ret);
293 ufshcd_wb_toggle_flush(hba, true);
294}
295
38135535
SJ		 296static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
297{
298 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
299 scsi_unblock_requests(hba->host);
300}
301
302static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
303{
304 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
305 scsi_block_requests(hba->host);
306}
307
6667e6d9
OS		 308static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
309 const char *str)
310{
311 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
312
313 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
314}
315
316static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
317 const char *str)
318{
319 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
320
321 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
322}
323
324static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
325 const char *str)
326{
6667e6d9 327 int off = (int)tag - hba->nutrs;
391e388f 328 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 329
391e388f
CH		 330 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
331 &descp->input_param1);
6667e6d9
OS		 332}
333
aa5c6979
SC		 334static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
335 struct uic_command *ucmd,
336 const char *str)
337{
338 u32 cmd;
339
340 if (!trace_ufshcd_uic_command_enabled())
341 return;
342
343 if (!strcmp(str, "send"))
344 cmd = ucmd->command;
345 else
346 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
347
348 trace_ufshcd_uic_command(dev_name(hba->dev), str, cmd,
349 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
350 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
351 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
352}
353
1a07f2d9
LS		 354static void ufshcd_add_command_trace(struct ufs_hba *hba,
355 unsigned int tag, const char *str)
356{
357 sector_t lba = -1;
69a314d6 358 u8 opcode = 0, group_id = 0;
1a07f2d9 359 u32 intr, doorbell;
e7c3b379 360 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
e4d2add7 361 struct scsi_cmnd *cmd = lrbp->cmd;
1a07f2d9
LS		 362 int transfer_len = -1;
363
e7c3b379
OS		 364 if (!trace_ufshcd_command_enabled()) {
365 /* trace UPIU W/O tracing command */
e4d2add7 366 if (cmd)
e7c3b379 367 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9 368 return;
e7c3b379 369 }
1a07f2d9 370
e4d2add7 371 if (cmd) { /* data phase exists */
e7c3b379
OS		 372 /* trace UPIU also */
373 ufshcd_add_cmd_upiu_trace(hba, tag, str);
e4d2add7 374 opcode = cmd->cmnd[0];
1a07f2d9
LS		 375 if ((opcode == READ_10) || (opcode == WRITE_10)) {
376 /*
377 * Currently we only fully trace read(10) and write(10)
378 * commands
379 */
e4d2add7
BVA		 380 if (cmd->request && cmd->request->bio)
381 lba = cmd->request->bio->bi_iter.bi_sector;
1a07f2d9
LS		 382 transfer_len = be32_to_cpu(
383 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
69a314d6
JK		 384 if (opcode == WRITE_10)
385 group_id = lrbp->cmd->cmnd[6];
3754cde8
LL		 386 } else if (opcode == UNMAP) {
387 if (cmd->request) {
388 lba = scsi_get_lba(cmd);
389 transfer_len = blk_rq_bytes(cmd->request);
390 }
1a07f2d9
LS		 391 }
392 }
393
394 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
395 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
396 trace_ufshcd_command(dev_name(hba->dev), str, tag,
69a314d6 397 doorbell, transfer_len, intr, lba, opcode, group_id);
1a07f2d9
LS		 398}
399
ff8e20c6
DR		 400static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
401{
402 struct ufs_clk_info *clki;
403 struct list_head *head = &hba->clk_list_head;
404
566ec9ad 405 if (list_empty(head))
ff8e20c6
DR		 406 return;
407
408 list_for_each_entry(clki, head, list) {
409 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
410 clki->max_freq)
411 dev_err(hba->dev, "clk: %s, rate: %u\n",
412 clki->name, clki->curr_freq);
413 }
414}
415
48d5b973
SC		 416static void ufshcd_print_err_hist(struct ufs_hba *hba,
417 struct ufs_err_reg_hist *err_hist,
418 char *err_name)
ff8e20c6
DR		 419{
420 int i;
27752647 421 bool found = false;
ff8e20c6 422
48d5b973
SC		 423 for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
424 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
ff8e20c6 425
645728a6 426 if (err_hist->tstamp[p] == 0)
ff8e20c6 427 continue;
c5397f13 428 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
ff8e20c6 429 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
27752647 430 found = true;
ff8e20c6 431 }
27752647
SC		 432
433 if (!found)
fd1fb4d5 434 dev_err(hba->dev, "No record of %s\n", err_name);
ff8e20c6
DR		 435}
436
66cc820f
DR		 437static void ufshcd_print_host_regs(struct ufs_hba *hba)
438{
ba80917d 439 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
ff8e20c6 440
48d5b973
SC		 441 ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
442 ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
443 ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
444 ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
445 ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
d3c615bf
SC		 446 ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
447 "auto_hibern8_err");
8808b4e9
SC		 448 ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
449 ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
450 "link_startup_fail");
451 ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
452 ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
453 "suspend_fail");
454 ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
455 ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
456 ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
ff8e20c6 457
7c486d91 458 ufshcd_vops_dbg_register_dump(hba);
66cc820f
DR		 459}
460
461static
462void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
463{
464 struct ufshcd_lrb *lrbp;
7fabb77b 465 int prdt_length;
66cc820f
DR		 466 int tag;
467
468 for_each_set_bit(tag, &bitmap, hba->nutrs) {
469 lrbp = &hba->lrb[tag];
470
ff8e20c6
DR		 471 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
472 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 473 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
474 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 475 dev_err(hba->dev,
476 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
477 tag, (u64)lrbp->utrd_dma_addr);
478
66cc820f
DR		 479 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
480 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 481 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
482 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 483 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
484 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 485 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
486 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 487 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
488 sizeof(struct utp_upiu_rsp));
66cc820f 489
7fabb77b
GB		 490 prdt_length = le16_to_cpu(
491 lrbp->utr_descriptor_ptr->prd_table_length);
cc770ce3
EB		 492 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
493 prdt_length /= sizeof(struct ufshcd_sg_entry);
494
7fabb77b
GB		 495 dev_err(hba->dev,
496 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
497 tag, prdt_length,
498 (u64)lrbp->ucd_prdt_dma_addr);
499
500 if (pr_prdt)
66cc820f 501 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 502 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 503 }
504}
505
506static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
507{
66cc820f
DR		 508 int tag;
509
510 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 511 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
512
66cc820f 513 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 514 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 515 }
516}
517
6ba65588
GB		 518static void ufshcd_print_host_state(struct ufs_hba *hba)
519{
3f8af604
CG		 520 struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
521
6ba65588 522 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 523 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
524 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 525 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
526 hba->saved_err, hba->saved_uic_err);
527 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
528 hba->curr_dev_pwr_mode, hba->uic_link_state);
529 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
530 hba->pm_op_in_progress, hba->is_sys_suspended);
531 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
532 hba->auto_bkops_enabled, hba->host->host_self_blocked);
533 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
3f8af604
CG		 534 dev_err(hba->dev,
535 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
536 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
537 hba->ufs_stats.hibern8_exit_cnt);
538 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
539 ktime_to_us(hba->ufs_stats.last_intr_ts),
540 hba->ufs_stats.last_intr_status);
6ba65588
GB		 541 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
542 hba->eh_flags, hba->req_abort_count);
3f8af604
CG		 543 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
544 hba->ufs_version, hba->capabilities, hba->caps);
6ba65588
GB		 545 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
546 hba->dev_quirks);
3f8af604
CG		 547 if (sdev_ufs)
548 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
549 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
550
551 ufshcd_print_clk_freqs(hba);
6ba65588
GB		 552}
553
ff8e20c6
DR		 554/**
555 * ufshcd_print_pwr_info - print power params as saved in hba
556 * power info
557 * @hba: per-adapter instance
558 */
559static void ufshcd_print_pwr_info(struct ufs_hba *hba)
560{
561 static const char * const names[] = {
562 "INVALID MODE",
563 "FAST MODE",
564 "SLOW_MODE",
565 "INVALID MODE",
566 "FASTAUTO_MODE",
567 "SLOWAUTO_MODE",
568 "INVALID MODE",
569 };
570
571 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
572 __func__,
573 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
574 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
575 names[hba->pwr_info.pwr_rx],
576 names[hba->pwr_info.pwr_tx],
577 hba->pwr_info.hs_rate);
578}
579
5c955c10
SC		 580void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
581{
582 if (!us)
583 return;
584
585 if (us < 10)
586 udelay(us);
587 else
588 usleep_range(us, us + tolerance);
589}
590EXPORT_SYMBOL_GPL(ufshcd_delay_us);
591
5cac1095 592/**
5a0b0cb9 593 * ufshcd_wait_for_register - wait for register value to change
5cac1095
BVA		 594 * @hba: per-adapter interface
595 * @reg: mmio register offset
596 * @mask: mask to apply to the read register value
597 * @val: value to wait for
598 * @interval_us: polling interval in microseconds
599 * @timeout_ms: timeout in milliseconds
5a0b0cb9 600 *
5cac1095
BVA		 601 * Return:
602 * -ETIMEDOUT on error, zero on success.
5a0b0cb9 603 */
596585a2
YG		 604int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
605 u32 val, unsigned long interval_us,
5cac1095 606 unsigned long timeout_ms)
5a0b0cb9
SRT		 607{
608 int err = 0;
609 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
610
611 /* ignore bits that we don't intend to wait on */
612 val = val & mask;
613
614 while ((ufshcd_readl(hba, reg) & mask) != val) {
5cac1095 615 usleep_range(interval_us, interval_us + 50);
5a0b0cb9
SRT		 616 if (time_after(jiffies, timeout)) {
617 if ((ufshcd_readl(hba, reg) & mask) != val)
618 err = -ETIMEDOUT;
619 break;
620 }
621 }
622
623 return err;
624}
625
2fbd009b
SJ		 626/**
627 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 628 * @hba: Pointer to adapter instance
2fbd009b
SJ		 629 *
630 * Returns interrupt bit mask per version
631 */
632static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
633{
c01848c6
YG		 634 u32 intr_mask = 0;
635
636 switch (hba->ufs_version) {
637 case UFSHCI_VERSION_10:
638 intr_mask = INTERRUPT_MASK_ALL_VER_10;
639 break;
c01848c6
YG		 640 case UFSHCI_VERSION_11:
641 case UFSHCI_VERSION_20:
642 intr_mask = INTERRUPT_MASK_ALL_VER_11;
643 break;
c01848c6
YG		 644 case UFSHCI_VERSION_21:
645 default:
646 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 647 break;
c01848c6
YG		 648 }
649
650 return intr_mask;
2fbd009b
SJ		 651}
652
7a3e97b0
SY		 653/**
654 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 655 * @hba: Pointer to adapter instance
7a3e97b0
SY		 656 *
657 * Returns UFSHCI version supported by the controller
658 */
659static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
660{
0263bcd0
YG		 661 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
662 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 663
b873a275 664 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 665}
666
667/**
668 * ufshcd_is_device_present - Check if any device connected to
669 * the host controller
5c0c28a8 670 * @hba: pointer to adapter instance
7a3e97b0 671 *
c9e6010b 672 * Returns true if device present, false if no device detected
7a3e97b0 673 */
c9e6010b 674static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 675{
5c0c28a8 676 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 677 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 678}
679
680/**
681 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 682 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 683 *
684 * This function is used to get the OCS field from UTRD
685 * Returns the OCS field in the UTRD
686 */
687static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
688{
e8c8e82a 689 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 690}
691
7a3e97b0
SY		 692/**
693 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
694 * @hba: per adapter instance
695 * @pos: position of the bit to be cleared
696 */
697static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
698{
87183841
AA		 699 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
700 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
701 else
702 ufshcd_writel(hba, ~(1 << pos),
703 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
1399c5b0
AA		 704}
705
706/**
707 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
708 * @hba: per adapter instance
709 * @pos: position of the bit to be cleared
710 */
711static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
712{
87183841
AA		 713 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
714 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
715 else
716 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 717}
718
a48353f6
YG		 719/**
720 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
721 * @hba: per adapter instance
722 * @tag: position of the bit to be cleared
723 */
724static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
725{
726 __clear_bit(tag, &hba->outstanding_reqs);
727}
728
7a3e97b0
SY		 729/**
730 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
731 * @reg: Register value of host controller status
732 *
733 * Returns integer, 0 on Success and positive value if failed
734 */
735static inline int ufshcd_get_lists_status(u32 reg)
736{
6cf16115 737 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 738}
739
740/**
741 * ufshcd_get_uic_cmd_result - Get the UIC command result
742 * @hba: Pointer to adapter instance
743 *
744 * This function gets the result of UIC command completion
745 * Returns 0 on success, non zero value on error
746 */
747static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
748{
b873a275 749 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 750 MASK_UIC_COMMAND_RESULT;
751}
752
12b4fdb4
SJ		 753/**
754 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
755 * @hba: Pointer to adapter instance
756 *
757 * This function gets UIC command argument3
758 * Returns 0 on success, non zero value on error
759 */
760static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
761{
762 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
763}
764
7a3e97b0 765/**
5a0b0cb9 766 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 767 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 768 */
769static inline int
5a0b0cb9 770ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 771{
5a0b0cb9 772 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 773}
774
775/**
776 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
777 * @ucd_rsp_ptr: pointer to response UPIU
778 *
779 * This function gets the response status and scsi_status from response UPIU
780 * Returns the response result code.
781 */
782static inline int
783ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
784{
785 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
786}
787
1c2623c5
SJ		 788/*
789 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
790 * from response UPIU
791 * @ucd_rsp_ptr: pointer to response UPIU
792 *
793 * Return the data segment length.
794 */
795static inline unsigned int
796ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
797{
798 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
799 MASK_RSP_UPIU_DATA_SEG_LEN;
800}
801
66ec6d59
SRT		 802/**
803 * ufshcd_is_exception_event - Check if the device raised an exception event
804 * @ucd_rsp_ptr: pointer to response UPIU
805 *
806 * The function checks if the device raised an exception event indicated in
807 * the Device Information field of response UPIU.
808 *
809 * Returns true if exception is raised, false otherwise.
810 */
811static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
812{
813 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
814 MASK_RSP_EXCEPTION_EVENT ? true : false;
815}
816
7a3e97b0 817/**
7d568652 818 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 819 * @hba: per adapter instance
7a3e97b0
SY		 820 */
821static inline void
7d568652 822ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 823{
7d568652
SJ		 824 ufshcd_writel(hba, INT_AGGR_ENABLE |
825 INT_AGGR_COUNTER_AND_TIMER_RESET,
826 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
827}
828
829/**
830 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
831 * @hba: per adapter instance
832 * @cnt: Interrupt aggregation counter threshold
833 * @tmout: Interrupt aggregation timeout value
834 */
835static inline void
836ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
837{
838 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
839 INT_AGGR_COUNTER_THLD_VAL(cnt) |
840 INT_AGGR_TIMEOUT_VAL(tmout),
841 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 842}
843
b852190e
YG		 844/**
845 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
846 * @hba: per adapter instance
847 */
848static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
849{
850 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
851}
852
7a3e97b0
SY		 853/**
854 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
855 * When run-stop registers are set to 1, it indicates the
856 * host controller that it can process the requests
857 * @hba: per adapter instance
858 */
859static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
860{
b873a275
SJ		 861 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
862 REG_UTP_TASK_REQ_LIST_RUN_STOP);
863 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
864 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 865}
866
7a3e97b0
SY		 867/**
868 * ufshcd_hba_start - Start controller initialization sequence
869 * @hba: per adapter instance
870 */
871static inline void ufshcd_hba_start(struct ufs_hba *hba)
872{
df043c74
ST		 873 u32 val = CONTROLLER_ENABLE;
874
875 if (ufshcd_crypto_enable(hba))
876 val |= CRYPTO_GENERAL_ENABLE;
877
878 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 879}
880
881/**
882 * ufshcd_is_hba_active - Get controller state
883 * @hba: per adapter instance
884 *
c9e6010b 885 * Returns false if controller is active, true otherwise
7a3e97b0 886 */
c9e6010b 887static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 888{
4a8eec2b
TK		 889 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
890 ? false : true;
7a3e97b0
SY		 891}
892
37113106
YG		 893u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
894{
895 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
896 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
897 (hba->ufs_version == UFSHCI_VERSION_11))
898 return UFS_UNIPRO_VER_1_41;
899 else
900 return UFS_UNIPRO_VER_1_6;
901}
902EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
903
904static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
905{
906 /*
907 * If both host and device support UniPro ver1.6 or later, PA layer
908 * parameters tuning happens during link startup itself.
909 *
910 * We can manually tune PA layer parameters if either host or device
911 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
912 * logic simple, we will only do manual tuning if local unipro version
913 * doesn't support ver1.6 or later.
914 */
915 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
916 return true;
917 else
918 return false;
919}
920
394b949f
SJ		 921/**
922 * ufshcd_set_clk_freq - set UFS controller clock frequencies
923 * @hba: per adapter instance
924 * @scale_up: If True, set max possible frequency othewise set low frequency
925 *
926 * Returns 0 if successful
927 * Returns < 0 for any other errors
928 */
929static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
a3cd5ec5 930{
931 int ret = 0;
932 struct ufs_clk_info *clki;
933 struct list_head *head = &hba->clk_list_head;
a3cd5ec5 934
566ec9ad 935 if (list_empty(head))
a3cd5ec5 936 goto out;
937
a3cd5ec5 938 list_for_each_entry(clki, head, list) {
939 if (!IS_ERR_OR_NULL(clki->clk)) {
940 if (scale_up && clki->max_freq) {
941 if (clki->curr_freq == clki->max_freq)
942 continue;
943
a3cd5ec5 944 ret = clk_set_rate(clki->clk, clki->max_freq);
945 if (ret) {
946 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
947 __func__, clki->name,
948 clki->max_freq, ret);
949 break;
950 }
951 trace_ufshcd_clk_scaling(dev_name(hba->dev),
952 "scaled up", clki->name,
953 clki->curr_freq,
954 clki->max_freq);
955
956 clki->curr_freq = clki->max_freq;
957
958 } else if (!scale_up && clki->min_freq) {
959 if (clki->curr_freq == clki->min_freq)
960 continue;
961
a3cd5ec5 962 ret = clk_set_rate(clki->clk, clki->min_freq);
963 if (ret) {
964 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
965 __func__, clki->name,
966 clki->min_freq, ret);
967 break;
968 }
969 trace_ufshcd_clk_scaling(dev_name(hba->dev),
970 "scaled down", clki->name,
971 clki->curr_freq,
972 clki->min_freq);
973 clki->curr_freq = clki->min_freq;
974 }
975 }
976 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
977 clki->name, clk_get_rate(clki->clk));
978 }
979
394b949f
SJ		 980out:
981 return ret;
982}
983
984/**
985 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
986 * @hba: per adapter instance
987 * @scale_up: True if scaling up and false if scaling down
988 *
989 * Returns 0 if successful
990 * Returns < 0 for any other errors
991 */
992static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
993{
994 int ret = 0;
995 ktime_t start = ktime_get();
996
997 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
998 if (ret)
999 goto out;
1000
1001 ret = ufshcd_set_clk_freq(hba, scale_up);
1002 if (ret)
1003 goto out;
1004
a3cd5ec5 1005 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
394b949f
SJ		 1006 if (ret)
1007 ufshcd_set_clk_freq(hba, !scale_up);
a3cd5ec5 1008
1009out:
394b949f 1010 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
a3cd5ec5 1011 (scale_up ? "up" : "down"),
1012 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1013 return ret;
1014}
1015
1016/**
1017 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1018 * @hba: per adapter instance
1019 * @scale_up: True if scaling up and false if scaling down
1020 *
1021 * Returns true if scaling is required, false otherwise.
1022 */
1023static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1024 bool scale_up)
1025{
1026 struct ufs_clk_info *clki;
1027 struct list_head *head = &hba->clk_list_head;
1028
566ec9ad 1029 if (list_empty(head))
a3cd5ec5 1030 return false;
1031
1032 list_for_each_entry(clki, head, list) {
1033 if (!IS_ERR_OR_NULL(clki->clk)) {
1034 if (scale_up && clki->max_freq) {
1035 if (clki->curr_freq == clki->max_freq)
1036 continue;
1037 return true;
1038 } else if (!scale_up && clki->min_freq) {
1039 if (clki->curr_freq == clki->min_freq)
1040 continue;
1041 return true;
1042 }
1043 }
1044 }
1045
1046 return false;
1047}
1048
1049static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1050 u64 wait_timeout_us)
1051{
1052 unsigned long flags;
1053 int ret = 0;
1054 u32 tm_doorbell;
1055 u32 tr_doorbell;
1056 bool timeout = false, do_last_check = false;
1057 ktime_t start;
1058
1059 ufshcd_hold(hba, false);
1060 spin_lock_irqsave(hba->host->host_lock, flags);
1061 /*
1062 * Wait for all the outstanding tasks/transfer requests.
1063 * Verify by checking the doorbell registers are clear.
1064 */
1065 start = ktime_get();
1066 do {
1067 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1068 ret = -EBUSY;
1069 goto out;
1070 }
1071
1072 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1073 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1074 if (!tm_doorbell && !tr_doorbell) {
1075 timeout = false;
1076 break;
1077 } else if (do_last_check) {
1078 break;
1079 }
1080
1081 spin_unlock_irqrestore(hba->host->host_lock, flags);
1082 schedule();
1083 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1084 wait_timeout_us) {
1085 timeout = true;
1086 /*
1087 * We might have scheduled out for long time so make
1088 * sure to check if doorbells are cleared by this time
1089 * or not.
1090 */
1091 do_last_check = true;
1092 }
1093 spin_lock_irqsave(hba->host->host_lock, flags);
1094 } while (tm_doorbell || tr_doorbell);
1095
1096 if (timeout) {
1097 dev_err(hba->dev,
1098 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1099 __func__, tm_doorbell, tr_doorbell);
1100 ret = -EBUSY;
1101 }
1102out:
1103 spin_unlock_irqrestore(hba->host->host_lock, flags);
1104 ufshcd_release(hba);
1105 return ret;
1106}
1107
1108/**
1109 * ufshcd_scale_gear - scale up/down UFS gear
1110 * @hba: per adapter instance
1111 * @scale_up: True for scaling up gear and false for scaling down
1112 *
1113 * Returns 0 for success,
1114 * Returns -EBUSY if scaling can't happen at this time
1115 * Returns non-zero for any other errors
1116 */
1117static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1118{
1119 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1120 int ret = 0;
1121 struct ufs_pa_layer_attr new_pwr_info;
1122
1123 if (scale_up) {
1124 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1125 sizeof(struct ufs_pa_layer_attr));
1126 } else {
1127 memcpy(&new_pwr_info, &hba->pwr_info,
1128 sizeof(struct ufs_pa_layer_attr));
1129
1130 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1131 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1132 /* save the current power mode */
1133 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1134 &hba->pwr_info,
1135 sizeof(struct ufs_pa_layer_attr));
1136
1137 /* scale down gear */
1138 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1139 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1140 }
1141 }
1142
1143 /* check if the power mode needs to be changed or not? */
6a9df818 1144 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
a3cd5ec5 1145 if (ret)
1146 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1147 __func__, ret,
1148 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1149 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1150
1151 return ret;
1152}
1153
1154static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1155{
1156 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1157 int ret = 0;
1158 /*
1159 * make sure that there are no outstanding requests when
1160 * clock scaling is in progress
1161 */
38135535 1162 ufshcd_scsi_block_requests(hba);
a3cd5ec5 1163 down_write(&hba->clk_scaling_lock);
1164 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1165 ret = -EBUSY;
1166 up_write(&hba->clk_scaling_lock);
38135535 1167 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1168 }
1169
1170 return ret;
1171}
1172
1173static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1174{
1175 up_write(&hba->clk_scaling_lock);
38135535 1176 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1177}
1178
1179/**
1180 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1181 * @hba: per adapter instance
1182 * @scale_up: True for scaling up and false for scalin down
1183 *
1184 * Returns 0 for success,
1185 * Returns -EBUSY if scaling can't happen at this time
1186 * Returns non-zero for any other errors
1187 */
1188static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1189{
1190 int ret = 0;
1191
401f1e44 1192 /* let's not get into low power until clock scaling is completed */
1193 ufshcd_hold(hba, false);
1194
a3cd5ec5 1195 ret = ufshcd_clock_scaling_prepare(hba);
1196 if (ret)
394b949f 1197 goto out;
a3cd5ec5 1198
1199 /* scale down the gear before scaling down clocks */
1200 if (!scale_up) {
1201 ret = ufshcd_scale_gear(hba, false);
1202 if (ret)
394b949f 1203 goto out_unprepare;
a3cd5ec5 1204 }
1205
1206 ret = ufshcd_scale_clks(hba, scale_up);
1207 if (ret) {
1208 if (!scale_up)
1209 ufshcd_scale_gear(hba, true);
394b949f 1210 goto out_unprepare;
a3cd5ec5 1211 }
1212
1213 /* scale up the gear after scaling up clocks */
1214 if (scale_up) {
1215 ret = ufshcd_scale_gear(hba, true);
3d17b9b5 1216 if (ret) {
a3cd5ec5 1217 ufshcd_scale_clks(hba, false);
3d17b9b5
AD		 1218 goto out_unprepare;
1219 }
a3cd5ec5 1220 }
1221
3d17b9b5
AD		 1222 /* Enable Write Booster if we have scaled up else disable it */
1223 up_write(&hba->clk_scaling_lock);
1224 ufshcd_wb_ctrl(hba, scale_up);
1225 down_write(&hba->clk_scaling_lock);
1226
394b949f 1227out_unprepare:
a3cd5ec5 1228 ufshcd_clock_scaling_unprepare(hba);
394b949f 1229out:
401f1e44 1230 ufshcd_release(hba);
a3cd5ec5 1231 return ret;
1232}
1233
401f1e44 1234static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1235{
1236 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1237 clk_scaling.suspend_work);
1238 unsigned long irq_flags;
1239
1240 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1241 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1242 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1243 return;
1244 }
1245 hba->clk_scaling.is_suspended = true;
1246 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1247
1248 __ufshcd_suspend_clkscaling(hba);
1249}
1250
1251static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1252{
1253 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1254 clk_scaling.resume_work);
1255 unsigned long irq_flags;
1256
1257 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1258 if (!hba->clk_scaling.is_suspended) {
1259 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1260 return;
1261 }
1262 hba->clk_scaling.is_suspended = false;
1263 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1264
1265 devfreq_resume_device(hba->devfreq);
1266}
1267
a3cd5ec5 1268static int ufshcd_devfreq_target(struct device *dev,
1269 unsigned long *freq, u32 flags)
1270{
1271 int ret = 0;
1272 struct ufs_hba *hba = dev_get_drvdata(dev);
1273 ktime_t start;
401f1e44 1274 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1275 struct list_head *clk_list = &hba->clk_list_head;
1276 struct ufs_clk_info *clki;
a3cd5ec5 1277 unsigned long irq_flags;
1278
1279 if (!ufshcd_is_clkscaling_supported(hba))
1280 return -EINVAL;
1281
91831d33
AD		 1282 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1283 /* Override with the closest supported frequency */
1284 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
a3cd5ec5 1285 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1286 if (ufshcd_eh_in_progress(hba)) {
1287 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1288 return 0;
1289 }
1290
401f1e44 1291 if (!hba->clk_scaling.active_reqs)
1292 sched_clk_scaling_suspend_work = true;
1293
092b4558
BA		 1294 if (list_empty(clk_list)) {
1295 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1296 goto out;
1297 }
1298
91831d33 1299 /* Decide based on the rounded-off frequency and update */
092b4558 1300 scale_up = (*freq == clki->max_freq) ? true : false;
91831d33
AD		 1301 if (!scale_up)
1302 *freq = clki->min_freq;
1303 /* Update the frequency */
401f1e44 1304 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1305 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1306 ret = 0;
1307 goto out; /* no state change required */
a3cd5ec5 1308 }
1309 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1310
1311 start = ktime_get();
a3cd5ec5 1312 ret = ufshcd_devfreq_scale(hba, scale_up);
1313
a3cd5ec5 1314 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1315 (scale_up ? "up" : "down"),
1316 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1317
401f1e44 1318out:
1319 if (sched_clk_scaling_suspend_work)
1320 queue_work(hba->clk_scaling.workq,
1321 &hba->clk_scaling.suspend_work);
1322
a3cd5ec5 1323 return ret;
1324}
1325
7252a360
BVA		 1326static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1327{
1328 int *busy = priv;
1329
1330 WARN_ON_ONCE(reserved);
1331 (*busy)++;
1332 return false;
1333}
1334
1335/* Whether or not any tag is in use by a request that is in progress. */
1336static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1337{
1338 struct request_queue *q = hba->cmd_queue;
1339 int busy = 0;
1340
1341 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1342 return busy;
1343}
a3cd5ec5 1344
1345static int ufshcd_devfreq_get_dev_status(struct device *dev,
1346 struct devfreq_dev_status *stat)
1347{
1348 struct ufs_hba *hba = dev_get_drvdata(dev);
1349 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1350 unsigned long flags;
91831d33
AD		 1351 struct list_head *clk_list = &hba->clk_list_head;
1352 struct ufs_clk_info *clki;
b1bf66d1 1353 ktime_t curr_t;
a3cd5ec5 1354
1355 if (!ufshcd_is_clkscaling_supported(hba))
1356 return -EINVAL;
1357
1358 memset(stat, 0, sizeof(*stat));
1359
1360 spin_lock_irqsave(hba->host->host_lock, flags);
b1bf66d1 1361 curr_t = ktime_get();
a3cd5ec5 1362 if (!scaling->window_start_t)
1363 goto start_window;
1364
91831d33
AD		 1365 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1366 /*
1367 * If current frequency is 0, then the ondemand governor considers
1368 * there's no initial frequency set. And it always requests to set
1369 * to max. frequency.
1370 */
1371 stat->current_frequency = clki->curr_freq;
a3cd5ec5 1372 if (scaling->is_busy_started)
b1bf66d1
SC		 1373 scaling->tot_busy_t += ktime_us_delta(curr_t,
1374 scaling->busy_start_t);
a3cd5ec5 1375
b1bf66d1 1376 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
a3cd5ec5 1377 stat->busy_time = scaling->tot_busy_t;
1378start_window:
b1bf66d1 1379 scaling->window_start_t = curr_t;
a3cd5ec5 1380 scaling->tot_busy_t = 0;
1381
1382 if (hba->outstanding_reqs) {
b1bf66d1 1383 scaling->busy_start_t = curr_t;
a3cd5ec5 1384 scaling->is_busy_started = true;
1385 } else {
1386 scaling->busy_start_t = 0;
1387 scaling->is_busy_started = false;
1388 }
1389 spin_unlock_irqrestore(hba->host->host_lock, flags);
1390 return 0;
1391}
1392
deac444f
BA		 1393static int ufshcd_devfreq_init(struct ufs_hba *hba)
1394{
092b4558
BA		 1395 struct list_head *clk_list = &hba->clk_list_head;
1396 struct ufs_clk_info *clki;
deac444f
BA		 1397 struct devfreq *devfreq;
1398 int ret;
1399
092b4558
BA		 1400 /* Skip devfreq if we don't have any clocks in the list */
1401 if (list_empty(clk_list))
1402 return 0;
1403
1404 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1405 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1406 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1407
90b8491c
SC		 1408 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1409 &hba->vps->ondemand_data);
092b4558 1410 devfreq = devfreq_add_device(hba->dev,
90b8491c 1411 &hba->vps->devfreq_profile,
deac444f 1412 DEVFREQ_GOV_SIMPLE_ONDEMAND,
90b8491c 1413 &hba->vps->ondemand_data);
deac444f
BA		 1414 if (IS_ERR(devfreq)) {
1415 ret = PTR_ERR(devfreq);
1416 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1417
1418 dev_pm_opp_remove(hba->dev, clki->min_freq);
1419 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1420 return ret;
1421 }
1422
1423 hba->devfreq = devfreq;
1424
1425 return 0;
1426}
1427
092b4558
BA		 1428static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1429{
1430 struct list_head *clk_list = &hba->clk_list_head;
1431 struct ufs_clk_info *clki;
1432
1433 if (!hba->devfreq)
1434 return;
1435
1436 devfreq_remove_device(hba->devfreq);
1437 hba->devfreq = NULL;
1438
1439 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1440 dev_pm_opp_remove(hba->dev, clki->min_freq);
1441 dev_pm_opp_remove(hba->dev, clki->max_freq);
1442}
1443
401f1e44 1444static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1445{
1446 unsigned long flags;
1447
1448 devfreq_suspend_device(hba->devfreq);
1449 spin_lock_irqsave(hba->host->host_lock, flags);
1450 hba->clk_scaling.window_start_t = 0;
1451 spin_unlock_irqrestore(hba->host->host_lock, flags);
1452}
a3cd5ec5 1453
a508253d
GB		 1454static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1455{
401f1e44 1456 unsigned long flags;
1457 bool suspend = false;
1458
fcb0c4b0
ST		 1459 if (!ufshcd_is_clkscaling_supported(hba))
1460 return;
1461
401f1e44 1462 spin_lock_irqsave(hba->host->host_lock, flags);
1463 if (!hba->clk_scaling.is_suspended) {
1464 suspend = true;
1465 hba->clk_scaling.is_suspended = true;
1466 }
1467 spin_unlock_irqrestore(hba->host->host_lock, flags);
1468
1469 if (suspend)
1470 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1471}
1472
1473static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1474{
401f1e44 1475 unsigned long flags;
1476 bool resume = false;
1477
1478 if (!ufshcd_is_clkscaling_supported(hba))
1479 return;
1480
1481 spin_lock_irqsave(hba->host->host_lock, flags);
1482 if (hba->clk_scaling.is_suspended) {
1483 resume = true;
1484 hba->clk_scaling.is_suspended = false;
1485 }
1486 spin_unlock_irqrestore(hba->host->host_lock, flags);
1487
1488 if (resume)
1489 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1490}
1491
1492static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1493 struct device_attribute *attr, char *buf)
1494{
1495 struct ufs_hba *hba = dev_get_drvdata(dev);
1496
1497 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1498}
1499
1500static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1501 struct device_attribute *attr, const char *buf, size_t count)
1502{
1503 struct ufs_hba *hba = dev_get_drvdata(dev);
1504 u32 value;
1505 int err;
1506
1507 if (kstrtou32(buf, 0, &value))
1508 return -EINVAL;
1509
1510 value = !!value;
1511 if (value == hba->clk_scaling.is_allowed)
1512 goto out;
1513
1514 pm_runtime_get_sync(hba->dev);
1515 ufshcd_hold(hba, false);
1516
401f1e44 1517 cancel_work_sync(&hba->clk_scaling.suspend_work);
1518 cancel_work_sync(&hba->clk_scaling.resume_work);
1519
1520 hba->clk_scaling.is_allowed = value;
1521
fcb0c4b0
ST		 1522 if (value) {
1523 ufshcd_resume_clkscaling(hba);
1524 } else {
1525 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1526 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1527 if (err)
1528 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1529 __func__, err);
1530 }
fcb0c4b0
ST		 1531
1532 ufshcd_release(hba);
1533 pm_runtime_put_sync(hba->dev);
1534out:
1535 return count;
a508253d
GB		 1536}
1537
a3cd5ec5 1538static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1539{
1540 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1541 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1542 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1543 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1544 hba->clk_scaling.enable_attr.attr.mode = 0644;
1545 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1546 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1547}
1548
1ab27c9c
ST		 1549static void ufshcd_ungate_work(struct work_struct *work)
1550{
1551 int ret;
1552 unsigned long flags;
1553 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1554 clk_gating.ungate_work);
1555
1556 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1557
1558 spin_lock_irqsave(hba->host->host_lock, flags);
1559 if (hba->clk_gating.state == CLKS_ON) {
1560 spin_unlock_irqrestore(hba->host->host_lock, flags);
1561 goto unblock_reqs;
1562 }
1563
1564 spin_unlock_irqrestore(hba->host->host_lock, flags);
dd7143e2 1565 ufshcd_hba_vreg_set_hpm(hba);
1ab27c9c
ST		 1566 ufshcd_setup_clocks(hba, true);
1567
8b0bbf00
SC		 1568 ufshcd_enable_irq(hba);
1569
1ab27c9c
ST		 1570 /* Exit from hibern8 */
1571 if (ufshcd_can_hibern8_during_gating(hba)) {
1572 /* Prevent gating in this path */
1573 hba->clk_gating.is_suspended = true;
1574 if (ufshcd_is_link_hibern8(hba)) {
1575 ret = ufshcd_uic_hibern8_exit(hba);
1576 if (ret)
1577 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1578 __func__, ret);
1579 else
1580 ufshcd_set_link_active(hba);
1581 }
1582 hba->clk_gating.is_suspended = false;
1583 }
1584unblock_reqs:
38135535 1585 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1586}
1587
1588/**
1589 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1590 * Also, exit from hibern8 mode and set the link as active.
1591 * @hba: per adapter instance
1592 * @async: This indicates whether caller should ungate clocks asynchronously.
1593 */
1594int ufshcd_hold(struct ufs_hba *hba, bool async)
1595{
1596 int rc = 0;
93b6c5db 1597 bool flush_result;
1ab27c9c
ST		 1598 unsigned long flags;
1599
1600 if (!ufshcd_is_clkgating_allowed(hba))
1601 goto out;
1ab27c9c
ST		 1602 spin_lock_irqsave(hba->host->host_lock, flags);
1603 hba->clk_gating.active_reqs++;
1604
856b3483 1605start:
1ab27c9c
ST		 1606 switch (hba->clk_gating.state) {
1607 case CLKS_ON:
f2a785ac
VG		 1608 /*
1609 * Wait for the ungate work to complete if in progress.
1610 * Though the clocks may be in ON state, the link could
1611 * still be in hibner8 state if hibern8 is allowed
1612 * during clock gating.
1613 * Make sure we exit hibern8 state also in addition to
1614 * clocks being ON.
1615 */
1616 if (ufshcd_can_hibern8_during_gating(hba) &&
1617 ufshcd_is_link_hibern8(hba)) {
c63d6099
CG		 1618 if (async) {
1619 rc = -EAGAIN;
1620 hba->clk_gating.active_reqs--;
1621 break;
1622 }
f2a785ac 1623 spin_unlock_irqrestore(hba->host->host_lock, flags);
93b6c5db
SC		 1624 flush_result = flush_work(&hba->clk_gating.ungate_work);
1625 if (hba->clk_gating.is_suspended && !flush_result)
1626 goto out;
f2a785ac
VG		 1627 spin_lock_irqsave(hba->host->host_lock, flags);
1628 goto start;
1629 }
1ab27c9c
ST		 1630 break;
1631 case REQ_CLKS_OFF:
1632 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1633 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1634 trace_ufshcd_clk_gating(dev_name(hba->dev),
1635 hba->clk_gating.state);
1ab27c9c
ST		 1636 break;
1637 }
1638 /*
9c490d2d 1639 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1640 * currently running. Hence, fall through to cancel gating
1641 * work and to enable clocks.
1642 */
df561f66 1643 fallthrough;
1ab27c9c 1644 case CLKS_OFF:
38135535 1645 ufshcd_scsi_block_requests(hba);
1ab27c9c 1646 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47 1647 trace_ufshcd_clk_gating(dev_name(hba->dev),
1648 hba->clk_gating.state);
10e5e375
VV		 1649 queue_work(hba->clk_gating.clk_gating_workq,
1650 &hba->clk_gating.ungate_work);
1ab27c9c
ST		 1651 /*
1652 * fall through to check if we should wait for this
1653 * work to be done or not.
1654 */
df561f66 1655 fallthrough;
1ab27c9c
ST		 1656 case REQ_CLKS_ON:
1657 if (async) {
1658 rc = -EAGAIN;
1659 hba->clk_gating.active_reqs--;
1660 break;
1661 }
1662
1663 spin_unlock_irqrestore(hba->host->host_lock, flags);
1664 flush_work(&hba->clk_gating.ungate_work);
1665 /* Make sure state is CLKS_ON before returning */
856b3483 1666 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1667 goto start;
1668 default:
1669 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1670 __func__, hba->clk_gating.state);
1671 break;
1672 }
1673 spin_unlock_irqrestore(hba->host->host_lock, flags);
1674out:
1675 return rc;
1676}
6e3fd44d 1677EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1678
1679static void ufshcd_gate_work(struct work_struct *work)
1680{
1681 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1682 clk_gating.gate_work.work);
1683 unsigned long flags;
4db7a236 1684 int ret;
1ab27c9c
ST		 1685
1686 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1687 /*
1688 * In case you are here to cancel this work the gating state
1689 * would be marked as REQ_CLKS_ON. In this case save time by
1690 * skipping the gating work and exit after changing the clock
1691 * state to CLKS_ON.
1692 */
1693 if (hba->clk_gating.is_suspended ||
18f01374 1694 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1695 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1696 trace_ufshcd_clk_gating(dev_name(hba->dev),
1697 hba->clk_gating.state);
1ab27c9c
ST		 1698 goto rel_lock;
1699 }
1700
1701 if (hba->clk_gating.active_reqs
1702 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1703 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1ab27c9c
ST		 1704 || hba->active_uic_cmd || hba->uic_async_done)
1705 goto rel_lock;
1706
1707 spin_unlock_irqrestore(hba->host->host_lock, flags);
1708
1709 /* put the link into hibern8 mode before turning off clocks */
1710 if (ufshcd_can_hibern8_during_gating(hba)) {
4db7a236
CG		 1711 ret = ufshcd_uic_hibern8_enter(hba);
1712 if (ret) {
1ab27c9c 1713 hba->clk_gating.state = CLKS_ON;
4db7a236
CG		 1714 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1715 __func__, ret);
7ff5ab47 1716 trace_ufshcd_clk_gating(dev_name(hba->dev),
1717 hba->clk_gating.state);
1ab27c9c
ST		 1718 goto out;
1719 }
1720 ufshcd_set_link_hibern8(hba);
1721 }
1722
8b0bbf00
SC		 1723 ufshcd_disable_irq(hba);
1724
1ab27c9c
ST		 1725 if (!ufshcd_is_link_active(hba))
1726 ufshcd_setup_clocks(hba, false);
1727 else
1728 /* If link is active, device ref_clk can't be switched off */
1729 __ufshcd_setup_clocks(hba, false, true);
1730
dd7143e2
CG		 1731 /* Put the host controller in low power mode if possible */
1732 ufshcd_hba_vreg_set_lpm(hba);
1ab27c9c
ST		 1733 /*
1734 * In case you are here to cancel this work the gating state
1735 * would be marked as REQ_CLKS_ON. In this case keep the state
1736 * as REQ_CLKS_ON which would anyway imply that clocks are off
1737 * and a request to turn them on is pending. By doing this way,
1738 * we keep the state machine in tact and this would ultimately
1739 * prevent from doing cancel work multiple times when there are
1740 * new requests arriving before the current cancel work is done.
1741 */
1742 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1743 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1744 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 1745 trace_ufshcd_clk_gating(dev_name(hba->dev),
1746 hba->clk_gating.state);
1747 }
1ab27c9c
ST		 1748rel_lock:
1749 spin_unlock_irqrestore(hba->host->host_lock, flags);
1750out:
1751 return;
1752}
1753
1754/* host lock must be held before calling this variant */
1755static void __ufshcd_release(struct ufs_hba *hba)
1756{
1757 if (!ufshcd_is_clkgating_allowed(hba))
1758 return;
1759
1760 hba->clk_gating.active_reqs--;
1761
4db7a236
CG		 1762 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1763 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
8eb456be 1764 hba->outstanding_tasks ||
fd62de11
JK		 1765 hba->active_uic_cmd || hba->uic_async_done ||
1766 hba->clk_gating.state == CLKS_OFF)
1ab27c9c
ST		 1767 return;
1768
1769 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1770 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1771 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1772 &hba->clk_gating.gate_work,
1773 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1774}
1775
1776void ufshcd_release(struct ufs_hba *hba)
1777{
1778 unsigned long flags;
1779
1780 spin_lock_irqsave(hba->host->host_lock, flags);
1781 __ufshcd_release(hba);
1782 spin_unlock_irqrestore(hba->host->host_lock, flags);
1783}
6e3fd44d 1784EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1785
1786static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1787 struct device_attribute *attr, char *buf)
1788{
1789 struct ufs_hba *hba = dev_get_drvdata(dev);
1790
1791 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1792}
1793
1794static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1795 struct device_attribute *attr, const char *buf, size_t count)
1796{
1797 struct ufs_hba *hba = dev_get_drvdata(dev);
1798 unsigned long flags, value;
1799
1800 if (kstrtoul(buf, 0, &value))
1801 return -EINVAL;
1802
1803 spin_lock_irqsave(hba->host->host_lock, flags);
1804 hba->clk_gating.delay_ms = value;
1805 spin_unlock_irqrestore(hba->host->host_lock, flags);
1806 return count;
1807}
1808
b427411a
ST		 1809static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1810 struct device_attribute *attr, char *buf)
1811{
1812 struct ufs_hba *hba = dev_get_drvdata(dev);
1813
1814 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1815}
1816
1817static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1818 struct device_attribute *attr, const char *buf, size_t count)
1819{
1820 struct ufs_hba *hba = dev_get_drvdata(dev);
1821 unsigned long flags;
1822 u32 value;
1823
1824 if (kstrtou32(buf, 0, &value))
1825 return -EINVAL;
1826
1827 value = !!value;
b6645112
JK		 1828
1829 spin_lock_irqsave(hba->host->host_lock, flags);
b427411a
ST		 1830 if (value == hba->clk_gating.is_enabled)
1831 goto out;
1832
b6645112
JK		 1833 if (value)
1834 __ufshcd_release(hba);
1835 else
b427411a 1836 hba->clk_gating.active_reqs++;
b427411a
ST		 1837
1838 hba->clk_gating.is_enabled = value;
1839out:
b6645112 1840 spin_unlock_irqrestore(hba->host->host_lock, flags);
b427411a
ST		 1841 return count;
1842}
1843
eebcc196
VG		 1844static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1845{
1846 char wq_name[sizeof("ufs_clkscaling_00")];
1847
1848 if (!ufshcd_is_clkscaling_supported(hba))
1849 return;
1850
1851 INIT_WORK(&hba->clk_scaling.suspend_work,
1852 ufshcd_clk_scaling_suspend_work);
1853 INIT_WORK(&hba->clk_scaling.resume_work,
1854 ufshcd_clk_scaling_resume_work);
1855
1856 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1857 hba->host->host_no);
1858 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1859
1860 ufshcd_clkscaling_init_sysfs(hba);
1861}
1862
1863static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1864{
1865 if (!ufshcd_is_clkscaling_supported(hba))
1866 return;
1867
1868 destroy_workqueue(hba->clk_scaling.workq);
1869 ufshcd_devfreq_remove(hba);
1870}
1871
1ab27c9c
ST		 1872static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1873{
10e5e375
VV		 1874 char wq_name[sizeof("ufs_clk_gating_00")];
1875
1ab27c9c
ST		 1876 if (!ufshcd_is_clkgating_allowed(hba))
1877 return;
1878
2dec9475
CG		 1879 hba->clk_gating.state = CLKS_ON;
1880
1ab27c9c
ST		 1881 hba->clk_gating.delay_ms = 150;
1882 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1883 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1884
10e5e375
VV		 1885 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1886 hba->host->host_no);
1887 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
e93e6e49 1888 WQ_MEM_RECLAIM | WQ_HIGHPRI);
10e5e375 1889
b427411a
ST		 1890 hba->clk_gating.is_enabled = true;
1891
1ab27c9c
ST		 1892 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1893 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1894 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1895 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
b427411a 1896 hba->clk_gating.delay_attr.attr.mode = 0644;
1ab27c9c
ST		 1897 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1898 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
b427411a
ST		 1899
1900 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1901 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1902 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1903 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1904 hba->clk_gating.enable_attr.attr.mode = 0644;
1905 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1906 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1ab27c9c
ST		 1907}
1908
1909static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1910{
1911 if (!ufshcd_is_clkgating_allowed(hba))
1912 return;
1913 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
b427411a 1914 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
97cd6805
AM		 1915 cancel_work_sync(&hba->clk_gating.ungate_work);
1916 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1917 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1918}
1919
856b3483
ST		 1920/* Must be called with host lock acquired */
1921static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1922{
401f1e44 1923 bool queue_resume_work = false;
b1bf66d1 1924 ktime_t curr_t = ktime_get();
401f1e44 1925
fcb0c4b0 1926 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1927 return;
1928
401f1e44 1929 if (!hba->clk_scaling.active_reqs++)
1930 queue_resume_work = true;
1931
1932 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1933 return;
1934
1935 if (queue_resume_work)
1936 queue_work(hba->clk_scaling.workq,
1937 &hba->clk_scaling.resume_work);
1938
1939 if (!hba->clk_scaling.window_start_t) {
b1bf66d1 1940 hba->clk_scaling.window_start_t = curr_t;
401f1e44 1941 hba->clk_scaling.tot_busy_t = 0;
1942 hba->clk_scaling.is_busy_started = false;
1943 }
1944
856b3483 1945 if (!hba->clk_scaling.is_busy_started) {
b1bf66d1 1946 hba->clk_scaling.busy_start_t = curr_t;
856b3483
ST		 1947 hba->clk_scaling.is_busy_started = true;
1948 }
1949}
1950
1951static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1952{
1953 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1954
fcb0c4b0 1955 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1956 return;
1957
1958 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1959 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1960 scaling->busy_start_t));
8b0e1953 1961 scaling->busy_start_t = 0;
856b3483
ST		 1962 scaling->is_busy_started = false;
1963 }
1964}
7a3e97b0
SY		 1965/**
1966 * ufshcd_send_command - Send SCSI or device management commands
1967 * @hba: per adapter instance
1968 * @task_tag: Task tag of the command
1969 */
1970static inline
1971void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1972{
6edfdcfe
SC		 1973 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
1974
1975 lrbp->issue_time_stamp = ktime_get();
1976 lrbp->compl_time_stamp = ktime_set(0, 0);
1977 ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false));
eacf36f5 1978 ufshcd_add_command_trace(hba, task_tag, "send");
856b3483 1979 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 1980 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 1981 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 1982 /* Make sure that doorbell is committed immediately */
1983 wmb();
7a3e97b0
SY		 1984}
1985
1986/**
1987 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 1988 * @lrbp: pointer to local reference block
7a3e97b0
SY		 1989 */
1990static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1991{
1992 int len;
1c2623c5
SJ		 1993 if (lrbp->sense_buffer &&
1994 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 1995 int len_to_copy;
1996
5a0b0cb9 1997 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 1998 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 1999
09a5a24f
AA		 2000 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2001 len_to_copy);
7a3e97b0
SY		 2002 }
2003}
2004
68078d5c
DR		 2005/**
2006 * ufshcd_copy_query_response() - Copy the Query Response and the data
2007 * descriptor
2008 * @hba: per adapter instance
8aa29f19 2009 * @lrbp: pointer to local reference block
68078d5c
DR		 2010 */
2011static
c6d4a831 2012int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 2013{
2014 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2015
68078d5c 2016 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 2017
68078d5c 2018 /* Get the descriptor */
1c90836f
AA		 2019 if (hba->dev_cmd.query.descriptor &&
2020 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 2021 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 2022 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 2023 u16 resp_len;
2024 u16 buf_len;
68078d5c
DR		 2025
2026 /* data segment length */
c6d4a831 2027 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 2028 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 2029 buf_len = be16_to_cpu(
2030 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 2031 if (likely(buf_len >= resp_len)) {
2032 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2033 } else {
2034 dev_warn(hba->dev,
3d4881d1
BH		 2035 "%s: rsp size %d is bigger than buffer size %d",
2036 __func__, resp_len, buf_len);
c6d4a831
DR		 2037 return -EINVAL;
2038 }
68078d5c 2039 }
c6d4a831
DR		 2040
2041 return 0;
68078d5c
DR		 2042}
2043
7a3e97b0
SY		 2044/**
2045 * ufshcd_hba_capabilities - Read controller capabilities
2046 * @hba: per adapter instance
df043c74
ST		 2047 *
2048 * Return: 0 on success, negative on error.
7a3e97b0 2049 */
df043c74 2050static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
7a3e97b0 2051{
df043c74
ST		 2052 int err;
2053
b873a275 2054 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 2055
2056 /* nutrs and nutmrs are 0 based values */
2057 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2058 hba->nutmrs =
2059 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
df043c74
ST		 2060
2061 /* Read crypto capabilities */
2062 err = ufshcd_hba_init_crypto_capabilities(hba);
2063 if (err)
2064 dev_err(hba->dev, "crypto setup failed\n");
2065
2066 return err;
7a3e97b0
SY		 2067}
2068
2069/**
6ccf44fe
SJ		 2070 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2071 * to accept UIC commands
7a3e97b0 2072 * @hba: per adapter instance
6ccf44fe
SJ		 2073 * Return true on success, else false
2074 */
2075static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2076{
2077 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2078 return true;
2079 else
2080 return false;
2081}
2082
53b3d9c3
SJ		 2083/**
2084 * ufshcd_get_upmcrs - Get the power mode change request status
2085 * @hba: Pointer to adapter instance
2086 *
2087 * This function gets the UPMCRS field of HCS register
2088 * Returns value of UPMCRS field
2089 */
2090static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2091{
2092 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2093}
2094
6ccf44fe
SJ		 2095/**
2096 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2097 * @hba: per adapter instance
2098 * @uic_cmd: UIC command
2099 *
2100 * Mutex must be held.
7a3e97b0
SY		 2101 */
2102static inline void
6ccf44fe 2103ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 2104{
6ccf44fe
SJ		 2105 WARN_ON(hba->active_uic_cmd);
2106
2107 hba->active_uic_cmd = uic_cmd;
2108
7a3e97b0 2109 /* Write Args */
6ccf44fe
SJ		 2110 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2111 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2112 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0 2113
aa5c6979
SC		 2114 ufshcd_add_uic_command_trace(hba, uic_cmd, "send");
2115
7a3e97b0 2116 /* Write UIC Cmd */
6ccf44fe 2117 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 2118 REG_UIC_COMMAND);
7a3e97b0
SY		 2119}
2120
6ccf44fe
SJ		 2121/**
2122 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2123 * @hba: per adapter instance
8aa29f19 2124 * @uic_cmd: UIC command
6ccf44fe
SJ		 2125 *
2126 * Must be called with mutex held.
2127 * Returns 0 only if success.
2128 */
2129static int
2130ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2131{
2132 int ret;
2133 unsigned long flags;
2134
2135 if (wait_for_completion_timeout(&uic_cmd->done,
2136 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2137 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2138 else
2139 ret = -ETIMEDOUT;
2140
2141 spin_lock_irqsave(hba->host->host_lock, flags);
2142 hba->active_uic_cmd = NULL;
2143 spin_unlock_irqrestore(hba->host->host_lock, flags);
2144
2145 return ret;
2146}
2147
2148/**
2149 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2150 * @hba: per adapter instance
2151 * @uic_cmd: UIC command
d75f7fe4 2152 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2153 *
2154 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2155 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2156 * Returns 0 only if success.
2157 */
2158static int
d75f7fe4
YG		 2159__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2160 bool completion)
6ccf44fe 2161{
6ccf44fe
SJ		 2162 if (!ufshcd_ready_for_uic_cmd(hba)) {
2163 dev_err(hba->dev,
2164 "Controller not ready to accept UIC commands\n");
2165 return -EIO;
2166 }
2167
d75f7fe4
YG		 2168 if (completion)
2169 init_completion(&uic_cmd->done);
6ccf44fe 2170
6ccf44fe 2171 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2172
57d104c1 2173 return 0;
6ccf44fe
SJ		 2174}
2175
2176/**
2177 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2178 * @hba: per adapter instance
2179 * @uic_cmd: UIC command
2180 *
2181 * Returns 0 only if success.
2182 */
e77044c5 2183int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2184{
2185 int ret;
57d104c1 2186 unsigned long flags;
6ccf44fe 2187
1ab27c9c 2188 ufshcd_hold(hba, false);
6ccf44fe 2189 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2190 ufshcd_add_delay_before_dme_cmd(hba);
2191
57d104c1 2192 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2193 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2194 spin_unlock_irqrestore(hba->host->host_lock, flags);
2195 if (!ret)
2196 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2197
6ccf44fe
SJ		 2198 mutex_unlock(&hba->uic_cmd_mutex);
2199
1ab27c9c 2200 ufshcd_release(hba);
6ccf44fe
SJ		 2201 return ret;
2202}
2203
7a3e97b0
SY		 2204/**
2205 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2206 * @hba: per adapter instance
2207 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2208 *
2209 * Returns 0 in case of success, non-zero value in case of failure
2210 */
75b1cc4a 2211static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2212{
2213 struct ufshcd_sg_entry *prd_table;
2214 struct scatterlist *sg;
2215 struct scsi_cmnd *cmd;
2216 int sg_segments;
2217 int i;
2218
2219 cmd = lrbp->cmd;
2220 sg_segments = scsi_dma_map(cmd);
2221 if (sg_segments < 0)
2222 return sg_segments;
2223
2224 if (sg_segments) {
26f968d7
AA		 2225
2226 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2227 lrbp->utr_descriptor_ptr->prd_table_length =
2228 cpu_to_le16((sg_segments *
2229 sizeof(struct ufshcd_sg_entry)));
2230 else
2231 lrbp->utr_descriptor_ptr->prd_table_length =
2232 cpu_to_le16((u16) (sg_segments));
7a3e97b0
SY		 2233
2234 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2235
2236 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2237 prd_table[i].size =
2238 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2239 prd_table[i].base_addr =
2240 cpu_to_le32(lower_32_bits(sg->dma_address));
2241 prd_table[i].upper_addr =
2242 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2243 prd_table[i].reserved = 0;
7a3e97b0
SY		 2244 }
2245 } else {
2246 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2247 }
2248
2249 return 0;
2250}
2251
2252/**
2fbd009b 2253 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2254 * @hba: per adapter instance
2fbd009b 2255 * @intrs: interrupt bits
7a3e97b0 2256 */
2fbd009b 2257static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2258{
2fbd009b
SJ		 2259 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2260
2261 if (hba->ufs_version == UFSHCI_VERSION_10) {
2262 u32 rw;
2263 rw = set & INTERRUPT_MASK_RW_VER_10;
2264 set = rw | ((set ^ intrs) & intrs);
2265 } else {
2266 set |= intrs;
2267 }
2268
2269 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2270}
2271
2272/**
2273 * ufshcd_disable_intr - disable interrupts
2274 * @hba: per adapter instance
2275 * @intrs: interrupt bits
2276 */
2277static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2278{
2279 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2280
2281 if (hba->ufs_version == UFSHCI_VERSION_10) {
2282 u32 rw;
2283 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2284 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2285 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2286
2287 } else {
2288 set &= ~intrs;
7a3e97b0 2289 }
2fbd009b
SJ		 2290
2291 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2292}
2293
5a0b0cb9
SRT		 2294/**
2295 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2296 * descriptor according to request
2297 * @lrbp: pointer to local reference block
2298 * @upiu_flags: flags required in the header
2299 * @cmd_dir: requests data direction
2300 */
2301static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
a23064c4 2302 u8 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2303{
2304 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2305 u32 data_direction;
2306 u32 dword_0;
df043c74
ST		 2307 u32 dword_1 = 0;
2308 u32 dword_3 = 0;
5a0b0cb9
SRT		 2309
2310 if (cmd_dir == DMA_FROM_DEVICE) {
2311 data_direction = UTP_DEVICE_TO_HOST;
2312 *upiu_flags = UPIU_CMD_FLAGS_READ;
2313 } else if (cmd_dir == DMA_TO_DEVICE) {
2314 data_direction = UTP_HOST_TO_DEVICE;
2315 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2316 } else {
2317 data_direction = UTP_NO_DATA_TRANSFER;
2318 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2319 }
2320
2321 dword_0 = data_direction | (lrbp->command_type
2322 << UPIU_COMMAND_TYPE_OFFSET);
2323 if (lrbp->intr_cmd)
2324 dword_0 |= UTP_REQ_DESC_INT_CMD;
2325
df043c74
ST		 2326 /* Prepare crypto related dwords */
2327 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2328
5a0b0cb9
SRT		 2329 /* Transfer request descriptor header fields */
2330 req_desc->header.dword_0 = cpu_to_le32(dword_0);
df043c74 2331 req_desc->header.dword_1 = cpu_to_le32(dword_1);
5a0b0cb9
SRT		 2332 /*
2333 * assigning invalid value for command status. Controller
2334 * updates OCS on command completion, with the command
2335 * status
2336 */
2337 req_desc->header.dword_2 =
2338 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
df043c74 2339 req_desc->header.dword_3 = cpu_to_le32(dword_3);
51047266
YG		 2340
2341 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2342}
2343
2344/**
2345 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2346 * for scsi commands
8aa29f19
BVA		 2347 * @lrbp: local reference block pointer
2348 * @upiu_flags: flags
5a0b0cb9
SRT		 2349 */
2350static
a23064c4 2351void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
5a0b0cb9 2352{
1b21b8f0 2353 struct scsi_cmnd *cmd = lrbp->cmd;
5a0b0cb9 2354 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2355 unsigned short cdb_len;
5a0b0cb9
SRT		 2356
2357 /* command descriptor fields */
2358 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2359 UPIU_TRANSACTION_COMMAND, upiu_flags,
2360 lrbp->lun, lrbp->task_tag);
2361 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2362 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2363
2364 /* Total EHS length and Data segment length will be zero */
2365 ucd_req_ptr->header.dword_2 = 0;
2366
1b21b8f0 2367 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
5a0b0cb9 2368
1b21b8f0 2369 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
a851b2bd 2370 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1b21b8f0 2371 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
52ac95fe
YG		 2372
2373 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2374}
2375
68078d5c
DR		 2376/**
2377 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2378 * for query requsts
2379 * @hba: UFS hba
2380 * @lrbp: local reference block pointer
2381 * @upiu_flags: flags
2382 */
2383static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
a23064c4 2384 struct ufshcd_lrb *lrbp, u8 upiu_flags)
68078d5c
DR		 2385{
2386 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2387 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2388 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2389
2390 /* Query request header */
2391 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2392 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2393 lrbp->lun, lrbp->task_tag);
2394 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2395 0, query->request.query_func, 0, 0);
2396
6861285c
ZL		 2397 /* Data segment length only need for WRITE_DESC */
2398 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2399 ucd_req_ptr->header.dword_2 =
2400 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2401 else
2402 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2403
2404 /* Copy the Query Request buffer as is */
2405 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2406 QUERY_OSF_SIZE);
68078d5c
DR		 2407
2408 /* Copy the Descriptor */
c6d4a831 2409 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2410 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2411
51047266 2412 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2413}
2414
5a0b0cb9
SRT		 2415static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2416{
2417 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2418
2419 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2420
2421 /* command descriptor fields */
2422 ucd_req_ptr->header.dword_0 =
2423 UPIU_HEADER_DWORD(
2424 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2425 /* clear rest of the fields of basic header */
2426 ucd_req_ptr->header.dword_1 = 0;
2427 ucd_req_ptr->header.dword_2 = 0;
2428
2429 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2430}
2431
7a3e97b0 2432/**
f273c54b 2433 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
300bb13f 2434 * for Device Management Purposes
8aa29f19
BVA		 2435 * @hba: per adapter instance
2436 * @lrbp: pointer to local reference block
7a3e97b0 2437 */
f273c54b
BH		 2438static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2439 struct ufshcd_lrb *lrbp)
7a3e97b0 2440{
a23064c4 2441 u8 upiu_flags;
5a0b0cb9 2442 int ret = 0;
7a3e97b0 2443
83dc7e3d 2444 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2445 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2446 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2447 else
2448 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2449
2450 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2451 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2452 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2453 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2454 ufshcd_prepare_utp_nop_upiu(lrbp);
2455 else
2456 ret = -EINVAL;
2457
2458 return ret;
2459}
2460
2461/**
2462 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2463 * for SCSI Purposes
8aa29f19
BVA		 2464 * @hba: per adapter instance
2465 * @lrbp: pointer to local reference block
300bb13f
JP		 2466 */
2467static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2468{
a23064c4 2469 u8 upiu_flags;
300bb13f
JP		 2470 int ret = 0;
2471
83dc7e3d 2472 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2473 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2474 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2475 else
2476 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2477
2478 if (likely(lrbp->cmd)) {
2479 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2480 lrbp->cmd->sc_data_direction);
2481 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2482 } else {
2483 ret = -EINVAL;
2484 }
5a0b0cb9
SRT		 2485
2486 return ret;
7a3e97b0
SY		 2487}
2488
2a8fa600
SJ		 2489/**
2490 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2491 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2492 *
2493 * Returns SCSI W-LUN id
2494 */
2495static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2496{
2497 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2498}
2499
4d2b8d40
BVA		 2500static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2501{
2502 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2503 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2504 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2505 i * sizeof(struct utp_transfer_cmd_desc);
2506 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2507 response_upiu);
2508 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2509
2510 lrb->utr_descriptor_ptr = utrdlp + i;
2511 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2512 i * sizeof(struct utp_transfer_req_desc);
2513 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2514 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2515 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2516 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2517 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2518 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2519}
2520
7a3e97b0
SY		 2521/**
2522 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2523 * @host: SCSI host pointer
7a3e97b0 2524 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2525 *
2526 * Returns 0 for success, non-zero in case of failure
2527 */
2528static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2529{
2530 struct ufshcd_lrb *lrbp;
2531 struct ufs_hba *hba;
2532 unsigned long flags;
2533 int tag;
2534 int err = 0;
2535
2536 hba = shost_priv(host);
2537
2538 tag = cmd->request->tag;
14497328
YG		 2539 if (!ufshcd_valid_tag(hba, tag)) {
2540 dev_err(hba->dev,
2541 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2542 __func__, tag, cmd, cmd->request);
2543 BUG();
2544 }
7a3e97b0 2545
a3cd5ec5 2546 if (!down_read_trylock(&hba->clk_scaling_lock))
2547 return SCSI_MLQUEUE_HOST_BUSY;
2548
7fabb77b
GB		 2549 hba->req_abort_count = 0;
2550
1ab27c9c
ST		 2551 err = ufshcd_hold(hba, true);
2552 if (err) {
2553 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2554 goto out;
2555 }
2dec9475
CG		 2556 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2557 (hba->clk_gating.state != CLKS_ON));
1ab27c9c 2558
7a3e97b0
SY		 2559 lrbp = &hba->lrb[tag];
2560
5a0b0cb9 2561 WARN_ON(lrbp->cmd);
7a3e97b0 2562 lrbp->cmd = cmd;
09a5a24f 2563 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2564 lrbp->sense_buffer = cmd->sense_buffer;
2565 lrbp->task_tag = tag;
0ce147d4 2566 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2567 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
df043c74
ST		 2568
2569 ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
2570
e0b299e3 2571 lrbp->req_abort_skip = false;
7a3e97b0 2572
300bb13f
JP		 2573 ufshcd_comp_scsi_upiu(hba, lrbp);
2574
75b1cc4a 2575 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2576 if (err) {
2577 lrbp->cmd = NULL;
17c7d35f 2578 ufshcd_release(hba);
7a3e97b0 2579 goto out;
5a0b0cb9 2580 }
ad1a1b9c
GB		 2581 /* Make sure descriptors are ready before ringing the doorbell */
2582 wmb();
7a3e97b0 2583
7a3e97b0 2584 spin_lock_irqsave(hba->host->host_lock, flags);
5586dd8e
CG		 2585 switch (hba->ufshcd_state) {
2586 case UFSHCD_STATE_OPERATIONAL:
2587 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2588 break;
2589 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2590 /*
2591 * pm_runtime_get_sync() is used at error handling preparation
2592 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2593 * PM ops, it can never be finished if we let SCSI layer keep
2594 * retrying it, which gets err handler stuck forever. Neither
2595 * can we let the scsi cmd pass through, because UFS is in bad
2596 * state, the scsi cmd may eventually time out, which will get
2597 * err handler blocked for too long. So, just fail the scsi cmd
2598 * sent from PM ops, err handler can recover PM error anyways.
2599 */
2600 if (hba->pm_op_in_progress) {
2601 hba->force_reset = true;
2602 set_host_byte(cmd, DID_BAD_TARGET);
2603 goto out_compl_cmd;
2604 }
2605 fallthrough;
2606 case UFSHCD_STATE_RESET:
2607 err = SCSI_MLQUEUE_HOST_BUSY;
2608 goto out_compl_cmd;
2609 case UFSHCD_STATE_ERROR:
2610 set_host_byte(cmd, DID_ERROR);
2611 goto out_compl_cmd;
2612 default:
2613 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2614 __func__, hba->ufshcd_state);
2615 set_host_byte(cmd, DID_BAD_TARGET);
2616 goto out_compl_cmd;
2617 }
7a3e97b0
SY		 2618 ufshcd_send_command(hba, tag);
2619 spin_unlock_irqrestore(hba->host->host_lock, flags);
5586dd8e
CG		 2620 goto out;
2621
2622out_compl_cmd:
2623 scsi_dma_unmap(lrbp->cmd);
2624 lrbp->cmd = NULL;
2625 spin_unlock_irqrestore(hba->host->host_lock, flags);
2626 ufshcd_release(hba);
2627 if (!err)
2628 cmd->scsi_done(cmd);
7a3e97b0 2629out:
a3cd5ec5 2630 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2631 return err;
2632}
2633
5a0b0cb9
SRT		 2634static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2635 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2636{
2637 lrbp->cmd = NULL;
2638 lrbp->sense_bufflen = 0;
2639 lrbp->sense_buffer = NULL;
2640 lrbp->task_tag = tag;
2641 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9 2642 lrbp->intr_cmd = true; /* No interrupt aggregation */
df043c74 2643 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5a0b0cb9
SRT		 2644 hba->dev_cmd.type = cmd_type;
2645
f273c54b 2646 return ufshcd_compose_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2647}
2648
2649static int
2650ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2651{
2652 int err = 0;
2653 unsigned long flags;
2654 u32 mask = 1 << tag;
2655
2656 /* clear outstanding transaction before retry */
2657 spin_lock_irqsave(hba->host->host_lock, flags);
2658 ufshcd_utrl_clear(hba, tag);
2659 spin_unlock_irqrestore(hba->host->host_lock, flags);
2660
2661 /*
2662 * wait for for h/w to clear corresponding bit in door-bell.
2663 * max. wait is 1 sec.
2664 */
2665 err = ufshcd_wait_for_register(hba,
2666 REG_UTP_TRANSFER_REQ_DOOR_BELL,
5cac1095 2667 mask, ~mask, 1000, 1000);
5a0b0cb9
SRT		 2668
2669 return err;
2670}
2671
c6d4a831
DR		 2672static int
2673ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2674{
2675 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2676
2677 /* Get the UPIU response */
2678 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2679 UPIU_RSP_CODE_OFFSET;
2680 return query_res->response;
2681}
2682
5a0b0cb9
SRT		 2683/**
2684 * ufshcd_dev_cmd_completion() - handles device management command responses
2685 * @hba: per adapter instance
2686 * @lrbp: pointer to local reference block
2687 */
2688static int
2689ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2690{
2691 int resp;
2692 int err = 0;
2693
ff8e20c6 2694 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2695 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2696
2697 switch (resp) {
2698 case UPIU_TRANSACTION_NOP_IN:
2699 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2700 err = -EINVAL;
2701 dev_err(hba->dev, "%s: unexpected response %x\n",
2702 __func__, resp);
2703 }
2704 break;
68078d5c 2705 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2706 err = ufshcd_check_query_response(hba, lrbp);
2707 if (!err)
2708 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2709 break;
5a0b0cb9
SRT		 2710 case UPIU_TRANSACTION_REJECT_UPIU:
2711 /* TODO: handle Reject UPIU Response */
2712 err = -EPERM;
2713 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2714 __func__);
2715 break;
2716 default:
2717 err = -EINVAL;
2718 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2719 __func__, resp);
2720 break;
2721 }
2722
2723 return err;
2724}
2725
2726static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2727 struct ufshcd_lrb *lrbp, int max_timeout)
2728{
2729 int err = 0;
2730 unsigned long time_left;
2731 unsigned long flags;
2732
2733 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2734 msecs_to_jiffies(max_timeout));
2735
ad1a1b9c
GB		 2736 /* Make sure descriptors are ready before ringing the doorbell */
2737 wmb();
5a0b0cb9
SRT		 2738 spin_lock_irqsave(hba->host->host_lock, flags);
2739 hba->dev_cmd.complete = NULL;
2740 if (likely(time_left)) {
2741 err = ufshcd_get_tr_ocs(lrbp);
2742 if (!err)
2743 err = ufshcd_dev_cmd_completion(hba, lrbp);
2744 }
2745 spin_unlock_irqrestore(hba->host->host_lock, flags);
2746
2747 if (!time_left) {
2748 err = -ETIMEDOUT;
a48353f6
YG		 2749 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2750 __func__, lrbp->task_tag);
5a0b0cb9 2751 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2752 /* successfully cleared the command, retry if needed */
5a0b0cb9 2753 err = -EAGAIN;
a48353f6
YG		 2754 /*
2755 * in case of an error, after clearing the doorbell,
2756 * we also need to clear the outstanding_request
2757 * field in hba
2758 */
2759 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2760 }
2761
2762 return err;
2763}
2764
5a0b0cb9
SRT		 2765/**
2766 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2767 * @hba: UFS hba
2768 * @cmd_type: specifies the type (NOP, Query...)
2769 * @timeout: time in seconds
5a0b0cb9 2770 *
68078d5c
DR		 2771 * NOTE: Since there is only one available tag for device management commands,
2772 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2773 */
2774static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2775 enum dev_cmd_type cmd_type, int timeout)
2776{
7252a360
BVA		 2777 struct request_queue *q = hba->cmd_queue;
2778 struct request *req;
5a0b0cb9
SRT		 2779 struct ufshcd_lrb *lrbp;
2780 int err;
2781 int tag;
2782 struct completion wait;
2783 unsigned long flags;
2784
a3cd5ec5 2785 down_read(&hba->clk_scaling_lock);
2786
5a0b0cb9
SRT		 2787 /*
2788 * Get free slot, sleep if slots are unavailable.
2789 * Even though we use wait_event() which sleeps indefinitely,
2790 * the maximum wait time is bounded by SCSI request timeout.
2791 */
7252a360 2792 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 2793 if (IS_ERR(req)) {
2794 err = PTR_ERR(req);
2795 goto out_unlock;
2796 }
7252a360
BVA		 2797 tag = req->tag;
2798 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5a0b0cb9
SRT		 2799
2800 init_completion(&wait);
2801 lrbp = &hba->lrb[tag];
2802 WARN_ON(lrbp->cmd);
2803 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2804 if (unlikely(err))
2805 goto out_put_tag;
2806
2807 hba->dev_cmd.complete = &wait;
2808
6667e6d9 2809 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
e3dfdc53
YG		 2810 /* Make sure descriptors are ready before ringing the doorbell */
2811 wmb();
5a0b0cb9
SRT		 2812 spin_lock_irqsave(hba->host->host_lock, flags);
2813 ufshcd_send_command(hba, tag);
2814 spin_unlock_irqrestore(hba->host->host_lock, flags);
2815
2816 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2817
6667e6d9
OS		 2818 ufshcd_add_query_upiu_trace(hba, tag,
2819 err ? "query_complete_err" : "query_complete");
2820
5a0b0cb9 2821out_put_tag:
7252a360 2822 blk_put_request(req);
bb14dd15 2823out_unlock:
a3cd5ec5 2824 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2825 return err;
2826}
2827
d44a5f98
DR		 2828/**
2829 * ufshcd_init_query() - init the query response and request parameters
2830 * @hba: per-adapter instance
2831 * @request: address of the request pointer to be initialized
2832 * @response: address of the response pointer to be initialized
2833 * @opcode: operation to perform
2834 * @idn: flag idn to access
2835 * @index: LU number to access
2836 * @selector: query/flag/descriptor further identification
2837 */
2838static inline void ufshcd_init_query(struct ufs_hba *hba,
2839 struct ufs_query_req **request, struct ufs_query_res **response,
2840 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2841{
2842 *request = &hba->dev_cmd.query.request;
2843 *response = &hba->dev_cmd.query.response;
2844 memset(*request, 0, sizeof(struct ufs_query_req));
2845 memset(*response, 0, sizeof(struct ufs_query_res));
2846 (*request)->upiu_req.opcode = opcode;
2847 (*request)->upiu_req.idn = idn;
2848 (*request)->upiu_req.index = index;
2849 (*request)->upiu_req.selector = selector;
2850}
2851
dc3c8d3a 2852static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1f34eedf 2853 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
dc3c8d3a
YG		 2854{
2855 int ret;
2856 int retries;
2857
2858 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1f34eedf 2859 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
dc3c8d3a
YG		 2860 if (ret)
2861 dev_dbg(hba->dev,
2862 "%s: failed with error %d, retries %d\n",
2863 __func__, ret, retries);
2864 else
2865 break;
2866 }
2867
2868 if (ret)
2869 dev_err(hba->dev,
2870 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2871 __func__, opcode, idn, ret, retries);
2872 return ret;
2873}
2874
68078d5c
DR		 2875/**
2876 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2877 * @hba: per-adapter instance
2878 * @opcode: flag query to perform
2879 * @idn: flag idn to access
1f34eedf 2880 * @index: flag index to access
8aa29f19 2881 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2882 *
2883 * Returns 0 for success, non-zero in case of failure
2884 */
dc3c8d3a 2885int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1f34eedf 2886 enum flag_idn idn, u8 index, bool *flag_res)
68078d5c 2887{
d44a5f98
DR		 2888 struct ufs_query_req *request = NULL;
2889 struct ufs_query_res *response = NULL;
1f34eedf 2890 int err, selector = 0;
e5ad406c 2891 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2892
2893 BUG_ON(!hba);
2894
1ab27c9c 2895 ufshcd_hold(hba, false);
68078d5c 2896 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2897 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2898 selector);
68078d5c
DR		 2899
2900 switch (opcode) {
2901 case UPIU_QUERY_OPCODE_SET_FLAG:
2902 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2903 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2904 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2905 break;
2906 case UPIU_QUERY_OPCODE_READ_FLAG:
2907 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2908 if (!flag_res) {
2909 /* No dummy reads */
2910 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2911 __func__);
2912 err = -EINVAL;
2913 goto out_unlock;
2914 }
2915 break;
2916 default:
2917 dev_err(hba->dev,
2918 "%s: Expected query flag opcode but got = %d\n",
2919 __func__, opcode);
2920 err = -EINVAL;
2921 goto out_unlock;
2922 }
68078d5c 2923
e5ad406c 2924 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 2925
2926 if (err) {
2927 dev_err(hba->dev,
2928 "%s: Sending flag query for idn %d failed, err = %d\n",
2929 __func__, idn, err);
2930 goto out_unlock;
2931 }
2932
2933 if (flag_res)
e8c8e82a 2934 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 2935 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2936
2937out_unlock:
2938 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 2939 ufshcd_release(hba);
68078d5c
DR		 2940 return err;
2941}
2942
66ec6d59
SRT		 2943/**
2944 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 2945 * @hba: per-adapter instance
2946 * @opcode: attribute opcode
2947 * @idn: attribute idn to access
2948 * @index: index field
2949 * @selector: selector field
2950 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 2951 *
2952 * Returns 0 for success, non-zero in case of failure
2953*/
ec92b59c
SN		 2954int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2955 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 2956{
d44a5f98
DR		 2957 struct ufs_query_req *request = NULL;
2958 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 2959 int err;
2960
2961 BUG_ON(!hba);
2962
1ab27c9c 2963 ufshcd_hold(hba, false);
66ec6d59
SRT		 2964 if (!attr_val) {
2965 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2966 __func__, opcode);
2967 err = -EINVAL;
2968 goto out;
2969 }
2970
2971 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2972 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2973 selector);
66ec6d59
SRT		 2974
2975 switch (opcode) {
2976 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2977 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 2978 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 2979 break;
2980 case UPIU_QUERY_OPCODE_READ_ATTR:
2981 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2982 break;
2983 default:
2984 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2985 __func__, opcode);
2986 err = -EINVAL;
2987 goto out_unlock;
2988 }
2989
d44a5f98 2990 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 2991
2992 if (err) {
4b761b58
YG		 2993 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2994 __func__, opcode, idn, index, err);
66ec6d59
SRT		 2995 goto out_unlock;
2996 }
2997
e8c8e82a 2998 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 2999
3000out_unlock:
3001 mutex_unlock(&hba->dev_cmd.lock);
3002out:
1ab27c9c 3003 ufshcd_release(hba);
66ec6d59
SRT		 3004 return err;
3005}
3006
5e86ae44
YG		 3007/**
3008 * ufshcd_query_attr_retry() - API function for sending query
3009 * attribute with retries
3010 * @hba: per-adapter instance
3011 * @opcode: attribute opcode
3012 * @idn: attribute idn to access
3013 * @index: index field
3014 * @selector: selector field
3015 * @attr_val: the attribute value after the query request
3016 * completes
3017 *
3018 * Returns 0 for success, non-zero in case of failure
3019*/
3020static int ufshcd_query_attr_retry(struct ufs_hba *hba,
3021 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3022 u32 *attr_val)
3023{
3024 int ret = 0;
3025 u32 retries;
3026
68c9fcfd 3027 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
5e86ae44
YG		 3028 ret = ufshcd_query_attr(hba, opcode, idn, index,
3029 selector, attr_val);
3030 if (ret)
3031 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3032 __func__, ret, retries);
3033 else
3034 break;
3035 }
3036
3037 if (ret)
3038 dev_err(hba->dev,
3039 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3040 __func__, idn, ret, QUERY_REQ_RETRIES);
3041 return ret;
3042}
3043
a70e91b8 3044static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 3045 enum query_opcode opcode, enum desc_idn idn, u8 index,
3046 u8 selector, u8 *desc_buf, int *buf_len)
3047{
3048 struct ufs_query_req *request = NULL;
3049 struct ufs_query_res *response = NULL;
3050 int err;
3051
3052 BUG_ON(!hba);
3053
1ab27c9c 3054 ufshcd_hold(hba, false);
d44a5f98
DR		 3055 if (!desc_buf) {
3056 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3057 __func__, opcode);
3058 err = -EINVAL;
3059 goto out;
3060 }
3061
a4b0e8a4 3062 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 3063 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3064 __func__, *buf_len);
3065 err = -EINVAL;
3066 goto out;
3067 }
3068
3069 mutex_lock(&hba->dev_cmd.lock);
3070 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3071 selector);
3072 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 3073 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 3074
3075 switch (opcode) {
3076 case UPIU_QUERY_OPCODE_WRITE_DESC:
3077 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3078 break;
3079 case UPIU_QUERY_OPCODE_READ_DESC:
3080 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3081 break;
3082 default:
3083 dev_err(hba->dev,
3084 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3085 __func__, opcode);
3086 err = -EINVAL;
3087 goto out_unlock;
3088 }
3089
3090 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3091
3092 if (err) {
4b761b58
YG		 3093 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3094 __func__, opcode, idn, index, err);
d44a5f98
DR		 3095 goto out_unlock;
3096 }
3097
ea2aab24 3098 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 3099
3100out_unlock:
cfcbae38 3101 hba->dev_cmd.query.descriptor = NULL;
d44a5f98
DR		 3102 mutex_unlock(&hba->dev_cmd.lock);
3103out:
1ab27c9c 3104 ufshcd_release(hba);
d44a5f98
DR		 3105 return err;
3106}
3107
a70e91b8 3108/**
8aa29f19
BVA		 3109 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3110 * @hba: per-adapter instance
3111 * @opcode: attribute opcode
3112 * @idn: attribute idn to access
3113 * @index: index field
3114 * @selector: selector field
3115 * @desc_buf: the buffer that contains the descriptor
3116 * @buf_len: length parameter passed to the device
a70e91b8
YG		 3117 *
3118 * Returns 0 for success, non-zero in case of failure.
3119 * The buf_len parameter will contain, on return, the length parameter
3120 * received on the response.
3121 */
2238d31c
SN		 3122int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3123 enum query_opcode opcode,
3124 enum desc_idn idn, u8 index,
3125 u8 selector,
3126 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 3127{
3128 int err;
3129 int retries;
3130
3131 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3132 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3133 selector, desc_buf, buf_len);
3134 if (!err || err == -EINVAL)
3135 break;
3136 }
3137
3138 return err;
3139}
a70e91b8 3140
a4b0e8a4
PM		 3141/**
3142 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3143 * @hba: Pointer to adapter instance
3144 * @desc_id: descriptor idn value
3145 * @desc_len: mapped desc length (out)
a4b0e8a4 3146 */
7a0bf85b
BH		 3147void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3148 int *desc_len)
a4b0e8a4 3149{
7a0bf85b
BH		 3150 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3151 desc_id == QUERY_DESC_IDN_RFU_1)
a4b0e8a4 3152 *desc_len = 0;
7a0bf85b
BH		 3153 else
3154 *desc_len = hba->desc_size[desc_id];
a4b0e8a4
PM		 3155}
3156EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3157
7a0bf85b 3158static void ufshcd_update_desc_length(struct ufs_hba *hba,
72fb690e 3159 enum desc_idn desc_id, int desc_index,
7a0bf85b
BH		 3160 unsigned char desc_len)
3161{
3162 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
72fb690e
BH		 3163 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3164 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3165 * than the RPMB unit, however, both descriptors share the same
3166 * desc_idn, to cover both unit descriptors with one length, we
3167 * choose the normal unit descriptor length by desc_index.
3168 */
7a0bf85b
BH		 3169 hba->desc_size[desc_id] = desc_len;
3170}
3171
da461cec
SJ		 3172/**
3173 * ufshcd_read_desc_param - read the specified descriptor parameter
3174 * @hba: Pointer to adapter instance
3175 * @desc_id: descriptor idn value
3176 * @desc_index: descriptor index
3177 * @param_offset: offset of the parameter to read
3178 * @param_read_buf: pointer to buffer where parameter would be read
3179 * @param_size: sizeof(param_read_buf)
3180 *
3181 * Return 0 in case of success, non-zero otherwise
3182 */
45bced87
SN		 3183int ufshcd_read_desc_param(struct ufs_hba *hba,
3184 enum desc_idn desc_id,
3185 int desc_index,
3186 u8 param_offset,
3187 u8 *param_read_buf,
3188 u8 param_size)
da461cec
SJ		 3189{
3190 int ret;
3191 u8 *desc_buf;
a4b0e8a4 3192 int buff_len;
da461cec
SJ		 3193 bool is_kmalloc = true;
3194
a4b0e8a4
PM		 3195 /* Safety check */
3196 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3197 return -EINVAL;
3198
7a0bf85b
BH		 3199 /* Get the length of descriptor */
3200 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3201 if (!buff_len) {
3202 dev_err(hba->dev, "%s: Failed to get desc length", __func__);
3203 return -EINVAL;
a4b0e8a4
PM		 3204 }
3205
3206 /* Check whether we need temp memory */
3207 if (param_offset != 0 || param_size < buff_len) {
da461cec
SJ		 3208 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3209 if (!desc_buf)
3210 return -ENOMEM;
a4b0e8a4
PM		 3211 } else {
3212 desc_buf = param_read_buf;
3213 is_kmalloc = false;
da461cec
SJ		 3214 }
3215
a4b0e8a4 3216 /* Request for full descriptor */
a70e91b8 3217 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3218 desc_id, desc_index, 0,
3219 desc_buf, &buff_len);
da461cec 3220
bde44bb6 3221 if (ret) {
3222 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3223 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3224 goto out;
3225 }
3226
bde44bb6 3227 /* Sanity check */
3228 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3229 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3230 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3231 ret = -EINVAL;
3232 goto out;
3233 }
3234
7a0bf85b
BH		 3235 /* Update descriptor length */
3236 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
72fb690e 3237 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
7a0bf85b 3238
a4b0e8a4 3239 /* Check wherher we will not copy more data, than available */
cbe193f6
BH		 3240 if (is_kmalloc && (param_offset + param_size) > buff_len)
3241 param_size = buff_len - param_offset;
bde44bb6 3242
da461cec
SJ		 3243 if (is_kmalloc)
3244 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3245out:
3246 if (is_kmalloc)
3247 kfree(desc_buf);
3248 return ret;
3249}
3250
4b828fe1
TW		 3251/**
3252 * struct uc_string_id - unicode string
3253 *
3254 * @len: size of this descriptor inclusive
3255 * @type: descriptor type
3256 * @uc: unicode string character
3257 */
3258struct uc_string_id {
3259 u8 len;
3260 u8 type;
ec38c0ad 3261 wchar_t uc[];
4b828fe1
TW		 3262} __packed;
3263
3264/* replace non-printable or non-ASCII characters with spaces */
3265static inline char ufshcd_remove_non_printable(u8 ch)
3266{
3267 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3268}
3269
b573d484
YG		 3270/**
3271 * ufshcd_read_string_desc - read string descriptor
3272 * @hba: pointer to adapter instance
3273 * @desc_index: descriptor index
4b828fe1
TW		 3274 * @buf: pointer to buffer where descriptor would be read,
3275 * the caller should free the memory.
b573d484 3276 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3277 * null terminated string.
b573d484 3278 *
4b828fe1
TW		 3279 * Return:
3280 * * string size on success.
3281 * * -ENOMEM: on allocation failure
3282 * * -EINVAL: on a wrong parameter
b573d484 3283 */
4b828fe1
TW		 3284int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3285 u8 **buf, bool ascii)
b573d484 3286{
4b828fe1
TW		 3287 struct uc_string_id *uc_str;
3288 u8 *str;
3289 int ret;
b573d484 3290
4b828fe1
TW		 3291 if (!buf)
3292 return -EINVAL;
b573d484 3293
4b828fe1
TW		 3294 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3295 if (!uc_str)
3296 return -ENOMEM;
b573d484 3297
c4607a09
BH		 3298 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3299 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
4b828fe1
TW		 3300 if (ret < 0) {
3301 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3302 QUERY_REQ_RETRIES, ret);
3303 str = NULL;
3304 goto out;
3305 }
3306
3307 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3308 dev_dbg(hba->dev, "String Desc is of zero length\n");
3309 str = NULL;
3310 ret = 0;
b573d484
YG		 3311 goto out;
3312 }
3313
3314 if (ascii) {
4b828fe1 3315 ssize_t ascii_len;
b573d484 3316 int i;
b573d484 3317 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3318 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3319 str = kzalloc(ascii_len, GFP_KERNEL);
3320 if (!str) {
3321 ret = -ENOMEM;
fcbefc3b 3322 goto out;
b573d484
YG		 3323 }
3324
3325 /*
3326 * the descriptor contains string in UTF16 format
3327 * we need to convert to utf-8 so it can be displayed
3328 */
4b828fe1
TW		 3329 ret = utf16s_to_utf8s(uc_str->uc,
3330 uc_str->len - QUERY_DESC_HDR_SIZE,
3331 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3332
3333 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3334 for (i = 0; i < ret; i++)
3335 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3336
4b828fe1
TW		 3337 str[ret++] = '\0';
3338
3339 } else {
5f57704d 3340 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3341 if (!str) {
3342 ret = -ENOMEM;
3343 goto out;
3344 }
4b828fe1 3345 ret = uc_str->len;
b573d484
YG		 3346 }
3347out:
4b828fe1
TW		 3348 *buf = str;
3349 kfree(uc_str);
3350 return ret;
b573d484 3351}
b573d484 3352
da461cec
SJ		 3353/**
3354 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3355 * @hba: Pointer to adapter instance
3356 * @lun: lun id
3357 * @param_offset: offset of the parameter to read
3358 * @param_read_buf: pointer to buffer where parameter would be read
3359 * @param_size: sizeof(param_read_buf)
3360 *
3361 * Return 0 in case of success, non-zero otherwise
3362 */
3363static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3364 int lun,
3365 enum unit_desc_param param_offset,
3366 u8 *param_read_buf,
3367 u32 param_size)
3368{
3369 /*
3370 * Unit descriptors are only available for general purpose LUs (LUN id
3371 * from 0 to 7) and RPMB Well known LU.
3372 */
1baa8011 3373 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
da461cec
SJ		 3374 return -EOPNOTSUPP;
3375
3376 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3377 param_offset, param_read_buf, param_size);
3378}
3379
09f17791
CG		 3380static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3381{
3382 int err = 0;
3383 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3384
3385 if (hba->dev_info.wspecversion >= 0x300) {
3386 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3387 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3388 &gating_wait);
3389 if (err)
3390 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3391 err, gating_wait);
3392
3393 if (gating_wait == 0) {
3394 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3395 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3396 gating_wait);
3397 }
3398
3399 hba->dev_info.clk_gating_wait_us = gating_wait;
3400 }
3401
3402 return err;
3403}
3404
7a3e97b0
SY		 3405/**
3406 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3407 * @hba: per adapter instance
3408 *
3409 * 1. Allocate DMA memory for Command Descriptor array
3410 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3411 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3412 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3413 * (UTMRDL)
3414 * 4. Allocate memory for local reference block(lrb).
3415 *
3416 * Returns 0 for success, non-zero in case of failure
3417 */
3418static int ufshcd_memory_alloc(struct ufs_hba *hba)
3419{
3420 size_t utmrdl_size, utrdl_size, ucdl_size;
3421
3422 /* Allocate memory for UTP command descriptors */
3423 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3424 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3425 ucdl_size,
3426 &hba->ucdl_dma_addr,
3427 GFP_KERNEL);
7a3e97b0
SY		 3428
3429 /*
3430 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3431 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3432 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3433 * be aligned to 128 bytes as well
3434 */
3435 if (!hba->ucdl_base_addr ||
3436 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3437 dev_err(hba->dev,
7a3e97b0
SY		 3438 "Command Descriptor Memory allocation failed\n");
3439 goto out;
3440 }
3441
3442 /*
3443 * Allocate memory for UTP Transfer descriptors
3444 * UFSHCI requires 1024 byte alignment of UTRD
3445 */
3446 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3447 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3448 utrdl_size,
3449 &hba->utrdl_dma_addr,
3450 GFP_KERNEL);
7a3e97b0
SY		 3451 if (!hba->utrdl_base_addr ||
3452 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3453 dev_err(hba->dev,
7a3e97b0
SY		 3454 "Transfer Descriptor Memory allocation failed\n");
3455 goto out;
3456 }
3457
3458 /*
3459 * Allocate memory for UTP Task Management descriptors
3460 * UFSHCI requires 1024 byte alignment of UTMRD
3461 */
3462 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3463 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3464 utmrdl_size,
3465 &hba->utmrdl_dma_addr,
3466 GFP_KERNEL);
7a3e97b0
SY		 3467 if (!hba->utmrdl_base_addr ||
3468 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3469 dev_err(hba->dev,
7a3e97b0
SY		 3470 "Task Management Descriptor Memory allocation failed\n");
3471 goto out;
3472 }
3473
3474 /* Allocate memory for local reference block */
a86854d0
KC		 3475 hba->lrb = devm_kcalloc(hba->dev,
3476 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3477 GFP_KERNEL);
7a3e97b0 3478 if (!hba->lrb) {
3b1d0580 3479 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3480 goto out;
3481 }
3482 return 0;
3483out:
7a3e97b0
SY		 3484 return -ENOMEM;
3485}
3486
3487/**
3488 * ufshcd_host_memory_configure - configure local reference block with
3489 * memory offsets
3490 * @hba: per adapter instance
3491 *
3492 * Configure Host memory space
3493 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3494 * address.
3495 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3496 * and PRDT offset.
3497 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3498 * into local reference block.
3499 */
3500static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3501{
7a3e97b0
SY		 3502 struct utp_transfer_req_desc *utrdlp;
3503 dma_addr_t cmd_desc_dma_addr;
3504 dma_addr_t cmd_desc_element_addr;
3505 u16 response_offset;
3506 u16 prdt_offset;
3507 int cmd_desc_size;
3508 int i;
3509
3510 utrdlp = hba->utrdl_base_addr;
7a3e97b0
SY		 3511
3512 response_offset =
3513 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3514 prdt_offset =
3515 offsetof(struct utp_transfer_cmd_desc, prd_table);
3516
3517 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3518 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3519
3520 for (i = 0; i < hba->nutrs; i++) {
3521 /* Configure UTRD with command descriptor base address */
3522 cmd_desc_element_addr =
3523 (cmd_desc_dma_addr + (cmd_desc_size * i));
3524 utrdlp[i].command_desc_base_addr_lo =
3525 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3526 utrdlp[i].command_desc_base_addr_hi =
3527 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3528
3529 /* Response upiu and prdt offset should be in double words */
26f968d7
AA		 3530 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3531 utrdlp[i].response_upiu_offset =
3532 cpu_to_le16(response_offset);
3533 utrdlp[i].prd_table_offset =
3534 cpu_to_le16(prdt_offset);
3535 utrdlp[i].response_upiu_length =
3536 cpu_to_le16(ALIGNED_UPIU_SIZE);
3537 } else {
3538 utrdlp[i].response_upiu_offset =
3539 cpu_to_le16(response_offset >> 2);
3540 utrdlp[i].prd_table_offset =
3541 cpu_to_le16(prdt_offset >> 2);
3542 utrdlp[i].response_upiu_length =
3543 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3544 }
7a3e97b0 3545
4d2b8d40 3546 ufshcd_init_lrb(hba, &hba->lrb[i], i);
7a3e97b0
SY		 3547 }
3548}
3549
3550/**
3551 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3552 * @hba: per adapter instance
3553 *
3554 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3555 * in order to initialize the Unipro link startup procedure.
3556 * Once the Unipro links are up, the device connected to the controller
3557 * is detected.
3558 *
3559 * Returns 0 on success, non-zero value on failure
3560 */
3561static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3562{
6ccf44fe
SJ		 3563 struct uic_command uic_cmd = {0};
3564 int ret;
7a3e97b0 3565
6ccf44fe 3566 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3567
6ccf44fe
SJ		 3568 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3569 if (ret)
ff8e20c6 3570 dev_dbg(hba->dev,
6ccf44fe
SJ		 3571 "dme-link-startup: error code %d\n", ret);
3572 return ret;
7a3e97b0 3573}
39bf2d83
AA		 3574/**
3575 * ufshcd_dme_reset - UIC command for DME_RESET
3576 * @hba: per adapter instance
3577 *
3578 * DME_RESET command is issued in order to reset UniPro stack.
3579 * This function now deals with cold reset.
3580 *
3581 * Returns 0 on success, non-zero value on failure
3582 */
3583static int ufshcd_dme_reset(struct ufs_hba *hba)
3584{
3585 struct uic_command uic_cmd = {0};
3586 int ret;
3587
3588 uic_cmd.command = UIC_CMD_DME_RESET;
3589
3590 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3591 if (ret)
3592 dev_err(hba->dev,
3593 "dme-reset: error code %d\n", ret);
3594
3595 return ret;
3596}
3597
fc85a74e
SC		 3598int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3599 int agreed_gear,
3600 int adapt_val)
3601{
3602 int ret;
3603
3604 if (agreed_gear != UFS_HS_G4)
66df79cc 3605 adapt_val = PA_NO_ADAPT;
fc85a74e
SC		 3606
3607 ret = ufshcd_dme_set(hba,
3608 UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3609 adapt_val);
3610 return ret;
3611}
3612EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3613
39bf2d83
AA		 3614/**
3615 * ufshcd_dme_enable - UIC command for DME_ENABLE
3616 * @hba: per adapter instance
3617 *
3618 * DME_ENABLE command is issued in order to enable UniPro stack.
3619 *
3620 * Returns 0 on success, non-zero value on failure
3621 */
3622static int ufshcd_dme_enable(struct ufs_hba *hba)
3623{
3624 struct uic_command uic_cmd = {0};
3625 int ret;
3626
3627 uic_cmd.command = UIC_CMD_DME_ENABLE;
3628
3629 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3630 if (ret)
3631 dev_err(hba->dev,
3632 "dme-reset: error code %d\n", ret);
3633
3634 return ret;
3635}
7a3e97b0 3636
cad2e03d
YG		 3637static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3638{
3639 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3640 unsigned long min_sleep_time_us;
3641
3642 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3643 return;
3644
3645 /*
3646 * last_dme_cmd_tstamp will be 0 only for 1st call to
3647 * this function
3648 */
3649 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3650 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3651 } else {
3652 unsigned long delta =
3653 (unsigned long) ktime_to_us(
3654 ktime_sub(ktime_get(),
3655 hba->last_dme_cmd_tstamp));
3656
3657 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3658 min_sleep_time_us =
3659 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3660 else
3661 return; /* no more delay required */
3662 }
3663
3664 /* allow sleep for extra 50us if needed */
3665 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3666}
3667
12b4fdb4
SJ		 3668/**
3669 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3670 * @hba: per adapter instance
3671 * @attr_sel: uic command argument1
3672 * @attr_set: attribute set type as uic command argument2
3673 * @mib_val: setting value as uic command argument3
3674 * @peer: indicate whether peer or local
3675 *
3676 * Returns 0 on success, non-zero value on failure
3677 */
3678int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3679 u8 attr_set, u32 mib_val, u8 peer)
3680{
3681 struct uic_command uic_cmd = {0};
3682 static const char *const action[] = {
3683 "dme-set",
3684 "dme-peer-set"
3685 };
3686 const char *set = action[!!peer];
3687 int ret;
64238fbd 3688 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3689
3690 uic_cmd.command = peer ?
3691 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3692 uic_cmd.argument1 = attr_sel;
3693 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3694 uic_cmd.argument3 = mib_val;
3695
64238fbd
YG		 3696 do {
3697 /* for peer attributes we retry upon failure */
3698 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3699 if (ret)
3700 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3701 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3702 } while (ret && peer && --retries);
3703
f37e9f8c 3704 if (ret)
64238fbd 3705 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3706 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3707 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3708
3709 return ret;
3710}
3711EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3712
3713/**
3714 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3715 * @hba: per adapter instance
3716 * @attr_sel: uic command argument1
3717 * @mib_val: the value of the attribute as returned by the UIC command
3718 * @peer: indicate whether peer or local
3719 *
3720 * Returns 0 on success, non-zero value on failure
3721 */
3722int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3723 u32 *mib_val, u8 peer)
3724{
3725 struct uic_command uic_cmd = {0};
3726 static const char *const action[] = {
3727 "dme-get",
3728 "dme-peer-get"
3729 };
3730 const char *get = action[!!peer];
3731 int ret;
64238fbd 3732 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3733 struct ufs_pa_layer_attr orig_pwr_info;
3734 struct ufs_pa_layer_attr temp_pwr_info;
3735 bool pwr_mode_change = false;
3736
3737 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3738 orig_pwr_info = hba->pwr_info;
3739 temp_pwr_info = orig_pwr_info;
3740
3741 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3742 orig_pwr_info.pwr_rx == FAST_MODE) {
3743 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3744 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3745 pwr_mode_change = true;
3746 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3747 orig_pwr_info.pwr_rx == SLOW_MODE) {
3748 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3749 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3750 pwr_mode_change = true;
3751 }
3752 if (pwr_mode_change) {
3753 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3754 if (ret)
3755 goto out;
3756 }
3757 }
12b4fdb4
SJ		 3758
3759 uic_cmd.command = peer ?
3760 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3761 uic_cmd.argument1 = attr_sel;
3762
64238fbd
YG		 3763 do {
3764 /* for peer attributes we retry upon failure */
3765 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3766 if (ret)
3767 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3768 get, UIC_GET_ATTR_ID(attr_sel), ret);
3769 } while (ret && peer && --retries);
3770
f37e9f8c 3771 if (ret)
64238fbd 3772 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3773 get, UIC_GET_ATTR_ID(attr_sel),
3774 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3775
64238fbd 3776 if (mib_val && !ret)
12b4fdb4 3777 *mib_val = uic_cmd.argument3;
874237f7
YG		 3778
3779 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3780 && pwr_mode_change)
3781 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3782out:
3783 return ret;
3784}
3785EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3786
53b3d9c3 3787/**
57d104c1
SJ		 3788 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3789 * state) and waits for it to take effect.
3790 *
53b3d9c3 3791 * @hba: per adapter instance
57d104c1
SJ		 3792 * @cmd: UIC command to execute
3793 *
3794 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3795 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3796 * and device UniPro link and hence it's final completion would be indicated by
3797 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3798 * addition to normal UIC command completion Status (UCCS). This function only
3799 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3800 *
3801 * Returns 0 on success, non-zero value on failure
3802 */
57d104c1 3803static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3804{
57d104c1 3805 struct completion uic_async_done;
53b3d9c3
SJ		 3806 unsigned long flags;
3807 u8 status;
3808 int ret;
d75f7fe4 3809 bool reenable_intr = false;
53b3d9c3 3810
53b3d9c3 3811 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3812 init_completion(&uic_async_done);
cad2e03d 3813 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3814
3815 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236
CG		 3816 if (ufshcd_is_link_broken(hba)) {
3817 ret = -ENOLINK;
3818 goto out_unlock;
3819 }
57d104c1 3820 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3821 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3822 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3823 /*
3824 * Make sure UIC command completion interrupt is disabled before
3825 * issuing UIC command.
3826 */
3827 wmb();
3828 reenable_intr = true;
57d104c1 3829 }
d75f7fe4
YG		 3830 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3831 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3832 if (ret) {
3833 dev_err(hba->dev,
3834 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3835 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3836 goto out;
3837 }
3838
57d104c1 3839 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3840 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3841 dev_err(hba->dev,
57d104c1
SJ		 3842 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3843 cmd->command, cmd->argument3);
53b3d9c3
SJ		 3844 ret = -ETIMEDOUT;
3845 goto out;
3846 }
3847
3848 status = ufshcd_get_upmcrs(hba);
3849 if (status != PWR_LOCAL) {
3850 dev_err(hba->dev,
479da360 3851 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3852 cmd->command, status);
53b3d9c3
SJ		 3853 ret = (status != PWR_OK) ? status : -1;
3854 }
3855out:
7942f7b5
VG		 3856 if (ret) {
3857 ufshcd_print_host_state(hba);
3858 ufshcd_print_pwr_info(hba);
3859 ufshcd_print_host_regs(hba);
3860 }
3861
53b3d9c3 3862 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3863 hba->active_uic_cmd = NULL;
57d104c1 3864 hba->uic_async_done = NULL;
d75f7fe4
YG		 3865 if (reenable_intr)
3866 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4db7a236
CG		 3867 if (ret) {
3868 ufshcd_set_link_broken(hba);
3869 ufshcd_schedule_eh_work(hba);
3870 }
3871out_unlock:
53b3d9c3
SJ		 3872 spin_unlock_irqrestore(hba->host->host_lock, flags);
3873 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3874
53b3d9c3
SJ		 3875 return ret;
3876}
3877
57d104c1
SJ		 3878/**
3879 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3880 * using DME_SET primitives.
3881 * @hba: per adapter instance
3882 * @mode: powr mode value
3883 *
3884 * Returns 0 on success, non-zero value on failure
3885 */
3886static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3887{
3888 struct uic_command uic_cmd = {0};
1ab27c9c 3889 int ret;
57d104c1 3890
c3a2f9ee
YG		 3891 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3892 ret = ufshcd_dme_set(hba,
3893 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3894 if (ret) {
3895 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3896 __func__, ret);
3897 goto out;
3898 }
3899 }
3900
57d104c1
SJ		 3901 uic_cmd.command = UIC_CMD_DME_SET;
3902 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3903 uic_cmd.argument3 = mode;
1ab27c9c
ST		 3904 ufshcd_hold(hba, false);
3905 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3906 ufshcd_release(hba);
57d104c1 3907
c3a2f9ee 3908out:
1ab27c9c 3909 return ret;
57d104c1
SJ		 3910}
3911
087c5efa 3912int ufshcd_link_recovery(struct ufs_hba *hba)
53c12d0e
YG		 3913{
3914 int ret;
3915 unsigned long flags;
3916
3917 spin_lock_irqsave(hba->host->host_lock, flags);
3918 hba->ufshcd_state = UFSHCD_STATE_RESET;
3919 ufshcd_set_eh_in_progress(hba);
3920 spin_unlock_irqrestore(hba->host->host_lock, flags);
3921
ebdd1dfd
CG		 3922 /* Reset the attached device */
3923 ufshcd_vops_device_reset(hba);
3924
53c12d0e
YG		 3925 ret = ufshcd_host_reset_and_restore(hba);
3926
3927 spin_lock_irqsave(hba->host->host_lock, flags);
3928 if (ret)
3929 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3930 ufshcd_clear_eh_in_progress(hba);
3931 spin_unlock_irqrestore(hba->host->host_lock, flags);
3932
3933 if (ret)
3934 dev_err(hba->dev, "%s: link recovery failed, err %d",
3935 __func__, ret);
3936
3937 return ret;
3938}
087c5efa 3939EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
53c12d0e 3940
4db7a236 3941static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 3942{
87d0b4a6 3943 int ret;
57d104c1 3944 struct uic_command uic_cmd = {0};
911a0771 3945 ktime_t start = ktime_get();
57d104c1 3946
ee32c909
KK		 3947 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3948
57d104c1 3949 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 3950 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3951 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3952 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 3953
4db7a236 3954 if (ret)
87d0b4a6
YG		 3955 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3956 __func__, ret);
4db7a236 3957 else
ee32c909
KK		 3958 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3959 POST_CHANGE);
53c12d0e 3960
87d0b4a6
YG		 3961 return ret;
3962}
3963
9d19bf7a 3964int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
57d104c1
SJ		 3965{
3966 struct uic_command uic_cmd = {0};
3967 int ret;
911a0771 3968 ktime_t start = ktime_get();
57d104c1 3969
ee32c909
KK		 3970 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3971
57d104c1
SJ		 3972 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3973 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3974 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3975 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3976
57d104c1 3977 if (ret) {
53c12d0e
YG		 3978 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3979 __func__, ret);
ff8e20c6 3980 } else {
ee32c909
KK		 3981 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3982 POST_CHANGE);
ff8e20c6
DR		 3983 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3984 hba->ufs_stats.hibern8_exit_cnt++;
3985 }
57d104c1
SJ		 3986
3987 return ret;
3988}
9d19bf7a 3989EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
57d104c1 3990
ba7af5ec
SC		 3991void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
3992{
3993 unsigned long flags;
be7594a4 3994 bool update = false;
ba7af5ec 3995
be7594a4 3996 if (!ufshcd_is_auto_hibern8_supported(hba))
ba7af5ec
SC		 3997 return;
3998
3999 spin_lock_irqsave(hba->host->host_lock, flags);
be7594a4
CG		 4000 if (hba->ahit != ahit) {
4001 hba->ahit = ahit;
4002 update = true;
4003 }
ba7af5ec 4004 spin_unlock_irqrestore(hba->host->host_lock, flags);
be7594a4
CG		 4005
4006 if (update && !pm_runtime_suspended(hba->dev)) {
4007 pm_runtime_get_sync(hba->dev);
4008 ufshcd_hold(hba, false);
4009 ufshcd_auto_hibern8_enable(hba);
4010 ufshcd_release(hba);
4011 pm_runtime_put(hba->dev);
4012 }
ba7af5ec
SC		 4013}
4014EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4015
71d848b8 4016void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378
AH		 4017{
4018 unsigned long flags;
4019
499f7a96 4020 if (!ufshcd_is_auto_hibern8_supported(hba))
ad448378
AH		 4021 return;
4022
4023 spin_lock_irqsave(hba->host->host_lock, flags);
4024 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4025 spin_unlock_irqrestore(hba->host->host_lock, flags);
4026}
4027
5064636c
YG		 4028 /**
4029 * ufshcd_init_pwr_info - setting the POR (power on reset)
4030 * values in hba power info
4031 * @hba: per-adapter instance
4032 */
4033static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4034{
4035 hba->pwr_info.gear_rx = UFS_PWM_G1;
4036 hba->pwr_info.gear_tx = UFS_PWM_G1;
4037 hba->pwr_info.lane_rx = 1;
4038 hba->pwr_info.lane_tx = 1;
4039 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4040 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4041 hba->pwr_info.hs_rate = 0;
4042}
4043
d3e89bac 4044/**
7eb584db
DR		 4045 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4046 * @hba: per-adapter instance
d3e89bac 4047 */
7eb584db 4048static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 4049{
7eb584db
DR		 4050 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4051
4052 if (hba->max_pwr_info.is_valid)
4053 return 0;
4054
2349b533 4055 pwr_info->pwr_tx = FAST_MODE;
4056 pwr_info->pwr_rx = FAST_MODE;
7eb584db 4057 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 4058
4059 /* Get the connected lane count */
7eb584db
DR		 4060 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4061 &pwr_info->lane_rx);
4062 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4063 &pwr_info->lane_tx);
4064
4065 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4066 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4067 __func__,
4068 pwr_info->lane_rx,
4069 pwr_info->lane_tx);
4070 return -EINVAL;
4071 }
d3e89bac
SJ		 4072
4073 /*
4074 * First, get the maximum gears of HS speed.
4075 * If a zero value, it means there is no HSGEAR capability.
4076 * Then, get the maximum gears of PWM speed.
4077 */
7eb584db
DR		 4078 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4079 if (!pwr_info->gear_rx) {
4080 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4081 &pwr_info->gear_rx);
4082 if (!pwr_info->gear_rx) {
4083 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4084 __func__, pwr_info->gear_rx);
4085 return -EINVAL;
4086 }
2349b533 4087 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 4088 }
4089
7eb584db
DR		 4090 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4091 &pwr_info->gear_tx);
4092 if (!pwr_info->gear_tx) {
d3e89bac 4093 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 4094 &pwr_info->gear_tx);
4095 if (!pwr_info->gear_tx) {
4096 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4097 __func__, pwr_info->gear_tx);
4098 return -EINVAL;
4099 }
2349b533 4100 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 4101 }
4102
4103 hba->max_pwr_info.is_valid = true;
4104 return 0;
4105}
4106
4107static int ufshcd_change_power_mode(struct ufs_hba *hba,
4108 struct ufs_pa_layer_attr *pwr_mode)
4109{
4110 int ret;
4111
4112 /* if already configured to the requested pwr_mode */
2355b66e
CG		 4113 if (!hba->force_pmc &&
4114 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
7eb584db
DR		 4115 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4116 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4117 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4118 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4119 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4120 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4121 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4122 return 0;
d3e89bac
SJ		 4123 }
4124
4125 /*
4126 * Configure attributes for power mode change with below.
4127 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4128 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4129 * - PA_HSSERIES
4130 */
7eb584db
DR		 4131 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4132 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4133 pwr_mode->lane_rx);
4134 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4135 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4136 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4137 else
4138 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4139
7eb584db
DR		 4140 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4141 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4142 pwr_mode->lane_tx);
4143 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4144 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4145 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4146 else
4147 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4148
7eb584db
DR		 4149 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4150 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4151 pwr_mode->pwr_rx == FAST_MODE ||
4152 pwr_mode->pwr_tx == FAST_MODE)
4153 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4154 pwr_mode->hs_rate);
d3e89bac 4155
08342537
CG		 4156 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4157 DL_FC0ProtectionTimeOutVal_Default);
4158 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4159 DL_TC0ReplayTimeOutVal_Default);
4160 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4161 DL_AFC0ReqTimeOutVal_Default);
4162 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4163 DL_FC1ProtectionTimeOutVal_Default);
4164 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4165 DL_TC1ReplayTimeOutVal_Default);
4166 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4167 DL_AFC1ReqTimeOutVal_Default);
4168
4169 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4170 DL_FC0ProtectionTimeOutVal_Default);
4171 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4172 DL_TC0ReplayTimeOutVal_Default);
4173 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4174 DL_AFC0ReqTimeOutVal_Default);
4175
7eb584db
DR		 4176 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4177 | pwr_mode->pwr_tx);
4178
4179 if (ret) {
d3e89bac 4180 dev_err(hba->dev,
7eb584db
DR		 4181 "%s: power mode change failed %d\n", __func__, ret);
4182 } else {
0263bcd0
YG		 4183 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4184 pwr_mode);
7eb584db
DR		 4185
4186 memcpy(&hba->pwr_info, pwr_mode,
4187 sizeof(struct ufs_pa_layer_attr));
4188 }
4189
4190 return ret;
4191}
4192
4193/**
4194 * ufshcd_config_pwr_mode - configure a new power mode
4195 * @hba: per-adapter instance
4196 * @desired_pwr_mode: desired power configuration
4197 */
0d846e70 4198int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4199 struct ufs_pa_layer_attr *desired_pwr_mode)
4200{
4201 struct ufs_pa_layer_attr final_params = { 0 };
4202 int ret;
4203
0263bcd0
YG		 4204 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4205 desired_pwr_mode, &final_params);
4206
4207 if (ret)
7eb584db
DR		 4208 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4209
4210 ret = ufshcd_change_power_mode(hba, &final_params);
d3e89bac
SJ		 4211
4212 return ret;
4213}
0d846e70 4214EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4215
68078d5c
DR		 4216/**
4217 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4218 * @hba: per-adapter instance
68078d5c
DR		 4219 *
4220 * Set fDeviceInit flag and poll until device toggles it.
4221 */
4222static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4223{
dc3c8d3a 4224 int err;
7dfdcc39 4225 bool flag_res = true;
29707fab 4226 ktime_t timeout;
68078d5c 4227
dc3c8d3a 4228 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 4229 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
68078d5c
DR		 4230 if (err) {
4231 dev_err(hba->dev,
4232 "%s setting fDeviceInit flag failed with error %d\n",
4233 __func__, err);
4234 goto out;
4235 }
4236
29707fab
KK		 4237 /* Poll fDeviceInit flag to be cleared */
4238 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4239 do {
4240 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4241 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4242 if (!flag_res)
4243 break;
4244 usleep_range(5000, 10000);
4245 } while (ktime_before(ktime_get(), timeout));
dc3c8d3a 4246
29707fab 4247 if (err) {
68078d5c 4248 dev_err(hba->dev,
29707fab
KK		 4249 "%s reading fDeviceInit flag failed with error %d\n",
4250 __func__, err);
4251 } else if (flag_res) {
68078d5c 4252 dev_err(hba->dev,
29707fab
KK		 4253 "%s fDeviceInit was not cleared by the device\n",
4254 __func__);
4255 err = -EBUSY;
4256 }
68078d5c
DR		 4257out:
4258 return err;
4259}
4260
7a3e97b0
SY		 4261/**
4262 * ufshcd_make_hba_operational - Make UFS controller operational
4263 * @hba: per adapter instance
4264 *
4265 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4266 * 1. Enable required interrupts
4267 * 2. Configure interrupt aggregation
897efe62 4268 * 3. Program UTRL and UTMRL base address
5c0c28a8 4269 * 4. Configure run-stop-registers
7a3e97b0
SY		 4270 *
4271 * Returns 0 on success, non-zero value on failure
4272 */
9d19bf7a 4273int ufshcd_make_hba_operational(struct ufs_hba *hba)
7a3e97b0
SY		 4274{
4275 int err = 0;
4276 u32 reg;
4277
6ccf44fe
SJ		 4278 /* Enable required interrupts */
4279 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4280
4281 /* Configure interrupt aggregation */
b852190e
YG		 4282 if (ufshcd_is_intr_aggr_allowed(hba))
4283 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4284 else
4285 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4286
4287 /* Configure UTRL and UTMRL base address registers */
4288 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4289 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4290 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4291 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4292 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4293 REG_UTP_TASK_REQ_LIST_BASE_L);
4294 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4295 REG_UTP_TASK_REQ_LIST_BASE_H);
4296
897efe62
YG		 4297 /*
4298 * Make sure base address and interrupt setup are updated before
4299 * enabling the run/stop registers below.
4300 */
4301 wmb();
4302
7a3e97b0
SY		 4303 /*
4304 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4305 */
5c0c28a8 4306 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4307 if (!(ufshcd_get_lists_status(reg))) {
4308 ufshcd_enable_run_stop_reg(hba);
4309 } else {
3b1d0580 4310 dev_err(hba->dev,
7a3e97b0
SY		 4311 "Host controller not ready to process requests");
4312 err = -EIO;
7a3e97b0
SY		 4313 }
4314
7a3e97b0
SY		 4315 return err;
4316}
9d19bf7a 4317EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
7a3e97b0 4318
596585a2
YG		 4319/**
4320 * ufshcd_hba_stop - Send controller to reset state
4321 * @hba: per adapter instance
596585a2 4322 */
5cac1095 4323static inline void ufshcd_hba_stop(struct ufs_hba *hba)
596585a2 4324{
5cac1095 4325 unsigned long flags;
596585a2
YG		 4326 int err;
4327
5cac1095
BVA		 4328 /*
4329 * Obtain the host lock to prevent that the controller is disabled
4330 * while the UFS interrupt handler is active on another CPU.
4331 */
4332 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 4333 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
5cac1095
BVA		 4334 spin_unlock_irqrestore(hba->host->host_lock, flags);
4335
596585a2
YG		 4336 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4337 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
5cac1095 4338 10, 1);
596585a2
YG		 4339 if (err)
4340 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4341}
4342
7a3e97b0 4343/**
39bf2d83 4344 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4345 * @hba: per adapter instance
4346 *
4347 * The controller resets itself and controller firmware initialization
4348 * sequence kicks off. When controller is ready it will set
4349 * the Host Controller Enable bit to 1.
4350 *
4351 * Returns 0 on success, non-zero value on failure
4352 */
39bf2d83 4353static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0 4354{
6081b12c
SC		 4355 int retry_outer = 3;
4356 int retry_inner;
7a3e97b0 4357
6081b12c 4358start:
596585a2 4359 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4360 /* change controller state to "reset state" */
5cac1095 4361 ufshcd_hba_stop(hba);
7a3e97b0 4362
57d104c1
SJ		 4363 /* UniPro link is disabled at this point */
4364 ufshcd_set_link_off(hba);
4365
0263bcd0 4366 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4367
7a3e97b0
SY		 4368 /* start controller initialization sequence */
4369 ufshcd_hba_start(hba);
4370
4371 /*
4372 * To initialize a UFS host controller HCE bit must be set to 1.
4373 * During initialization the HCE bit value changes from 1->0->1.
4374 * When the host controller completes initialization sequence
4375 * it sets the value of HCE bit to 1. The same HCE bit is read back
4376 * to check if the controller has completed initialization sequence.
4377 * So without this delay the value HCE = 1, set in the previous
4378 * instruction might be read back.
4379 * This delay can be changed based on the controller.
4380 */
90b8491c 4381 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
7a3e97b0
SY		 4382
4383 /* wait for the host controller to complete initialization */
6081b12c 4384 retry_inner = 50;
7a3e97b0 4385 while (ufshcd_is_hba_active(hba)) {
6081b12c
SC		 4386 if (retry_inner) {
4387 retry_inner--;
7a3e97b0 4388 } else {
3b1d0580 4389 dev_err(hba->dev,
7a3e97b0 4390 "Controller enable failed\n");
6081b12c
SC		 4391 if (retry_outer) {
4392 retry_outer--;
4393 goto start;
4394 }
7a3e97b0
SY		 4395 return -EIO;
4396 }
9fc305ef 4397 usleep_range(1000, 1100);
7a3e97b0 4398 }
5c0c28a8 4399
1d337ec2 4400 /* enable UIC related interrupts */
57d104c1 4401 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4402
0263bcd0 4403 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4404
7a3e97b0
SY		 4405 return 0;
4406}
39bf2d83
AA		 4407
4408int ufshcd_hba_enable(struct ufs_hba *hba)
4409{
4410 int ret;
4411
4412 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4413 ufshcd_set_link_off(hba);
4414 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4415
4416 /* enable UIC related interrupts */
4417 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4418 ret = ufshcd_dme_reset(hba);
4419 if (!ret) {
4420 ret = ufshcd_dme_enable(hba);
4421 if (!ret)
4422 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4423 if (ret)
4424 dev_err(hba->dev,
4425 "Host controller enable failed with non-hce\n");
4426 }
4427 } else {
4428 ret = ufshcd_hba_execute_hce(hba);
4429 }
4430
4431 return ret;
4432}
9d19bf7a
SC		 4433EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4434
7ca38cf3
YG		 4435static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4436{
ba0320fb 4437 int tx_lanes = 0, i, err = 0;
7ca38cf3
YG		 4438
4439 if (!peer)
4440 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4441 &tx_lanes);
4442 else
4443 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4444 &tx_lanes);
4445 for (i = 0; i < tx_lanes; i++) {
4446 if (!peer)
4447 err = ufshcd_dme_set(hba,
4448 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4449 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4450 0);
4451 else
4452 err = ufshcd_dme_peer_set(hba,
4453 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4454 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4455 0);
4456 if (err) {
4457 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4458 __func__, peer, i, err);
4459 break;
4460 }
4461 }
4462
4463 return err;
4464}
4465
4466static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4467{
4468 return ufshcd_disable_tx_lcc(hba, true);
4469}
4470
a5fe372d
SC		 4471void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4472 u32 reg)
8808b4e9
SC		 4473{
4474 reg_hist->reg[reg_hist->pos] = reg;
4475 reg_hist->tstamp[reg_hist->pos] = ktime_get();
4476 reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4477}
a5fe372d 4478EXPORT_SYMBOL_GPL(ufshcd_update_reg_hist);
8808b4e9 4479
7a3e97b0 4480/**
6ccf44fe 4481 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4482 * @hba: per adapter instance
4483 *
6ccf44fe 4484 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4485 */
6ccf44fe 4486static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4487{
6ccf44fe 4488 int ret;
1d337ec2 4489 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4490 bool link_startup_again = false;
7a3e97b0 4491
7caf489b 4492 /*
4493 * If UFS device isn't active then we will have to issue link startup
4494 * 2 times to make sure the device state move to active.
4495 */
4496 if (!ufshcd_is_ufs_dev_active(hba))
4497 link_startup_again = true;
7a3e97b0 4498
7caf489b 4499link_startup:
1d337ec2 4500 do {
0263bcd0 4501 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4502
1d337ec2 4503 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4504
1d337ec2
SRT		 4505 /* check if device is detected by inter-connect layer */
4506 if (!ret && !ufshcd_is_device_present(hba)) {
8808b4e9
SC		 4507 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4508 0);
1d337ec2
SRT		 4509 dev_err(hba->dev, "%s: Device not present\n", __func__);
4510 ret = -ENXIO;
4511 goto out;
4512 }
6ccf44fe 4513
1d337ec2
SRT		 4514 /*
4515 * DME link lost indication is only received when link is up,
4516 * but we can't be sure if the link is up until link startup
4517 * succeeds. So reset the local Uni-Pro and try again.
4518 */
8808b4e9
SC		 4519 if (ret && ufshcd_hba_enable(hba)) {
4520 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4521 (u32)ret);
1d337ec2 4522 goto out;
8808b4e9 4523 }
1d337ec2
SRT		 4524 } while (ret && retries--);
4525
8808b4e9 4526 if (ret) {
1d337ec2 4527 /* failed to get the link up... retire */
8808b4e9
SC		 4528 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4529 (u32)ret);
5c0c28a8 4530 goto out;
8808b4e9 4531 }
5c0c28a8 4532
7caf489b 4533 if (link_startup_again) {
4534 link_startup_again = false;
4535 retries = DME_LINKSTARTUP_RETRIES;
4536 goto link_startup;
4537 }
4538
d2aebb9b 4539 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4540 ufshcd_init_pwr_info(hba);
4541 ufshcd_print_pwr_info(hba);
4542
7ca38cf3
YG		 4543 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4544 ret = ufshcd_disable_device_tx_lcc(hba);
4545 if (ret)
4546 goto out;
4547 }
4548
5c0c28a8 4549 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4550 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4551 if (ret)
4552 goto out;
7a3e97b0 4553
2355b66e
CG		 4554 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4555 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5c0c28a8 4556 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4557out:
7942f7b5 4558 if (ret) {
6ccf44fe 4559 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4560 ufshcd_print_host_state(hba);
4561 ufshcd_print_pwr_info(hba);
4562 ufshcd_print_host_regs(hba);
4563 }
6ccf44fe 4564 return ret;
7a3e97b0
SY		 4565}
4566
5a0b0cb9
SRT		 4567/**
4568 * ufshcd_verify_dev_init() - Verify device initialization
4569 * @hba: per-adapter instance
4570 *
4571 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4572 * device Transport Protocol (UTP) layer is ready after a reset.
4573 * If the UTP layer at the device side is not initialized, it may
4574 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4575 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4576 */
4577static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4578{
4579 int err = 0;
4580 int retries;
4581
1ab27c9c 4582 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4583 mutex_lock(&hba->dev_cmd.lock);
4584 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4585 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4586 NOP_OUT_TIMEOUT);
4587
4588 if (!err || err == -ETIMEDOUT)
4589 break;
4590
4591 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4592 }
4593 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4594 ufshcd_release(hba);
5a0b0cb9
SRT		 4595
4596 if (err)
4597 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4598 return err;
4599}
4600
0ce147d4
SJ		 4601/**
4602 * ufshcd_set_queue_depth - set lun queue depth
4603 * @sdev: pointer to SCSI device
4604 *
4605 * Read bLUQueueDepth value and activate scsi tagged command
4606 * queueing. For WLUN, queue depth is set to 1. For best-effort
4607 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4608 * value that host can queue.
4609 */
4610static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4611{
4612 int ret = 0;
4613 u8 lun_qdepth;
4614 struct ufs_hba *hba;
4615
4616 hba = shost_priv(sdev->host);
4617
4618 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4619 ret = ufshcd_read_unit_desc_param(hba,
4620 ufshcd_scsi_to_upiu_lun(sdev->lun),
4621 UNIT_DESC_PARAM_LU_Q_DEPTH,
4622 &lun_qdepth,
4623 sizeof(lun_qdepth));
0ce147d4
SJ		 4624
4625 /* Some WLUN doesn't support unit descriptor */
4626 if (ret == -EOPNOTSUPP)
4627 lun_qdepth = 1;
4628 else if (!lun_qdepth)
4629 /* eventually, we can figure out the real queue depth */
4630 lun_qdepth = hba->nutrs;
4631 else
4632 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4633
4634 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4635 __func__, lun_qdepth);
db5ed4df 4636 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4637}
4638
57d104c1
SJ		 4639/*
4640 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4641 * @hba: per-adapter instance
4642 * @lun: UFS device lun id
4643 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4644 *
4645 * Returns 0 in case of success and b_lu_write_protect status would be returned
4646 * @b_lu_write_protect parameter.
4647 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4648 * Returns -EINVAL in case of invalid parameters passed to this function.
4649 */
4650static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4651 u8 lun,
4652 u8 *b_lu_write_protect)
4653{
4654 int ret;
4655
4656 if (!b_lu_write_protect)
4657 ret = -EINVAL;
4658 /*
4659 * According to UFS device spec, RPMB LU can't be write
4660 * protected so skip reading bLUWriteProtect parameter for
4661 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4662 */
1baa8011 4663 else if (lun >= hba->dev_info.max_lu_supported)
57d104c1
SJ		 4664 ret = -ENOTSUPP;
4665 else
4666 ret = ufshcd_read_unit_desc_param(hba,
4667 lun,
4668 UNIT_DESC_PARAM_LU_WR_PROTECT,
4669 b_lu_write_protect,
4670 sizeof(*b_lu_write_protect));
4671 return ret;
4672}
4673
4674/**
4675 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4676 * status
4677 * @hba: per-adapter instance
4678 * @sdev: pointer to SCSI device
4679 *
4680 */
4681static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4682 struct scsi_device *sdev)
4683{
4684 if (hba->dev_info.f_power_on_wp_en &&
4685 !hba->dev_info.is_lu_power_on_wp) {
4686 u8 b_lu_write_protect;
4687
4688 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4689 &b_lu_write_protect) &&
4690 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4691 hba->dev_info.is_lu_power_on_wp = true;
4692 }
4693}
4694
7a3e97b0
SY		 4695/**
4696 * ufshcd_slave_alloc - handle initial SCSI device configurations
4697 * @sdev: pointer to SCSI device
4698 *
4699 * Returns success
4700 */
4701static int ufshcd_slave_alloc(struct scsi_device *sdev)
4702{
4703 struct ufs_hba *hba;
4704
4705 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4706
4707 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4708 sdev->use_10_for_ms = 1;
a3a76391
CG		 4709
4710 /* DBD field should be set to 1 in mode sense(10) */
4711 sdev->set_dbd_for_ms = 1;
7a3e97b0 4712
e8e7f271
SRT		 4713 /* allow SCSI layer to restart the device in case of errors */
4714 sdev->allow_restart = 1;
4264fd61 4715
b2a6c522
SRT		 4716 /* REPORT SUPPORTED OPERATION CODES is not supported */
4717 sdev->no_report_opcodes = 1;
4718
84af7e8b
SRT		 4719 /* WRITE_SAME command is not supported */
4720 sdev->no_write_same = 1;
e8e7f271 4721
0ce147d4 4722 ufshcd_set_queue_depth(sdev);
4264fd61 4723
57d104c1
SJ		 4724 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4725
7a3e97b0
SY		 4726 return 0;
4727}
4728
4264fd61
SRT		 4729/**
4730 * ufshcd_change_queue_depth - change queue depth
4731 * @sdev: pointer to SCSI device
4732 * @depth: required depth to set
4264fd61 4733 *
db5ed4df 4734 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4735 */
db5ed4df 4736static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4737{
4738 struct ufs_hba *hba = shost_priv(sdev->host);
4739
4740 if (depth > hba->nutrs)
4741 depth = hba->nutrs;
db5ed4df 4742 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4743}
4744
eeda4749
AM		 4745/**
4746 * ufshcd_slave_configure - adjust SCSI device configurations
4747 * @sdev: pointer to SCSI device
4748 */
4749static int ufshcd_slave_configure(struct scsi_device *sdev)
4750{
49615ba1 4751 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 4752 struct request_queue *q = sdev->request_queue;
4753
4754 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
49615ba1
SC		 4755
4756 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4757 sdev->rpm_autosuspend = 1;
4758
df043c74
ST		 4759 ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
4760
eeda4749
AM		 4761 return 0;
4762}
4763
7a3e97b0
SY		 4764/**
4765 * ufshcd_slave_destroy - remove SCSI device configurations
4766 * @sdev: pointer to SCSI device
4767 */
4768static void ufshcd_slave_destroy(struct scsi_device *sdev)
4769{
4770 struct ufs_hba *hba;
4771
4772 hba = shost_priv(sdev->host);
0ce147d4 4773 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4774 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4775 unsigned long flags;
4776
4777 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4778 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4779 spin_unlock_irqrestore(hba->host->host_lock, flags);
4780 }
7a3e97b0
SY		 4781}
4782
7a3e97b0
SY		 4783/**
4784 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4785 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4786 * @scsi_status: SCSI command status
4787 *
4788 * Returns value base on SCSI command status
4789 */
4790static inline int
4791ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4792{
4793 int result = 0;
4794
4795 switch (scsi_status) {
7a3e97b0 4796 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4797 ufshcd_copy_sense_data(lrbp);
df561f66 4798 fallthrough;
1c2623c5 4799 case SAM_STAT_GOOD:
7a3e97b0
SY		 4800 result |= DID_OK << 16 |
4801 COMMAND_COMPLETE << 8 |
1c2623c5 4802 scsi_status;
7a3e97b0
SY		 4803 break;
4804 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4805 case SAM_STAT_BUSY:
7a3e97b0 4806 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4807 ufshcd_copy_sense_data(lrbp);
4808 result |= scsi_status;
7a3e97b0
SY		 4809 break;
4810 default:
4811 result |= DID_ERROR << 16;
4812 break;
4813 } /* end of switch */
4814
4815 return result;
4816}
4817
4818/**
4819 * ufshcd_transfer_rsp_status - Get overall status of the response
4820 * @hba: per adapter instance
8aa29f19 4821 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4822 *
4823 * Returns result of the command to notify SCSI midlayer
4824 */
4825static inline int
4826ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4827{
4828 int result = 0;
4829 int scsi_status;
4830 int ocs;
4831
4832 /* overall command status of utrd */
4833 ocs = ufshcd_get_tr_ocs(lrbp);
4834
d779a6e9
KK		 4835 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
4836 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
4837 MASK_RSP_UPIU_RESULT)
4838 ocs = OCS_SUCCESS;
4839 }
4840
7a3e97b0
SY		 4841 switch (ocs) {
4842 case OCS_SUCCESS:
5a0b0cb9 4843 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4844 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4845 switch (result) {
4846 case UPIU_TRANSACTION_RESPONSE:
4847 /*
4848 * get the response UPIU result to extract
4849 * the SCSI command status
4850 */
4851 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4852
4853 /*
4854 * get the result based on SCSI status response
4855 * to notify the SCSI midlayer of the command status
4856 */
4857 scsi_status = result & MASK_SCSI_STATUS;
4858 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4859
f05ac2e5
YG		 4860 /*
4861 * Currently we are only supporting BKOPs exception
4862 * events hence we can ignore BKOPs exception event
4863 * during power management callbacks. BKOPs exception
4864 * event is not expected to be raised in runtime suspend
4865 * callback as it allows the urgent bkops.
4866 * During system suspend, we are anyway forcefully
4867 * disabling the bkops and if urgent bkops is needed
4868 * it will be enabled on system resume. Long term
4869 * solution could be to abort the system suspend if
4870 * UFS device needs urgent BKOPs.
4871 */
4872 if (!hba->pm_op_in_progress &&
2824ec9f
SL		 4873 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr) &&
4874 schedule_work(&hba->eeh_work)) {
4875 /*
4876 * Prevent suspend once eeh_work is scheduled
4877 * to avoid deadlock between ufshcd_suspend
4878 * and exception event handler.
4879 */
4880 pm_runtime_get_noresume(hba->dev);
4881 }
5a0b0cb9
SRT		 4882 break;
4883 case UPIU_TRANSACTION_REJECT_UPIU:
4884 /* TODO: handle Reject UPIU Response */
4885 result = DID_ERROR << 16;
3b1d0580 4886 dev_err(hba->dev,
5a0b0cb9
SRT		 4887 "Reject UPIU not fully implemented\n");
4888 break;
4889 default:
5a0b0cb9
SRT		 4890 dev_err(hba->dev,
4891 "Unexpected request response code = %x\n",
4892 result);
e0347d89 4893 result = DID_ERROR << 16;
7a3e97b0
SY		 4894 break;
4895 }
7a3e97b0
SY		 4896 break;
4897 case OCS_ABORTED:
4898 result |= DID_ABORT << 16;
4899 break;
e8e7f271
SRT		 4900 case OCS_INVALID_COMMAND_STATUS:
4901 result |= DID_REQUEUE << 16;
4902 break;
7a3e97b0
SY		 4903 case OCS_INVALID_CMD_TABLE_ATTR:
4904 case OCS_INVALID_PRDT_ATTR:
4905 case OCS_MISMATCH_DATA_BUF_SIZE:
4906 case OCS_MISMATCH_RESP_UPIU_SIZE:
4907 case OCS_PEER_COMM_FAILURE:
4908 case OCS_FATAL_ERROR:
5e7341e1
ST		 4909 case OCS_DEVICE_FATAL_ERROR:
4910 case OCS_INVALID_CRYPTO_CONFIG:
4911 case OCS_GENERAL_CRYPTO_ERROR:
7a3e97b0
SY		 4912 default:
4913 result |= DID_ERROR << 16;
3b1d0580 4914 dev_err(hba->dev,
ff8e20c6
DR		 4915 "OCS error from controller = %x for tag %d\n",
4916 ocs, lrbp->task_tag);
4917 ufshcd_print_host_regs(hba);
6ba65588 4918 ufshcd_print_host_state(hba);
7a3e97b0
SY		 4919 break;
4920 } /* end of switch */
4921
2df74b69 4922 if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
66cc820f 4923 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 4924 return result;
4925}
4926
6ccf44fe
SJ		 4927/**
4928 * ufshcd_uic_cmd_compl - handle completion of uic command
4929 * @hba: per adapter instance
53b3d9c3 4930 * @intr_status: interrupt status generated by the controller
9333d775
VG		 4931 *
4932 * Returns
4933 * IRQ_HANDLED - If interrupt is valid
4934 * IRQ_NONE - If invalid interrupt
6ccf44fe 4935 */
9333d775 4936static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 4937{
9333d775
VG		 4938 irqreturn_t retval = IRQ_NONE;
4939
53b3d9c3 4940 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 4941 hba->active_uic_cmd->argument2 |=
4942 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 4943 hba->active_uic_cmd->argument3 =
4944 ufshcd_get_dme_attr_val(hba);
6ccf44fe 4945 complete(&hba->active_uic_cmd->done);
9333d775 4946 retval = IRQ_HANDLED;
6ccf44fe 4947 }
53b3d9c3 4948
9333d775 4949 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
57d104c1 4950 complete(hba->uic_async_done);
9333d775
VG		 4951 retval = IRQ_HANDLED;
4952 }
aa5c6979
SC		 4953
4954 if (retval == IRQ_HANDLED)
4955 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
4956 "complete");
9333d775 4957 return retval;
6ccf44fe
SJ		 4958}
4959
7a3e97b0 4960/**
9a47ec7c 4961 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 4962 * @hba: per adapter instance
9a47ec7c 4963 * @completed_reqs: requests to complete
7a3e97b0 4964 */
9a47ec7c
YG		 4965static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4966 unsigned long completed_reqs)
7a3e97b0 4967{
5a0b0cb9
SRT		 4968 struct ufshcd_lrb *lrbp;
4969 struct scsi_cmnd *cmd;
7a3e97b0
SY		 4970 int result;
4971 int index;
e9d501b1 4972
e9d501b1
DR		 4973 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4974 lrbp = &hba->lrb[index];
a3170376 4975 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 4976 cmd = lrbp->cmd;
4977 if (cmd) {
1a07f2d9 4978 ufshcd_add_command_trace(hba, index, "complete");
e9d501b1
DR		 4979 result = ufshcd_transfer_rsp_status(hba, lrbp);
4980 scsi_dma_unmap(cmd);
4981 cmd->result = result;
4982 /* Mark completed command as NULL in LRB */
4983 lrbp->cmd = NULL;
e9d501b1
DR		 4984 /* Do not touch lrbp after scsi done */
4985 cmd->scsi_done(cmd);
1ab27c9c 4986 __ufshcd_release(hba);
300bb13f
JP		 4987 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4988 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
1a07f2d9
LS		 4989 if (hba->dev_cmd.complete) {
4990 ufshcd_add_command_trace(hba, index,
4991 "dev_complete");
e9d501b1 4992 complete(hba->dev_cmd.complete);
1a07f2d9 4993 }
e9d501b1 4994 }
401f1e44 4995 if (ufshcd_is_clkscaling_supported(hba))
4996 hba->clk_scaling.active_reqs--;
e9d501b1 4997 }
7a3e97b0
SY		 4998
4999 /* clear corresponding bits of completed commands */
5000 hba->outstanding_reqs ^= completed_reqs;
5001
856b3483 5002 ufshcd_clk_scaling_update_busy(hba);
7a3e97b0
SY		 5003}
5004
9a47ec7c
YG		 5005/**
5006 * ufshcd_transfer_req_compl - handle SCSI and query command completion
5007 * @hba: per adapter instance
9333d775
VG		 5008 *
5009 * Returns
5010 * IRQ_HANDLED - If interrupt is valid
5011 * IRQ_NONE - If invalid interrupt
9a47ec7c 5012 */
9333d775 5013static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c
YG		 5014{
5015 unsigned long completed_reqs;
5016 u32 tr_doorbell;
5017
5018 /* Resetting interrupt aggregation counters first and reading the
5019 * DOOR_BELL afterward allows us to handle all the completed requests.
5020 * In order to prevent other interrupts starvation the DB is read once
5021 * after reset. The down side of this solution is the possibility of
5022 * false interrupt if device completes another request after resetting
5023 * aggregation and before reading the DB.
5024 */
b638b5eb
AA		 5025 if (ufshcd_is_intr_aggr_allowed(hba) &&
5026 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 5027 ufshcd_reset_intr_aggr(hba);
5028
5029 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5030 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
5031
9333d775
VG		 5032 if (completed_reqs) {
5033 __ufshcd_transfer_req_compl(hba, completed_reqs);
5034 return IRQ_HANDLED;
5035 } else {
5036 return IRQ_NONE;
5037 }
9a47ec7c
YG		 5038}
5039
66ec6d59
SRT		 5040/**
5041 * ufshcd_disable_ee - disable exception event
5042 * @hba: per-adapter instance
5043 * @mask: exception event to disable
5044 *
5045 * Disables exception event in the device so that the EVENT_ALERT
5046 * bit is not set.
5047 *
5048 * Returns zero on success, non-zero error value on failure.
5049 */
5050static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5051{
5052 int err = 0;
5053 u32 val;
5054
5055 if (!(hba->ee_ctrl_mask & mask))
5056 goto out;
5057
5058 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 5059 val &= MASK_EE_STATUS;
5e86ae44 5060 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 5061 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5062 if (!err)
5063 hba->ee_ctrl_mask &= ~mask;
5064out:
5065 return err;
5066}
5067
5068/**
5069 * ufshcd_enable_ee - enable exception event
5070 * @hba: per-adapter instance
5071 * @mask: exception event to enable
5072 *
5073 * Enable corresponding exception event in the device to allow
5074 * device to alert host in critical scenarios.
5075 *
5076 * Returns zero on success, non-zero error value on failure.
5077 */
5078static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5079{
5080 int err = 0;
5081 u32 val;
5082
5083 if (hba->ee_ctrl_mask & mask)
5084 goto out;
5085
5086 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 5087 val &= MASK_EE_STATUS;
5e86ae44 5088 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 5089 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5090 if (!err)
5091 hba->ee_ctrl_mask |= mask;
5092out:
5093 return err;
5094}
5095
5096/**
5097 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5098 * @hba: per-adapter instance
5099 *
5100 * Allow device to manage background operations on its own. Enabling
5101 * this might lead to inconsistent latencies during normal data transfers
5102 * as the device is allowed to manage its own way of handling background
5103 * operations.
5104 *
5105 * Returns zero on success, non-zero on failure.
5106 */
5107static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5108{
5109 int err = 0;
5110
5111 if (hba->auto_bkops_enabled)
5112 goto out;
5113
dc3c8d3a 5114 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 5115 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5116 if (err) {
5117 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5118 __func__, err);
5119 goto out;
5120 }
5121
5122 hba->auto_bkops_enabled = true;
7ff5ab47 5123 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 5124
5125 /* No need of URGENT_BKOPS exception from the device */
5126 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5127 if (err)
5128 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5129 __func__, err);
5130out:
5131 return err;
5132}
5133
5134/**
5135 * ufshcd_disable_auto_bkops - block device in doing background operations
5136 * @hba: per-adapter instance
5137 *
5138 * Disabling background operations improves command response latency but
5139 * has drawback of device moving into critical state where the device is
5140 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5141 * host is idle so that BKOPS are managed effectively without any negative
5142 * impacts.
5143 *
5144 * Returns zero on success, non-zero on failure.
5145 */
5146static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5147{
5148 int err = 0;
5149
5150 if (!hba->auto_bkops_enabled)
5151 goto out;
5152
5153 /*
5154 * If host assisted BKOPs is to be enabled, make sure
5155 * urgent bkops exception is allowed.
5156 */
5157 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5158 if (err) {
5159 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5160 __func__, err);
5161 goto out;
5162 }
5163
dc3c8d3a 5164 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
1f34eedf 5165 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5166 if (err) {
5167 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5168 __func__, err);
5169 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5170 goto out;
5171 }
5172
5173 hba->auto_bkops_enabled = false;
7ff5ab47 5174 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 5175 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5176out:
5177 return err;
5178}
5179
5180/**
4e768e76 5181 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 5182 * @hba: per adapter instance
5183 *
5184 * After a device reset the device may toggle the BKOPS_EN flag
5185 * to default value. The s/w tracking variables should be updated
4e768e76 5186 * as well. This function would change the auto-bkops state based on
5187 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5188 */
4e768e76 5189static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5190{
4e768e76 5191 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5192 hba->auto_bkops_enabled = false;
5193 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5194 ufshcd_enable_auto_bkops(hba);
5195 } else {
5196 hba->auto_bkops_enabled = true;
5197 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5198 ufshcd_disable_auto_bkops(hba);
5199 }
7b6668d8 5200 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
24366c2a 5201 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5202}
5203
5204static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5205{
5e86ae44 5206 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5207 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5208}
5209
5210/**
57d104c1 5211 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5212 * @hba: per-adapter instance
57d104c1 5213 * @status: bkops_status value
66ec6d59 5214 *
57d104c1
SJ		 5215 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5216 * flag in the device to permit background operations if the device
5217 * bkops_status is greater than or equal to "status" argument passed to
5218 * this function, disable otherwise.
5219 *
5220 * Returns 0 for success, non-zero in case of failure.
5221 *
5222 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5223 * to know whether auto bkops is enabled or disabled after this function
5224 * returns control to it.
66ec6d59 5225 */
57d104c1
SJ		 5226static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5227 enum bkops_status status)
66ec6d59
SRT		 5228{
5229 int err;
57d104c1 5230 u32 curr_status = 0;
66ec6d59 5231
57d104c1 5232 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5233 if (err) {
5234 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5235 __func__, err);
5236 goto out;
57d104c1
SJ		 5237 } else if (curr_status > BKOPS_STATUS_MAX) {
5238 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5239 __func__, curr_status);
5240 err = -EINVAL;
5241 goto out;
66ec6d59
SRT		 5242 }
5243
57d104c1 5244 if (curr_status >= status)
66ec6d59 5245 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5246 else
5247 err = ufshcd_disable_auto_bkops(hba);
66ec6d59
SRT		 5248out:
5249 return err;
5250}
5251
57d104c1
SJ		 5252/**
5253 * ufshcd_urgent_bkops - handle urgent bkops exception event
5254 * @hba: per-adapter instance
5255 *
5256 * Enable fBackgroundOpsEn flag in the device to permit background
5257 * operations.
5258 *
5259 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5260 * and negative error value for any other failure.
5261 */
5262static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5263{
afdfff59 5264 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5265}
5266
66ec6d59
SRT		 5267static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5268{
5e86ae44 5269 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5270 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5271}
5272
afdfff59
YG		 5273static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5274{
5275 int err;
5276 u32 curr_status = 0;
5277
5278 if (hba->is_urgent_bkops_lvl_checked)
5279 goto enable_auto_bkops;
5280
5281 err = ufshcd_get_bkops_status(hba, &curr_status);
5282 if (err) {
5283 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5284 __func__, err);
5285 goto out;
5286 }
5287
5288 /*
5289 * We are seeing that some devices are raising the urgent bkops
5290 * exception events even when BKOPS status doesn't indicate performace
5291 * impacted or critical. Handle these device by determining their urgent
5292 * bkops status at runtime.
5293 */
5294 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5295 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5296 __func__, curr_status);
5297 /* update the current status as the urgent bkops level */
5298 hba->urgent_bkops_lvl = curr_status;
5299 hba->is_urgent_bkops_lvl_checked = true;
5300 }
5301
5302enable_auto_bkops:
5303 err = ufshcd_enable_auto_bkops(hba);
5304out:
5305 if (err < 0)
5306 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5307 __func__, err);
5308}
5309
3d17b9b5
AD		 5310static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable)
5311{
5312 int ret;
6f8d5a6a 5313 u8 index;
3d17b9b5
AD		 5314 enum query_opcode opcode;
5315
79e3520f 5316 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5317 return 0;
5318
5319 if (!(enable ^ hba->wb_enabled))
5320 return 0;
5321 if (enable)
5322 opcode = UPIU_QUERY_OPCODE_SET_FLAG;
5323 else
5324 opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5325
e31011ab 5326 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5327 ret = ufshcd_query_flag_retry(hba, opcode,
6f8d5a6a 5328 QUERY_FLAG_IDN_WB_EN, index, NULL);
3d17b9b5
AD		 5329 if (ret) {
5330 dev_err(hba->dev, "%s write booster %s failed %d\n",
5331 __func__, enable ? "enable" : "disable", ret);
5332 return ret;
5333 }
5334
5335 hba->wb_enabled = enable;
5336 dev_dbg(hba->dev, "%s write booster %s %d\n",
5337 __func__, enable ? "enable" : "disable", ret);
5338
5339 return ret;
5340}
5341
5342static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5343{
5344 int val;
6f8d5a6a 5345 u8 index;
3d17b9b5
AD		 5346
5347 if (set)
5348 val = UPIU_QUERY_OPCODE_SET_FLAG;
5349 else
5350 val = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5351
e31011ab 5352 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5353 return ufshcd_query_flag_retry(hba, val,
6f8d5a6a
SC		 5354 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8,
5355 index, NULL);
3d17b9b5
AD		 5356}
5357
5358static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
5359{
5df6f2de
KK		 5360 if (hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL)
5361 return;
5362
3d17b9b5
AD		 5363 if (enable)
5364 ufshcd_wb_buf_flush_enable(hba);
5365 else
5366 ufshcd_wb_buf_flush_disable(hba);
5367
5368}
5369
5370static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba)
5371{
5372 int ret;
6f8d5a6a 5373 u8 index;
3d17b9b5 5374
79e3520f 5375 if (!ufshcd_is_wb_allowed(hba) || hba->wb_buf_flush_enabled)
3d17b9b5
AD		 5376 return 0;
5377
e31011ab 5378 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5379 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 5380 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
6f8d5a6a 5381 index, NULL);
3d17b9b5
AD		 5382 if (ret)
5383 dev_err(hba->dev, "%s WB - buf flush enable failed %d\n",
5384 __func__, ret);
5385 else
5386 hba->wb_buf_flush_enabled = true;
5387
5388 dev_dbg(hba->dev, "WB - Flush enabled: %d\n", ret);
5389 return ret;
5390}
5391
5392static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba)
5393{
5394 int ret;
6f8d5a6a 5395 u8 index;
3d17b9b5 5396
79e3520f 5397 if (!ufshcd_is_wb_allowed(hba) || !hba->wb_buf_flush_enabled)
3d17b9b5
AD		 5398 return 0;
5399
e31011ab 5400 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5401 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
6f8d5a6a
SC		 5402 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
5403 index, NULL);
3d17b9b5
AD		 5404 if (ret) {
5405 dev_warn(hba->dev, "%s: WB - buf flush disable failed %d\n",
5406 __func__, ret);
5407 } else {
5408 hba->wb_buf_flush_enabled = false;
5409 dev_dbg(hba->dev, "WB - Flush disabled: %d\n", ret);
5410 }
5411
5412 return ret;
5413}
5414
5415static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5416 u32 avail_buf)
5417{
5418 u32 cur_buf;
5419 int ret;
e31011ab 5420 u8 index;
3d17b9b5 5421
e31011ab 5422 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5423 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5424 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
e31011ab 5425 index, 0, &cur_buf);
3d17b9b5
AD		 5426 if (ret) {
5427 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5428 __func__, ret);
5429 return false;
5430 }
5431
5432 if (!cur_buf) {
5433 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5434 cur_buf);
5435 return false;
5436 }
d14734ae
SC		 5437 /* Let it continue to flush when available buffer exceeds threshold */
5438 if (avail_buf < hba->vps->wb_flush_threshold)
3d17b9b5
AD		 5439 return true;
5440
5441 return false;
5442}
5443
51dd905b 5444static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
3d17b9b5
AD		 5445{
5446 int ret;
5447 u32 avail_buf;
e31011ab 5448 u8 index;
3d17b9b5 5449
79e3520f 5450 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5451 return false;
5452 /*
5453 * The ufs device needs the vcc to be ON to flush.
5454 * With user-space reduction enabled, it's enough to enable flush
5455 * by checking only the available buffer. The threshold
5456 * defined here is > 90% full.
5457 * With user-space preserved enabled, the current-buffer
5458 * should be checked too because the wb buffer size can reduce
5459 * when disk tends to be full. This info is provided by current
5460 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5461 * keeping vcc on when current buffer is empty.
5462 */
e31011ab 5463 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5464 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5465 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
e31011ab 5466 index, 0, &avail_buf);
3d17b9b5
AD		 5467 if (ret) {
5468 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5469 __func__, ret);
5470 return false;
5471 }
5472
5473 if (!hba->dev_info.b_presrv_uspc_en) {
d14734ae 5474 if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
3d17b9b5
AD		 5475 return true;
5476 return false;
5477 }
5478
5479 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5480}
5481
51dd905b
SC		 5482static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5483{
5484 struct ufs_hba *hba = container_of(to_delayed_work(work),
5485 struct ufs_hba,
5486 rpm_dev_flush_recheck_work);
5487 /*
5488 * To prevent unnecessary VCC power drain after device finishes
5489 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5490 * after a certain delay to recheck the threshold by next runtime
5491 * suspend.
5492 */
5493 pm_runtime_get_sync(hba->dev);
5494 pm_runtime_put_sync(hba->dev);
5495}
5496
66ec6d59
SRT		 5497/**
5498 * ufshcd_exception_event_handler - handle exceptions raised by device
5499 * @work: pointer to work data
5500 *
5501 * Read bExceptionEventStatus attribute from the device and handle the
5502 * exception event accordingly.
5503 */
5504static void ufshcd_exception_event_handler(struct work_struct *work)
5505{
5506 struct ufs_hba *hba;
5507 int err;
5508 u32 status = 0;
5509 hba = container_of(work, struct ufs_hba, eeh_work);
5510
62694735 5511 pm_runtime_get_sync(hba->dev);
03e1d28e 5512 ufshcd_scsi_block_requests(hba);
66ec6d59
SRT		 5513 err = ufshcd_get_ee_status(hba, &status);
5514 if (err) {
5515 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5516 __func__, err);
5517 goto out;
5518 }
5519
5520 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5521
5522 if (status & MASK_EE_URGENT_BKOPS)
5523 ufshcd_bkops_exception_event_handler(hba);
5524
66ec6d59 5525out:
03e1d28e 5526 ufshcd_scsi_unblock_requests(hba);
2824ec9f
SL		 5527 /*
5528 * pm_runtime_get_noresume is called while scheduling
5529 * eeh_work to avoid suspend racing with exception work.
5530 * Hence decrement usage counter using pm_runtime_put_noidle
5531 * to allow suspend on completion of exception event handler.
5532 */
5533 pm_runtime_put_noidle(hba->dev);
5534 pm_runtime_put(hba->dev);
66ec6d59
SRT		 5535 return;
5536}
5537
9a47ec7c
YG		 5538/* Complete requests that have door-bell cleared */
5539static void ufshcd_complete_requests(struct ufs_hba *hba)
5540{
5541 ufshcd_transfer_req_compl(hba);
5542 ufshcd_tmc_handler(hba);
5543}
5544
583fa62d
YG		 5545/**
5546 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5547 * to recover from the DL NAC errors or not.
5548 * @hba: per-adapter instance
5549 *
5550 * Returns true if error handling is required, false otherwise
5551 */
5552static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5553{
5554 unsigned long flags;
5555 bool err_handling = true;
5556
5557 spin_lock_irqsave(hba->host->host_lock, flags);
5558 /*
5559 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5560 * device fatal error and/or DL NAC & REPLAY timeout errors.
5561 */
5562 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5563 goto out;
5564
5565 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5566 ((hba->saved_err & UIC_ERROR) &&
5567 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5568 goto out;
5569
5570 if ((hba->saved_err & UIC_ERROR) &&
5571 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5572 int err;
5573 /*
5574 * wait for 50ms to see if we can get any other errors or not.
5575 */
5576 spin_unlock_irqrestore(hba->host->host_lock, flags);
5577 msleep(50);
5578 spin_lock_irqsave(hba->host->host_lock, flags);
5579
5580 /*
5581 * now check if we have got any other severe errors other than
5582 * DL NAC error?
5583 */
5584 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5585 ((hba->saved_err & UIC_ERROR) &&
5586 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5587 goto out;
5588
5589 /*
5590 * As DL NAC is the only error received so far, send out NOP
5591 * command to confirm if link is still active or not.
5592 * - If we don't get any response then do error recovery.
5593 * - If we get response then clear the DL NAC error bit.
5594 */
5595
5596 spin_unlock_irqrestore(hba->host->host_lock, flags);
5597 err = ufshcd_verify_dev_init(hba);
5598 spin_lock_irqsave(hba->host->host_lock, flags);
5599
5600 if (err)
5601 goto out;
5602
5603 /* Link seems to be alive hence ignore the DL NAC errors */
5604 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5605 hba->saved_err &= ~UIC_ERROR;
5606 /* clear NAC error */
5607 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
b0008625 5608 if (!hba->saved_uic_err)
583fa62d 5609 err_handling = false;
583fa62d
YG		 5610 }
5611out:
5612 spin_unlock_irqrestore(hba->host->host_lock, flags);
5613 return err_handling;
5614}
5615
4db7a236
CG		 5616/* host lock must be held before calling this func */
5617static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
5618{
5619 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
5620 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
5621}
5622
5623/* host lock must be held before calling this func */
5624static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
5625{
5626 /* handle fatal errors only when link is not in error state */
5627 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
5586dd8e
CG		 5628 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5629 ufshcd_is_saved_err_fatal(hba))
5630 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
5631 else
5632 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
5633 queue_work(hba->eh_wq, &hba->eh_work);
4db7a236
CG		 5634 }
5635}
5636
c72e79c0
CG		 5637static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
5638{
5639 pm_runtime_get_sync(hba->dev);
5640 if (pm_runtime_suspended(hba->dev)) {
5641 /*
5642 * Don't assume anything of pm_runtime_get_sync(), if
5643 * resume fails, irq and clocks can be OFF, and powers
5644 * can be OFF or in LPM.
5645 */
5646 ufshcd_setup_hba_vreg(hba, true);
5647 ufshcd_enable_irq(hba);
5648 ufshcd_setup_vreg(hba, true);
5649 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5650 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5651 ufshcd_hold(hba, false);
5652 if (!ufshcd_is_clkgating_allowed(hba))
5653 ufshcd_setup_clocks(hba, true);
5654 ufshcd_release(hba);
5655 ufshcd_vops_resume(hba, UFS_RUNTIME_PM);
5656 } else {
5657 ufshcd_hold(hba, false);
5658 if (hba->clk_scaling.is_allowed) {
5659 cancel_work_sync(&hba->clk_scaling.suspend_work);
5660 cancel_work_sync(&hba->clk_scaling.resume_work);
5661 ufshcd_suspend_clkscaling(hba);
5662 }
5663 }
5664}
5665
5666static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
5667{
5668 ufshcd_release(hba);
5669 if (hba->clk_scaling.is_allowed)
5670 ufshcd_resume_clkscaling(hba);
5671 pm_runtime_put(hba->dev);
5672}
5673
5674static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
5675{
5676 return (hba->ufshcd_state == UFSHCD_STATE_ERROR ||
5677 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
5678 ufshcd_is_link_broken(hba))));
5679}
5680
5681#ifdef CONFIG_PM
5682static void ufshcd_recover_pm_error(struct ufs_hba *hba)
5683{
5684 struct Scsi_Host *shost = hba->host;
5685 struct scsi_device *sdev;
5686 struct request_queue *q;
5687 int ret;
5688
5689 /*
5690 * Set RPM status of hba device to RPM_ACTIVE,
5691 * this also clears its runtime error.
5692 */
5693 ret = pm_runtime_set_active(hba->dev);
5694 /*
5695 * If hba device had runtime error, we also need to resume those
5696 * scsi devices under hba in case any of them has failed to be
5697 * resumed due to hba runtime resume failure. This is to unblock
5698 * blk_queue_enter in case there are bios waiting inside it.
5699 */
5700 if (!ret) {
5701 shost_for_each_device(sdev, shost) {
5702 q = sdev->request_queue;
5703 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
5704 q->rpm_status == RPM_SUSPENDING))
5705 pm_request_resume(q->dev);
5706 }
5707 }
5708}
5709#else
5710static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
5711{
5712}
5713#endif
5714
2355b66e
CG		 5715static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
5716{
5717 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
5718 u32 mode;
5719
5720 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
5721
5722 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
5723 return true;
5724
5725 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
5726 return true;
5727
5728 return false;
5729}
5730
7a3e97b0 5731/**
e8e7f271
SRT		 5732 * ufshcd_err_handler - handle UFS errors that require s/w attention
5733 * @work: pointer to work structure
7a3e97b0 5734 */
e8e7f271 5735static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5736{
5737 struct ufs_hba *hba;
e8e7f271 5738 unsigned long flags;
307348f6
CG		 5739 bool err_xfer = false;
5740 bool err_tm = false;
2355b66e 5741 int err = 0, pmc_err;
e8e7f271 5742 int tag;
2355b66e 5743 bool needs_reset = false, needs_restore = false;
e8e7f271
SRT		 5744
5745 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5746
e8e7f271 5747 spin_lock_irqsave(hba->host->host_lock, flags);
c72e79c0 5748 if (ufshcd_err_handling_should_stop(hba)) {
4db7a236
CG		 5749 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5750 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5751 spin_unlock_irqrestore(hba->host->host_lock, flags);
4db7a236
CG		 5752 return;
5753 }
5754 ufshcd_set_eh_in_progress(hba);
5755 spin_unlock_irqrestore(hba->host->host_lock, flags);
c72e79c0 5756 ufshcd_err_handling_prepare(hba);
e8e7f271 5757 spin_lock_irqsave(hba->host->host_lock, flags);
5586dd8e 5758 ufshcd_scsi_block_requests(hba);
c72e79c0
CG		 5759 /*
5760 * A full reset and restore might have happened after preparation
5761 * is finished, double check whether we should stop.
5762 */
5763 if (ufshcd_err_handling_should_stop(hba)) {
5764 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5765 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
e8e7f271 5766 goto out;
c72e79c0 5767 }
e8e7f271 5768 hba->ufshcd_state = UFSHCD_STATE_RESET;
e8e7f271
SRT		 5769
5770 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5771 ufshcd_complete_requests(hba);
583fa62d
YG		 5772
5773 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5774 bool ret;
5775
5776 spin_unlock_irqrestore(hba->host->host_lock, flags);
5777 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5778 ret = ufshcd_quirk_dl_nac_errors(hba);
5779 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236 5780 if (!ret && !hba->force_reset && ufshcd_is_link_active(hba))
583fa62d
YG		 5781 goto skip_err_handling;
5782 }
4db7a236
CG		 5783
5784 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5785 ufshcd_is_saved_err_fatal(hba) ||
9a47ec7c 5786 ((hba->saved_err & UIC_ERROR) &&
4db7a236
CG		 5787 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5788 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
9a47ec7c 5789 needs_reset = true;
e8e7f271 5790
2355b66e
CG		 5791 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
5792 (hba->saved_uic_err &&
5793 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
c3be8d1e
CG		 5794 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
5795
5796 spin_unlock_irqrestore(hba->host->host_lock, flags);
5797 ufshcd_print_host_state(hba);
5798 ufshcd_print_pwr_info(hba);
5799 ufshcd_print_host_regs(hba);
5800 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5801 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
5802 spin_lock_irqsave(hba->host->host_lock, flags);
5803 }
5804
9a47ec7c
YG		 5805 /*
5806 * if host reset is required then skip clearing the pending
2df74b69
CG		 5807 * transfers forcefully because they will get cleared during
5808 * host reset and restore
9a47ec7c
YG		 5809 */
5810 if (needs_reset)
2355b66e 5811 goto do_reset;
9a47ec7c 5812
2355b66e
CG		 5813 /*
5814 * If LINERESET was caught, UFS might have been put to PWM mode,
5815 * check if power mode restore is needed.
5816 */
5817 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
5818 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5819 if (!hba->saved_uic_err)
5820 hba->saved_err &= ~UIC_ERROR;
5821 spin_unlock_irqrestore(hba->host->host_lock, flags);
5822 if (ufshcd_is_pwr_mode_restore_needed(hba))
5823 needs_restore = true;
5824 spin_lock_irqsave(hba->host->host_lock, flags);
5825 if (!hba->saved_err && !needs_restore)
5826 goto skip_err_handling;
5827 }
9a47ec7c 5828
2355b66e 5829 hba->silence_err_logs = true;
9a47ec7c
YG		 5830 /* release lock as clear command might sleep */
5831 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5832 /* Clear pending transfer requests */
9a47ec7c 5833 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
307348f6 5834 if (ufshcd_try_to_abort_task(hba, tag)) {
9a47ec7c
YG		 5835 err_xfer = true;
5836 goto lock_skip_pending_xfer_clear;
5837 }
5838 }
e8e7f271
SRT		 5839
5840 /* Clear pending task management requests */
9a47ec7c
YG		 5841 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5842 if (ufshcd_clear_tm_cmd(hba, tag)) {
5843 err_tm = true;
5844 goto lock_skip_pending_xfer_clear;
5845 }
5846 }
e8e7f271 5847
9a47ec7c 5848lock_skip_pending_xfer_clear:
e8e7f271 5849 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5850
9a47ec7c
YG		 5851 /* Complete the requests that are cleared by s/w */
5852 ufshcd_complete_requests(hba);
2355b66e 5853 hba->silence_err_logs = false;
9a47ec7c 5854
2355b66e 5855 if (err_xfer || err_tm) {
9a47ec7c 5856 needs_reset = true;
2355b66e
CG		 5857 goto do_reset;
5858 }
9a47ec7c 5859
2355b66e
CG		 5860 /*
5861 * After all reqs and tasks are cleared from doorbell,
5862 * now it is safe to retore power mode.
5863 */
5864 if (needs_restore) {
5865 spin_unlock_irqrestore(hba->host->host_lock, flags);
5866 /*
5867 * Hold the scaling lock just in case dev cmds
5868 * are sent via bsg and/or sysfs.
5869 */
5870 down_write(&hba->clk_scaling_lock);
5871 hba->force_pmc = true;
5872 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
5873 if (pmc_err) {
5874 needs_reset = true;
5875 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
5876 __func__, pmc_err);
5877 }
5878 hba->force_pmc = false;
5879 ufshcd_print_pwr_info(hba);
5880 up_write(&hba->clk_scaling_lock);
5881 spin_lock_irqsave(hba->host->host_lock, flags);
5882 }
9a47ec7c 5883
2355b66e 5884do_reset:
e8e7f271 5885 /* Fatal errors need reset */
9a47ec7c
YG		 5886 if (needs_reset) {
5887 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5888
5889 /*
5890 * ufshcd_reset_and_restore() does the link reinitialization
5891 * which will need atleast one empty doorbell slot to send the
5892 * device management commands (NOP and query commands).
5893 * If there is no slot empty at this moment then free up last
5894 * slot forcefully.
5895 */
5896 if (hba->outstanding_reqs == max_doorbells)
5897 __ufshcd_transfer_req_compl(hba,
5898 (1UL << (hba->nutrs - 1)));
5899
4db7a236 5900 hba->force_reset = false;
9a47ec7c 5901 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5902 err = ufshcd_reset_and_restore(hba);
4db7a236
CG		 5903 if (err)
5904 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
5905 __func__, err);
c72e79c0
CG		 5906 else
5907 ufshcd_recover_pm_error(hba);
9a47ec7c 5908 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5909 }
9a47ec7c 5910
583fa62d 5911skip_err_handling:
9a47ec7c 5912 if (!needs_reset) {
4db7a236
CG		 5913 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
5914 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
9a47ec7c
YG		 5915 if (hba->saved_err || hba->saved_uic_err)
5916 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5917 __func__, hba->saved_err, hba->saved_uic_err);
5918 }
5919
e8e7f271 5920out:
e8e7f271 5921 ufshcd_clear_eh_in_progress(hba);
9a47ec7c 5922 spin_unlock_irqrestore(hba->host->host_lock, flags);
38135535 5923 ufshcd_scsi_unblock_requests(hba);
c72e79c0 5924 ufshcd_err_handling_unprepare(hba);
7a3e97b0
SY		 5925}
5926
5927/**
e8e7f271
SRT		 5928 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5929 * @hba: per-adapter instance
9333d775
VG		 5930 *
5931 * Returns
5932 * IRQ_HANDLED - If interrupt is valid
5933 * IRQ_NONE - If invalid interrupt
7a3e97b0 5934 */
9333d775 5935static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 5936{
5937 u32 reg;
9333d775 5938 irqreturn_t retval = IRQ_NONE;
7a3e97b0 5939
2355b66e 5940 /* PHY layer error */
fb7b45f0 5941 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
fb7b45f0 5942 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
2355b66e
CG		 5943 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
5944 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
fb7b45f0
DR		 5945 /*
5946 * To know whether this error is fatal or not, DB timeout
5947 * must be checked but this error is handled separately.
5948 */
2355b66e
CG		 5949 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
5950 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
5951 __func__);
5952
5953 /* Got a LINERESET indication. */
5954 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
5955 struct uic_command *cmd = NULL;
5956
5957 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
5958 if (hba->uic_async_done && hba->active_uic_cmd)
5959 cmd = hba->active_uic_cmd;
5960 /*
5961 * Ignore the LINERESET during power mode change
5962 * operation via DME_SET command.
5963 */
5964 if (cmd && (cmd->command == UIC_CMD_DME_SET))
5965 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5966 }
9333d775 5967 retval |= IRQ_HANDLED;
ff8e20c6 5968 }
fb7b45f0 5969
e8e7f271
SRT		 5970 /* PA_INIT_ERROR is fatal and needs UIC reset */
5971 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 5972 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5973 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5974 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
ff8e20c6 5975
9333d775
VG		 5976 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5977 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5978 else if (hba->dev_quirks &
5979 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5980 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5981 hba->uic_error |=
5982 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5983 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5984 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5985 }
5986 retval |= IRQ_HANDLED;
583fa62d 5987 }
e8e7f271
SRT		 5988
5989 /* UIC NL/TL/DME errors needs software retry */
5990 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 5991 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5992 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5993 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
e8e7f271 5994 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 5995 retval |= IRQ_HANDLED;
ff8e20c6 5996 }
e8e7f271
SRT		 5997
5998 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 5999 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6000 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
48d5b973 6001 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
e8e7f271 6002 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 6003 retval |= IRQ_HANDLED;
ff8e20c6 6004 }
e8e7f271
SRT		 6005
6006 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 6007 if ((reg & UIC_DME_ERROR) &&
6008 (reg & UIC_DME_ERROR_CODE_MASK)) {
48d5b973 6009 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
e8e7f271 6010 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 6011 retval |= IRQ_HANDLED;
ff8e20c6 6012 }
e8e7f271
SRT		 6013
6014 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6015 __func__, hba->uic_error);
9333d775 6016 return retval;
e8e7f271
SRT		 6017}
6018
82174440
SC		 6019static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
6020 u32 intr_mask)
6021{
5a244e0e
SC		 6022 if (!ufshcd_is_auto_hibern8_supported(hba) ||
6023 !ufshcd_is_auto_hibern8_enabled(hba))
82174440
SC		 6024 return false;
6025
6026 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
6027 return false;
6028
6029 if (hba->active_uic_cmd &&
6030 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
6031 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
6032 return false;
6033
6034 return true;
6035}
6036
e8e7f271
SRT		 6037/**
6038 * ufshcd_check_errors - Check for errors that need s/w attention
6039 * @hba: per-adapter instance
9333d775
VG		 6040 *
6041 * Returns
6042 * IRQ_HANDLED - If interrupt is valid
6043 * IRQ_NONE - If invalid interrupt
e8e7f271 6044 */
9333d775 6045static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
e8e7f271
SRT		 6046{
6047 bool queue_eh_work = false;
9333d775 6048 irqreturn_t retval = IRQ_NONE;
e8e7f271 6049
d3c615bf
SC		 6050 if (hba->errors & INT_FATAL_ERRORS) {
6051 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
e8e7f271 6052 queue_eh_work = true;
d3c615bf 6053 }
7a3e97b0
SY		 6054
6055 if (hba->errors & UIC_ERROR) {
e8e7f271 6056 hba->uic_error = 0;
9333d775 6057 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 6058 if (hba->uic_error)
6059 queue_eh_work = true;
7a3e97b0 6060 }
e8e7f271 6061
82174440
SC		 6062 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6063 dev_err(hba->dev,
6064 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6065 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6066 "Enter" : "Exit",
6067 hba->errors, ufshcd_get_upmcrs(hba));
d3c615bf
SC		 6068 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
6069 hba->errors);
4db7a236 6070 ufshcd_set_link_broken(hba);
82174440
SC		 6071 queue_eh_work = true;
6072 }
6073
e8e7f271 6074 if (queue_eh_work) {
9a47ec7c
YG		 6075 /*
6076 * update the transfer error masks to sticky bits, let's do this
6077 * irrespective of current ufshcd_state.
6078 */
6079 hba->saved_err |= hba->errors;
6080 hba->saved_uic_err |= hba->uic_error;
6081
4db7a236 6082 /* dump controller state before resetting */
2355b66e
CG		 6083 if ((hba->saved_err & (INT_FATAL_ERRORS)) ||
6084 (hba->saved_uic_err &&
6085 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
4db7a236 6086 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
66cc820f
DR		 6087 __func__, hba->saved_err,
6088 hba->saved_uic_err);
c3be8d1e
CG		 6089 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6090 "host_regs: ");
4db7a236 6091 ufshcd_print_pwr_info(hba);
e8e7f271 6092 }
4db7a236 6093 ufshcd_schedule_eh_work(hba);
9333d775 6094 retval |= IRQ_HANDLED;
3441da7d 6095 }
e8e7f271
SRT		 6096 /*
6097 * if (!queue_eh_work) -
6098 * Other errors are either non-fatal where host recovers
6099 * itself without s/w intervention or errors that will be
6100 * handled by the SCSI core layer.
6101 */
9333d775 6102 return retval;
7a3e97b0
SY		 6103}
6104
69a6c269
BVA		 6105struct ctm_info {
6106 struct ufs_hba *hba;
6107 unsigned long pending;
6108 unsigned int ncpl;
6109};
6110
6111static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
6112{
6113 struct ctm_info *const ci = priv;
6114 struct completion *c;
6115
6116 WARN_ON_ONCE(reserved);
6117 if (test_bit(req->tag, &ci->pending))
6118 return true;
6119 ci->ncpl++;
6120 c = req->end_io_data;
6121 if (c)
6122 complete(c);
6123 return true;
6124}
6125
7a3e97b0
SY		 6126/**
6127 * ufshcd_tmc_handler - handle task management function completion
6128 * @hba: per adapter instance
9333d775
VG		 6129 *
6130 * Returns
6131 * IRQ_HANDLED - If interrupt is valid
6132 * IRQ_NONE - If invalid interrupt
7a3e97b0 6133 */
9333d775 6134static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 6135{
69a6c269
BVA		 6136 struct request_queue *q = hba->tmf_queue;
6137 struct ctm_info ci = {
6138 .hba = hba,
6139 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
6140 };
7a3e97b0 6141
69a6c269
BVA		 6142 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
6143 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
7a3e97b0
SY		 6144}
6145
6146/**
6147 * ufshcd_sl_intr - Interrupt service routine
6148 * @hba: per adapter instance
6149 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 6150 *
6151 * Returns
6152 * IRQ_HANDLED - If interrupt is valid
6153 * IRQ_NONE - If invalid interrupt
7a3e97b0 6154 */
9333d775 6155static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 6156{
9333d775
VG		 6157 irqreturn_t retval = IRQ_NONE;
6158
7a3e97b0 6159 hba->errors = UFSHCD_ERROR_MASK & intr_status;
82174440
SC		 6160
6161 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
6162 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
6163
7a3e97b0 6164 if (hba->errors)
9333d775 6165 retval |= ufshcd_check_errors(hba);
7a3e97b0 6166
53b3d9c3 6167 if (intr_status & UFSHCD_UIC_MASK)
9333d775 6168 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 6169
6170 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 6171 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 6172
6173 if (intr_status & UTP_TRANSFER_REQ_COMPL)
9333d775
VG		 6174 retval |= ufshcd_transfer_req_compl(hba);
6175
6176 return retval;
7a3e97b0
SY		 6177}
6178
6179/**
6180 * ufshcd_intr - Main interrupt service routine
6181 * @irq: irq number
6182 * @__hba: pointer to adapter instance
6183 *
9333d775
VG		 6184 * Returns
6185 * IRQ_HANDLED - If interrupt is valid
6186 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 6187 */
6188static irqreturn_t ufshcd_intr(int irq, void *__hba)
6189{
127d5f7c 6190 u32 intr_status, enabled_intr_status = 0;
7a3e97b0
SY		 6191 irqreturn_t retval = IRQ_NONE;
6192 struct ufs_hba *hba = __hba;
7f6ba4f1 6193 int retries = hba->nutrs;
7a3e97b0
SY		 6194
6195 spin_lock(hba->host->host_lock);
b873a275 6196 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
3f8af604
CG		 6197 hba->ufs_stats.last_intr_status = intr_status;
6198 hba->ufs_stats.last_intr_ts = ktime_get();
7a3e97b0 6199
7f6ba4f1
VG		 6200 /*
6201 * There could be max of hba->nutrs reqs in flight and in worst case
6202 * if the reqs get finished 1 by 1 after the interrupt status is
6203 * read, make sure we handle them by checking the interrupt status
6204 * again in a loop until we process all of the reqs before returning.
6205 */
127d5f7c 6206 while (intr_status && retries--) {
7f6ba4f1
VG		 6207 enabled_intr_status =
6208 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6209 if (intr_status)
6210 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 6211 if (enabled_intr_status)
6212 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 6213
6214 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
127d5f7c 6215 }
d75f7fe4 6216
6337f58c 6217 if (enabled_intr_status && retval == IRQ_NONE) {
9333d775
VG		 6218 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
6219 __func__, intr_status);
6220 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6221 }
6222
7a3e97b0
SY		 6223 spin_unlock(hba->host->host_lock);
6224 return retval;
6225}
6226
e2933132
SRT		 6227static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6228{
6229 int err = 0;
6230 u32 mask = 1 << tag;
6231 unsigned long flags;
6232
6233 if (!test_bit(tag, &hba->outstanding_tasks))
6234 goto out;
6235
6236 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 6237 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 6238 spin_unlock_irqrestore(hba->host->host_lock, flags);
6239
6240 /* poll for max. 1 sec to clear door bell register by h/w */
6241 err = ufshcd_wait_for_register(hba,
6242 REG_UTP_TASK_REQ_DOOR_BELL,
5cac1095 6243 mask, 0, 1000, 1000);
e2933132
SRT		 6244out:
6245 return err;
6246}
6247
c6049cd9
CH		 6248static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6249 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 6250{
69a6c269 6251 struct request_queue *q = hba->tmf_queue;
c6049cd9 6252 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 6253 DECLARE_COMPLETION_ONSTACK(wait);
6254 struct request *req;
7a3e97b0 6255 unsigned long flags;
c6049cd9 6256 int free_slot, task_tag, err;
7a3e97b0 6257
e2933132
SRT		 6258 /*
6259 * Get free slot, sleep if slots are unavailable.
6260 * Even though we use wait_event() which sleeps indefinitely,
6261 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
6262 */
69a6c269
BVA		 6263 req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
6264 req->end_io_data = &wait;
6265 free_slot = req->tag;
6266 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
1ab27c9c 6267 ufshcd_hold(hba, false);
7a3e97b0 6268
e2933132 6269 spin_lock_irqsave(host->host_lock, flags);
e2933132 6270 task_tag = hba->nutrs + free_slot;
7a3e97b0 6271
c6049cd9
CH		 6272 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
6273
6274 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 6275 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
6276
7a3e97b0
SY		 6277 /* send command to the controller */
6278 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 6279
6280 /* Make sure descriptors are ready before ringing the task doorbell */
6281 wmb();
6282
b873a275 6283 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 6284 /* Make sure that doorbell is committed immediately */
6285 wmb();
7a3e97b0
SY		 6286
6287 spin_unlock_irqrestore(host->host_lock, flags);
6288
6667e6d9
OS		 6289 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
6290
7a3e97b0 6291 /* wait until the task management command is completed */
69a6c269 6292 err = wait_for_completion_io_timeout(&wait,
e2933132 6293 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 6294 if (!err) {
69a6c269
BVA		 6295 /*
6296 * Make sure that ufshcd_compl_tm() does not trigger a
6297 * use-after-free.
6298 */
6299 req->end_io_data = NULL;
6667e6d9 6300 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
e2933132
SRT		 6301 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6302 __func__, tm_function);
6303 if (ufshcd_clear_tm_cmd(hba, free_slot))
6304 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
6305 __func__, free_slot);
6306 err = -ETIMEDOUT;
6307 } else {
c6049cd9
CH		 6308 err = 0;
6309 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
6310
6667e6d9 6311 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
7a3e97b0 6312 }
e2933132 6313
b557217c
SC		 6314 spin_lock_irqsave(hba->host->host_lock, flags);
6315 __clear_bit(free_slot, &hba->outstanding_tasks);
6316 spin_unlock_irqrestore(hba->host->host_lock, flags);
6317
69a6c269 6318 blk_put_request(req);
e2933132 6319
1ab27c9c 6320 ufshcd_release(hba);
7a3e97b0
SY		 6321 return err;
6322}
6323
c6049cd9
CH		 6324/**
6325 * ufshcd_issue_tm_cmd - issues task management commands to controller
6326 * @hba: per adapter instance
6327 * @lun_id: LUN ID to which TM command is sent
6328 * @task_id: task ID to which the TM command is applicable
6329 * @tm_function: task management function opcode
6330 * @tm_response: task management service response return value
6331 *
6332 * Returns non-zero value on error, zero on success.
6333 */
6334static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6335 u8 tm_function, u8 *tm_response)
6336{
6337 struct utp_task_req_desc treq = { { 0 }, };
6338 int ocs_value, err;
6339
6340 /* Configure task request descriptor */
6341 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6342 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6343
6344 /* Configure task request UPIU */
6345 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6346 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6347 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6348
6349 /*
6350 * The host shall provide the same value for LUN field in the basic
6351 * header and for Input Parameter.
6352 */
6353 treq.input_param1 = cpu_to_be32(lun_id);
6354 treq.input_param2 = cpu_to_be32(task_id);
6355
6356 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6357 if (err == -ETIMEDOUT)
6358 return err;
6359
6360 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6361 if (ocs_value != OCS_SUCCESS)
6362 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6363 __func__, ocs_value);
6364 else if (tm_response)
6365 *tm_response = be32_to_cpu(treq.output_param1) &
6366 MASK_TM_SERVICE_RESP;
6367 return err;
6368}
6369
5e0a86ee
AA		 6370/**
6371 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6372 * @hba: per-adapter instance
6373 * @req_upiu: upiu request
6374 * @rsp_upiu: upiu reply
5e0a86ee
AA		 6375 * @desc_buff: pointer to descriptor buffer, NULL if NA
6376 * @buff_len: descriptor size, 0 if NA
d0e9760d 6377 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 6378 * @desc_op: descriptor operation
6379 *
6380 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6381 * Therefore, it "rides" the device management infrastructure: uses its tag and
6382 * tasks work queues.
6383 *
6384 * Since there is only one available tag for device management commands,
6385 * the caller is expected to hold the hba->dev_cmd.lock mutex.
6386 */
6387static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6388 struct utp_upiu_req *req_upiu,
6389 struct utp_upiu_req *rsp_upiu,
6390 u8 *desc_buff, int *buff_len,
7f674c38 6391 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 6392 enum query_opcode desc_op)
6393{
7252a360
BVA		 6394 struct request_queue *q = hba->cmd_queue;
6395 struct request *req;
5e0a86ee
AA		 6396 struct ufshcd_lrb *lrbp;
6397 int err = 0;
6398 int tag;
6399 struct completion wait;
6400 unsigned long flags;
a23064c4 6401 u8 upiu_flags;
5e0a86ee
AA		 6402
6403 down_read(&hba->clk_scaling_lock);
6404
7252a360 6405 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 6406 if (IS_ERR(req)) {
6407 err = PTR_ERR(req);
6408 goto out_unlock;
6409 }
7252a360
BVA		 6410 tag = req->tag;
6411 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5e0a86ee
AA		 6412
6413 init_completion(&wait);
6414 lrbp = &hba->lrb[tag];
6415 WARN_ON(lrbp->cmd);
6416
6417 lrbp->cmd = NULL;
6418 lrbp->sense_bufflen = 0;
6419 lrbp->sense_buffer = NULL;
6420 lrbp->task_tag = tag;
6421 lrbp->lun = 0;
6422 lrbp->intr_cmd = true;
df043c74 6423 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5e0a86ee
AA		 6424 hba->dev_cmd.type = cmd_type;
6425
6426 switch (hba->ufs_version) {
6427 case UFSHCI_VERSION_10:
6428 case UFSHCI_VERSION_11:
6429 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6430 break;
6431 default:
6432 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6433 break;
6434 }
6435
6436 /* update the task tag in the request upiu */
6437 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6438
6439 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6440
6441 /* just copy the upiu request as it is */
6442 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6443 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6444 /* The Data Segment Area is optional depending upon the query
6445 * function value. for WRITE DESCRIPTOR, the data segment
6446 * follows right after the tsf.
6447 */
6448 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6449 *buff_len = 0;
6450 }
6451
6452 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6453
6454 hba->dev_cmd.complete = &wait;
6455
6456 /* Make sure descriptors are ready before ringing the doorbell */
6457 wmb();
6458 spin_lock_irqsave(hba->host->host_lock, flags);
6459 ufshcd_send_command(hba, tag);
6460 spin_unlock_irqrestore(hba->host->host_lock, flags);
6461
6462 /*
6463 * ignore the returning value here - ufshcd_check_query_response is
6464 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6465 * read the response directly ignoring all errors.
6466 */
6467 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6468
6469 /* just copy the upiu response as it is */
6470 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 6471 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6472 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6473 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6474 MASK_QUERY_DATA_SEG_LEN;
6475
6476 if (*buff_len >= resp_len) {
6477 memcpy(desc_buff, descp, resp_len);
6478 *buff_len = resp_len;
6479 } else {
3d4881d1
BH		 6480 dev_warn(hba->dev,
6481 "%s: rsp size %d is bigger than buffer size %d",
6482 __func__, resp_len, *buff_len);
4bbbe242
AA		 6483 *buff_len = 0;
6484 err = -EINVAL;
6485 }
6486 }
5e0a86ee 6487
7252a360 6488 blk_put_request(req);
bb14dd15 6489out_unlock:
5e0a86ee
AA		 6490 up_read(&hba->clk_scaling_lock);
6491 return err;
6492}
6493
6494/**
6495 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6496 * @hba: per-adapter instance
6497 * @req_upiu: upiu request
6498 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
6499 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
6500 * @desc_buff: pointer to descriptor buffer, NULL if NA
6501 * @buff_len: descriptor size, 0 if NA
6502 * @desc_op: descriptor operation
6503 *
6504 * Supports UTP Transfer requests (nop and query), and UTP Task
6505 * Management requests.
6506 * It is up to the caller to fill the upiu conent properly, as it will
6507 * be copied without any further input validations.
6508 */
6509int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6510 struct utp_upiu_req *req_upiu,
6511 struct utp_upiu_req *rsp_upiu,
6512 int msgcode,
6513 u8 *desc_buff, int *buff_len,
6514 enum query_opcode desc_op)
6515{
6516 int err;
7f674c38 6517 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee
AA		 6518 struct utp_task_req_desc treq = { { 0 }, };
6519 int ocs_value;
6520 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6521
5e0a86ee
AA		 6522 switch (msgcode) {
6523 case UPIU_TRANSACTION_NOP_OUT:
6524 cmd_type = DEV_CMD_TYPE_NOP;
df561f66 6525 fallthrough;
5e0a86ee
AA		 6526 case UPIU_TRANSACTION_QUERY_REQ:
6527 ufshcd_hold(hba, false);
6528 mutex_lock(&hba->dev_cmd.lock);
6529 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6530 desc_buff, buff_len,
6531 cmd_type, desc_op);
6532 mutex_unlock(&hba->dev_cmd.lock);
6533 ufshcd_release(hba);
6534
6535 break;
6536 case UPIU_TRANSACTION_TASK_REQ:
6537 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6538 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6539
6540 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6541
6542 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6543 if (err == -ETIMEDOUT)
6544 break;
6545
6546 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6547 if (ocs_value != OCS_SUCCESS) {
6548 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6549 ocs_value);
6550 break;
6551 }
6552
6553 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6554
6555 break;
6556 default:
6557 err = -EINVAL;
6558
6559 break;
6560 }
6561
5e0a86ee
AA		 6562 return err;
6563}
6564
7a3e97b0 6565/**
3441da7d
SRT		 6566 * ufshcd_eh_device_reset_handler - device reset handler registered to
6567 * scsi layer.
7a3e97b0
SY		 6568 * @cmd: SCSI command pointer
6569 *
6570 * Returns SUCCESS/FAILED
6571 */
3441da7d 6572static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6573{
6574 struct Scsi_Host *host;
6575 struct ufs_hba *hba;
6576 unsigned int tag;
6577 u32 pos;
6578 int err;
e2933132
SRT		 6579 u8 resp = 0xF;
6580 struct ufshcd_lrb *lrbp;
3441da7d 6581 unsigned long flags;
7a3e97b0
SY		 6582
6583 host = cmd->device->host;
6584 hba = shost_priv(host);
6585 tag = cmd->request->tag;
6586
e2933132
SRT		 6587 lrbp = &hba->lrb[tag];
6588 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6589 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6590 if (!err)
6591 err = resp;
7a3e97b0 6592 goto out;
e2933132 6593 }
7a3e97b0 6594
3441da7d
SRT		 6595 /* clear the commands that were pending for corresponding LUN */
6596 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6597 if (hba->lrb[pos].lun == lrbp->lun) {
6598 err = ufshcd_clear_cmd(hba, pos);
6599 if (err)
6600 break;
7a3e97b0 6601 }
3441da7d
SRT		 6602 }
6603 spin_lock_irqsave(host->host_lock, flags);
6604 ufshcd_transfer_req_compl(hba);
6605 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 6606
7a3e97b0 6607out:
7fabb77b 6608 hba->req_abort_count = 0;
8808b4e9 6609 ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
3441da7d
SRT		 6610 if (!err) {
6611 err = SUCCESS;
6612 } else {
6613 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6614 err = FAILED;
6615 }
7a3e97b0
SY		 6616 return err;
6617}
6618
e0b299e3
GB		 6619static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6620{
6621 struct ufshcd_lrb *lrbp;
6622 int tag;
6623
6624 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6625 lrbp = &hba->lrb[tag];
6626 lrbp->req_abort_skip = true;
6627 }
6628}
6629
7a3e97b0 6630/**
307348f6 6631 * ufshcd_try_to_abort_task - abort a specific task
d23ec0b6
LJ		 6632 * @hba: Pointer to adapter instance
6633 * @tag: Task tag/index to be aborted
7a3e97b0 6634 *
f20810d8
SRT		 6635 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6636 * command, and in host controller by clearing the door-bell register. There can
6637 * be race between controller sending the command to the device while abort is
6638 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6639 * really issued and then try to abort it.
6640 *
307348f6
CG		 6641 * Returns zero on success, non-zero on failure
6642 */
6643static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
6644{
6645 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6646 int err = 0;
6647 int poll_cnt;
6648 u8 resp = 0xF;
6649 u32 reg;
6650
6651 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6652 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6653 UFS_QUERY_TASK, &resp);
6654 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6655 /* cmd pending in the device */
6656 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6657 __func__, tag);
6658 break;
6659 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6660 /*
6661 * cmd not pending in the device, check if it is
6662 * in transition.
6663 */
6664 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6665 __func__, tag);
6666 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6667 if (reg & (1 << tag)) {
6668 /* sleep for max. 200us to stabilize */
6669 usleep_range(100, 200);
6670 continue;
6671 }
6672 /* command completed already */
6673 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6674 __func__, tag);
6675 goto out;
6676 } else {
6677 dev_err(hba->dev,
6678 "%s: no response from device. tag = %d, err %d\n",
6679 __func__, tag, err);
6680 if (!err)
6681 err = resp; /* service response error */
6682 goto out;
6683 }
6684 }
6685
6686 if (!poll_cnt) {
6687 err = -EBUSY;
6688 goto out;
6689 }
6690
6691 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6692 UFS_ABORT_TASK, &resp);
6693 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6694 if (!err) {
6695 err = resp; /* service response error */
6696 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6697 __func__, tag, err);
6698 }
6699 goto out;
6700 }
6701
6702 err = ufshcd_clear_cmd(hba, tag);
6703 if (err)
6704 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6705 __func__, tag, err);
6706
6707out:
6708 return err;
6709}
6710
6711/**
6712 * ufshcd_abort - scsi host template eh_abort_handler callback
6713 * @cmd: SCSI command pointer
6714 *
7a3e97b0
SY		 6715 * Returns SUCCESS/FAILED
6716 */
6717static int ufshcd_abort(struct scsi_cmnd *cmd)
6718{
6719 struct Scsi_Host *host;
6720 struct ufs_hba *hba;
6721 unsigned long flags;
6722 unsigned int tag;
f20810d8 6723 int err = 0;
e2933132 6724 struct ufshcd_lrb *lrbp;
e9d501b1 6725 u32 reg;
7a3e97b0
SY		 6726
6727 host = cmd->device->host;
6728 hba = shost_priv(host);
6729 tag = cmd->request->tag;
e7d38257 6730 lrbp = &hba->lrb[tag];
14497328
YG		 6731 if (!ufshcd_valid_tag(hba, tag)) {
6732 dev_err(hba->dev,
6733 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6734 __func__, tag, cmd, cmd->request);
6735 BUG();
6736 }
7a3e97b0 6737
e7d38257
DR		 6738 /*
6739 * Task abort to the device W-LUN is illegal. When this command
6740 * will fail, due to spec violation, scsi err handling next step
6741 * will be to send LU reset which, again, is a spec violation.
6742 * To avoid these unnecessary/illegal step we skip to the last error
6743 * handling stage: reset and restore.
6744 */
6745 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6746 return ufshcd_eh_host_reset_handler(cmd);
6747
1ab27c9c 6748 ufshcd_hold(hba, false);
14497328 6749 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 6750 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 6751 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6752 dev_err(hba->dev,
6753 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6754 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 6755 goto out;
14497328 6756 }
7a3e97b0 6757
66cc820f 6758 /* Print Transfer Request of aborted task */
d87a1f6d 6759 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 6760
7fabb77b
GB		 6761 /*
6762 * Print detailed info about aborted request.
6763 * As more than one request might get aborted at the same time,
6764 * print full information only for the first aborted request in order
6765 * to reduce repeated printouts. For other aborted requests only print
6766 * basic details.
6767 */
6768 scsi_print_command(hba->lrb[tag].cmd);
6769 if (!hba->req_abort_count) {
8808b4e9 6770 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
7fabb77b 6771 ufshcd_print_host_regs(hba);
6ba65588 6772 ufshcd_print_host_state(hba);
7fabb77b
GB		 6773 ufshcd_print_pwr_info(hba);
6774 ufshcd_print_trs(hba, 1 << tag, true);
6775 } else {
6776 ufshcd_print_trs(hba, 1 << tag, false);
6777 }
6778 hba->req_abort_count++;
e0b299e3 6779
d87a1f6d
BH		 6780 if (!(reg & (1 << tag))) {
6781 dev_err(hba->dev,
6782 "%s: cmd was completed, but without a notifying intr, tag = %d",
6783 __func__, tag);
6784 goto cleanup;
6785 }
6786
e0b299e3 6787 /* Skip task abort in case previous aborts failed and report failure */
02f74150 6788 if (lrbp->req_abort_skip)
e0b299e3 6789 err = -EIO;
02f74150
MP		 6790 else
6791 err = ufshcd_try_to_abort_task(hba, tag);
f20810d8 6792
f20810d8 6793 if (!err) {
b10178ee 6794cleanup:
02f74150
MP		 6795 spin_lock_irqsave(host->host_lock, flags);
6796 __ufshcd_transfer_req_compl(hba, (1UL << tag));
6797 spin_unlock_irqrestore(host->host_lock, flags);
7a3e97b0 6798out:
f20810d8
SRT		 6799 err = SUCCESS;
6800 } else {
6801 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6802 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6803 err = FAILED;
6804 }
6805
1ab27c9c
ST		 6806 /*
6807 * This ufshcd_release() corresponds to the original scsi cmd that got
6808 * aborted here (as we won't get any IRQ for it).
6809 */
6810 ufshcd_release(hba);
7a3e97b0
SY		 6811 return err;
6812}
6813
3441da7d
SRT		 6814/**
6815 * ufshcd_host_reset_and_restore - reset and restore host controller
6816 * @hba: per-adapter instance
6817 *
6818 * Note that host controller reset may issue DME_RESET to
6819 * local and remote (device) Uni-Pro stack and the attributes
6820 * are reset to default state.
6821 *
6822 * Returns zero on success, non-zero on failure
6823 */
6824static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6825{
6826 int err;
3441da7d
SRT		 6827 unsigned long flags;
6828
2df74b69
CG		 6829 /*
6830 * Stop the host controller and complete the requests
6831 * cleared by h/w
6832 */
5cac1095
BVA		 6833 ufshcd_hba_stop(hba);
6834
3441da7d 6835 spin_lock_irqsave(hba->host->host_lock, flags);
2df74b69
CG		 6836 hba->silence_err_logs = true;
6837 ufshcd_complete_requests(hba);
6838 hba->silence_err_logs = false;
3441da7d
SRT		 6839 spin_unlock_irqrestore(hba->host->host_lock, flags);
6840
a3cd5ec5 6841 /* scale up clocks to max frequency before full reinitialization */
394b949f 6842 ufshcd_set_clk_freq(hba, true);
a3cd5ec5 6843
3441da7d
SRT		 6844 err = ufshcd_hba_enable(hba);
6845 if (err)
6846 goto out;
6847
6848 /* Establish the link again and restore the device */
1b9e2141 6849 err = ufshcd_probe_hba(hba, false);
1d337ec2 6850
3441da7d
SRT		 6851out:
6852 if (err)
6853 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
8808b4e9 6854 ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
3441da7d
SRT		 6855 return err;
6856}
6857
6858/**
6859 * ufshcd_reset_and_restore - reset and re-initialize host/device
6860 * @hba: per-adapter instance
6861 *
6862 * Reset and recover device, host and re-establish link. This
6863 * is helpful to recover the communication in fatal error conditions.
6864 *
6865 * Returns zero on success, non-zero on failure
6866 */
6867static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6868{
4db7a236
CG		 6869 u32 saved_err;
6870 u32 saved_uic_err;
3441da7d 6871 int err = 0;
4db7a236 6872 unsigned long flags;
1d337ec2 6873 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 6874
4db7a236
CG		 6875 /*
6876 * This is a fresh start, cache and clear saved error first,
6877 * in case new error generated during reset and restore.
6878 */
6879 spin_lock_irqsave(hba->host->host_lock, flags);
6880 saved_err = hba->saved_err;
6881 saved_uic_err = hba->saved_uic_err;
6882 hba->saved_err = 0;
6883 hba->saved_uic_err = 0;
6884 spin_unlock_irqrestore(hba->host->host_lock, flags);
6885
1d337ec2 6886 do {
d8d9f793
BA		 6887 /* Reset the attached device */
6888 ufshcd_vops_device_reset(hba);
6889
1d337ec2
SRT		 6890 err = ufshcd_host_reset_and_restore(hba);
6891 } while (err && --retries);
3441da7d 6892
4db7a236
CG		 6893 spin_lock_irqsave(hba->host->host_lock, flags);
6894 /*
6895 * Inform scsi mid-layer that we did reset and allow to handle
6896 * Unit Attention properly.
6897 */
6898 scsi_report_bus_reset(hba->host, 0);
6899 if (err) {
6900 hba->saved_err |= saved_err;
6901 hba->saved_uic_err |= saved_uic_err;
6902 }
6903 spin_unlock_irqrestore(hba->host->host_lock, flags);
6904
3441da7d
SRT		 6905 return err;
6906}
6907
6908/**
6909 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 6910 * @cmd: SCSI command pointer
3441da7d
SRT		 6911 *
6912 * Returns SUCCESS/FAILED
6913 */
6914static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6915{
4db7a236 6916 int err = SUCCESS;
3441da7d
SRT		 6917 unsigned long flags;
6918 struct ufs_hba *hba;
6919
6920 hba = shost_priv(cmd->device->host);
6921
4db7a236
CG		 6922 spin_lock_irqsave(hba->host->host_lock, flags);
6923 hba->force_reset = true;
6924 ufshcd_schedule_eh_work(hba);
6925 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
3441da7d
SRT		 6926 spin_unlock_irqrestore(hba->host->host_lock, flags);
6927
4db7a236 6928 flush_work(&hba->eh_work);
3441da7d
SRT		 6929
6930 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236 6931 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
3441da7d 6932 err = FAILED;
3441da7d
SRT		 6933 spin_unlock_irqrestore(hba->host->host_lock, flags);
6934
6935 return err;
6936}
6937
3a4bf06d
YG		 6938/**
6939 * ufshcd_get_max_icc_level - calculate the ICC level
6940 * @sup_curr_uA: max. current supported by the regulator
6941 * @start_scan: row at the desc table to start scan from
6942 * @buff: power descriptor buffer
6943 *
6944 * Returns calculated max ICC level for specific regulator
6945 */
6946static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6947{
6948 int i;
6949 int curr_uA;
6950 u16 data;
6951 u16 unit;
6952
6953 for (i = start_scan; i >= 0; i--) {
d79713f9 6954 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 6955 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6956 ATTR_ICC_LVL_UNIT_OFFSET;
6957 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6958 switch (unit) {
6959 case UFSHCD_NANO_AMP:
6960 curr_uA = curr_uA / 1000;
6961 break;
6962 case UFSHCD_MILI_AMP:
6963 curr_uA = curr_uA * 1000;
6964 break;
6965 case UFSHCD_AMP:
6966 curr_uA = curr_uA * 1000 * 1000;
6967 break;
6968 case UFSHCD_MICRO_AMP:
6969 default:
6970 break;
6971 }
6972 if (sup_curr_uA >= curr_uA)
6973 break;
6974 }
6975 if (i < 0) {
6976 i = 0;
6977 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6978 }
6979
6980 return (u32)i;
6981}
6982
6983/**
6984 * ufshcd_calc_icc_level - calculate the max ICC level
6985 * In case regulators are not initialized we'll return 0
6986 * @hba: per-adapter instance
6987 * @desc_buf: power descriptor buffer to extract ICC levels from.
6988 * @len: length of desc_buff
6989 *
6990 * Returns calculated ICC level
6991 */
6992static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6993 u8 *desc_buf, int len)
6994{
6995 u32 icc_level = 0;
6996
6997 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6998 !hba->vreg_info.vccq2) {
6999 dev_err(hba->dev,
7000 "%s: Regulator capability was not set, actvIccLevel=%d",
7001 __func__, icc_level);
7002 goto out;
7003 }
7004
0487fff7 7005 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 7006 icc_level = ufshcd_get_max_icc_level(
7007 hba->vreg_info.vcc->max_uA,
7008 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7009 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7010
0487fff7 7011 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 7012 icc_level = ufshcd_get_max_icc_level(
7013 hba->vreg_info.vccq->max_uA,
7014 icc_level,
7015 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7016
0487fff7 7017 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 7018 icc_level = ufshcd_get_max_icc_level(
7019 hba->vreg_info.vccq2->max_uA,
7020 icc_level,
7021 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7022out:
7023 return icc_level;
7024}
7025
e89860f1 7026static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
3a4bf06d
YG		 7027{
7028 int ret;
7a0bf85b 7029 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
bbe21d7a 7030 u8 *desc_buf;
e89860f1 7031 u32 icc_level;
bbe21d7a
KC		 7032
7033 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7034 if (!desc_buf)
7035 return;
3a4bf06d 7036
c4607a09
BH		 7037 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7038 desc_buf, buff_len);
3a4bf06d
YG		 7039 if (ret) {
7040 dev_err(hba->dev,
7041 "%s: Failed reading power descriptor.len = %d ret = %d",
7042 __func__, buff_len, ret);
bbe21d7a 7043 goto out;
3a4bf06d
YG		 7044 }
7045
e89860f1
CG		 7046 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7047 buff_len);
7048 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
3a4bf06d 7049
dbd34a61 7050 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
e89860f1 7051 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
3a4bf06d
YG		 7052
7053 if (ret)
7054 dev_err(hba->dev,
7055 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
e89860f1 7056 __func__, icc_level, ret);
3a4bf06d 7057
bbe21d7a
KC		 7058out:
7059 kfree(desc_buf);
3a4bf06d
YG		 7060}
7061
fb276f77
CG		 7062static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7063{
7064 scsi_autopm_get_device(sdev);
7065 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7066 if (sdev->rpm_autosuspend)
7067 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7068 RPM_AUTOSUSPEND_DELAY_MS);
7069 scsi_autopm_put_device(sdev);
7070}
7071
2a8fa600
SJ		 7072/**
7073 * ufshcd_scsi_add_wlus - Adds required W-LUs
7074 * @hba: per-adapter instance
7075 *
7076 * UFS device specification requires the UFS devices to support 4 well known
7077 * logical units:
7078 * "REPORT_LUNS" (address: 01h)
7079 * "UFS Device" (address: 50h)
7080 * "RPMB" (address: 44h)
7081 * "BOOT" (address: 30h)
7082 * UFS device's power management needs to be controlled by "POWER CONDITION"
7083 * field of SSU (START STOP UNIT) command. But this "power condition" field
7084 * will take effect only when its sent to "UFS device" well known logical unit
7085 * hence we require the scsi_device instance to represent this logical unit in
7086 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 7087 *
2a8fa600
SJ		 7088 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7089 * Block) LU so user space process can control this LU. User space may also
7090 * want to have access to BOOT LU.
8aa29f19 7091 *
2a8fa600
SJ		 7092 * This function adds scsi device instances for each of all well known LUs
7093 * (except "REPORT LUNS" LU).
7094 *
7095 * Returns zero on success (all required W-LUs are added successfully),
7096 * non-zero error value on failure (if failed to add any of the required W-LU).
7097 */
7098static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7099{
7100 int ret = 0;
7c48bfd0 7101 struct scsi_device *sdev_boot;
2a8fa600
SJ		 7102
7103 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
7104 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7105 if (IS_ERR(hba->sdev_ufs_device)) {
7106 ret = PTR_ERR(hba->sdev_ufs_device);
7107 hba->sdev_ufs_device = NULL;
7108 goto out;
7109 }
fb276f77 7110 ufshcd_blk_pm_runtime_init(hba->sdev_ufs_device);
7c48bfd0 7111 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 7112
4f3e900b 7113 hba->sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 7114 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
4f3e900b
JK		 7115 if (IS_ERR(hba->sdev_rpmb)) {
7116 ret = PTR_ERR(hba->sdev_rpmb);
3d21fbde 7117 goto remove_sdev_ufs_device;
2a8fa600 7118 }
4f3e900b
JK		 7119 ufshcd_blk_pm_runtime_init(hba->sdev_rpmb);
7120 scsi_device_put(hba->sdev_rpmb);
3d21fbde
HK		 7121
7122 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7123 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
fb276f77 7124 if (IS_ERR(sdev_boot)) {
3d21fbde 7125 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
fb276f77
CG		 7126 } else {
7127 ufshcd_blk_pm_runtime_init(sdev_boot);
3d21fbde 7128 scsi_device_put(sdev_boot);
fb276f77 7129 }
2a8fa600
SJ		 7130 goto out;
7131
2a8fa600
SJ		 7132remove_sdev_ufs_device:
7133 scsi_remove_device(hba->sdev_ufs_device);
7134out:
7135 return ret;
7136}
7137
3d17b9b5
AD		 7138static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7139{
a7f1e69d 7140 struct ufs_dev_info *dev_info = &hba->dev_info;
6f8d5a6a
SC		 7141 u8 lun;
7142 u32 d_lu_wb_buf_alloc;
7143
817d7e14
SC		 7144 if (!ufshcd_is_wb_allowed(hba))
7145 return;
a7f1e69d
SC		 7146 /*
7147 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7148 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7149 * enabled
7150 */
7151 if (!(dev_info->wspecversion >= 0x310 ||
7152 dev_info->wspecversion == 0x220 ||
7153 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7154 goto wb_disabled;
817d7e14 7155
7a0bf85b
BH		 7156 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7157 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
817d7e14
SC		 7158 goto wb_disabled;
7159
a7f1e69d 7160 dev_info->d_ext_ufs_feature_sup =
3d17b9b5
AD		 7161 get_unaligned_be32(desc_buf +
7162 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
817d7e14 7163
a7f1e69d 7164 if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
817d7e14
SC		 7165 goto wb_disabled;
7166
3d17b9b5
AD		 7167 /*
7168 * WB may be supported but not configured while provisioning.
7169 * The spec says, in dedicated wb buffer mode,
7170 * a max of 1 lun would have wb buffer configured.
7171 * Now only shared buffer mode is supported.
7172 */
a7f1e69d 7173 dev_info->b_wb_buffer_type =
3d17b9b5
AD		 7174 desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7175
a7f1e69d 7176 dev_info->b_presrv_uspc_en =
3d17b9b5
AD		 7177 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7178
a7f1e69d
SC		 7179 if (dev_info->b_wb_buffer_type == WB_BUF_MODE_SHARED) {
7180 dev_info->d_wb_alloc_units =
6f8d5a6a
SC		 7181 get_unaligned_be32(desc_buf +
7182 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS);
a7f1e69d 7183 if (!dev_info->d_wb_alloc_units)
6f8d5a6a
SC		 7184 goto wb_disabled;
7185 } else {
7186 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7187 d_lu_wb_buf_alloc = 0;
7188 ufshcd_read_unit_desc_param(hba,
7189 lun,
7190 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7191 (u8 *)&d_lu_wb_buf_alloc,
7192 sizeof(d_lu_wb_buf_alloc));
7193 if (d_lu_wb_buf_alloc) {
a7f1e69d 7194 dev_info->wb_dedicated_lu = lun;
6f8d5a6a
SC		 7195 break;
7196 }
7197 }
817d7e14 7198
6f8d5a6a
SC		 7199 if (!d_lu_wb_buf_alloc)
7200 goto wb_disabled;
7201 }
817d7e14
SC		 7202 return;
7203
7204wb_disabled:
7205 hba->caps &= ~UFSHCD_CAP_WB_EN;
7206}
7207
8db269a5 7208void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
817d7e14
SC		 7209{
7210 struct ufs_dev_fix *f;
7211 struct ufs_dev_info *dev_info = &hba->dev_info;
7212
8db269a5
SC		 7213 if (!fixups)
7214 return;
7215
7216 for (f = fixups; f->quirk; f++) {
817d7e14
SC		 7217 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7218 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7219 ((dev_info->model &&
7220 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7221 !strcmp(f->model, UFS_ANY_MODEL)))
7222 hba->dev_quirks |= f->quirk;
7223 }
3d17b9b5 7224}
8db269a5 7225EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
3d17b9b5 7226
c28c00ba
SC		 7227static void ufs_fixup_device_setup(struct ufs_hba *hba)
7228{
7229 /* fix by general quirk table */
8db269a5 7230 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
c28c00ba
SC		 7231
7232 /* allow vendors to fix quirks */
7233 ufshcd_vops_fixup_dev_quirks(hba);
7234}
7235
09750066 7236static int ufs_get_device_desc(struct ufs_hba *hba)
c58ab7aa
YG		 7237{
7238 int err;
7239 u8 model_index;
bbe21d7a 7240 u8 *desc_buf;
09750066 7241 struct ufs_dev_info *dev_info = &hba->dev_info;
4b828fe1 7242
458a45f5 7243 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
bbe21d7a
KC		 7244 if (!desc_buf) {
7245 err = -ENOMEM;
7246 goto out;
7247 }
c58ab7aa 7248
c4607a09 7249 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7a0bf85b 7250 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
c58ab7aa
YG		 7251 if (err) {
7252 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7253 __func__, err);
7254 goto out;
7255 }
7256
7257 /*
7258 * getting vendor (manufacturerID) and Bank Index in big endian
7259 * format
7260 */
09750066 7261 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 7262 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7263
09f17791
CG		 7264 /* getting Specification Version in big endian format */
7265 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7266 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7267
c58ab7aa 7268 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
3d17b9b5 7269
4b828fe1 7270 err = ufshcd_read_string_desc(hba, model_index,
09750066 7271 &dev_info->model, SD_ASCII_STD);
4b828fe1 7272 if (err < 0) {
c58ab7aa
YG		 7273 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7274 __func__, err);
7275 goto out;
7276 }
7277
817d7e14
SC		 7278 ufs_fixup_device_setup(hba);
7279
a7f1e69d 7280 ufshcd_wb_probe(hba, desc_buf);
817d7e14 7281
4b828fe1
TW		 7282 /*
7283 * ufshcd_read_string_desc returns size of the string
7284 * reset the error value
7285 */
7286 err = 0;
c58ab7aa
YG		 7287
7288out:
bbe21d7a 7289 kfree(desc_buf);
c58ab7aa
YG		 7290 return err;
7291}
7292
09750066 7293static void ufs_put_device_desc(struct ufs_hba *hba)
4b828fe1 7294{
09750066
BH		 7295 struct ufs_dev_info *dev_info = &hba->dev_info;
7296
7297 kfree(dev_info->model);
7298 dev_info->model = NULL;
4b828fe1
TW		 7299}
7300
37113106
YG		 7301/**
7302 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7303 * @hba: per-adapter instance
7304 *
7305 * PA_TActivate parameter can be tuned manually if UniPro version is less than
7306 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7307 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7308 * the hibern8 exit latency.
7309 *
7310 * Returns zero on success, non-zero error value on failure.
7311 */
7312static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7313{
7314 int ret = 0;
7315 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7316
7317 ret = ufshcd_dme_peer_get(hba,
7318 UIC_ARG_MIB_SEL(
7319 RX_MIN_ACTIVATETIME_CAPABILITY,
7320 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7321 &peer_rx_min_activatetime);
7322 if (ret)
7323 goto out;
7324
7325 /* make sure proper unit conversion is applied */
7326 tuned_pa_tactivate =
7327 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7328 / PA_TACTIVATE_TIME_UNIT_US);
7329 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7330 tuned_pa_tactivate);
7331
7332out:
7333 return ret;
7334}
7335
7336/**
7337 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7338 * @hba: per-adapter instance
7339 *
7340 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7341 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7342 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7343 * This optimal value can help reduce the hibern8 exit latency.
7344 *
7345 * Returns zero on success, non-zero error value on failure.
7346 */
7347static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7348{
7349 int ret = 0;
7350 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7351 u32 max_hibern8_time, tuned_pa_hibern8time;
7352
7353 ret = ufshcd_dme_get(hba,
7354 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7355 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7356 &local_tx_hibern8_time_cap);
7357 if (ret)
7358 goto out;
7359
7360 ret = ufshcd_dme_peer_get(hba,
7361 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7362 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7363 &peer_rx_hibern8_time_cap);
7364 if (ret)
7365 goto out;
7366
7367 max_hibern8_time = max(local_tx_hibern8_time_cap,
7368 peer_rx_hibern8_time_cap);
7369 /* make sure proper unit conversion is applied */
7370 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7371 / PA_HIBERN8_TIME_UNIT_US);
7372 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7373 tuned_pa_hibern8time);
7374out:
7375 return ret;
7376}
7377
c6a6db43 7378/**
7379 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7380 * less than device PA_TACTIVATE time.
7381 * @hba: per-adapter instance
7382 *
7383 * Some UFS devices require host PA_TACTIVATE to be lower than device
7384 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7385 * for such devices.
7386 *
7387 * Returns zero on success, non-zero error value on failure.
7388 */
7389static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7390{
7391 int ret = 0;
7392 u32 granularity, peer_granularity;
7393 u32 pa_tactivate, peer_pa_tactivate;
7394 u32 pa_tactivate_us, peer_pa_tactivate_us;
7395 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7396
7397 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7398 &granularity);
7399 if (ret)
7400 goto out;
7401
7402 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7403 &peer_granularity);
7404 if (ret)
7405 goto out;
7406
7407 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7408 (granularity > PA_GRANULARITY_MAX_VAL)) {
7409 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7410 __func__, granularity);
7411 return -EINVAL;
7412 }
7413
7414 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7415 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7416 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7417 __func__, peer_granularity);
7418 return -EINVAL;
7419 }
7420
7421 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7422 if (ret)
7423 goto out;
7424
7425 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7426 &peer_pa_tactivate);
7427 if (ret)
7428 goto out;
7429
7430 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7431 peer_pa_tactivate_us = peer_pa_tactivate *
7432 gran_to_us_table[peer_granularity - 1];
7433
7434 if (pa_tactivate_us > peer_pa_tactivate_us) {
7435 u32 new_peer_pa_tactivate;
7436
7437 new_peer_pa_tactivate = pa_tactivate_us /
7438 gran_to_us_table[peer_granularity - 1];
7439 new_peer_pa_tactivate++;
7440 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7441 new_peer_pa_tactivate);
7442 }
7443
7444out:
7445 return ret;
7446}
7447
09750066 7448static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
37113106
YG		 7449{
7450 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7451 ufshcd_tune_pa_tactivate(hba);
7452 ufshcd_tune_pa_hibern8time(hba);
7453 }
7454
e91ed9e0
CG		 7455 ufshcd_vops_apply_dev_quirks(hba);
7456
37113106
YG		 7457 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7458 /* set 1ms timeout for PA_TACTIVATE */
7459 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43 7460
7461 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7462 ufshcd_quirk_tune_host_pa_tactivate(hba);
37113106
YG		 7463}
7464
ff8e20c6
DR		 7465static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7466{
ff8e20c6
DR		 7467 hba->ufs_stats.hibern8_exit_cnt = 0;
7468 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 7469 hba->req_abort_count = 0;
ff8e20c6
DR		 7470}
7471
731f0621
BH		 7472static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7473{
7474 int err;
7475 size_t buff_len;
7476 u8 *desc_buf;
7477
7a0bf85b 7478 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
731f0621
BH		 7479 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7480 if (!desc_buf) {
7481 err = -ENOMEM;
7482 goto out;
7483 }
7484
c4607a09
BH		 7485 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7486 desc_buf, buff_len);
731f0621
BH		 7487 if (err) {
7488 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7489 __func__, err);
7490 goto out;
7491 }
7492
7493 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7494 hba->dev_info.max_lu_supported = 32;
7495 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7496 hba->dev_info.max_lu_supported = 8;
7497
7498out:
7499 kfree(desc_buf);
7500 return err;
7501}
7502
9e1e8a75
SJ		 7503static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7504 {19200000, REF_CLK_FREQ_19_2_MHZ},
7505 {26000000, REF_CLK_FREQ_26_MHZ},
7506 {38400000, REF_CLK_FREQ_38_4_MHZ},
7507 {52000000, REF_CLK_FREQ_52_MHZ},
7508 {0, REF_CLK_FREQ_INVAL},
7509};
7510
7511static enum ufs_ref_clk_freq
7512ufs_get_bref_clk_from_hz(unsigned long freq)
7513{
7514 int i;
7515
7516 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7517 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7518 return ufs_ref_clk_freqs[i].val;
7519
7520 return REF_CLK_FREQ_INVAL;
7521}
7522
7523void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7524{
7525 unsigned long freq;
7526
7527 freq = clk_get_rate(refclk);
7528
7529 hba->dev_ref_clk_freq =
7530 ufs_get_bref_clk_from_hz(freq);
7531
7532 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7533 dev_err(hba->dev,
7534 "invalid ref_clk setting = %ld\n", freq);
7535}
7536
7537static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
7538{
7539 int err;
7540 u32 ref_clk;
7541 u32 freq = hba->dev_ref_clk_freq;
7542
7543 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
7544 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
7545
7546 if (err) {
7547 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
7548 err);
7549 goto out;
7550 }
7551
7552 if (ref_clk == freq)
7553 goto out; /* nothing to update */
7554
7555 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7556 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
7557
7558 if (err) {
7559 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
7560 ufs_ref_clk_freqs[freq].freq_hz);
7561 goto out;
7562 }
7563
7564 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
7565 ufs_ref_clk_freqs[freq].freq_hz);
7566
7567out:
7568 return err;
7569}
7570
1b9e2141
BH		 7571static int ufshcd_device_params_init(struct ufs_hba *hba)
7572{
7573 bool flag;
7a0bf85b 7574 int ret, i;
1b9e2141 7575
7a0bf85b
BH		 7576 /* Init device descriptor sizes */
7577 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
7578 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
1b9e2141 7579
731f0621
BH		 7580 /* Init UFS geometry descriptor related parameters */
7581 ret = ufshcd_device_geo_params_init(hba);
7582 if (ret)
7583 goto out;
7584
1b9e2141
BH		 7585 /* Check and apply UFS device quirks */
7586 ret = ufs_get_device_desc(hba);
7587 if (ret) {
7588 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
7589 __func__, ret);
7590 goto out;
7591 }
7592
09f17791
CG		 7593 ufshcd_get_ref_clk_gating_wait(hba);
7594
1b9e2141 7595 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
1f34eedf 7596 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
1b9e2141
BH		 7597 hba->dev_info.f_power_on_wp_en = flag;
7598
2b35b2ad
BH		 7599 /* Probe maximum power mode co-supported by both UFS host and device */
7600 if (ufshcd_get_max_pwr_mode(hba))
7601 dev_err(hba->dev,
7602 "%s: Failed getting max supported power mode\n",
7603 __func__);
1b9e2141
BH		 7604out:
7605 return ret;
7606}
7607
7608/**
7609 * ufshcd_add_lus - probe and add UFS logical units
7610 * @hba: per-adapter instance
7611 */
7612static int ufshcd_add_lus(struct ufs_hba *hba)
7613{
7614 int ret;
7615
1b9e2141
BH		 7616 /* Add required well known logical units to scsi mid layer */
7617 ret = ufshcd_scsi_add_wlus(hba);
7618 if (ret)
7619 goto out;
7620
7621 /* Initialize devfreq after UFS device is detected */
7622 if (ufshcd_is_clkscaling_supported(hba)) {
7623 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7624 &hba->pwr_info,
7625 sizeof(struct ufs_pa_layer_attr));
7626 hba->clk_scaling.saved_pwr_info.is_valid = true;
7627 if (!hba->devfreq) {
7628 ret = ufshcd_devfreq_init(hba);
7629 if (ret)
7630 goto out;
7631 }
7632
7633 hba->clk_scaling.is_allowed = true;
7634 }
7635
7636 ufs_bsg_probe(hba);
7637 scsi_scan_host(hba->host);
7638 pm_runtime_put_sync(hba->dev);
7639
1b9e2141
BH		 7640out:
7641 return ret;
7642}
7643
4f3e900b
JK		 7644static int
7645ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp);
7646
7647static int ufshcd_clear_ua_wlun(struct ufs_hba *hba, u8 wlun)
7648{
7649 struct scsi_device *sdp;
7650 unsigned long flags;
7651 int ret = 0;
7652
7653 spin_lock_irqsave(hba->host->host_lock, flags);
7654 if (wlun == UFS_UPIU_UFS_DEVICE_WLUN)
7655 sdp = hba->sdev_ufs_device;
7656 else if (wlun == UFS_UPIU_RPMB_WLUN)
7657 sdp = hba->sdev_rpmb;
7658 else
7659 BUG_ON(1);
7660 if (sdp) {
7661 ret = scsi_device_get(sdp);
7662 if (!ret && !scsi_device_online(sdp)) {
7663 ret = -ENODEV;
7664 scsi_device_put(sdp);
7665 }
7666 } else {
7667 ret = -ENODEV;
7668 }
7669 spin_unlock_irqrestore(hba->host->host_lock, flags);
7670 if (ret)
7671 goto out_err;
7672
7673 ret = ufshcd_send_request_sense(hba, sdp);
7674 scsi_device_put(sdp);
7675out_err:
7676 if (ret)
7677 dev_err(hba->dev, "%s: UAC clear LU=%x ret = %d\n",
7678 __func__, wlun, ret);
7679 return ret;
7680}
7681
7682static int ufshcd_clear_ua_wluns(struct ufs_hba *hba)
7683{
7684 int ret = 0;
7685
7686 if (!hba->wlun_dev_clr_ua)
7687 goto out;
7688
7689 ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN);
7690 if (!ret)
7691 ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN);
7692 if (!ret)
7693 hba->wlun_dev_clr_ua = false;
7694out:
7695 if (ret)
7696 dev_err(hba->dev, "%s: Failed to clear UAC WLUNS ret = %d\n",
7697 __func__, ret);
7698 return ret;
7699}
7700
6ccf44fe 7701/**
1d337ec2
SRT		 7702 * ufshcd_probe_hba - probe hba to detect device and initialize
7703 * @hba: per-adapter instance
1b9e2141 7704 * @async: asynchronous execution or not
1d337ec2
SRT		 7705 *
7706 * Execute link-startup and verify device initialization
6ccf44fe 7707 */
1b9e2141 7708static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
6ccf44fe 7709{
6ccf44fe 7710 int ret;
4db7a236 7711 unsigned long flags;
7ff5ab47 7712 ktime_t start = ktime_get();
6ccf44fe
SJ		 7713
7714 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 7715 if (ret)
7716 goto out;
7717
ff8e20c6
DR		 7718 /* Debug counters initialization */
7719 ufshcd_clear_dbg_ufs_stats(hba);
7720
57d104c1
SJ		 7721 /* UniPro link is active now */
7722 ufshcd_set_link_active(hba);
d3e89bac 7723
1b9e2141 7724 /* Verify device initialization by sending NOP OUT UPIU */
5a0b0cb9
SRT		 7725 ret = ufshcd_verify_dev_init(hba);
7726 if (ret)
7727 goto out;
68078d5c 7728
1b9e2141 7729 /* Initiate UFS initialization, and waiting until completion */
68078d5c
DR		 7730 ret = ufshcd_complete_dev_init(hba);
7731 if (ret)
7732 goto out;
5a0b0cb9 7733
1b9e2141
BH		 7734 /*
7735 * Initialize UFS device parameters used by driver, these
7736 * parameters are associated with UFS descriptors.
7737 */
7738 if (async) {
7739 ret = ufshcd_device_params_init(hba);
7740 if (ret)
7741 goto out;
93fdd5ac
TW		 7742 }
7743
09750066 7744 ufshcd_tune_unipro_params(hba);
4b828fe1 7745
57d104c1
SJ		 7746 /* UFS device is also active now */
7747 ufshcd_set_ufs_dev_active(hba);
66ec6d59 7748 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 7749 hba->wlun_dev_clr_ua = true;
7750
2b35b2ad
BH		 7751 /* Gear up to HS gear if supported */
7752 if (hba->max_pwr_info.is_valid) {
9e1e8a75
SJ		 7753 /*
7754 * Set the right value to bRefClkFreq before attempting to
7755 * switch to HS gears.
7756 */
7757 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7758 ufshcd_set_dev_ref_clk(hba);
7eb584db 7759 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 7760 if (ret) {
7eb584db
DR		 7761 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7762 __func__, ret);
8643ae66
DL		 7763 goto out;
7764 }
6a9df818 7765 ufshcd_print_pwr_info(hba);
7eb584db 7766 }
57d104c1 7767
e89860f1
CG		 7768 /*
7769 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
7770 * and for removable UFS card as well, hence always set the parameter.
7771 * Note: Error handler may issue the device reset hence resetting
7772 * bActiveICCLevel as well so it is always safe to set this here.
7773 */
7774 ufshcd_set_active_icc_lvl(hba);
7775
3d17b9b5 7776 ufshcd_wb_config(hba);
71d848b8
CG		 7777 /* Enable Auto-Hibernate if configured */
7778 ufshcd_auto_hibern8_enable(hba);
7779
5a0b0cb9 7780out:
4db7a236
CG		 7781 spin_lock_irqsave(hba->host->host_lock, flags);
7782 if (ret)
7783 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7784 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
7785 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7786 spin_unlock_irqrestore(hba->host->host_lock, flags);
1d337ec2 7787
7ff5ab47 7788 trace_ufshcd_init(dev_name(hba->dev), ret,
7789 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7790 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 7791 return ret;
7792}
7793
7794/**
7795 * ufshcd_async_scan - asynchronous execution for probing hba
7796 * @data: data pointer to pass to this function
7797 * @cookie: cookie data
7798 */
7799static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7800{
7801 struct ufs_hba *hba = (struct ufs_hba *)data;
1b9e2141 7802 int ret;
1d337ec2 7803
1b9e2141
BH		 7804 /* Initialize hba, detect and initialize UFS device */
7805 ret = ufshcd_probe_hba(hba, true);
7806 if (ret)
7807 goto out;
7808
7809 /* Probe and add UFS logical units */
7810 ret = ufshcd_add_lus(hba);
7811out:
7812 /*
7813 * If we failed to initialize the device or the device is not
7814 * present, turn off the power/clocks etc.
7815 */
7816 if (ret) {
7817 pm_runtime_put_sync(hba->dev);
7818 ufshcd_exit_clk_scaling(hba);
7819 ufshcd_hba_exit(hba);
4f3e900b
JK		 7820 } else {
7821 ufshcd_clear_ua_wluns(hba);
1b9e2141 7822 }
6ccf44fe
SJ		 7823}
7824
d829fc8a
SN		 7825static const struct attribute_group *ufshcd_driver_groups[] = {
7826 &ufs_sysfs_unit_descriptor_group,
ec92b59c 7827 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 7828 NULL,
7829};
7830
90b8491c
SC		 7831static struct ufs_hba_variant_params ufs_hba_vps = {
7832 .hba_enable_delay_us = 1000,
d14734ae 7833 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
90b8491c
SC		 7834 .devfreq_profile.polling_ms = 100,
7835 .devfreq_profile.target = ufshcd_devfreq_target,
7836 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
7837 .ondemand_data.upthreshold = 70,
7838 .ondemand_data.downdifferential = 5,
7839};
7840
7a3e97b0
SY		 7841static struct scsi_host_template ufshcd_driver_template = {
7842 .module = THIS_MODULE,
7843 .name = UFSHCD,
7844 .proc_name = UFSHCD,
7845 .queuecommand = ufshcd_queuecommand,
7846 .slave_alloc = ufshcd_slave_alloc,
eeda4749 7847 .slave_configure = ufshcd_slave_configure,
7a3e97b0 7848 .slave_destroy = ufshcd_slave_destroy,
4264fd61 7849 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 7850 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 7851 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7852 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
7a3e97b0
SY		 7853 .this_id = -1,
7854 .sg_tablesize = SG_ALL,
7855 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
7856 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 7857 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 7858 .max_host_blocked = 1,
c40ecc12 7859 .track_queue_depth = 1,
d829fc8a 7860 .sdev_groups = ufshcd_driver_groups,
4af14d11 7861 .dma_boundary = PAGE_SIZE - 1,
49615ba1 7862 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 7863};
7864
57d104c1
SJ		 7865static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7866 int ua)
7867{
7b16a07c 7868 int ret;
57d104c1 7869
7b16a07c
BA		 7870 if (!vreg)
7871 return 0;
57d104c1 7872
0487fff7
SC		 7873 /*
7874 * "set_load" operation shall be required on those regulators
7875 * which specifically configured current limitation. Otherwise
7876 * zero max_uA may cause unexpected behavior when regulator is
7877 * enabled or set as high power mode.
7878 */
7879 if (!vreg->max_uA)
7880 return 0;
7881
7b16a07c
BA		 7882 ret = regulator_set_load(vreg->reg, ua);
7883 if (ret < 0) {
7884 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7885 __func__, vreg->name, ua, ret);
57d104c1
SJ		 7886 }
7887
7888 return ret;
7889}
7890
7891static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7892 struct ufs_vreg *vreg)
7893{
73067981 7894 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 7895}
7896
7897static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7898 struct ufs_vreg *vreg)
7899{
7c7cfdcf
AH		 7900 if (!vreg)
7901 return 0;
7902
73067981 7903 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 7904}
7905
aa497613
SRT		 7906static int ufshcd_config_vreg(struct device *dev,
7907 struct ufs_vreg *vreg, bool on)
7908{
7909 int ret = 0;
72753590
GS		 7910 struct regulator *reg;
7911 const char *name;
aa497613
SRT		 7912 int min_uV, uA_load;
7913
7914 BUG_ON(!vreg);
7915
72753590
GS		 7916 reg = vreg->reg;
7917 name = vreg->name;
7918
aa497613 7919 if (regulator_count_voltages(reg) > 0) {
90d88f47
AD		 7920 uA_load = on ? vreg->max_uA : 0;
7921 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7922 if (ret)
7923 goto out;
7924
3b141e8c
SC		 7925 if (vreg->min_uV && vreg->max_uV) {
7926 min_uV = on ? vreg->min_uV : 0;
7927 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
b0008625 7928 if (ret)
3b141e8c
SC		 7929 dev_err(dev,
7930 "%s: %s set voltage failed, err=%d\n",
aa497613 7931 __func__, name, ret);
aa497613 7932 }
aa497613
SRT		 7933 }
7934out:
7935 return ret;
7936}
7937
7938static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7939{
7940 int ret = 0;
7941
73067981 7942 if (!vreg || vreg->enabled)
aa497613
SRT		 7943 goto out;
7944
7945 ret = ufshcd_config_vreg(dev, vreg, true);
7946 if (!ret)
7947 ret = regulator_enable(vreg->reg);
7948
7949 if (!ret)
7950 vreg->enabled = true;
7951 else
7952 dev_err(dev, "%s: %s enable failed, err=%d\n",
7953 __func__, vreg->name, ret);
7954out:
7955 return ret;
7956}
7957
7958static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7959{
7960 int ret = 0;
7961
73067981 7962 if (!vreg || !vreg->enabled)
aa497613
SRT		 7963 goto out;
7964
7965 ret = regulator_disable(vreg->reg);
7966
7967 if (!ret) {
7968 /* ignore errors on applying disable config */
7969 ufshcd_config_vreg(dev, vreg, false);
7970 vreg->enabled = false;
7971 } else {
7972 dev_err(dev, "%s: %s disable failed, err=%d\n",
7973 __func__, vreg->name, ret);
7974 }
7975out:
7976 return ret;
7977}
7978
7979static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7980{
7981 int ret = 0;
7982 struct device *dev = hba->dev;
7983 struct ufs_vreg_info *info = &hba->vreg_info;
7984
aa497613
SRT		 7985 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7986 if (ret)
7987 goto out;
7988
7989 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7990 if (ret)
7991 goto out;
7992
7993 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
aa497613
SRT		 7994
7995out:
7996 if (ret) {
7997 ufshcd_toggle_vreg(dev, info->vccq2, false);
7998 ufshcd_toggle_vreg(dev, info->vccq, false);
7999 ufshcd_toggle_vreg(dev, info->vcc, false);
8000 }
8001 return ret;
8002}
8003
6a771a65
RS		 8004static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8005{
8006 struct ufs_vreg_info *info = &hba->vreg_info;
8007
60b7b823 8008 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 8009}
8010
aa497613
SRT		 8011static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8012{
8013 int ret = 0;
8014
8015 if (!vreg)
8016 goto out;
8017
8018 vreg->reg = devm_regulator_get(dev, vreg->name);
8019 if (IS_ERR(vreg->reg)) {
8020 ret = PTR_ERR(vreg->reg);
8021 dev_err(dev, "%s: %s get failed, err=%d\n",
8022 __func__, vreg->name, ret);
8023 }
8024out:
8025 return ret;
8026}
8027
8028static int ufshcd_init_vreg(struct ufs_hba *hba)
8029{
8030 int ret = 0;
8031 struct device *dev = hba->dev;
8032 struct ufs_vreg_info *info = &hba->vreg_info;
8033
aa497613
SRT		 8034 ret = ufshcd_get_vreg(dev, info->vcc);
8035 if (ret)
8036 goto out;
8037
8038 ret = ufshcd_get_vreg(dev, info->vccq);
b0008625
BH		 8039 if (!ret)
8040 ret = ufshcd_get_vreg(dev, info->vccq2);
aa497613
SRT		 8041out:
8042 return ret;
8043}
8044
6a771a65
RS		 8045static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8046{
8047 struct ufs_vreg_info *info = &hba->vreg_info;
8048
8049 if (info)
8050 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8051
8052 return 0;
8053}
8054
57d104c1
SJ		 8055static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
8056 bool skip_ref_clk)
c6e79dac
SRT		 8057{
8058 int ret = 0;
8059 struct ufs_clk_info *clki;
8060 struct list_head *head = &hba->clk_list_head;
1ab27c9c 8061 unsigned long flags;
911a0771 8062 ktime_t start = ktime_get();
8063 bool clk_state_changed = false;
c6e79dac 8064
566ec9ad 8065 if (list_empty(head))
c6e79dac
SRT		 8066 goto out;
8067
38f3242e
CG		 8068 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8069 if (ret)
8070 return ret;
1e879e8f 8071
c6e79dac
SRT		 8072 list_for_each_entry(clki, head, list) {
8073 if (!IS_ERR_OR_NULL(clki->clk)) {
57d104c1
SJ		 8074 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
8075 continue;
8076
911a0771 8077 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 8078 if (on && !clki->enabled) {
8079 ret = clk_prepare_enable(clki->clk);
8080 if (ret) {
8081 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8082 __func__, clki->name, ret);
8083 goto out;
8084 }
8085 } else if (!on && clki->enabled) {
8086 clk_disable_unprepare(clki->clk);
8087 }
8088 clki->enabled = on;
8089 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8090 clki->name, on ? "en" : "dis");
8091 }
8092 }
1ab27c9c 8093
38f3242e
CG		 8094 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8095 if (ret)
8096 return ret;
1e879e8f 8097
c6e79dac
SRT		 8098out:
8099 if (ret) {
8100 list_for_each_entry(clki, head, list) {
8101 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8102 clk_disable_unprepare(clki->clk);
8103 }
7ff5ab47 8104 } else if (!ret && on) {
1ab27c9c
ST		 8105 spin_lock_irqsave(hba->host->host_lock, flags);
8106 hba->clk_gating.state = CLKS_ON;
7ff5ab47 8107 trace_ufshcd_clk_gating(dev_name(hba->dev),
8108 hba->clk_gating.state);
1ab27c9c 8109 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 8110 }
7ff5ab47 8111
911a0771 8112 if (clk_state_changed)
8113 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8114 (on ? "on" : "off"),
8115 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 8116 return ret;
8117}
8118
57d104c1
SJ		 8119static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8120{
8121 return __ufshcd_setup_clocks(hba, on, false);
8122}
8123
c6e79dac
SRT		 8124static int ufshcd_init_clocks(struct ufs_hba *hba)
8125{
8126 int ret = 0;
8127 struct ufs_clk_info *clki;
8128 struct device *dev = hba->dev;
8129 struct list_head *head = &hba->clk_list_head;
8130
566ec9ad 8131 if (list_empty(head))
c6e79dac
SRT		 8132 goto out;
8133
8134 list_for_each_entry(clki, head, list) {
8135 if (!clki->name)
8136 continue;
8137
8138 clki->clk = devm_clk_get(dev, clki->name);
8139 if (IS_ERR(clki->clk)) {
8140 ret = PTR_ERR(clki->clk);
8141 dev_err(dev, "%s: %s clk get failed, %d\n",
8142 __func__, clki->name, ret);
8143 goto out;
8144 }
8145
9e1e8a75
SJ		 8146 /*
8147 * Parse device ref clk freq as per device tree "ref_clk".
8148 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8149 * in ufshcd_alloc_host().
8150 */
8151 if (!strcmp(clki->name, "ref_clk"))
8152 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8153
c6e79dac
SRT		 8154 if (clki->max_freq) {
8155 ret = clk_set_rate(clki->clk, clki->max_freq);
8156 if (ret) {
8157 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8158 __func__, clki->name,
8159 clki->max_freq, ret);
8160 goto out;
8161 }
856b3483 8162 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 8163 }
8164 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8165 clki->name, clk_get_rate(clki->clk));
8166 }
8167out:
8168 return ret;
8169}
8170
5c0c28a8
SRT		 8171static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8172{
8173 int err = 0;
8174
8175 if (!hba->vops)
8176 goto out;
8177
0263bcd0
YG		 8178 err = ufshcd_vops_init(hba);
8179 if (err)
8180 goto out;
5c0c28a8 8181
0263bcd0
YG		 8182 err = ufshcd_vops_setup_regulators(hba, true);
8183 if (err)
b0008625 8184 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 8185out:
8186 if (err)
8187 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 8188 __func__, ufshcd_get_var_name(hba), err);
5c0c28a8
SRT		 8189 return err;
8190}
8191
8192static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8193{
8194 if (!hba->vops)
8195 return;
8196
0263bcd0 8197 ufshcd_vops_setup_regulators(hba, false);
5c0c28a8 8198
0263bcd0 8199 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 8200}
8201
aa497613
SRT		 8202static int ufshcd_hba_init(struct ufs_hba *hba)
8203{
8204 int err;
8205
6a771a65
RS		 8206 /*
8207 * Handle host controller power separately from the UFS device power
8208 * rails as it will help controlling the UFS host controller power
8209 * collapse easily which is different than UFS device power collapse.
8210 * Also, enable the host controller power before we go ahead with rest
8211 * of the initialization here.
8212 */
8213 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 8214 if (err)
8215 goto out;
8216
6a771a65 8217 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 8218 if (err)
8219 goto out;
8220
6a771a65
RS		 8221 err = ufshcd_init_clocks(hba);
8222 if (err)
8223 goto out_disable_hba_vreg;
8224
8225 err = ufshcd_setup_clocks(hba, true);
8226 if (err)
8227 goto out_disable_hba_vreg;
8228
c6e79dac
SRT		 8229 err = ufshcd_init_vreg(hba);
8230 if (err)
8231 goto out_disable_clks;
8232
8233 err = ufshcd_setup_vreg(hba, true);
8234 if (err)
8235 goto out_disable_clks;
8236
aa497613
SRT		 8237 err = ufshcd_variant_hba_init(hba);
8238 if (err)
8239 goto out_disable_vreg;
8240
1d337ec2 8241 hba->is_powered = true;
aa497613
SRT		 8242 goto out;
8243
8244out_disable_vreg:
8245 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 8246out_disable_clks:
8247 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 8248out_disable_hba_vreg:
8249 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 8250out:
8251 return err;
8252}
8253
8254static void ufshcd_hba_exit(struct ufs_hba *hba)
8255{
1d337ec2
SRT		 8256 if (hba->is_powered) {
8257 ufshcd_variant_hba_exit(hba);
8258 ufshcd_setup_vreg(hba, false);
a508253d 8259 ufshcd_suspend_clkscaling(hba);
eebcc196 8260 if (ufshcd_is_clkscaling_supported(hba))
0701e49d 8261 if (hba->devfreq)
8262 ufshcd_suspend_clkscaling(hba);
1d337ec2
SRT		 8263 ufshcd_setup_clocks(hba, false);
8264 ufshcd_setup_hba_vreg(hba, false);
8265 hba->is_powered = false;
09750066 8266 ufs_put_device_desc(hba);
1d337ec2 8267 }
aa497613
SRT		 8268}
8269
57d104c1
SJ		 8270static int
8271ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
8272{
8273 unsigned char cmd[6] = {REQUEST_SENSE,
8274 0,
8275 0,
8276 0,
09a5a24f 8277 UFS_SENSE_SIZE,
57d104c1
SJ		 8278 0};
8279 char *buffer;
8280 int ret;
8281
09a5a24f 8282 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 8283 if (!buffer) {
8284 ret = -ENOMEM;
8285 goto out;
8286 }
8287
fcbfffe2 8288 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 8289 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 8290 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 8291 if (ret)
8292 pr_err("%s: failed with err %d\n", __func__, ret);
8293
8294 kfree(buffer);
8295out:
8296 return ret;
8297}
8298
8299/**
8300 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8301 * power mode
8302 * @hba: per adapter instance
8303 * @pwr_mode: device power mode to set
8304 *
8305 * Returns 0 if requested power mode is set successfully
8306 * Returns non-zero if failed to set the requested power mode
8307 */
8308static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8309 enum ufs_dev_pwr_mode pwr_mode)
8310{
8311 unsigned char cmd[6] = { START_STOP };
8312 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 8313 struct scsi_device *sdp;
8314 unsigned long flags;
57d104c1
SJ		 8315 int ret;
8316
7c48bfd0
AM		 8317 spin_lock_irqsave(hba->host->host_lock, flags);
8318 sdp = hba->sdev_ufs_device;
8319 if (sdp) {
8320 ret = scsi_device_get(sdp);
8321 if (!ret && !scsi_device_online(sdp)) {
8322 ret = -ENODEV;
8323 scsi_device_put(sdp);
8324 }
8325 } else {
8326 ret = -ENODEV;
8327 }
8328 spin_unlock_irqrestore(hba->host->host_lock, flags);
8329
8330 if (ret)
8331 return ret;
57d104c1
SJ		 8332
8333 /*
8334 * If scsi commands fail, the scsi mid-layer schedules scsi error-
8335 * handling, which would wait for host to be resumed. Since we know
8336 * we are functional while we are here, skip host resume in error
8337 * handling context.
8338 */
8339 hba->host->eh_noresume = 1;
8340 if (hba->wlun_dev_clr_ua) {
8341 ret = ufshcd_send_request_sense(hba, sdp);
8342 if (ret)
8343 goto out;
8344 /* Unit attention condition is cleared now */
8345 hba->wlun_dev_clr_ua = false;
8346 }
8347
8348 cmd[4] = pwr_mode << 4;
8349
8350 /*
8351 * Current function would be generally called from the power management
e8064021 8352 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 8353 * already suspended childs.
8354 */
fcbfffe2
CH		 8355 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8356 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 8357 if (ret) {
8358 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 8359 "START_STOP failed for power mode: %d, result %x\n",
8360 pwr_mode, ret);
c65be1a6 8361 if (driver_byte(ret) == DRIVER_SENSE)
21045519 8362 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 8363 }
8364
8365 if (!ret)
8366 hba->curr_dev_pwr_mode = pwr_mode;
8367out:
7c48bfd0 8368 scsi_device_put(sdp);
57d104c1
SJ		 8369 hba->host->eh_noresume = 0;
8370 return ret;
8371}
8372
8373static int ufshcd_link_state_transition(struct ufs_hba *hba,
8374 enum uic_link_state req_link_state,
8375 int check_for_bkops)
8376{
8377 int ret = 0;
8378
8379 if (req_link_state == hba->uic_link_state)
8380 return 0;
8381
8382 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8383 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236 8384 if (!ret) {
57d104c1 8385 ufshcd_set_link_hibern8(hba);
4db7a236
CG		 8386 } else {
8387 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8388 __func__, ret);
57d104c1 8389 goto out;
4db7a236 8390 }
57d104c1
SJ		 8391 }
8392 /*
8393 * If autobkops is enabled, link can't be turned off because
fe1d4c2e
AH		 8394 * turning off the link would also turn off the device, except in the
8395 * case of DeepSleep where the device is expected to remain powered.
57d104c1
SJ		 8396 */
8397 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
dc30c9e6 8398 (!check_for_bkops || !hba->auto_bkops_enabled)) {
f3099fbd
YG		 8399 /*
8400 * Let's make sure that link is in low power mode, we are doing
8401 * this currently by putting the link in Hibern8. Otherway to
8402 * put the link in low power mode is to send the DME end point
8403 * to device and then send the DME reset command to local
8404 * unipro. But putting the link in hibern8 is much faster.
fe1d4c2e
AH		 8405 *
8406 * Note also that putting the link in Hibern8 is a requirement
8407 * for entering DeepSleep.
f3099fbd
YG		 8408 */
8409 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236
CG		 8410 if (ret) {
8411 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8412 __func__, ret);
f3099fbd 8413 goto out;
4db7a236 8414 }
57d104c1
SJ		 8415 /*
8416 * Change controller state to "reset state" which
8417 * should also put the link in off/reset state
8418 */
5cac1095 8419 ufshcd_hba_stop(hba);
57d104c1
SJ		 8420 /*
8421 * TODO: Check if we need any delay to make sure that
8422 * controller is reset
8423 */
8424 ufshcd_set_link_off(hba);
8425 }
8426
8427out:
8428 return ret;
8429}
8430
8431static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8432{
c4df6eed
SC		 8433 bool vcc_off = false;
8434
b799fdf7
YG		 8435 /*
8436 * It seems some UFS devices may keep drawing more than sleep current
8437 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8438 * To avoid this situation, add 2ms delay before putting these UFS
8439 * rails in LPM mode.
8440 */
8441 if (!ufshcd_is_link_active(hba) &&
8442 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8443 usleep_range(2000, 2100);
8444
57d104c1
SJ		 8445 /*
8446 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8447 * power.
8448 *
8449 * If UFS device and link is in OFF state, all power supplies (VCC,
8450 * VCCQ, VCCQ2) can be turned off if power on write protect is not
8451 * required. If UFS link is inactive (Hibern8 or OFF state) and device
8452 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8453 *
8454 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8455 * in low power state which would save some power.
3d17b9b5
AD		 8456 *
8457 * If Write Booster is enabled and the device needs to flush the WB
8458 * buffer OR if bkops status is urgent for WB, keep Vcc on.
57d104c1
SJ		 8459 */
8460 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8461 !hba->dev_info.is_lu_power_on_wp) {
8462 ufshcd_setup_vreg(hba, false);
c4df6eed 8463 vcc_off = true;
57d104c1 8464 } else if (!ufshcd_is_ufs_dev_active(hba)) {
51dd905b 8465 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
c4df6eed 8466 vcc_off = true;
57d104c1
SJ		 8467 if (!ufshcd_is_link_active(hba)) {
8468 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8469 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8470 }
8471 }
c4df6eed
SC		 8472
8473 /*
8474 * Some UFS devices require delay after VCC power rail is turned-off.
8475 */
8476 if (vcc_off && hba->vreg_info.vcc &&
8477 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8478 usleep_range(5000, 5100);
57d104c1
SJ		 8479}
8480
8481static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8482{
8483 int ret = 0;
8484
8485 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8486 !hba->dev_info.is_lu_power_on_wp) {
8487 ret = ufshcd_setup_vreg(hba, true);
8488 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 8489 if (!ret && !ufshcd_is_link_active(hba)) {
8490 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8491 if (ret)
8492 goto vcc_disable;
8493 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8494 if (ret)
8495 goto vccq_lpm;
8496 }
69d72ac8 8497 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 8498 }
8499 goto out;
8500
8501vccq_lpm:
8502 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8503vcc_disable:
8504 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8505out:
8506 return ret;
8507}
8508
8509static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8510{
dd7143e2 8511 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
57d104c1
SJ		 8512 ufshcd_setup_hba_vreg(hba, false);
8513}
8514
8515static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8516{
dd7143e2 8517 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
57d104c1
SJ		 8518 ufshcd_setup_hba_vreg(hba, true);
8519}
8520
7a3e97b0 8521/**
57d104c1 8522 * ufshcd_suspend - helper function for suspend operations
3b1d0580 8523 * @hba: per adapter instance
57d104c1
SJ		 8524 * @pm_op: desired low power operation type
8525 *
8526 * This function will try to put the UFS device and link into low power
8527 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
8528 * (System PM level).
8529 *
8530 * If this function is called during shutdown, it will make sure that
8531 * both UFS device and UFS link is powered off.
7a3e97b0 8532 *
57d104c1
SJ		 8533 * NOTE: UFS device & link must be active before we enter in this function.
8534 *
8535 * Returns 0 for success and non-zero for failure
7a3e97b0 8536 */
57d104c1 8537static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8538{
57d104c1 8539 int ret = 0;
fe1d4c2e 8540 int check_for_bkops;
57d104c1
SJ		 8541 enum ufs_pm_level pm_lvl;
8542 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8543 enum uic_link_state req_link_state;
8544
8545 hba->pm_op_in_progress = 1;
8546 if (!ufshcd_is_shutdown_pm(pm_op)) {
8547 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
8548 hba->rpm_lvl : hba->spm_lvl;
8549 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8550 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8551 } else {
8552 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8553 req_link_state = UIC_LINK_OFF_STATE;
8554 }
8555
7a3e97b0 8556 /*
57d104c1
SJ		 8557 * If we can't transition into any of the low power modes
8558 * just gate the clocks.
7a3e97b0 8559 */
1ab27c9c
ST		 8560 ufshcd_hold(hba, false);
8561 hba->clk_gating.is_suspended = true;
8562
401f1e44 8563 if (hba->clk_scaling.is_allowed) {
8564 cancel_work_sync(&hba->clk_scaling.suspend_work);
8565 cancel_work_sync(&hba->clk_scaling.resume_work);
8566 ufshcd_suspend_clkscaling(hba);
8567 }
d6fcf81a 8568
57d104c1
SJ		 8569 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8570 req_link_state == UIC_LINK_ACTIVE_STATE) {
8571 goto disable_clks;
8572 }
7a3e97b0 8573
57d104c1
SJ		 8574 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8575 (req_link_state == hba->uic_link_state))
d6fcf81a 8576 goto enable_gating;
57d104c1
SJ		 8577
8578 /* UFS device & link must be active before we enter in this function */
8579 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8580 ret = -EINVAL;
d6fcf81a 8581 goto enable_gating;
57d104c1
SJ		 8582 }
8583
8584 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 8585 if (ufshcd_can_autobkops_during_suspend(hba)) {
8586 /*
8587 * The device is idle with no requests in the queue,
8588 * allow background operations if bkops status shows
8589 * that performance might be impacted.
8590 */
8591 ret = ufshcd_urgent_bkops(hba);
8592 if (ret)
8593 goto enable_gating;
8594 } else {
8595 /* make sure that auto bkops is disabled */
8596 ufshcd_disable_auto_bkops(hba);
8597 }
3d17b9b5 8598 /*
51dd905b
SC		 8599 * If device needs to do BKOP or WB buffer flush during
8600 * Hibern8, keep device power mode as "active power mode"
8601 * and VCC supply.
3d17b9b5 8602 */
51dd905b
SC		 8603 hba->dev_info.b_rpm_dev_flush_capable =
8604 hba->auto_bkops_enabled ||
8605 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8606 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8607 ufshcd_is_auto_hibern8_enabled(hba))) &&
8608 ufshcd_wb_need_flush(hba));
8609 }
8610
8611 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
8612 if ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
8613 !ufshcd_is_runtime_pm(pm_op)) {
8614 /* ensure that bkops is disabled */
8615 ufshcd_disable_auto_bkops(hba);
8616 }
57d104c1 8617
51dd905b
SC		 8618 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8619 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8620 if (ret)
8621 goto enable_gating;
8622 }
57d104c1
SJ		 8623 }
8624
2824ec9f 8625 flush_work(&hba->eeh_work);
fe1d4c2e
AH		 8626
8627 /*
8628 * In the case of DeepSleep, the device is expected to remain powered
8629 * with the link off, so do not check for bkops.
8630 */
8631 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
8632 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
57d104c1
SJ		 8633 if (ret)
8634 goto set_dev_active;
8635
8636 ufshcd_vreg_set_lpm(hba);
8637
8638disable_clks:
8639 /*
8640 * Call vendor specific suspend callback. As these callbacks may access
8641 * vendor specific host controller register space call them before the
8642 * host clocks are ON.
8643 */
0263bcd0
YG		 8644 ret = ufshcd_vops_suspend(hba, pm_op);
8645 if (ret)
8646 goto set_link_active;
dcb6cec5
SC		 8647 /*
8648 * Disable the host irq as host controller as there won't be any
8649 * host controller transaction expected till resume.
8650 */
8651 ufshcd_disable_irq(hba);
57d104c1 8652
57d104c1
SJ		 8653 if (!ufshcd_is_link_active(hba))
8654 ufshcd_setup_clocks(hba, false);
8655 else
8656 /* If link is active, device ref_clk can't be switched off */
8657 __ufshcd_setup_clocks(hba, false, true);
8658
2dec9475
CG		 8659 if (ufshcd_is_clkgating_allowed(hba)) {
8660 hba->clk_gating.state = CLKS_OFF;
8661 trace_ufshcd_clk_gating(dev_name(hba->dev),
8662 hba->clk_gating.state);
8663 }
dcb6cec5 8664
57d104c1
SJ		 8665 /* Put the host controller in low power mode if possible */
8666 ufshcd_hba_vreg_set_lpm(hba);
8667 goto out;
8668
57d104c1 8669set_link_active:
401f1e44 8670 if (hba->clk_scaling.is_allowed)
8671 ufshcd_resume_clkscaling(hba);
57d104c1 8672 ufshcd_vreg_set_hpm(hba);
fe1d4c2e
AH		 8673 /*
8674 * Device hardware reset is required to exit DeepSleep. Also, for
8675 * DeepSleep, the link is off so host reset and restore will be done
8676 * further below.
8677 */
8678 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
8679 ufshcd_vops_device_reset(hba);
8680 WARN_ON(!ufshcd_is_link_off(hba));
8681 }
57d104c1
SJ		 8682 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
8683 ufshcd_set_link_active(hba);
8684 else if (ufshcd_is_link_off(hba))
8685 ufshcd_host_reset_and_restore(hba);
8686set_dev_active:
fe1d4c2e
AH		 8687 /* Can also get here needing to exit DeepSleep */
8688 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
8689 ufshcd_vops_device_reset(hba);
8690 ufshcd_host_reset_and_restore(hba);
8691 }
57d104c1
SJ		 8692 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
8693 ufshcd_disable_auto_bkops(hba);
1ab27c9c 8694enable_gating:
401f1e44 8695 if (hba->clk_scaling.is_allowed)
8696 ufshcd_resume_clkscaling(hba);
1ab27c9c 8697 hba->clk_gating.is_suspended = false;
51dd905b 8698 hba->dev_info.b_rpm_dev_flush_capable = false;
1ab27c9c 8699 ufshcd_release(hba);
57d104c1 8700out:
51dd905b
SC		 8701 if (hba->dev_info.b_rpm_dev_flush_capable) {
8702 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
8703 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
8704 }
8705
57d104c1 8706 hba->pm_op_in_progress = 0;
51dd905b 8707
8808b4e9
SC		 8708 if (ret)
8709 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
57d104c1 8710 return ret;
7a3e97b0
SY		 8711}
8712
8713/**
57d104c1 8714 * ufshcd_resume - helper function for resume operations
3b1d0580 8715 * @hba: per adapter instance
57d104c1 8716 * @pm_op: runtime PM or system PM
7a3e97b0 8717 *
57d104c1
SJ		 8718 * This function basically brings the UFS device, UniPro link and controller
8719 * to active state.
8720 *
8721 * Returns 0 for success and non-zero for failure
7a3e97b0 8722 */
57d104c1 8723static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8724{
57d104c1
SJ		 8725 int ret;
8726 enum uic_link_state old_link_state;
8727
8728 hba->pm_op_in_progress = 1;
8729 old_link_state = hba->uic_link_state;
8730
8731 ufshcd_hba_vreg_set_hpm(hba);
8732 /* Make sure clocks are enabled before accessing controller */
8733 ret = ufshcd_setup_clocks(hba, true);
8734 if (ret)
8735 goto out;
8736
57d104c1 8737 /* enable the host irq as host controller would be active soon */
5231d38c 8738 ufshcd_enable_irq(hba);
57d104c1
SJ		 8739
8740 ret = ufshcd_vreg_set_hpm(hba);
8741 if (ret)
8742 goto disable_irq_and_vops_clks;
8743
7a3e97b0 8744 /*
57d104c1
SJ		 8745 * Call vendor specific resume callback. As these callbacks may access
8746 * vendor specific host controller register space call them when the
8747 * host clocks are ON.
7a3e97b0 8748 */
0263bcd0
YG		 8749 ret = ufshcd_vops_resume(hba, pm_op);
8750 if (ret)
8751 goto disable_vreg;
57d104c1 8752
fe1d4c2e
AH		 8753 /* For DeepSleep, the only supported option is to have the link off */
8754 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
8755
57d104c1
SJ		 8756 if (ufshcd_is_link_hibern8(hba)) {
8757 ret = ufshcd_uic_hibern8_exit(hba);
4db7a236 8758 if (!ret) {
57d104c1 8759 ufshcd_set_link_active(hba);
4db7a236
CG		 8760 } else {
8761 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
8762 __func__, ret);
57d104c1 8763 goto vendor_suspend;
4db7a236 8764 }
57d104c1 8765 } else if (ufshcd_is_link_off(hba)) {
57d104c1 8766 /*
089f5b64
AD		 8767 * A full initialization of the host and the device is
8768 * required since the link was put to off during suspend.
fe1d4c2e
AH		 8769 * Note, in the case of DeepSleep, the device will exit
8770 * DeepSleep due to device reset.
089f5b64
AD		 8771 */
8772 ret = ufshcd_reset_and_restore(hba);
8773 /*
8774 * ufshcd_reset_and_restore() should have already
57d104c1
SJ		 8775 * set the link state as active
8776 */
8777 if (ret || !ufshcd_is_link_active(hba))
8778 goto vendor_suspend;
8779 }
8780
8781 if (!ufshcd_is_ufs_dev_active(hba)) {
8782 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8783 if (ret)
8784 goto set_old_link_state;
8785 }
8786
4e768e76 8787 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8788 ufshcd_enable_auto_bkops(hba);
8789 else
8790 /*
8791 * If BKOPs operations are urgently needed at this moment then
8792 * keep auto-bkops enabled or else disable it.
8793 */
8794 ufshcd_urgent_bkops(hba);
8795
1ab27c9c
ST		 8796 hba->clk_gating.is_suspended = false;
8797
fcb0c4b0
ST		 8798 if (hba->clk_scaling.is_allowed)
8799 ufshcd_resume_clkscaling(hba);
856b3483 8800
ad448378
AH		 8801 /* Enable Auto-Hibernate if configured */
8802 ufshcd_auto_hibern8_enable(hba);
8803
51dd905b
SC		 8804 if (hba->dev_info.b_rpm_dev_flush_capable) {
8805 hba->dev_info.b_rpm_dev_flush_capable = false;
8806 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
8807 }
8808
71d848b8
CG		 8809 /* Schedule clock gating in case of no access to UFS device yet */
8810 ufshcd_release(hba);
8811
57d104c1
SJ		 8812 goto out;
8813
8814set_old_link_state:
8815 ufshcd_link_state_transition(hba, old_link_state, 0);
8816vendor_suspend:
0263bcd0 8817 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 8818disable_vreg:
8819 ufshcd_vreg_set_lpm(hba);
8820disable_irq_and_vops_clks:
8821 ufshcd_disable_irq(hba);
401f1e44 8822 if (hba->clk_scaling.is_allowed)
8823 ufshcd_suspend_clkscaling(hba);
57d104c1 8824 ufshcd_setup_clocks(hba, false);
2dec9475
CG		 8825 if (ufshcd_is_clkgating_allowed(hba)) {
8826 hba->clk_gating.state = CLKS_OFF;
8827 trace_ufshcd_clk_gating(dev_name(hba->dev),
8828 hba->clk_gating.state);
8829 }
57d104c1
SJ		 8830out:
8831 hba->pm_op_in_progress = 0;
8808b4e9
SC		 8832 if (ret)
8833 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
57d104c1
SJ		 8834 return ret;
8835}
8836
8837/**
8838 * ufshcd_system_suspend - system suspend routine
8839 * @hba: per adapter instance
57d104c1
SJ		 8840 *
8841 * Check the description of ufshcd_suspend() function for more details.
8842 *
8843 * Returns 0 for success and non-zero for failure
8844 */
8845int ufshcd_system_suspend(struct ufs_hba *hba)
8846{
8847 int ret = 0;
7ff5ab47 8848 ktime_t start = ktime_get();
57d104c1
SJ		 8849
8850 if (!hba || !hba->is_powered)
233b594b 8851 return 0;
57d104c1 8852
0b257734 8853 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8854 hba->curr_dev_pwr_mode) &&
8855 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8856 hba->uic_link_state))
8857 goto out;
57d104c1 8858
0b257734 8859 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 8860 /*
8861 * UFS device and/or UFS link low power states during runtime
8862 * suspend seems to be different than what is expected during
8863 * system suspend. Hence runtime resume the devic & link and
8864 * let the system suspend low power states to take effect.
8865 * TODO: If resume takes longer time, we might have optimize
8866 * it in future by not resuming everything if possible.
8867 */
8868 ret = ufshcd_runtime_resume(hba);
8869 if (ret)
8870 goto out;
8871 }
8872
8873 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8874out:
7ff5ab47 8875 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8876 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8877 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 8878 if (!ret)
8879 hba->is_sys_suspended = true;
57d104c1
SJ		 8880 return ret;
8881}
8882EXPORT_SYMBOL(ufshcd_system_suspend);
8883
8884/**
8885 * ufshcd_system_resume - system resume routine
8886 * @hba: per adapter instance
8887 *
8888 * Returns 0 for success and non-zero for failure
8889 */
7a3e97b0 8890
57d104c1
SJ		 8891int ufshcd_system_resume(struct ufs_hba *hba)
8892{
7ff5ab47 8893 int ret = 0;
8894 ktime_t start = ktime_get();
8895
e3ce73d6
YG		 8896 if (!hba)
8897 return -EINVAL;
8898
8899 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 8900 /*
8901 * Let the runtime resume take care of resuming
8902 * if runtime suspended.
8903 */
7ff5ab47 8904 goto out;
8905 else
8906 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8907out:
8908 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8909 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8910 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 8911 if (!ret)
8912 hba->is_sys_suspended = false;
7ff5ab47 8913 return ret;
7a3e97b0 8914}
57d104c1 8915EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 8916
57d104c1
SJ		 8917/**
8918 * ufshcd_runtime_suspend - runtime suspend routine
8919 * @hba: per adapter instance
8920 *
8921 * Check the description of ufshcd_suspend() function for more details.
8922 *
8923 * Returns 0 for success and non-zero for failure
8924 */
66ec6d59
SRT		 8925int ufshcd_runtime_suspend(struct ufs_hba *hba)
8926{
7ff5ab47 8927 int ret = 0;
8928 ktime_t start = ktime_get();
8929
e3ce73d6
YG		 8930 if (!hba)
8931 return -EINVAL;
8932
8933 if (!hba->is_powered)
7ff5ab47 8934 goto out;
8935 else
8936 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8937out:
8938 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8939 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8940 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8941 return ret;
66ec6d59
SRT		 8942}
8943EXPORT_SYMBOL(ufshcd_runtime_suspend);
8944
57d104c1
SJ		 8945/**
8946 * ufshcd_runtime_resume - runtime resume routine
8947 * @hba: per adapter instance
8948 *
8949 * This function basically brings the UFS device, UniPro link and controller
8950 * to active state. Following operations are done in this function:
8951 *
8952 * 1. Turn on all the controller related clocks
8953 * 2. Bring the UniPro link out of Hibernate state
8954 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8955 * to active state.
8956 * 4. If auto-bkops is enabled on the device, disable it.
8957 *
8958 * So following would be the possible power state after this function return
8959 * successfully:
8960 * S1: UFS device in Active state with VCC rail ON
8961 * UniPro link in Active state
8962 * All the UFS/UniPro controller clocks are ON
8963 *
8964 * Returns 0 for success and non-zero for failure
8965 */
66ec6d59
SRT		 8966int ufshcd_runtime_resume(struct ufs_hba *hba)
8967{
7ff5ab47 8968 int ret = 0;
8969 ktime_t start = ktime_get();
8970
e3ce73d6
YG		 8971 if (!hba)
8972 return -EINVAL;
8973
8974 if (!hba->is_powered)
7ff5ab47 8975 goto out;
8976 else
8977 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8978out:
8979 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8980 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8981 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8982 return ret;
66ec6d59
SRT		 8983}
8984EXPORT_SYMBOL(ufshcd_runtime_resume);
8985
8986int ufshcd_runtime_idle(struct ufs_hba *hba)
8987{
8988 return 0;
8989}
8990EXPORT_SYMBOL(ufshcd_runtime_idle);
8991
57d104c1
SJ		 8992/**
8993 * ufshcd_shutdown - shutdown routine
8994 * @hba: per adapter instance
8995 *
8996 * This function would power off both UFS device and UFS link.
8997 *
8998 * Returns 0 always to allow force shutdown even in case of errors.
8999 */
9000int ufshcd_shutdown(struct ufs_hba *hba)
9001{
9002 int ret = 0;
9003
f51913ee
SC		 9004 if (!hba->is_powered)
9005 goto out;
9006
57d104c1
SJ		 9007 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9008 goto out;
9009
9010 if (pm_runtime_suspended(hba->dev)) {
9011 ret = ufshcd_runtime_resume(hba);
9012 if (ret)
9013 goto out;
9014 }
9015
9016 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
9017out:
9018 if (ret)
9019 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
9020 /* allow force shutdown even in case of errors */
9021 return 0;
9022}
9023EXPORT_SYMBOL(ufshcd_shutdown);
9024
7a3e97b0 9025/**
3b1d0580 9026 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 9027 * data structure memory
8aa29f19 9028 * @hba: per adapter instance
7a3e97b0 9029 */
3b1d0580 9030void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 9031{
df032bf2 9032 ufs_bsg_remove(hba);
cbb6813e 9033 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 9034 blk_cleanup_queue(hba->tmf_queue);
9035 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360 9036 blk_cleanup_queue(hba->cmd_queue);
cfdf9c91 9037 scsi_remove_host(hba->host);
7a3e97b0 9038 /* disable interrupts */
2fbd009b 9039 ufshcd_disable_intr(hba, hba->intr_mask);
5cac1095 9040 ufshcd_hba_stop(hba);
7a3e97b0 9041
eebcc196 9042 ufshcd_exit_clk_scaling(hba);
1ab27c9c 9043 ufshcd_exit_clk_gating(hba);
fcb0c4b0
ST		 9044 if (ufshcd_is_clkscaling_supported(hba))
9045 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
aa497613 9046 ufshcd_hba_exit(hba);
3b1d0580
VH		 9047}
9048EXPORT_SYMBOL_GPL(ufshcd_remove);
9049
47555a5c
YG		 9050/**
9051 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9052 * @hba: pointer to Host Bus Adapter (HBA)
9053 */
9054void ufshcd_dealloc_host(struct ufs_hba *hba)
9055{
df043c74 9056 ufshcd_crypto_destroy_keyslot_manager(hba);
47555a5c
YG		 9057 scsi_host_put(hba->host);
9058}
9059EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9060
ca3d7bf9
AM		 9061/**
9062 * ufshcd_set_dma_mask - Set dma mask based on the controller
9063 * addressing capability
9064 * @hba: per adapter instance
9065 *
9066 * Returns 0 for success, non-zero for failure
9067 */
9068static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9069{
9070 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9071 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9072 return 0;
9073 }
9074 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9075}
9076
7a3e97b0 9077/**
5c0c28a8 9078 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 9079 * @dev: pointer to device handle
9080 * @hba_handle: driver private handle
7a3e97b0
SY		 9081 * Returns 0 on success, non-zero value on failure
9082 */
5c0c28a8 9083int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 9084{
9085 struct Scsi_Host *host;
9086 struct ufs_hba *hba;
5c0c28a8 9087 int err = 0;
7a3e97b0 9088
3b1d0580
VH		 9089 if (!dev) {
9090 dev_err(dev,
9091 "Invalid memory reference for dev is NULL\n");
9092 err = -ENODEV;
7a3e97b0
SY		 9093 goto out_error;
9094 }
9095
7a3e97b0
SY		 9096 host = scsi_host_alloc(&ufshcd_driver_template,
9097 sizeof(struct ufs_hba));
9098 if (!host) {
3b1d0580 9099 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 9100 err = -ENOMEM;
3b1d0580 9101 goto out_error;
7a3e97b0
SY		 9102 }
9103 hba = shost_priv(host);
7a3e97b0 9104 hba->host = host;
3b1d0580 9105 hba->dev = dev;
5c0c28a8 9106 *hba_handle = hba;
9e1e8a75 9107 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 9108
566ec9ad
SM		 9109 INIT_LIST_HEAD(&hba->clk_list_head);
9110
5c0c28a8
SRT		 9111out_error:
9112 return err;
9113}
9114EXPORT_SYMBOL(ufshcd_alloc_host);
9115
69a6c269
BVA		 9116/* This function exists because blk_mq_alloc_tag_set() requires this. */
9117static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9118 const struct blk_mq_queue_data *qd)
9119{
9120 WARN_ON_ONCE(true);
9121 return BLK_STS_NOTSUPP;
9122}
9123
9124static const struct blk_mq_ops ufshcd_tmf_ops = {
9125 .queue_rq = ufshcd_queue_tmf,
9126};
9127
5c0c28a8
SRT		 9128/**
9129 * ufshcd_init - Driver initialization routine
9130 * @hba: per-adapter instance
9131 * @mmio_base: base register address
9132 * @irq: Interrupt line of device
9133 * Returns 0 on success, non-zero value on failure
9134 */
9135int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9136{
9137 int err;
9138 struct Scsi_Host *host = hba->host;
9139 struct device *dev = hba->dev;
4db7a236 9140 char eh_wq_name[sizeof("ufs_eh_wq_00")];
5c0c28a8
SRT		 9141
9142 if (!mmio_base) {
9143 dev_err(hba->dev,
9144 "Invalid memory reference for mmio_base is NULL\n");
9145 err = -ENODEV;
9146 goto out_error;
9147 }
9148
3b1d0580
VH		 9149 hba->mmio_base = mmio_base;
9150 hba->irq = irq;
90b8491c 9151 hba->vps = &ufs_hba_vps;
7a3e97b0 9152
aa497613 9153 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 9154 if (err)
9155 goto out_error;
9156
7a3e97b0 9157 /* Read capabilities registers */
df043c74
ST		 9158 err = ufshcd_hba_capabilities(hba);
9159 if (err)
9160 goto out_disable;
7a3e97b0
SY		 9161
9162 /* Get UFS version supported by the controller */
9163 hba->ufs_version = ufshcd_get_ufs_version(hba);
9164
c01848c6
YG		 9165 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
9166 (hba->ufs_version != UFSHCI_VERSION_11) &&
9167 (hba->ufs_version != UFSHCI_VERSION_20) &&
9168 (hba->ufs_version != UFSHCI_VERSION_21))
9169 dev_err(hba->dev, "invalid UFS version 0x%x\n",
9170 hba->ufs_version);
9171
2fbd009b
SJ		 9172 /* Get Interrupt bit mask per version */
9173 hba->intr_mask = ufshcd_get_intr_mask(hba);
9174
ca3d7bf9
AM		 9175 err = ufshcd_set_dma_mask(hba);
9176 if (err) {
9177 dev_err(hba->dev, "set dma mask failed\n");
9178 goto out_disable;
9179 }
9180
7a3e97b0
SY		 9181 /* Allocate memory for host memory space */
9182 err = ufshcd_memory_alloc(hba);
9183 if (err) {
3b1d0580
VH		 9184 dev_err(hba->dev, "Memory allocation failed\n");
9185 goto out_disable;
7a3e97b0
SY		 9186 }
9187
9188 /* Configure LRB */
9189 ufshcd_host_memory_configure(hba);
9190
9191 host->can_queue = hba->nutrs;
9192 host->cmd_per_lun = hba->nutrs;
9193 host->max_id = UFSHCD_MAX_ID;
0ce147d4 9194 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 9195 host->max_channel = UFSHCD_MAX_CHANNEL;
9196 host->unique_id = host->host_no;
a851b2bd 9197 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 9198
7eb584db
DR		 9199 hba->max_pwr_info.is_valid = false;
9200
7a3e97b0 9201 /* Initialize work queues */
4db7a236
CG		 9202 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9203 hba->host->host_no);
9204 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9205 if (!hba->eh_wq) {
9206 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9207 __func__);
9208 err = -ENOMEM;
9209 goto out_disable;
9210 }
e8e7f271 9211 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 9212 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 9213
6ccf44fe
SJ		 9214 /* Initialize UIC command mutex */
9215 mutex_init(&hba->uic_cmd_mutex);
9216
5a0b0cb9
SRT		 9217 /* Initialize mutex for device management commands */
9218 mutex_init(&hba->dev_cmd.lock);
9219
a3cd5ec5 9220 init_rwsem(&hba->clk_scaling_lock);
9221
1ab27c9c 9222 ufshcd_init_clk_gating(hba);
199ef13c 9223
eebcc196
VG		 9224 ufshcd_init_clk_scaling(hba);
9225
199ef13c
YG		 9226 /*
9227 * In order to avoid any spurious interrupt immediately after
9228 * registering UFS controller interrupt handler, clear any pending UFS
9229 * interrupt status and disable all the UFS interrupts.
9230 */
9231 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9232 REG_INTERRUPT_STATUS);
9233 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9234 /*
9235 * Make sure that UFS interrupts are disabled and any pending interrupt
9236 * status is cleared before registering UFS interrupt handler.
9237 */
9238 mb();
9239
7a3e97b0 9240 /* IRQ registration */
2953f850 9241 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 9242 if (err) {
3b1d0580 9243 dev_err(hba->dev, "request irq failed\n");
1ab27c9c 9244 goto exit_gating;
57d104c1
SJ		 9245 } else {
9246 hba->is_irq_enabled = true;
7a3e97b0
SY		 9247 }
9248
3b1d0580 9249 err = scsi_add_host(host, hba->dev);
7a3e97b0 9250 if (err) {
3b1d0580 9251 dev_err(hba->dev, "scsi_add_host failed\n");
1ab27c9c 9252 goto exit_gating;
7a3e97b0
SY		 9253 }
9254
7252a360
BVA		 9255 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
9256 if (IS_ERR(hba->cmd_queue)) {
9257 err = PTR_ERR(hba->cmd_queue);
9258 goto out_remove_scsi_host;
9259 }
9260
69a6c269
BVA		 9261 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9262 .nr_hw_queues = 1,
9263 .queue_depth = hba->nutmrs,
9264 .ops = &ufshcd_tmf_ops,
9265 .flags = BLK_MQ_F_NO_SCHED,
9266 };
9267 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9268 if (err < 0)
9269 goto free_cmd_queue;
9270 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9271 if (IS_ERR(hba->tmf_queue)) {
9272 err = PTR_ERR(hba->tmf_queue);
9273 goto free_tmf_tag_set;
9274 }
9275
d8d9f793
BA		 9276 /* Reset the attached device */
9277 ufshcd_vops_device_reset(hba);
9278
df043c74
ST		 9279 ufshcd_init_crypto(hba);
9280
6ccf44fe
SJ		 9281 /* Host controller enable */
9282 err = ufshcd_hba_enable(hba);
7a3e97b0 9283 if (err) {
6ccf44fe 9284 dev_err(hba->dev, "Host controller enable failed\n");
66cc820f 9285 ufshcd_print_host_regs(hba);
6ba65588 9286 ufshcd_print_host_state(hba);
69a6c269 9287 goto free_tmf_queue;
7a3e97b0 9288 }
6ccf44fe 9289
0c8f7586 9290 /*
9291 * Set the default power management level for runtime and system PM.
9292 * Default power saving mode is to keep UFS link in Hibern8 state
9293 * and UFS device in sleep state.
9294 */
9295 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9296 UFS_SLEEP_PWR_MODE,
9297 UIC_LINK_HIBERN8_STATE);
9298 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9299 UFS_SLEEP_PWR_MODE,
9300 UIC_LINK_HIBERN8_STATE);
9301
51dd905b
SC		 9302 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9303 ufshcd_rpm_dev_flush_recheck_work);
9304
ad448378 9305 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 9306 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 9307 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9308 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9309 }
9310
62694735
SRT		 9311 /* Hold auto suspend until async scan completes */
9312 pm_runtime_get_sync(dev);
38135535 9313 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 9314 /*
7caf489b 9315 * We are assuming that device wasn't put in sleep/power-down
9316 * state exclusively during the boot stage before kernel.
9317 * This assumption helps avoid doing link startup twice during
9318 * ufshcd_probe_hba().
57d104c1 9319 */
7caf489b 9320 ufshcd_set_ufs_dev_active(hba);
57d104c1 9321
6ccf44fe 9322 async_schedule(ufshcd_async_scan, hba);
cbb6813e 9323 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 9324
7a3e97b0
SY		 9325 return 0;
9326
69a6c269
BVA		 9327free_tmf_queue:
9328 blk_cleanup_queue(hba->tmf_queue);
9329free_tmf_tag_set:
9330 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360
BVA		 9331free_cmd_queue:
9332 blk_cleanup_queue(hba->cmd_queue);
3b1d0580
VH		 9333out_remove_scsi_host:
9334 scsi_remove_host(hba->host);
1ab27c9c 9335exit_gating:
eebcc196 9336 ufshcd_exit_clk_scaling(hba);
1ab27c9c 9337 ufshcd_exit_clk_gating(hba);
3b1d0580 9338out_disable:
57d104c1 9339 hba->is_irq_enabled = false;
aa497613 9340 ufshcd_hba_exit(hba);
3b1d0580
VH		 9341out_error:
9342 return err;
9343}
9344EXPORT_SYMBOL_GPL(ufshcd_init);
9345
3b1d0580
VH		 9346MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9347MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 9348MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 9349MODULE_LICENSE("GPL");
9350MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Linux 6.x block layer and io_uring tree(s)