projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: ufs: Remove redundant checks of !hba in suspend/resume callbacks
[linux-2.6-block.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
67351119 1// SPDX-License-Identifier: GPL-2.0-or-later
7a3e97b0 2/*
e0eca63e 3 * Universal Flash Storage Host controller driver Core
3b1d0580 4 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 6 *
3b1d0580
VH		 7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 10 */
11
6ccf44fe 12#include <linux/async.h>
856b3483 13#include <linux/devfreq.h>
b573d484 14#include <linux/nls.h>
54b879b7 15#include <linux/of.h>
ad448378 16#include <linux/bitfield.h>
fb276f77 17#include <linux/blk-pm.h>
c72e79c0 18#include <linux/blkdev.h>
e0eca63e 19#include "ufshcd.h"
c58ab7aa 20#include "ufs_quirks.h"
53b3d9c3 21#include "unipro.h"
cbb6813e 22#include "ufs-sysfs.h"
b6cacaf2 23#include "ufs-debugfs.h"
df032bf2 24#include "ufs_bsg.h"
df043c74 25#include "ufshcd-crypto.h"
3d17b9b5
AD		 26#include <asm/unaligned.h>
27#include <linux/blkdev.h>
7a3e97b0 28
7ff5ab47 29#define CREATE_TRACE_POINTS
30#include <trace/events/ufs.h>
31
2fbd009b
SJ		 32#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
33 UTP_TASK_REQ_COMPL |\
34 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 35/* UIC command timeout, unit: ms */
36#define UIC_CMD_TIMEOUT 500
2fbd009b 37
5a0b0cb9
SRT		 38/* NOP OUT retries waiting for NOP IN response */
39#define NOP_OUT_RETRIES 10
782e2efb
DP		 40/* Timeout after 50 msecs if NOP OUT hangs without response */
41#define NOP_OUT_TIMEOUT 50 /* msecs */
5a0b0cb9 42
68078d5c 43/* Query request retries */
10fe5888 44#define QUERY_REQ_RETRIES 3
68078d5c 45/* Query request timeout */
10fe5888 46#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 47
e2933132
SRT		 48/* Task management command timeout */
49#define TM_CMD_TIMEOUT 100 /* msecs */
50
64238fbd
YG		 51/* maximum number of retries for a general UIC command */
52#define UFS_UIC_COMMAND_RETRIES 3
53
1d337ec2
SRT		 54/* maximum number of link-startup retries */
55#define DME_LINKSTARTUP_RETRIES 3
56
87d0b4a6
YG		 57/* Maximum retries for Hibern8 enter */
58#define UIC_HIBERN8_ENTER_RETRIES 3
59
1d337ec2
SRT		 60/* maximum number of reset retries before giving up */
61#define MAX_HOST_RESET_RETRIES 5
62
68078d5c
DR		 63/* Expose the flag value from utp_upiu_query.value */
64#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
65
7d568652
SJ		 66/* Interrupt aggregation default timeout, unit: 40us */
67#define INT_AGGR_DEF_TO 0x02
68
49615ba1
SC		 69/* default delay of autosuspend: 2000 ms */
70#define RPM_AUTOSUSPEND_DELAY_MS 2000
71
51dd905b
SC		 72/* Default delay of RPM device flush delayed work */
73#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
74
09f17791
CG		 75/* Default value of wait time before gating device ref clock */
76#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
77
29707fab
KK		 78/* Polling time to wait for fDeviceInit */
79#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
80
aa497613
SRT		 81#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
82 ({ \
83 int _ret; \
84 if (_on) \
85 _ret = ufshcd_enable_vreg(_dev, _vreg); \
86 else \
87 _ret = ufshcd_disable_vreg(_dev, _vreg); \
88 _ret; \
89 })
90
ba80917d
TW		 91#define ufshcd_hex_dump(prefix_str, buf, len) do { \
92 size_t __len = (len); \
93 print_hex_dump(KERN_ERR, prefix_str, \
94 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
95 16, 4, buf, __len, false); \
96} while (0)
97
98int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
99 const char *prefix)
100{
d6724756
MG		 101 u32 *regs;
102 size_t pos;
103
104 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
105 return -EINVAL;
ba80917d 106
cddaebaf 107 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 108 if (!regs)
109 return -ENOMEM;
110
d6724756
MG		 111 for (pos = 0; pos < len; pos += 4)
112 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
113
ba80917d
TW		 114 ufshcd_hex_dump(prefix, regs, len);
115 kfree(regs);
116
117 return 0;
118}
119EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 120
7a3e97b0
SY		 121enum {
122 UFSHCD_MAX_CHANNEL = 0,
123 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 124 UFSHCD_CMD_PER_LUN = 32,
125 UFSHCD_CAN_QUEUE = 32,
126};
127
128/* UFSHCD states */
129enum {
7a3e97b0
SY		 130 UFSHCD_STATE_RESET,
131 UFSHCD_STATE_ERROR,
3441da7d 132 UFSHCD_STATE_OPERATIONAL,
5586dd8e
CG		 133 UFSHCD_STATE_EH_SCHEDULED_FATAL,
134 UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
3441da7d
SRT		 135};
136
137/* UFSHCD error handling flags */
138enum {
139 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 140};
141
e8e7f271
SRT		 142/* UFSHCD UIC layer error flags */
143enum {
144 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 145 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
146 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
147 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
148 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
149 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
2355b66e 150 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
e8e7f271
SRT		 151};
152
3441da7d 153#define ufshcd_set_eh_in_progress(h) \
9c490d2d 154 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 155#define ufshcd_eh_in_progress(h) \
9c490d2d 156 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 157#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 158 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 159
cbb6813e 160struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 161 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
162 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
163 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
164 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
165 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
166 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
fe1d4c2e
AH		 167 /*
168 * For DeepSleep, the link is first put in hibern8 and then off.
169 * Leaving the link in hibern8 is not supported.
170 */
171 {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
57d104c1
SJ		 172};
173
174static inline enum ufs_dev_pwr_mode
175ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
176{
177 return ufs_pm_lvl_states[lvl].dev_state;
178}
179
180static inline enum uic_link_state
181ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
182{
183 return ufs_pm_lvl_states[lvl].link_state;
184}
185
0c8f7586 186static inline enum ufs_pm_level
187ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
188 enum uic_link_state link_state)
189{
190 enum ufs_pm_level lvl;
191
192 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
193 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
194 (ufs_pm_lvl_states[lvl].link_state == link_state))
195 return lvl;
196 }
197
198 /* if no match found, return the level 0 */
199 return UFS_PM_LVL_0;
200}
201
56d4a186
SJ		 202static struct ufs_dev_fix ufs_fixups[] = {
203 /* UFS cards deviations table */
c0a18ee0
SC		 204 UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
205 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186 206 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
ed0b40ff
SC		 207 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
208 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
56d4a186 209 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
ed0b40ff
SC		 210 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
211 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
212 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
213 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 214 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
215 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
216 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
217 UFS_DEVICE_QUIRK_PA_TACTIVATE),
218 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
219 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 220 END_FIX
221};
222
9333d775 223static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 224static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 225static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 226static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 227static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2 228static void ufshcd_hba_exit(struct ufs_hba *hba);
1918651f 229static int ufshcd_clear_ua_wluns(struct ufs_hba *hba);
1b9e2141 230static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
1ab27c9c 231static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
1ab27c9c 232static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 233static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 234static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 235static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
236static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 237static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 238static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 239static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 240static int ufshcd_change_power_mode(struct ufs_hba *hba,
241 struct ufs_pa_layer_attr *pwr_mode);
4db7a236 242static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
c72e79c0
CG		 243static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
244static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
245static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
246 struct ufs_vreg *vreg);
307348f6 247static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
3d17b9b5 248static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
d3ba622d 249static inline int ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
dd7143e2
CG		 250static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
251static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
3d17b9b5 252
14497328
YG		 253static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
254{
255 return tag >= 0 && tag < hba->nutrs;
256}
57d104c1 257
5231d38c 258static inline void ufshcd_enable_irq(struct ufs_hba *hba)
57d104c1 259{
57d104c1 260 if (!hba->is_irq_enabled) {
5231d38c 261 enable_irq(hba->irq);
57d104c1
SJ		 262 hba->is_irq_enabled = true;
263 }
57d104c1
SJ		 264}
265
266static inline void ufshcd_disable_irq(struct ufs_hba *hba)
267{
268 if (hba->is_irq_enabled) {
5231d38c 269 disable_irq(hba->irq);
57d104c1
SJ		 270 hba->is_irq_enabled = false;
271 }
272}
3441da7d 273
3d17b9b5
AD		 274static inline void ufshcd_wb_config(struct ufs_hba *hba)
275{
276 int ret;
277
79e3520f 278 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 279 return;
280
281 ret = ufshcd_wb_ctrl(hba, true);
282 if (ret)
283 dev_err(hba->dev, "%s: Enable WB failed: %d\n", __func__, ret);
284 else
285 dev_info(hba->dev, "%s: Write Booster Configured\n", __func__);
286 ret = ufshcd_wb_toggle_flush_during_h8(hba, true);
287 if (ret)
288 dev_err(hba->dev, "%s: En WB flush during H8: failed: %d\n",
289 __func__, ret);
21acf460
SC		 290 if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
291 ufshcd_wb_toggle_flush(hba, true);
3d17b9b5
AD		 292}
293
38135535
SJ		 294static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
295{
296 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
297 scsi_unblock_requests(hba->host);
298}
299
300static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
301{
302 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
303 scsi_block_requests(hba->host);
304}
305
6667e6d9 306static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
28fa68fc 307 enum ufs_trace_str_t str_t)
6667e6d9
OS		 308{
309 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
310
9d5095e7
BH		 311 if (!trace_ufshcd_upiu_enabled())
312 return;
313
867fdc2d
BH		 314 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq->header, &rq->sc.cdb,
315 UFS_TSF_CDB);
6667e6d9
OS		 316}
317
fb475b74
AA		 318static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
319 enum ufs_trace_str_t str_t,
320 struct utp_upiu_req *rq_rsp)
6667e6d9 321{
9d5095e7
BH		 322 if (!trace_ufshcd_upiu_enabled())
323 return;
6667e6d9 324
be20b51c 325 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
867fdc2d 326 &rq_rsp->qr, UFS_TSF_OSF);
6667e6d9
OS		 327}
328
329static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
28fa68fc 330 enum ufs_trace_str_t str_t)
6667e6d9 331{
6667e6d9 332 int off = (int)tag - hba->nutrs;
391e388f 333 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 334
9d5095e7
BH		 335 if (!trace_ufshcd_upiu_enabled())
336 return;
337
0ed083e9
BH		 338 if (str_t == UFS_TM_SEND)
339 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &descp->req_header,
867fdc2d 340 &descp->input_param1, UFS_TSF_TM_INPUT);
0ed083e9
BH		 341 else
342 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &descp->rsp_header,
867fdc2d 343 &descp->output_param1, UFS_TSF_TM_OUTPUT);
6667e6d9
OS		 344}
345
aa5c6979
SC		 346static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
347 struct uic_command *ucmd,
28fa68fc 348 enum ufs_trace_str_t str_t)
aa5c6979
SC		 349{
350 u32 cmd;
351
352 if (!trace_ufshcd_uic_command_enabled())
353 return;
354
28fa68fc 355 if (str_t == UFS_CMD_SEND)
aa5c6979
SC		 356 cmd = ucmd->command;
357 else
358 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
359
28fa68fc 360 trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
aa5c6979
SC		 361 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
362 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
363 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
364}
365
28fa68fc
BH		 366static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
367 enum ufs_trace_str_t str_t)
1a07f2d9
LS		 368{
369 sector_t lba = -1;
69a314d6 370 u8 opcode = 0, group_id = 0;
1a07f2d9 371 u32 intr, doorbell;
e7c3b379 372 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
e4d2add7 373 struct scsi_cmnd *cmd = lrbp->cmd;
1a07f2d9
LS		 374 int transfer_len = -1;
375
e7c3b379
OS		 376 if (!trace_ufshcd_command_enabled()) {
377 /* trace UPIU W/O tracing command */
e4d2add7 378 if (cmd)
28fa68fc 379 ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
1a07f2d9 380 return;
e7c3b379 381 }
1a07f2d9 382
e4d2add7 383 if (cmd) { /* data phase exists */
e7c3b379 384 /* trace UPIU also */
28fa68fc 385 ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
e4d2add7 386 opcode = cmd->cmnd[0];
1a07f2d9
LS		 387 if ((opcode == READ_10) || (opcode == WRITE_10)) {
388 /*
389 * Currently we only fully trace read(10) and write(10)
390 * commands
391 */
e4d2add7
BVA		 392 if (cmd->request && cmd->request->bio)
393 lba = cmd->request->bio->bi_iter.bi_sector;
1a07f2d9
LS		 394 transfer_len = be32_to_cpu(
395 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
69a314d6
JK		 396 if (opcode == WRITE_10)
397 group_id = lrbp->cmd->cmnd[6];
3754cde8
LL		 398 } else if (opcode == UNMAP) {
399 if (cmd->request) {
400 lba = scsi_get_lba(cmd);
401 transfer_len = blk_rq_bytes(cmd->request);
402 }
1a07f2d9
LS		 403 }
404 }
405
406 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
407 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
28fa68fc 408 trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
69a314d6 409 doorbell, transfer_len, intr, lba, opcode, group_id);
1a07f2d9
LS		 410}
411
ff8e20c6
DR		 412static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
413{
414 struct ufs_clk_info *clki;
415 struct list_head *head = &hba->clk_list_head;
416
566ec9ad 417 if (list_empty(head))
ff8e20c6
DR		 418 return;
419
420 list_for_each_entry(clki, head, list) {
421 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
422 clki->max_freq)
423 dev_err(hba->dev, "clk: %s, rate: %u\n",
424 clki->name, clki->curr_freq);
425 }
426}
427
e965e5e0
SC		 428static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
429 char *err_name)
ff8e20c6
DR		 430{
431 int i;
27752647 432 bool found = false;
e965e5e0 433 struct ufs_event_hist *e;
ff8e20c6 434
e965e5e0
SC		 435 if (id >= UFS_EVT_CNT)
436 return;
ff8e20c6 437
e965e5e0 438 e = &hba->ufs_stats.event[id];
ff8e20c6 439
e965e5e0
SC		 440 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
441 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
442
443 if (e->tstamp[p] == 0)
ff8e20c6 444 continue;
c5397f13 445 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
e965e5e0 446 e->val[p], ktime_to_us(e->tstamp[p]));
27752647 447 found = true;
ff8e20c6 448 }
27752647
SC		 449
450 if (!found)
fd1fb4d5 451 dev_err(hba->dev, "No record of %s\n", err_name);
bafd09f8
DH		 452 else
453 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
ff8e20c6
DR		 454}
455
e965e5e0 456static void ufshcd_print_evt_hist(struct ufs_hba *hba)
66cc820f 457{
ba80917d 458 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
ff8e20c6 459
e965e5e0
SC		 460 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
461 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
462 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
463 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
464 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
465 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
466 "auto_hibern8_err");
467 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
468 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
469 "link_startup_fail");
470 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
471 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
472 "suspend_fail");
473 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
474 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
475 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
ff8e20c6 476
7c486d91 477 ufshcd_vops_dbg_register_dump(hba);
66cc820f
DR		 478}
479
480static
481void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
482{
483 struct ufshcd_lrb *lrbp;
7fabb77b 484 int prdt_length;
66cc820f
DR		 485 int tag;
486
487 for_each_set_bit(tag, &bitmap, hba->nutrs) {
488 lrbp = &hba->lrb[tag];
489
ff8e20c6
DR		 490 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
491 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 492 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
493 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 494 dev_err(hba->dev,
495 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
496 tag, (u64)lrbp->utrd_dma_addr);
497
66cc820f
DR		 498 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
499 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 500 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
501 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 502 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
503 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 504 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
505 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 506 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
507 sizeof(struct utp_upiu_rsp));
66cc820f 508
7fabb77b
GB		 509 prdt_length = le16_to_cpu(
510 lrbp->utr_descriptor_ptr->prd_table_length);
cc770ce3
EB		 511 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
512 prdt_length /= sizeof(struct ufshcd_sg_entry);
513
7fabb77b
GB		 514 dev_err(hba->dev,
515 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
516 tag, prdt_length,
517 (u64)lrbp->ucd_prdt_dma_addr);
518
519 if (pr_prdt)
66cc820f 520 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 521 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 522 }
523}
524
525static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
526{
66cc820f
DR		 527 int tag;
528
529 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 530 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
531
66cc820f 532 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 533 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 534 }
535}
536
6ba65588
GB		 537static void ufshcd_print_host_state(struct ufs_hba *hba)
538{
3f8af604
CG		 539 struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
540
6ba65588 541 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 542 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
543 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 544 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
545 hba->saved_err, hba->saved_uic_err);
546 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
547 hba->curr_dev_pwr_mode, hba->uic_link_state);
548 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
549 hba->pm_op_in_progress, hba->is_sys_suspended);
550 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
551 hba->auto_bkops_enabled, hba->host->host_self_blocked);
552 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
3f8af604
CG		 553 dev_err(hba->dev,
554 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
555 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
556 hba->ufs_stats.hibern8_exit_cnt);
557 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
558 ktime_to_us(hba->ufs_stats.last_intr_ts),
559 hba->ufs_stats.last_intr_status);
6ba65588
GB		 560 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
561 hba->eh_flags, hba->req_abort_count);
3f8af604
CG		 562 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
563 hba->ufs_version, hba->capabilities, hba->caps);
6ba65588
GB		 564 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
565 hba->dev_quirks);
3f8af604
CG		 566 if (sdev_ufs)
567 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
568 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
569
570 ufshcd_print_clk_freqs(hba);
6ba65588
GB		 571}
572
ff8e20c6
DR		 573/**
574 * ufshcd_print_pwr_info - print power params as saved in hba
575 * power info
576 * @hba: per-adapter instance
577 */
578static void ufshcd_print_pwr_info(struct ufs_hba *hba)
579{
580 static const char * const names[] = {
581 "INVALID MODE",
582 "FAST MODE",
583 "SLOW_MODE",
584 "INVALID MODE",
585 "FASTAUTO_MODE",
586 "SLOWAUTO_MODE",
587 "INVALID MODE",
588 };
589
590 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
591 __func__,
592 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
593 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
594 names[hba->pwr_info.pwr_rx],
595 names[hba->pwr_info.pwr_tx],
596 hba->pwr_info.hs_rate);
597}
598
31a5d9ca
SC		 599static void ufshcd_device_reset(struct ufs_hba *hba)
600{
601 int err;
602
603 err = ufshcd_vops_device_reset(hba);
604
605 if (!err) {
606 ufshcd_set_ufs_dev_active(hba);
607 if (ufshcd_is_wb_allowed(hba)) {
4cd48995
BH		 608 hba->dev_info.wb_enabled = false;
609 hba->dev_info.wb_buf_flush_enabled = false;
31a5d9ca
SC		 610 }
611 }
612 if (err != -EOPNOTSUPP)
613 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
614}
615
5c955c10
SC		 616void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
617{
618 if (!us)
619 return;
620
621 if (us < 10)
622 udelay(us);
623 else
624 usleep_range(us, us + tolerance);
625}
626EXPORT_SYMBOL_GPL(ufshcd_delay_us);
627
5cac1095 628/**
5a0b0cb9 629 * ufshcd_wait_for_register - wait for register value to change
5cac1095
BVA		 630 * @hba: per-adapter interface
631 * @reg: mmio register offset
632 * @mask: mask to apply to the read register value
633 * @val: value to wait for
634 * @interval_us: polling interval in microseconds
635 * @timeout_ms: timeout in milliseconds
5a0b0cb9 636 *
5cac1095
BVA		 637 * Return:
638 * -ETIMEDOUT on error, zero on success.
5a0b0cb9 639 */
596585a2
YG		 640int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
641 u32 val, unsigned long interval_us,
5cac1095 642 unsigned long timeout_ms)
5a0b0cb9
SRT		 643{
644 int err = 0;
645 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
646
647 /* ignore bits that we don't intend to wait on */
648 val = val & mask;
649
650 while ((ufshcd_readl(hba, reg) & mask) != val) {
5cac1095 651 usleep_range(interval_us, interval_us + 50);
5a0b0cb9
SRT		 652 if (time_after(jiffies, timeout)) {
653 if ((ufshcd_readl(hba, reg) & mask) != val)
654 err = -ETIMEDOUT;
655 break;
656 }
657 }
658
659 return err;
660}
661
2fbd009b
SJ		 662/**
663 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 664 * @hba: Pointer to adapter instance
2fbd009b
SJ		 665 *
666 * Returns interrupt bit mask per version
667 */
668static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
669{
c01848c6
YG		 670 u32 intr_mask = 0;
671
672 switch (hba->ufs_version) {
673 case UFSHCI_VERSION_10:
674 intr_mask = INTERRUPT_MASK_ALL_VER_10;
675 break;
c01848c6
YG		 676 case UFSHCI_VERSION_11:
677 case UFSHCI_VERSION_20:
678 intr_mask = INTERRUPT_MASK_ALL_VER_11;
679 break;
c01848c6
YG		 680 case UFSHCI_VERSION_21:
681 default:
682 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 683 break;
c01848c6
YG		 684 }
685
686 return intr_mask;
2fbd009b
SJ		 687}
688
7a3e97b0
SY		 689/**
690 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 691 * @hba: Pointer to adapter instance
7a3e97b0
SY		 692 *
693 * Returns UFSHCI version supported by the controller
694 */
695static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
696{
0263bcd0
YG		 697 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
698 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 699
b873a275 700 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 701}
702
703/**
704 * ufshcd_is_device_present - Check if any device connected to
705 * the host controller
5c0c28a8 706 * @hba: pointer to adapter instance
7a3e97b0 707 *
c9e6010b 708 * Returns true if device present, false if no device detected
7a3e97b0 709 */
c9e6010b 710static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 711{
5c0c28a8 712 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 713 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 714}
715
716/**
717 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 718 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 719 *
720 * This function is used to get the OCS field from UTRD
721 * Returns the OCS field in the UTRD
722 */
723static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
724{
e8c8e82a 725 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 726}
727
7a3e97b0
SY		 728/**
729 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
730 * @hba: per adapter instance
731 * @pos: position of the bit to be cleared
732 */
733static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
734{
87183841
AA		 735 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
736 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
737 else
738 ufshcd_writel(hba, ~(1 << pos),
739 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
1399c5b0
AA		 740}
741
742/**
743 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
744 * @hba: per adapter instance
745 * @pos: position of the bit to be cleared
746 */
747static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
748{
87183841
AA		 749 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
750 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
751 else
752 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 753}
754
a48353f6
YG		 755/**
756 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
757 * @hba: per adapter instance
758 * @tag: position of the bit to be cleared
759 */
760static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
761{
762 __clear_bit(tag, &hba->outstanding_reqs);
763}
764
7a3e97b0
SY		 765/**
766 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
767 * @reg: Register value of host controller status
768 *
769 * Returns integer, 0 on Success and positive value if failed
770 */
771static inline int ufshcd_get_lists_status(u32 reg)
772{
6cf16115 773 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 774}
775
776/**
777 * ufshcd_get_uic_cmd_result - Get the UIC command result
778 * @hba: Pointer to adapter instance
779 *
780 * This function gets the result of UIC command completion
781 * Returns 0 on success, non zero value on error
782 */
783static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
784{
b873a275 785 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 786 MASK_UIC_COMMAND_RESULT;
787}
788
12b4fdb4
SJ		 789/**
790 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
791 * @hba: Pointer to adapter instance
792 *
793 * This function gets UIC command argument3
794 * Returns 0 on success, non zero value on error
795 */
796static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
797{
798 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
799}
800
7a3e97b0 801/**
5a0b0cb9 802 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 803 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 804 */
805static inline int
5a0b0cb9 806ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 807{
5a0b0cb9 808 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 809}
810
811/**
812 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
813 * @ucd_rsp_ptr: pointer to response UPIU
814 *
815 * This function gets the response status and scsi_status from response UPIU
816 * Returns the response result code.
817 */
818static inline int
819ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
820{
821 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
822}
823
1c2623c5
SJ		 824/*
825 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
826 * from response UPIU
827 * @ucd_rsp_ptr: pointer to response UPIU
828 *
829 * Return the data segment length.
830 */
831static inline unsigned int
832ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
833{
834 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
835 MASK_RSP_UPIU_DATA_SEG_LEN;
836}
837
66ec6d59
SRT		 838/**
839 * ufshcd_is_exception_event - Check if the device raised an exception event
840 * @ucd_rsp_ptr: pointer to response UPIU
841 *
842 * The function checks if the device raised an exception event indicated in
843 * the Device Information field of response UPIU.
844 *
845 * Returns true if exception is raised, false otherwise.
846 */
847static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
848{
849 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
850 MASK_RSP_EXCEPTION_EVENT ? true : false;
851}
852
7a3e97b0 853/**
7d568652 854 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 855 * @hba: per adapter instance
7a3e97b0
SY		 856 */
857static inline void
7d568652 858ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 859{
7d568652
SJ		 860 ufshcd_writel(hba, INT_AGGR_ENABLE |
861 INT_AGGR_COUNTER_AND_TIMER_RESET,
862 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
863}
864
865/**
866 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
867 * @hba: per adapter instance
868 * @cnt: Interrupt aggregation counter threshold
869 * @tmout: Interrupt aggregation timeout value
870 */
871static inline void
872ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
873{
874 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
875 INT_AGGR_COUNTER_THLD_VAL(cnt) |
876 INT_AGGR_TIMEOUT_VAL(tmout),
877 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 878}
879
b852190e
YG		 880/**
881 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
882 * @hba: per adapter instance
883 */
884static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
885{
886 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
887}
888
7a3e97b0
SY		 889/**
890 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
891 * When run-stop registers are set to 1, it indicates the
892 * host controller that it can process the requests
893 * @hba: per adapter instance
894 */
895static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
896{
b873a275
SJ		 897 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
898 REG_UTP_TASK_REQ_LIST_RUN_STOP);
899 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
900 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 901}
902
7a3e97b0
SY		 903/**
904 * ufshcd_hba_start - Start controller initialization sequence
905 * @hba: per adapter instance
906 */
907static inline void ufshcd_hba_start(struct ufs_hba *hba)
908{
df043c74
ST		 909 u32 val = CONTROLLER_ENABLE;
910
911 if (ufshcd_crypto_enable(hba))
912 val |= CRYPTO_GENERAL_ENABLE;
913
914 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 915}
916
917/**
918 * ufshcd_is_hba_active - Get controller state
919 * @hba: per adapter instance
920 *
c9e6010b 921 * Returns false if controller is active, true otherwise
7a3e97b0 922 */
c9e6010b 923static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 924{
4a8eec2b
TK		 925 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
926 ? false : true;
7a3e97b0
SY		 927}
928
37113106
YG		 929u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
930{
931 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
932 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
933 (hba->ufs_version == UFSHCI_VERSION_11))
934 return UFS_UNIPRO_VER_1_41;
935 else
936 return UFS_UNIPRO_VER_1_6;
937}
938EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
939
940static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
941{
942 /*
943 * If both host and device support UniPro ver1.6 or later, PA layer
944 * parameters tuning happens during link startup itself.
945 *
946 * We can manually tune PA layer parameters if either host or device
947 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
948 * logic simple, we will only do manual tuning if local unipro version
949 * doesn't support ver1.6 or later.
950 */
951 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
952 return true;
953 else
954 return false;
955}
956
394b949f
SJ		 957/**
958 * ufshcd_set_clk_freq - set UFS controller clock frequencies
959 * @hba: per adapter instance
960 * @scale_up: If True, set max possible frequency othewise set low frequency
961 *
962 * Returns 0 if successful
963 * Returns < 0 for any other errors
964 */
965static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
a3cd5ec5 966{
967 int ret = 0;
968 struct ufs_clk_info *clki;
969 struct list_head *head = &hba->clk_list_head;
a3cd5ec5 970
566ec9ad 971 if (list_empty(head))
a3cd5ec5 972 goto out;
973
a3cd5ec5 974 list_for_each_entry(clki, head, list) {
975 if (!IS_ERR_OR_NULL(clki->clk)) {
976 if (scale_up && clki->max_freq) {
977 if (clki->curr_freq == clki->max_freq)
978 continue;
979
a3cd5ec5 980 ret = clk_set_rate(clki->clk, clki->max_freq);
981 if (ret) {
982 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
983 __func__, clki->name,
984 clki->max_freq, ret);
985 break;
986 }
987 trace_ufshcd_clk_scaling(dev_name(hba->dev),
988 "scaled up", clki->name,
989 clki->curr_freq,
990 clki->max_freq);
991
992 clki->curr_freq = clki->max_freq;
993
994 } else if (!scale_up && clki->min_freq) {
995 if (clki->curr_freq == clki->min_freq)
996 continue;
997
a3cd5ec5 998 ret = clk_set_rate(clki->clk, clki->min_freq);
999 if (ret) {
1000 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1001 __func__, clki->name,
1002 clki->min_freq, ret);
1003 break;
1004 }
1005 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1006 "scaled down", clki->name,
1007 clki->curr_freq,
1008 clki->min_freq);
1009 clki->curr_freq = clki->min_freq;
1010 }
1011 }
1012 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1013 clki->name, clk_get_rate(clki->clk));
1014 }
1015
394b949f
SJ		 1016out:
1017 return ret;
1018}
1019
1020/**
1021 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1022 * @hba: per adapter instance
1023 * @scale_up: True if scaling up and false if scaling down
1024 *
1025 * Returns 0 if successful
1026 * Returns < 0 for any other errors
1027 */
1028static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1029{
1030 int ret = 0;
1031 ktime_t start = ktime_get();
1032
1033 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1034 if (ret)
1035 goto out;
1036
1037 ret = ufshcd_set_clk_freq(hba, scale_up);
1038 if (ret)
1039 goto out;
1040
a3cd5ec5 1041 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
394b949f
SJ		 1042 if (ret)
1043 ufshcd_set_clk_freq(hba, !scale_up);
a3cd5ec5 1044
1045out:
394b949f 1046 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
a3cd5ec5 1047 (scale_up ? "up" : "down"),
1048 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1049 return ret;
1050}
1051
1052/**
1053 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1054 * @hba: per adapter instance
1055 * @scale_up: True if scaling up and false if scaling down
1056 *
1057 * Returns true if scaling is required, false otherwise.
1058 */
1059static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1060 bool scale_up)
1061{
1062 struct ufs_clk_info *clki;
1063 struct list_head *head = &hba->clk_list_head;
1064
566ec9ad 1065 if (list_empty(head))
a3cd5ec5 1066 return false;
1067
1068 list_for_each_entry(clki, head, list) {
1069 if (!IS_ERR_OR_NULL(clki->clk)) {
1070 if (scale_up && clki->max_freq) {
1071 if (clki->curr_freq == clki->max_freq)
1072 continue;
1073 return true;
1074 } else if (!scale_up && clki->min_freq) {
1075 if (clki->curr_freq == clki->min_freq)
1076 continue;
1077 return true;
1078 }
1079 }
1080 }
1081
1082 return false;
1083}
1084
1085static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1086 u64 wait_timeout_us)
1087{
1088 unsigned long flags;
1089 int ret = 0;
1090 u32 tm_doorbell;
1091 u32 tr_doorbell;
1092 bool timeout = false, do_last_check = false;
1093 ktime_t start;
1094
1095 ufshcd_hold(hba, false);
1096 spin_lock_irqsave(hba->host->host_lock, flags);
1097 /*
1098 * Wait for all the outstanding tasks/transfer requests.
1099 * Verify by checking the doorbell registers are clear.
1100 */
1101 start = ktime_get();
1102 do {
1103 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1104 ret = -EBUSY;
1105 goto out;
1106 }
1107
1108 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1109 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1110 if (!tm_doorbell && !tr_doorbell) {
1111 timeout = false;
1112 break;
1113 } else if (do_last_check) {
1114 break;
1115 }
1116
1117 spin_unlock_irqrestore(hba->host->host_lock, flags);
1118 schedule();
1119 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1120 wait_timeout_us) {
1121 timeout = true;
1122 /*
1123 * We might have scheduled out for long time so make
1124 * sure to check if doorbells are cleared by this time
1125 * or not.
1126 */
1127 do_last_check = true;
1128 }
1129 spin_lock_irqsave(hba->host->host_lock, flags);
1130 } while (tm_doorbell || tr_doorbell);
1131
1132 if (timeout) {
1133 dev_err(hba->dev,
1134 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1135 __func__, tm_doorbell, tr_doorbell);
1136 ret = -EBUSY;
1137 }
1138out:
1139 spin_unlock_irqrestore(hba->host->host_lock, flags);
1140 ufshcd_release(hba);
1141 return ret;
1142}
1143
1144/**
1145 * ufshcd_scale_gear - scale up/down UFS gear
1146 * @hba: per adapter instance
1147 * @scale_up: True for scaling up gear and false for scaling down
1148 *
1149 * Returns 0 for success,
1150 * Returns -EBUSY if scaling can't happen at this time
1151 * Returns non-zero for any other errors
1152 */
1153static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1154{
a3cd5ec5 1155 int ret = 0;
1156 struct ufs_pa_layer_attr new_pwr_info;
1157
1158 if (scale_up) {
1159 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1160 sizeof(struct ufs_pa_layer_attr));
1161 } else {
1162 memcpy(&new_pwr_info, &hba->pwr_info,
1163 sizeof(struct ufs_pa_layer_attr));
1164
29b87e92
CG		 1165 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1166 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
a3cd5ec5 1167 /* save the current power mode */
1168 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1169 &hba->pwr_info,
1170 sizeof(struct ufs_pa_layer_attr));
1171
1172 /* scale down gear */
29b87e92
CG		 1173 new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1174 new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
a3cd5ec5 1175 }
1176 }
1177
1178 /* check if the power mode needs to be changed or not? */
6a9df818 1179 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
a3cd5ec5 1180 if (ret)
1181 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1182 __func__, ret,
1183 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1184 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1185
1186 return ret;
1187}
1188
1189static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1190{
1191 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1192 int ret = 0;
1193 /*
1194 * make sure that there are no outstanding requests when
1195 * clock scaling is in progress
1196 */
38135535 1197 ufshcd_scsi_block_requests(hba);
a3cd5ec5 1198 down_write(&hba->clk_scaling_lock);
0e9d4ca4
CG		 1199
1200 if (!hba->clk_scaling.is_allowed ||
1201 ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
a3cd5ec5 1202 ret = -EBUSY;
1203 up_write(&hba->clk_scaling_lock);
38135535 1204 ufshcd_scsi_unblock_requests(hba);
0e9d4ca4 1205 goto out;
a3cd5ec5 1206 }
1207
0e9d4ca4
CG		 1208 /* let's not get into low power until clock scaling is completed */
1209 ufshcd_hold(hba, false);
1210
1211out:
a3cd5ec5 1212 return ret;
1213}
1214
0e9d4ca4 1215static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, bool writelock)
a3cd5ec5 1216{
0e9d4ca4
CG		 1217 if (writelock)
1218 up_write(&hba->clk_scaling_lock);
1219 else
1220 up_read(&hba->clk_scaling_lock);
38135535 1221 ufshcd_scsi_unblock_requests(hba);
0e9d4ca4 1222 ufshcd_release(hba);
a3cd5ec5 1223}
1224
1225/**
1226 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1227 * @hba: per adapter instance
1228 * @scale_up: True for scaling up and false for scalin down
1229 *
1230 * Returns 0 for success,
1231 * Returns -EBUSY if scaling can't happen at this time
1232 * Returns non-zero for any other errors
1233 */
1234static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1235{
1236 int ret = 0;
0e9d4ca4 1237 bool is_writelock = true;
401f1e44 1238
a3cd5ec5 1239 ret = ufshcd_clock_scaling_prepare(hba);
1240 if (ret)
0e9d4ca4 1241 return ret;
a3cd5ec5 1242
1243 /* scale down the gear before scaling down clocks */
1244 if (!scale_up) {
1245 ret = ufshcd_scale_gear(hba, false);
1246 if (ret)
394b949f 1247 goto out_unprepare;
a3cd5ec5 1248 }
1249
1250 ret = ufshcd_scale_clks(hba, scale_up);
1251 if (ret) {
1252 if (!scale_up)
1253 ufshcd_scale_gear(hba, true);
394b949f 1254 goto out_unprepare;
a3cd5ec5 1255 }
1256
1257 /* scale up the gear after scaling up clocks */
1258 if (scale_up) {
1259 ret = ufshcd_scale_gear(hba, true);
3d17b9b5 1260 if (ret) {
a3cd5ec5 1261 ufshcd_scale_clks(hba, false);
3d17b9b5
AD		 1262 goto out_unprepare;
1263 }
a3cd5ec5 1264 }
1265
3d17b9b5 1266 /* Enable Write Booster if we have scaled up else disable it */
0e9d4ca4
CG		 1267 downgrade_write(&hba->clk_scaling_lock);
1268 is_writelock = false;
3d17b9b5 1269 ufshcd_wb_ctrl(hba, scale_up);
3d17b9b5 1270
394b949f 1271out_unprepare:
0e9d4ca4 1272 ufshcd_clock_scaling_unprepare(hba, is_writelock);
a3cd5ec5 1273 return ret;
1274}
1275
401f1e44 1276static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1277{
1278 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1279 clk_scaling.suspend_work);
1280 unsigned long irq_flags;
1281
1282 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1283 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1284 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1285 return;
1286 }
1287 hba->clk_scaling.is_suspended = true;
1288 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1289
1290 __ufshcd_suspend_clkscaling(hba);
1291}
1292
1293static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1294{
1295 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1296 clk_scaling.resume_work);
1297 unsigned long irq_flags;
1298
1299 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1300 if (!hba->clk_scaling.is_suspended) {
1301 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1302 return;
1303 }
1304 hba->clk_scaling.is_suspended = false;
1305 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1306
1307 devfreq_resume_device(hba->devfreq);
1308}
1309
a3cd5ec5 1310static int ufshcd_devfreq_target(struct device *dev,
1311 unsigned long *freq, u32 flags)
1312{
1313 int ret = 0;
1314 struct ufs_hba *hba = dev_get_drvdata(dev);
1315 ktime_t start;
401f1e44 1316 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1317 struct list_head *clk_list = &hba->clk_list_head;
1318 struct ufs_clk_info *clki;
a3cd5ec5 1319 unsigned long irq_flags;
1320
1321 if (!ufshcd_is_clkscaling_supported(hba))
1322 return -EINVAL;
1323
91831d33
AD		 1324 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1325 /* Override with the closest supported frequency */
1326 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
a3cd5ec5 1327 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1328 if (ufshcd_eh_in_progress(hba)) {
1329 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1330 return 0;
1331 }
1332
401f1e44 1333 if (!hba->clk_scaling.active_reqs)
1334 sched_clk_scaling_suspend_work = true;
1335
092b4558
BA		 1336 if (list_empty(clk_list)) {
1337 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1338 goto out;
1339 }
1340
91831d33 1341 /* Decide based on the rounded-off frequency and update */
092b4558 1342 scale_up = (*freq == clki->max_freq) ? true : false;
91831d33
AD		 1343 if (!scale_up)
1344 *freq = clki->min_freq;
1345 /* Update the frequency */
401f1e44 1346 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1347 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1348 ret = 0;
1349 goto out; /* no state change required */
a3cd5ec5 1350 }
1351 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1352
1353 start = ktime_get();
a3cd5ec5 1354 ret = ufshcd_devfreq_scale(hba, scale_up);
1355
a3cd5ec5 1356 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1357 (scale_up ? "up" : "down"),
1358 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1359
401f1e44 1360out:
1361 if (sched_clk_scaling_suspend_work)
1362 queue_work(hba->clk_scaling.workq,
1363 &hba->clk_scaling.suspend_work);
1364
a3cd5ec5 1365 return ret;
1366}
1367
7252a360
BVA		 1368static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1369{
1370 int *busy = priv;
1371
1372 WARN_ON_ONCE(reserved);
1373 (*busy)++;
1374 return false;
1375}
1376
1377/* Whether or not any tag is in use by a request that is in progress. */
1378static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1379{
1380 struct request_queue *q = hba->cmd_queue;
1381 int busy = 0;
1382
1383 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1384 return busy;
1385}
a3cd5ec5 1386
1387static int ufshcd_devfreq_get_dev_status(struct device *dev,
1388 struct devfreq_dev_status *stat)
1389{
1390 struct ufs_hba *hba = dev_get_drvdata(dev);
1391 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1392 unsigned long flags;
91831d33
AD		 1393 struct list_head *clk_list = &hba->clk_list_head;
1394 struct ufs_clk_info *clki;
b1bf66d1 1395 ktime_t curr_t;
a3cd5ec5 1396
1397 if (!ufshcd_is_clkscaling_supported(hba))
1398 return -EINVAL;
1399
1400 memset(stat, 0, sizeof(*stat));
1401
1402 spin_lock_irqsave(hba->host->host_lock, flags);
b1bf66d1 1403 curr_t = ktime_get();
a3cd5ec5 1404 if (!scaling->window_start_t)
1405 goto start_window;
1406
91831d33
AD		 1407 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1408 /*
1409 * If current frequency is 0, then the ondemand governor considers
1410 * there's no initial frequency set. And it always requests to set
1411 * to max. frequency.
1412 */
1413 stat->current_frequency = clki->curr_freq;
a3cd5ec5 1414 if (scaling->is_busy_started)
b1bf66d1
SC		 1415 scaling->tot_busy_t += ktime_us_delta(curr_t,
1416 scaling->busy_start_t);
a3cd5ec5 1417
b1bf66d1 1418 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
a3cd5ec5 1419 stat->busy_time = scaling->tot_busy_t;
1420start_window:
b1bf66d1 1421 scaling->window_start_t = curr_t;
a3cd5ec5 1422 scaling->tot_busy_t = 0;
1423
1424 if (hba->outstanding_reqs) {
b1bf66d1 1425 scaling->busy_start_t = curr_t;
a3cd5ec5 1426 scaling->is_busy_started = true;
1427 } else {
1428 scaling->busy_start_t = 0;
1429 scaling->is_busy_started = false;
1430 }
1431 spin_unlock_irqrestore(hba->host->host_lock, flags);
1432 return 0;
1433}
1434
deac444f
BA		 1435static int ufshcd_devfreq_init(struct ufs_hba *hba)
1436{
092b4558
BA		 1437 struct list_head *clk_list = &hba->clk_list_head;
1438 struct ufs_clk_info *clki;
deac444f
BA		 1439 struct devfreq *devfreq;
1440 int ret;
1441
092b4558
BA		 1442 /* Skip devfreq if we don't have any clocks in the list */
1443 if (list_empty(clk_list))
1444 return 0;
1445
1446 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1447 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1448 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1449
90b8491c
SC		 1450 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1451 &hba->vps->ondemand_data);
092b4558 1452 devfreq = devfreq_add_device(hba->dev,
90b8491c 1453 &hba->vps->devfreq_profile,
deac444f 1454 DEVFREQ_GOV_SIMPLE_ONDEMAND,
90b8491c 1455 &hba->vps->ondemand_data);
deac444f
BA		 1456 if (IS_ERR(devfreq)) {
1457 ret = PTR_ERR(devfreq);
1458 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1459
1460 dev_pm_opp_remove(hba->dev, clki->min_freq);
1461 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1462 return ret;
1463 }
1464
1465 hba->devfreq = devfreq;
1466
1467 return 0;
1468}
1469
092b4558
BA		 1470static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1471{
1472 struct list_head *clk_list = &hba->clk_list_head;
1473 struct ufs_clk_info *clki;
1474
1475 if (!hba->devfreq)
1476 return;
1477
1478 devfreq_remove_device(hba->devfreq);
1479 hba->devfreq = NULL;
1480
1481 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1482 dev_pm_opp_remove(hba->dev, clki->min_freq);
1483 dev_pm_opp_remove(hba->dev, clki->max_freq);
1484}
1485
401f1e44 1486static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1487{
1488 unsigned long flags;
1489
1490 devfreq_suspend_device(hba->devfreq);
1491 spin_lock_irqsave(hba->host->host_lock, flags);
1492 hba->clk_scaling.window_start_t = 0;
1493 spin_unlock_irqrestore(hba->host->host_lock, flags);
1494}
a3cd5ec5 1495
a508253d
GB		 1496static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1497{
401f1e44 1498 unsigned long flags;
1499 bool suspend = false;
1500
f9a7fa34
SC		 1501 cancel_work_sync(&hba->clk_scaling.suspend_work);
1502 cancel_work_sync(&hba->clk_scaling.resume_work);
fcb0c4b0 1503
401f1e44 1504 spin_lock_irqsave(hba->host->host_lock, flags);
1505 if (!hba->clk_scaling.is_suspended) {
1506 suspend = true;
1507 hba->clk_scaling.is_suspended = true;
1508 }
1509 spin_unlock_irqrestore(hba->host->host_lock, flags);
1510
1511 if (suspend)
1512 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1513}
1514
1515static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1516{
401f1e44 1517 unsigned long flags;
1518 bool resume = false;
1519
401f1e44 1520 spin_lock_irqsave(hba->host->host_lock, flags);
1521 if (hba->clk_scaling.is_suspended) {
1522 resume = true;
1523 hba->clk_scaling.is_suspended = false;
1524 }
1525 spin_unlock_irqrestore(hba->host->host_lock, flags);
1526
1527 if (resume)
1528 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1529}
1530
1531static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1532 struct device_attribute *attr, char *buf)
1533{
1534 struct ufs_hba *hba = dev_get_drvdata(dev);
1535
0e9d4ca4 1536 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_enabled);
fcb0c4b0
ST		 1537}
1538
1539static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1540 struct device_attribute *attr, const char *buf, size_t count)
1541{
1542 struct ufs_hba *hba = dev_get_drvdata(dev);
1543 u32 value;
9cd20d3f 1544 int err = 0;
fcb0c4b0
ST		 1545
1546 if (kstrtou32(buf, 0, &value))
1547 return -EINVAL;
1548
9cd20d3f
CG		 1549 down(&hba->host_sem);
1550 if (!ufshcd_is_user_access_allowed(hba)) {
1551 err = -EBUSY;
1552 goto out;
1553 }
1554
fcb0c4b0 1555 value = !!value;
0e9d4ca4 1556 if (value == hba->clk_scaling.is_enabled)
fcb0c4b0
ST		 1557 goto out;
1558
1559 pm_runtime_get_sync(hba->dev);
1560 ufshcd_hold(hba, false);
1561
0e9d4ca4 1562 hba->clk_scaling.is_enabled = value;
401f1e44 1563
fcb0c4b0
ST		 1564 if (value) {
1565 ufshcd_resume_clkscaling(hba);
1566 } else {
1567 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1568 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1569 if (err)
1570 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1571 __func__, err);
1572 }
fcb0c4b0
ST		 1573
1574 ufshcd_release(hba);
1575 pm_runtime_put_sync(hba->dev);
1576out:
9cd20d3f
CG		 1577 up(&hba->host_sem);
1578 return err ? err : count;
a508253d
GB		 1579}
1580
4543d9d7 1581static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
a3cd5ec5 1582{
1583 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1584 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1585 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1586 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1587 hba->clk_scaling.enable_attr.attr.mode = 0644;
1588 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1589 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1590}
1591
4543d9d7
CG		 1592static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1593{
1594 if (hba->clk_scaling.enable_attr.attr.name)
1595 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1596}
1597
1598static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1599{
1600 char wq_name[sizeof("ufs_clkscaling_00")];
1601
1602 if (!ufshcd_is_clkscaling_supported(hba))
1603 return;
1604
80d892f4
CG		 1605 if (!hba->clk_scaling.min_gear)
1606 hba->clk_scaling.min_gear = UFS_HS_G1;
1607
4543d9d7
CG		 1608 INIT_WORK(&hba->clk_scaling.suspend_work,
1609 ufshcd_clk_scaling_suspend_work);
1610 INIT_WORK(&hba->clk_scaling.resume_work,
1611 ufshcd_clk_scaling_resume_work);
1612
1613 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1614 hba->host->host_no);
1615 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1616
1617 hba->clk_scaling.is_initialized = true;
1618}
1619
1620static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1621{
1622 if (!hba->clk_scaling.is_initialized)
1623 return;
1624
1625 ufshcd_remove_clk_scaling_sysfs(hba);
1626 destroy_workqueue(hba->clk_scaling.workq);
1627 ufshcd_devfreq_remove(hba);
1628 hba->clk_scaling.is_initialized = false;
1629}
1630
1ab27c9c
ST		 1631static void ufshcd_ungate_work(struct work_struct *work)
1632{
1633 int ret;
1634 unsigned long flags;
1635 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1636 clk_gating.ungate_work);
1637
1638 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1639
1640 spin_lock_irqsave(hba->host->host_lock, flags);
1641 if (hba->clk_gating.state == CLKS_ON) {
1642 spin_unlock_irqrestore(hba->host->host_lock, flags);
1643 goto unblock_reqs;
1644 }
1645
1646 spin_unlock_irqrestore(hba->host->host_lock, flags);
dd7143e2 1647 ufshcd_hba_vreg_set_hpm(hba);
1ab27c9c
ST		 1648 ufshcd_setup_clocks(hba, true);
1649
8b0bbf00
SC		 1650 ufshcd_enable_irq(hba);
1651
1ab27c9c
ST		 1652 /* Exit from hibern8 */
1653 if (ufshcd_can_hibern8_during_gating(hba)) {
1654 /* Prevent gating in this path */
1655 hba->clk_gating.is_suspended = true;
1656 if (ufshcd_is_link_hibern8(hba)) {
1657 ret = ufshcd_uic_hibern8_exit(hba);
1658 if (ret)
1659 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1660 __func__, ret);
1661 else
1662 ufshcd_set_link_active(hba);
1663 }
1664 hba->clk_gating.is_suspended = false;
1665 }
1666unblock_reqs:
38135535 1667 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1668}
1669
1670/**
1671 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1672 * Also, exit from hibern8 mode and set the link as active.
1673 * @hba: per adapter instance
1674 * @async: This indicates whether caller should ungate clocks asynchronously.
1675 */
1676int ufshcd_hold(struct ufs_hba *hba, bool async)
1677{
1678 int rc = 0;
93b6c5db 1679 bool flush_result;
1ab27c9c
ST		 1680 unsigned long flags;
1681
1682 if (!ufshcd_is_clkgating_allowed(hba))
1683 goto out;
1ab27c9c
ST		 1684 spin_lock_irqsave(hba->host->host_lock, flags);
1685 hba->clk_gating.active_reqs++;
1686
856b3483 1687start:
1ab27c9c
ST		 1688 switch (hba->clk_gating.state) {
1689 case CLKS_ON:
f2a785ac
VG		 1690 /*
1691 * Wait for the ungate work to complete if in progress.
1692 * Though the clocks may be in ON state, the link could
1693 * still be in hibner8 state if hibern8 is allowed
1694 * during clock gating.
1695 * Make sure we exit hibern8 state also in addition to
1696 * clocks being ON.
1697 */
1698 if (ufshcd_can_hibern8_during_gating(hba) &&
1699 ufshcd_is_link_hibern8(hba)) {
c63d6099
CG		 1700 if (async) {
1701 rc = -EAGAIN;
1702 hba->clk_gating.active_reqs--;
1703 break;
1704 }
f2a785ac 1705 spin_unlock_irqrestore(hba->host->host_lock, flags);
93b6c5db
SC		 1706 flush_result = flush_work(&hba->clk_gating.ungate_work);
1707 if (hba->clk_gating.is_suspended && !flush_result)
1708 goto out;
f2a785ac
VG		 1709 spin_lock_irqsave(hba->host->host_lock, flags);
1710 goto start;
1711 }
1ab27c9c
ST		 1712 break;
1713 case REQ_CLKS_OFF:
1714 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1715 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1716 trace_ufshcd_clk_gating(dev_name(hba->dev),
1717 hba->clk_gating.state);
1ab27c9c
ST		 1718 break;
1719 }
1720 /*
9c490d2d 1721 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1722 * currently running. Hence, fall through to cancel gating
1723 * work and to enable clocks.
1724 */
df561f66 1725 fallthrough;
1ab27c9c 1726 case CLKS_OFF:
1ab27c9c 1727 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47 1728 trace_ufshcd_clk_gating(dev_name(hba->dev),
1729 hba->clk_gating.state);
da3fecb0
CG		 1730 if (queue_work(hba->clk_gating.clk_gating_workq,
1731 &hba->clk_gating.ungate_work))
1732 ufshcd_scsi_block_requests(hba);
1ab27c9c
ST		 1733 /*
1734 * fall through to check if we should wait for this
1735 * work to be done or not.
1736 */
df561f66 1737 fallthrough;
1ab27c9c
ST		 1738 case REQ_CLKS_ON:
1739 if (async) {
1740 rc = -EAGAIN;
1741 hba->clk_gating.active_reqs--;
1742 break;
1743 }
1744
1745 spin_unlock_irqrestore(hba->host->host_lock, flags);
1746 flush_work(&hba->clk_gating.ungate_work);
1747 /* Make sure state is CLKS_ON before returning */
856b3483 1748 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1749 goto start;
1750 default:
1751 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1752 __func__, hba->clk_gating.state);
1753 break;
1754 }
1755 spin_unlock_irqrestore(hba->host->host_lock, flags);
1756out:
1757 return rc;
1758}
6e3fd44d 1759EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1760
1761static void ufshcd_gate_work(struct work_struct *work)
1762{
1763 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1764 clk_gating.gate_work.work);
1765 unsigned long flags;
4db7a236 1766 int ret;
1ab27c9c
ST		 1767
1768 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1769 /*
1770 * In case you are here to cancel this work the gating state
1771 * would be marked as REQ_CLKS_ON. In this case save time by
1772 * skipping the gating work and exit after changing the clock
1773 * state to CLKS_ON.
1774 */
1775 if (hba->clk_gating.is_suspended ||
18f01374 1776 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1777 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1778 trace_ufshcd_clk_gating(dev_name(hba->dev),
1779 hba->clk_gating.state);
1ab27c9c
ST		 1780 goto rel_lock;
1781 }
1782
1783 if (hba->clk_gating.active_reqs
1784 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1785 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1ab27c9c
ST		 1786 || hba->active_uic_cmd || hba->uic_async_done)
1787 goto rel_lock;
1788
1789 spin_unlock_irqrestore(hba->host->host_lock, flags);
1790
1791 /* put the link into hibern8 mode before turning off clocks */
1792 if (ufshcd_can_hibern8_during_gating(hba)) {
4db7a236
CG		 1793 ret = ufshcd_uic_hibern8_enter(hba);
1794 if (ret) {
1ab27c9c 1795 hba->clk_gating.state = CLKS_ON;
4db7a236
CG		 1796 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1797 __func__, ret);
7ff5ab47 1798 trace_ufshcd_clk_gating(dev_name(hba->dev),
1799 hba->clk_gating.state);
1ab27c9c
ST		 1800 goto out;
1801 }
1802 ufshcd_set_link_hibern8(hba);
1803 }
1804
8b0bbf00
SC		 1805 ufshcd_disable_irq(hba);
1806
81309c24 1807 ufshcd_setup_clocks(hba, false);
1ab27c9c 1808
dd7143e2
CG		 1809 /* Put the host controller in low power mode if possible */
1810 ufshcd_hba_vreg_set_lpm(hba);
1ab27c9c
ST		 1811 /*
1812 * In case you are here to cancel this work the gating state
1813 * would be marked as REQ_CLKS_ON. In this case keep the state
1814 * as REQ_CLKS_ON which would anyway imply that clocks are off
1815 * and a request to turn them on is pending. By doing this way,
1816 * we keep the state machine in tact and this would ultimately
1817 * prevent from doing cancel work multiple times when there are
1818 * new requests arriving before the current cancel work is done.
1819 */
1820 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1821 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1822 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 1823 trace_ufshcd_clk_gating(dev_name(hba->dev),
1824 hba->clk_gating.state);
1825 }
1ab27c9c
ST		 1826rel_lock:
1827 spin_unlock_irqrestore(hba->host->host_lock, flags);
1828out:
1829 return;
1830}
1831
1832/* host lock must be held before calling this variant */
1833static void __ufshcd_release(struct ufs_hba *hba)
1834{
1835 if (!ufshcd_is_clkgating_allowed(hba))
1836 return;
1837
1838 hba->clk_gating.active_reqs--;
1839
4db7a236
CG		 1840 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1841 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
8eb456be 1842 hba->outstanding_tasks ||
fd62de11
JK		 1843 hba->active_uic_cmd || hba->uic_async_done ||
1844 hba->clk_gating.state == CLKS_OFF)
1ab27c9c
ST		 1845 return;
1846
1847 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1848 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1849 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1850 &hba->clk_gating.gate_work,
1851 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1852}
1853
1854void ufshcd_release(struct ufs_hba *hba)
1855{
1856 unsigned long flags;
1857
1858 spin_lock_irqsave(hba->host->host_lock, flags);
1859 __ufshcd_release(hba);
1860 spin_unlock_irqrestore(hba->host->host_lock, flags);
1861}
6e3fd44d 1862EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1863
1864static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1865 struct device_attribute *attr, char *buf)
1866{
1867 struct ufs_hba *hba = dev_get_drvdata(dev);
1868
bafd09f8 1869 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1ab27c9c
ST		 1870}
1871
1872static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1873 struct device_attribute *attr, const char *buf, size_t count)
1874{
1875 struct ufs_hba *hba = dev_get_drvdata(dev);
1876 unsigned long flags, value;
1877
1878 if (kstrtoul(buf, 0, &value))
1879 return -EINVAL;
1880
1881 spin_lock_irqsave(hba->host->host_lock, flags);
1882 hba->clk_gating.delay_ms = value;
1883 spin_unlock_irqrestore(hba->host->host_lock, flags);
1884 return count;
1885}
1886
b427411a
ST		 1887static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1888 struct device_attribute *attr, char *buf)
1889{
1890 struct ufs_hba *hba = dev_get_drvdata(dev);
1891
bafd09f8 1892 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
b427411a
ST		 1893}
1894
1895static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1896 struct device_attribute *attr, const char *buf, size_t count)
1897{
1898 struct ufs_hba *hba = dev_get_drvdata(dev);
1899 unsigned long flags;
1900 u32 value;
1901
1902 if (kstrtou32(buf, 0, &value))
1903 return -EINVAL;
1904
1905 value = !!value;
b6645112
JK		 1906
1907 spin_lock_irqsave(hba->host->host_lock, flags);
b427411a
ST		 1908 if (value == hba->clk_gating.is_enabled)
1909 goto out;
1910
b6645112
JK		 1911 if (value)
1912 __ufshcd_release(hba);
1913 else
b427411a 1914 hba->clk_gating.active_reqs++;
b427411a
ST		 1915
1916 hba->clk_gating.is_enabled = value;
1917out:
b6645112 1918 spin_unlock_irqrestore(hba->host->host_lock, flags);
b427411a
ST		 1919 return count;
1920}
1921
4543d9d7 1922static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
eebcc196 1923{
4543d9d7
CG		 1924 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1925 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1926 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1927 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1928 hba->clk_gating.delay_attr.attr.mode = 0644;
1929 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1930 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
eebcc196 1931
4543d9d7
CG		 1932 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1933 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1934 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1935 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1936 hba->clk_gating.enable_attr.attr.mode = 0644;
1937 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1938 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
eebcc196
VG		 1939}
1940
4543d9d7 1941static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
eebcc196 1942{
4543d9d7
CG		 1943 if (hba->clk_gating.delay_attr.attr.name)
1944 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1945 if (hba->clk_gating.enable_attr.attr.name)
1946 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
eebcc196
VG		 1947}
1948
1ab27c9c
ST		 1949static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1950{
10e5e375
VV		 1951 char wq_name[sizeof("ufs_clk_gating_00")];
1952
1ab27c9c
ST		 1953 if (!ufshcd_is_clkgating_allowed(hba))
1954 return;
1955
2dec9475
CG		 1956 hba->clk_gating.state = CLKS_ON;
1957
1ab27c9c
ST		 1958 hba->clk_gating.delay_ms = 150;
1959 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1960 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1961
10e5e375
VV		 1962 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1963 hba->host->host_no);
1964 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
e93e6e49 1965 WQ_MEM_RECLAIM | WQ_HIGHPRI);
10e5e375 1966
4543d9d7 1967 ufshcd_init_clk_gating_sysfs(hba);
b427411a 1968
4543d9d7
CG		 1969 hba->clk_gating.is_enabled = true;
1970 hba->clk_gating.is_initialized = true;
1ab27c9c
ST		 1971}
1972
1973static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1974{
4543d9d7 1975 if (!hba->clk_gating.is_initialized)
1ab27c9c 1976 return;
4543d9d7 1977 ufshcd_remove_clk_gating_sysfs(hba);
97cd6805
AM		 1978 cancel_work_sync(&hba->clk_gating.ungate_work);
1979 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1980 destroy_workqueue(hba->clk_gating.clk_gating_workq);
4543d9d7 1981 hba->clk_gating.is_initialized = false;
1ab27c9c
ST		 1982}
1983
856b3483
ST		 1984/* Must be called with host lock acquired */
1985static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1986{
401f1e44 1987 bool queue_resume_work = false;
b1bf66d1 1988 ktime_t curr_t = ktime_get();
401f1e44 1989
fcb0c4b0 1990 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1991 return;
1992
401f1e44 1993 if (!hba->clk_scaling.active_reqs++)
1994 queue_resume_work = true;
1995
0e9d4ca4 1996 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress)
401f1e44 1997 return;
1998
1999 if (queue_resume_work)
2000 queue_work(hba->clk_scaling.workq,
2001 &hba->clk_scaling.resume_work);
2002
2003 if (!hba->clk_scaling.window_start_t) {
b1bf66d1 2004 hba->clk_scaling.window_start_t = curr_t;
401f1e44 2005 hba->clk_scaling.tot_busy_t = 0;
2006 hba->clk_scaling.is_busy_started = false;
2007 }
2008
856b3483 2009 if (!hba->clk_scaling.is_busy_started) {
b1bf66d1 2010 hba->clk_scaling.busy_start_t = curr_t;
856b3483
ST		 2011 hba->clk_scaling.is_busy_started = true;
2012 }
2013}
2014
2015static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2016{
2017 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2018
fcb0c4b0 2019 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 2020 return;
2021
2022 if (!hba->outstanding_reqs && scaling->is_busy_started) {
2023 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2024 scaling->busy_start_t));
8b0e1953 2025 scaling->busy_start_t = 0;
856b3483
ST		 2026 scaling->is_busy_started = false;
2027 }
2028}
7a3e97b0
SY		 2029/**
2030 * ufshcd_send_command - Send SCSI or device management commands
2031 * @hba: per adapter instance
2032 * @task_tag: Task tag of the command
2033 */
2034static inline
2035void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2036{
6edfdcfe
SC		 2037 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2038
2039 lrbp->issue_time_stamp = ktime_get();
2040 lrbp->compl_time_stamp = ktime_set(0, 0);
2041 ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false));
28fa68fc 2042 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
856b3483 2043 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 2044 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 2045 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 2046 /* Make sure that doorbell is committed immediately */
2047 wmb();
7a3e97b0
SY		 2048}
2049
2050/**
2051 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 2052 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2053 */
2054static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2055{
2056 int len;
1c2623c5
SJ		 2057 if (lrbp->sense_buffer &&
2058 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 2059 int len_to_copy;
2060
5a0b0cb9 2061 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 2062 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 2063
09a5a24f
AA		 2064 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2065 len_to_copy);
7a3e97b0
SY		 2066 }
2067}
2068
68078d5c
DR		 2069/**
2070 * ufshcd_copy_query_response() - Copy the Query Response and the data
2071 * descriptor
2072 * @hba: per adapter instance
8aa29f19 2073 * @lrbp: pointer to local reference block
68078d5c
DR		 2074 */
2075static
c6d4a831 2076int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 2077{
2078 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2079
68078d5c 2080 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 2081
68078d5c 2082 /* Get the descriptor */
1c90836f
AA		 2083 if (hba->dev_cmd.query.descriptor &&
2084 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 2085 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 2086 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 2087 u16 resp_len;
2088 u16 buf_len;
68078d5c
DR		 2089
2090 /* data segment length */
c6d4a831 2091 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 2092 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 2093 buf_len = be16_to_cpu(
2094 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 2095 if (likely(buf_len >= resp_len)) {
2096 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2097 } else {
2098 dev_warn(hba->dev,
3d4881d1
BH		 2099 "%s: rsp size %d is bigger than buffer size %d",
2100 __func__, resp_len, buf_len);
c6d4a831
DR		 2101 return -EINVAL;
2102 }
68078d5c 2103 }
c6d4a831
DR		 2104
2105 return 0;
68078d5c
DR		 2106}
2107
7a3e97b0
SY		 2108/**
2109 * ufshcd_hba_capabilities - Read controller capabilities
2110 * @hba: per adapter instance
df043c74
ST		 2111 *
2112 * Return: 0 on success, negative on error.
7a3e97b0 2113 */
df043c74 2114static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
7a3e97b0 2115{
df043c74
ST		 2116 int err;
2117
b873a275 2118 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 2119
2120 /* nutrs and nutmrs are 0 based values */
2121 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2122 hba->nutmrs =
2123 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
df043c74
ST		 2124
2125 /* Read crypto capabilities */
2126 err = ufshcd_hba_init_crypto_capabilities(hba);
2127 if (err)
2128 dev_err(hba->dev, "crypto setup failed\n");
2129
2130 return err;
7a3e97b0
SY		 2131}
2132
2133/**
6ccf44fe
SJ		 2134 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2135 * to accept UIC commands
7a3e97b0 2136 * @hba: per adapter instance
6ccf44fe
SJ		 2137 * Return true on success, else false
2138 */
2139static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2140{
2141 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2142 return true;
2143 else
2144 return false;
2145}
2146
53b3d9c3
SJ		 2147/**
2148 * ufshcd_get_upmcrs - Get the power mode change request status
2149 * @hba: Pointer to adapter instance
2150 *
2151 * This function gets the UPMCRS field of HCS register
2152 * Returns value of UPMCRS field
2153 */
2154static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2155{
2156 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2157}
2158
6ccf44fe
SJ		 2159/**
2160 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2161 * @hba: per adapter instance
2162 * @uic_cmd: UIC command
2163 *
2164 * Mutex must be held.
7a3e97b0
SY		 2165 */
2166static inline void
6ccf44fe 2167ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 2168{
6ccf44fe
SJ		 2169 WARN_ON(hba->active_uic_cmd);
2170
2171 hba->active_uic_cmd = uic_cmd;
2172
7a3e97b0 2173 /* Write Args */
6ccf44fe
SJ		 2174 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2175 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2176 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0 2177
28fa68fc 2178 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
aa5c6979 2179
7a3e97b0 2180 /* Write UIC Cmd */
6ccf44fe 2181 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 2182 REG_UIC_COMMAND);
7a3e97b0
SY		 2183}
2184
6ccf44fe
SJ		 2185/**
2186 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2187 * @hba: per adapter instance
8aa29f19 2188 * @uic_cmd: UIC command
6ccf44fe
SJ		 2189 *
2190 * Must be called with mutex held.
2191 * Returns 0 only if success.
2192 */
2193static int
2194ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2195{
2196 int ret;
2197 unsigned long flags;
2198
2199 if (wait_for_completion_timeout(&uic_cmd->done,
0f52fcb9 2200 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
6ccf44fe 2201 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
0f52fcb9 2202 } else {
6ccf44fe 2203 ret = -ETIMEDOUT;
0f52fcb9
CG		 2204 dev_err(hba->dev,
2205 "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2206 uic_cmd->command, uic_cmd->argument3);
2207
2208 if (!uic_cmd->cmd_active) {
2209 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2210 __func__);
2211 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2212 }
2213 }
6ccf44fe
SJ		 2214
2215 spin_lock_irqsave(hba->host->host_lock, flags);
2216 hba->active_uic_cmd = NULL;
2217 spin_unlock_irqrestore(hba->host->host_lock, flags);
2218
2219 return ret;
2220}
2221
2222/**
2223 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2224 * @hba: per adapter instance
2225 * @uic_cmd: UIC command
d75f7fe4 2226 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2227 *
2228 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2229 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2230 * Returns 0 only if success.
2231 */
2232static int
d75f7fe4
YG		 2233__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2234 bool completion)
6ccf44fe 2235{
6ccf44fe
SJ		 2236 if (!ufshcd_ready_for_uic_cmd(hba)) {
2237 dev_err(hba->dev,
2238 "Controller not ready to accept UIC commands\n");
2239 return -EIO;
2240 }
2241
d75f7fe4
YG		 2242 if (completion)
2243 init_completion(&uic_cmd->done);
6ccf44fe 2244
0f52fcb9 2245 uic_cmd->cmd_active = 1;
6ccf44fe 2246 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2247
57d104c1 2248 return 0;
6ccf44fe
SJ		 2249}
2250
2251/**
2252 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2253 * @hba: per adapter instance
2254 * @uic_cmd: UIC command
2255 *
2256 * Returns 0 only if success.
2257 */
e77044c5 2258int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2259{
2260 int ret;
57d104c1 2261 unsigned long flags;
6ccf44fe 2262
1ab27c9c 2263 ufshcd_hold(hba, false);
6ccf44fe 2264 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2265 ufshcd_add_delay_before_dme_cmd(hba);
2266
57d104c1 2267 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2268 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2269 spin_unlock_irqrestore(hba->host->host_lock, flags);
2270 if (!ret)
2271 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2272
6ccf44fe
SJ		 2273 mutex_unlock(&hba->uic_cmd_mutex);
2274
1ab27c9c 2275 ufshcd_release(hba);
6ccf44fe
SJ		 2276 return ret;
2277}
2278
7a3e97b0
SY		 2279/**
2280 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2281 * @hba: per adapter instance
2282 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2283 *
2284 * Returns 0 in case of success, non-zero value in case of failure
2285 */
75b1cc4a 2286static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2287{
2288 struct ufshcd_sg_entry *prd_table;
2289 struct scatterlist *sg;
2290 struct scsi_cmnd *cmd;
2291 int sg_segments;
2292 int i;
2293
2294 cmd = lrbp->cmd;
2295 sg_segments = scsi_dma_map(cmd);
2296 if (sg_segments < 0)
2297 return sg_segments;
2298
2299 if (sg_segments) {
26f968d7
AA		 2300
2301 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2302 lrbp->utr_descriptor_ptr->prd_table_length =
2303 cpu_to_le16((sg_segments *
2304 sizeof(struct ufshcd_sg_entry)));
2305 else
2306 lrbp->utr_descriptor_ptr->prd_table_length =
2307 cpu_to_le16((u16) (sg_segments));
7a3e97b0
SY		 2308
2309 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2310
2311 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2312 prd_table[i].size =
2313 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2314 prd_table[i].base_addr =
2315 cpu_to_le32(lower_32_bits(sg->dma_address));
2316 prd_table[i].upper_addr =
2317 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2318 prd_table[i].reserved = 0;
7a3e97b0
SY		 2319 }
2320 } else {
2321 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2322 }
2323
2324 return 0;
2325}
2326
2327/**
2fbd009b 2328 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2329 * @hba: per adapter instance
2fbd009b 2330 * @intrs: interrupt bits
7a3e97b0 2331 */
2fbd009b 2332static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2333{
2fbd009b
SJ		 2334 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2335
2336 if (hba->ufs_version == UFSHCI_VERSION_10) {
2337 u32 rw;
2338 rw = set & INTERRUPT_MASK_RW_VER_10;
2339 set = rw | ((set ^ intrs) & intrs);
2340 } else {
2341 set |= intrs;
2342 }
2343
2344 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2345}
2346
2347/**
2348 * ufshcd_disable_intr - disable interrupts
2349 * @hba: per adapter instance
2350 * @intrs: interrupt bits
2351 */
2352static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2353{
2354 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2355
2356 if (hba->ufs_version == UFSHCI_VERSION_10) {
2357 u32 rw;
2358 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2359 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2360 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2361
2362 } else {
2363 set &= ~intrs;
7a3e97b0 2364 }
2fbd009b
SJ		 2365
2366 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2367}
2368
5a0b0cb9
SRT		 2369/**
2370 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2371 * descriptor according to request
2372 * @lrbp: pointer to local reference block
2373 * @upiu_flags: flags required in the header
2374 * @cmd_dir: requests data direction
2375 */
2376static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
a23064c4 2377 u8 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2378{
2379 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2380 u32 data_direction;
2381 u32 dword_0;
df043c74
ST		 2382 u32 dword_1 = 0;
2383 u32 dword_3 = 0;
5a0b0cb9
SRT		 2384
2385 if (cmd_dir == DMA_FROM_DEVICE) {
2386 data_direction = UTP_DEVICE_TO_HOST;
2387 *upiu_flags = UPIU_CMD_FLAGS_READ;
2388 } else if (cmd_dir == DMA_TO_DEVICE) {
2389 data_direction = UTP_HOST_TO_DEVICE;
2390 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2391 } else {
2392 data_direction = UTP_NO_DATA_TRANSFER;
2393 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2394 }
2395
2396 dword_0 = data_direction | (lrbp->command_type
2397 << UPIU_COMMAND_TYPE_OFFSET);
2398 if (lrbp->intr_cmd)
2399 dword_0 |= UTP_REQ_DESC_INT_CMD;
2400
df043c74
ST		 2401 /* Prepare crypto related dwords */
2402 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2403
5a0b0cb9
SRT		 2404 /* Transfer request descriptor header fields */
2405 req_desc->header.dword_0 = cpu_to_le32(dword_0);
df043c74 2406 req_desc->header.dword_1 = cpu_to_le32(dword_1);
5a0b0cb9
SRT		 2407 /*
2408 * assigning invalid value for command status. Controller
2409 * updates OCS on command completion, with the command
2410 * status
2411 */
2412 req_desc->header.dword_2 =
2413 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
df043c74 2414 req_desc->header.dword_3 = cpu_to_le32(dword_3);
51047266
YG		 2415
2416 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2417}
2418
2419/**
2420 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2421 * for scsi commands
8aa29f19
BVA		 2422 * @lrbp: local reference block pointer
2423 * @upiu_flags: flags
5a0b0cb9
SRT		 2424 */
2425static
a23064c4 2426void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
5a0b0cb9 2427{
1b21b8f0 2428 struct scsi_cmnd *cmd = lrbp->cmd;
5a0b0cb9 2429 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2430 unsigned short cdb_len;
5a0b0cb9
SRT		 2431
2432 /* command descriptor fields */
2433 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2434 UPIU_TRANSACTION_COMMAND, upiu_flags,
2435 lrbp->lun, lrbp->task_tag);
2436 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2437 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2438
2439 /* Total EHS length and Data segment length will be zero */
2440 ucd_req_ptr->header.dword_2 = 0;
2441
1b21b8f0 2442 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
5a0b0cb9 2443
1b21b8f0 2444 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
a851b2bd 2445 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1b21b8f0 2446 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
52ac95fe
YG		 2447
2448 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2449}
2450
68078d5c
DR		 2451/**
2452 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2453 * for query requsts
2454 * @hba: UFS hba
2455 * @lrbp: local reference block pointer
2456 * @upiu_flags: flags
2457 */
2458static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
a23064c4 2459 struct ufshcd_lrb *lrbp, u8 upiu_flags)
68078d5c
DR		 2460{
2461 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2462 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2463 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2464
2465 /* Query request header */
2466 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2467 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2468 lrbp->lun, lrbp->task_tag);
2469 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2470 0, query->request.query_func, 0, 0);
2471
6861285c
ZL		 2472 /* Data segment length only need for WRITE_DESC */
2473 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2474 ucd_req_ptr->header.dword_2 =
2475 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2476 else
2477 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2478
2479 /* Copy the Query Request buffer as is */
2480 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2481 QUERY_OSF_SIZE);
68078d5c
DR		 2482
2483 /* Copy the Descriptor */
c6d4a831 2484 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2485 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2486
51047266 2487 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2488}
2489
5a0b0cb9
SRT		 2490static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2491{
2492 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2493
2494 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2495
2496 /* command descriptor fields */
2497 ucd_req_ptr->header.dword_0 =
2498 UPIU_HEADER_DWORD(
2499 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2500 /* clear rest of the fields of basic header */
2501 ucd_req_ptr->header.dword_1 = 0;
2502 ucd_req_ptr->header.dword_2 = 0;
2503
2504 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2505}
2506
7a3e97b0 2507/**
f273c54b 2508 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
300bb13f 2509 * for Device Management Purposes
8aa29f19
BVA		 2510 * @hba: per adapter instance
2511 * @lrbp: pointer to local reference block
7a3e97b0 2512 */
f273c54b
BH		 2513static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2514 struct ufshcd_lrb *lrbp)
7a3e97b0 2515{
a23064c4 2516 u8 upiu_flags;
5a0b0cb9 2517 int ret = 0;
7a3e97b0 2518
83dc7e3d 2519 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2520 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2521 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2522 else
2523 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2524
2525 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2526 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2527 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2528 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2529 ufshcd_prepare_utp_nop_upiu(lrbp);
2530 else
2531 ret = -EINVAL;
2532
2533 return ret;
2534}
2535
2536/**
2537 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2538 * for SCSI Purposes
8aa29f19
BVA		 2539 * @hba: per adapter instance
2540 * @lrbp: pointer to local reference block
300bb13f
JP		 2541 */
2542static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2543{
a23064c4 2544 u8 upiu_flags;
300bb13f
JP		 2545 int ret = 0;
2546
83dc7e3d 2547 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2548 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2549 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2550 else
2551 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2552
2553 if (likely(lrbp->cmd)) {
2554 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2555 lrbp->cmd->sc_data_direction);
2556 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2557 } else {
2558 ret = -EINVAL;
2559 }
5a0b0cb9
SRT		 2560
2561 return ret;
7a3e97b0
SY		 2562}
2563
2a8fa600
SJ		 2564/**
2565 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2566 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2567 *
2568 * Returns SCSI W-LUN id
2569 */
2570static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2571{
2572 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2573}
2574
4d2b8d40
BVA		 2575static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2576{
2577 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2578 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2579 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2580 i * sizeof(struct utp_transfer_cmd_desc);
2581 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2582 response_upiu);
2583 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2584
2585 lrb->utr_descriptor_ptr = utrdlp + i;
2586 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2587 i * sizeof(struct utp_transfer_req_desc);
2588 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2589 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2590 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2591 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2592 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2593 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2594}
2595
7a3e97b0
SY		 2596/**
2597 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2598 * @host: SCSI host pointer
7a3e97b0 2599 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2600 *
2601 * Returns 0 for success, non-zero in case of failure
2602 */
2603static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2604{
2605 struct ufshcd_lrb *lrbp;
2606 struct ufs_hba *hba;
2607 unsigned long flags;
2608 int tag;
2609 int err = 0;
2610
2611 hba = shost_priv(host);
2612
2613 tag = cmd->request->tag;
14497328
YG		 2614 if (!ufshcd_valid_tag(hba, tag)) {
2615 dev_err(hba->dev,
2616 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2617 __func__, tag, cmd, cmd->request);
2618 BUG();
2619 }
7a3e97b0 2620
a3cd5ec5 2621 if (!down_read_trylock(&hba->clk_scaling_lock))
2622 return SCSI_MLQUEUE_HOST_BUSY;
2623
7fabb77b
GB		 2624 hba->req_abort_count = 0;
2625
1ab27c9c
ST		 2626 err = ufshcd_hold(hba, true);
2627 if (err) {
2628 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2629 goto out;
2630 }
2dec9475
CG		 2631 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2632 (hba->clk_gating.state != CLKS_ON));
1ab27c9c 2633
7a3e97b0 2634 lrbp = &hba->lrb[tag];
7a7e66c6
CG		 2635 if (unlikely(lrbp->in_use)) {
2636 if (hba->pm_op_in_progress)
2637 set_host_byte(cmd, DID_BAD_TARGET);
2638 else
2639 err = SCSI_MLQUEUE_HOST_BUSY;
2640 ufshcd_release(hba);
2641 goto out;
2642 }
7a3e97b0 2643
5a0b0cb9 2644 WARN_ON(lrbp->cmd);
7a3e97b0 2645 lrbp->cmd = cmd;
09a5a24f 2646 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2647 lrbp->sense_buffer = cmd->sense_buffer;
2648 lrbp->task_tag = tag;
0ce147d4 2649 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2650 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
df043c74
ST		 2651
2652 ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
2653
e0b299e3 2654 lrbp->req_abort_skip = false;
7a3e97b0 2655
300bb13f
JP		 2656 ufshcd_comp_scsi_upiu(hba, lrbp);
2657
75b1cc4a 2658 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2659 if (err) {
2660 lrbp->cmd = NULL;
17c7d35f 2661 ufshcd_release(hba);
7a3e97b0 2662 goto out;
5a0b0cb9 2663 }
ad1a1b9c
GB		 2664 /* Make sure descriptors are ready before ringing the doorbell */
2665 wmb();
7a3e97b0 2666
7a3e97b0 2667 spin_lock_irqsave(hba->host->host_lock, flags);
5586dd8e
CG		 2668 switch (hba->ufshcd_state) {
2669 case UFSHCD_STATE_OPERATIONAL:
2670 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2671 break;
2672 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2673 /*
2674 * pm_runtime_get_sync() is used at error handling preparation
2675 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2676 * PM ops, it can never be finished if we let SCSI layer keep
2677 * retrying it, which gets err handler stuck forever. Neither
2678 * can we let the scsi cmd pass through, because UFS is in bad
2679 * state, the scsi cmd may eventually time out, which will get
2680 * err handler blocked for too long. So, just fail the scsi cmd
2681 * sent from PM ops, err handler can recover PM error anyways.
2682 */
2683 if (hba->pm_op_in_progress) {
2684 hba->force_reset = true;
2685 set_host_byte(cmd, DID_BAD_TARGET);
2686 goto out_compl_cmd;
2687 }
2688 fallthrough;
2689 case UFSHCD_STATE_RESET:
2690 err = SCSI_MLQUEUE_HOST_BUSY;
2691 goto out_compl_cmd;
2692 case UFSHCD_STATE_ERROR:
2693 set_host_byte(cmd, DID_ERROR);
2694 goto out_compl_cmd;
2695 default:
2696 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2697 __func__, hba->ufshcd_state);
2698 set_host_byte(cmd, DID_BAD_TARGET);
2699 goto out_compl_cmd;
2700 }
7a3e97b0
SY		 2701 ufshcd_send_command(hba, tag);
2702 spin_unlock_irqrestore(hba->host->host_lock, flags);
5586dd8e
CG		 2703 goto out;
2704
2705out_compl_cmd:
2706 scsi_dma_unmap(lrbp->cmd);
2707 lrbp->cmd = NULL;
2708 spin_unlock_irqrestore(hba->host->host_lock, flags);
2709 ufshcd_release(hba);
2710 if (!err)
2711 cmd->scsi_done(cmd);
7a3e97b0 2712out:
a3cd5ec5 2713 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2714 return err;
2715}
2716
5a0b0cb9
SRT		 2717static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2718 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2719{
2720 lrbp->cmd = NULL;
2721 lrbp->sense_bufflen = 0;
2722 lrbp->sense_buffer = NULL;
2723 lrbp->task_tag = tag;
2724 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9 2725 lrbp->intr_cmd = true; /* No interrupt aggregation */
df043c74 2726 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5a0b0cb9
SRT		 2727 hba->dev_cmd.type = cmd_type;
2728
f273c54b 2729 return ufshcd_compose_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2730}
2731
2732static int
2733ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2734{
2735 int err = 0;
2736 unsigned long flags;
2737 u32 mask = 1 << tag;
2738
2739 /* clear outstanding transaction before retry */
2740 spin_lock_irqsave(hba->host->host_lock, flags);
2741 ufshcd_utrl_clear(hba, tag);
2742 spin_unlock_irqrestore(hba->host->host_lock, flags);
2743
2744 /*
2745 * wait for for h/w to clear corresponding bit in door-bell.
2746 * max. wait is 1 sec.
2747 */
2748 err = ufshcd_wait_for_register(hba,
2749 REG_UTP_TRANSFER_REQ_DOOR_BELL,
5cac1095 2750 mask, ~mask, 1000, 1000);
5a0b0cb9
SRT		 2751
2752 return err;
2753}
2754
c6d4a831
DR		 2755static int
2756ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2757{
2758 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2759
2760 /* Get the UPIU response */
2761 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2762 UPIU_RSP_CODE_OFFSET;
2763 return query_res->response;
2764}
2765
5a0b0cb9
SRT		 2766/**
2767 * ufshcd_dev_cmd_completion() - handles device management command responses
2768 * @hba: per adapter instance
2769 * @lrbp: pointer to local reference block
2770 */
2771static int
2772ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2773{
2774 int resp;
2775 int err = 0;
2776
ff8e20c6 2777 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2778 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2779
2780 switch (resp) {
2781 case UPIU_TRANSACTION_NOP_IN:
2782 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2783 err = -EINVAL;
2784 dev_err(hba->dev, "%s: unexpected response %x\n",
2785 __func__, resp);
2786 }
2787 break;
68078d5c 2788 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2789 err = ufshcd_check_query_response(hba, lrbp);
2790 if (!err)
2791 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2792 break;
5a0b0cb9
SRT		 2793 case UPIU_TRANSACTION_REJECT_UPIU:
2794 /* TODO: handle Reject UPIU Response */
2795 err = -EPERM;
2796 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2797 __func__);
2798 break;
2799 default:
2800 err = -EINVAL;
2801 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2802 __func__, resp);
2803 break;
2804 }
2805
2806 return err;
2807}
2808
2809static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2810 struct ufshcd_lrb *lrbp, int max_timeout)
2811{
2812 int err = 0;
2813 unsigned long time_left;
2814 unsigned long flags;
2815
2816 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2817 msecs_to_jiffies(max_timeout));
2818
ad1a1b9c
GB		 2819 /* Make sure descriptors are ready before ringing the doorbell */
2820 wmb();
5a0b0cb9
SRT		 2821 spin_lock_irqsave(hba->host->host_lock, flags);
2822 hba->dev_cmd.complete = NULL;
2823 if (likely(time_left)) {
2824 err = ufshcd_get_tr_ocs(lrbp);
2825 if (!err)
2826 err = ufshcd_dev_cmd_completion(hba, lrbp);
2827 }
2828 spin_unlock_irqrestore(hba->host->host_lock, flags);
2829
2830 if (!time_left) {
2831 err = -ETIMEDOUT;
a48353f6
YG		 2832 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2833 __func__, lrbp->task_tag);
5a0b0cb9 2834 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2835 /* successfully cleared the command, retry if needed */
5a0b0cb9 2836 err = -EAGAIN;
a48353f6
YG		 2837 /*
2838 * in case of an error, after clearing the doorbell,
2839 * we also need to clear the outstanding_request
2840 * field in hba
2841 */
2842 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2843 }
2844
2845 return err;
2846}
2847
5a0b0cb9
SRT		 2848/**
2849 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2850 * @hba: UFS hba
2851 * @cmd_type: specifies the type (NOP, Query...)
2852 * @timeout: time in seconds
5a0b0cb9 2853 *
68078d5c
DR		 2854 * NOTE: Since there is only one available tag for device management commands,
2855 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2856 */
2857static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2858 enum dev_cmd_type cmd_type, int timeout)
2859{
7252a360
BVA		 2860 struct request_queue *q = hba->cmd_queue;
2861 struct request *req;
5a0b0cb9
SRT		 2862 struct ufshcd_lrb *lrbp;
2863 int err;
2864 int tag;
2865 struct completion wait;
2866 unsigned long flags;
2867
a3cd5ec5 2868 down_read(&hba->clk_scaling_lock);
2869
5a0b0cb9
SRT		 2870 /*
2871 * Get free slot, sleep if slots are unavailable.
2872 * Even though we use wait_event() which sleeps indefinitely,
2873 * the maximum wait time is bounded by SCSI request timeout.
2874 */
7252a360 2875 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 2876 if (IS_ERR(req)) {
2877 err = PTR_ERR(req);
2878 goto out_unlock;
2879 }
7252a360
BVA		 2880 tag = req->tag;
2881 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5a0b0cb9
SRT		 2882
2883 init_completion(&wait);
2884 lrbp = &hba->lrb[tag];
7a7e66c6
CG		 2885 if (unlikely(lrbp->in_use)) {
2886 err = -EBUSY;
2887 goto out;
2888 }
2889
5a0b0cb9
SRT		 2890 WARN_ON(lrbp->cmd);
2891 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2892 if (unlikely(err))
2893 goto out_put_tag;
2894
2895 hba->dev_cmd.complete = &wait;
2896
fb475b74 2897 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
e3dfdc53
YG		 2898 /* Make sure descriptors are ready before ringing the doorbell */
2899 wmb();
5a0b0cb9
SRT		 2900 spin_lock_irqsave(hba->host->host_lock, flags);
2901 ufshcd_send_command(hba, tag);
2902 spin_unlock_irqrestore(hba->host->host_lock, flags);
2903
2904 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2905
7a7e66c6 2906out:
fb475b74
AA		 2907 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
2908 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
6667e6d9 2909
5a0b0cb9 2910out_put_tag:
7252a360 2911 blk_put_request(req);
bb14dd15 2912out_unlock:
a3cd5ec5 2913 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2914 return err;
2915}
2916
d44a5f98
DR		 2917/**
2918 * ufshcd_init_query() - init the query response and request parameters
2919 * @hba: per-adapter instance
2920 * @request: address of the request pointer to be initialized
2921 * @response: address of the response pointer to be initialized
2922 * @opcode: operation to perform
2923 * @idn: flag idn to access
2924 * @index: LU number to access
2925 * @selector: query/flag/descriptor further identification
2926 */
2927static inline void ufshcd_init_query(struct ufs_hba *hba,
2928 struct ufs_query_req **request, struct ufs_query_res **response,
2929 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2930{
2931 *request = &hba->dev_cmd.query.request;
2932 *response = &hba->dev_cmd.query.response;
2933 memset(*request, 0, sizeof(struct ufs_query_req));
2934 memset(*response, 0, sizeof(struct ufs_query_res));
2935 (*request)->upiu_req.opcode = opcode;
2936 (*request)->upiu_req.idn = idn;
2937 (*request)->upiu_req.index = index;
2938 (*request)->upiu_req.selector = selector;
2939}
2940
dc3c8d3a 2941static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1f34eedf 2942 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
dc3c8d3a
YG		 2943{
2944 int ret;
2945 int retries;
2946
2947 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1f34eedf 2948 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
dc3c8d3a
YG		 2949 if (ret)
2950 dev_dbg(hba->dev,
2951 "%s: failed with error %d, retries %d\n",
2952 __func__, ret, retries);
2953 else
2954 break;
2955 }
2956
2957 if (ret)
2958 dev_err(hba->dev,
2959 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2960 __func__, opcode, idn, ret, retries);
2961 return ret;
2962}
2963
68078d5c
DR		 2964/**
2965 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2966 * @hba: per-adapter instance
2967 * @opcode: flag query to perform
2968 * @idn: flag idn to access
1f34eedf 2969 * @index: flag index to access
8aa29f19 2970 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2971 *
2972 * Returns 0 for success, non-zero in case of failure
2973 */
dc3c8d3a 2974int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1f34eedf 2975 enum flag_idn idn, u8 index, bool *flag_res)
68078d5c 2976{
d44a5f98
DR		 2977 struct ufs_query_req *request = NULL;
2978 struct ufs_query_res *response = NULL;
1f34eedf 2979 int err, selector = 0;
e5ad406c 2980 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2981
2982 BUG_ON(!hba);
2983
1ab27c9c 2984 ufshcd_hold(hba, false);
68078d5c 2985 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2986 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2987 selector);
68078d5c
DR		 2988
2989 switch (opcode) {
2990 case UPIU_QUERY_OPCODE_SET_FLAG:
2991 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2992 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2993 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2994 break;
2995 case UPIU_QUERY_OPCODE_READ_FLAG:
2996 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2997 if (!flag_res) {
2998 /* No dummy reads */
2999 dev_err(hba->dev, "%s: Invalid argument for read request\n",
3000 __func__);
3001 err = -EINVAL;
3002 goto out_unlock;
3003 }
3004 break;
3005 default:
3006 dev_err(hba->dev,
3007 "%s: Expected query flag opcode but got = %d\n",
3008 __func__, opcode);
3009 err = -EINVAL;
3010 goto out_unlock;
3011 }
68078d5c 3012
e5ad406c 3013 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 3014
3015 if (err) {
3016 dev_err(hba->dev,
3017 "%s: Sending flag query for idn %d failed, err = %d\n",
3018 __func__, idn, err);
3019 goto out_unlock;
3020 }
3021
3022 if (flag_res)
e8c8e82a 3023 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 3024 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3025
3026out_unlock:
3027 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 3028 ufshcd_release(hba);
68078d5c
DR		 3029 return err;
3030}
3031
66ec6d59
SRT		 3032/**
3033 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 3034 * @hba: per-adapter instance
3035 * @opcode: attribute opcode
3036 * @idn: attribute idn to access
3037 * @index: index field
3038 * @selector: selector field
3039 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 3040 *
3041 * Returns 0 for success, non-zero in case of failure
3042*/
ec92b59c
SN		 3043int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3044 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 3045{
d44a5f98
DR		 3046 struct ufs_query_req *request = NULL;
3047 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 3048 int err;
3049
3050 BUG_ON(!hba);
3051
3052 if (!attr_val) {
3053 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3054 __func__, opcode);
8ca1a40b 3055 return -EINVAL;
66ec6d59
SRT		 3056 }
3057
8ca1a40b 3058 ufshcd_hold(hba, false);
3059
66ec6d59 3060 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 3061 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3062 selector);
66ec6d59
SRT		 3063
3064 switch (opcode) {
3065 case UPIU_QUERY_OPCODE_WRITE_ATTR:
3066 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 3067 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 3068 break;
3069 case UPIU_QUERY_OPCODE_READ_ATTR:
3070 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3071 break;
3072 default:
3073 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3074 __func__, opcode);
3075 err = -EINVAL;
3076 goto out_unlock;
3077 }
3078
d44a5f98 3079 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 3080
3081 if (err) {
4b761b58
YG		 3082 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3083 __func__, opcode, idn, index, err);
66ec6d59
SRT		 3084 goto out_unlock;
3085 }
3086
e8c8e82a 3087 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 3088
3089out_unlock:
3090 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 3091 ufshcd_release(hba);
66ec6d59
SRT		 3092 return err;
3093}
3094
5e86ae44
YG		 3095/**
3096 * ufshcd_query_attr_retry() - API function for sending query
3097 * attribute with retries
3098 * @hba: per-adapter instance
3099 * @opcode: attribute opcode
3100 * @idn: attribute idn to access
3101 * @index: index field
3102 * @selector: selector field
3103 * @attr_val: the attribute value after the query request
3104 * completes
3105 *
3106 * Returns 0 for success, non-zero in case of failure
3107*/
3108static int ufshcd_query_attr_retry(struct ufs_hba *hba,
3109 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3110 u32 *attr_val)
3111{
3112 int ret = 0;
3113 u32 retries;
3114
68c9fcfd 3115 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
5e86ae44
YG		 3116 ret = ufshcd_query_attr(hba, opcode, idn, index,
3117 selector, attr_val);
3118 if (ret)
3119 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3120 __func__, ret, retries);
3121 else
3122 break;
3123 }
3124
3125 if (ret)
3126 dev_err(hba->dev,
3127 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3128 __func__, idn, ret, QUERY_REQ_RETRIES);
3129 return ret;
3130}
3131
a70e91b8 3132static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 3133 enum query_opcode opcode, enum desc_idn idn, u8 index,
3134 u8 selector, u8 *desc_buf, int *buf_len)
3135{
3136 struct ufs_query_req *request = NULL;
3137 struct ufs_query_res *response = NULL;
3138 int err;
3139
3140 BUG_ON(!hba);
3141
3142 if (!desc_buf) {
3143 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3144 __func__, opcode);
8ca1a40b 3145 return -EINVAL;
d44a5f98
DR		 3146 }
3147
a4b0e8a4 3148 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 3149 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3150 __func__, *buf_len);
8ca1a40b 3151 return -EINVAL;
d44a5f98
DR		 3152 }
3153
8ca1a40b 3154 ufshcd_hold(hba, false);
3155
d44a5f98
DR		 3156 mutex_lock(&hba->dev_cmd.lock);
3157 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3158 selector);
3159 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 3160 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 3161
3162 switch (opcode) {
3163 case UPIU_QUERY_OPCODE_WRITE_DESC:
3164 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3165 break;
3166 case UPIU_QUERY_OPCODE_READ_DESC:
3167 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3168 break;
3169 default:
3170 dev_err(hba->dev,
3171 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3172 __func__, opcode);
3173 err = -EINVAL;
3174 goto out_unlock;
3175 }
3176
3177 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3178
3179 if (err) {
4b761b58
YG		 3180 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3181 __func__, opcode, idn, index, err);
d44a5f98
DR		 3182 goto out_unlock;
3183 }
3184
ea2aab24 3185 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 3186
3187out_unlock:
cfcbae38 3188 hba->dev_cmd.query.descriptor = NULL;
d44a5f98 3189 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 3190 ufshcd_release(hba);
d44a5f98
DR		 3191 return err;
3192}
3193
a70e91b8 3194/**
8aa29f19
BVA		 3195 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3196 * @hba: per-adapter instance
3197 * @opcode: attribute opcode
3198 * @idn: attribute idn to access
3199 * @index: index field
3200 * @selector: selector field
3201 * @desc_buf: the buffer that contains the descriptor
3202 * @buf_len: length parameter passed to the device
a70e91b8
YG		 3203 *
3204 * Returns 0 for success, non-zero in case of failure.
3205 * The buf_len parameter will contain, on return, the length parameter
3206 * received on the response.
3207 */
2238d31c
SN		 3208int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3209 enum query_opcode opcode,
3210 enum desc_idn idn, u8 index,
3211 u8 selector,
3212 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 3213{
3214 int err;
3215 int retries;
3216
3217 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3218 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3219 selector, desc_buf, buf_len);
3220 if (!err || err == -EINVAL)
3221 break;
3222 }
3223
3224 return err;
3225}
a70e91b8 3226
a4b0e8a4
PM		 3227/**
3228 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3229 * @hba: Pointer to adapter instance
3230 * @desc_id: descriptor idn value
3231 * @desc_len: mapped desc length (out)
a4b0e8a4 3232 */
7a0bf85b
BH		 3233void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3234 int *desc_len)
a4b0e8a4 3235{
7a0bf85b
BH		 3236 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3237 desc_id == QUERY_DESC_IDN_RFU_1)
a4b0e8a4 3238 *desc_len = 0;
7a0bf85b
BH		 3239 else
3240 *desc_len = hba->desc_size[desc_id];
a4b0e8a4
PM		 3241}
3242EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3243
7a0bf85b 3244static void ufshcd_update_desc_length(struct ufs_hba *hba,
72fb690e 3245 enum desc_idn desc_id, int desc_index,
7a0bf85b
BH		 3246 unsigned char desc_len)
3247{
3248 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
72fb690e
BH		 3249 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3250 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3251 * than the RPMB unit, however, both descriptors share the same
3252 * desc_idn, to cover both unit descriptors with one length, we
3253 * choose the normal unit descriptor length by desc_index.
3254 */
7a0bf85b
BH		 3255 hba->desc_size[desc_id] = desc_len;
3256}
3257
da461cec
SJ		 3258/**
3259 * ufshcd_read_desc_param - read the specified descriptor parameter
3260 * @hba: Pointer to adapter instance
3261 * @desc_id: descriptor idn value
3262 * @desc_index: descriptor index
3263 * @param_offset: offset of the parameter to read
3264 * @param_read_buf: pointer to buffer where parameter would be read
3265 * @param_size: sizeof(param_read_buf)
3266 *
3267 * Return 0 in case of success, non-zero otherwise
3268 */
45bced87
SN		 3269int ufshcd_read_desc_param(struct ufs_hba *hba,
3270 enum desc_idn desc_id,
3271 int desc_index,
3272 u8 param_offset,
3273 u8 *param_read_buf,
3274 u8 param_size)
da461cec
SJ		 3275{
3276 int ret;
3277 u8 *desc_buf;
a4b0e8a4 3278 int buff_len;
da461cec
SJ		 3279 bool is_kmalloc = true;
3280
a4b0e8a4
PM		 3281 /* Safety check */
3282 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3283 return -EINVAL;
3284
7a0bf85b
BH		 3285 /* Get the length of descriptor */
3286 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3287 if (!buff_len) {
1699f980
CG		 3288 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3289 return -EINVAL;
3290 }
3291
3292 if (param_offset >= buff_len) {
3293 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3294 __func__, param_offset, desc_id, buff_len);
7a0bf85b 3295 return -EINVAL;
a4b0e8a4
PM		 3296 }
3297
3298 /* Check whether we need temp memory */
3299 if (param_offset != 0 || param_size < buff_len) {
1699f980 3300 desc_buf = kzalloc(buff_len, GFP_KERNEL);
da461cec
SJ		 3301 if (!desc_buf)
3302 return -ENOMEM;
a4b0e8a4
PM		 3303 } else {
3304 desc_buf = param_read_buf;
3305 is_kmalloc = false;
da461cec
SJ		 3306 }
3307
a4b0e8a4 3308 /* Request for full descriptor */
a70e91b8 3309 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3310 desc_id, desc_index, 0,
3311 desc_buf, &buff_len);
da461cec 3312
bde44bb6 3313 if (ret) {
1699f980 3314 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
bde44bb6 3315 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3316 goto out;
3317 }
3318
bde44bb6 3319 /* Sanity check */
3320 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
1699f980 3321 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
bde44bb6 3322 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3323 ret = -EINVAL;
3324 goto out;
3325 }
3326
7a0bf85b
BH		 3327 /* Update descriptor length */
3328 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
72fb690e 3329 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
7a0bf85b 3330
1699f980
CG		 3331 if (is_kmalloc) {
3332 /* Make sure we don't copy more data than available */
3333 if (param_offset + param_size > buff_len)
3334 param_size = buff_len - param_offset;
da461cec 3335 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
1699f980 3336 }
da461cec
SJ		 3337out:
3338 if (is_kmalloc)
3339 kfree(desc_buf);
3340 return ret;
3341}
3342
4b828fe1
TW		 3343/**
3344 * struct uc_string_id - unicode string
3345 *
3346 * @len: size of this descriptor inclusive
3347 * @type: descriptor type
3348 * @uc: unicode string character
3349 */
3350struct uc_string_id {
3351 u8 len;
3352 u8 type;
ec38c0ad 3353 wchar_t uc[];
4b828fe1
TW		 3354} __packed;
3355
3356/* replace non-printable or non-ASCII characters with spaces */
3357static inline char ufshcd_remove_non_printable(u8 ch)
3358{
3359 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3360}
3361
b573d484
YG		 3362/**
3363 * ufshcd_read_string_desc - read string descriptor
3364 * @hba: pointer to adapter instance
3365 * @desc_index: descriptor index
4b828fe1
TW		 3366 * @buf: pointer to buffer where descriptor would be read,
3367 * the caller should free the memory.
b573d484 3368 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3369 * null terminated string.
b573d484 3370 *
4b828fe1
TW		 3371 * Return:
3372 * * string size on success.
3373 * * -ENOMEM: on allocation failure
3374 * * -EINVAL: on a wrong parameter
b573d484 3375 */
4b828fe1
TW		 3376int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3377 u8 **buf, bool ascii)
b573d484 3378{
4b828fe1
TW		 3379 struct uc_string_id *uc_str;
3380 u8 *str;
3381 int ret;
b573d484 3382
4b828fe1
TW		 3383 if (!buf)
3384 return -EINVAL;
b573d484 3385
4b828fe1
TW		 3386 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3387 if (!uc_str)
3388 return -ENOMEM;
b573d484 3389
c4607a09
BH		 3390 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3391 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
4b828fe1
TW		 3392 if (ret < 0) {
3393 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3394 QUERY_REQ_RETRIES, ret);
3395 str = NULL;
3396 goto out;
3397 }
3398
3399 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3400 dev_dbg(hba->dev, "String Desc is of zero length\n");
3401 str = NULL;
3402 ret = 0;
b573d484
YG		 3403 goto out;
3404 }
3405
3406 if (ascii) {
4b828fe1 3407 ssize_t ascii_len;
b573d484 3408 int i;
b573d484 3409 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3410 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3411 str = kzalloc(ascii_len, GFP_KERNEL);
3412 if (!str) {
3413 ret = -ENOMEM;
fcbefc3b 3414 goto out;
b573d484
YG		 3415 }
3416
3417 /*
3418 * the descriptor contains string in UTF16 format
3419 * we need to convert to utf-8 so it can be displayed
3420 */
4b828fe1
TW		 3421 ret = utf16s_to_utf8s(uc_str->uc,
3422 uc_str->len - QUERY_DESC_HDR_SIZE,
3423 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3424
3425 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3426 for (i = 0; i < ret; i++)
3427 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3428
4b828fe1
TW		 3429 str[ret++] = '\0';
3430
3431 } else {
5f57704d 3432 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3433 if (!str) {
3434 ret = -ENOMEM;
3435 goto out;
3436 }
4b828fe1 3437 ret = uc_str->len;
b573d484
YG		 3438 }
3439out:
4b828fe1
TW		 3440 *buf = str;
3441 kfree(uc_str);
3442 return ret;
b573d484 3443}
b573d484 3444
da461cec
SJ		 3445/**
3446 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3447 * @hba: Pointer to adapter instance
3448 * @lun: lun id
3449 * @param_offset: offset of the parameter to read
3450 * @param_read_buf: pointer to buffer where parameter would be read
3451 * @param_size: sizeof(param_read_buf)
3452 *
3453 * Return 0 in case of success, non-zero otherwise
3454 */
3455static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3456 int lun,
3457 enum unit_desc_param param_offset,
3458 u8 *param_read_buf,
3459 u32 param_size)
3460{
3461 /*
3462 * Unit descriptors are only available for general purpose LUs (LUN id
3463 * from 0 to 7) and RPMB Well known LU.
3464 */
a2fca52e 3465 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
da461cec
SJ		 3466 return -EOPNOTSUPP;
3467
3468 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3469 param_offset, param_read_buf, param_size);
3470}
3471
09f17791
CG		 3472static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3473{
3474 int err = 0;
3475 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3476
3477 if (hba->dev_info.wspecversion >= 0x300) {
3478 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3479 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3480 &gating_wait);
3481 if (err)
3482 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3483 err, gating_wait);
3484
3485 if (gating_wait == 0) {
3486 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3487 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3488 gating_wait);
3489 }
3490
3491 hba->dev_info.clk_gating_wait_us = gating_wait;
3492 }
3493
3494 return err;
3495}
3496
7a3e97b0
SY		 3497/**
3498 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3499 * @hba: per adapter instance
3500 *
3501 * 1. Allocate DMA memory for Command Descriptor array
3502 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3503 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3504 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3505 * (UTMRDL)
3506 * 4. Allocate memory for local reference block(lrb).
3507 *
3508 * Returns 0 for success, non-zero in case of failure
3509 */
3510static int ufshcd_memory_alloc(struct ufs_hba *hba)
3511{
3512 size_t utmrdl_size, utrdl_size, ucdl_size;
3513
3514 /* Allocate memory for UTP command descriptors */
3515 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3516 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3517 ucdl_size,
3518 &hba->ucdl_dma_addr,
3519 GFP_KERNEL);
7a3e97b0
SY		 3520
3521 /*
3522 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3523 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3524 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3525 * be aligned to 128 bytes as well
3526 */
3527 if (!hba->ucdl_base_addr ||
3528 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3529 dev_err(hba->dev,
7a3e97b0
SY		 3530 "Command Descriptor Memory allocation failed\n");
3531 goto out;
3532 }
3533
3534 /*
3535 * Allocate memory for UTP Transfer descriptors
3536 * UFSHCI requires 1024 byte alignment of UTRD
3537 */
3538 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3539 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3540 utrdl_size,
3541 &hba->utrdl_dma_addr,
3542 GFP_KERNEL);
7a3e97b0
SY		 3543 if (!hba->utrdl_base_addr ||
3544 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3545 dev_err(hba->dev,
7a3e97b0
SY		 3546 "Transfer Descriptor Memory allocation failed\n");
3547 goto out;
3548 }
3549
3550 /*
3551 * Allocate memory for UTP Task Management descriptors
3552 * UFSHCI requires 1024 byte alignment of UTMRD
3553 */
3554 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3555 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3556 utmrdl_size,
3557 &hba->utmrdl_dma_addr,
3558 GFP_KERNEL);
7a3e97b0
SY		 3559 if (!hba->utmrdl_base_addr ||
3560 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3561 dev_err(hba->dev,
7a3e97b0
SY		 3562 "Task Management Descriptor Memory allocation failed\n");
3563 goto out;
3564 }
3565
3566 /* Allocate memory for local reference block */
a86854d0
KC		 3567 hba->lrb = devm_kcalloc(hba->dev,
3568 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3569 GFP_KERNEL);
7a3e97b0 3570 if (!hba->lrb) {
3b1d0580 3571 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3572 goto out;
3573 }
3574 return 0;
3575out:
7a3e97b0
SY		 3576 return -ENOMEM;
3577}
3578
3579/**
3580 * ufshcd_host_memory_configure - configure local reference block with
3581 * memory offsets
3582 * @hba: per adapter instance
3583 *
3584 * Configure Host memory space
3585 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3586 * address.
3587 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3588 * and PRDT offset.
3589 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3590 * into local reference block.
3591 */
3592static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3593{
7a3e97b0
SY		 3594 struct utp_transfer_req_desc *utrdlp;
3595 dma_addr_t cmd_desc_dma_addr;
3596 dma_addr_t cmd_desc_element_addr;
3597 u16 response_offset;
3598 u16 prdt_offset;
3599 int cmd_desc_size;
3600 int i;
3601
3602 utrdlp = hba->utrdl_base_addr;
7a3e97b0
SY		 3603
3604 response_offset =
3605 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3606 prdt_offset =
3607 offsetof(struct utp_transfer_cmd_desc, prd_table);
3608
3609 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3610 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3611
3612 for (i = 0; i < hba->nutrs; i++) {
3613 /* Configure UTRD with command descriptor base address */
3614 cmd_desc_element_addr =
3615 (cmd_desc_dma_addr + (cmd_desc_size * i));
3616 utrdlp[i].command_desc_base_addr_lo =
3617 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3618 utrdlp[i].command_desc_base_addr_hi =
3619 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3620
3621 /* Response upiu and prdt offset should be in double words */
26f968d7
AA		 3622 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3623 utrdlp[i].response_upiu_offset =
3624 cpu_to_le16(response_offset);
3625 utrdlp[i].prd_table_offset =
3626 cpu_to_le16(prdt_offset);
3627 utrdlp[i].response_upiu_length =
3628 cpu_to_le16(ALIGNED_UPIU_SIZE);
3629 } else {
3630 utrdlp[i].response_upiu_offset =
3631 cpu_to_le16(response_offset >> 2);
3632 utrdlp[i].prd_table_offset =
3633 cpu_to_le16(prdt_offset >> 2);
3634 utrdlp[i].response_upiu_length =
3635 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3636 }
7a3e97b0 3637
4d2b8d40 3638 ufshcd_init_lrb(hba, &hba->lrb[i], i);
7a3e97b0
SY		 3639 }
3640}
3641
3642/**
3643 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3644 * @hba: per adapter instance
3645 *
3646 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3647 * in order to initialize the Unipro link startup procedure.
3648 * Once the Unipro links are up, the device connected to the controller
3649 * is detected.
3650 *
3651 * Returns 0 on success, non-zero value on failure
3652 */
3653static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3654{
6ccf44fe
SJ		 3655 struct uic_command uic_cmd = {0};
3656 int ret;
7a3e97b0 3657
6ccf44fe 3658 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3659
6ccf44fe
SJ		 3660 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3661 if (ret)
ff8e20c6 3662 dev_dbg(hba->dev,
6ccf44fe
SJ		 3663 "dme-link-startup: error code %d\n", ret);
3664 return ret;
7a3e97b0 3665}
39bf2d83
AA		 3666/**
3667 * ufshcd_dme_reset - UIC command for DME_RESET
3668 * @hba: per adapter instance
3669 *
3670 * DME_RESET command is issued in order to reset UniPro stack.
3671 * This function now deals with cold reset.
3672 *
3673 * Returns 0 on success, non-zero value on failure
3674 */
3675static int ufshcd_dme_reset(struct ufs_hba *hba)
3676{
3677 struct uic_command uic_cmd = {0};
3678 int ret;
3679
3680 uic_cmd.command = UIC_CMD_DME_RESET;
3681
3682 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3683 if (ret)
3684 dev_err(hba->dev,
3685 "dme-reset: error code %d\n", ret);
3686
3687 return ret;
3688}
3689
fc85a74e
SC		 3690int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3691 int agreed_gear,
3692 int adapt_val)
3693{
3694 int ret;
3695
3696 if (agreed_gear != UFS_HS_G4)
66df79cc 3697 adapt_val = PA_NO_ADAPT;
fc85a74e
SC		 3698
3699 ret = ufshcd_dme_set(hba,
3700 UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3701 adapt_val);
3702 return ret;
3703}
3704EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3705
39bf2d83
AA		 3706/**
3707 * ufshcd_dme_enable - UIC command for DME_ENABLE
3708 * @hba: per adapter instance
3709 *
3710 * DME_ENABLE command is issued in order to enable UniPro stack.
3711 *
3712 * Returns 0 on success, non-zero value on failure
3713 */
3714static int ufshcd_dme_enable(struct ufs_hba *hba)
3715{
3716 struct uic_command uic_cmd = {0};
3717 int ret;
3718
3719 uic_cmd.command = UIC_CMD_DME_ENABLE;
3720
3721 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3722 if (ret)
3723 dev_err(hba->dev,
1fa05700 3724 "dme-enable: error code %d\n", ret);
39bf2d83
AA		 3725
3726 return ret;
3727}
7a3e97b0 3728
cad2e03d
YG		 3729static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3730{
3731 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3732 unsigned long min_sleep_time_us;
3733
3734 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3735 return;
3736
3737 /*
3738 * last_dme_cmd_tstamp will be 0 only for 1st call to
3739 * this function
3740 */
3741 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3742 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3743 } else {
3744 unsigned long delta =
3745 (unsigned long) ktime_to_us(
3746 ktime_sub(ktime_get(),
3747 hba->last_dme_cmd_tstamp));
3748
3749 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3750 min_sleep_time_us =
3751 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3752 else
3753 return; /* no more delay required */
3754 }
3755
3756 /* allow sleep for extra 50us if needed */
3757 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3758}
3759
12b4fdb4
SJ		 3760/**
3761 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3762 * @hba: per adapter instance
3763 * @attr_sel: uic command argument1
3764 * @attr_set: attribute set type as uic command argument2
3765 * @mib_val: setting value as uic command argument3
3766 * @peer: indicate whether peer or local
3767 *
3768 * Returns 0 on success, non-zero value on failure
3769 */
3770int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3771 u8 attr_set, u32 mib_val, u8 peer)
3772{
3773 struct uic_command uic_cmd = {0};
3774 static const char *const action[] = {
3775 "dme-set",
3776 "dme-peer-set"
3777 };
3778 const char *set = action[!!peer];
3779 int ret;
64238fbd 3780 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3781
3782 uic_cmd.command = peer ?
3783 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3784 uic_cmd.argument1 = attr_sel;
3785 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3786 uic_cmd.argument3 = mib_val;
3787
64238fbd
YG		 3788 do {
3789 /* for peer attributes we retry upon failure */
3790 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3791 if (ret)
3792 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3793 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3794 } while (ret && peer && --retries);
3795
f37e9f8c 3796 if (ret)
64238fbd 3797 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3798 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3799 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3800
3801 return ret;
3802}
3803EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3804
3805/**
3806 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3807 * @hba: per adapter instance
3808 * @attr_sel: uic command argument1
3809 * @mib_val: the value of the attribute as returned by the UIC command
3810 * @peer: indicate whether peer or local
3811 *
3812 * Returns 0 on success, non-zero value on failure
3813 */
3814int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3815 u32 *mib_val, u8 peer)
3816{
3817 struct uic_command uic_cmd = {0};
3818 static const char *const action[] = {
3819 "dme-get",
3820 "dme-peer-get"
3821 };
3822 const char *get = action[!!peer];
3823 int ret;
64238fbd 3824 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3825 struct ufs_pa_layer_attr orig_pwr_info;
3826 struct ufs_pa_layer_attr temp_pwr_info;
3827 bool pwr_mode_change = false;
3828
3829 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3830 orig_pwr_info = hba->pwr_info;
3831 temp_pwr_info = orig_pwr_info;
3832
3833 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3834 orig_pwr_info.pwr_rx == FAST_MODE) {
3835 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3836 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3837 pwr_mode_change = true;
3838 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3839 orig_pwr_info.pwr_rx == SLOW_MODE) {
3840 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3841 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3842 pwr_mode_change = true;
3843 }
3844 if (pwr_mode_change) {
3845 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3846 if (ret)
3847 goto out;
3848 }
3849 }
12b4fdb4
SJ		 3850
3851 uic_cmd.command = peer ?
3852 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3853 uic_cmd.argument1 = attr_sel;
3854
64238fbd
YG		 3855 do {
3856 /* for peer attributes we retry upon failure */
3857 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3858 if (ret)
3859 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3860 get, UIC_GET_ATTR_ID(attr_sel), ret);
3861 } while (ret && peer && --retries);
3862
f37e9f8c 3863 if (ret)
64238fbd 3864 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3865 get, UIC_GET_ATTR_ID(attr_sel),
3866 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3867
64238fbd 3868 if (mib_val && !ret)
12b4fdb4 3869 *mib_val = uic_cmd.argument3;
874237f7
YG		 3870
3871 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3872 && pwr_mode_change)
3873 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3874out:
3875 return ret;
3876}
3877EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3878
53b3d9c3 3879/**
57d104c1
SJ		 3880 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3881 * state) and waits for it to take effect.
3882 *
53b3d9c3 3883 * @hba: per adapter instance
57d104c1
SJ		 3884 * @cmd: UIC command to execute
3885 *
3886 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3887 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3888 * and device UniPro link and hence it's final completion would be indicated by
3889 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3890 * addition to normal UIC command completion Status (UCCS). This function only
3891 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3892 *
3893 * Returns 0 on success, non-zero value on failure
3894 */
57d104c1 3895static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3896{
57d104c1 3897 struct completion uic_async_done;
53b3d9c3
SJ		 3898 unsigned long flags;
3899 u8 status;
3900 int ret;
d75f7fe4 3901 bool reenable_intr = false;
53b3d9c3 3902
53b3d9c3 3903 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3904 init_completion(&uic_async_done);
cad2e03d 3905 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3906
3907 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236
CG		 3908 if (ufshcd_is_link_broken(hba)) {
3909 ret = -ENOLINK;
3910 goto out_unlock;
3911 }
57d104c1 3912 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3913 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3914 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3915 /*
3916 * Make sure UIC command completion interrupt is disabled before
3917 * issuing UIC command.
3918 */
3919 wmb();
3920 reenable_intr = true;
57d104c1 3921 }
d75f7fe4
YG		 3922 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3923 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3924 if (ret) {
3925 dev_err(hba->dev,
3926 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3927 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3928 goto out;
3929 }
3930
57d104c1 3931 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3932 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3933 dev_err(hba->dev,
57d104c1
SJ		 3934 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3935 cmd->command, cmd->argument3);
0f52fcb9
CG		 3936
3937 if (!cmd->cmd_active) {
3938 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
3939 __func__);
3940 goto check_upmcrs;
3941 }
3942
53b3d9c3
SJ		 3943 ret = -ETIMEDOUT;
3944 goto out;
3945 }
3946
0f52fcb9 3947check_upmcrs:
53b3d9c3
SJ		 3948 status = ufshcd_get_upmcrs(hba);
3949 if (status != PWR_LOCAL) {
3950 dev_err(hba->dev,
479da360 3951 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3952 cmd->command, status);
53b3d9c3
SJ		 3953 ret = (status != PWR_OK) ? status : -1;
3954 }
3955out:
7942f7b5
VG		 3956 if (ret) {
3957 ufshcd_print_host_state(hba);
3958 ufshcd_print_pwr_info(hba);
e965e5e0 3959 ufshcd_print_evt_hist(hba);
7942f7b5
VG		 3960 }
3961
53b3d9c3 3962 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3963 hba->active_uic_cmd = NULL;
57d104c1 3964 hba->uic_async_done = NULL;
d75f7fe4
YG		 3965 if (reenable_intr)
3966 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4db7a236
CG		 3967 if (ret) {
3968 ufshcd_set_link_broken(hba);
3969 ufshcd_schedule_eh_work(hba);
3970 }
3971out_unlock:
53b3d9c3
SJ		 3972 spin_unlock_irqrestore(hba->host->host_lock, flags);
3973 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3974
53b3d9c3
SJ		 3975 return ret;
3976}
3977
57d104c1
SJ		 3978/**
3979 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3980 * using DME_SET primitives.
3981 * @hba: per adapter instance
3982 * @mode: powr mode value
3983 *
3984 * Returns 0 on success, non-zero value on failure
3985 */
3986static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3987{
3988 struct uic_command uic_cmd = {0};
1ab27c9c 3989 int ret;
57d104c1 3990
c3a2f9ee
YG		 3991 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3992 ret = ufshcd_dme_set(hba,
3993 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3994 if (ret) {
3995 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3996 __func__, ret);
3997 goto out;
3998 }
3999 }
4000
57d104c1
SJ		 4001 uic_cmd.command = UIC_CMD_DME_SET;
4002 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4003 uic_cmd.argument3 = mode;
1ab27c9c
ST		 4004 ufshcd_hold(hba, false);
4005 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4006 ufshcd_release(hba);
57d104c1 4007
c3a2f9ee 4008out:
1ab27c9c 4009 return ret;
57d104c1
SJ		 4010}
4011
087c5efa 4012int ufshcd_link_recovery(struct ufs_hba *hba)
53c12d0e
YG		 4013{
4014 int ret;
4015 unsigned long flags;
4016
4017 spin_lock_irqsave(hba->host->host_lock, flags);
4018 hba->ufshcd_state = UFSHCD_STATE_RESET;
4019 ufshcd_set_eh_in_progress(hba);
4020 spin_unlock_irqrestore(hba->host->host_lock, flags);
4021
ebdd1dfd 4022 /* Reset the attached device */
31a5d9ca 4023 ufshcd_device_reset(hba);
ebdd1dfd 4024
53c12d0e
YG		 4025 ret = ufshcd_host_reset_and_restore(hba);
4026
4027 spin_lock_irqsave(hba->host->host_lock, flags);
4028 if (ret)
4029 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4030 ufshcd_clear_eh_in_progress(hba);
4031 spin_unlock_irqrestore(hba->host->host_lock, flags);
4032
4033 if (ret)
4034 dev_err(hba->dev, "%s: link recovery failed, err %d",
4035 __func__, ret);
4ee7ee53
JK		 4036 else
4037 ufshcd_clear_ua_wluns(hba);
53c12d0e
YG		 4038
4039 return ret;
4040}
087c5efa 4041EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
53c12d0e 4042
4db7a236 4043static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 4044{
87d0b4a6 4045 int ret;
57d104c1 4046 struct uic_command uic_cmd = {0};
911a0771 4047 ktime_t start = ktime_get();
57d104c1 4048
ee32c909
KK		 4049 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4050
57d104c1 4051 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 4052 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 4053 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4054 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 4055
4db7a236 4056 if (ret)
87d0b4a6
YG		 4057 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4058 __func__, ret);
4db7a236 4059 else
ee32c909
KK		 4060 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4061 POST_CHANGE);
53c12d0e 4062
87d0b4a6
YG		 4063 return ret;
4064}
4065
9d19bf7a 4066int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
57d104c1
SJ		 4067{
4068 struct uic_command uic_cmd = {0};
4069 int ret;
911a0771 4070 ktime_t start = ktime_get();
57d104c1 4071
ee32c909
KK		 4072 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4073
57d104c1
SJ		 4074 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4075 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 4076 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4077 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4078
57d104c1 4079 if (ret) {
53c12d0e
YG		 4080 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4081 __func__, ret);
ff8e20c6 4082 } else {
ee32c909
KK		 4083 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4084 POST_CHANGE);
ff8e20c6
DR		 4085 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
4086 hba->ufs_stats.hibern8_exit_cnt++;
4087 }
57d104c1
SJ		 4088
4089 return ret;
4090}
9d19bf7a 4091EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
57d104c1 4092
ba7af5ec
SC		 4093void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4094{
4095 unsigned long flags;
be7594a4 4096 bool update = false;
ba7af5ec 4097
be7594a4 4098 if (!ufshcd_is_auto_hibern8_supported(hba))
ba7af5ec
SC		 4099 return;
4100
4101 spin_lock_irqsave(hba->host->host_lock, flags);
be7594a4
CG		 4102 if (hba->ahit != ahit) {
4103 hba->ahit = ahit;
4104 update = true;
4105 }
ba7af5ec 4106 spin_unlock_irqrestore(hba->host->host_lock, flags);
be7594a4
CG		 4107
4108 if (update && !pm_runtime_suspended(hba->dev)) {
4109 pm_runtime_get_sync(hba->dev);
4110 ufshcd_hold(hba, false);
4111 ufshcd_auto_hibern8_enable(hba);
4112 ufshcd_release(hba);
4113 pm_runtime_put(hba->dev);
4114 }
ba7af5ec
SC		 4115}
4116EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4117
71d848b8 4118void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378
AH		 4119{
4120 unsigned long flags;
4121
499f7a96 4122 if (!ufshcd_is_auto_hibern8_supported(hba))
ad448378
AH		 4123 return;
4124
4125 spin_lock_irqsave(hba->host->host_lock, flags);
4126 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4127 spin_unlock_irqrestore(hba->host->host_lock, flags);
4128}
4129
5064636c
YG		 4130 /**
4131 * ufshcd_init_pwr_info - setting the POR (power on reset)
4132 * values in hba power info
4133 * @hba: per-adapter instance
4134 */
4135static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4136{
4137 hba->pwr_info.gear_rx = UFS_PWM_G1;
4138 hba->pwr_info.gear_tx = UFS_PWM_G1;
4139 hba->pwr_info.lane_rx = 1;
4140 hba->pwr_info.lane_tx = 1;
4141 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4142 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4143 hba->pwr_info.hs_rate = 0;
4144}
4145
d3e89bac 4146/**
7eb584db
DR		 4147 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4148 * @hba: per-adapter instance
d3e89bac 4149 */
7eb584db 4150static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 4151{
7eb584db
DR		 4152 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4153
4154 if (hba->max_pwr_info.is_valid)
4155 return 0;
4156
2349b533 4157 pwr_info->pwr_tx = FAST_MODE;
4158 pwr_info->pwr_rx = FAST_MODE;
7eb584db 4159 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 4160
4161 /* Get the connected lane count */
7eb584db
DR		 4162 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4163 &pwr_info->lane_rx);
4164 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4165 &pwr_info->lane_tx);
4166
4167 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4168 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4169 __func__,
4170 pwr_info->lane_rx,
4171 pwr_info->lane_tx);
4172 return -EINVAL;
4173 }
d3e89bac
SJ		 4174
4175 /*
4176 * First, get the maximum gears of HS speed.
4177 * If a zero value, it means there is no HSGEAR capability.
4178 * Then, get the maximum gears of PWM speed.
4179 */
7eb584db
DR		 4180 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4181 if (!pwr_info->gear_rx) {
4182 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4183 &pwr_info->gear_rx);
4184 if (!pwr_info->gear_rx) {
4185 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4186 __func__, pwr_info->gear_rx);
4187 return -EINVAL;
4188 }
2349b533 4189 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 4190 }
4191
7eb584db
DR		 4192 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4193 &pwr_info->gear_tx);
4194 if (!pwr_info->gear_tx) {
d3e89bac 4195 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 4196 &pwr_info->gear_tx);
4197 if (!pwr_info->gear_tx) {
4198 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4199 __func__, pwr_info->gear_tx);
4200 return -EINVAL;
4201 }
2349b533 4202 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 4203 }
4204
4205 hba->max_pwr_info.is_valid = true;
4206 return 0;
4207}
4208
4209static int ufshcd_change_power_mode(struct ufs_hba *hba,
4210 struct ufs_pa_layer_attr *pwr_mode)
4211{
4212 int ret;
4213
4214 /* if already configured to the requested pwr_mode */
2355b66e
CG		 4215 if (!hba->force_pmc &&
4216 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
7eb584db
DR		 4217 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4218 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4219 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4220 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4221 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4222 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4223 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4224 return 0;
d3e89bac
SJ		 4225 }
4226
4227 /*
4228 * Configure attributes for power mode change with below.
4229 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4230 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4231 * - PA_HSSERIES
4232 */
7eb584db
DR		 4233 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4234 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4235 pwr_mode->lane_rx);
4236 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4237 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4238 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4239 else
4240 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4241
7eb584db
DR		 4242 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4243 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4244 pwr_mode->lane_tx);
4245 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4246 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4247 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4248 else
4249 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4250
7eb584db
DR		 4251 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4252 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4253 pwr_mode->pwr_rx == FAST_MODE ||
4254 pwr_mode->pwr_tx == FAST_MODE)
4255 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4256 pwr_mode->hs_rate);
d3e89bac 4257
b1d0d2eb
KK		 4258 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4259 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4260 DL_FC0ProtectionTimeOutVal_Default);
4261 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4262 DL_TC0ReplayTimeOutVal_Default);
4263 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4264 DL_AFC0ReqTimeOutVal_Default);
4265 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4266 DL_FC1ProtectionTimeOutVal_Default);
4267 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4268 DL_TC1ReplayTimeOutVal_Default);
4269 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4270 DL_AFC1ReqTimeOutVal_Default);
4271
4272 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4273 DL_FC0ProtectionTimeOutVal_Default);
4274 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4275 DL_TC0ReplayTimeOutVal_Default);
4276 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4277 DL_AFC0ReqTimeOutVal_Default);
4278 }
08342537 4279
7eb584db
DR		 4280 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4281 | pwr_mode->pwr_tx);
4282
4283 if (ret) {
d3e89bac 4284 dev_err(hba->dev,
7eb584db
DR		 4285 "%s: power mode change failed %d\n", __func__, ret);
4286 } else {
0263bcd0
YG		 4287 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4288 pwr_mode);
7eb584db
DR		 4289
4290 memcpy(&hba->pwr_info, pwr_mode,
4291 sizeof(struct ufs_pa_layer_attr));
4292 }
4293
4294 return ret;
4295}
4296
4297/**
4298 * ufshcd_config_pwr_mode - configure a new power mode
4299 * @hba: per-adapter instance
4300 * @desired_pwr_mode: desired power configuration
4301 */
0d846e70 4302int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4303 struct ufs_pa_layer_attr *desired_pwr_mode)
4304{
4305 struct ufs_pa_layer_attr final_params = { 0 };
4306 int ret;
4307
0263bcd0
YG		 4308 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4309 desired_pwr_mode, &final_params);
4310
4311 if (ret)
7eb584db
DR		 4312 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4313
4314 ret = ufshcd_change_power_mode(hba, &final_params);
d3e89bac
SJ		 4315
4316 return ret;
4317}
0d846e70 4318EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4319
68078d5c
DR		 4320/**
4321 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4322 * @hba: per-adapter instance
68078d5c
DR		 4323 *
4324 * Set fDeviceInit flag and poll until device toggles it.
4325 */
4326static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4327{
dc3c8d3a 4328 int err;
7dfdcc39 4329 bool flag_res = true;
29707fab 4330 ktime_t timeout;
68078d5c 4331
dc3c8d3a 4332 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 4333 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
68078d5c
DR		 4334 if (err) {
4335 dev_err(hba->dev,
4336 "%s setting fDeviceInit flag failed with error %d\n",
4337 __func__, err);
4338 goto out;
4339 }
4340
29707fab
KK		 4341 /* Poll fDeviceInit flag to be cleared */
4342 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4343 do {
4344 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4345 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4346 if (!flag_res)
4347 break;
4348 usleep_range(5000, 10000);
4349 } while (ktime_before(ktime_get(), timeout));
dc3c8d3a 4350
29707fab 4351 if (err) {
68078d5c 4352 dev_err(hba->dev,
29707fab
KK		 4353 "%s reading fDeviceInit flag failed with error %d\n",
4354 __func__, err);
4355 } else if (flag_res) {
68078d5c 4356 dev_err(hba->dev,
29707fab
KK		 4357 "%s fDeviceInit was not cleared by the device\n",
4358 __func__);
4359 err = -EBUSY;
4360 }
68078d5c
DR		 4361out:
4362 return err;
4363}
4364
7a3e97b0
SY		 4365/**
4366 * ufshcd_make_hba_operational - Make UFS controller operational
4367 * @hba: per adapter instance
4368 *
4369 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4370 * 1. Enable required interrupts
4371 * 2. Configure interrupt aggregation
897efe62 4372 * 3. Program UTRL and UTMRL base address
5c0c28a8 4373 * 4. Configure run-stop-registers
7a3e97b0
SY		 4374 *
4375 * Returns 0 on success, non-zero value on failure
4376 */
9d19bf7a 4377int ufshcd_make_hba_operational(struct ufs_hba *hba)
7a3e97b0
SY		 4378{
4379 int err = 0;
4380 u32 reg;
4381
6ccf44fe
SJ		 4382 /* Enable required interrupts */
4383 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4384
4385 /* Configure interrupt aggregation */
b852190e
YG		 4386 if (ufshcd_is_intr_aggr_allowed(hba))
4387 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4388 else
4389 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4390
4391 /* Configure UTRL and UTMRL base address registers */
4392 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4393 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4394 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4395 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4396 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4397 REG_UTP_TASK_REQ_LIST_BASE_L);
4398 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4399 REG_UTP_TASK_REQ_LIST_BASE_H);
4400
897efe62
YG		 4401 /*
4402 * Make sure base address and interrupt setup are updated before
4403 * enabling the run/stop registers below.
4404 */
4405 wmb();
4406
7a3e97b0
SY		 4407 /*
4408 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4409 */
5c0c28a8 4410 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4411 if (!(ufshcd_get_lists_status(reg))) {
4412 ufshcd_enable_run_stop_reg(hba);
4413 } else {
3b1d0580 4414 dev_err(hba->dev,
7a3e97b0
SY		 4415 "Host controller not ready to process requests");
4416 err = -EIO;
7a3e97b0
SY		 4417 }
4418
7a3e97b0
SY		 4419 return err;
4420}
9d19bf7a 4421EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
7a3e97b0 4422
596585a2
YG		 4423/**
4424 * ufshcd_hba_stop - Send controller to reset state
4425 * @hba: per adapter instance
596585a2 4426 */
5cac1095 4427static inline void ufshcd_hba_stop(struct ufs_hba *hba)
596585a2 4428{
5cac1095 4429 unsigned long flags;
596585a2
YG		 4430 int err;
4431
5cac1095
BVA		 4432 /*
4433 * Obtain the host lock to prevent that the controller is disabled
4434 * while the UFS interrupt handler is active on another CPU.
4435 */
4436 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 4437 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
5cac1095
BVA		 4438 spin_unlock_irqrestore(hba->host->host_lock, flags);
4439
596585a2
YG		 4440 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4441 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
5cac1095 4442 10, 1);
596585a2
YG		 4443 if (err)
4444 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4445}
4446
7a3e97b0 4447/**
39bf2d83 4448 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4449 * @hba: per adapter instance
4450 *
4451 * The controller resets itself and controller firmware initialization
4452 * sequence kicks off. When controller is ready it will set
4453 * the Host Controller Enable bit to 1.
4454 *
4455 * Returns 0 on success, non-zero value on failure
4456 */
39bf2d83 4457static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0 4458{
6081b12c
SC		 4459 int retry_outer = 3;
4460 int retry_inner;
7a3e97b0 4461
6081b12c 4462start:
596585a2 4463 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4464 /* change controller state to "reset state" */
5cac1095 4465 ufshcd_hba_stop(hba);
7a3e97b0 4466
57d104c1
SJ		 4467 /* UniPro link is disabled at this point */
4468 ufshcd_set_link_off(hba);
4469
0263bcd0 4470 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4471
7a3e97b0
SY		 4472 /* start controller initialization sequence */
4473 ufshcd_hba_start(hba);
4474
4475 /*
4476 * To initialize a UFS host controller HCE bit must be set to 1.
4477 * During initialization the HCE bit value changes from 1->0->1.
4478 * When the host controller completes initialization sequence
4479 * it sets the value of HCE bit to 1. The same HCE bit is read back
4480 * to check if the controller has completed initialization sequence.
4481 * So without this delay the value HCE = 1, set in the previous
4482 * instruction might be read back.
4483 * This delay can be changed based on the controller.
4484 */
90b8491c 4485 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
7a3e97b0
SY		 4486
4487 /* wait for the host controller to complete initialization */
6081b12c 4488 retry_inner = 50;
7a3e97b0 4489 while (ufshcd_is_hba_active(hba)) {
6081b12c
SC		 4490 if (retry_inner) {
4491 retry_inner--;
7a3e97b0 4492 } else {
3b1d0580 4493 dev_err(hba->dev,
7a3e97b0 4494 "Controller enable failed\n");
6081b12c
SC		 4495 if (retry_outer) {
4496 retry_outer--;
4497 goto start;
4498 }
7a3e97b0
SY		 4499 return -EIO;
4500 }
9fc305ef 4501 usleep_range(1000, 1100);
7a3e97b0 4502 }
5c0c28a8 4503
1d337ec2 4504 /* enable UIC related interrupts */
57d104c1 4505 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4506
0263bcd0 4507 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4508
7a3e97b0
SY		 4509 return 0;
4510}
39bf2d83
AA		 4511
4512int ufshcd_hba_enable(struct ufs_hba *hba)
4513{
4514 int ret;
4515
4516 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4517 ufshcd_set_link_off(hba);
4518 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4519
4520 /* enable UIC related interrupts */
4521 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4522 ret = ufshcd_dme_reset(hba);
4523 if (!ret) {
4524 ret = ufshcd_dme_enable(hba);
4525 if (!ret)
4526 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4527 if (ret)
4528 dev_err(hba->dev,
4529 "Host controller enable failed with non-hce\n");
4530 }
4531 } else {
4532 ret = ufshcd_hba_execute_hce(hba);
4533 }
4534
4535 return ret;
4536}
9d19bf7a
SC		 4537EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4538
7ca38cf3
YG		 4539static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4540{
ba0320fb 4541 int tx_lanes = 0, i, err = 0;
7ca38cf3
YG		 4542
4543 if (!peer)
4544 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4545 &tx_lanes);
4546 else
4547 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4548 &tx_lanes);
4549 for (i = 0; i < tx_lanes; i++) {
4550 if (!peer)
4551 err = ufshcd_dme_set(hba,
4552 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4553 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4554 0);
4555 else
4556 err = ufshcd_dme_peer_set(hba,
4557 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4558 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4559 0);
4560 if (err) {
4561 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4562 __func__, peer, i, err);
4563 break;
4564 }
4565 }
4566
4567 return err;
4568}
4569
4570static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4571{
4572 return ufshcd_disable_tx_lcc(hba, true);
4573}
4574
e965e5e0 4575void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
8808b4e9 4576{
e965e5e0
SC		 4577 struct ufs_event_hist *e;
4578
4579 if (id >= UFS_EVT_CNT)
4580 return;
4581
4582 e = &hba->ufs_stats.event[id];
4583 e->val[e->pos] = val;
4584 e->tstamp[e->pos] = ktime_get();
b6cacaf2 4585 e->cnt += 1;
e965e5e0 4586 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
172614a9
SC		 4587
4588 ufshcd_vops_event_notify(hba, id, &val);
8808b4e9 4589}
e965e5e0 4590EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
8808b4e9 4591
7a3e97b0 4592/**
6ccf44fe 4593 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4594 * @hba: per adapter instance
4595 *
6ccf44fe 4596 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4597 */
6ccf44fe 4598static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4599{
6ccf44fe 4600 int ret;
1d337ec2 4601 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4602 bool link_startup_again = false;
7a3e97b0 4603
7caf489b 4604 /*
4605 * If UFS device isn't active then we will have to issue link startup
4606 * 2 times to make sure the device state move to active.
4607 */
4608 if (!ufshcd_is_ufs_dev_active(hba))
4609 link_startup_again = true;
7a3e97b0 4610
7caf489b 4611link_startup:
1d337ec2 4612 do {
0263bcd0 4613 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4614
1d337ec2 4615 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4616
1d337ec2
SRT		 4617 /* check if device is detected by inter-connect layer */
4618 if (!ret && !ufshcd_is_device_present(hba)) {
e965e5e0
SC		 4619 ufshcd_update_evt_hist(hba,
4620 UFS_EVT_LINK_STARTUP_FAIL,
8808b4e9 4621 0);
1d337ec2
SRT		 4622 dev_err(hba->dev, "%s: Device not present\n", __func__);
4623 ret = -ENXIO;
4624 goto out;
4625 }
6ccf44fe 4626
1d337ec2
SRT		 4627 /*
4628 * DME link lost indication is only received when link is up,
4629 * but we can't be sure if the link is up until link startup
4630 * succeeds. So reset the local Uni-Pro and try again.
4631 */
8808b4e9 4632 if (ret && ufshcd_hba_enable(hba)) {
e965e5e0
SC		 4633 ufshcd_update_evt_hist(hba,
4634 UFS_EVT_LINK_STARTUP_FAIL,
8808b4e9 4635 (u32)ret);
1d337ec2 4636 goto out;
8808b4e9 4637 }
1d337ec2
SRT		 4638 } while (ret && retries--);
4639
8808b4e9 4640 if (ret) {
1d337ec2 4641 /* failed to get the link up... retire */
e965e5e0
SC		 4642 ufshcd_update_evt_hist(hba,
4643 UFS_EVT_LINK_STARTUP_FAIL,
8808b4e9 4644 (u32)ret);
5c0c28a8 4645 goto out;
8808b4e9 4646 }
5c0c28a8 4647
7caf489b 4648 if (link_startup_again) {
4649 link_startup_again = false;
4650 retries = DME_LINKSTARTUP_RETRIES;
4651 goto link_startup;
4652 }
4653
d2aebb9b 4654 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4655 ufshcd_init_pwr_info(hba);
4656 ufshcd_print_pwr_info(hba);
4657
7ca38cf3
YG		 4658 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4659 ret = ufshcd_disable_device_tx_lcc(hba);
4660 if (ret)
4661 goto out;
4662 }
4663
5c0c28a8 4664 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4665 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4666 if (ret)
4667 goto out;
7a3e97b0 4668
2355b66e
CG		 4669 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4670 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5c0c28a8 4671 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4672out:
7942f7b5 4673 if (ret) {
6ccf44fe 4674 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4675 ufshcd_print_host_state(hba);
4676 ufshcd_print_pwr_info(hba);
e965e5e0 4677 ufshcd_print_evt_hist(hba);
7942f7b5 4678 }
6ccf44fe 4679 return ret;
7a3e97b0
SY		 4680}
4681
5a0b0cb9
SRT		 4682/**
4683 * ufshcd_verify_dev_init() - Verify device initialization
4684 * @hba: per-adapter instance
4685 *
4686 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4687 * device Transport Protocol (UTP) layer is ready after a reset.
4688 * If the UTP layer at the device side is not initialized, it may
4689 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4690 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4691 */
4692static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4693{
4694 int err = 0;
4695 int retries;
4696
1ab27c9c 4697 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4698 mutex_lock(&hba->dev_cmd.lock);
4699 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4700 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4701 NOP_OUT_TIMEOUT);
4702
4703 if (!err || err == -ETIMEDOUT)
4704 break;
4705
4706 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4707 }
4708 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4709 ufshcd_release(hba);
5a0b0cb9
SRT		 4710
4711 if (err)
4712 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4713 return err;
4714}
4715
0ce147d4
SJ		 4716/**
4717 * ufshcd_set_queue_depth - set lun queue depth
4718 * @sdev: pointer to SCSI device
4719 *
4720 * Read bLUQueueDepth value and activate scsi tagged command
4721 * queueing. For WLUN, queue depth is set to 1. For best-effort
4722 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4723 * value that host can queue.
4724 */
4725static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4726{
4727 int ret = 0;
4728 u8 lun_qdepth;
4729 struct ufs_hba *hba;
4730
4731 hba = shost_priv(sdev->host);
4732
4733 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4734 ret = ufshcd_read_unit_desc_param(hba,
4735 ufshcd_scsi_to_upiu_lun(sdev->lun),
4736 UNIT_DESC_PARAM_LU_Q_DEPTH,
4737 &lun_qdepth,
4738 sizeof(lun_qdepth));
0ce147d4
SJ		 4739
4740 /* Some WLUN doesn't support unit descriptor */
4741 if (ret == -EOPNOTSUPP)
4742 lun_qdepth = 1;
4743 else if (!lun_qdepth)
4744 /* eventually, we can figure out the real queue depth */
4745 lun_qdepth = hba->nutrs;
4746 else
4747 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4748
4749 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4750 __func__, lun_qdepth);
db5ed4df 4751 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4752}
4753
57d104c1
SJ		 4754/*
4755 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4756 * @hba: per-adapter instance
4757 * @lun: UFS device lun id
4758 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4759 *
4760 * Returns 0 in case of success and b_lu_write_protect status would be returned
4761 * @b_lu_write_protect parameter.
4762 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4763 * Returns -EINVAL in case of invalid parameters passed to this function.
4764 */
4765static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4766 u8 lun,
4767 u8 *b_lu_write_protect)
4768{
4769 int ret;
4770
4771 if (!b_lu_write_protect)
4772 ret = -EINVAL;
4773 /*
4774 * According to UFS device spec, RPMB LU can't be write
4775 * protected so skip reading bLUWriteProtect parameter for
4776 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4777 */
1baa8011 4778 else if (lun >= hba->dev_info.max_lu_supported)
57d104c1
SJ		 4779 ret = -ENOTSUPP;
4780 else
4781 ret = ufshcd_read_unit_desc_param(hba,
4782 lun,
4783 UNIT_DESC_PARAM_LU_WR_PROTECT,
4784 b_lu_write_protect,
4785 sizeof(*b_lu_write_protect));
4786 return ret;
4787}
4788
4789/**
4790 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4791 * status
4792 * @hba: per-adapter instance
4793 * @sdev: pointer to SCSI device
4794 *
4795 */
4796static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4797 struct scsi_device *sdev)
4798{
4799 if (hba->dev_info.f_power_on_wp_en &&
4800 !hba->dev_info.is_lu_power_on_wp) {
4801 u8 b_lu_write_protect;
4802
4803 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4804 &b_lu_write_protect) &&
4805 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4806 hba->dev_info.is_lu_power_on_wp = true;
4807 }
4808}
4809
7a3e97b0
SY		 4810/**
4811 * ufshcd_slave_alloc - handle initial SCSI device configurations
4812 * @sdev: pointer to SCSI device
4813 *
4814 * Returns success
4815 */
4816static int ufshcd_slave_alloc(struct scsi_device *sdev)
4817{
4818 struct ufs_hba *hba;
4819
4820 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4821
4822 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4823 sdev->use_10_for_ms = 1;
a3a76391
CG		 4824
4825 /* DBD field should be set to 1 in mode sense(10) */
4826 sdev->set_dbd_for_ms = 1;
7a3e97b0 4827
e8e7f271
SRT		 4828 /* allow SCSI layer to restart the device in case of errors */
4829 sdev->allow_restart = 1;
4264fd61 4830
b2a6c522
SRT		 4831 /* REPORT SUPPORTED OPERATION CODES is not supported */
4832 sdev->no_report_opcodes = 1;
4833
84af7e8b
SRT		 4834 /* WRITE_SAME command is not supported */
4835 sdev->no_write_same = 1;
e8e7f271 4836
0ce147d4 4837 ufshcd_set_queue_depth(sdev);
4264fd61 4838
57d104c1
SJ		 4839 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4840
7a3e97b0
SY		 4841 return 0;
4842}
4843
4264fd61
SRT		 4844/**
4845 * ufshcd_change_queue_depth - change queue depth
4846 * @sdev: pointer to SCSI device
4847 * @depth: required depth to set
4264fd61 4848 *
db5ed4df 4849 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4850 */
db5ed4df 4851static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4852{
4853 struct ufs_hba *hba = shost_priv(sdev->host);
4854
4855 if (depth > hba->nutrs)
4856 depth = hba->nutrs;
db5ed4df 4857 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4858}
4859
eeda4749
AM		 4860/**
4861 * ufshcd_slave_configure - adjust SCSI device configurations
4862 * @sdev: pointer to SCSI device
4863 */
4864static int ufshcd_slave_configure(struct scsi_device *sdev)
4865{
49615ba1 4866 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 4867 struct request_queue *q = sdev->request_queue;
4868
4869 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
2b2bfc8a
KK		 4870 if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
4871 blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
49615ba1
SC		 4872
4873 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4874 sdev->rpm_autosuspend = 1;
4875
df043c74
ST		 4876 ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
4877
eeda4749
AM		 4878 return 0;
4879}
4880
7a3e97b0
SY		 4881/**
4882 * ufshcd_slave_destroy - remove SCSI device configurations
4883 * @sdev: pointer to SCSI device
4884 */
4885static void ufshcd_slave_destroy(struct scsi_device *sdev)
4886{
4887 struct ufs_hba *hba;
4888
4889 hba = shost_priv(sdev->host);
0ce147d4 4890 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4891 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4892 unsigned long flags;
4893
4894 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4895 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4896 spin_unlock_irqrestore(hba->host->host_lock, flags);
4897 }
7a3e97b0
SY		 4898}
4899
7a3e97b0
SY		 4900/**
4901 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4902 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4903 * @scsi_status: SCSI command status
4904 *
4905 * Returns value base on SCSI command status
4906 */
4907static inline int
4908ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4909{
4910 int result = 0;
4911
4912 switch (scsi_status) {
7a3e97b0 4913 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4914 ufshcd_copy_sense_data(lrbp);
df561f66 4915 fallthrough;
1c2623c5 4916 case SAM_STAT_GOOD:
db83d8a5 4917 result |= DID_OK << 16 | scsi_status;
7a3e97b0
SY		 4918 break;
4919 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4920 case SAM_STAT_BUSY:
7a3e97b0 4921 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4922 ufshcd_copy_sense_data(lrbp);
4923 result |= scsi_status;
7a3e97b0
SY		 4924 break;
4925 default:
4926 result |= DID_ERROR << 16;
4927 break;
4928 } /* end of switch */
4929
4930 return result;
4931}
4932
4933/**
4934 * ufshcd_transfer_rsp_status - Get overall status of the response
4935 * @hba: per adapter instance
8aa29f19 4936 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4937 *
4938 * Returns result of the command to notify SCSI midlayer
4939 */
4940static inline int
4941ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4942{
4943 int result = 0;
4944 int scsi_status;
4945 int ocs;
4946
4947 /* overall command status of utrd */
4948 ocs = ufshcd_get_tr_ocs(lrbp);
4949
d779a6e9
KK		 4950 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
4951 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
4952 MASK_RSP_UPIU_RESULT)
4953 ocs = OCS_SUCCESS;
4954 }
4955
7a3e97b0
SY		 4956 switch (ocs) {
4957 case OCS_SUCCESS:
5a0b0cb9 4958 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4959 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4960 switch (result) {
4961 case UPIU_TRANSACTION_RESPONSE:
4962 /*
4963 * get the response UPIU result to extract
4964 * the SCSI command status
4965 */
4966 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4967
4968 /*
4969 * get the result based on SCSI status response
4970 * to notify the SCSI midlayer of the command status
4971 */
4972 scsi_status = result & MASK_SCSI_STATUS;
4973 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4974
f05ac2e5
YG		 4975 /*
4976 * Currently we are only supporting BKOPs exception
4977 * events hence we can ignore BKOPs exception event
4978 * during power management callbacks. BKOPs exception
4979 * event is not expected to be raised in runtime suspend
4980 * callback as it allows the urgent bkops.
4981 * During system suspend, we are anyway forcefully
4982 * disabling the bkops and if urgent bkops is needed
4983 * it will be enabled on system resume. Long term
4984 * solution could be to abort the system suspend if
4985 * UFS device needs urgent BKOPs.
4986 */
4987 if (!hba->pm_op_in_progress &&
aa53f580 4988 !ufshcd_eh_in_progress(hba) &&
2824ec9f
SL		 4989 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr) &&
4990 schedule_work(&hba->eeh_work)) {
4991 /*
4992 * Prevent suspend once eeh_work is scheduled
4993 * to avoid deadlock between ufshcd_suspend
4994 * and exception event handler.
4995 */
4996 pm_runtime_get_noresume(hba->dev);
4997 }
5a0b0cb9
SRT		 4998 break;
4999 case UPIU_TRANSACTION_REJECT_UPIU:
5000 /* TODO: handle Reject UPIU Response */
5001 result = DID_ERROR << 16;
3b1d0580 5002 dev_err(hba->dev,
5a0b0cb9
SRT		 5003 "Reject UPIU not fully implemented\n");
5004 break;
5005 default:
5a0b0cb9
SRT		 5006 dev_err(hba->dev,
5007 "Unexpected request response code = %x\n",
5008 result);
e0347d89 5009 result = DID_ERROR << 16;
7a3e97b0
SY		 5010 break;
5011 }
7a3e97b0
SY		 5012 break;
5013 case OCS_ABORTED:
5014 result |= DID_ABORT << 16;
5015 break;
e8e7f271
SRT		 5016 case OCS_INVALID_COMMAND_STATUS:
5017 result |= DID_REQUEUE << 16;
5018 break;
7a3e97b0
SY		 5019 case OCS_INVALID_CMD_TABLE_ATTR:
5020 case OCS_INVALID_PRDT_ATTR:
5021 case OCS_MISMATCH_DATA_BUF_SIZE:
5022 case OCS_MISMATCH_RESP_UPIU_SIZE:
5023 case OCS_PEER_COMM_FAILURE:
5024 case OCS_FATAL_ERROR:
5e7341e1
ST		 5025 case OCS_DEVICE_FATAL_ERROR:
5026 case OCS_INVALID_CRYPTO_CONFIG:
5027 case OCS_GENERAL_CRYPTO_ERROR:
7a3e97b0
SY		 5028 default:
5029 result |= DID_ERROR << 16;
3b1d0580 5030 dev_err(hba->dev,
ff8e20c6
DR		 5031 "OCS error from controller = %x for tag %d\n",
5032 ocs, lrbp->task_tag);
e965e5e0 5033 ufshcd_print_evt_hist(hba);
6ba65588 5034 ufshcd_print_host_state(hba);
7a3e97b0
SY		 5035 break;
5036 } /* end of switch */
5037
eeb1b55b
JK		 5038 if ((host_byte(result) != DID_OK) &&
5039 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
66cc820f 5040 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 5041 return result;
5042}
5043
6ccf44fe
SJ		 5044/**
5045 * ufshcd_uic_cmd_compl - handle completion of uic command
5046 * @hba: per adapter instance
53b3d9c3 5047 * @intr_status: interrupt status generated by the controller
9333d775
VG		 5048 *
5049 * Returns
5050 * IRQ_HANDLED - If interrupt is valid
5051 * IRQ_NONE - If invalid interrupt
6ccf44fe 5052 */
9333d775 5053static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 5054{
9333d775
VG		 5055 irqreturn_t retval = IRQ_NONE;
5056
53b3d9c3 5057 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 5058 hba->active_uic_cmd->argument2 |=
5059 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 5060 hba->active_uic_cmd->argument3 =
5061 ufshcd_get_dme_attr_val(hba);
0f52fcb9
CG		 5062 if (!hba->uic_async_done)
5063 hba->active_uic_cmd->cmd_active = 0;
6ccf44fe 5064 complete(&hba->active_uic_cmd->done);
9333d775 5065 retval = IRQ_HANDLED;
6ccf44fe 5066 }
53b3d9c3 5067
9333d775 5068 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
0f52fcb9 5069 hba->active_uic_cmd->cmd_active = 0;
57d104c1 5070 complete(hba->uic_async_done);
9333d775
VG		 5071 retval = IRQ_HANDLED;
5072 }
aa5c6979
SC		 5073
5074 if (retval == IRQ_HANDLED)
5075 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
28fa68fc 5076 UFS_CMD_COMP);
9333d775 5077 return retval;
6ccf44fe
SJ		 5078}
5079
7a3e97b0 5080/**
9a47ec7c 5081 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 5082 * @hba: per adapter instance
9a47ec7c 5083 * @completed_reqs: requests to complete
7a3e97b0 5084 */
9a47ec7c
YG		 5085static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5086 unsigned long completed_reqs)
7a3e97b0 5087{
5a0b0cb9
SRT		 5088 struct ufshcd_lrb *lrbp;
5089 struct scsi_cmnd *cmd;
7a3e97b0
SY		 5090 int result;
5091 int index;
7a7e66c6 5092 bool update_scaling = false;
e9d501b1 5093
e9d501b1
DR		 5094 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5095 lrbp = &hba->lrb[index];
7a7e66c6 5096 lrbp->in_use = false;
a3170376 5097 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 5098 cmd = lrbp->cmd;
5099 if (cmd) {
28fa68fc 5100 ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
e9d501b1
DR		 5101 result = ufshcd_transfer_rsp_status(hba, lrbp);
5102 scsi_dma_unmap(cmd);
5103 cmd->result = result;
5104 /* Mark completed command as NULL in LRB */
5105 lrbp->cmd = NULL;
e9d501b1
DR		 5106 /* Do not touch lrbp after scsi done */
5107 cmd->scsi_done(cmd);
1ab27c9c 5108 __ufshcd_release(hba);
7a7e66c6 5109 update_scaling = true;
300bb13f
JP		 5110 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5111 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
1a07f2d9
LS		 5112 if (hba->dev_cmd.complete) {
5113 ufshcd_add_command_trace(hba, index,
28fa68fc 5114 UFS_DEV_COMP);
e9d501b1 5115 complete(hba->dev_cmd.complete);
7a7e66c6 5116 update_scaling = true;
1a07f2d9 5117 }
e9d501b1 5118 }
7a7e66c6 5119 if (ufshcd_is_clkscaling_supported(hba) && update_scaling)
401f1e44 5120 hba->clk_scaling.active_reqs--;
e9d501b1 5121 }
7a3e97b0
SY		 5122
5123 /* clear corresponding bits of completed commands */
5124 hba->outstanding_reqs ^= completed_reqs;
5125
856b3483 5126 ufshcd_clk_scaling_update_busy(hba);
7a3e97b0
SY		 5127}
5128
9a47ec7c
YG		 5129/**
5130 * ufshcd_transfer_req_compl - handle SCSI and query command completion
5131 * @hba: per adapter instance
9333d775
VG		 5132 *
5133 * Returns
5134 * IRQ_HANDLED - If interrupt is valid
5135 * IRQ_NONE - If invalid interrupt
9a47ec7c 5136 */
9333d775 5137static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c
YG		 5138{
5139 unsigned long completed_reqs;
5140 u32 tr_doorbell;
5141
5142 /* Resetting interrupt aggregation counters first and reading the
5143 * DOOR_BELL afterward allows us to handle all the completed requests.
5144 * In order to prevent other interrupts starvation the DB is read once
5145 * after reset. The down side of this solution is the possibility of
5146 * false interrupt if device completes another request after resetting
5147 * aggregation and before reading the DB.
5148 */
b638b5eb
AA		 5149 if (ufshcd_is_intr_aggr_allowed(hba) &&
5150 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 5151 ufshcd_reset_intr_aggr(hba);
5152
5153 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5154 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
5155
9333d775
VG		 5156 if (completed_reqs) {
5157 __ufshcd_transfer_req_compl(hba, completed_reqs);
5158 return IRQ_HANDLED;
5159 } else {
5160 return IRQ_NONE;
5161 }
9a47ec7c
YG		 5162}
5163
66ec6d59
SRT		 5164/**
5165 * ufshcd_disable_ee - disable exception event
5166 * @hba: per-adapter instance
5167 * @mask: exception event to disable
5168 *
5169 * Disables exception event in the device so that the EVENT_ALERT
5170 * bit is not set.
5171 *
5172 * Returns zero on success, non-zero error value on failure.
5173 */
5174static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5175{
5176 int err = 0;
5177 u32 val;
5178
5179 if (!(hba->ee_ctrl_mask & mask))
5180 goto out;
5181
5182 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 5183 val &= MASK_EE_STATUS;
5e86ae44 5184 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 5185 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5186 if (!err)
5187 hba->ee_ctrl_mask &= ~mask;
5188out:
5189 return err;
5190}
5191
5192/**
5193 * ufshcd_enable_ee - enable exception event
5194 * @hba: per-adapter instance
5195 * @mask: exception event to enable
5196 *
5197 * Enable corresponding exception event in the device to allow
5198 * device to alert host in critical scenarios.
5199 *
5200 * Returns zero on success, non-zero error value on failure.
5201 */
5202static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5203{
5204 int err = 0;
5205 u32 val;
5206
5207 if (hba->ee_ctrl_mask & mask)
5208 goto out;
5209
5210 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 5211 val &= MASK_EE_STATUS;
5e86ae44 5212 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 5213 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5214 if (!err)
5215 hba->ee_ctrl_mask |= mask;
5216out:
5217 return err;
5218}
5219
5220/**
5221 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5222 * @hba: per-adapter instance
5223 *
5224 * Allow device to manage background operations on its own. Enabling
5225 * this might lead to inconsistent latencies during normal data transfers
5226 * as the device is allowed to manage its own way of handling background
5227 * operations.
5228 *
5229 * Returns zero on success, non-zero on failure.
5230 */
5231static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5232{
5233 int err = 0;
5234
5235 if (hba->auto_bkops_enabled)
5236 goto out;
5237
dc3c8d3a 5238 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 5239 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5240 if (err) {
5241 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5242 __func__, err);
5243 goto out;
5244 }
5245
5246 hba->auto_bkops_enabled = true;
7ff5ab47 5247 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 5248
5249 /* No need of URGENT_BKOPS exception from the device */
5250 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5251 if (err)
5252 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5253 __func__, err);
5254out:
5255 return err;
5256}
5257
5258/**
5259 * ufshcd_disable_auto_bkops - block device in doing background operations
5260 * @hba: per-adapter instance
5261 *
5262 * Disabling background operations improves command response latency but
5263 * has drawback of device moving into critical state where the device is
5264 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5265 * host is idle so that BKOPS are managed effectively without any negative
5266 * impacts.
5267 *
5268 * Returns zero on success, non-zero on failure.
5269 */
5270static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5271{
5272 int err = 0;
5273
5274 if (!hba->auto_bkops_enabled)
5275 goto out;
5276
5277 /*
5278 * If host assisted BKOPs is to be enabled, make sure
5279 * urgent bkops exception is allowed.
5280 */
5281 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5282 if (err) {
5283 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5284 __func__, err);
5285 goto out;
5286 }
5287
dc3c8d3a 5288 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
1f34eedf 5289 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5290 if (err) {
5291 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5292 __func__, err);
5293 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5294 goto out;
5295 }
5296
5297 hba->auto_bkops_enabled = false;
7ff5ab47 5298 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 5299 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5300out:
5301 return err;
5302}
5303
5304/**
4e768e76 5305 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 5306 * @hba: per adapter instance
5307 *
5308 * After a device reset the device may toggle the BKOPS_EN flag
5309 * to default value. The s/w tracking variables should be updated
4e768e76 5310 * as well. This function would change the auto-bkops state based on
5311 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5312 */
4e768e76 5313static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5314{
4e768e76 5315 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5316 hba->auto_bkops_enabled = false;
5317 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5318 ufshcd_enable_auto_bkops(hba);
5319 } else {
5320 hba->auto_bkops_enabled = true;
5321 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5322 ufshcd_disable_auto_bkops(hba);
5323 }
7b6668d8 5324 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
24366c2a 5325 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5326}
5327
5328static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5329{
5e86ae44 5330 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5331 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5332}
5333
5334/**
57d104c1 5335 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5336 * @hba: per-adapter instance
57d104c1 5337 * @status: bkops_status value
66ec6d59 5338 *
57d104c1
SJ		 5339 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5340 * flag in the device to permit background operations if the device
5341 * bkops_status is greater than or equal to "status" argument passed to
5342 * this function, disable otherwise.
5343 *
5344 * Returns 0 for success, non-zero in case of failure.
5345 *
5346 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5347 * to know whether auto bkops is enabled or disabled after this function
5348 * returns control to it.
66ec6d59 5349 */
57d104c1
SJ		 5350static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5351 enum bkops_status status)
66ec6d59
SRT		 5352{
5353 int err;
57d104c1 5354 u32 curr_status = 0;
66ec6d59 5355
57d104c1 5356 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5357 if (err) {
5358 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5359 __func__, err);
5360 goto out;
57d104c1
SJ		 5361 } else if (curr_status > BKOPS_STATUS_MAX) {
5362 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5363 __func__, curr_status);
5364 err = -EINVAL;
5365 goto out;
66ec6d59
SRT		 5366 }
5367
57d104c1 5368 if (curr_status >= status)
66ec6d59 5369 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5370 else
5371 err = ufshcd_disable_auto_bkops(hba);
66ec6d59
SRT		 5372out:
5373 return err;
5374}
5375
57d104c1
SJ		 5376/**
5377 * ufshcd_urgent_bkops - handle urgent bkops exception event
5378 * @hba: per-adapter instance
5379 *
5380 * Enable fBackgroundOpsEn flag in the device to permit background
5381 * operations.
5382 *
5383 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5384 * and negative error value for any other failure.
5385 */
5386static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5387{
afdfff59 5388 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5389}
5390
66ec6d59
SRT		 5391static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5392{
5e86ae44 5393 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5394 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5395}
5396
afdfff59
YG		 5397static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5398{
5399 int err;
5400 u32 curr_status = 0;
5401
5402 if (hba->is_urgent_bkops_lvl_checked)
5403 goto enable_auto_bkops;
5404
5405 err = ufshcd_get_bkops_status(hba, &curr_status);
5406 if (err) {
5407 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5408 __func__, err);
5409 goto out;
5410 }
5411
5412 /*
5413 * We are seeing that some devices are raising the urgent bkops
5414 * exception events even when BKOPS status doesn't indicate performace
5415 * impacted or critical. Handle these device by determining their urgent
5416 * bkops status at runtime.
5417 */
5418 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5419 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5420 __func__, curr_status);
5421 /* update the current status as the urgent bkops level */
5422 hba->urgent_bkops_lvl = curr_status;
5423 hba->is_urgent_bkops_lvl_checked = true;
5424 }
5425
5426enable_auto_bkops:
5427 err = ufshcd_enable_auto_bkops(hba);
5428out:
5429 if (err < 0)
5430 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5431 __func__, err);
5432}
5433
8e834ca5 5434int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable)
3d17b9b5
AD		 5435{
5436 int ret;
6f8d5a6a 5437 u8 index;
3d17b9b5
AD		 5438 enum query_opcode opcode;
5439
79e3520f 5440 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5441 return 0;
5442
4cd48995 5443 if (!(enable ^ hba->dev_info.wb_enabled))
3d17b9b5
AD		 5444 return 0;
5445 if (enable)
5446 opcode = UPIU_QUERY_OPCODE_SET_FLAG;
5447 else
5448 opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5449
e31011ab 5450 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5451 ret = ufshcd_query_flag_retry(hba, opcode,
6f8d5a6a 5452 QUERY_FLAG_IDN_WB_EN, index, NULL);
3d17b9b5
AD		 5453 if (ret) {
5454 dev_err(hba->dev, "%s write booster %s failed %d\n",
5455 __func__, enable ? "enable" : "disable", ret);
5456 return ret;
5457 }
5458
4cd48995 5459 hba->dev_info.wb_enabled = enable;
3d17b9b5
AD		 5460 dev_dbg(hba->dev, "%s write booster %s %d\n",
5461 __func__, enable ? "enable" : "disable", ret);
5462
5463 return ret;
5464}
5465
5466static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5467{
5468 int val;
6f8d5a6a 5469 u8 index;
3d17b9b5
AD		 5470
5471 if (set)
5472 val = UPIU_QUERY_OPCODE_SET_FLAG;
5473 else
5474 val = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5475
e31011ab 5476 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5477 return ufshcd_query_flag_retry(hba, val,
6f8d5a6a
SC		 5478 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8,
5479 index, NULL);
3d17b9b5
AD		 5480}
5481
d3ba622d 5482static inline int ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
3d17b9b5
AD		 5483{
5484 int ret;
6f8d5a6a 5485 u8 index;
d3ba622d 5486 enum query_opcode opcode;
3d17b9b5 5487
d3ba622d
BH		 5488 if (!ufshcd_is_wb_allowed(hba) ||
5489 hba->dev_info.wb_buf_flush_enabled == enable)
3d17b9b5
AD		 5490 return 0;
5491
d3ba622d
BH		 5492 if (enable)
5493 opcode = UPIU_QUERY_OPCODE_SET_FLAG;
3d17b9b5 5494 else
d3ba622d 5495 opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
3d17b9b5 5496
e31011ab 5497 index = ufshcd_wb_get_query_index(hba);
d3ba622d
BH		 5498 ret = ufshcd_query_flag_retry(hba, opcode,
5499 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN, index,
5500 NULL);
3d17b9b5 5501 if (ret) {
d3ba622d
BH		 5502 dev_err(hba->dev, "%s WB-Buf Flush %s failed %d\n", __func__,
5503 enable ? "enable" : "disable", ret);
5504 goto out;
3d17b9b5
AD		 5505 }
5506
d3ba622d
BH		 5507 hba->dev_info.wb_buf_flush_enabled = enable;
5508
5509 dev_dbg(hba->dev, "WB-Buf Flush %s\n", enable ? "enabled" : "disabled");
5510out:
3d17b9b5 5511 return ret;
d3ba622d 5512
3d17b9b5
AD		 5513}
5514
5515static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5516 u32 avail_buf)
5517{
5518 u32 cur_buf;
5519 int ret;
e31011ab 5520 u8 index;
3d17b9b5 5521
e31011ab 5522 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5523 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5524 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
e31011ab 5525 index, 0, &cur_buf);
3d17b9b5
AD		 5526 if (ret) {
5527 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5528 __func__, ret);
5529 return false;
5530 }
5531
5532 if (!cur_buf) {
5533 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5534 cur_buf);
5535 return false;
5536 }
d14734ae
SC		 5537 /* Let it continue to flush when available buffer exceeds threshold */
5538 if (avail_buf < hba->vps->wb_flush_threshold)
3d17b9b5
AD		 5539 return true;
5540
5541 return false;
5542}
5543
51dd905b 5544static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
3d17b9b5
AD		 5545{
5546 int ret;
5547 u32 avail_buf;
e31011ab 5548 u8 index;
3d17b9b5 5549
79e3520f 5550 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5551 return false;
5552 /*
5553 * The ufs device needs the vcc to be ON to flush.
5554 * With user-space reduction enabled, it's enough to enable flush
5555 * by checking only the available buffer. The threshold
5556 * defined here is > 90% full.
5557 * With user-space preserved enabled, the current-buffer
5558 * should be checked too because the wb buffer size can reduce
5559 * when disk tends to be full. This info is provided by current
5560 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5561 * keeping vcc on when current buffer is empty.
5562 */
e31011ab 5563 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5564 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5565 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
e31011ab 5566 index, 0, &avail_buf);
3d17b9b5
AD		 5567 if (ret) {
5568 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5569 __func__, ret);
5570 return false;
5571 }
5572
5573 if (!hba->dev_info.b_presrv_uspc_en) {
d14734ae 5574 if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
3d17b9b5
AD		 5575 return true;
5576 return false;
5577 }
5578
5579 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5580}
5581
51dd905b
SC		 5582static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5583{
5584 struct ufs_hba *hba = container_of(to_delayed_work(work),
5585 struct ufs_hba,
5586 rpm_dev_flush_recheck_work);
5587 /*
5588 * To prevent unnecessary VCC power drain after device finishes
5589 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5590 * after a certain delay to recheck the threshold by next runtime
5591 * suspend.
5592 */
5593 pm_runtime_get_sync(hba->dev);
5594 pm_runtime_put_sync(hba->dev);
5595}
5596
66ec6d59
SRT		 5597/**
5598 * ufshcd_exception_event_handler - handle exceptions raised by device
5599 * @work: pointer to work data
5600 *
5601 * Read bExceptionEventStatus attribute from the device and handle the
5602 * exception event accordingly.
5603 */
5604static void ufshcd_exception_event_handler(struct work_struct *work)
5605{
5606 struct ufs_hba *hba;
5607 int err;
5608 u32 status = 0;
5609 hba = container_of(work, struct ufs_hba, eeh_work);
5610
62694735 5611 pm_runtime_get_sync(hba->dev);
03e1d28e 5612 ufshcd_scsi_block_requests(hba);
66ec6d59
SRT		 5613 err = ufshcd_get_ee_status(hba, &status);
5614 if (err) {
5615 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5616 __func__, err);
5617 goto out;
5618 }
5619
5620 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5621
5622 if (status & MASK_EE_URGENT_BKOPS)
5623 ufshcd_bkops_exception_event_handler(hba);
5624
66ec6d59 5625out:
03e1d28e 5626 ufshcd_scsi_unblock_requests(hba);
2824ec9f
SL		 5627 /*
5628 * pm_runtime_get_noresume is called while scheduling
5629 * eeh_work to avoid suspend racing with exception work.
5630 * Hence decrement usage counter using pm_runtime_put_noidle
5631 * to allow suspend on completion of exception event handler.
5632 */
5633 pm_runtime_put_noidle(hba->dev);
5634 pm_runtime_put(hba->dev);
66ec6d59
SRT		 5635 return;
5636}
5637
9a47ec7c
YG		 5638/* Complete requests that have door-bell cleared */
5639static void ufshcd_complete_requests(struct ufs_hba *hba)
5640{
5641 ufshcd_transfer_req_compl(hba);
5642 ufshcd_tmc_handler(hba);
5643}
5644
583fa62d
YG		 5645/**
5646 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5647 * to recover from the DL NAC errors or not.
5648 * @hba: per-adapter instance
5649 *
5650 * Returns true if error handling is required, false otherwise
5651 */
5652static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5653{
5654 unsigned long flags;
5655 bool err_handling = true;
5656
5657 spin_lock_irqsave(hba->host->host_lock, flags);
5658 /*
5659 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5660 * device fatal error and/or DL NAC & REPLAY timeout errors.
5661 */
5662 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5663 goto out;
5664
5665 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5666 ((hba->saved_err & UIC_ERROR) &&
5667 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5668 goto out;
5669
5670 if ((hba->saved_err & UIC_ERROR) &&
5671 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5672 int err;
5673 /*
5674 * wait for 50ms to see if we can get any other errors or not.
5675 */
5676 spin_unlock_irqrestore(hba->host->host_lock, flags);
5677 msleep(50);
5678 spin_lock_irqsave(hba->host->host_lock, flags);
5679
5680 /*
5681 * now check if we have got any other severe errors other than
5682 * DL NAC error?
5683 */
5684 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5685 ((hba->saved_err & UIC_ERROR) &&
5686 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5687 goto out;
5688
5689 /*
5690 * As DL NAC is the only error received so far, send out NOP
5691 * command to confirm if link is still active or not.
5692 * - If we don't get any response then do error recovery.
5693 * - If we get response then clear the DL NAC error bit.
5694 */
5695
5696 spin_unlock_irqrestore(hba->host->host_lock, flags);
5697 err = ufshcd_verify_dev_init(hba);
5698 spin_lock_irqsave(hba->host->host_lock, flags);
5699
5700 if (err)
5701 goto out;
5702
5703 /* Link seems to be alive hence ignore the DL NAC errors */
5704 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5705 hba->saved_err &= ~UIC_ERROR;
5706 /* clear NAC error */
5707 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
b0008625 5708 if (!hba->saved_uic_err)
583fa62d 5709 err_handling = false;
583fa62d
YG		 5710 }
5711out:
5712 spin_unlock_irqrestore(hba->host->host_lock, flags);
5713 return err_handling;
5714}
5715
4db7a236
CG		 5716/* host lock must be held before calling this func */
5717static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
5718{
5719 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
5720 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
5721}
5722
5723/* host lock must be held before calling this func */
5724static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
5725{
5726 /* handle fatal errors only when link is not in error state */
5727 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
5586dd8e
CG		 5728 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5729 ufshcd_is_saved_err_fatal(hba))
5730 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
5731 else
5732 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
5733 queue_work(hba->eh_wq, &hba->eh_work);
4db7a236
CG		 5734 }
5735}
5736
348e1bc5
SC		 5737static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
5738{
5739 down_write(&hba->clk_scaling_lock);
5740 hba->clk_scaling.is_allowed = allow;
5741 up_write(&hba->clk_scaling_lock);
5742}
5743
5744static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
5745{
5746 if (suspend) {
5747 if (hba->clk_scaling.is_enabled)
5748 ufshcd_suspend_clkscaling(hba);
5749 ufshcd_clk_scaling_allow(hba, false);
5750 } else {
5751 ufshcd_clk_scaling_allow(hba, true);
5752 if (hba->clk_scaling.is_enabled)
5753 ufshcd_resume_clkscaling(hba);
5754 }
5755}
5756
c72e79c0
CG		 5757static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
5758{
5759 pm_runtime_get_sync(hba->dev);
88a92d6a
CG		 5760 if (pm_runtime_status_suspended(hba->dev) || hba->is_sys_suspended) {
5761 enum ufs_pm_op pm_op;
5762
c72e79c0
CG		 5763 /*
5764 * Don't assume anything of pm_runtime_get_sync(), if
5765 * resume fails, irq and clocks can be OFF, and powers
5766 * can be OFF or in LPM.
5767 */
5768 ufshcd_setup_hba_vreg(hba, true);
5769 ufshcd_enable_irq(hba);
5770 ufshcd_setup_vreg(hba, true);
5771 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5772 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5773 ufshcd_hold(hba, false);
5774 if (!ufshcd_is_clkgating_allowed(hba))
5775 ufshcd_setup_clocks(hba, true);
5776 ufshcd_release(hba);
88a92d6a
CG		 5777 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
5778 ufshcd_vops_resume(hba, pm_op);
c72e79c0
CG		 5779 } else {
5780 ufshcd_hold(hba, false);
348e1bc5
SC		 5781 if (ufshcd_is_clkscaling_supported(hba) &&
5782 hba->clk_scaling.is_enabled)
c72e79c0 5783 ufshcd_suspend_clkscaling(hba);
348e1bc5 5784 ufshcd_clk_scaling_allow(hba, false);
c72e79c0 5785 }
aa53f580
CG		 5786 ufshcd_scsi_block_requests(hba);
5787 /* Drain ufshcd_queuecommand() */
5788 down_write(&hba->clk_scaling_lock);
5789 up_write(&hba->clk_scaling_lock);
5790 cancel_work_sync(&hba->eeh_work);
c72e79c0
CG		 5791}
5792
5793static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
5794{
aa53f580 5795 ufshcd_scsi_unblock_requests(hba);
c72e79c0 5796 ufshcd_release(hba);
348e1bc5
SC		 5797 if (ufshcd_is_clkscaling_supported(hba))
5798 ufshcd_clk_scaling_suspend(hba, false);
aa53f580 5799 ufshcd_clear_ua_wluns(hba);
c72e79c0
CG		 5800 pm_runtime_put(hba->dev);
5801}
5802
5803static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
5804{
9cd20d3f
CG		 5805 return (!hba->is_powered || hba->shutting_down ||
5806 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
c72e79c0 5807 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
9cd20d3f 5808 ufshcd_is_link_broken(hba))));
c72e79c0
CG		 5809}
5810
5811#ifdef CONFIG_PM
5812static void ufshcd_recover_pm_error(struct ufs_hba *hba)
5813{
5814 struct Scsi_Host *shost = hba->host;
5815 struct scsi_device *sdev;
5816 struct request_queue *q;
5817 int ret;
5818
88a92d6a 5819 hba->is_sys_suspended = false;
c72e79c0
CG		 5820 /*
5821 * Set RPM status of hba device to RPM_ACTIVE,
5822 * this also clears its runtime error.
5823 */
5824 ret = pm_runtime_set_active(hba->dev);
5825 /*
5826 * If hba device had runtime error, we also need to resume those
5827 * scsi devices under hba in case any of them has failed to be
5828 * resumed due to hba runtime resume failure. This is to unblock
5829 * blk_queue_enter in case there are bios waiting inside it.
5830 */
5831 if (!ret) {
5832 shost_for_each_device(sdev, shost) {
5833 q = sdev->request_queue;
5834 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
5835 q->rpm_status == RPM_SUSPENDING))
5836 pm_request_resume(q->dev);
5837 }
5838 }
5839}
5840#else
5841static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
5842{
5843}
5844#endif
5845
2355b66e
CG		 5846static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
5847{
5848 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
5849 u32 mode;
5850
5851 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
5852
5853 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
5854 return true;
5855
5856 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
5857 return true;
5858
5859 return false;
5860}
5861
7a3e97b0 5862/**
e8e7f271
SRT		 5863 * ufshcd_err_handler - handle UFS errors that require s/w attention
5864 * @work: pointer to work structure
7a3e97b0 5865 */
e8e7f271 5866static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5867{
5868 struct ufs_hba *hba;
e8e7f271 5869 unsigned long flags;
307348f6
CG		 5870 bool err_xfer = false;
5871 bool err_tm = false;
2355b66e 5872 int err = 0, pmc_err;
e8e7f271 5873 int tag;
2355b66e 5874 bool needs_reset = false, needs_restore = false;
e8e7f271
SRT		 5875
5876 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5877
9cd20d3f 5878 down(&hba->host_sem);
e8e7f271 5879 spin_lock_irqsave(hba->host->host_lock, flags);
c72e79c0 5880 if (ufshcd_err_handling_should_stop(hba)) {
4db7a236
CG		 5881 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5882 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5883 spin_unlock_irqrestore(hba->host->host_lock, flags);
9cd20d3f 5884 up(&hba->host_sem);
4db7a236
CG		 5885 return;
5886 }
5887 ufshcd_set_eh_in_progress(hba);
5888 spin_unlock_irqrestore(hba->host->host_lock, flags);
c72e79c0 5889 ufshcd_err_handling_prepare(hba);
e8e7f271 5890 spin_lock_irqsave(hba->host->host_lock, flags);
aa53f580
CG		 5891 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5892 hba->ufshcd_state = UFSHCD_STATE_RESET;
e8e7f271
SRT		 5893
5894 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5895 ufshcd_complete_requests(hba);
583fa62d 5896
88a92d6a
CG		 5897 /*
5898 * A full reset and restore might have happened after preparation
5899 * is finished, double check whether we should stop.
5900 */
5901 if (ufshcd_err_handling_should_stop(hba))
5902 goto skip_err_handling;
5903
583fa62d
YG		 5904 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5905 bool ret;
5906
5907 spin_unlock_irqrestore(hba->host->host_lock, flags);
5908 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5909 ret = ufshcd_quirk_dl_nac_errors(hba);
5910 spin_lock_irqsave(hba->host->host_lock, flags);
88a92d6a 5911 if (!ret && ufshcd_err_handling_should_stop(hba))
583fa62d
YG		 5912 goto skip_err_handling;
5913 }
4db7a236 5914
2355b66e
CG		 5915 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
5916 (hba->saved_uic_err &&
5917 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
c3be8d1e
CG		 5918 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
5919
5920 spin_unlock_irqrestore(hba->host->host_lock, flags);
5921 ufshcd_print_host_state(hba);
5922 ufshcd_print_pwr_info(hba);
e965e5e0 5923 ufshcd_print_evt_hist(hba);
c3be8d1e
CG		 5924 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5925 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
5926 spin_lock_irqsave(hba->host->host_lock, flags);
5927 }
5928
9a47ec7c
YG		 5929 /*
5930 * if host reset is required then skip clearing the pending
2df74b69
CG		 5931 * transfers forcefully because they will get cleared during
5932 * host reset and restore
9a47ec7c 5933 */
88a92d6a
CG		 5934 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5935 ufshcd_is_saved_err_fatal(hba) ||
5936 ((hba->saved_err & UIC_ERROR) &&
5937 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5938 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
5939 needs_reset = true;
2355b66e 5940 goto do_reset;
88a92d6a 5941 }
9a47ec7c 5942
2355b66e
CG		 5943 /*
5944 * If LINERESET was caught, UFS might have been put to PWM mode,
5945 * check if power mode restore is needed.
5946 */
5947 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
5948 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5949 if (!hba->saved_uic_err)
5950 hba->saved_err &= ~UIC_ERROR;
5951 spin_unlock_irqrestore(hba->host->host_lock, flags);
5952 if (ufshcd_is_pwr_mode_restore_needed(hba))
5953 needs_restore = true;
5954 spin_lock_irqsave(hba->host->host_lock, flags);
5955 if (!hba->saved_err && !needs_restore)
5956 goto skip_err_handling;
5957 }
9a47ec7c 5958
2355b66e 5959 hba->silence_err_logs = true;
9a47ec7c
YG		 5960 /* release lock as clear command might sleep */
5961 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5962 /* Clear pending transfer requests */
9a47ec7c 5963 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
307348f6 5964 if (ufshcd_try_to_abort_task(hba, tag)) {
9a47ec7c
YG		 5965 err_xfer = true;
5966 goto lock_skip_pending_xfer_clear;
5967 }
5968 }
e8e7f271
SRT		 5969
5970 /* Clear pending task management requests */
9a47ec7c
YG		 5971 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5972 if (ufshcd_clear_tm_cmd(hba, tag)) {
5973 err_tm = true;
5974 goto lock_skip_pending_xfer_clear;
5975 }
5976 }
e8e7f271 5977
9a47ec7c 5978lock_skip_pending_xfer_clear:
e8e7f271 5979 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5980
9a47ec7c
YG		 5981 /* Complete the requests that are cleared by s/w */
5982 ufshcd_complete_requests(hba);
2355b66e 5983 hba->silence_err_logs = false;
9a47ec7c 5984
2355b66e 5985 if (err_xfer || err_tm) {
9a47ec7c 5986 needs_reset = true;
2355b66e
CG		 5987 goto do_reset;
5988 }
9a47ec7c 5989
2355b66e
CG		 5990 /*
5991 * After all reqs and tasks are cleared from doorbell,
5992 * now it is safe to retore power mode.
5993 */
5994 if (needs_restore) {
5995 spin_unlock_irqrestore(hba->host->host_lock, flags);
5996 /*
5997 * Hold the scaling lock just in case dev cmds
5998 * are sent via bsg and/or sysfs.
5999 */
6000 down_write(&hba->clk_scaling_lock);
6001 hba->force_pmc = true;
6002 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6003 if (pmc_err) {
6004 needs_reset = true;
6005 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6006 __func__, pmc_err);
6007 }
6008 hba->force_pmc = false;
6009 ufshcd_print_pwr_info(hba);
6010 up_write(&hba->clk_scaling_lock);
6011 spin_lock_irqsave(hba->host->host_lock, flags);
6012 }
9a47ec7c 6013
2355b66e 6014do_reset:
e8e7f271 6015 /* Fatal errors need reset */
9a47ec7c
YG		 6016 if (needs_reset) {
6017 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
6018
6019 /*
6020 * ufshcd_reset_and_restore() does the link reinitialization
6021 * which will need atleast one empty doorbell slot to send the
6022 * device management commands (NOP and query commands).
6023 * If there is no slot empty at this moment then free up last
6024 * slot forcefully.
6025 */
6026 if (hba->outstanding_reqs == max_doorbells)
6027 __ufshcd_transfer_req_compl(hba,
6028 (1UL << (hba->nutrs - 1)));
6029
4db7a236 6030 hba->force_reset = false;
9a47ec7c 6031 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 6032 err = ufshcd_reset_and_restore(hba);
4db7a236
CG		 6033 if (err)
6034 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6035 __func__, err);
c72e79c0
CG		 6036 else
6037 ufshcd_recover_pm_error(hba);
9a47ec7c 6038 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 6039 }
9a47ec7c 6040
583fa62d 6041skip_err_handling:
9a47ec7c 6042 if (!needs_reset) {
4db7a236
CG		 6043 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6044 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
9a47ec7c
YG		 6045 if (hba->saved_err || hba->saved_uic_err)
6046 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6047 __func__, hba->saved_err, hba->saved_uic_err);
6048 }
e8e7f271 6049 ufshcd_clear_eh_in_progress(hba);
9a47ec7c 6050 spin_unlock_irqrestore(hba->host->host_lock, flags);
c72e79c0 6051 ufshcd_err_handling_unprepare(hba);
9cd20d3f 6052 up(&hba->host_sem);
7a3e97b0
SY		 6053}
6054
6055/**
e8e7f271
SRT		 6056 * ufshcd_update_uic_error - check and set fatal UIC error flags.
6057 * @hba: per-adapter instance
9333d775
VG		 6058 *
6059 * Returns
6060 * IRQ_HANDLED - If interrupt is valid
6061 * IRQ_NONE - If invalid interrupt
7a3e97b0 6062 */
9333d775 6063static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 6064{
6065 u32 reg;
9333d775 6066 irqreturn_t retval = IRQ_NONE;
7a3e97b0 6067
2355b66e 6068 /* PHY layer error */
fb7b45f0 6069 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
fb7b45f0 6070 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
2355b66e 6071 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6072 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
fb7b45f0
DR		 6073 /*
6074 * To know whether this error is fatal or not, DB timeout
6075 * must be checked but this error is handled separately.
6076 */
2355b66e
CG		 6077 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6078 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6079 __func__);
6080
6081 /* Got a LINERESET indication. */
6082 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6083 struct uic_command *cmd = NULL;
6084
6085 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6086 if (hba->uic_async_done && hba->active_uic_cmd)
6087 cmd = hba->active_uic_cmd;
6088 /*
6089 * Ignore the LINERESET during power mode change
6090 * operation via DME_SET command.
6091 */
6092 if (cmd && (cmd->command == UIC_CMD_DME_SET))
6093 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6094 }
9333d775 6095 retval |= IRQ_HANDLED;
ff8e20c6 6096 }
fb7b45f0 6097
e8e7f271
SRT		 6098 /* PA_INIT_ERROR is fatal and needs UIC reset */
6099 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 6100 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6101 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6102 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
ff8e20c6 6103
9333d775
VG		 6104 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6105 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6106 else if (hba->dev_quirks &
6107 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6108 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6109 hba->uic_error |=
6110 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6111 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6112 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6113 }
6114 retval |= IRQ_HANDLED;
583fa62d 6115 }
e8e7f271
SRT		 6116
6117 /* UIC NL/TL/DME errors needs software retry */
6118 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 6119 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6120 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6121 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
e8e7f271 6122 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 6123 retval |= IRQ_HANDLED;
ff8e20c6 6124 }
e8e7f271
SRT		 6125
6126 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 6127 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6128 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6129 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
e8e7f271 6130 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 6131 retval |= IRQ_HANDLED;
ff8e20c6 6132 }
e8e7f271
SRT		 6133
6134 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 6135 if ((reg & UIC_DME_ERROR) &&
6136 (reg & UIC_DME_ERROR_CODE_MASK)) {
e965e5e0 6137 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
e8e7f271 6138 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 6139 retval |= IRQ_HANDLED;
ff8e20c6 6140 }
e8e7f271
SRT		 6141
6142 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6143 __func__, hba->uic_error);
9333d775 6144 return retval;
e8e7f271
SRT		 6145}
6146
82174440
SC		 6147static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
6148 u32 intr_mask)
6149{
5a244e0e
SC		 6150 if (!ufshcd_is_auto_hibern8_supported(hba) ||
6151 !ufshcd_is_auto_hibern8_enabled(hba))
82174440
SC		 6152 return false;
6153
6154 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
6155 return false;
6156
6157 if (hba->active_uic_cmd &&
6158 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
6159 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
6160 return false;
6161
6162 return true;
6163}
6164
e8e7f271
SRT		 6165/**
6166 * ufshcd_check_errors - Check for errors that need s/w attention
6167 * @hba: per-adapter instance
9333d775
VG		 6168 *
6169 * Returns
6170 * IRQ_HANDLED - If interrupt is valid
6171 * IRQ_NONE - If invalid interrupt
e8e7f271 6172 */
9333d775 6173static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
e8e7f271
SRT		 6174{
6175 bool queue_eh_work = false;
9333d775 6176 irqreturn_t retval = IRQ_NONE;
e8e7f271 6177
d3c615bf 6178 if (hba->errors & INT_FATAL_ERRORS) {
e965e5e0
SC		 6179 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6180 hba->errors);
e8e7f271 6181 queue_eh_work = true;
d3c615bf 6182 }
7a3e97b0
SY		 6183
6184 if (hba->errors & UIC_ERROR) {
e8e7f271 6185 hba->uic_error = 0;
9333d775 6186 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 6187 if (hba->uic_error)
6188 queue_eh_work = true;
7a3e97b0 6189 }
e8e7f271 6190
82174440
SC		 6191 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6192 dev_err(hba->dev,
6193 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6194 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6195 "Enter" : "Exit",
6196 hba->errors, ufshcd_get_upmcrs(hba));
e965e5e0 6197 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
d3c615bf 6198 hba->errors);
4db7a236 6199 ufshcd_set_link_broken(hba);
82174440
SC		 6200 queue_eh_work = true;
6201 }
6202
e8e7f271 6203 if (queue_eh_work) {
9a47ec7c
YG		 6204 /*
6205 * update the transfer error masks to sticky bits, let's do this
6206 * irrespective of current ufshcd_state.
6207 */
6208 hba->saved_err |= hba->errors;
6209 hba->saved_uic_err |= hba->uic_error;
6210
4db7a236 6211 /* dump controller state before resetting */
ace3804b
CG		 6212 if ((hba->saved_err &
6213 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
2355b66e
CG		 6214 (hba->saved_uic_err &&
6215 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
4db7a236 6216 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
66cc820f
DR		 6217 __func__, hba->saved_err,
6218 hba->saved_uic_err);
c3be8d1e
CG		 6219 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6220 "host_regs: ");
4db7a236 6221 ufshcd_print_pwr_info(hba);
e8e7f271 6222 }
4db7a236 6223 ufshcd_schedule_eh_work(hba);
9333d775 6224 retval |= IRQ_HANDLED;
3441da7d 6225 }
e8e7f271
SRT		 6226 /*
6227 * if (!queue_eh_work) -
6228 * Other errors are either non-fatal where host recovers
6229 * itself without s/w intervention or errors that will be
6230 * handled by the SCSI core layer.
6231 */
9333d775 6232 return retval;
7a3e97b0
SY		 6233}
6234
69a6c269
BVA		 6235struct ctm_info {
6236 struct ufs_hba *hba;
6237 unsigned long pending;
6238 unsigned int ncpl;
6239};
6240
6241static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
6242{
6243 struct ctm_info *const ci = priv;
6244 struct completion *c;
6245
6246 WARN_ON_ONCE(reserved);
6247 if (test_bit(req->tag, &ci->pending))
6248 return true;
6249 ci->ncpl++;
6250 c = req->end_io_data;
6251 if (c)
6252 complete(c);
6253 return true;
6254}
6255
7a3e97b0
SY		 6256/**
6257 * ufshcd_tmc_handler - handle task management function completion
6258 * @hba: per adapter instance
9333d775
VG		 6259 *
6260 * Returns
6261 * IRQ_HANDLED - If interrupt is valid
6262 * IRQ_NONE - If invalid interrupt
7a3e97b0 6263 */
9333d775 6264static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 6265{
69a6c269
BVA		 6266 struct request_queue *q = hba->tmf_queue;
6267 struct ctm_info ci = {
6268 .hba = hba,
6269 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
6270 };
7a3e97b0 6271
69a6c269
BVA		 6272 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
6273 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
7a3e97b0
SY		 6274}
6275
6276/**
6277 * ufshcd_sl_intr - Interrupt service routine
6278 * @hba: per adapter instance
6279 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 6280 *
6281 * Returns
6282 * IRQ_HANDLED - If interrupt is valid
6283 * IRQ_NONE - If invalid interrupt
7a3e97b0 6284 */
9333d775 6285static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 6286{
9333d775
VG		 6287 irqreturn_t retval = IRQ_NONE;
6288
7a3e97b0 6289 hba->errors = UFSHCD_ERROR_MASK & intr_status;
82174440
SC		 6290
6291 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
6292 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
6293
7a3e97b0 6294 if (hba->errors)
9333d775 6295 retval |= ufshcd_check_errors(hba);
7a3e97b0 6296
53b3d9c3 6297 if (intr_status & UFSHCD_UIC_MASK)
9333d775 6298 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 6299
6300 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 6301 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 6302
6303 if (intr_status & UTP_TRANSFER_REQ_COMPL)
9333d775
VG		 6304 retval |= ufshcd_transfer_req_compl(hba);
6305
6306 return retval;
7a3e97b0
SY		 6307}
6308
6309/**
6310 * ufshcd_intr - Main interrupt service routine
6311 * @irq: irq number
6312 * @__hba: pointer to adapter instance
6313 *
9333d775
VG		 6314 * Returns
6315 * IRQ_HANDLED - If interrupt is valid
6316 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 6317 */
6318static irqreturn_t ufshcd_intr(int irq, void *__hba)
6319{
127d5f7c 6320 u32 intr_status, enabled_intr_status = 0;
7a3e97b0
SY		 6321 irqreturn_t retval = IRQ_NONE;
6322 struct ufs_hba *hba = __hba;
7f6ba4f1 6323 int retries = hba->nutrs;
7a3e97b0
SY		 6324
6325 spin_lock(hba->host->host_lock);
b873a275 6326 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
3f8af604
CG		 6327 hba->ufs_stats.last_intr_status = intr_status;
6328 hba->ufs_stats.last_intr_ts = ktime_get();
7a3e97b0 6329
7f6ba4f1
VG		 6330 /*
6331 * There could be max of hba->nutrs reqs in flight and in worst case
6332 * if the reqs get finished 1 by 1 after the interrupt status is
6333 * read, make sure we handle them by checking the interrupt status
6334 * again in a loop until we process all of the reqs before returning.
6335 */
127d5f7c 6336 while (intr_status && retries--) {
7f6ba4f1
VG		 6337 enabled_intr_status =
6338 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
60ec3755 6339 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 6340 if (enabled_intr_status)
6341 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 6342
6343 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
127d5f7c 6344 }
d75f7fe4 6345
eeb1b55b
JK		 6346 if (enabled_intr_status && retval == IRQ_NONE &&
6347 !ufshcd_eh_in_progress(hba)) {
6348 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6349 __func__,
6350 intr_status,
6351 hba->ufs_stats.last_intr_status,
6352 enabled_intr_status);
9333d775
VG		 6353 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6354 }
6355
7a3e97b0
SY		 6356 spin_unlock(hba->host->host_lock);
6357 return retval;
6358}
6359
e2933132
SRT		 6360static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6361{
6362 int err = 0;
6363 u32 mask = 1 << tag;
6364 unsigned long flags;
6365
6366 if (!test_bit(tag, &hba->outstanding_tasks))
6367 goto out;
6368
6369 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 6370 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 6371 spin_unlock_irqrestore(hba->host->host_lock, flags);
6372
6373 /* poll for max. 1 sec to clear door bell register by h/w */
6374 err = ufshcd_wait_for_register(hba,
6375 REG_UTP_TASK_REQ_DOOR_BELL,
5cac1095 6376 mask, 0, 1000, 1000);
e2933132
SRT		 6377out:
6378 return err;
6379}
6380
c6049cd9
CH		 6381static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6382 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 6383{
69a6c269 6384 struct request_queue *q = hba->tmf_queue;
c6049cd9 6385 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 6386 DECLARE_COMPLETION_ONSTACK(wait);
6387 struct request *req;
7a3e97b0 6388 unsigned long flags;
c6049cd9 6389 int free_slot, task_tag, err;
7a3e97b0 6390
e2933132
SRT		 6391 /*
6392 * Get free slot, sleep if slots are unavailable.
6393 * Even though we use wait_event() which sleeps indefinitely,
6394 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
6395 */
eeb1b55b
JK		 6396 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
6397 if (IS_ERR(req))
6398 return PTR_ERR(req);
6399
69a6c269
BVA		 6400 req->end_io_data = &wait;
6401 free_slot = req->tag;
6402 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
1ab27c9c 6403 ufshcd_hold(hba, false);
7a3e97b0 6404
e2933132 6405 spin_lock_irqsave(host->host_lock, flags);
e2933132 6406 task_tag = hba->nutrs + free_slot;
7a3e97b0 6407
c6049cd9
CH		 6408 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
6409
6410 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 6411 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
6412
7a3e97b0
SY		 6413 /* send command to the controller */
6414 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 6415
6416 /* Make sure descriptors are ready before ringing the task doorbell */
6417 wmb();
6418
b873a275 6419 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 6420 /* Make sure that doorbell is committed immediately */
6421 wmb();
7a3e97b0
SY		 6422
6423 spin_unlock_irqrestore(host->host_lock, flags);
6424
28fa68fc 6425 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
6667e6d9 6426
7a3e97b0 6427 /* wait until the task management command is completed */
69a6c269 6428 err = wait_for_completion_io_timeout(&wait,
e2933132 6429 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 6430 if (!err) {
69a6c269
BVA		 6431 /*
6432 * Make sure that ufshcd_compl_tm() does not trigger a
6433 * use-after-free.
6434 */
6435 req->end_io_data = NULL;
28fa68fc 6436 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
e2933132
SRT		 6437 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6438 __func__, tm_function);
6439 if (ufshcd_clear_tm_cmd(hba, free_slot))
6440 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
6441 __func__, free_slot);
6442 err = -ETIMEDOUT;
6443 } else {
c6049cd9
CH		 6444 err = 0;
6445 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
6446
28fa68fc 6447 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
7a3e97b0 6448 }
e2933132 6449
b557217c
SC		 6450 spin_lock_irqsave(hba->host->host_lock, flags);
6451 __clear_bit(free_slot, &hba->outstanding_tasks);
6452 spin_unlock_irqrestore(hba->host->host_lock, flags);
6453
69a6c269 6454 blk_put_request(req);
e2933132 6455
1ab27c9c 6456 ufshcd_release(hba);
7a3e97b0
SY		 6457 return err;
6458}
6459
c6049cd9
CH		 6460/**
6461 * ufshcd_issue_tm_cmd - issues task management commands to controller
6462 * @hba: per adapter instance
6463 * @lun_id: LUN ID to which TM command is sent
6464 * @task_id: task ID to which the TM command is applicable
6465 * @tm_function: task management function opcode
6466 * @tm_response: task management service response return value
6467 *
6468 * Returns non-zero value on error, zero on success.
6469 */
6470static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6471 u8 tm_function, u8 *tm_response)
6472{
6473 struct utp_task_req_desc treq = { { 0 }, };
6474 int ocs_value, err;
6475
6476 /* Configure task request descriptor */
6477 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6478 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6479
6480 /* Configure task request UPIU */
6481 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6482 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6483 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6484
6485 /*
6486 * The host shall provide the same value for LUN field in the basic
6487 * header and for Input Parameter.
6488 */
6489 treq.input_param1 = cpu_to_be32(lun_id);
6490 treq.input_param2 = cpu_to_be32(task_id);
6491
6492 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6493 if (err == -ETIMEDOUT)
6494 return err;
6495
6496 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6497 if (ocs_value != OCS_SUCCESS)
6498 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6499 __func__, ocs_value);
6500 else if (tm_response)
6501 *tm_response = be32_to_cpu(treq.output_param1) &
6502 MASK_TM_SERVICE_RESP;
6503 return err;
6504}
6505
5e0a86ee
AA		 6506/**
6507 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6508 * @hba: per-adapter instance
6509 * @req_upiu: upiu request
6510 * @rsp_upiu: upiu reply
5e0a86ee
AA		 6511 * @desc_buff: pointer to descriptor buffer, NULL if NA
6512 * @buff_len: descriptor size, 0 if NA
d0e9760d 6513 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 6514 * @desc_op: descriptor operation
6515 *
6516 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6517 * Therefore, it "rides" the device management infrastructure: uses its tag and
6518 * tasks work queues.
6519 *
6520 * Since there is only one available tag for device management commands,
6521 * the caller is expected to hold the hba->dev_cmd.lock mutex.
6522 */
6523static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6524 struct utp_upiu_req *req_upiu,
6525 struct utp_upiu_req *rsp_upiu,
6526 u8 *desc_buff, int *buff_len,
7f674c38 6527 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 6528 enum query_opcode desc_op)
6529{
7252a360
BVA		 6530 struct request_queue *q = hba->cmd_queue;
6531 struct request *req;
5e0a86ee
AA		 6532 struct ufshcd_lrb *lrbp;
6533 int err = 0;
6534 int tag;
6535 struct completion wait;
6536 unsigned long flags;
a23064c4 6537 u8 upiu_flags;
5e0a86ee
AA		 6538
6539 down_read(&hba->clk_scaling_lock);
6540
7252a360 6541 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 6542 if (IS_ERR(req)) {
6543 err = PTR_ERR(req);
6544 goto out_unlock;
6545 }
7252a360
BVA		 6546 tag = req->tag;
6547 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5e0a86ee
AA		 6548
6549 init_completion(&wait);
6550 lrbp = &hba->lrb[tag];
7a7e66c6
CG		 6551 if (unlikely(lrbp->in_use)) {
6552 err = -EBUSY;
6553 goto out;
6554 }
5e0a86ee 6555
7a7e66c6 6556 WARN_ON(lrbp->cmd);
5e0a86ee
AA		 6557 lrbp->cmd = NULL;
6558 lrbp->sense_bufflen = 0;
6559 lrbp->sense_buffer = NULL;
6560 lrbp->task_tag = tag;
6561 lrbp->lun = 0;
6562 lrbp->intr_cmd = true;
df043c74 6563 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5e0a86ee
AA		 6564 hba->dev_cmd.type = cmd_type;
6565
6566 switch (hba->ufs_version) {
6567 case UFSHCI_VERSION_10:
6568 case UFSHCI_VERSION_11:
6569 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6570 break;
6571 default:
6572 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6573 break;
6574 }
6575
6576 /* update the task tag in the request upiu */
6577 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6578
6579 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6580
6581 /* just copy the upiu request as it is */
6582 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6583 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6584 /* The Data Segment Area is optional depending upon the query
6585 * function value. for WRITE DESCRIPTOR, the data segment
6586 * follows right after the tsf.
6587 */
6588 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6589 *buff_len = 0;
6590 }
6591
6592 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6593
6594 hba->dev_cmd.complete = &wait;
6595
6596 /* Make sure descriptors are ready before ringing the doorbell */
6597 wmb();
6598 spin_lock_irqsave(hba->host->host_lock, flags);
6599 ufshcd_send_command(hba, tag);
6600 spin_unlock_irqrestore(hba->host->host_lock, flags);
6601
6602 /*
6603 * ignore the returning value here - ufshcd_check_query_response is
6604 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6605 * read the response directly ignoring all errors.
6606 */
6607 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6608
6609 /* just copy the upiu response as it is */
6610 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 6611 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6612 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6613 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6614 MASK_QUERY_DATA_SEG_LEN;
6615
6616 if (*buff_len >= resp_len) {
6617 memcpy(desc_buff, descp, resp_len);
6618 *buff_len = resp_len;
6619 } else {
3d4881d1
BH		 6620 dev_warn(hba->dev,
6621 "%s: rsp size %d is bigger than buffer size %d",
6622 __func__, resp_len, *buff_len);
4bbbe242
AA		 6623 *buff_len = 0;
6624 err = -EINVAL;
6625 }
6626 }
5e0a86ee 6627
7a7e66c6 6628out:
7252a360 6629 blk_put_request(req);
bb14dd15 6630out_unlock:
5e0a86ee
AA		 6631 up_read(&hba->clk_scaling_lock);
6632 return err;
6633}
6634
6635/**
6636 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6637 * @hba: per-adapter instance
6638 * @req_upiu: upiu request
6639 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
6640 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
6641 * @desc_buff: pointer to descriptor buffer, NULL if NA
6642 * @buff_len: descriptor size, 0 if NA
6643 * @desc_op: descriptor operation
6644 *
6645 * Supports UTP Transfer requests (nop and query), and UTP Task
6646 * Management requests.
6647 * It is up to the caller to fill the upiu conent properly, as it will
6648 * be copied without any further input validations.
6649 */
6650int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6651 struct utp_upiu_req *req_upiu,
6652 struct utp_upiu_req *rsp_upiu,
6653 int msgcode,
6654 u8 *desc_buff, int *buff_len,
6655 enum query_opcode desc_op)
6656{
6657 int err;
7f674c38 6658 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee
AA		 6659 struct utp_task_req_desc treq = { { 0 }, };
6660 int ocs_value;
6661 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6662
5e0a86ee
AA		 6663 switch (msgcode) {
6664 case UPIU_TRANSACTION_NOP_OUT:
6665 cmd_type = DEV_CMD_TYPE_NOP;
df561f66 6666 fallthrough;
5e0a86ee
AA		 6667 case UPIU_TRANSACTION_QUERY_REQ:
6668 ufshcd_hold(hba, false);
6669 mutex_lock(&hba->dev_cmd.lock);
6670 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6671 desc_buff, buff_len,
6672 cmd_type, desc_op);
6673 mutex_unlock(&hba->dev_cmd.lock);
6674 ufshcd_release(hba);
6675
6676 break;
6677 case UPIU_TRANSACTION_TASK_REQ:
6678 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6679 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6680
6681 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6682
6683 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6684 if (err == -ETIMEDOUT)
6685 break;
6686
6687 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6688 if (ocs_value != OCS_SUCCESS) {
6689 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6690 ocs_value);
6691 break;
6692 }
6693
6694 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6695
6696 break;
6697 default:
6698 err = -EINVAL;
6699
6700 break;
6701 }
6702
5e0a86ee
AA		 6703 return err;
6704}
6705
7a3e97b0 6706/**
3441da7d
SRT		 6707 * ufshcd_eh_device_reset_handler - device reset handler registered to
6708 * scsi layer.
7a3e97b0
SY		 6709 * @cmd: SCSI command pointer
6710 *
6711 * Returns SUCCESS/FAILED
6712 */
3441da7d 6713static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6714{
6715 struct Scsi_Host *host;
6716 struct ufs_hba *hba;
7a3e97b0
SY		 6717 u32 pos;
6718 int err;
35fc4cd3 6719 u8 resp = 0xF, lun;
3441da7d 6720 unsigned long flags;
7a3e97b0
SY		 6721
6722 host = cmd->device->host;
6723 hba = shost_priv(host);
7a3e97b0 6724
35fc4cd3
CG		 6725 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
6726 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
e2933132 6727 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6728 if (!err)
6729 err = resp;
7a3e97b0 6730 goto out;
e2933132 6731 }
7a3e97b0 6732
3441da7d
SRT		 6733 /* clear the commands that were pending for corresponding LUN */
6734 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
35fc4cd3 6735 if (hba->lrb[pos].lun == lun) {
3441da7d
SRT		 6736 err = ufshcd_clear_cmd(hba, pos);
6737 if (err)
6738 break;
7a3e97b0 6739 }
3441da7d
SRT		 6740 }
6741 spin_lock_irqsave(host->host_lock, flags);
6742 ufshcd_transfer_req_compl(hba);
6743 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 6744
7a3e97b0 6745out:
7fabb77b 6746 hba->req_abort_count = 0;
e965e5e0 6747 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
3441da7d
SRT		 6748 if (!err) {
6749 err = SUCCESS;
6750 } else {
6751 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6752 err = FAILED;
6753 }
7a3e97b0
SY		 6754 return err;
6755}
6756
e0b299e3
GB		 6757static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6758{
6759 struct ufshcd_lrb *lrbp;
6760 int tag;
6761
6762 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6763 lrbp = &hba->lrb[tag];
6764 lrbp->req_abort_skip = true;
6765 }
6766}
6767
7a3e97b0 6768/**
307348f6 6769 * ufshcd_try_to_abort_task - abort a specific task
d23ec0b6
LJ		 6770 * @hba: Pointer to adapter instance
6771 * @tag: Task tag/index to be aborted
7a3e97b0 6772 *
f20810d8
SRT		 6773 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6774 * command, and in host controller by clearing the door-bell register. There can
6775 * be race between controller sending the command to the device while abort is
6776 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6777 * really issued and then try to abort it.
6778 *
307348f6
CG		 6779 * Returns zero on success, non-zero on failure
6780 */
6781static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
6782{
6783 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6784 int err = 0;
6785 int poll_cnt;
6786 u8 resp = 0xF;
6787 u32 reg;
6788
6789 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6790 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6791 UFS_QUERY_TASK, &resp);
6792 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6793 /* cmd pending in the device */
6794 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6795 __func__, tag);
6796 break;
6797 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6798 /*
6799 * cmd not pending in the device, check if it is
6800 * in transition.
6801 */
6802 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6803 __func__, tag);
6804 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6805 if (reg & (1 << tag)) {
6806 /* sleep for max. 200us to stabilize */
6807 usleep_range(100, 200);
6808 continue;
6809 }
6810 /* command completed already */
6811 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6812 __func__, tag);
6813 goto out;
6814 } else {
6815 dev_err(hba->dev,
6816 "%s: no response from device. tag = %d, err %d\n",
6817 __func__, tag, err);
6818 if (!err)
6819 err = resp; /* service response error */
6820 goto out;
6821 }
6822 }
6823
6824 if (!poll_cnt) {
6825 err = -EBUSY;
6826 goto out;
6827 }
6828
6829 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6830 UFS_ABORT_TASK, &resp);
6831 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6832 if (!err) {
6833 err = resp; /* service response error */
6834 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6835 __func__, tag, err);
6836 }
6837 goto out;
6838 }
6839
6840 err = ufshcd_clear_cmd(hba, tag);
6841 if (err)
6842 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6843 __func__, tag, err);
6844
6845out:
6846 return err;
6847}
6848
6849/**
6850 * ufshcd_abort - scsi host template eh_abort_handler callback
6851 * @cmd: SCSI command pointer
6852 *
7a3e97b0
SY		 6853 * Returns SUCCESS/FAILED
6854 */
6855static int ufshcd_abort(struct scsi_cmnd *cmd)
6856{
6857 struct Scsi_Host *host;
6858 struct ufs_hba *hba;
6859 unsigned long flags;
6860 unsigned int tag;
f20810d8 6861 int err = 0;
e2933132 6862 struct ufshcd_lrb *lrbp;
e9d501b1 6863 u32 reg;
7a3e97b0
SY		 6864
6865 host = cmd->device->host;
6866 hba = shost_priv(host);
6867 tag = cmd->request->tag;
e7d38257 6868 lrbp = &hba->lrb[tag];
14497328
YG		 6869 if (!ufshcd_valid_tag(hba, tag)) {
6870 dev_err(hba->dev,
6871 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6872 __func__, tag, cmd, cmd->request);
6873 BUG();
6874 }
7a3e97b0 6875
1ab27c9c 6876 ufshcd_hold(hba, false);
14497328 6877 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 6878 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 6879 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6880 dev_err(hba->dev,
6881 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6882 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 6883 goto out;
14497328 6884 }
7a3e97b0 6885
66cc820f 6886 /* Print Transfer Request of aborted task */
d87a1f6d 6887 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 6888
7fabb77b
GB		 6889 /*
6890 * Print detailed info about aborted request.
6891 * As more than one request might get aborted at the same time,
6892 * print full information only for the first aborted request in order
6893 * to reduce repeated printouts. For other aborted requests only print
6894 * basic details.
6895 */
7a7e66c6 6896 scsi_print_command(cmd);
7fabb77b 6897 if (!hba->req_abort_count) {
e965e5e0
SC		 6898 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
6899 ufshcd_print_evt_hist(hba);
6ba65588 6900 ufshcd_print_host_state(hba);
7fabb77b
GB		 6901 ufshcd_print_pwr_info(hba);
6902 ufshcd_print_trs(hba, 1 << tag, true);
6903 } else {
6904 ufshcd_print_trs(hba, 1 << tag, false);
6905 }
6906 hba->req_abort_count++;
e0b299e3 6907
d87a1f6d
BH		 6908 if (!(reg & (1 << tag))) {
6909 dev_err(hba->dev,
6910 "%s: cmd was completed, but without a notifying intr, tag = %d",
6911 __func__, tag);
6912 goto cleanup;
6913 }
6914
7a7e66c6
CG		 6915 /*
6916 * Task abort to the device W-LUN is illegal. When this command
6917 * will fail, due to spec violation, scsi err handling next step
6918 * will be to send LU reset which, again, is a spec violation.
6919 * To avoid these unnecessary/illegal steps, first we clean up
6920 * the lrb taken by this cmd and mark the lrb as in_use, then
6921 * queue the eh_work and bail.
6922 */
6923 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
6924 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
6925 spin_lock_irqsave(host->host_lock, flags);
6926 if (lrbp->cmd) {
6927 __ufshcd_transfer_req_compl(hba, (1UL << tag));
6928 __set_bit(tag, &hba->outstanding_reqs);
6929 lrbp->in_use = true;
6930 hba->force_reset = true;
6931 ufshcd_schedule_eh_work(hba);
6932 }
6933
6934 spin_unlock_irqrestore(host->host_lock, flags);
6935 goto out;
6936 }
6937
e0b299e3 6938 /* Skip task abort in case previous aborts failed and report failure */
02f74150 6939 if (lrbp->req_abort_skip)
e0b299e3 6940 err = -EIO;
02f74150
MP		 6941 else
6942 err = ufshcd_try_to_abort_task(hba, tag);
f20810d8 6943
f20810d8 6944 if (!err) {
b10178ee 6945cleanup:
02f74150
MP		 6946 spin_lock_irqsave(host->host_lock, flags);
6947 __ufshcd_transfer_req_compl(hba, (1UL << tag));
6948 spin_unlock_irqrestore(host->host_lock, flags);
7a3e97b0 6949out:
f20810d8
SRT		 6950 err = SUCCESS;
6951 } else {
6952 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6953 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6954 err = FAILED;
6955 }
6956
1ab27c9c
ST		 6957 /*
6958 * This ufshcd_release() corresponds to the original scsi cmd that got
6959 * aborted here (as we won't get any IRQ for it).
6960 */
6961 ufshcd_release(hba);
7a3e97b0
SY		 6962 return err;
6963}
6964
3441da7d
SRT		 6965/**
6966 * ufshcd_host_reset_and_restore - reset and restore host controller
6967 * @hba: per-adapter instance
6968 *
6969 * Note that host controller reset may issue DME_RESET to
6970 * local and remote (device) Uni-Pro stack and the attributes
6971 * are reset to default state.
6972 *
6973 * Returns zero on success, non-zero on failure
6974 */
6975static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6976{
6977 int err;
3441da7d
SRT		 6978 unsigned long flags;
6979
2df74b69
CG		 6980 /*
6981 * Stop the host controller and complete the requests
6982 * cleared by h/w
6983 */
5cac1095
BVA		 6984 ufshcd_hba_stop(hba);
6985
3441da7d 6986 spin_lock_irqsave(hba->host->host_lock, flags);
2df74b69
CG		 6987 hba->silence_err_logs = true;
6988 ufshcd_complete_requests(hba);
6989 hba->silence_err_logs = false;
3441da7d
SRT		 6990 spin_unlock_irqrestore(hba->host->host_lock, flags);
6991
a3cd5ec5 6992 /* scale up clocks to max frequency before full reinitialization */
394b949f 6993 ufshcd_set_clk_freq(hba, true);
a3cd5ec5 6994
3441da7d 6995 err = ufshcd_hba_enable(hba);
3441da7d
SRT		 6996
6997 /* Establish the link again and restore the device */
1918651f 6998 if (!err)
4ee7ee53
JK		 6999 err = ufshcd_probe_hba(hba, false);
7000
3441da7d
SRT		 7001 if (err)
7002 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
e965e5e0 7003 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
3441da7d
SRT		 7004 return err;
7005}
7006
7007/**
7008 * ufshcd_reset_and_restore - reset and re-initialize host/device
7009 * @hba: per-adapter instance
7010 *
7011 * Reset and recover device, host and re-establish link. This
7012 * is helpful to recover the communication in fatal error conditions.
7013 *
7014 * Returns zero on success, non-zero on failure
7015 */
7016static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7017{
4db7a236
CG		 7018 u32 saved_err;
7019 u32 saved_uic_err;
3441da7d 7020 int err = 0;
4db7a236 7021 unsigned long flags;
1d337ec2 7022 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 7023
4db7a236
CG		 7024 /*
7025 * This is a fresh start, cache and clear saved error first,
7026 * in case new error generated during reset and restore.
7027 */
7028 spin_lock_irqsave(hba->host->host_lock, flags);
7029 saved_err = hba->saved_err;
7030 saved_uic_err = hba->saved_uic_err;
7031 hba->saved_err = 0;
7032 hba->saved_uic_err = 0;
7033 spin_unlock_irqrestore(hba->host->host_lock, flags);
7034
1d337ec2 7035 do {
d8d9f793 7036 /* Reset the attached device */
31a5d9ca 7037 ufshcd_device_reset(hba);
d8d9f793 7038
1d337ec2
SRT		 7039 err = ufshcd_host_reset_and_restore(hba);
7040 } while (err && --retries);
3441da7d 7041
4db7a236
CG		 7042 spin_lock_irqsave(hba->host->host_lock, flags);
7043 /*
7044 * Inform scsi mid-layer that we did reset and allow to handle
7045 * Unit Attention properly.
7046 */
7047 scsi_report_bus_reset(hba->host, 0);
7048 if (err) {
88a92d6a 7049 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4db7a236
CG		 7050 hba->saved_err |= saved_err;
7051 hba->saved_uic_err |= saved_uic_err;
7052 }
7053 spin_unlock_irqrestore(hba->host->host_lock, flags);
7054
3441da7d
SRT		 7055 return err;
7056}
7057
7058/**
7059 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 7060 * @cmd: SCSI command pointer
3441da7d
SRT		 7061 *
7062 * Returns SUCCESS/FAILED
7063 */
7064static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7065{
4db7a236 7066 int err = SUCCESS;
3441da7d
SRT		 7067 unsigned long flags;
7068 struct ufs_hba *hba;
7069
7070 hba = shost_priv(cmd->device->host);
7071
4db7a236
CG		 7072 spin_lock_irqsave(hba->host->host_lock, flags);
7073 hba->force_reset = true;
7074 ufshcd_schedule_eh_work(hba);
7075 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
3441da7d
SRT		 7076 spin_unlock_irqrestore(hba->host->host_lock, flags);
7077
4db7a236 7078 flush_work(&hba->eh_work);
3441da7d
SRT		 7079
7080 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236 7081 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
3441da7d 7082 err = FAILED;
3441da7d
SRT		 7083 spin_unlock_irqrestore(hba->host->host_lock, flags);
7084
7085 return err;
7086}
7087
3a4bf06d
YG		 7088/**
7089 * ufshcd_get_max_icc_level - calculate the ICC level
7090 * @sup_curr_uA: max. current supported by the regulator
7091 * @start_scan: row at the desc table to start scan from
7092 * @buff: power descriptor buffer
7093 *
7094 * Returns calculated max ICC level for specific regulator
7095 */
7096static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
7097{
7098 int i;
7099 int curr_uA;
7100 u16 data;
7101 u16 unit;
7102
7103 for (i = start_scan; i >= 0; i--) {
d79713f9 7104 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 7105 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7106 ATTR_ICC_LVL_UNIT_OFFSET;
7107 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7108 switch (unit) {
7109 case UFSHCD_NANO_AMP:
7110 curr_uA = curr_uA / 1000;
7111 break;
7112 case UFSHCD_MILI_AMP:
7113 curr_uA = curr_uA * 1000;
7114 break;
7115 case UFSHCD_AMP:
7116 curr_uA = curr_uA * 1000 * 1000;
7117 break;
7118 case UFSHCD_MICRO_AMP:
7119 default:
7120 break;
7121 }
7122 if (sup_curr_uA >= curr_uA)
7123 break;
7124 }
7125 if (i < 0) {
7126 i = 0;
7127 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7128 }
7129
7130 return (u32)i;
7131}
7132
7133/**
7134 * ufshcd_calc_icc_level - calculate the max ICC level
7135 * In case regulators are not initialized we'll return 0
7136 * @hba: per-adapter instance
7137 * @desc_buf: power descriptor buffer to extract ICC levels from.
7138 * @len: length of desc_buff
7139 *
7140 * Returns calculated ICC level
7141 */
7142static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7143 u8 *desc_buf, int len)
7144{
7145 u32 icc_level = 0;
7146
7147 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7148 !hba->vreg_info.vccq2) {
7149 dev_err(hba->dev,
7150 "%s: Regulator capability was not set, actvIccLevel=%d",
7151 __func__, icc_level);
7152 goto out;
7153 }
7154
0487fff7 7155 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 7156 icc_level = ufshcd_get_max_icc_level(
7157 hba->vreg_info.vcc->max_uA,
7158 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7159 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7160
0487fff7 7161 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 7162 icc_level = ufshcd_get_max_icc_level(
7163 hba->vreg_info.vccq->max_uA,
7164 icc_level,
7165 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7166
0487fff7 7167 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 7168 icc_level = ufshcd_get_max_icc_level(
7169 hba->vreg_info.vccq2->max_uA,
7170 icc_level,
7171 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7172out:
7173 return icc_level;
7174}
7175
e89860f1 7176static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
3a4bf06d
YG		 7177{
7178 int ret;
7a0bf85b 7179 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
bbe21d7a 7180 u8 *desc_buf;
e89860f1 7181 u32 icc_level;
bbe21d7a
KC		 7182
7183 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7184 if (!desc_buf)
7185 return;
3a4bf06d 7186
c4607a09
BH		 7187 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7188 desc_buf, buff_len);
3a4bf06d
YG		 7189 if (ret) {
7190 dev_err(hba->dev,
7191 "%s: Failed reading power descriptor.len = %d ret = %d",
7192 __func__, buff_len, ret);
bbe21d7a 7193 goto out;
3a4bf06d
YG		 7194 }
7195
e89860f1
CG		 7196 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7197 buff_len);
7198 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
3a4bf06d 7199
dbd34a61 7200 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
e89860f1 7201 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
3a4bf06d
YG		 7202
7203 if (ret)
7204 dev_err(hba->dev,
7205 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
e89860f1 7206 __func__, icc_level, ret);
3a4bf06d 7207
bbe21d7a
KC		 7208out:
7209 kfree(desc_buf);
3a4bf06d
YG		 7210}
7211
fb276f77
CG		 7212static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7213{
7214 scsi_autopm_get_device(sdev);
7215 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7216 if (sdev->rpm_autosuspend)
7217 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7218 RPM_AUTOSUSPEND_DELAY_MS);
7219 scsi_autopm_put_device(sdev);
7220}
7221
2a8fa600
SJ		 7222/**
7223 * ufshcd_scsi_add_wlus - Adds required W-LUs
7224 * @hba: per-adapter instance
7225 *
7226 * UFS device specification requires the UFS devices to support 4 well known
7227 * logical units:
7228 * "REPORT_LUNS" (address: 01h)
7229 * "UFS Device" (address: 50h)
7230 * "RPMB" (address: 44h)
7231 * "BOOT" (address: 30h)
7232 * UFS device's power management needs to be controlled by "POWER CONDITION"
7233 * field of SSU (START STOP UNIT) command. But this "power condition" field
7234 * will take effect only when its sent to "UFS device" well known logical unit
7235 * hence we require the scsi_device instance to represent this logical unit in
7236 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 7237 *
2a8fa600
SJ		 7238 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7239 * Block) LU so user space process can control this LU. User space may also
7240 * want to have access to BOOT LU.
8aa29f19 7241 *
2a8fa600
SJ		 7242 * This function adds scsi device instances for each of all well known LUs
7243 * (except "REPORT LUNS" LU).
7244 *
7245 * Returns zero on success (all required W-LUs are added successfully),
7246 * non-zero error value on failure (if failed to add any of the required W-LU).
7247 */
7248static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7249{
7250 int ret = 0;
7c48bfd0 7251 struct scsi_device *sdev_boot;
2a8fa600
SJ		 7252
7253 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
7254 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7255 if (IS_ERR(hba->sdev_ufs_device)) {
7256 ret = PTR_ERR(hba->sdev_ufs_device);
7257 hba->sdev_ufs_device = NULL;
7258 goto out;
7259 }
fb276f77 7260 ufshcd_blk_pm_runtime_init(hba->sdev_ufs_device);
7c48bfd0 7261 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 7262
4f3e900b 7263 hba->sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 7264 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
4f3e900b
JK		 7265 if (IS_ERR(hba->sdev_rpmb)) {
7266 ret = PTR_ERR(hba->sdev_rpmb);
3d21fbde 7267 goto remove_sdev_ufs_device;
2a8fa600 7268 }
4f3e900b
JK		 7269 ufshcd_blk_pm_runtime_init(hba->sdev_rpmb);
7270 scsi_device_put(hba->sdev_rpmb);
3d21fbde
HK		 7271
7272 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7273 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
fb276f77 7274 if (IS_ERR(sdev_boot)) {
3d21fbde 7275 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
fb276f77
CG		 7276 } else {
7277 ufshcd_blk_pm_runtime_init(sdev_boot);
3d21fbde 7278 scsi_device_put(sdev_boot);
fb276f77 7279 }
2a8fa600
SJ		 7280 goto out;
7281
2a8fa600
SJ		 7282remove_sdev_ufs_device:
7283 scsi_remove_device(hba->sdev_ufs_device);
7284out:
7285 return ret;
7286}
7287
3d17b9b5
AD		 7288static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7289{
a7f1e69d 7290 struct ufs_dev_info *dev_info = &hba->dev_info;
6f8d5a6a
SC		 7291 u8 lun;
7292 u32 d_lu_wb_buf_alloc;
e8d03813 7293 u32 ext_ufs_feature;
6f8d5a6a 7294
817d7e14
SC		 7295 if (!ufshcd_is_wb_allowed(hba))
7296 return;
a7f1e69d
SC		 7297 /*
7298 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7299 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7300 * enabled
7301 */
7302 if (!(dev_info->wspecversion >= 0x310 ||
7303 dev_info->wspecversion == 0x220 ||
7304 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7305 goto wb_disabled;
817d7e14 7306
7a0bf85b
BH		 7307 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7308 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
817d7e14
SC		 7309 goto wb_disabled;
7310
e8d03813
BH		 7311 ext_ufs_feature = get_unaligned_be32(desc_buf +
7312 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
817d7e14 7313
e8d03813 7314 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
817d7e14
SC		 7315 goto wb_disabled;
7316
3d17b9b5 7317 /*
ae1ce1fc
BH		 7318 * WB may be supported but not configured while provisioning. The spec
7319 * says, in dedicated wb buffer mode, a max of 1 lun would have wb
7320 * buffer configured.
3d17b9b5 7321 */
4cd48995 7322 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
3d17b9b5 7323
a7f1e69d 7324 dev_info->b_presrv_uspc_en =
3d17b9b5
AD		 7325 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7326
4cd48995 7327 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
e8d03813
BH		 7328 if (!get_unaligned_be32(desc_buf +
7329 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
6f8d5a6a
SC		 7330 goto wb_disabled;
7331 } else {
7332 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7333 d_lu_wb_buf_alloc = 0;
7334 ufshcd_read_unit_desc_param(hba,
7335 lun,
7336 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7337 (u8 *)&d_lu_wb_buf_alloc,
7338 sizeof(d_lu_wb_buf_alloc));
7339 if (d_lu_wb_buf_alloc) {
a7f1e69d 7340 dev_info->wb_dedicated_lu = lun;
6f8d5a6a
SC		 7341 break;
7342 }
7343 }
817d7e14 7344
6f8d5a6a
SC		 7345 if (!d_lu_wb_buf_alloc)
7346 goto wb_disabled;
7347 }
817d7e14
SC		 7348 return;
7349
7350wb_disabled:
7351 hba->caps &= ~UFSHCD_CAP_WB_EN;
7352}
7353
8db269a5 7354void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
817d7e14
SC		 7355{
7356 struct ufs_dev_fix *f;
7357 struct ufs_dev_info *dev_info = &hba->dev_info;
7358
8db269a5
SC		 7359 if (!fixups)
7360 return;
7361
7362 for (f = fixups; f->quirk; f++) {
817d7e14
SC		 7363 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7364 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7365 ((dev_info->model &&
7366 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7367 !strcmp(f->model, UFS_ANY_MODEL)))
7368 hba->dev_quirks |= f->quirk;
7369 }
3d17b9b5 7370}
8db269a5 7371EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
3d17b9b5 7372
c28c00ba
SC		 7373static void ufs_fixup_device_setup(struct ufs_hba *hba)
7374{
7375 /* fix by general quirk table */
8db269a5 7376 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
c28c00ba
SC		 7377
7378 /* allow vendors to fix quirks */
7379 ufshcd_vops_fixup_dev_quirks(hba);
7380}
7381
09750066 7382static int ufs_get_device_desc(struct ufs_hba *hba)
c58ab7aa
YG		 7383{
7384 int err;
7385 u8 model_index;
bbe21d7a 7386 u8 *desc_buf;
09750066 7387 struct ufs_dev_info *dev_info = &hba->dev_info;
4b828fe1 7388
458a45f5 7389 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
bbe21d7a
KC		 7390 if (!desc_buf) {
7391 err = -ENOMEM;
7392 goto out;
7393 }
c58ab7aa 7394
c4607a09 7395 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7a0bf85b 7396 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
c58ab7aa
YG		 7397 if (err) {
7398 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7399 __func__, err);
7400 goto out;
7401 }
7402
7403 /*
7404 * getting vendor (manufacturerID) and Bank Index in big endian
7405 * format
7406 */
09750066 7407 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 7408 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7409
09f17791
CG		 7410 /* getting Specification Version in big endian format */
7411 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7412 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7413
c58ab7aa 7414 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
3d17b9b5 7415
4b828fe1 7416 err = ufshcd_read_string_desc(hba, model_index,
09750066 7417 &dev_info->model, SD_ASCII_STD);
4b828fe1 7418 if (err < 0) {
c58ab7aa
YG		 7419 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7420 __func__, err);
7421 goto out;
7422 }
7423
817d7e14
SC		 7424 ufs_fixup_device_setup(hba);
7425
a7f1e69d 7426 ufshcd_wb_probe(hba, desc_buf);
817d7e14 7427
4b828fe1
TW		 7428 /*
7429 * ufshcd_read_string_desc returns size of the string
7430 * reset the error value
7431 */
7432 err = 0;
c58ab7aa
YG		 7433
7434out:
bbe21d7a 7435 kfree(desc_buf);
c58ab7aa
YG		 7436 return err;
7437}
7438
09750066 7439static void ufs_put_device_desc(struct ufs_hba *hba)
4b828fe1 7440{
09750066
BH		 7441 struct ufs_dev_info *dev_info = &hba->dev_info;
7442
7443 kfree(dev_info->model);
7444 dev_info->model = NULL;
4b828fe1
TW		 7445}
7446
37113106
YG		 7447/**
7448 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7449 * @hba: per-adapter instance
7450 *
7451 * PA_TActivate parameter can be tuned manually if UniPro version is less than
7452 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7453 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7454 * the hibern8 exit latency.
7455 *
7456 * Returns zero on success, non-zero error value on failure.
7457 */
7458static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7459{
7460 int ret = 0;
7461 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7462
7463 ret = ufshcd_dme_peer_get(hba,
7464 UIC_ARG_MIB_SEL(
7465 RX_MIN_ACTIVATETIME_CAPABILITY,
7466 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7467 &peer_rx_min_activatetime);
7468 if (ret)
7469 goto out;
7470
7471 /* make sure proper unit conversion is applied */
7472 tuned_pa_tactivate =
7473 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7474 / PA_TACTIVATE_TIME_UNIT_US);
7475 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7476 tuned_pa_tactivate);
7477
7478out:
7479 return ret;
7480}
7481
7482/**
7483 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7484 * @hba: per-adapter instance
7485 *
7486 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7487 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7488 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7489 * This optimal value can help reduce the hibern8 exit latency.
7490 *
7491 * Returns zero on success, non-zero error value on failure.
7492 */
7493static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7494{
7495 int ret = 0;
7496 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7497 u32 max_hibern8_time, tuned_pa_hibern8time;
7498
7499 ret = ufshcd_dme_get(hba,
7500 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7501 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7502 &local_tx_hibern8_time_cap);
7503 if (ret)
7504 goto out;
7505
7506 ret = ufshcd_dme_peer_get(hba,
7507 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7508 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7509 &peer_rx_hibern8_time_cap);
7510 if (ret)
7511 goto out;
7512
7513 max_hibern8_time = max(local_tx_hibern8_time_cap,
7514 peer_rx_hibern8_time_cap);
7515 /* make sure proper unit conversion is applied */
7516 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7517 / PA_HIBERN8_TIME_UNIT_US);
7518 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7519 tuned_pa_hibern8time);
7520out:
7521 return ret;
7522}
7523
c6a6db43 7524/**
7525 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7526 * less than device PA_TACTIVATE time.
7527 * @hba: per-adapter instance
7528 *
7529 * Some UFS devices require host PA_TACTIVATE to be lower than device
7530 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7531 * for such devices.
7532 *
7533 * Returns zero on success, non-zero error value on failure.
7534 */
7535static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7536{
7537 int ret = 0;
7538 u32 granularity, peer_granularity;
7539 u32 pa_tactivate, peer_pa_tactivate;
7540 u32 pa_tactivate_us, peer_pa_tactivate_us;
7541 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7542
7543 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7544 &granularity);
7545 if (ret)
7546 goto out;
7547
7548 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7549 &peer_granularity);
7550 if (ret)
7551 goto out;
7552
7553 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7554 (granularity > PA_GRANULARITY_MAX_VAL)) {
7555 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7556 __func__, granularity);
7557 return -EINVAL;
7558 }
7559
7560 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7561 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7562 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7563 __func__, peer_granularity);
7564 return -EINVAL;
7565 }
7566
7567 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7568 if (ret)
7569 goto out;
7570
7571 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7572 &peer_pa_tactivate);
7573 if (ret)
7574 goto out;
7575
7576 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7577 peer_pa_tactivate_us = peer_pa_tactivate *
7578 gran_to_us_table[peer_granularity - 1];
7579
7580 if (pa_tactivate_us > peer_pa_tactivate_us) {
7581 u32 new_peer_pa_tactivate;
7582
7583 new_peer_pa_tactivate = pa_tactivate_us /
7584 gran_to_us_table[peer_granularity - 1];
7585 new_peer_pa_tactivate++;
7586 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7587 new_peer_pa_tactivate);
7588 }
7589
7590out:
7591 return ret;
7592}
7593
09750066 7594static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
37113106
YG		 7595{
7596 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7597 ufshcd_tune_pa_tactivate(hba);
7598 ufshcd_tune_pa_hibern8time(hba);
7599 }
7600
e91ed9e0
CG		 7601 ufshcd_vops_apply_dev_quirks(hba);
7602
37113106
YG		 7603 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7604 /* set 1ms timeout for PA_TACTIVATE */
7605 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43 7606
7607 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7608 ufshcd_quirk_tune_host_pa_tactivate(hba);
37113106
YG		 7609}
7610
ff8e20c6
DR		 7611static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7612{
ff8e20c6
DR		 7613 hba->ufs_stats.hibern8_exit_cnt = 0;
7614 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 7615 hba->req_abort_count = 0;
ff8e20c6
DR		 7616}
7617
731f0621
BH		 7618static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7619{
7620 int err;
7621 size_t buff_len;
7622 u8 *desc_buf;
7623
7a0bf85b 7624 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
731f0621
BH		 7625 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7626 if (!desc_buf) {
7627 err = -ENOMEM;
7628 goto out;
7629 }
7630
c4607a09
BH		 7631 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7632 desc_buf, buff_len);
731f0621
BH		 7633 if (err) {
7634 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7635 __func__, err);
7636 goto out;
7637 }
7638
7639 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7640 hba->dev_info.max_lu_supported = 32;
7641 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7642 hba->dev_info.max_lu_supported = 8;
7643
7644out:
7645 kfree(desc_buf);
7646 return err;
7647}
7648
9e1e8a75
SJ		 7649static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7650 {19200000, REF_CLK_FREQ_19_2_MHZ},
7651 {26000000, REF_CLK_FREQ_26_MHZ},
7652 {38400000, REF_CLK_FREQ_38_4_MHZ},
7653 {52000000, REF_CLK_FREQ_52_MHZ},
7654 {0, REF_CLK_FREQ_INVAL},
7655};
7656
7657static enum ufs_ref_clk_freq
7658ufs_get_bref_clk_from_hz(unsigned long freq)
7659{
7660 int i;
7661
7662 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7663 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7664 return ufs_ref_clk_freqs[i].val;
7665
7666 return REF_CLK_FREQ_INVAL;
7667}
7668
7669void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7670{
7671 unsigned long freq;
7672
7673 freq = clk_get_rate(refclk);
7674
7675 hba->dev_ref_clk_freq =
7676 ufs_get_bref_clk_from_hz(freq);
7677
7678 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7679 dev_err(hba->dev,
7680 "invalid ref_clk setting = %ld\n", freq);
7681}
7682
7683static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
7684{
7685 int err;
7686 u32 ref_clk;
7687 u32 freq = hba->dev_ref_clk_freq;
7688
7689 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
7690 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
7691
7692 if (err) {
7693 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
7694 err);
7695 goto out;
7696 }
7697
7698 if (ref_clk == freq)
7699 goto out; /* nothing to update */
7700
7701 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7702 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
7703
7704 if (err) {
7705 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
7706 ufs_ref_clk_freqs[freq].freq_hz);
7707 goto out;
7708 }
7709
7710 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
7711 ufs_ref_clk_freqs[freq].freq_hz);
7712
7713out:
7714 return err;
7715}
7716
1b9e2141
BH		 7717static int ufshcd_device_params_init(struct ufs_hba *hba)
7718{
7719 bool flag;
7a0bf85b 7720 int ret, i;
1b9e2141 7721
7a0bf85b
BH		 7722 /* Init device descriptor sizes */
7723 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
7724 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
1b9e2141 7725
731f0621
BH		 7726 /* Init UFS geometry descriptor related parameters */
7727 ret = ufshcd_device_geo_params_init(hba);
7728 if (ret)
7729 goto out;
7730
1b9e2141
BH		 7731 /* Check and apply UFS device quirks */
7732 ret = ufs_get_device_desc(hba);
7733 if (ret) {
7734 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
7735 __func__, ret);
7736 goto out;
7737 }
7738
09f17791
CG		 7739 ufshcd_get_ref_clk_gating_wait(hba);
7740
1b9e2141 7741 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
1f34eedf 7742 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
1b9e2141
BH		 7743 hba->dev_info.f_power_on_wp_en = flag;
7744
2b35b2ad
BH		 7745 /* Probe maximum power mode co-supported by both UFS host and device */
7746 if (ufshcd_get_max_pwr_mode(hba))
7747 dev_err(hba->dev,
7748 "%s: Failed getting max supported power mode\n",
7749 __func__);
1b9e2141
BH		 7750out:
7751 return ret;
7752}
7753
7754/**
7755 * ufshcd_add_lus - probe and add UFS logical units
7756 * @hba: per-adapter instance
7757 */
7758static int ufshcd_add_lus(struct ufs_hba *hba)
7759{
7760 int ret;
7761
1b9e2141
BH		 7762 /* Add required well known logical units to scsi mid layer */
7763 ret = ufshcd_scsi_add_wlus(hba);
7764 if (ret)
7765 goto out;
7766
4ee7ee53
JK		 7767 ufshcd_clear_ua_wluns(hba);
7768
1b9e2141
BH		 7769 /* Initialize devfreq after UFS device is detected */
7770 if (ufshcd_is_clkscaling_supported(hba)) {
7771 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7772 &hba->pwr_info,
7773 sizeof(struct ufs_pa_layer_attr));
7774 hba->clk_scaling.saved_pwr_info.is_valid = true;
1b9e2141 7775 hba->clk_scaling.is_allowed = true;
1b9e2141 7776
b058fa86
SC		 7777 ret = ufshcd_devfreq_init(hba);
7778 if (ret)
7779 goto out;
7780
7781 hba->clk_scaling.is_enabled = true;
7782 ufshcd_init_clk_scaling_sysfs(hba);
1b9e2141
BH		 7783 }
7784
7785 ufs_bsg_probe(hba);
7786 scsi_scan_host(hba->host);
7787 pm_runtime_put_sync(hba->dev);
7788
1b9e2141
BH		 7789out:
7790 return ret;
7791}
7792
4f3e900b
JK		 7793static int
7794ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp);
7795
7796static int ufshcd_clear_ua_wlun(struct ufs_hba *hba, u8 wlun)
7797{
7798 struct scsi_device *sdp;
7799 unsigned long flags;
7800 int ret = 0;
7801
7802 spin_lock_irqsave(hba->host->host_lock, flags);
7803 if (wlun == UFS_UPIU_UFS_DEVICE_WLUN)
7804 sdp = hba->sdev_ufs_device;
7805 else if (wlun == UFS_UPIU_RPMB_WLUN)
7806 sdp = hba->sdev_rpmb;
7807 else
4c60244d 7808 BUG();
4f3e900b
JK		 7809 if (sdp) {
7810 ret = scsi_device_get(sdp);
7811 if (!ret && !scsi_device_online(sdp)) {
7812 ret = -ENODEV;
7813 scsi_device_put(sdp);
7814 }
7815 } else {
7816 ret = -ENODEV;
7817 }
7818 spin_unlock_irqrestore(hba->host->host_lock, flags);
7819 if (ret)
7820 goto out_err;
7821
7822 ret = ufshcd_send_request_sense(hba, sdp);
7823 scsi_device_put(sdp);
7824out_err:
7825 if (ret)
7826 dev_err(hba->dev, "%s: UAC clear LU=%x ret = %d\n",
7827 __func__, wlun, ret);
7828 return ret;
7829}
7830
7831static int ufshcd_clear_ua_wluns(struct ufs_hba *hba)
7832{
7833 int ret = 0;
7834
7835 if (!hba->wlun_dev_clr_ua)
7836 goto out;
7837
7838 ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN);
7839 if (!ret)
7840 ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN);
7841 if (!ret)
7842 hba->wlun_dev_clr_ua = false;
7843out:
7844 if (ret)
7845 dev_err(hba->dev, "%s: Failed to clear UAC WLUNS ret = %d\n",
7846 __func__, ret);
7847 return ret;
7848}
7849
6ccf44fe 7850/**
1d337ec2
SRT		 7851 * ufshcd_probe_hba - probe hba to detect device and initialize
7852 * @hba: per-adapter instance
1b9e2141 7853 * @async: asynchronous execution or not
1d337ec2
SRT		 7854 *
7855 * Execute link-startup and verify device initialization
6ccf44fe 7856 */
1b9e2141 7857static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
6ccf44fe 7858{
6ccf44fe 7859 int ret;
4db7a236 7860 unsigned long flags;
7ff5ab47 7861 ktime_t start = ktime_get();
6ccf44fe 7862
aa53f580
CG		 7863 hba->ufshcd_state = UFSHCD_STATE_RESET;
7864
6ccf44fe 7865 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 7866 if (ret)
7867 goto out;
7868
ff8e20c6
DR		 7869 /* Debug counters initialization */
7870 ufshcd_clear_dbg_ufs_stats(hba);
7871
57d104c1
SJ		 7872 /* UniPro link is active now */
7873 ufshcd_set_link_active(hba);
d3e89bac 7874
1b9e2141 7875 /* Verify device initialization by sending NOP OUT UPIU */
5a0b0cb9
SRT		 7876 ret = ufshcd_verify_dev_init(hba);
7877 if (ret)
7878 goto out;
68078d5c 7879
1b9e2141 7880 /* Initiate UFS initialization, and waiting until completion */
68078d5c
DR		 7881 ret = ufshcd_complete_dev_init(hba);
7882 if (ret)
7883 goto out;
5a0b0cb9 7884
1b9e2141
BH		 7885 /*
7886 * Initialize UFS device parameters used by driver, these
7887 * parameters are associated with UFS descriptors.
7888 */
7889 if (async) {
7890 ret = ufshcd_device_params_init(hba);
7891 if (ret)
7892 goto out;
93fdd5ac
TW		 7893 }
7894
09750066 7895 ufshcd_tune_unipro_params(hba);
4b828fe1 7896
57d104c1
SJ		 7897 /* UFS device is also active now */
7898 ufshcd_set_ufs_dev_active(hba);
66ec6d59 7899 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 7900 hba->wlun_dev_clr_ua = true;
7901
2b35b2ad
BH		 7902 /* Gear up to HS gear if supported */
7903 if (hba->max_pwr_info.is_valid) {
9e1e8a75
SJ		 7904 /*
7905 * Set the right value to bRefClkFreq before attempting to
7906 * switch to HS gears.
7907 */
7908 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7909 ufshcd_set_dev_ref_clk(hba);
7eb584db 7910 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 7911 if (ret) {
7eb584db
DR		 7912 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7913 __func__, ret);
8643ae66
DL		 7914 goto out;
7915 }
6a9df818 7916 ufshcd_print_pwr_info(hba);
7eb584db 7917 }
57d104c1 7918
e89860f1
CG		 7919 /*
7920 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
7921 * and for removable UFS card as well, hence always set the parameter.
7922 * Note: Error handler may issue the device reset hence resetting
7923 * bActiveICCLevel as well so it is always safe to set this here.
7924 */
7925 ufshcd_set_active_icc_lvl(hba);
7926
3d17b9b5 7927 ufshcd_wb_config(hba);
71d848b8
CG		 7928 /* Enable Auto-Hibernate if configured */
7929 ufshcd_auto_hibern8_enable(hba);
7930
5a0b0cb9 7931out:
4db7a236
CG		 7932 spin_lock_irqsave(hba->host->host_lock, flags);
7933 if (ret)
7934 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7935 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
7936 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7937 spin_unlock_irqrestore(hba->host->host_lock, flags);
1d337ec2 7938
7ff5ab47 7939 trace_ufshcd_init(dev_name(hba->dev), ret,
7940 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7941 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 7942 return ret;
7943}
7944
7945/**
7946 * ufshcd_async_scan - asynchronous execution for probing hba
7947 * @data: data pointer to pass to this function
7948 * @cookie: cookie data
7949 */
7950static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7951{
7952 struct ufs_hba *hba = (struct ufs_hba *)data;
1b9e2141 7953 int ret;
1d337ec2 7954
9cd20d3f 7955 down(&hba->host_sem);
1b9e2141
BH		 7956 /* Initialize hba, detect and initialize UFS device */
7957 ret = ufshcd_probe_hba(hba, true);
9cd20d3f 7958 up(&hba->host_sem);
1b9e2141
BH		 7959 if (ret)
7960 goto out;
7961
7962 /* Probe and add UFS logical units */
7963 ret = ufshcd_add_lus(hba);
7964out:
7965 /*
7966 * If we failed to initialize the device or the device is not
7967 * present, turn off the power/clocks etc.
7968 */
7969 if (ret) {
7970 pm_runtime_put_sync(hba->dev);
1b9e2141
BH		 7971 ufshcd_hba_exit(hba);
7972 }
6ccf44fe
SJ		 7973}
7974
d829fc8a
SN		 7975static const struct attribute_group *ufshcd_driver_groups[] = {
7976 &ufs_sysfs_unit_descriptor_group,
ec92b59c 7977 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 7978 NULL,
7979};
7980
90b8491c
SC		 7981static struct ufs_hba_variant_params ufs_hba_vps = {
7982 .hba_enable_delay_us = 1000,
d14734ae 7983 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
90b8491c
SC		 7984 .devfreq_profile.polling_ms = 100,
7985 .devfreq_profile.target = ufshcd_devfreq_target,
7986 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
7987 .ondemand_data.upthreshold = 70,
7988 .ondemand_data.downdifferential = 5,
7989};
7990
7a3e97b0
SY		 7991static struct scsi_host_template ufshcd_driver_template = {
7992 .module = THIS_MODULE,
7993 .name = UFSHCD,
7994 .proc_name = UFSHCD,
7995 .queuecommand = ufshcd_queuecommand,
7996 .slave_alloc = ufshcd_slave_alloc,
eeda4749 7997 .slave_configure = ufshcd_slave_configure,
7a3e97b0 7998 .slave_destroy = ufshcd_slave_destroy,
4264fd61 7999 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 8000 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 8001 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8002 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
7a3e97b0
SY		 8003 .this_id = -1,
8004 .sg_tablesize = SG_ALL,
8005 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
8006 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 8007 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 8008 .max_host_blocked = 1,
c40ecc12 8009 .track_queue_depth = 1,
d829fc8a 8010 .sdev_groups = ufshcd_driver_groups,
4af14d11 8011 .dma_boundary = PAGE_SIZE - 1,
49615ba1 8012 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 8013};
8014
57d104c1
SJ		 8015static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8016 int ua)
8017{
7b16a07c 8018 int ret;
57d104c1 8019
7b16a07c
BA		 8020 if (!vreg)
8021 return 0;
57d104c1 8022
0487fff7
SC		 8023 /*
8024 * "set_load" operation shall be required on those regulators
8025 * which specifically configured current limitation. Otherwise
8026 * zero max_uA may cause unexpected behavior when regulator is
8027 * enabled or set as high power mode.
8028 */
8029 if (!vreg->max_uA)
8030 return 0;
8031
7b16a07c
BA		 8032 ret = regulator_set_load(vreg->reg, ua);
8033 if (ret < 0) {
8034 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
8035 __func__, vreg->name, ua, ret);
57d104c1
SJ		 8036 }
8037
8038 return ret;
8039}
8040
8041static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
8042 struct ufs_vreg *vreg)
8043{
73067981 8044 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 8045}
8046
8047static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
8048 struct ufs_vreg *vreg)
8049{
7c7cfdcf
AH		 8050 if (!vreg)
8051 return 0;
8052
73067981 8053 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 8054}
8055
aa497613
SRT		 8056static int ufshcd_config_vreg(struct device *dev,
8057 struct ufs_vreg *vreg, bool on)
8058{
8059 int ret = 0;
72753590
GS		 8060 struct regulator *reg;
8061 const char *name;
aa497613
SRT		 8062 int min_uV, uA_load;
8063
8064 BUG_ON(!vreg);
8065
72753590
GS		 8066 reg = vreg->reg;
8067 name = vreg->name;
8068
aa497613 8069 if (regulator_count_voltages(reg) > 0) {
90d88f47
AD		 8070 uA_load = on ? vreg->max_uA : 0;
8071 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
8072 if (ret)
8073 goto out;
8074
3b141e8c
SC		 8075 if (vreg->min_uV && vreg->max_uV) {
8076 min_uV = on ? vreg->min_uV : 0;
8077 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
b0008625 8078 if (ret)
3b141e8c
SC		 8079 dev_err(dev,
8080 "%s: %s set voltage failed, err=%d\n",
aa497613 8081 __func__, name, ret);
aa497613 8082 }
aa497613
SRT		 8083 }
8084out:
8085 return ret;
8086}
8087
8088static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
8089{
8090 int ret = 0;
8091
73067981 8092 if (!vreg || vreg->enabled)
aa497613
SRT		 8093 goto out;
8094
8095 ret = ufshcd_config_vreg(dev, vreg, true);
8096 if (!ret)
8097 ret = regulator_enable(vreg->reg);
8098
8099 if (!ret)
8100 vreg->enabled = true;
8101 else
8102 dev_err(dev, "%s: %s enable failed, err=%d\n",
8103 __func__, vreg->name, ret);
8104out:
8105 return ret;
8106}
8107
8108static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
8109{
8110 int ret = 0;
8111
f8162ac7 8112 if (!vreg || !vreg->enabled || vreg->always_on)
aa497613
SRT		 8113 goto out;
8114
8115 ret = regulator_disable(vreg->reg);
8116
8117 if (!ret) {
8118 /* ignore errors on applying disable config */
8119 ufshcd_config_vreg(dev, vreg, false);
8120 vreg->enabled = false;
8121 } else {
8122 dev_err(dev, "%s: %s disable failed, err=%d\n",
8123 __func__, vreg->name, ret);
8124 }
8125out:
8126 return ret;
8127}
8128
8129static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
8130{
8131 int ret = 0;
8132 struct device *dev = hba->dev;
8133 struct ufs_vreg_info *info = &hba->vreg_info;
8134
aa497613
SRT		 8135 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
8136 if (ret)
8137 goto out;
8138
8139 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
8140 if (ret)
8141 goto out;
8142
8143 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
aa497613
SRT		 8144
8145out:
8146 if (ret) {
8147 ufshcd_toggle_vreg(dev, info->vccq2, false);
8148 ufshcd_toggle_vreg(dev, info->vccq, false);
8149 ufshcd_toggle_vreg(dev, info->vcc, false);
8150 }
8151 return ret;
8152}
8153
6a771a65
RS		 8154static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8155{
8156 struct ufs_vreg_info *info = &hba->vreg_info;
8157
60b7b823 8158 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 8159}
8160
aa497613
SRT		 8161static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8162{
8163 int ret = 0;
8164
8165 if (!vreg)
8166 goto out;
8167
8168 vreg->reg = devm_regulator_get(dev, vreg->name);
8169 if (IS_ERR(vreg->reg)) {
8170 ret = PTR_ERR(vreg->reg);
8171 dev_err(dev, "%s: %s get failed, err=%d\n",
8172 __func__, vreg->name, ret);
8173 }
8174out:
8175 return ret;
8176}
8177
8178static int ufshcd_init_vreg(struct ufs_hba *hba)
8179{
8180 int ret = 0;
8181 struct device *dev = hba->dev;
8182 struct ufs_vreg_info *info = &hba->vreg_info;
8183
aa497613
SRT		 8184 ret = ufshcd_get_vreg(dev, info->vcc);
8185 if (ret)
8186 goto out;
8187
8188 ret = ufshcd_get_vreg(dev, info->vccq);
b0008625
BH		 8189 if (!ret)
8190 ret = ufshcd_get_vreg(dev, info->vccq2);
aa497613
SRT		 8191out:
8192 return ret;
8193}
8194
6a771a65
RS		 8195static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8196{
8197 struct ufs_vreg_info *info = &hba->vreg_info;
8198
8199 if (info)
8200 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8201
8202 return 0;
8203}
8204
81309c24 8205static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
c6e79dac
SRT		 8206{
8207 int ret = 0;
8208 struct ufs_clk_info *clki;
8209 struct list_head *head = &hba->clk_list_head;
1ab27c9c 8210 unsigned long flags;
911a0771 8211 ktime_t start = ktime_get();
8212 bool clk_state_changed = false;
c6e79dac 8213
566ec9ad 8214 if (list_empty(head))
c6e79dac
SRT		 8215 goto out;
8216
38f3242e
CG		 8217 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8218 if (ret)
8219 return ret;
1e879e8f 8220
c6e79dac
SRT		 8221 list_for_each_entry(clki, head, list) {
8222 if (!IS_ERR_OR_NULL(clki->clk)) {
81309c24
CG		 8223 /*
8224 * Don't disable clocks which are needed
8225 * to keep the link active.
8226 */
8227 if (ufshcd_is_link_active(hba) &&
8228 clki->keep_link_active)
57d104c1
SJ		 8229 continue;
8230
911a0771 8231 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 8232 if (on && !clki->enabled) {
8233 ret = clk_prepare_enable(clki->clk);
8234 if (ret) {
8235 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8236 __func__, clki->name, ret);
8237 goto out;
8238 }
8239 } else if (!on && clki->enabled) {
8240 clk_disable_unprepare(clki->clk);
8241 }
8242 clki->enabled = on;
8243 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8244 clki->name, on ? "en" : "dis");
8245 }
8246 }
1ab27c9c 8247
38f3242e
CG		 8248 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8249 if (ret)
8250 return ret;
1e879e8f 8251
c6e79dac
SRT		 8252out:
8253 if (ret) {
8254 list_for_each_entry(clki, head, list) {
8255 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8256 clk_disable_unprepare(clki->clk);
8257 }
7ff5ab47 8258 } else if (!ret && on) {
1ab27c9c
ST		 8259 spin_lock_irqsave(hba->host->host_lock, flags);
8260 hba->clk_gating.state = CLKS_ON;
7ff5ab47 8261 trace_ufshcd_clk_gating(dev_name(hba->dev),
8262 hba->clk_gating.state);
1ab27c9c 8263 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 8264 }
7ff5ab47 8265
911a0771 8266 if (clk_state_changed)
8267 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8268 (on ? "on" : "off"),
8269 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 8270 return ret;
8271}
8272
8273static int ufshcd_init_clocks(struct ufs_hba *hba)
8274{
8275 int ret = 0;
8276 struct ufs_clk_info *clki;
8277 struct device *dev = hba->dev;
8278 struct list_head *head = &hba->clk_list_head;
8279
566ec9ad 8280 if (list_empty(head))
c6e79dac
SRT		 8281 goto out;
8282
8283 list_for_each_entry(clki, head, list) {
8284 if (!clki->name)
8285 continue;
8286
8287 clki->clk = devm_clk_get(dev, clki->name);
8288 if (IS_ERR(clki->clk)) {
8289 ret = PTR_ERR(clki->clk);
8290 dev_err(dev, "%s: %s clk get failed, %d\n",
8291 __func__, clki->name, ret);
8292 goto out;
8293 }
8294
9e1e8a75
SJ		 8295 /*
8296 * Parse device ref clk freq as per device tree "ref_clk".
8297 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8298 * in ufshcd_alloc_host().
8299 */
8300 if (!strcmp(clki->name, "ref_clk"))
8301 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8302
c6e79dac
SRT		 8303 if (clki->max_freq) {
8304 ret = clk_set_rate(clki->clk, clki->max_freq);
8305 if (ret) {
8306 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8307 __func__, clki->name,
8308 clki->max_freq, ret);
8309 goto out;
8310 }
856b3483 8311 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 8312 }
8313 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8314 clki->name, clk_get_rate(clki->clk));
8315 }
8316out:
8317 return ret;
8318}
8319
5c0c28a8
SRT		 8320static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8321{
8322 int err = 0;
8323
8324 if (!hba->vops)
8325 goto out;
8326
0263bcd0 8327 err = ufshcd_vops_init(hba);
5c0c28a8
SRT		 8328 if (err)
8329 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 8330 __func__, ufshcd_get_var_name(hba), err);
ade921a8 8331out:
5c0c28a8
SRT		 8332 return err;
8333}
8334
8335static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8336{
8337 if (!hba->vops)
8338 return;
8339
0263bcd0 8340 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 8341}
8342
aa497613
SRT		 8343static int ufshcd_hba_init(struct ufs_hba *hba)
8344{
8345 int err;
8346
6a771a65
RS		 8347 /*
8348 * Handle host controller power separately from the UFS device power
8349 * rails as it will help controlling the UFS host controller power
8350 * collapse easily which is different than UFS device power collapse.
8351 * Also, enable the host controller power before we go ahead with rest
8352 * of the initialization here.
8353 */
8354 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 8355 if (err)
8356 goto out;
8357
6a771a65 8358 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 8359 if (err)
8360 goto out;
8361
6a771a65
RS		 8362 err = ufshcd_init_clocks(hba);
8363 if (err)
8364 goto out_disable_hba_vreg;
8365
8366 err = ufshcd_setup_clocks(hba, true);
8367 if (err)
8368 goto out_disable_hba_vreg;
8369
c6e79dac
SRT		 8370 err = ufshcd_init_vreg(hba);
8371 if (err)
8372 goto out_disable_clks;
8373
8374 err = ufshcd_setup_vreg(hba, true);
8375 if (err)
8376 goto out_disable_clks;
8377
aa497613
SRT		 8378 err = ufshcd_variant_hba_init(hba);
8379 if (err)
8380 goto out_disable_vreg;
8381
b6cacaf2
AH		 8382 ufs_debugfs_hba_init(hba);
8383
1d337ec2 8384 hba->is_powered = true;
aa497613
SRT		 8385 goto out;
8386
8387out_disable_vreg:
8388 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 8389out_disable_clks:
8390 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 8391out_disable_hba_vreg:
8392 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 8393out:
8394 return err;
8395}
8396
8397static void ufshcd_hba_exit(struct ufs_hba *hba)
8398{
1d337ec2 8399 if (hba->is_powered) {
4543d9d7
CG		 8400 ufshcd_exit_clk_scaling(hba);
8401 ufshcd_exit_clk_gating(hba);
8402 if (hba->eh_wq)
8403 destroy_workqueue(hba->eh_wq);
b6cacaf2 8404 ufs_debugfs_hba_exit(hba);
1d337ec2
SRT		 8405 ufshcd_variant_hba_exit(hba);
8406 ufshcd_setup_vreg(hba, false);
8407 ufshcd_setup_clocks(hba, false);
8408 ufshcd_setup_hba_vreg(hba, false);
8409 hba->is_powered = false;
09750066 8410 ufs_put_device_desc(hba);
1d337ec2 8411 }
aa497613
SRT		 8412}
8413
57d104c1
SJ		 8414static int
8415ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
8416{
8417 unsigned char cmd[6] = {REQUEST_SENSE,
8418 0,
8419 0,
8420 0,
09a5a24f 8421 UFS_SENSE_SIZE,
57d104c1
SJ		 8422 0};
8423 char *buffer;
8424 int ret;
8425
09a5a24f 8426 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 8427 if (!buffer) {
8428 ret = -ENOMEM;
8429 goto out;
8430 }
8431
fcbfffe2 8432 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 8433 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 8434 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 8435 if (ret)
8436 pr_err("%s: failed with err %d\n", __func__, ret);
8437
8438 kfree(buffer);
8439out:
8440 return ret;
8441}
8442
8443/**
8444 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8445 * power mode
8446 * @hba: per adapter instance
8447 * @pwr_mode: device power mode to set
8448 *
8449 * Returns 0 if requested power mode is set successfully
8450 * Returns non-zero if failed to set the requested power mode
8451 */
8452static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8453 enum ufs_dev_pwr_mode pwr_mode)
8454{
8455 unsigned char cmd[6] = { START_STOP };
8456 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 8457 struct scsi_device *sdp;
8458 unsigned long flags;
57d104c1
SJ		 8459 int ret;
8460
7c48bfd0
AM		 8461 spin_lock_irqsave(hba->host->host_lock, flags);
8462 sdp = hba->sdev_ufs_device;
8463 if (sdp) {
8464 ret = scsi_device_get(sdp);
8465 if (!ret && !scsi_device_online(sdp)) {
8466 ret = -ENODEV;
8467 scsi_device_put(sdp);
8468 }
8469 } else {
8470 ret = -ENODEV;
8471 }
8472 spin_unlock_irqrestore(hba->host->host_lock, flags);
8473
8474 if (ret)
8475 return ret;
57d104c1
SJ		 8476
8477 /*
8478 * If scsi commands fail, the scsi mid-layer schedules scsi error-
8479 * handling, which would wait for host to be resumed. Since we know
8480 * we are functional while we are here, skip host resume in error
8481 * handling context.
8482 */
8483 hba->host->eh_noresume = 1;
1918651f 8484 ufshcd_clear_ua_wluns(hba);
57d104c1
SJ		 8485
8486 cmd[4] = pwr_mode << 4;
8487
8488 /*
8489 * Current function would be generally called from the power management
e8064021 8490 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 8491 * already suspended childs.
8492 */
fcbfffe2
CH		 8493 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8494 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 8495 if (ret) {
8496 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 8497 "START_STOP failed for power mode: %d, result %x\n",
8498 pwr_mode, ret);
c65be1a6 8499 if (driver_byte(ret) == DRIVER_SENSE)
21045519 8500 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 8501 }
8502
8503 if (!ret)
8504 hba->curr_dev_pwr_mode = pwr_mode;
1918651f 8505
7c48bfd0 8506 scsi_device_put(sdp);
57d104c1
SJ		 8507 hba->host->eh_noresume = 0;
8508 return ret;
8509}
8510
8511static int ufshcd_link_state_transition(struct ufs_hba *hba,
8512 enum uic_link_state req_link_state,
8513 int check_for_bkops)
8514{
8515 int ret = 0;
8516
8517 if (req_link_state == hba->uic_link_state)
8518 return 0;
8519
8520 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8521 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236 8522 if (!ret) {
57d104c1 8523 ufshcd_set_link_hibern8(hba);
4db7a236
CG		 8524 } else {
8525 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8526 __func__, ret);
57d104c1 8527 goto out;
4db7a236 8528 }
57d104c1
SJ		 8529 }
8530 /*
8531 * If autobkops is enabled, link can't be turned off because
fe1d4c2e
AH		 8532 * turning off the link would also turn off the device, except in the
8533 * case of DeepSleep where the device is expected to remain powered.
57d104c1
SJ		 8534 */
8535 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
dc30c9e6 8536 (!check_for_bkops || !hba->auto_bkops_enabled)) {
f3099fbd
YG		 8537 /*
8538 * Let's make sure that link is in low power mode, we are doing
8539 * this currently by putting the link in Hibern8. Otherway to
8540 * put the link in low power mode is to send the DME end point
8541 * to device and then send the DME reset command to local
8542 * unipro. But putting the link in hibern8 is much faster.
fe1d4c2e
AH		 8543 *
8544 * Note also that putting the link in Hibern8 is a requirement
8545 * for entering DeepSleep.
f3099fbd
YG		 8546 */
8547 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236
CG		 8548 if (ret) {
8549 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8550 __func__, ret);
f3099fbd 8551 goto out;
4db7a236 8552 }
57d104c1
SJ		 8553 /*
8554 * Change controller state to "reset state" which
8555 * should also put the link in off/reset state
8556 */
5cac1095 8557 ufshcd_hba_stop(hba);
57d104c1
SJ		 8558 /*
8559 * TODO: Check if we need any delay to make sure that
8560 * controller is reset
8561 */
8562 ufshcd_set_link_off(hba);
8563 }
8564
8565out:
8566 return ret;
8567}
8568
8569static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8570{
c4df6eed
SC		 8571 bool vcc_off = false;
8572
b799fdf7
YG		 8573 /*
8574 * It seems some UFS devices may keep drawing more than sleep current
8575 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8576 * To avoid this situation, add 2ms delay before putting these UFS
8577 * rails in LPM mode.
8578 */
8579 if (!ufshcd_is_link_active(hba) &&
8580 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8581 usleep_range(2000, 2100);
8582
57d104c1
SJ		 8583 /*
8584 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8585 * power.
8586 *
8587 * If UFS device and link is in OFF state, all power supplies (VCC,
8588 * VCCQ, VCCQ2) can be turned off if power on write protect is not
8589 * required. If UFS link is inactive (Hibern8 or OFF state) and device
8590 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8591 *
8592 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8593 * in low power state which would save some power.
3d17b9b5
AD		 8594 *
8595 * If Write Booster is enabled and the device needs to flush the WB
8596 * buffer OR if bkops status is urgent for WB, keep Vcc on.
57d104c1
SJ		 8597 */
8598 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8599 !hba->dev_info.is_lu_power_on_wp) {
8600 ufshcd_setup_vreg(hba, false);
c4df6eed 8601 vcc_off = true;
57d104c1 8602 } else if (!ufshcd_is_ufs_dev_active(hba)) {
51dd905b 8603 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
c4df6eed 8604 vcc_off = true;
57d104c1
SJ		 8605 if (!ufshcd_is_link_active(hba)) {
8606 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8607 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8608 }
8609 }
c4df6eed
SC		 8610
8611 /*
8612 * Some UFS devices require delay after VCC power rail is turned-off.
8613 */
8614 if (vcc_off && hba->vreg_info.vcc &&
8615 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8616 usleep_range(5000, 5100);
57d104c1
SJ		 8617}
8618
8619static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8620{
8621 int ret = 0;
8622
8623 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8624 !hba->dev_info.is_lu_power_on_wp) {
8625 ret = ufshcd_setup_vreg(hba, true);
8626 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 8627 if (!ret && !ufshcd_is_link_active(hba)) {
8628 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8629 if (ret)
8630 goto vcc_disable;
8631 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8632 if (ret)
8633 goto vccq_lpm;
8634 }
69d72ac8 8635 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 8636 }
8637 goto out;
8638
8639vccq_lpm:
8640 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8641vcc_disable:
8642 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8643out:
8644 return ret;
8645}
8646
8647static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8648{
dd7143e2 8649 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
57d104c1
SJ		 8650 ufshcd_setup_hba_vreg(hba, false);
8651}
8652
8653static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8654{
dd7143e2 8655 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
57d104c1
SJ		 8656 ufshcd_setup_hba_vreg(hba, true);
8657}
8658
7a3e97b0 8659/**
57d104c1 8660 * ufshcd_suspend - helper function for suspend operations
3b1d0580 8661 * @hba: per adapter instance
57d104c1
SJ		 8662 * @pm_op: desired low power operation type
8663 *
8664 * This function will try to put the UFS device and link into low power
8665 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
8666 * (System PM level).
8667 *
8668 * If this function is called during shutdown, it will make sure that
8669 * both UFS device and UFS link is powered off.
7a3e97b0 8670 *
57d104c1
SJ		 8671 * NOTE: UFS device & link must be active before we enter in this function.
8672 *
8673 * Returns 0 for success and non-zero for failure
7a3e97b0 8674 */
57d104c1 8675static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8676{
57d104c1 8677 int ret = 0;
fe1d4c2e 8678 int check_for_bkops;
57d104c1
SJ		 8679 enum ufs_pm_level pm_lvl;
8680 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8681 enum uic_link_state req_link_state;
8682
8683 hba->pm_op_in_progress = 1;
8684 if (!ufshcd_is_shutdown_pm(pm_op)) {
8685 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
8686 hba->rpm_lvl : hba->spm_lvl;
8687 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8688 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8689 } else {
8690 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8691 req_link_state = UIC_LINK_OFF_STATE;
8692 }
8693
7a3e97b0 8694 /*
57d104c1
SJ		 8695 * If we can't transition into any of the low power modes
8696 * just gate the clocks.
7a3e97b0 8697 */
1ab27c9c
ST		 8698 ufshcd_hold(hba, false);
8699 hba->clk_gating.is_suspended = true;
8700
348e1bc5
SC		 8701 if (ufshcd_is_clkscaling_supported(hba))
8702 ufshcd_clk_scaling_suspend(hba, true);
d6fcf81a 8703
57d104c1
SJ		 8704 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8705 req_link_state == UIC_LINK_ACTIVE_STATE) {
8706 goto disable_clks;
8707 }
7a3e97b0 8708
57d104c1
SJ		 8709 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8710 (req_link_state == hba->uic_link_state))
d6fcf81a 8711 goto enable_gating;
57d104c1
SJ		 8712
8713 /* UFS device & link must be active before we enter in this function */
8714 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8715 ret = -EINVAL;
d6fcf81a 8716 goto enable_gating;
57d104c1
SJ		 8717 }
8718
8719 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 8720 if (ufshcd_can_autobkops_during_suspend(hba)) {
8721 /*
8722 * The device is idle with no requests in the queue,
8723 * allow background operations if bkops status shows
8724 * that performance might be impacted.
8725 */
8726 ret = ufshcd_urgent_bkops(hba);
8727 if (ret)
8728 goto enable_gating;
8729 } else {
8730 /* make sure that auto bkops is disabled */
8731 ufshcd_disable_auto_bkops(hba);
8732 }
3d17b9b5 8733 /*
51dd905b
SC		 8734 * If device needs to do BKOP or WB buffer flush during
8735 * Hibern8, keep device power mode as "active power mode"
8736 * and VCC supply.
3d17b9b5 8737 */
51dd905b
SC		 8738 hba->dev_info.b_rpm_dev_flush_capable =
8739 hba->auto_bkops_enabled ||
8740 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8741 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8742 ufshcd_is_auto_hibern8_enabled(hba))) &&
8743 ufshcd_wb_need_flush(hba));
8744 }
8745
6948a96a
KK		 8746 flush_work(&hba->eeh_work);
8747
51dd905b 8748 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
939785d3 8749 if (!ufshcd_is_runtime_pm(pm_op))
51dd905b
SC		 8750 /* ensure that bkops is disabled */
8751 ufshcd_disable_auto_bkops(hba);
57d104c1 8752
51dd905b
SC		 8753 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8754 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8755 if (ret)
8756 goto enable_gating;
8757 }
57d104c1
SJ		 8758 }
8759
fe1d4c2e
AH		 8760 /*
8761 * In the case of DeepSleep, the device is expected to remain powered
8762 * with the link off, so do not check for bkops.
8763 */
8764 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
8765 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
57d104c1
SJ		 8766 if (ret)
8767 goto set_dev_active;
8768
57d104c1
SJ		 8769disable_clks:
8770 /*
8771 * Call vendor specific suspend callback. As these callbacks may access
8772 * vendor specific host controller register space call them before the
8773 * host clocks are ON.
8774 */
0263bcd0
YG		 8775 ret = ufshcd_vops_suspend(hba, pm_op);
8776 if (ret)
8777 goto set_link_active;
dcb6cec5
SC		 8778 /*
8779 * Disable the host irq as host controller as there won't be any
8780 * host controller transaction expected till resume.
8781 */
8782 ufshcd_disable_irq(hba);
57d104c1 8783
81309c24 8784 ufshcd_setup_clocks(hba, false);
57d104c1 8785
2dec9475
CG		 8786 if (ufshcd_is_clkgating_allowed(hba)) {
8787 hba->clk_gating.state = CLKS_OFF;
8788 trace_ufshcd_clk_gating(dev_name(hba->dev),
8789 hba->clk_gating.state);
8790 }
dcb6cec5 8791
528db9e5
ZC		 8792 ufshcd_vreg_set_lpm(hba);
8793
57d104c1
SJ		 8794 /* Put the host controller in low power mode if possible */
8795 ufshcd_hba_vreg_set_lpm(hba);
8796 goto out;
8797
57d104c1
SJ		 8798set_link_active:
8799 ufshcd_vreg_set_hpm(hba);
fe1d4c2e
AH		 8800 /*
8801 * Device hardware reset is required to exit DeepSleep. Also, for
8802 * DeepSleep, the link is off so host reset and restore will be done
8803 * further below.
8804 */
8805 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
31a5d9ca 8806 ufshcd_device_reset(hba);
fe1d4c2e
AH		 8807 WARN_ON(!ufshcd_is_link_off(hba));
8808 }
57d104c1
SJ		 8809 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
8810 ufshcd_set_link_active(hba);
8811 else if (ufshcd_is_link_off(hba))
8812 ufshcd_host_reset_and_restore(hba);
8813set_dev_active:
fe1d4c2e
AH		 8814 /* Can also get here needing to exit DeepSleep */
8815 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
31a5d9ca 8816 ufshcd_device_reset(hba);
fe1d4c2e
AH		 8817 ufshcd_host_reset_and_restore(hba);
8818 }
57d104c1
SJ		 8819 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
8820 ufshcd_disable_auto_bkops(hba);
1ab27c9c 8821enable_gating:
348e1bc5
SC		 8822 if (ufshcd_is_clkscaling_supported(hba))
8823 ufshcd_clk_scaling_suspend(hba, false);
8824
1ab27c9c 8825 hba->clk_gating.is_suspended = false;
51dd905b 8826 hba->dev_info.b_rpm_dev_flush_capable = false;
4ee7ee53 8827 ufshcd_clear_ua_wluns(hba);
1ab27c9c 8828 ufshcd_release(hba);
57d104c1 8829out:
51dd905b
SC		 8830 if (hba->dev_info.b_rpm_dev_flush_capable) {
8831 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
8832 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
8833 }
8834
57d104c1 8835 hba->pm_op_in_progress = 0;
51dd905b 8836
8808b4e9 8837 if (ret)
e965e5e0 8838 ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret);
57d104c1 8839 return ret;
7a3e97b0
SY		 8840}
8841
8842/**
57d104c1 8843 * ufshcd_resume - helper function for resume operations
3b1d0580 8844 * @hba: per adapter instance
57d104c1 8845 * @pm_op: runtime PM or system PM
7a3e97b0 8846 *
57d104c1
SJ		 8847 * This function basically brings the UFS device, UniPro link and controller
8848 * to active state.
8849 *
8850 * Returns 0 for success and non-zero for failure
7a3e97b0 8851 */
57d104c1 8852static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8853{
57d104c1
SJ		 8854 int ret;
8855 enum uic_link_state old_link_state;
8856
8857 hba->pm_op_in_progress = 1;
8858 old_link_state = hba->uic_link_state;
8859
8860 ufshcd_hba_vreg_set_hpm(hba);
528db9e5
ZC		 8861 ret = ufshcd_vreg_set_hpm(hba);
8862 if (ret)
8863 goto out;
8864
57d104c1
SJ		 8865 /* Make sure clocks are enabled before accessing controller */
8866 ret = ufshcd_setup_clocks(hba, true);
8867 if (ret)
528db9e5 8868 goto disable_vreg;
57d104c1 8869
57d104c1 8870 /* enable the host irq as host controller would be active soon */
5231d38c 8871 ufshcd_enable_irq(hba);
57d104c1 8872
7a3e97b0 8873 /*
57d104c1
SJ		 8874 * Call vendor specific resume callback. As these callbacks may access
8875 * vendor specific host controller register space call them when the
8876 * host clocks are ON.
7a3e97b0 8877 */
0263bcd0
YG		 8878 ret = ufshcd_vops_resume(hba, pm_op);
8879 if (ret)
528db9e5 8880 goto disable_irq_and_vops_clks;
57d104c1 8881
fe1d4c2e
AH		 8882 /* For DeepSleep, the only supported option is to have the link off */
8883 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
8884
57d104c1
SJ		 8885 if (ufshcd_is_link_hibern8(hba)) {
8886 ret = ufshcd_uic_hibern8_exit(hba);
4db7a236 8887 if (!ret) {
57d104c1 8888 ufshcd_set_link_active(hba);
4db7a236
CG		 8889 } else {
8890 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
8891 __func__, ret);
57d104c1 8892 goto vendor_suspend;
4db7a236 8893 }
57d104c1 8894 } else if (ufshcd_is_link_off(hba)) {
57d104c1 8895 /*
089f5b64
AD		 8896 * A full initialization of the host and the device is
8897 * required since the link was put to off during suspend.
fe1d4c2e
AH		 8898 * Note, in the case of DeepSleep, the device will exit
8899 * DeepSleep due to device reset.
089f5b64
AD		 8900 */
8901 ret = ufshcd_reset_and_restore(hba);
8902 /*
8903 * ufshcd_reset_and_restore() should have already
57d104c1
SJ		 8904 * set the link state as active
8905 */
8906 if (ret || !ufshcd_is_link_active(hba))
8907 goto vendor_suspend;
8908 }
8909
8910 if (!ufshcd_is_ufs_dev_active(hba)) {
8911 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8912 if (ret)
8913 goto set_old_link_state;
8914 }
8915
4e768e76 8916 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8917 ufshcd_enable_auto_bkops(hba);
8918 else
8919 /*
8920 * If BKOPs operations are urgently needed at this moment then
8921 * keep auto-bkops enabled or else disable it.
8922 */
8923 ufshcd_urgent_bkops(hba);
8924
1ab27c9c
ST		 8925 hba->clk_gating.is_suspended = false;
8926
348e1bc5
SC		 8927 if (ufshcd_is_clkscaling_supported(hba))
8928 ufshcd_clk_scaling_suspend(hba, false);
856b3483 8929
ad448378
AH		 8930 /* Enable Auto-Hibernate if configured */
8931 ufshcd_auto_hibern8_enable(hba);
8932
51dd905b
SC		 8933 if (hba->dev_info.b_rpm_dev_flush_capable) {
8934 hba->dev_info.b_rpm_dev_flush_capable = false;
8935 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
8936 }
8937
4ee7ee53
JK		 8938 ufshcd_clear_ua_wluns(hba);
8939
71d848b8
CG		 8940 /* Schedule clock gating in case of no access to UFS device yet */
8941 ufshcd_release(hba);
8942
57d104c1
SJ		 8943 goto out;
8944
8945set_old_link_state:
8946 ufshcd_link_state_transition(hba, old_link_state, 0);
8947vendor_suspend:
0263bcd0 8948 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 8949disable_irq_and_vops_clks:
8950 ufshcd_disable_irq(hba);
57d104c1 8951 ufshcd_setup_clocks(hba, false);
2dec9475
CG		 8952 if (ufshcd_is_clkgating_allowed(hba)) {
8953 hba->clk_gating.state = CLKS_OFF;
8954 trace_ufshcd_clk_gating(dev_name(hba->dev),
8955 hba->clk_gating.state);
8956 }
528db9e5
ZC		 8957disable_vreg:
8958 ufshcd_vreg_set_lpm(hba);
57d104c1
SJ		 8959out:
8960 hba->pm_op_in_progress = 0;
8808b4e9 8961 if (ret)
e965e5e0 8962 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
57d104c1
SJ		 8963 return ret;
8964}
8965
8966/**
8967 * ufshcd_system_suspend - system suspend routine
8968 * @hba: per adapter instance
57d104c1
SJ		 8969 *
8970 * Check the description of ufshcd_suspend() function for more details.
8971 *
8972 * Returns 0 for success and non-zero for failure
8973 */
8974int ufshcd_system_suspend(struct ufs_hba *hba)
8975{
8976 int ret = 0;
7ff5ab47 8977 ktime_t start = ktime_get();
57d104c1 8978
9cd20d3f 8979 down(&hba->host_sem);
fb7afe24
CG		 8980
8981 if (!hba->is_powered)
233b594b 8982 return 0;
57d104c1 8983
0b257734 8984 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8985 hba->curr_dev_pwr_mode) &&
8986 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
1d53864c
SC		 8987 hba->uic_link_state) &&
8988 !hba->dev_info.b_rpm_dev_flush_capable)
0b257734 8989 goto out;
57d104c1 8990
0b257734 8991 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 8992 /*
8993 * UFS device and/or UFS link low power states during runtime
8994 * suspend seems to be different than what is expected during
8995 * system suspend. Hence runtime resume the devic & link and
8996 * let the system suspend low power states to take effect.
8997 * TODO: If resume takes longer time, we might have optimize
8998 * it in future by not resuming everything if possible.
8999 */
9000 ret = ufshcd_runtime_resume(hba);
9001 if (ret)
9002 goto out;
9003 }
9004
9005 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
9006out:
7ff5ab47 9007 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
9008 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9009 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 9010 if (!ret)
9011 hba->is_sys_suspended = true;
88a92d6a 9012 else
9cd20d3f 9013 up(&hba->host_sem);
57d104c1
SJ		 9014 return ret;
9015}
9016EXPORT_SYMBOL(ufshcd_system_suspend);
9017
9018/**
9019 * ufshcd_system_resume - system resume routine
9020 * @hba: per adapter instance
9021 *
9022 * Returns 0 for success and non-zero for failure
9023 */
7a3e97b0 9024
57d104c1
SJ		 9025int ufshcd_system_resume(struct ufs_hba *hba)
9026{
7ff5ab47 9027 int ret = 0;
9028 ktime_t start = ktime_get();
9029
e3ce73d6 9030 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 9031 /*
9032 * Let the runtime resume take care of resuming
9033 * if runtime suspended.
9034 */
7ff5ab47 9035 goto out;
9036 else
9037 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
9038out:
9039 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
9040 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9041 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 9042 if (!ret)
9043 hba->is_sys_suspended = false;
9cd20d3f 9044 up(&hba->host_sem);
7ff5ab47 9045 return ret;
7a3e97b0 9046}
57d104c1 9047EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 9048
57d104c1
SJ		 9049/**
9050 * ufshcd_runtime_suspend - runtime suspend routine
9051 * @hba: per adapter instance
9052 *
9053 * Check the description of ufshcd_suspend() function for more details.
9054 *
9055 * Returns 0 for success and non-zero for failure
9056 */
66ec6d59
SRT		 9057int ufshcd_runtime_suspend(struct ufs_hba *hba)
9058{
7ff5ab47 9059 int ret = 0;
9060 ktime_t start = ktime_get();
9061
e3ce73d6 9062 if (!hba->is_powered)
7ff5ab47 9063 goto out;
9064 else
9065 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
9066out:
9067 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
9068 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9069 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 9070 return ret;
66ec6d59
SRT		 9071}
9072EXPORT_SYMBOL(ufshcd_runtime_suspend);
9073
57d104c1
SJ		 9074/**
9075 * ufshcd_runtime_resume - runtime resume routine
9076 * @hba: per adapter instance
9077 *
9078 * This function basically brings the UFS device, UniPro link and controller
9079 * to active state. Following operations are done in this function:
9080 *
9081 * 1. Turn on all the controller related clocks
9082 * 2. Bring the UniPro link out of Hibernate state
9083 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
9084 * to active state.
9085 * 4. If auto-bkops is enabled on the device, disable it.
9086 *
9087 * So following would be the possible power state after this function return
9088 * successfully:
9089 * S1: UFS device in Active state with VCC rail ON
9090 * UniPro link in Active state
9091 * All the UFS/UniPro controller clocks are ON
9092 *
9093 * Returns 0 for success and non-zero for failure
9094 */
66ec6d59
SRT		 9095int ufshcd_runtime_resume(struct ufs_hba *hba)
9096{
7ff5ab47 9097 int ret = 0;
9098 ktime_t start = ktime_get();
9099
e3ce73d6 9100 if (!hba->is_powered)
7ff5ab47 9101 goto out;
9102 else
9103 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
9104out:
9105 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
9106 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9107 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 9108 return ret;
66ec6d59
SRT		 9109}
9110EXPORT_SYMBOL(ufshcd_runtime_resume);
9111
9112int ufshcd_runtime_idle(struct ufs_hba *hba)
9113{
9114 return 0;
9115}
9116EXPORT_SYMBOL(ufshcd_runtime_idle);
9117
57d104c1
SJ		 9118/**
9119 * ufshcd_shutdown - shutdown routine
9120 * @hba: per adapter instance
9121 *
9122 * This function would power off both UFS device and UFS link.
9123 *
9124 * Returns 0 always to allow force shutdown even in case of errors.
9125 */
9126int ufshcd_shutdown(struct ufs_hba *hba)
9127{
9128 int ret = 0;
9129
9cd20d3f
CG		 9130 down(&hba->host_sem);
9131 hba->shutting_down = true;
9132 up(&hba->host_sem);
9133
f51913ee
SC		 9134 if (!hba->is_powered)
9135 goto out;
9136
57d104c1
SJ		 9137 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9138 goto out;
9139
e92643db 9140 pm_runtime_get_sync(hba->dev);
57d104c1
SJ		 9141
9142 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
9143out:
9144 if (ret)
9145 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
88a92d6a 9146 hba->is_powered = false;
57d104c1
SJ		 9147 /* allow force shutdown even in case of errors */
9148 return 0;
9149}
9150EXPORT_SYMBOL(ufshcd_shutdown);
9151
7a3e97b0 9152/**
3b1d0580 9153 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 9154 * data structure memory
8aa29f19 9155 * @hba: per adapter instance
7a3e97b0 9156 */
3b1d0580 9157void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 9158{
df032bf2 9159 ufs_bsg_remove(hba);
cbb6813e 9160 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 9161 blk_cleanup_queue(hba->tmf_queue);
9162 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360 9163 blk_cleanup_queue(hba->cmd_queue);
cfdf9c91 9164 scsi_remove_host(hba->host);
7a3e97b0 9165 /* disable interrupts */
2fbd009b 9166 ufshcd_disable_intr(hba, hba->intr_mask);
5cac1095 9167 ufshcd_hba_stop(hba);
aa497613 9168 ufshcd_hba_exit(hba);
3b1d0580
VH		 9169}
9170EXPORT_SYMBOL_GPL(ufshcd_remove);
9171
47555a5c
YG		 9172/**
9173 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9174 * @hba: pointer to Host Bus Adapter (HBA)
9175 */
9176void ufshcd_dealloc_host(struct ufs_hba *hba)
9177{
9178 scsi_host_put(hba->host);
9179}
9180EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9181
ca3d7bf9
AM		 9182/**
9183 * ufshcd_set_dma_mask - Set dma mask based on the controller
9184 * addressing capability
9185 * @hba: per adapter instance
9186 *
9187 * Returns 0 for success, non-zero for failure
9188 */
9189static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9190{
9191 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9192 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9193 return 0;
9194 }
9195 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9196}
9197
7a3e97b0 9198/**
5c0c28a8 9199 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 9200 * @dev: pointer to device handle
9201 * @hba_handle: driver private handle
7a3e97b0
SY		 9202 * Returns 0 on success, non-zero value on failure
9203 */
5c0c28a8 9204int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 9205{
9206 struct Scsi_Host *host;
9207 struct ufs_hba *hba;
5c0c28a8 9208 int err = 0;
7a3e97b0 9209
3b1d0580
VH		 9210 if (!dev) {
9211 dev_err(dev,
9212 "Invalid memory reference for dev is NULL\n");
9213 err = -ENODEV;
7a3e97b0
SY		 9214 goto out_error;
9215 }
9216
7a3e97b0
SY		 9217 host = scsi_host_alloc(&ufshcd_driver_template,
9218 sizeof(struct ufs_hba));
9219 if (!host) {
3b1d0580 9220 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 9221 err = -ENOMEM;
3b1d0580 9222 goto out_error;
7a3e97b0
SY		 9223 }
9224 hba = shost_priv(host);
7a3e97b0 9225 hba->host = host;
3b1d0580 9226 hba->dev = dev;
5c0c28a8 9227 *hba_handle = hba;
9e1e8a75 9228 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 9229
566ec9ad
SM		 9230 INIT_LIST_HEAD(&hba->clk_list_head);
9231
5c0c28a8
SRT		 9232out_error:
9233 return err;
9234}
9235EXPORT_SYMBOL(ufshcd_alloc_host);
9236
69a6c269
BVA		 9237/* This function exists because blk_mq_alloc_tag_set() requires this. */
9238static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9239 const struct blk_mq_queue_data *qd)
9240{
9241 WARN_ON_ONCE(true);
9242 return BLK_STS_NOTSUPP;
9243}
9244
9245static const struct blk_mq_ops ufshcd_tmf_ops = {
9246 .queue_rq = ufshcd_queue_tmf,
9247};
9248
5c0c28a8
SRT		 9249/**
9250 * ufshcd_init - Driver initialization routine
9251 * @hba: per-adapter instance
9252 * @mmio_base: base register address
9253 * @irq: Interrupt line of device
9254 * Returns 0 on success, non-zero value on failure
9255 */
9256int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9257{
9258 int err;
9259 struct Scsi_Host *host = hba->host;
9260 struct device *dev = hba->dev;
4db7a236 9261 char eh_wq_name[sizeof("ufs_eh_wq_00")];
5c0c28a8
SRT		 9262
9263 if (!mmio_base) {
9264 dev_err(hba->dev,
9265 "Invalid memory reference for mmio_base is NULL\n");
9266 err = -ENODEV;
9267 goto out_error;
9268 }
9269
3b1d0580
VH		 9270 hba->mmio_base = mmio_base;
9271 hba->irq = irq;
90b8491c 9272 hba->vps = &ufs_hba_vps;
7a3e97b0 9273
aa497613 9274 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 9275 if (err)
9276 goto out_error;
9277
7a3e97b0 9278 /* Read capabilities registers */
df043c74
ST		 9279 err = ufshcd_hba_capabilities(hba);
9280 if (err)
9281 goto out_disable;
7a3e97b0
SY		 9282
9283 /* Get UFS version supported by the controller */
9284 hba->ufs_version = ufshcd_get_ufs_version(hba);
9285
c01848c6
YG		 9286 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
9287 (hba->ufs_version != UFSHCI_VERSION_11) &&
9288 (hba->ufs_version != UFSHCI_VERSION_20) &&
9289 (hba->ufs_version != UFSHCI_VERSION_21))
9290 dev_err(hba->dev, "invalid UFS version 0x%x\n",
9291 hba->ufs_version);
9292
2fbd009b
SJ		 9293 /* Get Interrupt bit mask per version */
9294 hba->intr_mask = ufshcd_get_intr_mask(hba);
9295
ca3d7bf9
AM		 9296 err = ufshcd_set_dma_mask(hba);
9297 if (err) {
9298 dev_err(hba->dev, "set dma mask failed\n");
9299 goto out_disable;
9300 }
9301
7a3e97b0
SY		 9302 /* Allocate memory for host memory space */
9303 err = ufshcd_memory_alloc(hba);
9304 if (err) {
3b1d0580
VH		 9305 dev_err(hba->dev, "Memory allocation failed\n");
9306 goto out_disable;
7a3e97b0
SY		 9307 }
9308
9309 /* Configure LRB */
9310 ufshcd_host_memory_configure(hba);
9311
9312 host->can_queue = hba->nutrs;
9313 host->cmd_per_lun = hba->nutrs;
9314 host->max_id = UFSHCD_MAX_ID;
0ce147d4 9315 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 9316 host->max_channel = UFSHCD_MAX_CHANNEL;
9317 host->unique_id = host->host_no;
a851b2bd 9318 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 9319
7eb584db
DR		 9320 hba->max_pwr_info.is_valid = false;
9321
7a3e97b0 9322 /* Initialize work queues */
4db7a236
CG		 9323 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9324 hba->host->host_no);
9325 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9326 if (!hba->eh_wq) {
9327 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9328 __func__);
9329 err = -ENOMEM;
9330 goto out_disable;
9331 }
e8e7f271 9332 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 9333 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 9334
9cd20d3f 9335 sema_init(&hba->host_sem, 1);
88a92d6a 9336
6ccf44fe
SJ		 9337 /* Initialize UIC command mutex */
9338 mutex_init(&hba->uic_cmd_mutex);
9339
5a0b0cb9
SRT		 9340 /* Initialize mutex for device management commands */
9341 mutex_init(&hba->dev_cmd.lock);
9342
a3cd5ec5 9343 init_rwsem(&hba->clk_scaling_lock);
9344
1ab27c9c 9345 ufshcd_init_clk_gating(hba);
199ef13c 9346
eebcc196
VG		 9347 ufshcd_init_clk_scaling(hba);
9348
199ef13c
YG		 9349 /*
9350 * In order to avoid any spurious interrupt immediately after
9351 * registering UFS controller interrupt handler, clear any pending UFS
9352 * interrupt status and disable all the UFS interrupts.
9353 */
9354 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9355 REG_INTERRUPT_STATUS);
9356 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9357 /*
9358 * Make sure that UFS interrupts are disabled and any pending interrupt
9359 * status is cleared before registering UFS interrupt handler.
9360 */
9361 mb();
9362
7a3e97b0 9363 /* IRQ registration */
2953f850 9364 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 9365 if (err) {
3b1d0580 9366 dev_err(hba->dev, "request irq failed\n");
4543d9d7 9367 goto out_disable;
57d104c1
SJ		 9368 } else {
9369 hba->is_irq_enabled = true;
7a3e97b0
SY		 9370 }
9371
3b1d0580 9372 err = scsi_add_host(host, hba->dev);
7a3e97b0 9373 if (err) {
3b1d0580 9374 dev_err(hba->dev, "scsi_add_host failed\n");
4543d9d7 9375 goto out_disable;
7a3e97b0
SY		 9376 }
9377
7252a360
BVA		 9378 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
9379 if (IS_ERR(hba->cmd_queue)) {
9380 err = PTR_ERR(hba->cmd_queue);
9381 goto out_remove_scsi_host;
9382 }
9383
69a6c269
BVA		 9384 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9385 .nr_hw_queues = 1,
9386 .queue_depth = hba->nutmrs,
9387 .ops = &ufshcd_tmf_ops,
9388 .flags = BLK_MQ_F_NO_SCHED,
9389 };
9390 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9391 if (err < 0)
9392 goto free_cmd_queue;
9393 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9394 if (IS_ERR(hba->tmf_queue)) {
9395 err = PTR_ERR(hba->tmf_queue);
9396 goto free_tmf_tag_set;
9397 }
9398
d8d9f793 9399 /* Reset the attached device */
31a5d9ca 9400 ufshcd_device_reset(hba);
d8d9f793 9401
df043c74
ST		 9402 ufshcd_init_crypto(hba);
9403
6ccf44fe
SJ		 9404 /* Host controller enable */
9405 err = ufshcd_hba_enable(hba);
7a3e97b0 9406 if (err) {
6ccf44fe 9407 dev_err(hba->dev, "Host controller enable failed\n");
e965e5e0 9408 ufshcd_print_evt_hist(hba);
6ba65588 9409 ufshcd_print_host_state(hba);
69a6c269 9410 goto free_tmf_queue;
7a3e97b0 9411 }
6ccf44fe 9412
0c8f7586 9413 /*
9414 * Set the default power management level for runtime and system PM.
9415 * Default power saving mode is to keep UFS link in Hibern8 state
9416 * and UFS device in sleep state.
9417 */
9418 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9419 UFS_SLEEP_PWR_MODE,
9420 UIC_LINK_HIBERN8_STATE);
9421 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9422 UFS_SLEEP_PWR_MODE,
9423 UIC_LINK_HIBERN8_STATE);
9424
51dd905b
SC		 9425 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9426 ufshcd_rpm_dev_flush_recheck_work);
9427
ad448378 9428 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 9429 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 9430 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9431 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9432 }
9433
62694735
SRT		 9434 /* Hold auto suspend until async scan completes */
9435 pm_runtime_get_sync(dev);
38135535 9436 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 9437 /*
7caf489b 9438 * We are assuming that device wasn't put in sleep/power-down
9439 * state exclusively during the boot stage before kernel.
9440 * This assumption helps avoid doing link startup twice during
9441 * ufshcd_probe_hba().
57d104c1 9442 */
7caf489b 9443 ufshcd_set_ufs_dev_active(hba);
57d104c1 9444
6ccf44fe 9445 async_schedule(ufshcd_async_scan, hba);
cbb6813e 9446 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 9447
7a3e97b0
SY		 9448 return 0;
9449
69a6c269
BVA		 9450free_tmf_queue:
9451 blk_cleanup_queue(hba->tmf_queue);
9452free_tmf_tag_set:
9453 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360
BVA		 9454free_cmd_queue:
9455 blk_cleanup_queue(hba->cmd_queue);
3b1d0580
VH		 9456out_remove_scsi_host:
9457 scsi_remove_host(hba->host);
3b1d0580 9458out_disable:
57d104c1 9459 hba->is_irq_enabled = false;
aa497613 9460 ufshcd_hba_exit(hba);
3b1d0580
VH		 9461out_error:
9462 return err;
9463}
9464EXPORT_SYMBOL_GPL(ufshcd_init);
9465
b6cacaf2
AH		 9466static int __init ufshcd_core_init(void)
9467{
9468 ufs_debugfs_init();
9469 return 0;
9470}
9471
9472static void __exit ufshcd_core_exit(void)
9473{
9474 ufs_debugfs_exit();
9475}
9476
9477module_init(ufshcd_core_init);
9478module_exit(ufshcd_core_exit);
9479
3b1d0580
VH		 9480MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9481MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 9482MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 9483MODULE_LICENSE("GPL");
9484MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Linux 6.x block layer and io_uring tree(s)