projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: core: Refine how we set tag_set NUMA node
[linux-block.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
67351119 1// SPDX-License-Identifier: GPL-2.0-or-later
7a3e97b0 2/*
e0eca63e 3 * Universal Flash Storage Host controller driver Core
3b1d0580 4 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 6 *
3b1d0580
VH		 7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 10 */
11
6ccf44fe 12#include <linux/async.h>
856b3483 13#include <linux/devfreq.h>
b573d484 14#include <linux/nls.h>
54b879b7 15#include <linux/of.h>
ad448378 16#include <linux/bitfield.h>
fb276f77 17#include <linux/blk-pm.h>
c72e79c0 18#include <linux/blkdev.h>
3f06f780
BVA		 19#include <linux/clk.h>
20#include <linux/delay.h>
21#include <linux/interrupt.h>
22#include <linux/module.h>
23#include <linux/nls.h>
24#include <linux/regulator/consumer.h>
25#include <scsi/scsi_cmnd.h>
26#include <scsi/scsi_dbg.h>
b294ff3e 27#include <scsi/scsi_driver.h>
3f06f780 28#include <scsi/scsi_eh.h>
4bc26113 29#include "ufshcd-priv.h"
c58ab7aa 30#include "ufs_quirks.h"
53b3d9c3 31#include "unipro.h"
cbb6813e 32#include "ufs-sysfs.h"
b6cacaf2 33#include "ufs-debugfs.h"
c11a1ae9 34#include "ufs-fault-injection.h"
df032bf2 35#include "ufs_bsg.h"
df043c74 36#include "ufshcd-crypto.h"
f02bc975 37#include "ufshpb.h"
3d17b9b5 38#include <asm/unaligned.h>
7a3e97b0 39
7ff5ab47 40#define CREATE_TRACE_POINTS
41#include <trace/events/ufs.h>
42
2fbd009b
SJ		 43#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
44 UTP_TASK_REQ_COMPL |\
45 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 46/* UIC command timeout, unit: ms */
47#define UIC_CMD_TIMEOUT 500
2fbd009b 48
5a0b0cb9
SRT		 49/* NOP OUT retries waiting for NOP IN response */
50#define NOP_OUT_RETRIES 10
782e2efb
DP		 51/* Timeout after 50 msecs if NOP OUT hangs without response */
52#define NOP_OUT_TIMEOUT 50 /* msecs */
5a0b0cb9 53
68078d5c 54/* Query request retries */
10fe5888 55#define QUERY_REQ_RETRIES 3
68078d5c 56/* Query request timeout */
10fe5888 57#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 58
e2933132
SRT		 59/* Task management command timeout */
60#define TM_CMD_TIMEOUT 100 /* msecs */
61
64238fbd
YG		 62/* maximum number of retries for a general UIC command */
63#define UFS_UIC_COMMAND_RETRIES 3
64
1d337ec2
SRT		 65/* maximum number of link-startup retries */
66#define DME_LINKSTARTUP_RETRIES 3
67
87d0b4a6
YG		 68/* Maximum retries for Hibern8 enter */
69#define UIC_HIBERN8_ENTER_RETRIES 3
70
1d337ec2
SRT		 71/* maximum number of reset retries before giving up */
72#define MAX_HOST_RESET_RETRIES 5
73
87bf6a6b
AH		 74/* Maximum number of error handler retries before giving up */
75#define MAX_ERR_HANDLER_RETRIES 5
76
68078d5c
DR		 77/* Expose the flag value from utp_upiu_query.value */
78#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
79
7d568652
SJ		 80/* Interrupt aggregation default timeout, unit: 40us */
81#define INT_AGGR_DEF_TO 0x02
82
49615ba1
SC		 83/* default delay of autosuspend: 2000 ms */
84#define RPM_AUTOSUSPEND_DELAY_MS 2000
85
51dd905b
SC		 86/* Default delay of RPM device flush delayed work */
87#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
88
09f17791
CG		 89/* Default value of wait time before gating device ref clock */
90#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
91
29707fab
KK		 92/* Polling time to wait for fDeviceInit */
93#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
94
aa497613
SRT		 95#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
96 ({ \
97 int _ret; \
98 if (_on) \
99 _ret = ufshcd_enable_vreg(_dev, _vreg); \
100 else \
101 _ret = ufshcd_disable_vreg(_dev, _vreg); \
102 _ret; \
103 })
104
ba80917d
TW		 105#define ufshcd_hex_dump(prefix_str, buf, len) do { \
106 size_t __len = (len); \
107 print_hex_dump(KERN_ERR, prefix_str, \
108 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
109 16, 4, buf, __len, false); \
110} while (0)
111
112int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
113 const char *prefix)
114{
d6724756
MG		 115 u32 *regs;
116 size_t pos;
117
118 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
119 return -EINVAL;
ba80917d 120
cddaebaf 121 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 122 if (!regs)
123 return -ENOMEM;
124
d6724756
MG		 125 for (pos = 0; pos < len; pos += 4)
126 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
127
ba80917d
TW		 128 ufshcd_hex_dump(prefix, regs, len);
129 kfree(regs);
130
131 return 0;
132}
133EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 134
7a3e97b0
SY		 135enum {
136 UFSHCD_MAX_CHANNEL = 0,
137 UFSHCD_MAX_ID = 1,
945c3cca
BVA		 138 UFSHCD_NUM_RESERVED = 1,
139 UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED,
140 UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED,
7a3e97b0
SY		 141};
142
4693fad7
BVA		 143static const char *const ufshcd_state_name[] = {
144 [UFSHCD_STATE_RESET] = "reset",
145 [UFSHCD_STATE_OPERATIONAL] = "operational",
146 [UFSHCD_STATE_ERROR] = "error",
147 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal",
148 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal",
149};
150
3441da7d
SRT		 151/* UFSHCD error handling flags */
152enum {
153 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 154};
155
e8e7f271
SRT		 156/* UFSHCD UIC layer error flags */
157enum {
158 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 159 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
160 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
161 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
162 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
163 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
2355b66e 164 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
e8e7f271
SRT		 165};
166
3441da7d 167#define ufshcd_set_eh_in_progress(h) \
9c490d2d 168 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 169#define ufshcd_eh_in_progress(h) \
9c490d2d 170 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 171#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 172 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 173
cbb6813e 174struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
e2ac7ab2
BVA		 175 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
176 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
177 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
178 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
179 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
180 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
fe1d4c2e
AH		 181 /*
182 * For DeepSleep, the link is first put in hibern8 and then off.
183 * Leaving the link in hibern8 is not supported.
184 */
e2ac7ab2 185 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
57d104c1
SJ		 186};
187
188static inline enum ufs_dev_pwr_mode
189ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
190{
191 return ufs_pm_lvl_states[lvl].dev_state;
192}
193
194static inline enum uic_link_state
195ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
196{
197 return ufs_pm_lvl_states[lvl].link_state;
198}
199
0c8f7586 200static inline enum ufs_pm_level
201ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
202 enum uic_link_state link_state)
203{
204 enum ufs_pm_level lvl;
205
206 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
207 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
208 (ufs_pm_lvl_states[lvl].link_state == link_state))
209 return lvl;
210 }
211
212 /* if no match found, return the level 0 */
213 return UFS_PM_LVL_0;
214}
215
aead21f3 216static const struct ufs_dev_quirk ufs_fixups[] = {
56d4a186 217 /* UFS cards deviations table */
dd2cf44f
BVA		 218 { .wmanufacturerid = UFS_VENDOR_MICRON,
219 .model = UFS_ANY_MODEL,
220 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
221 UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ },
222 { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
223 .model = UFS_ANY_MODEL,
224 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
225 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
226 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
227 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
228 .model = UFS_ANY_MODEL,
229 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
230 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
231 .model = "hB8aL1" /*H28U62301AMR*/,
232 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
233 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
234 .model = UFS_ANY_MODEL,
235 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
236 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
237 .model = "THGLF2G9C8KBADG",
238 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
239 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
240 .model = "THGLF2G9D8KBADG",
241 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
242 {}
56d4a186
SJ		 243};
244
9333d775 245static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 246static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 247static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 248static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 249static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2 250static void ufshcd_hba_exit(struct ufs_hba *hba);
68444d73 251static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
1ab27c9c 252static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
cad2e03d 253static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 254static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 255static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
256static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 257static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 258static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 259static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 260static int ufshcd_change_power_mode(struct ufs_hba *hba,
261 struct ufs_pa_layer_attr *pwr_mode);
c72e79c0
CG		 262static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
263static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
264static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
265 struct ufs_vreg *vreg);
307348f6 266static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
3b5f3c0d
YH		 267static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
268static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
dd7143e2
CG		 269static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
270static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
3d17b9b5 271
5231d38c 272static inline void ufshcd_enable_irq(struct ufs_hba *hba)
57d104c1 273{
57d104c1 274 if (!hba->is_irq_enabled) {
5231d38c 275 enable_irq(hba->irq);
57d104c1
SJ		 276 hba->is_irq_enabled = true;
277 }
57d104c1
SJ		 278}
279
280static inline void ufshcd_disable_irq(struct ufs_hba *hba)
281{
282 if (hba->is_irq_enabled) {
5231d38c 283 disable_irq(hba->irq);
57d104c1
SJ		 284 hba->is_irq_enabled = false;
285 }
286}
3441da7d 287
3d17b9b5
AD		 288static inline void ufshcd_wb_config(struct ufs_hba *hba)
289{
79e3520f 290 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 291 return;
292
3b5f3c0d
YH		 293 ufshcd_wb_toggle(hba, true);
294
295 ufshcd_wb_toggle_flush_during_h8(hba, true);
21acf460
SC		 296 if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
297 ufshcd_wb_toggle_flush(hba, true);
3d17b9b5
AD		 298}
299
38135535
SJ		 300static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
301{
302 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
303 scsi_unblock_requests(hba->host);
304}
305
306static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
307{
308 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
309 scsi_block_requests(hba->host);
310}
311
6667e6d9 312static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
28fa68fc 313 enum ufs_trace_str_t str_t)
6667e6d9
OS		 314{
315 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
89ac2c3b 316 struct utp_upiu_header *header;
6667e6d9 317
9d5095e7
BH		 318 if (!trace_ufshcd_upiu_enabled())
319 return;
320
89ac2c3b
BH		 321 if (str_t == UFS_CMD_SEND)
322 header = &rq->header;
323 else
324 header = &hba->lrb[tag].ucd_rsp_ptr->header;
325
326 trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
867fdc2d 327 UFS_TSF_CDB);
6667e6d9
OS		 328}
329
fb475b74
AA		 330static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
331 enum ufs_trace_str_t str_t,
332 struct utp_upiu_req *rq_rsp)
6667e6d9 333{
9d5095e7
BH		 334 if (!trace_ufshcd_upiu_enabled())
335 return;
6667e6d9 336
be20b51c 337 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
867fdc2d 338 &rq_rsp->qr, UFS_TSF_OSF);
6667e6d9
OS		 339}
340
341static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
28fa68fc 342 enum ufs_trace_str_t str_t)
6667e6d9 343{
e8c2da7e 344 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
6667e6d9 345
9d5095e7
BH		 346 if (!trace_ufshcd_upiu_enabled())
347 return;
348
0ed083e9 349 if (str_t == UFS_TM_SEND)
1352eec8
GS		 350 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
351 &descp->upiu_req.req_header,
352 &descp->upiu_req.input_param1,
353 UFS_TSF_TM_INPUT);
0ed083e9 354 else
1352eec8
GS		 355 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
356 &descp->upiu_rsp.rsp_header,
357 &descp->upiu_rsp.output_param1,
358 UFS_TSF_TM_OUTPUT);
6667e6d9
OS		 359}
360
aa5c6979
SC		 361static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
362 struct uic_command *ucmd,
28fa68fc 363 enum ufs_trace_str_t str_t)
aa5c6979
SC		 364{
365 u32 cmd;
366
367 if (!trace_ufshcd_uic_command_enabled())
368 return;
369
28fa68fc 370 if (str_t == UFS_CMD_SEND)
aa5c6979
SC		 371 cmd = ucmd->command;
372 else
373 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
374
28fa68fc 375 trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
aa5c6979
SC		 376 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
377 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
378 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
379}
380
28fa68fc
BH		 381static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
382 enum ufs_trace_str_t str_t)
1a07f2d9 383{
2bd3b6b7 384 u64 lba = 0;
69a314d6 385 u8 opcode = 0, group_id = 0;
1a07f2d9 386 u32 intr, doorbell;
e7c3b379 387 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
e4d2add7 388 struct scsi_cmnd *cmd = lrbp->cmd;
3f2c1002 389 struct request *rq = scsi_cmd_to_rq(cmd);
1a07f2d9
LS		 390 int transfer_len = -1;
391
44b5de36
BH		 392 if (!cmd)
393 return;
394
44b5de36
BH		 395 /* trace UPIU also */
396 ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
f0101af4
BH		 397 if (!trace_ufshcd_command_enabled())
398 return;
399
44b5de36 400 opcode = cmd->cmnd[0];
44b5de36
BH		 401
402 if (opcode == READ_10 || opcode == WRITE_10) {
403 /*
404 * Currently we only fully trace read(10) and write(10) commands
405 */
406 transfer_len =
407 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
2bd3b6b7 408 lba = scsi_get_lba(cmd);
44b5de36
BH		 409 if (opcode == WRITE_10)
410 group_id = lrbp->cmd->cmnd[6];
411 } else if (opcode == UNMAP) {
412 /*
413 * The number of Bytes to be unmapped beginning with the lba.
414 */
3f2c1002 415 transfer_len = blk_rq_bytes(rq);
2bd3b6b7 416 lba = scsi_get_lba(cmd);
1a07f2d9
LS		 417 }
418
419 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
420 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
28fa68fc 421 trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
69a314d6 422 doorbell, transfer_len, intr, lba, opcode, group_id);
1a07f2d9
LS		 423}
424
ff8e20c6
DR		 425static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
426{
427 struct ufs_clk_info *clki;
428 struct list_head *head = &hba->clk_list_head;
429
566ec9ad 430 if (list_empty(head))
ff8e20c6
DR		 431 return;
432
433 list_for_each_entry(clki, head, list) {
434 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
435 clki->max_freq)
436 dev_err(hba->dev, "clk: %s, rate: %u\n",
437 clki->name, clki->curr_freq);
438 }
439}
440
e965e5e0
SC		 441static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
442 char *err_name)
ff8e20c6
DR		 443{
444 int i;
27752647 445 bool found = false;
e965e5e0 446 struct ufs_event_hist *e;
ff8e20c6 447
e965e5e0
SC		 448 if (id >= UFS_EVT_CNT)
449 return;
ff8e20c6 450
e965e5e0 451 e = &hba->ufs_stats.event[id];
ff8e20c6 452
e965e5e0
SC		 453 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
454 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
455
456 if (e->tstamp[p] == 0)
ff8e20c6 457 continue;
c5397f13 458 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
e965e5e0 459 e->val[p], ktime_to_us(e->tstamp[p]));
27752647 460 found = true;
ff8e20c6 461 }
27752647
SC		 462
463 if (!found)
fd1fb4d5 464 dev_err(hba->dev, "No record of %s\n", err_name);
bafd09f8
DH		 465 else
466 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
ff8e20c6
DR		 467}
468
e965e5e0 469static void ufshcd_print_evt_hist(struct ufs_hba *hba)
66cc820f 470{
ba80917d 471 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
ff8e20c6 472
e965e5e0
SC		 473 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
474 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
475 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
476 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
477 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
478 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
479 "auto_hibern8_err");
480 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
481 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
482 "link_startup_fail");
483 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
484 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
485 "suspend_fail");
486 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
487 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
488 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
ff8e20c6 489
7c486d91 490 ufshcd_vops_dbg_register_dump(hba);
66cc820f
DR		 491}
492
493static
494void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
495{
496 struct ufshcd_lrb *lrbp;
7fabb77b 497 int prdt_length;
66cc820f
DR		 498 int tag;
499
500 for_each_set_bit(tag, &bitmap, hba->nutrs) {
501 lrbp = &hba->lrb[tag];
502
ff8e20c6
DR		 503 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
504 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 505 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
506 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 507 dev_err(hba->dev,
508 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
509 tag, (u64)lrbp->utrd_dma_addr);
510
66cc820f
DR		 511 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
512 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 513 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
514 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 515 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
516 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 517 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
518 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 519 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
520 sizeof(struct utp_upiu_rsp));
66cc820f 521
7fabb77b
GB		 522 prdt_length = le16_to_cpu(
523 lrbp->utr_descriptor_ptr->prd_table_length);
cc770ce3
EB		 524 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
525 prdt_length /= sizeof(struct ufshcd_sg_entry);
526
7fabb77b
GB		 527 dev_err(hba->dev,
528 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
529 tag, prdt_length,
530 (u64)lrbp->ucd_prdt_dma_addr);
531
532 if (pr_prdt)
66cc820f 533 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 534 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 535 }
536}
537
538static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
539{
66cc820f
DR		 540 int tag;
541
542 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 543 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
544
66cc820f 545 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 546 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 547 }
548}
549
6ba65588
GB		 550static void ufshcd_print_host_state(struct ufs_hba *hba)
551{
e2106584 552 struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
3f8af604 553
6ba65588 554 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 555 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
556 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 557 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
558 hba->saved_err, hba->saved_uic_err);
559 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
560 hba->curr_dev_pwr_mode, hba->uic_link_state);
561 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
562 hba->pm_op_in_progress, hba->is_sys_suspended);
563 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
564 hba->auto_bkops_enabled, hba->host->host_self_blocked);
565 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
3f8af604
CG		 566 dev_err(hba->dev,
567 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
568 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
569 hba->ufs_stats.hibern8_exit_cnt);
570 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
571 ktime_to_us(hba->ufs_stats.last_intr_ts),
572 hba->ufs_stats.last_intr_status);
6ba65588
GB		 573 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
574 hba->eh_flags, hba->req_abort_count);
3f8af604
CG		 575 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
576 hba->ufs_version, hba->capabilities, hba->caps);
6ba65588
GB		 577 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
578 hba->dev_quirks);
3f8af604
CG		 579 if (sdev_ufs)
580 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
581 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
582
583 ufshcd_print_clk_freqs(hba);
6ba65588
GB		 584}
585
ff8e20c6
DR		 586/**
587 * ufshcd_print_pwr_info - print power params as saved in hba
588 * power info
589 * @hba: per-adapter instance
590 */
591static void ufshcd_print_pwr_info(struct ufs_hba *hba)
592{
593 static const char * const names[] = {
594 "INVALID MODE",
595 "FAST MODE",
596 "SLOW_MODE",
597 "INVALID MODE",
598 "FASTAUTO_MODE",
599 "SLOWAUTO_MODE",
600 "INVALID MODE",
601 };
602
71bb9ab6
AH		 603 /*
604 * Using dev_dbg to avoid messages during runtime PM to avoid
605 * never-ending cycles of messages written back to storage by user space
606 * causing runtime resume, causing more messages and so on.
607 */
608 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
ff8e20c6
DR		 609 __func__,
610 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
611 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
612 names[hba->pwr_info.pwr_rx],
613 names[hba->pwr_info.pwr_tx],
614 hba->pwr_info.hs_rate);
615}
616
31a5d9ca
SC		 617static void ufshcd_device_reset(struct ufs_hba *hba)
618{
619 int err;
620
621 err = ufshcd_vops_device_reset(hba);
622
623 if (!err) {
624 ufshcd_set_ufs_dev_active(hba);
625 if (ufshcd_is_wb_allowed(hba)) {
4cd48995
BH		 626 hba->dev_info.wb_enabled = false;
627 hba->dev_info.wb_buf_flush_enabled = false;
31a5d9ca
SC		 628 }
629 }
630 if (err != -EOPNOTSUPP)
631 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
632}
633
5c955c10
SC		 634void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
635{
636 if (!us)
637 return;
638
639 if (us < 10)
640 udelay(us);
641 else
642 usleep_range(us, us + tolerance);
643}
644EXPORT_SYMBOL_GPL(ufshcd_delay_us);
645
5cac1095 646/**
5a0b0cb9 647 * ufshcd_wait_for_register - wait for register value to change
5cac1095
BVA		 648 * @hba: per-adapter interface
649 * @reg: mmio register offset
650 * @mask: mask to apply to the read register value
651 * @val: value to wait for
652 * @interval_us: polling interval in microseconds
653 * @timeout_ms: timeout in milliseconds
5a0b0cb9 654 *
5cac1095
BVA		 655 * Return:
656 * -ETIMEDOUT on error, zero on success.
5a0b0cb9 657 */
59a57bb7 658static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
596585a2 659 u32 val, unsigned long interval_us,
5cac1095 660 unsigned long timeout_ms)
5a0b0cb9
SRT		 661{
662 int err = 0;
663 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
664
665 /* ignore bits that we don't intend to wait on */
666 val = val & mask;
667
668 while ((ufshcd_readl(hba, reg) & mask) != val) {
5cac1095 669 usleep_range(interval_us, interval_us + 50);
5a0b0cb9
SRT		 670 if (time_after(jiffies, timeout)) {
671 if ((ufshcd_readl(hba, reg) & mask) != val)
672 err = -ETIMEDOUT;
673 break;
674 }
675 }
676
677 return err;
678}
679
2fbd009b
SJ		 680/**
681 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 682 * @hba: Pointer to adapter instance
2fbd009b
SJ		 683 *
684 * Returns interrupt bit mask per version
685 */
686static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
687{
51428818
CC		 688 if (hba->ufs_version == ufshci_version(1, 0))
689 return INTERRUPT_MASK_ALL_VER_10;
690 if (hba->ufs_version <= ufshci_version(2, 0))
691 return INTERRUPT_MASK_ALL_VER_11;
c01848c6 692
51428818 693 return INTERRUPT_MASK_ALL_VER_21;
2fbd009b
SJ		 694}
695
7a3e97b0
SY		 696/**
697 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 698 * @hba: Pointer to adapter instance
7a3e97b0
SY		 699 *
700 * Returns UFSHCI version supported by the controller
701 */
702static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
703{
51428818
CC		 704 u32 ufshci_ver;
705
0263bcd0 706 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
51428818
CC		 707 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
708 else
709 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
9949e702 710
51428818
CC		 711 /*
712 * UFSHCI v1.x uses a different version scheme, in order
713 * to allow the use of comparisons with the ufshci_version
714 * function, we convert it to the same scheme as ufs 2.0+.
715 */
716 if (ufshci_ver & 0x00010000)
717 return ufshci_version(1, ufshci_ver & 0x00000100);
718
719 return ufshci_ver;
7a3e97b0
SY		 720}
721
722/**
723 * ufshcd_is_device_present - Check if any device connected to
724 * the host controller
5c0c28a8 725 * @hba: pointer to adapter instance
7a3e97b0 726 *
c9e6010b 727 * Returns true if device present, false if no device detected
7a3e97b0 728 */
c9e6010b 729static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 730{
51d1628f 731 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
7a3e97b0
SY		 732}
733
734/**
735 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 736 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 737 *
738 * This function is used to get the OCS field from UTRD
739 * Returns the OCS field in the UTRD
740 */
957d63e7 741static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
7a3e97b0 742{
e8c8e82a 743 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 744}
745
7a3e97b0
SY		 746/**
747 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
748 * @hba: per adapter instance
749 * @pos: position of the bit to be cleared
750 */
751static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
752{
87183841
AA		 753 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
754 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
755 else
756 ufshcd_writel(hba, ~(1 << pos),
757 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
1399c5b0
AA		 758}
759
760/**
761 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
762 * @hba: per adapter instance
763 * @pos: position of the bit to be cleared
764 */
765static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
766{
87183841
AA		 767 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
768 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
769 else
770 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 771}
772
773/**
774 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
775 * @reg: Register value of host controller status
776 *
777 * Returns integer, 0 on Success and positive value if failed
778 */
779static inline int ufshcd_get_lists_status(u32 reg)
780{
6cf16115 781 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 782}
783
784/**
785 * ufshcd_get_uic_cmd_result - Get the UIC command result
786 * @hba: Pointer to adapter instance
787 *
788 * This function gets the result of UIC command completion
789 * Returns 0 on success, non zero value on error
790 */
791static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
792{
b873a275 793 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 794 MASK_UIC_COMMAND_RESULT;
795}
796
12b4fdb4
SJ		 797/**
798 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
799 * @hba: Pointer to adapter instance
800 *
801 * This function gets UIC command argument3
802 * Returns 0 on success, non zero value on error
803 */
804static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
805{
806 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
807}
808
7a3e97b0 809/**
5a0b0cb9 810 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 811 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 812 */
813static inline int
5a0b0cb9 814ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 815{
5a0b0cb9 816 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 817}
818
819/**
820 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
821 * @ucd_rsp_ptr: pointer to response UPIU
822 *
823 * This function gets the response status and scsi_status from response UPIU
824 * Returns the response result code.
825 */
826static inline int
827ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
828{
829 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
830}
831
1c2623c5
SJ		 832/*
833 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
834 * from response UPIU
835 * @ucd_rsp_ptr: pointer to response UPIU
836 *
837 * Return the data segment length.
838 */
839static inline unsigned int
840ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
841{
842 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
843 MASK_RSP_UPIU_DATA_SEG_LEN;
844}
845
66ec6d59
SRT		 846/**
847 * ufshcd_is_exception_event - Check if the device raised an exception event
848 * @ucd_rsp_ptr: pointer to response UPIU
849 *
850 * The function checks if the device raised an exception event indicated in
851 * the Device Information field of response UPIU.
852 *
853 * Returns true if exception is raised, false otherwise.
854 */
855static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
856{
857 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
51d1628f 858 MASK_RSP_EXCEPTION_EVENT;
66ec6d59
SRT		 859}
860
7a3e97b0 861/**
7d568652 862 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 863 * @hba: per adapter instance
7a3e97b0
SY		 864 */
865static inline void
7d568652 866ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 867{
7d568652
SJ		 868 ufshcd_writel(hba, INT_AGGR_ENABLE |
869 INT_AGGR_COUNTER_AND_TIMER_RESET,
870 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
871}
872
873/**
874 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
875 * @hba: per adapter instance
876 * @cnt: Interrupt aggregation counter threshold
877 * @tmout: Interrupt aggregation timeout value
878 */
879static inline void
880ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
881{
882 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
883 INT_AGGR_COUNTER_THLD_VAL(cnt) |
884 INT_AGGR_TIMEOUT_VAL(tmout),
885 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 886}
887
b852190e
YG		 888/**
889 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
890 * @hba: per adapter instance
891 */
892static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
893{
894 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
895}
896
7a3e97b0
SY		 897/**
898 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
899 * When run-stop registers are set to 1, it indicates the
900 * host controller that it can process the requests
901 * @hba: per adapter instance
902 */
903static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
904{
b873a275
SJ		 905 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
906 REG_UTP_TASK_REQ_LIST_RUN_STOP);
907 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
908 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 909}
910
7a3e97b0
SY		 911/**
912 * ufshcd_hba_start - Start controller initialization sequence
913 * @hba: per adapter instance
914 */
915static inline void ufshcd_hba_start(struct ufs_hba *hba)
916{
df043c74
ST		 917 u32 val = CONTROLLER_ENABLE;
918
919 if (ufshcd_crypto_enable(hba))
920 val |= CRYPTO_GENERAL_ENABLE;
921
922 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 923}
924
925/**
926 * ufshcd_is_hba_active - Get controller state
927 * @hba: per adapter instance
928 *
acbbfe48 929 * Returns true if and only if the controller is active.
7a3e97b0 930 */
c9e6010b 931static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 932{
acbbfe48 933 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
7a3e97b0
SY		 934}
935
37113106
YG		 936u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
937{
938 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
51428818 939 if (hba->ufs_version <= ufshci_version(1, 1))
37113106
YG		 940 return UFS_UNIPRO_VER_1_41;
941 else
942 return UFS_UNIPRO_VER_1_6;
943}
944EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
945
946static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
947{
948 /*
949 * If both host and device support UniPro ver1.6 or later, PA layer
950 * parameters tuning happens during link startup itself.
951 *
952 * We can manually tune PA layer parameters if either host or device
953 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
954 * logic simple, we will only do manual tuning if local unipro version
955 * doesn't support ver1.6 or later.
956 */
a858af9a 957 return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6;
37113106
YG		 958}
959
394b949f
SJ		 960/**
961 * ufshcd_set_clk_freq - set UFS controller clock frequencies
962 * @hba: per adapter instance
963 * @scale_up: If True, set max possible frequency othewise set low frequency
964 *
965 * Returns 0 if successful
966 * Returns < 0 for any other errors
967 */
968static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
a3cd5ec5 969{
970 int ret = 0;
971 struct ufs_clk_info *clki;
972 struct list_head *head = &hba->clk_list_head;
a3cd5ec5 973
566ec9ad 974 if (list_empty(head))
a3cd5ec5 975 goto out;
976
a3cd5ec5 977 list_for_each_entry(clki, head, list) {
978 if (!IS_ERR_OR_NULL(clki->clk)) {
979 if (scale_up && clki->max_freq) {
980 if (clki->curr_freq == clki->max_freq)
981 continue;
982
a3cd5ec5 983 ret = clk_set_rate(clki->clk, clki->max_freq);
984 if (ret) {
985 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
986 __func__, clki->name,
987 clki->max_freq, ret);
988 break;
989 }
990 trace_ufshcd_clk_scaling(dev_name(hba->dev),
991 "scaled up", clki->name,
992 clki->curr_freq,
993 clki->max_freq);
994
995 clki->curr_freq = clki->max_freq;
996
997 } else if (!scale_up && clki->min_freq) {
998 if (clki->curr_freq == clki->min_freq)
999 continue;
1000
a3cd5ec5 1001 ret = clk_set_rate(clki->clk, clki->min_freq);
1002 if (ret) {
1003 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1004 __func__, clki->name,
1005 clki->min_freq, ret);
1006 break;
1007 }
1008 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1009 "scaled down", clki->name,
1010 clki->curr_freq,
1011 clki->min_freq);
1012 clki->curr_freq = clki->min_freq;
1013 }
1014 }
1015 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1016 clki->name, clk_get_rate(clki->clk));
1017 }
1018
394b949f
SJ		 1019out:
1020 return ret;
1021}
1022
1023/**
1024 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1025 * @hba: per adapter instance
1026 * @scale_up: True if scaling up and false if scaling down
1027 *
1028 * Returns 0 if successful
1029 * Returns < 0 for any other errors
1030 */
1031static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1032{
1033 int ret = 0;
1034 ktime_t start = ktime_get();
1035
1036 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1037 if (ret)
1038 goto out;
1039
1040 ret = ufshcd_set_clk_freq(hba, scale_up);
1041 if (ret)
1042 goto out;
1043
a3cd5ec5 1044 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
394b949f
SJ		 1045 if (ret)
1046 ufshcd_set_clk_freq(hba, !scale_up);
a3cd5ec5 1047
1048out:
394b949f 1049 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
a3cd5ec5 1050 (scale_up ? "up" : "down"),
1051 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1052 return ret;
1053}
1054
1055/**
1056 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1057 * @hba: per adapter instance
1058 * @scale_up: True if scaling up and false if scaling down
1059 *
1060 * Returns true if scaling is required, false otherwise.
1061 */
1062static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1063 bool scale_up)
1064{
1065 struct ufs_clk_info *clki;
1066 struct list_head *head = &hba->clk_list_head;
1067
566ec9ad 1068 if (list_empty(head))
a3cd5ec5 1069 return false;
1070
1071 list_for_each_entry(clki, head, list) {
1072 if (!IS_ERR_OR_NULL(clki->clk)) {
1073 if (scale_up && clki->max_freq) {
1074 if (clki->curr_freq == clki->max_freq)
1075 continue;
1076 return true;
1077 } else if (!scale_up && clki->min_freq) {
1078 if (clki->curr_freq == clki->min_freq)
1079 continue;
1080 return true;
1081 }
1082 }
1083 }
1084
1085 return false;
1086}
1087
8d077ede
BVA		 1088/*
1089 * Determine the number of pending commands by counting the bits in the SCSI
1090 * device budget maps. This approach has been selected because a bit is set in
1091 * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1092 * flag. The host_self_blocked flag can be modified by calling
1093 * scsi_block_requests() or scsi_unblock_requests().
1094 */
1095static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1096{
1097 struct scsi_device *sdev;
1098 u32 pending = 0;
1099
99c66a88
BH		 1100 lockdep_assert_held(hba->host->host_lock);
1101 __shost_for_each_device(sdev, hba->host)
8d077ede
BVA		 1102 pending += sbitmap_weight(&sdev->budget_map);
1103
1104 return pending;
1105}
1106
a3cd5ec5 1107static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1108 u64 wait_timeout_us)
1109{
1110 unsigned long flags;
1111 int ret = 0;
1112 u32 tm_doorbell;
8d077ede 1113 u32 tr_pending;
a3cd5ec5 1114 bool timeout = false, do_last_check = false;
1115 ktime_t start;
1116
1117 ufshcd_hold(hba, false);
1118 spin_lock_irqsave(hba->host->host_lock, flags);
1119 /*
1120 * Wait for all the outstanding tasks/transfer requests.
1121 * Verify by checking the doorbell registers are clear.
1122 */
1123 start = ktime_get();
1124 do {
1125 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1126 ret = -EBUSY;
1127 goto out;
1128 }
1129
1130 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
8d077ede
BVA		 1131 tr_pending = ufshcd_pending_cmds(hba);
1132 if (!tm_doorbell && !tr_pending) {
a3cd5ec5 1133 timeout = false;
1134 break;
1135 } else if (do_last_check) {
1136 break;
1137 }
1138
1139 spin_unlock_irqrestore(hba->host->host_lock, flags);
1140 schedule();
1141 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1142 wait_timeout_us) {
1143 timeout = true;
1144 /*
1145 * We might have scheduled out for long time so make
1146 * sure to check if doorbells are cleared by this time
1147 * or not.
1148 */
1149 do_last_check = true;
1150 }
1151 spin_lock_irqsave(hba->host->host_lock, flags);
8d077ede 1152 } while (tm_doorbell || tr_pending);
a3cd5ec5 1153
1154 if (timeout) {
1155 dev_err(hba->dev,
1156 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
8d077ede 1157 __func__, tm_doorbell, tr_pending);
a3cd5ec5 1158 ret = -EBUSY;
1159 }
1160out:
1161 spin_unlock_irqrestore(hba->host->host_lock, flags);
1162 ufshcd_release(hba);
1163 return ret;
1164}
1165
1166/**
1167 * ufshcd_scale_gear - scale up/down UFS gear
1168 * @hba: per adapter instance
1169 * @scale_up: True for scaling up gear and false for scaling down
1170 *
1171 * Returns 0 for success,
1172 * Returns -EBUSY if scaling can't happen at this time
1173 * Returns non-zero for any other errors
1174 */
1175static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1176{
a3cd5ec5 1177 int ret = 0;
1178 struct ufs_pa_layer_attr new_pwr_info;
1179
1180 if (scale_up) {
1181 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1182 sizeof(struct ufs_pa_layer_attr));
1183 } else {
1184 memcpy(&new_pwr_info, &hba->pwr_info,
1185 sizeof(struct ufs_pa_layer_attr));
1186
29b87e92
CG		 1187 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1188 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
a3cd5ec5 1189 /* save the current power mode */
1190 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1191 &hba->pwr_info,
1192 sizeof(struct ufs_pa_layer_attr));
1193
1194 /* scale down gear */
29b87e92
CG		 1195 new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1196 new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
a3cd5ec5 1197 }
1198 }
1199
1200 /* check if the power mode needs to be changed or not? */
6a9df818 1201 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
a3cd5ec5 1202 if (ret)
1203 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1204 __func__, ret,
1205 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1206 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1207
1208 return ret;
1209}
1210
1211static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1212{
1213 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1214 int ret = 0;
1215 /*
1216 * make sure that there are no outstanding requests when
1217 * clock scaling is in progress
1218 */
38135535 1219 ufshcd_scsi_block_requests(hba);
a3cd5ec5 1220 down_write(&hba->clk_scaling_lock);
0e9d4ca4
CG		 1221
1222 if (!hba->clk_scaling.is_allowed ||
1223 ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
a3cd5ec5 1224 ret = -EBUSY;
1225 up_write(&hba->clk_scaling_lock);
38135535 1226 ufshcd_scsi_unblock_requests(hba);
0e9d4ca4 1227 goto out;
a3cd5ec5 1228 }
1229
0e9d4ca4
CG		 1230 /* let's not get into low power until clock scaling is completed */
1231 ufshcd_hold(hba, false);
1232
1233out:
a3cd5ec5 1234 return ret;
1235}
1236
0e9d4ca4 1237static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, bool writelock)
a3cd5ec5 1238{
0e9d4ca4
CG		 1239 if (writelock)
1240 up_write(&hba->clk_scaling_lock);
1241 else
1242 up_read(&hba->clk_scaling_lock);
38135535 1243 ufshcd_scsi_unblock_requests(hba);
0e9d4ca4 1244 ufshcd_release(hba);
a3cd5ec5 1245}
1246
1247/**
1248 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1249 * @hba: per adapter instance
1250 * @scale_up: True for scaling up and false for scalin down
1251 *
1252 * Returns 0 for success,
1253 * Returns -EBUSY if scaling can't happen at this time
1254 * Returns non-zero for any other errors
1255 */
1256static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1257{
1258 int ret = 0;
0e9d4ca4 1259 bool is_writelock = true;
401f1e44 1260
a3cd5ec5 1261 ret = ufshcd_clock_scaling_prepare(hba);
1262 if (ret)
0e9d4ca4 1263 return ret;
a3cd5ec5 1264
1265 /* scale down the gear before scaling down clocks */
1266 if (!scale_up) {
1267 ret = ufshcd_scale_gear(hba, false);
1268 if (ret)
394b949f 1269 goto out_unprepare;
a3cd5ec5 1270 }
1271
1272 ret = ufshcd_scale_clks(hba, scale_up);
1273 if (ret) {
1274 if (!scale_up)
1275 ufshcd_scale_gear(hba, true);
394b949f 1276 goto out_unprepare;
a3cd5ec5 1277 }
1278
1279 /* scale up the gear after scaling up clocks */
1280 if (scale_up) {
1281 ret = ufshcd_scale_gear(hba, true);
3d17b9b5 1282 if (ret) {
a3cd5ec5 1283 ufshcd_scale_clks(hba, false);
3d17b9b5
AD		 1284 goto out_unprepare;
1285 }
a3cd5ec5 1286 }
1287
3d17b9b5 1288 /* Enable Write Booster if we have scaled up else disable it */
0e9d4ca4
CG		 1289 downgrade_write(&hba->clk_scaling_lock);
1290 is_writelock = false;
3b5f3c0d 1291 ufshcd_wb_toggle(hba, scale_up);
3d17b9b5 1292
394b949f 1293out_unprepare:
0e9d4ca4 1294 ufshcd_clock_scaling_unprepare(hba, is_writelock);
a3cd5ec5 1295 return ret;
1296}
1297
401f1e44 1298static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1299{
1300 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1301 clk_scaling.suspend_work);
1302 unsigned long irq_flags;
1303
1304 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1305 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1306 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1307 return;
1308 }
1309 hba->clk_scaling.is_suspended = true;
1310 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1311
1312 __ufshcd_suspend_clkscaling(hba);
1313}
1314
1315static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1316{
1317 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1318 clk_scaling.resume_work);
1319 unsigned long irq_flags;
1320
1321 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1322 if (!hba->clk_scaling.is_suspended) {
1323 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1324 return;
1325 }
1326 hba->clk_scaling.is_suspended = false;
1327 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1328
1329 devfreq_resume_device(hba->devfreq);
1330}
1331
a3cd5ec5 1332static int ufshcd_devfreq_target(struct device *dev,
1333 unsigned long *freq, u32 flags)
1334{
1335 int ret = 0;
1336 struct ufs_hba *hba = dev_get_drvdata(dev);
1337 ktime_t start;
401f1e44 1338 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1339 struct list_head *clk_list = &hba->clk_list_head;
1340 struct ufs_clk_info *clki;
a3cd5ec5 1341 unsigned long irq_flags;
1342
1343 if (!ufshcd_is_clkscaling_supported(hba))
1344 return -EINVAL;
1345
91831d33
AD		 1346 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1347 /* Override with the closest supported frequency */
1348 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
a3cd5ec5 1349 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1350 if (ufshcd_eh_in_progress(hba)) {
1351 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1352 return 0;
1353 }
1354
401f1e44 1355 if (!hba->clk_scaling.active_reqs)
1356 sched_clk_scaling_suspend_work = true;
1357
092b4558
BA		 1358 if (list_empty(clk_list)) {
1359 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1360 goto out;
1361 }
1362
91831d33 1363 /* Decide based on the rounded-off frequency and update */
51d1628f 1364 scale_up = *freq == clki->max_freq;
91831d33
AD		 1365 if (!scale_up)
1366 *freq = clki->min_freq;
1367 /* Update the frequency */
401f1e44 1368 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1369 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1370 ret = 0;
1371 goto out; /* no state change required */
a3cd5ec5 1372 }
1373 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1374
1375 start = ktime_get();
a3cd5ec5 1376 ret = ufshcd_devfreq_scale(hba, scale_up);
1377
a3cd5ec5 1378 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1379 (scale_up ? "up" : "down"),
1380 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1381
401f1e44 1382out:
1383 if (sched_clk_scaling_suspend_work)
1384 queue_work(hba->clk_scaling.workq,
1385 &hba->clk_scaling.suspend_work);
1386
a3cd5ec5 1387 return ret;
1388}
1389
a3cd5ec5 1390static int ufshcd_devfreq_get_dev_status(struct device *dev,
1391 struct devfreq_dev_status *stat)
1392{
1393 struct ufs_hba *hba = dev_get_drvdata(dev);
1394 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1395 unsigned long flags;
91831d33
AD		 1396 struct list_head *clk_list = &hba->clk_list_head;
1397 struct ufs_clk_info *clki;
b1bf66d1 1398 ktime_t curr_t;
a3cd5ec5 1399
1400 if (!ufshcd_is_clkscaling_supported(hba))
1401 return -EINVAL;
1402
1403 memset(stat, 0, sizeof(*stat));
1404
1405 spin_lock_irqsave(hba->host->host_lock, flags);
b1bf66d1 1406 curr_t = ktime_get();
a3cd5ec5 1407 if (!scaling->window_start_t)
1408 goto start_window;
1409
91831d33
AD		 1410 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1411 /*
1412 * If current frequency is 0, then the ondemand governor considers
1413 * there's no initial frequency set. And it always requests to set
1414 * to max. frequency.
1415 */
1416 stat->current_frequency = clki->curr_freq;
a3cd5ec5 1417 if (scaling->is_busy_started)
b1bf66d1
SC		 1418 scaling->tot_busy_t += ktime_us_delta(curr_t,
1419 scaling->busy_start_t);
a3cd5ec5 1420
b1bf66d1 1421 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
a3cd5ec5 1422 stat->busy_time = scaling->tot_busy_t;
1423start_window:
b1bf66d1 1424 scaling->window_start_t = curr_t;
a3cd5ec5 1425 scaling->tot_busy_t = 0;
1426
1427 if (hba->outstanding_reqs) {
b1bf66d1 1428 scaling->busy_start_t = curr_t;
a3cd5ec5 1429 scaling->is_busy_started = true;
1430 } else {
1431 scaling->busy_start_t = 0;
1432 scaling->is_busy_started = false;
1433 }
1434 spin_unlock_irqrestore(hba->host->host_lock, flags);
1435 return 0;
1436}
1437
deac444f
BA		 1438static int ufshcd_devfreq_init(struct ufs_hba *hba)
1439{
092b4558
BA		 1440 struct list_head *clk_list = &hba->clk_list_head;
1441 struct ufs_clk_info *clki;
deac444f
BA		 1442 struct devfreq *devfreq;
1443 int ret;
1444
092b4558
BA		 1445 /* Skip devfreq if we don't have any clocks in the list */
1446 if (list_empty(clk_list))
1447 return 0;
1448
1449 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1450 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1451 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1452
90b8491c
SC		 1453 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1454 &hba->vps->ondemand_data);
092b4558 1455 devfreq = devfreq_add_device(hba->dev,
90b8491c 1456 &hba->vps->devfreq_profile,
deac444f 1457 DEVFREQ_GOV_SIMPLE_ONDEMAND,
90b8491c 1458 &hba->vps->ondemand_data);
deac444f
BA		 1459 if (IS_ERR(devfreq)) {
1460 ret = PTR_ERR(devfreq);
1461 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1462
1463 dev_pm_opp_remove(hba->dev, clki->min_freq);
1464 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1465 return ret;
1466 }
1467
1468 hba->devfreq = devfreq;
1469
1470 return 0;
1471}
1472
092b4558
BA		 1473static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1474{
1475 struct list_head *clk_list = &hba->clk_list_head;
1476 struct ufs_clk_info *clki;
1477
1478 if (!hba->devfreq)
1479 return;
1480
1481 devfreq_remove_device(hba->devfreq);
1482 hba->devfreq = NULL;
1483
1484 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1485 dev_pm_opp_remove(hba->dev, clki->min_freq);
1486 dev_pm_opp_remove(hba->dev, clki->max_freq);
1487}
1488
401f1e44 1489static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1490{
1491 unsigned long flags;
1492
1493 devfreq_suspend_device(hba->devfreq);
1494 spin_lock_irqsave(hba->host->host_lock, flags);
1495 hba->clk_scaling.window_start_t = 0;
1496 spin_unlock_irqrestore(hba->host->host_lock, flags);
1497}
a3cd5ec5 1498
a508253d
GB		 1499static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1500{
401f1e44 1501 unsigned long flags;
1502 bool suspend = false;
1503
f9a7fa34
SC		 1504 cancel_work_sync(&hba->clk_scaling.suspend_work);
1505 cancel_work_sync(&hba->clk_scaling.resume_work);
fcb0c4b0 1506
401f1e44 1507 spin_lock_irqsave(hba->host->host_lock, flags);
1508 if (!hba->clk_scaling.is_suspended) {
1509 suspend = true;
1510 hba->clk_scaling.is_suspended = true;
1511 }
1512 spin_unlock_irqrestore(hba->host->host_lock, flags);
1513
1514 if (suspend)
1515 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1516}
1517
1518static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1519{
401f1e44 1520 unsigned long flags;
1521 bool resume = false;
1522
401f1e44 1523 spin_lock_irqsave(hba->host->host_lock, flags);
1524 if (hba->clk_scaling.is_suspended) {
1525 resume = true;
1526 hba->clk_scaling.is_suspended = false;
1527 }
1528 spin_unlock_irqrestore(hba->host->host_lock, flags);
1529
1530 if (resume)
1531 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1532}
1533
1534static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1535 struct device_attribute *attr, char *buf)
1536{
1537 struct ufs_hba *hba = dev_get_drvdata(dev);
1538
1481b7fe 1539 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
fcb0c4b0
ST		 1540}
1541
1542static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1543 struct device_attribute *attr, const char *buf, size_t count)
1544{
1545 struct ufs_hba *hba = dev_get_drvdata(dev);
1546 u32 value;
9cd20d3f 1547 int err = 0;
fcb0c4b0
ST		 1548
1549 if (kstrtou32(buf, 0, &value))
1550 return -EINVAL;
1551
9cd20d3f
CG		 1552 down(&hba->host_sem);
1553 if (!ufshcd_is_user_access_allowed(hba)) {
1554 err = -EBUSY;
1555 goto out;
1556 }
1557
fcb0c4b0 1558 value = !!value;
0e9d4ca4 1559 if (value == hba->clk_scaling.is_enabled)
fcb0c4b0
ST		 1560 goto out;
1561
b294ff3e 1562 ufshcd_rpm_get_sync(hba);
fcb0c4b0
ST		 1563 ufshcd_hold(hba, false);
1564
0e9d4ca4 1565 hba->clk_scaling.is_enabled = value;
401f1e44 1566
fcb0c4b0
ST		 1567 if (value) {
1568 ufshcd_resume_clkscaling(hba);
1569 } else {
1570 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1571 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1572 if (err)
1573 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1574 __func__, err);
1575 }
fcb0c4b0
ST		 1576
1577 ufshcd_release(hba);
b294ff3e 1578 ufshcd_rpm_put_sync(hba);
fcb0c4b0 1579out:
9cd20d3f
CG		 1580 up(&hba->host_sem);
1581 return err ? err : count;
a508253d
GB		 1582}
1583
4543d9d7 1584static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
a3cd5ec5 1585{
1586 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1587 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1588 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1589 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1590 hba->clk_scaling.enable_attr.attr.mode = 0644;
1591 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1592 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1593}
1594
4543d9d7
CG		 1595static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1596{
1597 if (hba->clk_scaling.enable_attr.attr.name)
1598 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1599}
1600
1601static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1602{
1603 char wq_name[sizeof("ufs_clkscaling_00")];
1604
1605 if (!ufshcd_is_clkscaling_supported(hba))
1606 return;
1607
80d892f4
CG		 1608 if (!hba->clk_scaling.min_gear)
1609 hba->clk_scaling.min_gear = UFS_HS_G1;
1610
4543d9d7
CG		 1611 INIT_WORK(&hba->clk_scaling.suspend_work,
1612 ufshcd_clk_scaling_suspend_work);
1613 INIT_WORK(&hba->clk_scaling.resume_work,
1614 ufshcd_clk_scaling_resume_work);
1615
1616 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1617 hba->host->host_no);
1618 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1619
1620 hba->clk_scaling.is_initialized = true;
1621}
1622
1623static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1624{
1625 if (!hba->clk_scaling.is_initialized)
1626 return;
1627
1628 ufshcd_remove_clk_scaling_sysfs(hba);
1629 destroy_workqueue(hba->clk_scaling.workq);
1630 ufshcd_devfreq_remove(hba);
1631 hba->clk_scaling.is_initialized = false;
1632}
1633
1ab27c9c
ST		 1634static void ufshcd_ungate_work(struct work_struct *work)
1635{
1636 int ret;
1637 unsigned long flags;
1638 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1639 clk_gating.ungate_work);
1640
1641 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1642
1643 spin_lock_irqsave(hba->host->host_lock, flags);
1644 if (hba->clk_gating.state == CLKS_ON) {
1645 spin_unlock_irqrestore(hba->host->host_lock, flags);
1646 goto unblock_reqs;
1647 }
1648
1649 spin_unlock_irqrestore(hba->host->host_lock, flags);
dd7143e2 1650 ufshcd_hba_vreg_set_hpm(hba);
1ab27c9c
ST		 1651 ufshcd_setup_clocks(hba, true);
1652
8b0bbf00
SC		 1653 ufshcd_enable_irq(hba);
1654
1ab27c9c
ST		 1655 /* Exit from hibern8 */
1656 if (ufshcd_can_hibern8_during_gating(hba)) {
1657 /* Prevent gating in this path */
1658 hba->clk_gating.is_suspended = true;
1659 if (ufshcd_is_link_hibern8(hba)) {
1660 ret = ufshcd_uic_hibern8_exit(hba);
1661 if (ret)
1662 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1663 __func__, ret);
1664 else
1665 ufshcd_set_link_active(hba);
1666 }
1667 hba->clk_gating.is_suspended = false;
1668 }
1669unblock_reqs:
38135535 1670 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1671}
1672
1673/**
1674 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1675 * Also, exit from hibern8 mode and set the link as active.
1676 * @hba: per adapter instance
1677 * @async: This indicates whether caller should ungate clocks asynchronously.
1678 */
1679int ufshcd_hold(struct ufs_hba *hba, bool async)
1680{
1681 int rc = 0;
93b6c5db 1682 bool flush_result;
1ab27c9c
ST		 1683 unsigned long flags;
1684
3489c34b
BVA		 1685 if (!ufshcd_is_clkgating_allowed(hba) ||
1686 !hba->clk_gating.is_initialized)
1ab27c9c 1687 goto out;
1ab27c9c
ST		 1688 spin_lock_irqsave(hba->host->host_lock, flags);
1689 hba->clk_gating.active_reqs++;
1690
856b3483 1691start:
1ab27c9c
ST		 1692 switch (hba->clk_gating.state) {
1693 case CLKS_ON:
f2a785ac
VG		 1694 /*
1695 * Wait for the ungate work to complete if in progress.
1696 * Though the clocks may be in ON state, the link could
1697 * still be in hibner8 state if hibern8 is allowed
1698 * during clock gating.
1699 * Make sure we exit hibern8 state also in addition to
1700 * clocks being ON.
1701 */
1702 if (ufshcd_can_hibern8_during_gating(hba) &&
1703 ufshcd_is_link_hibern8(hba)) {
c63d6099
CG		 1704 if (async) {
1705 rc = -EAGAIN;
1706 hba->clk_gating.active_reqs--;
1707 break;
1708 }
f2a785ac 1709 spin_unlock_irqrestore(hba->host->host_lock, flags);
93b6c5db
SC		 1710 flush_result = flush_work(&hba->clk_gating.ungate_work);
1711 if (hba->clk_gating.is_suspended && !flush_result)
1712 goto out;
f2a785ac
VG		 1713 spin_lock_irqsave(hba->host->host_lock, flags);
1714 goto start;
1715 }
1ab27c9c
ST		 1716 break;
1717 case REQ_CLKS_OFF:
1718 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1719 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1720 trace_ufshcd_clk_gating(dev_name(hba->dev),
1721 hba->clk_gating.state);
1ab27c9c
ST		 1722 break;
1723 }
1724 /*
9c490d2d 1725 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1726 * currently running. Hence, fall through to cancel gating
1727 * work and to enable clocks.
1728 */
df561f66 1729 fallthrough;
1ab27c9c 1730 case CLKS_OFF:
1ab27c9c 1731 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47 1732 trace_ufshcd_clk_gating(dev_name(hba->dev),
1733 hba->clk_gating.state);
da3fecb0
CG		 1734 if (queue_work(hba->clk_gating.clk_gating_workq,
1735 &hba->clk_gating.ungate_work))
1736 ufshcd_scsi_block_requests(hba);
1ab27c9c
ST		 1737 /*
1738 * fall through to check if we should wait for this
1739 * work to be done or not.
1740 */
df561f66 1741 fallthrough;
1ab27c9c
ST		 1742 case REQ_CLKS_ON:
1743 if (async) {
1744 rc = -EAGAIN;
1745 hba->clk_gating.active_reqs--;
1746 break;
1747 }
1748
1749 spin_unlock_irqrestore(hba->host->host_lock, flags);
1750 flush_work(&hba->clk_gating.ungate_work);
1751 /* Make sure state is CLKS_ON before returning */
856b3483 1752 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1753 goto start;
1754 default:
1755 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1756 __func__, hba->clk_gating.state);
1757 break;
1758 }
1759 spin_unlock_irqrestore(hba->host->host_lock, flags);
1760out:
1761 return rc;
1762}
6e3fd44d 1763EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1764
1765static void ufshcd_gate_work(struct work_struct *work)
1766{
1767 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1768 clk_gating.gate_work.work);
1769 unsigned long flags;
4db7a236 1770 int ret;
1ab27c9c
ST		 1771
1772 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1773 /*
1774 * In case you are here to cancel this work the gating state
1775 * would be marked as REQ_CLKS_ON. In this case save time by
1776 * skipping the gating work and exit after changing the clock
1777 * state to CLKS_ON.
1778 */
1779 if (hba->clk_gating.is_suspended ||
18f01374 1780 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1781 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1782 trace_ufshcd_clk_gating(dev_name(hba->dev),
1783 hba->clk_gating.state);
1ab27c9c
ST		 1784 goto rel_lock;
1785 }
1786
1787 if (hba->clk_gating.active_reqs
1788 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
bd0b3538 1789 || hba->outstanding_reqs || hba->outstanding_tasks
1ab27c9c
ST		 1790 || hba->active_uic_cmd || hba->uic_async_done)
1791 goto rel_lock;
1792
1793 spin_unlock_irqrestore(hba->host->host_lock, flags);
1794
1795 /* put the link into hibern8 mode before turning off clocks */
1796 if (ufshcd_can_hibern8_during_gating(hba)) {
4db7a236
CG		 1797 ret = ufshcd_uic_hibern8_enter(hba);
1798 if (ret) {
1ab27c9c 1799 hba->clk_gating.state = CLKS_ON;
4db7a236
CG		 1800 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1801 __func__, ret);
7ff5ab47 1802 trace_ufshcd_clk_gating(dev_name(hba->dev),
1803 hba->clk_gating.state);
1ab27c9c
ST		 1804 goto out;
1805 }
1806 ufshcd_set_link_hibern8(hba);
1807 }
1808
8b0bbf00
SC		 1809 ufshcd_disable_irq(hba);
1810
81309c24 1811 ufshcd_setup_clocks(hba, false);
1ab27c9c 1812
dd7143e2
CG		 1813 /* Put the host controller in low power mode if possible */
1814 ufshcd_hba_vreg_set_lpm(hba);
1ab27c9c
ST		 1815 /*
1816 * In case you are here to cancel this work the gating state
1817 * would be marked as REQ_CLKS_ON. In this case keep the state
1818 * as REQ_CLKS_ON which would anyway imply that clocks are off
1819 * and a request to turn them on is pending. By doing this way,
1820 * we keep the state machine in tact and this would ultimately
1821 * prevent from doing cancel work multiple times when there are
1822 * new requests arriving before the current cancel work is done.
1823 */
1824 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1825 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1826 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 1827 trace_ufshcd_clk_gating(dev_name(hba->dev),
1828 hba->clk_gating.state);
1829 }
1ab27c9c
ST		 1830rel_lock:
1831 spin_unlock_irqrestore(hba->host->host_lock, flags);
1832out:
1833 return;
1834}
1835
1836/* host lock must be held before calling this variant */
1837static void __ufshcd_release(struct ufs_hba *hba)
1838{
1839 if (!ufshcd_is_clkgating_allowed(hba))
1840 return;
1841
1842 hba->clk_gating.active_reqs--;
1843
4db7a236
CG		 1844 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1845 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
3489c34b 1846 hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
fd62de11
JK		 1847 hba->active_uic_cmd || hba->uic_async_done ||
1848 hba->clk_gating.state == CLKS_OFF)
1ab27c9c
ST		 1849 return;
1850
1851 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1852 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1853 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1854 &hba->clk_gating.gate_work,
1855 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1856}
1857
1858void ufshcd_release(struct ufs_hba *hba)
1859{
1860 unsigned long flags;
1861
1862 spin_lock_irqsave(hba->host->host_lock, flags);
1863 __ufshcd_release(hba);
1864 spin_unlock_irqrestore(hba->host->host_lock, flags);
1865}
6e3fd44d 1866EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1867
1868static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1869 struct device_attribute *attr, char *buf)
1870{
1871 struct ufs_hba *hba = dev_get_drvdata(dev);
1872
bafd09f8 1873 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1ab27c9c
ST		 1874}
1875
ad8a647e
BVA		 1876void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
1877{
1878 struct ufs_hba *hba = dev_get_drvdata(dev);
1879 unsigned long flags;
1880
1881 spin_lock_irqsave(hba->host->host_lock, flags);
1882 hba->clk_gating.delay_ms = value;
1883 spin_unlock_irqrestore(hba->host->host_lock, flags);
1884}
1885EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
1886
1ab27c9c
ST		 1887static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1888 struct device_attribute *attr, const char *buf, size_t count)
1889{
ad8a647e 1890 unsigned long value;
1ab27c9c
ST		 1891
1892 if (kstrtoul(buf, 0, &value))
1893 return -EINVAL;
1894
ad8a647e 1895 ufshcd_clkgate_delay_set(dev, value);
1ab27c9c
ST		 1896 return count;
1897}
1898
b427411a
ST		 1899static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1900 struct device_attribute *attr, char *buf)
1901{
1902 struct ufs_hba *hba = dev_get_drvdata(dev);
1903
bafd09f8 1904 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
b427411a
ST		 1905}
1906
1907static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1908 struct device_attribute *attr, const char *buf, size_t count)
1909{
1910 struct ufs_hba *hba = dev_get_drvdata(dev);
1911 unsigned long flags;
1912 u32 value;
1913
1914 if (kstrtou32(buf, 0, &value))
1915 return -EINVAL;
1916
1917 value = !!value;
b6645112
JK		 1918
1919 spin_lock_irqsave(hba->host->host_lock, flags);
b427411a
ST		 1920 if (value == hba->clk_gating.is_enabled)
1921 goto out;
1922
b6645112
JK		 1923 if (value)
1924 __ufshcd_release(hba);
1925 else
b427411a 1926 hba->clk_gating.active_reqs++;
b427411a
ST		 1927
1928 hba->clk_gating.is_enabled = value;
1929out:
b6645112 1930 spin_unlock_irqrestore(hba->host->host_lock, flags);
b427411a
ST		 1931 return count;
1932}
1933
4543d9d7 1934static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
eebcc196 1935{
4543d9d7
CG		 1936 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1937 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1938 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1939 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1940 hba->clk_gating.delay_attr.attr.mode = 0644;
1941 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1942 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
eebcc196 1943
4543d9d7
CG		 1944 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1945 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1946 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1947 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1948 hba->clk_gating.enable_attr.attr.mode = 0644;
1949 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1950 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
eebcc196
VG		 1951}
1952
4543d9d7 1953static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
eebcc196 1954{
4543d9d7
CG		 1955 if (hba->clk_gating.delay_attr.attr.name)
1956 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1957 if (hba->clk_gating.enable_attr.attr.name)
1958 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
eebcc196
VG		 1959}
1960
1ab27c9c
ST		 1961static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1962{
10e5e375
VV		 1963 char wq_name[sizeof("ufs_clk_gating_00")];
1964
1ab27c9c
ST		 1965 if (!ufshcd_is_clkgating_allowed(hba))
1966 return;
1967
2dec9475
CG		 1968 hba->clk_gating.state = CLKS_ON;
1969
1ab27c9c
ST		 1970 hba->clk_gating.delay_ms = 150;
1971 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1972 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1973
10e5e375
VV		 1974 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1975 hba->host->host_no);
1976 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
e93e6e49 1977 WQ_MEM_RECLAIM | WQ_HIGHPRI);
10e5e375 1978
4543d9d7 1979 ufshcd_init_clk_gating_sysfs(hba);
b427411a 1980
4543d9d7
CG		 1981 hba->clk_gating.is_enabled = true;
1982 hba->clk_gating.is_initialized = true;
1ab27c9c
ST		 1983}
1984
1985static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1986{
4543d9d7 1987 if (!hba->clk_gating.is_initialized)
1ab27c9c 1988 return;
3489c34b 1989
4543d9d7 1990 ufshcd_remove_clk_gating_sysfs(hba);
3489c34b
BVA		 1991
1992 /* Ungate the clock if necessary. */
1993 ufshcd_hold(hba, false);
4543d9d7 1994 hba->clk_gating.is_initialized = false;
3489c34b
BVA		 1995 ufshcd_release(hba);
1996
1997 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1998}
1999
856b3483
ST		 2000/* Must be called with host lock acquired */
2001static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2002{
401f1e44 2003 bool queue_resume_work = false;
b1bf66d1 2004 ktime_t curr_t = ktime_get();
a45f9371 2005 unsigned long flags;
401f1e44 2006
fcb0c4b0 2007 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 2008 return;
2009
a45f9371 2010 spin_lock_irqsave(hba->host->host_lock, flags);
401f1e44 2011 if (!hba->clk_scaling.active_reqs++)
2012 queue_resume_work = true;
2013
a45f9371
CG		 2014 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2015 spin_unlock_irqrestore(hba->host->host_lock, flags);
401f1e44 2016 return;
a45f9371 2017 }
401f1e44 2018
2019 if (queue_resume_work)
2020 queue_work(hba->clk_scaling.workq,
2021 &hba->clk_scaling.resume_work);
2022
2023 if (!hba->clk_scaling.window_start_t) {
b1bf66d1 2024 hba->clk_scaling.window_start_t = curr_t;
401f1e44 2025 hba->clk_scaling.tot_busy_t = 0;
2026 hba->clk_scaling.is_busy_started = false;
2027 }
2028
856b3483 2029 if (!hba->clk_scaling.is_busy_started) {
b1bf66d1 2030 hba->clk_scaling.busy_start_t = curr_t;
856b3483
ST		 2031 hba->clk_scaling.is_busy_started = true;
2032 }
a45f9371 2033 spin_unlock_irqrestore(hba->host->host_lock, flags);
856b3483
ST		 2034}
2035
2036static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2037{
2038 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
a45f9371 2039 unsigned long flags;
856b3483 2040
fcb0c4b0 2041 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 2042 return;
2043
a45f9371
CG		 2044 spin_lock_irqsave(hba->host->host_lock, flags);
2045 hba->clk_scaling.active_reqs--;
856b3483
ST		 2046 if (!hba->outstanding_reqs && scaling->is_busy_started) {
2047 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2048 scaling->busy_start_t));
8b0e1953 2049 scaling->busy_start_t = 0;
856b3483
ST		 2050 scaling->is_busy_started = false;
2051 }
a45f9371 2052 spin_unlock_irqrestore(hba->host->host_lock, flags);
856b3483 2053}
1d8613a2
CG		 2054
2055static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2056{
2057 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2058 return READ;
2059 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2060 return WRITE;
2061 else
2062 return -EINVAL;
2063}
2064
2065static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2066 struct ufshcd_lrb *lrbp)
2067{
2068 struct ufs_hba_monitor *m = &hba->monitor;
2069
2070 return (m->enabled && lrbp && lrbp->cmd &&
2071 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2072 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2073}
2074
2075static void ufshcd_start_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2076{
2077 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
a45f9371 2078 unsigned long flags;
1d8613a2 2079
a45f9371 2080 spin_lock_irqsave(hba->host->host_lock, flags);
1d8613a2
CG		 2081 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2082 hba->monitor.busy_start_ts[dir] = ktime_get();
a45f9371 2083 spin_unlock_irqrestore(hba->host->host_lock, flags);
1d8613a2
CG		 2084}
2085
2086static void ufshcd_update_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2087{
2088 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
a45f9371 2089 unsigned long flags;
1d8613a2 2090
a45f9371 2091 spin_lock_irqsave(hba->host->host_lock, flags);
1d8613a2 2092 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
3f2c1002 2093 struct request *req = scsi_cmd_to_rq(lrbp->cmd);
1d8613a2
CG		 2094 struct ufs_hba_monitor *m = &hba->monitor;
2095 ktime_t now, inc, lat;
2096
2097 now = lrbp->compl_time_stamp;
2098 inc = ktime_sub(now, m->busy_start_ts[dir]);
2099 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2100 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2101
2102 /* Update latencies */
2103 m->nr_req[dir]++;
2104 lat = ktime_sub(now, lrbp->issue_time_stamp);
2105 m->lat_sum[dir] += lat;
2106 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2107 m->lat_max[dir] = lat;
2108 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2109 m->lat_min[dir] = lat;
2110
2111 m->nr_queued[dir]--;
2112 /* Push forward the busy start of monitor */
2113 m->busy_start_ts[dir] = now;
2114 }
a45f9371 2115 spin_unlock_irqrestore(hba->host->host_lock, flags);
856b3483 2116}
1d8613a2 2117
7a3e97b0
SY		 2118/**
2119 * ufshcd_send_command - Send SCSI or device management commands
2120 * @hba: per adapter instance
2121 * @task_tag: Task tag of the command
2122 */
2123static inline
2124void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2125{
6edfdcfe 2126 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
1f522c50 2127 unsigned long flags;
6edfdcfe
SC		 2128
2129 lrbp->issue_time_stamp = ktime_get();
2130 lrbp->compl_time_stamp = ktime_set(0, 0);
28fa68fc 2131 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
856b3483 2132 ufshcd_clk_scaling_start_busy(hba);
1d8613a2
CG		 2133 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2134 ufshcd_start_monitor(hba, lrbp);
169f5eb2
BVA		 2135
2136 spin_lock_irqsave(&hba->outstanding_lock, flags);
a024ad0d
BVA		 2137 if (hba->vops && hba->vops->setup_xfer_req)
2138 hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd);
169f5eb2 2139 __set_bit(task_tag, &hba->outstanding_reqs);
1f522c50 2140 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
169f5eb2
BVA		 2141 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2142
ad1a1b9c
GB		 2143 /* Make sure that doorbell is committed immediately */
2144 wmb();
7a3e97b0
SY		 2145}
2146
2147/**
2148 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 2149 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2150 */
2151static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2152{
1de4378f 2153 u8 *const sense_buffer = lrbp->cmd->sense_buffer;
7a3e97b0 2154 int len;
1de4378f
BVA		 2155
2156 if (sense_buffer &&
1c2623c5 2157 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 2158 int len_to_copy;
2159
5a0b0cb9 2160 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 2161 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 2162
1de4378f 2163 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
09a5a24f 2164 len_to_copy);
7a3e97b0
SY		 2165 }
2166}
2167
68078d5c
DR		 2168/**
2169 * ufshcd_copy_query_response() - Copy the Query Response and the data
2170 * descriptor
2171 * @hba: per adapter instance
8aa29f19 2172 * @lrbp: pointer to local reference block
68078d5c
DR		 2173 */
2174static
c6d4a831 2175int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 2176{
2177 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2178
68078d5c 2179 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 2180
68078d5c 2181 /* Get the descriptor */
1c90836f
AA		 2182 if (hba->dev_cmd.query.descriptor &&
2183 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 2184 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 2185 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 2186 u16 resp_len;
2187 u16 buf_len;
68078d5c
DR		 2188
2189 /* data segment length */
c6d4a831 2190 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 2191 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 2192 buf_len = be16_to_cpu(
2193 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 2194 if (likely(buf_len >= resp_len)) {
2195 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2196 } else {
2197 dev_warn(hba->dev,
3d4881d1
BH		 2198 "%s: rsp size %d is bigger than buffer size %d",
2199 __func__, resp_len, buf_len);
c6d4a831
DR		 2200 return -EINVAL;
2201 }
68078d5c 2202 }
c6d4a831
DR		 2203
2204 return 0;
68078d5c
DR		 2205}
2206
7a3e97b0
SY		 2207/**
2208 * ufshcd_hba_capabilities - Read controller capabilities
2209 * @hba: per adapter instance
df043c74
ST		 2210 *
2211 * Return: 0 on success, negative on error.
7a3e97b0 2212 */
df043c74 2213static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
7a3e97b0 2214{
df043c74
ST		 2215 int err;
2216
b873a275 2217 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 2218
2219 /* nutrs and nutmrs are 0 based values */
2220 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2221 hba->nutmrs =
2222 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
945c3cca 2223 hba->reserved_slot = hba->nutrs - 1;
df043c74
ST		 2224
2225 /* Read crypto capabilities */
2226 err = ufshcd_hba_init_crypto_capabilities(hba);
2227 if (err)
2228 dev_err(hba->dev, "crypto setup failed\n");
2229
2230 return err;
7a3e97b0
SY		 2231}
2232
2233/**
6ccf44fe
SJ		 2234 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2235 * to accept UIC commands
7a3e97b0 2236 * @hba: per adapter instance
6ccf44fe
SJ		 2237 * Return true on success, else false
2238 */
2239static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2240{
a858af9a 2241 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY;
6ccf44fe
SJ		 2242}
2243
53b3d9c3
SJ		 2244/**
2245 * ufshcd_get_upmcrs - Get the power mode change request status
2246 * @hba: Pointer to adapter instance
2247 *
2248 * This function gets the UPMCRS field of HCS register
2249 * Returns value of UPMCRS field
2250 */
2251static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2252{
2253 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2254}
2255
6ccf44fe 2256/**
35c7d874 2257 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
6ccf44fe
SJ		 2258 * @hba: per adapter instance
2259 * @uic_cmd: UIC command
7a3e97b0
SY		 2260 */
2261static inline void
6ccf44fe 2262ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 2263{
35c7d874
BVA		 2264 lockdep_assert_held(&hba->uic_cmd_mutex);
2265
6ccf44fe
SJ		 2266 WARN_ON(hba->active_uic_cmd);
2267
2268 hba->active_uic_cmd = uic_cmd;
2269
7a3e97b0 2270 /* Write Args */
6ccf44fe
SJ		 2271 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2272 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2273 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0 2274
28fa68fc 2275 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
aa5c6979 2276
7a3e97b0 2277 /* Write UIC Cmd */
6ccf44fe 2278 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 2279 REG_UIC_COMMAND);
7a3e97b0
SY		 2280}
2281
6ccf44fe 2282/**
35c7d874 2283 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
6ccf44fe 2284 * @hba: per adapter instance
8aa29f19 2285 * @uic_cmd: UIC command
6ccf44fe 2286 *
6ccf44fe
SJ		 2287 * Returns 0 only if success.
2288 */
2289static int
2290ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2291{
2292 int ret;
2293 unsigned long flags;
2294
35c7d874
BVA		 2295 lockdep_assert_held(&hba->uic_cmd_mutex);
2296
6ccf44fe 2297 if (wait_for_completion_timeout(&uic_cmd->done,
0f52fcb9 2298 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
6ccf44fe 2299 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
0f52fcb9 2300 } else {
6ccf44fe 2301 ret = -ETIMEDOUT;
0f52fcb9
CG		 2302 dev_err(hba->dev,
2303 "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2304 uic_cmd->command, uic_cmd->argument3);
2305
2306 if (!uic_cmd->cmd_active) {
2307 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2308 __func__);
2309 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2310 }
2311 }
6ccf44fe
SJ		 2312
2313 spin_lock_irqsave(hba->host->host_lock, flags);
2314 hba->active_uic_cmd = NULL;
2315 spin_unlock_irqrestore(hba->host->host_lock, flags);
2316
2317 return ret;
2318}
2319
2320/**
2321 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2322 * @hba: per adapter instance
2323 * @uic_cmd: UIC command
d75f7fe4 2324 * @completion: initialize the completion only if this is set to true
6ccf44fe 2325 *
6ccf44fe
SJ		 2326 * Returns 0 only if success.
2327 */
2328static int
d75f7fe4
YG		 2329__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2330 bool completion)
6ccf44fe 2331{
35c7d874
BVA		 2332 lockdep_assert_held(&hba->uic_cmd_mutex);
2333 lockdep_assert_held(hba->host->host_lock);
2334
6ccf44fe
SJ		 2335 if (!ufshcd_ready_for_uic_cmd(hba)) {
2336 dev_err(hba->dev,
2337 "Controller not ready to accept UIC commands\n");
2338 return -EIO;
2339 }
2340
d75f7fe4
YG		 2341 if (completion)
2342 init_completion(&uic_cmd->done);
6ccf44fe 2343
0f52fcb9 2344 uic_cmd->cmd_active = 1;
6ccf44fe 2345 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2346
57d104c1 2347 return 0;
6ccf44fe
SJ		 2348}
2349
2350/**
2351 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2352 * @hba: per adapter instance
2353 * @uic_cmd: UIC command
2354 *
2355 * Returns 0 only if success.
2356 */
e77044c5 2357int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2358{
2359 int ret;
57d104c1 2360 unsigned long flags;
6ccf44fe 2361
a22bcfdb 2362 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2363 return 0;
2364
1ab27c9c 2365 ufshcd_hold(hba, false);
6ccf44fe 2366 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2367 ufshcd_add_delay_before_dme_cmd(hba);
2368
57d104c1 2369 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2370 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2371 spin_unlock_irqrestore(hba->host->host_lock, flags);
2372 if (!ret)
2373 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2374
6ccf44fe
SJ		 2375 mutex_unlock(&hba->uic_cmd_mutex);
2376
1ab27c9c 2377 ufshcd_release(hba);
6ccf44fe
SJ		 2378 return ret;
2379}
2380
7a3e97b0
SY		 2381/**
2382 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2383 * @hba: per adapter instance
2384 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2385 *
2386 * Returns 0 in case of success, non-zero value in case of failure
2387 */
75b1cc4a 2388static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2389{
2390 struct ufshcd_sg_entry *prd_table;
2391 struct scatterlist *sg;
2392 struct scsi_cmnd *cmd;
2393 int sg_segments;
2394 int i;
2395
2396 cmd = lrbp->cmd;
2397 sg_segments = scsi_dma_map(cmd);
2398 if (sg_segments < 0)
2399 return sg_segments;
2400
2401 if (sg_segments) {
26f968d7
AA		 2402
2403 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2404 lrbp->utr_descriptor_ptr->prd_table_length =
2405 cpu_to_le16((sg_segments *
2406 sizeof(struct ufshcd_sg_entry)));
2407 else
2408 lrbp->utr_descriptor_ptr->prd_table_length =
3ad317a1 2409 cpu_to_le16(sg_segments);
7a3e97b0 2410
3ad317a1 2411 prd_table = lrbp->ucd_prdt_ptr;
7a3e97b0
SY		 2412
2413 scsi_for_each_sg(cmd, sg, sg_segments, i) {
1ea7d802
BVA		 2414 const unsigned int len = sg_dma_len(sg);
2415
2416 /*
2417 * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2418 * based value that indicates the length, in bytes, of
2419 * the data block. A maximum of length of 256KB may
2420 * exist for any entry. Bits 1:0 of this field shall be
2421 * 11b to indicate Dword granularity. A value of '3'
2422 * indicates 4 bytes, '7' indicates 8 bytes, etc."
2423 */
2424 WARN_ONCE(len > 256 * 1024, "len = %#x\n", len);
2425 prd_table[i].size = cpu_to_le32(len - 1);
2426 prd_table[i].addr = cpu_to_le64(sg->dma_address);
52ac95fe 2427 prd_table[i].reserved = 0;
7a3e97b0
SY		 2428 }
2429 } else {
2430 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2431 }
2432
2433 return 0;
2434}
2435
2436/**
2fbd009b 2437 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2438 * @hba: per adapter instance
2fbd009b 2439 * @intrs: interrupt bits
7a3e97b0 2440 */
2fbd009b 2441static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2442{
2fbd009b
SJ		 2443 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2444
51428818 2445 if (hba->ufs_version == ufshci_version(1, 0)) {
2fbd009b
SJ		 2446 u32 rw;
2447 rw = set & INTERRUPT_MASK_RW_VER_10;
2448 set = rw | ((set ^ intrs) & intrs);
2449 } else {
2450 set |= intrs;
2451 }
2452
2453 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2454}
2455
2456/**
2457 * ufshcd_disable_intr - disable interrupts
2458 * @hba: per adapter instance
2459 * @intrs: interrupt bits
2460 */
2461static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2462{
2463 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2464
51428818 2465 if (hba->ufs_version == ufshci_version(1, 0)) {
2fbd009b
SJ		 2466 u32 rw;
2467 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2468 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2469 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2470
2471 } else {
2472 set &= ~intrs;
7a3e97b0 2473 }
2fbd009b
SJ		 2474
2475 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2476}
2477
5a0b0cb9
SRT		 2478/**
2479 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2480 * descriptor according to request
2481 * @lrbp: pointer to local reference block
2482 * @upiu_flags: flags required in the header
2483 * @cmd_dir: requests data direction
2484 */
2485static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
a23064c4 2486 u8 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2487{
2488 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2489 u32 data_direction;
2490 u32 dword_0;
df043c74
ST		 2491 u32 dword_1 = 0;
2492 u32 dword_3 = 0;
5a0b0cb9
SRT		 2493
2494 if (cmd_dir == DMA_FROM_DEVICE) {
2495 data_direction = UTP_DEVICE_TO_HOST;
2496 *upiu_flags = UPIU_CMD_FLAGS_READ;
2497 } else if (cmd_dir == DMA_TO_DEVICE) {
2498 data_direction = UTP_HOST_TO_DEVICE;
2499 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2500 } else {
2501 data_direction = UTP_NO_DATA_TRANSFER;
2502 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2503 }
2504
2505 dword_0 = data_direction | (lrbp->command_type
2506 << UPIU_COMMAND_TYPE_OFFSET);
2507 if (lrbp->intr_cmd)
2508 dword_0 |= UTP_REQ_DESC_INT_CMD;
2509
df043c74
ST		 2510 /* Prepare crypto related dwords */
2511 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2512
5a0b0cb9
SRT		 2513 /* Transfer request descriptor header fields */
2514 req_desc->header.dword_0 = cpu_to_le32(dword_0);
df043c74 2515 req_desc->header.dword_1 = cpu_to_le32(dword_1);
5a0b0cb9
SRT		 2516 /*
2517 * assigning invalid value for command status. Controller
2518 * updates OCS on command completion, with the command
2519 * status
2520 */
2521 req_desc->header.dword_2 =
2522 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
df043c74 2523 req_desc->header.dword_3 = cpu_to_le32(dword_3);
51047266
YG		 2524
2525 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2526}
2527
2528/**
2529 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2530 * for scsi commands
8aa29f19
BVA		 2531 * @lrbp: local reference block pointer
2532 * @upiu_flags: flags
5a0b0cb9
SRT		 2533 */
2534static
a23064c4 2535void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
5a0b0cb9 2536{
1b21b8f0 2537 struct scsi_cmnd *cmd = lrbp->cmd;
5a0b0cb9 2538 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2539 unsigned short cdb_len;
5a0b0cb9
SRT		 2540
2541 /* command descriptor fields */
2542 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2543 UPIU_TRANSACTION_COMMAND, upiu_flags,
2544 lrbp->lun, lrbp->task_tag);
2545 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2546 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2547
2548 /* Total EHS length and Data segment length will be zero */
2549 ucd_req_ptr->header.dword_2 = 0;
2550
1b21b8f0 2551 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
5a0b0cb9 2552
1b21b8f0 2553 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
a851b2bd 2554 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1b21b8f0 2555 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
52ac95fe
YG		 2556
2557 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2558}
2559
68078d5c
DR		 2560/**
2561 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2562 * for query requsts
2563 * @hba: UFS hba
2564 * @lrbp: local reference block pointer
2565 * @upiu_flags: flags
2566 */
2567static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
a23064c4 2568 struct ufshcd_lrb *lrbp, u8 upiu_flags)
68078d5c
DR		 2569{
2570 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2571 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2572 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2573
2574 /* Query request header */
2575 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2576 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2577 lrbp->lun, lrbp->task_tag);
2578 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2579 0, query->request.query_func, 0, 0);
2580
6861285c
ZL		 2581 /* Data segment length only need for WRITE_DESC */
2582 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2583 ucd_req_ptr->header.dword_2 =
2584 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2585 else
2586 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2587
2588 /* Copy the Query Request buffer as is */
2589 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2590 QUERY_OSF_SIZE);
68078d5c
DR		 2591
2592 /* Copy the Descriptor */
c6d4a831 2593 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2594 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2595
51047266 2596 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2597}
2598
5a0b0cb9
SRT		 2599static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2600{
2601 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2602
2603 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2604
2605 /* command descriptor fields */
2606 ucd_req_ptr->header.dword_0 =
2607 UPIU_HEADER_DWORD(
2608 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2609 /* clear rest of the fields of basic header */
2610 ucd_req_ptr->header.dword_1 = 0;
2611 ucd_req_ptr->header.dword_2 = 0;
2612
2613 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2614}
2615
7a3e97b0 2616/**
f273c54b 2617 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
300bb13f 2618 * for Device Management Purposes
8aa29f19
BVA		 2619 * @hba: per adapter instance
2620 * @lrbp: pointer to local reference block
7a3e97b0 2621 */
f273c54b
BH		 2622static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2623 struct ufshcd_lrb *lrbp)
7a3e97b0 2624{
a23064c4 2625 u8 upiu_flags;
5a0b0cb9 2626 int ret = 0;
7a3e97b0 2627
51428818 2628 if (hba->ufs_version <= ufshci_version(1, 1))
300bb13f 2629 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2630 else
2631 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2632
2633 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2634 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2635 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2636 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2637 ufshcd_prepare_utp_nop_upiu(lrbp);
2638 else
2639 ret = -EINVAL;
2640
2641 return ret;
2642}
2643
2644/**
2645 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2646 * for SCSI Purposes
8aa29f19
BVA		 2647 * @hba: per adapter instance
2648 * @lrbp: pointer to local reference block
300bb13f
JP		 2649 */
2650static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2651{
a23064c4 2652 u8 upiu_flags;
300bb13f
JP		 2653 int ret = 0;
2654
51428818 2655 if (hba->ufs_version <= ufshci_version(1, 1))
300bb13f 2656 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2657 else
2658 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2659
2660 if (likely(lrbp->cmd)) {
2661 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2662 lrbp->cmd->sc_data_direction);
2663 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2664 } else {
2665 ret = -EINVAL;
2666 }
5a0b0cb9
SRT		 2667
2668 return ret;
7a3e97b0
SY		 2669}
2670
2a8fa600
SJ		 2671/**
2672 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2673 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2674 *
2675 * Returns SCSI W-LUN id
2676 */
2677static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2678{
2679 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2680}
2681
b294ff3e
AD		 2682static inline bool is_device_wlun(struct scsi_device *sdev)
2683{
2684 return sdev->lun ==
2685 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2686}
2687
eaab9b57
BVA		 2688/*
2689 * Associate the UFS controller queue with the default and poll HCTX types.
2690 * Initialize the mq_map[] arrays.
2691 */
2692static int ufshcd_map_queues(struct Scsi_Host *shost)
2693{
2694 int i, ret;
2695
2696 for (i = 0; i < shost->nr_maps; i++) {
2697 struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2698
2699 switch (i) {
2700 case HCTX_TYPE_DEFAULT:
2701 case HCTX_TYPE_POLL:
2702 map->nr_queues = 1;
2703 break;
2704 case HCTX_TYPE_READ:
2705 map->nr_queues = 0;
10af1156 2706 continue;
eaab9b57
BVA		 2707 default:
2708 WARN_ON_ONCE(true);
2709 }
2710 map->queue_offset = 0;
2711 ret = blk_mq_map_queues(map);
2712 WARN_ON_ONCE(ret);
2713 }
2714
2715 return 0;
2716}
2717
4d2b8d40
BVA		 2718static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2719{
2720 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2721 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2722 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2723 i * sizeof(struct utp_transfer_cmd_desc);
2724 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2725 response_upiu);
2726 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2727
2728 lrb->utr_descriptor_ptr = utrdlp + i;
2729 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2730 i * sizeof(struct utp_transfer_req_desc);
2731 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2732 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2733 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2734 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
3ad317a1 2735 lrb->ucd_prdt_ptr = cmd_descp[i].prd_table;
4d2b8d40
BVA		 2736 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2737}
2738
7a3e97b0
SY		 2739/**
2740 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2741 * @host: SCSI host pointer
7a3e97b0 2742 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2743 *
2744 * Returns 0 for success, non-zero in case of failure
2745 */
2746static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2747{
4728ab4a 2748 struct ufs_hba *hba = shost_priv(host);
3f2c1002 2749 int tag = scsi_cmd_to_rq(cmd)->tag;
7a3e97b0 2750 struct ufshcd_lrb *lrbp;
7a3e97b0
SY		 2751 int err = 0;
2752
eaab9b57 2753 WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag);
7a3e97b0 2754
5675c381
BVA		 2755 /*
2756 * Allows the UFS error handler to wait for prior ufshcd_queuecommand()
2757 * calls.
2758 */
2759 rcu_read_lock();
2760
a45f9371
CG		 2761 switch (hba->ufshcd_state) {
2762 case UFSHCD_STATE_OPERATIONAL:
d489f18a 2763 break;
a45f9371 2764 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
d489f18a
AH		 2765 /*
2766 * SCSI error handler can call ->queuecommand() while UFS error
2767 * handler is in progress. Error interrupts could change the
2768 * state from UFSHCD_STATE_RESET to
2769 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2770 * being issued in that case.
2771 */
2772 if (ufshcd_eh_in_progress(hba)) {
2773 err = SCSI_MLQUEUE_HOST_BUSY;
2774 goto out;
2775 }
a45f9371
CG		 2776 break;
2777 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2778 /*
2779 * pm_runtime_get_sync() is used at error handling preparation
2780 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2781 * PM ops, it can never be finished if we let SCSI layer keep
2782 * retrying it, which gets err handler stuck forever. Neither
2783 * can we let the scsi cmd pass through, because UFS is in bad
2784 * state, the scsi cmd may eventually time out, which will get
2785 * err handler blocked for too long. So, just fail the scsi cmd
2786 * sent from PM ops, err handler can recover PM error anyways.
2787 */
2788 if (hba->pm_op_in_progress) {
2789 hba->force_reset = true;
2790 set_host_byte(cmd, DID_BAD_TARGET);
35c3730a 2791 scsi_done(cmd);
a45f9371
CG		 2792 goto out;
2793 }
2794 fallthrough;
2795 case UFSHCD_STATE_RESET:
2796 err = SCSI_MLQUEUE_HOST_BUSY;
2797 goto out;
2798 case UFSHCD_STATE_ERROR:
2799 set_host_byte(cmd, DID_ERROR);
35c3730a 2800 scsi_done(cmd);
a45f9371 2801 goto out;
a45f9371
CG		 2802 }
2803
7fabb77b
GB		 2804 hba->req_abort_count = 0;
2805
1ab27c9c
ST		 2806 err = ufshcd_hold(hba, true);
2807 if (err) {
2808 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2809 goto out;
2810 }
2dec9475
CG		 2811 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2812 (hba->clk_gating.state != CLKS_ON));
1ab27c9c 2813
a45f9371 2814 lrbp = &hba->lrb[tag];
5a0b0cb9 2815 WARN_ON(lrbp->cmd);
7a3e97b0 2816 lrbp->cmd = cmd;
7a3e97b0 2817 lrbp->task_tag = tag;
0ce147d4 2818 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
51d1628f 2819 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
df043c74 2820
3f2c1002 2821 ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
df043c74 2822
e0b299e3 2823 lrbp->req_abort_skip = false;
7a3e97b0 2824
09d9e4d0 2825 ufshpb_prep(hba, lrbp);
2fff76f8 2826
300bb13f
JP		 2827 ufshcd_comp_scsi_upiu(hba, lrbp);
2828
75b1cc4a 2829 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2830 if (err) {
2831 lrbp->cmd = NULL;
17c7d35f 2832 ufshcd_release(hba);
7a3e97b0 2833 goto out;
5a0b0cb9 2834 }
7a3e97b0 2835
7a3e97b0 2836 ufshcd_send_command(hba, tag);
5675c381 2837
7a3e97b0 2838out:
5675c381
BVA		 2839 rcu_read_unlock();
2840
88b09900
AH		 2841 if (ufs_trigger_eh()) {
2842 unsigned long flags;
2843
2844 spin_lock_irqsave(hba->host->host_lock, flags);
2845 ufshcd_schedule_eh_work(hba);
2846 spin_unlock_irqrestore(hba->host->host_lock, flags);
2847 }
c11a1ae9 2848
7a3e97b0
SY		 2849 return err;
2850}
2851
5a0b0cb9
SRT		 2852static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2853 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2854{
2855 lrbp->cmd = NULL;
5a0b0cb9
SRT		 2856 lrbp->task_tag = tag;
2857 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9 2858 lrbp->intr_cmd = true; /* No interrupt aggregation */
df043c74 2859 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5a0b0cb9
SRT		 2860 hba->dev_cmd.type = cmd_type;
2861
f273c54b 2862 return ufshcd_compose_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2863}
2864
2865static int
2866ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2867{
2868 int err = 0;
2869 unsigned long flags;
2870 u32 mask = 1 << tag;
2871
2872 /* clear outstanding transaction before retry */
2873 spin_lock_irqsave(hba->host->host_lock, flags);
2874 ufshcd_utrl_clear(hba, tag);
2875 spin_unlock_irqrestore(hba->host->host_lock, flags);
2876
2877 /*
32424902 2878 * wait for h/w to clear corresponding bit in door-bell.
5a0b0cb9
SRT		 2879 * max. wait is 1 sec.
2880 */
2881 err = ufshcd_wait_for_register(hba,
2882 REG_UTP_TRANSFER_REQ_DOOR_BELL,
5cac1095 2883 mask, ~mask, 1000, 1000);
5a0b0cb9
SRT		 2884
2885 return err;
2886}
2887
c6d4a831
DR		 2888static int
2889ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2890{
2891 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2892
2893 /* Get the UPIU response */
2894 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2895 UPIU_RSP_CODE_OFFSET;
2896 return query_res->response;
2897}
2898
5a0b0cb9
SRT		 2899/**
2900 * ufshcd_dev_cmd_completion() - handles device management command responses
2901 * @hba: per adapter instance
2902 * @lrbp: pointer to local reference block
2903 */
2904static int
2905ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2906{
2907 int resp;
2908 int err = 0;
2909
ff8e20c6 2910 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2911 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2912
2913 switch (resp) {
2914 case UPIU_TRANSACTION_NOP_IN:
2915 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2916 err = -EINVAL;
2917 dev_err(hba->dev, "%s: unexpected response %x\n",
2918 __func__, resp);
2919 }
2920 break;
68078d5c 2921 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2922 err = ufshcd_check_query_response(hba, lrbp);
2923 if (!err)
2924 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2925 break;
5a0b0cb9
SRT		 2926 case UPIU_TRANSACTION_REJECT_UPIU:
2927 /* TODO: handle Reject UPIU Response */
2928 err = -EPERM;
2929 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2930 __func__);
2931 break;
2932 default:
2933 err = -EINVAL;
2934 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2935 __func__, resp);
2936 break;
2937 }
2938
2939 return err;
2940}
2941
2942static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2943 struct ufshcd_lrb *lrbp, int max_timeout)
2944{
2945 int err = 0;
2946 unsigned long time_left;
2947 unsigned long flags;
2948
2949 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2950 msecs_to_jiffies(max_timeout));
2951
2952 spin_lock_irqsave(hba->host->host_lock, flags);
2953 hba->dev_cmd.complete = NULL;
2954 if (likely(time_left)) {
2955 err = ufshcd_get_tr_ocs(lrbp);
2956 if (!err)
2957 err = ufshcd_dev_cmd_completion(hba, lrbp);
2958 }
2959 spin_unlock_irqrestore(hba->host->host_lock, flags);
2960
2961 if (!time_left) {
2962 err = -ETIMEDOUT;
a48353f6
YG		 2963 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2964 __func__, lrbp->task_tag);
5a0b0cb9 2965 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2966 /* successfully cleared the command, retry if needed */
5a0b0cb9 2967 err = -EAGAIN;
a48353f6
YG		 2968 /*
2969 * in case of an error, after clearing the doorbell,
2970 * we also need to clear the outstanding_request
2971 * field in hba
2972 */
169f5eb2
BVA		 2973 spin_lock_irqsave(&hba->outstanding_lock, flags);
2974 __clear_bit(lrbp->task_tag, &hba->outstanding_reqs);
2975 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5a0b0cb9
SRT		 2976 }
2977
2978 return err;
2979}
2980
5a0b0cb9
SRT		 2981/**
2982 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2983 * @hba: UFS hba
2984 * @cmd_type: specifies the type (NOP, Query...)
d0b2b70e 2985 * @timeout: timeout in milliseconds
5a0b0cb9 2986 *
68078d5c
DR		 2987 * NOTE: Since there is only one available tag for device management commands,
2988 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2989 */
2990static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2991 enum dev_cmd_type cmd_type, int timeout)
2992{
8a686f26 2993 DECLARE_COMPLETION_ONSTACK(wait);
945c3cca 2994 const u32 tag = hba->reserved_slot;
5a0b0cb9
SRT		 2995 struct ufshcd_lrb *lrbp;
2996 int err;
5a0b0cb9 2997
945c3cca
BVA		 2998 /* Protects use of hba->reserved_slot. */
2999 lockdep_assert_held(&hba->dev_cmd.lock);
a3cd5ec5 3000
945c3cca 3001 down_read(&hba->clk_scaling_lock);
5a0b0cb9 3002
a45f9371 3003 lrbp = &hba->lrb[tag];
5a0b0cb9
SRT		 3004 WARN_ON(lrbp->cmd);
3005 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3006 if (unlikely(err))
eb783bb8 3007 goto out;
5a0b0cb9
SRT		 3008
3009 hba->dev_cmd.complete = &wait;
3010
fb475b74 3011 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
5a0b0cb9 3012
a45f9371 3013 ufshcd_send_command(hba, tag);
5a0b0cb9 3014 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
fb475b74
AA		 3015 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3016 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
6667e6d9 3017
eb783bb8 3018out:
a3cd5ec5 3019 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 3020 return err;
3021}
3022
d44a5f98
DR		 3023/**
3024 * ufshcd_init_query() - init the query response and request parameters
3025 * @hba: per-adapter instance
3026 * @request: address of the request pointer to be initialized
3027 * @response: address of the response pointer to be initialized
3028 * @opcode: operation to perform
3029 * @idn: flag idn to access
3030 * @index: LU number to access
3031 * @selector: query/flag/descriptor further identification
3032 */
3033static inline void ufshcd_init_query(struct ufs_hba *hba,
3034 struct ufs_query_req **request, struct ufs_query_res **response,
3035 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3036{
3037 *request = &hba->dev_cmd.query.request;
3038 *response = &hba->dev_cmd.query.response;
3039 memset(*request, 0, sizeof(struct ufs_query_req));
3040 memset(*response, 0, sizeof(struct ufs_query_res));
3041 (*request)->upiu_req.opcode = opcode;
3042 (*request)->upiu_req.idn = idn;
3043 (*request)->upiu_req.index = index;
3044 (*request)->upiu_req.selector = selector;
3045}
3046
dc3c8d3a 3047static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1f34eedf 3048 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
dc3c8d3a
YG		 3049{
3050 int ret;
3051 int retries;
3052
3053 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1f34eedf 3054 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
dc3c8d3a
YG		 3055 if (ret)
3056 dev_dbg(hba->dev,
3057 "%s: failed with error %d, retries %d\n",
3058 __func__, ret, retries);
3059 else
3060 break;
3061 }
3062
3063 if (ret)
3064 dev_err(hba->dev,
3065 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
3066 __func__, opcode, idn, ret, retries);
3067 return ret;
3068}
3069
68078d5c
DR		 3070/**
3071 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 3072 * @hba: per-adapter instance
3073 * @opcode: flag query to perform
3074 * @idn: flag idn to access
1f34eedf 3075 * @index: flag index to access
8aa29f19 3076 * @flag_res: the flag value after the query request completes
68078d5c
DR		 3077 *
3078 * Returns 0 for success, non-zero in case of failure
3079 */
dc3c8d3a 3080int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1f34eedf 3081 enum flag_idn idn, u8 index, bool *flag_res)
68078d5c 3082{
d44a5f98
DR		 3083 struct ufs_query_req *request = NULL;
3084 struct ufs_query_res *response = NULL;
1f34eedf 3085 int err, selector = 0;
e5ad406c 3086 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 3087
3088 BUG_ON(!hba);
3089
1ab27c9c 3090 ufshcd_hold(hba, false);
68078d5c 3091 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 3092 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3093 selector);
68078d5c
DR		 3094
3095 switch (opcode) {
3096 case UPIU_QUERY_OPCODE_SET_FLAG:
3097 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3098 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3099 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3100 break;
3101 case UPIU_QUERY_OPCODE_READ_FLAG:
3102 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3103 if (!flag_res) {
3104 /* No dummy reads */
3105 dev_err(hba->dev, "%s: Invalid argument for read request\n",
3106 __func__);
3107 err = -EINVAL;
3108 goto out_unlock;
3109 }
3110 break;
3111 default:
3112 dev_err(hba->dev,
3113 "%s: Expected query flag opcode but got = %d\n",
3114 __func__, opcode);
3115 err = -EINVAL;
3116 goto out_unlock;
3117 }
68078d5c 3118
e5ad406c 3119 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 3120
3121 if (err) {
3122 dev_err(hba->dev,
3123 "%s: Sending flag query for idn %d failed, err = %d\n",
3124 __func__, idn, err);
3125 goto out_unlock;
3126 }
3127
3128 if (flag_res)
e8c8e82a 3129 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 3130 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3131
3132out_unlock:
3133 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 3134 ufshcd_release(hba);
68078d5c
DR		 3135 return err;
3136}
3137
66ec6d59
SRT		 3138/**
3139 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 3140 * @hba: per-adapter instance
3141 * @opcode: attribute opcode
3142 * @idn: attribute idn to access
3143 * @index: index field
3144 * @selector: selector field
3145 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 3146 *
3147 * Returns 0 for success, non-zero in case of failure
3148*/
ec92b59c
SN		 3149int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3150 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 3151{
d44a5f98
DR		 3152 struct ufs_query_req *request = NULL;
3153 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 3154 int err;
3155
3156 BUG_ON(!hba);
3157
3158 if (!attr_val) {
3159 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3160 __func__, opcode);
8ca1a40b 3161 return -EINVAL;
66ec6d59
SRT		 3162 }
3163
8ca1a40b 3164 ufshcd_hold(hba, false);
3165
66ec6d59 3166 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 3167 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3168 selector);
66ec6d59
SRT		 3169
3170 switch (opcode) {
3171 case UPIU_QUERY_OPCODE_WRITE_ATTR:
3172 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 3173 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 3174 break;
3175 case UPIU_QUERY_OPCODE_READ_ATTR:
3176 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3177 break;
3178 default:
3179 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3180 __func__, opcode);
3181 err = -EINVAL;
3182 goto out_unlock;
3183 }
3184
d44a5f98 3185 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 3186
3187 if (err) {
4b761b58
YG		 3188 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3189 __func__, opcode, idn, index, err);
66ec6d59
SRT		 3190 goto out_unlock;
3191 }
3192
e8c8e82a 3193 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 3194
3195out_unlock:
3196 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 3197 ufshcd_release(hba);
66ec6d59
SRT		 3198 return err;
3199}
3200
5e86ae44
YG		 3201/**
3202 * ufshcd_query_attr_retry() - API function for sending query
3203 * attribute with retries
3204 * @hba: per-adapter instance
3205 * @opcode: attribute opcode
3206 * @idn: attribute idn to access
3207 * @index: index field
3208 * @selector: selector field
3209 * @attr_val: the attribute value after the query request
3210 * completes
3211 *
3212 * Returns 0 for success, non-zero in case of failure
3213*/
41d8a933 3214int ufshcd_query_attr_retry(struct ufs_hba *hba,
5e86ae44
YG		 3215 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3216 u32 *attr_val)
3217{
3218 int ret = 0;
3219 u32 retries;
3220
68c9fcfd 3221 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
5e86ae44
YG		 3222 ret = ufshcd_query_attr(hba, opcode, idn, index,
3223 selector, attr_val);
3224 if (ret)
3225 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3226 __func__, ret, retries);
3227 else
3228 break;
3229 }
3230
3231 if (ret)
3232 dev_err(hba->dev,
3233 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3234 __func__, idn, ret, QUERY_REQ_RETRIES);
3235 return ret;
3236}
3237
a70e91b8 3238static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 3239 enum query_opcode opcode, enum desc_idn idn, u8 index,
3240 u8 selector, u8 *desc_buf, int *buf_len)
3241{
3242 struct ufs_query_req *request = NULL;
3243 struct ufs_query_res *response = NULL;
3244 int err;
3245
3246 BUG_ON(!hba);
3247
3248 if (!desc_buf) {
3249 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3250 __func__, opcode);
8ca1a40b 3251 return -EINVAL;
d44a5f98
DR		 3252 }
3253
a4b0e8a4 3254 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 3255 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3256 __func__, *buf_len);
8ca1a40b 3257 return -EINVAL;
d44a5f98
DR		 3258 }
3259
8ca1a40b 3260 ufshcd_hold(hba, false);
3261
d44a5f98
DR		 3262 mutex_lock(&hba->dev_cmd.lock);
3263 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3264 selector);
3265 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 3266 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 3267
3268 switch (opcode) {
3269 case UPIU_QUERY_OPCODE_WRITE_DESC:
3270 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3271 break;
3272 case UPIU_QUERY_OPCODE_READ_DESC:
3273 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3274 break;
3275 default:
3276 dev_err(hba->dev,
3277 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3278 __func__, opcode);
3279 err = -EINVAL;
3280 goto out_unlock;
3281 }
3282
3283 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3284
3285 if (err) {
4b761b58
YG		 3286 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3287 __func__, opcode, idn, index, err);
d44a5f98
DR		 3288 goto out_unlock;
3289 }
3290
ea2aab24 3291 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 3292
3293out_unlock:
cfcbae38 3294 hba->dev_cmd.query.descriptor = NULL;
d44a5f98 3295 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 3296 ufshcd_release(hba);
d44a5f98
DR		 3297 return err;
3298}
3299
a70e91b8 3300/**
8aa29f19
BVA		 3301 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3302 * @hba: per-adapter instance
3303 * @opcode: attribute opcode
3304 * @idn: attribute idn to access
3305 * @index: index field
3306 * @selector: selector field
3307 * @desc_buf: the buffer that contains the descriptor
3308 * @buf_len: length parameter passed to the device
a70e91b8
YG		 3309 *
3310 * Returns 0 for success, non-zero in case of failure.
3311 * The buf_len parameter will contain, on return, the length parameter
3312 * received on the response.
3313 */
2238d31c
SN		 3314int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3315 enum query_opcode opcode,
3316 enum desc_idn idn, u8 index,
3317 u8 selector,
3318 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 3319{
3320 int err;
3321 int retries;
3322
3323 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3324 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3325 selector, desc_buf, buf_len);
3326 if (!err || err == -EINVAL)
3327 break;
3328 }
3329
3330 return err;
3331}
a70e91b8 3332
a4b0e8a4
PM		 3333/**
3334 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3335 * @hba: Pointer to adapter instance
3336 * @desc_id: descriptor idn value
3337 * @desc_len: mapped desc length (out)
a4b0e8a4 3338 */
7a0bf85b
BH		 3339void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3340 int *desc_len)
a4b0e8a4 3341{
7a0bf85b
BH		 3342 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3343 desc_id == QUERY_DESC_IDN_RFU_1)
a4b0e8a4 3344 *desc_len = 0;
7a0bf85b
BH		 3345 else
3346 *desc_len = hba->desc_size[desc_id];
a4b0e8a4
PM		 3347}
3348EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3349
7a0bf85b 3350static void ufshcd_update_desc_length(struct ufs_hba *hba,
72fb690e 3351 enum desc_idn desc_id, int desc_index,
7a0bf85b
BH		 3352 unsigned char desc_len)
3353{
3354 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
72fb690e
BH		 3355 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3356 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3357 * than the RPMB unit, however, both descriptors share the same
3358 * desc_idn, to cover both unit descriptors with one length, we
3359 * choose the normal unit descriptor length by desc_index.
3360 */
7a0bf85b
BH		 3361 hba->desc_size[desc_id] = desc_len;
3362}
3363
da461cec
SJ		 3364/**
3365 * ufshcd_read_desc_param - read the specified descriptor parameter
3366 * @hba: Pointer to adapter instance
3367 * @desc_id: descriptor idn value
3368 * @desc_index: descriptor index
3369 * @param_offset: offset of the parameter to read
3370 * @param_read_buf: pointer to buffer where parameter would be read
3371 * @param_size: sizeof(param_read_buf)
3372 *
3373 * Return 0 in case of success, non-zero otherwise
3374 */
45bced87
SN		 3375int ufshcd_read_desc_param(struct ufs_hba *hba,
3376 enum desc_idn desc_id,
3377 int desc_index,
3378 u8 param_offset,
3379 u8 *param_read_buf,
3380 u8 param_size)
da461cec
SJ		 3381{
3382 int ret;
3383 u8 *desc_buf;
a4b0e8a4 3384 int buff_len;
da461cec
SJ		 3385 bool is_kmalloc = true;
3386
a4b0e8a4
PM		 3387 /* Safety check */
3388 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3389 return -EINVAL;
3390
7a0bf85b
BH		 3391 /* Get the length of descriptor */
3392 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3393 if (!buff_len) {
1699f980
CG		 3394 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3395 return -EINVAL;
3396 }
3397
3398 if (param_offset >= buff_len) {
3399 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3400 __func__, param_offset, desc_id, buff_len);
7a0bf85b 3401 return -EINVAL;
a4b0e8a4
PM		 3402 }
3403
3404 /* Check whether we need temp memory */
3405 if (param_offset != 0 || param_size < buff_len) {
1699f980 3406 desc_buf = kzalloc(buff_len, GFP_KERNEL);
da461cec
SJ		 3407 if (!desc_buf)
3408 return -ENOMEM;
a4b0e8a4
PM		 3409 } else {
3410 desc_buf = param_read_buf;
3411 is_kmalloc = false;
da461cec
SJ		 3412 }
3413
a4b0e8a4 3414 /* Request for full descriptor */
a70e91b8 3415 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3416 desc_id, desc_index, 0,
3417 desc_buf, &buff_len);
da461cec 3418
bde44bb6 3419 if (ret) {
1699f980 3420 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
bde44bb6 3421 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3422 goto out;
3423 }
3424
bde44bb6 3425 /* Sanity check */
3426 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
1699f980 3427 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
bde44bb6 3428 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3429 ret = -EINVAL;
3430 goto out;
3431 }
3432
7a0bf85b
BH		 3433 /* Update descriptor length */
3434 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
72fb690e 3435 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
7a0bf85b 3436
1699f980
CG		 3437 if (is_kmalloc) {
3438 /* Make sure we don't copy more data than available */
d3d9c457
BVA		 3439 if (param_offset >= buff_len)
3440 ret = -EINVAL;
3441 else
3442 memcpy(param_read_buf, &desc_buf[param_offset],
3443 min_t(u32, param_size, buff_len - param_offset));
1699f980 3444 }
da461cec
SJ		 3445out:
3446 if (is_kmalloc)
3447 kfree(desc_buf);
3448 return ret;
3449}
3450
4b828fe1
TW		 3451/**
3452 * struct uc_string_id - unicode string
3453 *
3454 * @len: size of this descriptor inclusive
3455 * @type: descriptor type
3456 * @uc: unicode string character
3457 */
3458struct uc_string_id {
3459 u8 len;
3460 u8 type;
ec38c0ad 3461 wchar_t uc[];
4b828fe1
TW		 3462} __packed;
3463
3464/* replace non-printable or non-ASCII characters with spaces */
3465static inline char ufshcd_remove_non_printable(u8 ch)
3466{
3467 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3468}
3469
b573d484
YG		 3470/**
3471 * ufshcd_read_string_desc - read string descriptor
3472 * @hba: pointer to adapter instance
3473 * @desc_index: descriptor index
4b828fe1
TW		 3474 * @buf: pointer to buffer where descriptor would be read,
3475 * the caller should free the memory.
b573d484 3476 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3477 * null terminated string.
b573d484 3478 *
4b828fe1
TW		 3479 * Return:
3480 * * string size on success.
3481 * * -ENOMEM: on allocation failure
3482 * * -EINVAL: on a wrong parameter
b573d484 3483 */
4b828fe1
TW		 3484int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3485 u8 **buf, bool ascii)
b573d484 3486{
4b828fe1
TW		 3487 struct uc_string_id *uc_str;
3488 u8 *str;
3489 int ret;
b573d484 3490
4b828fe1
TW		 3491 if (!buf)
3492 return -EINVAL;
b573d484 3493
4b828fe1
TW		 3494 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3495 if (!uc_str)
3496 return -ENOMEM;
b573d484 3497
c4607a09
BH		 3498 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3499 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
4b828fe1
TW		 3500 if (ret < 0) {
3501 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3502 QUERY_REQ_RETRIES, ret);
3503 str = NULL;
3504 goto out;
3505 }
3506
3507 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3508 dev_dbg(hba->dev, "String Desc is of zero length\n");
3509 str = NULL;
3510 ret = 0;
b573d484
YG		 3511 goto out;
3512 }
3513
3514 if (ascii) {
4b828fe1 3515 ssize_t ascii_len;
b573d484 3516 int i;
b573d484 3517 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3518 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3519 str = kzalloc(ascii_len, GFP_KERNEL);
3520 if (!str) {
3521 ret = -ENOMEM;
fcbefc3b 3522 goto out;
b573d484
YG		 3523 }
3524
3525 /*
3526 * the descriptor contains string in UTF16 format
3527 * we need to convert to utf-8 so it can be displayed
3528 */
4b828fe1
TW		 3529 ret = utf16s_to_utf8s(uc_str->uc,
3530 uc_str->len - QUERY_DESC_HDR_SIZE,
3531 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3532
3533 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3534 for (i = 0; i < ret; i++)
3535 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3536
4b828fe1
TW		 3537 str[ret++] = '\0';
3538
3539 } else {
5f57704d 3540 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3541 if (!str) {
3542 ret = -ENOMEM;
3543 goto out;
3544 }
4b828fe1 3545 ret = uc_str->len;
b573d484
YG		 3546 }
3547out:
4b828fe1
TW		 3548 *buf = str;
3549 kfree(uc_str);
3550 return ret;
b573d484 3551}
b573d484 3552
da461cec
SJ		 3553/**
3554 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3555 * @hba: Pointer to adapter instance
3556 * @lun: lun id
3557 * @param_offset: offset of the parameter to read
3558 * @param_read_buf: pointer to buffer where parameter would be read
3559 * @param_size: sizeof(param_read_buf)
3560 *
3561 * Return 0 in case of success, non-zero otherwise
3562 */
3563static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3564 int lun,
3565 enum unit_desc_param param_offset,
3566 u8 *param_read_buf,
3567 u32 param_size)
3568{
3569 /*
3570 * Unit descriptors are only available for general purpose LUs (LUN id
3571 * from 0 to 7) and RPMB Well known LU.
3572 */
a2fca52e 3573 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
da461cec
SJ		 3574 return -EOPNOTSUPP;
3575
3576 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3577 param_offset, param_read_buf, param_size);
3578}
3579
09f17791
CG		 3580static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3581{
3582 int err = 0;
3583 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3584
3585 if (hba->dev_info.wspecversion >= 0x300) {
3586 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3587 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3588 &gating_wait);
3589 if (err)
3590 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3591 err, gating_wait);
3592
3593 if (gating_wait == 0) {
3594 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3595 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3596 gating_wait);
3597 }
3598
3599 hba->dev_info.clk_gating_wait_us = gating_wait;
3600 }
3601
3602 return err;
3603}
3604
7a3e97b0
SY		 3605/**
3606 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3607 * @hba: per adapter instance
3608 *
3609 * 1. Allocate DMA memory for Command Descriptor array
3610 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3611 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3612 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3613 * (UTMRDL)
3614 * 4. Allocate memory for local reference block(lrb).
3615 *
3616 * Returns 0 for success, non-zero in case of failure
3617 */
3618static int ufshcd_memory_alloc(struct ufs_hba *hba)
3619{
3620 size_t utmrdl_size, utrdl_size, ucdl_size;
3621
3622 /* Allocate memory for UTP command descriptors */
3623 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3624 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3625 ucdl_size,
3626 &hba->ucdl_dma_addr,
3627 GFP_KERNEL);
7a3e97b0
SY		 3628
3629 /*
3630 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3631 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3632 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3633 * be aligned to 128 bytes as well
3634 */
3635 if (!hba->ucdl_base_addr ||
3636 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3637 dev_err(hba->dev,
7a3e97b0
SY		 3638 "Command Descriptor Memory allocation failed\n");
3639 goto out;
3640 }
3641
3642 /*
3643 * Allocate memory for UTP Transfer descriptors
3644 * UFSHCI requires 1024 byte alignment of UTRD
3645 */
3646 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3647 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3648 utrdl_size,
3649 &hba->utrdl_dma_addr,
3650 GFP_KERNEL);
7a3e97b0
SY		 3651 if (!hba->utrdl_base_addr ||
3652 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3653 dev_err(hba->dev,
7a3e97b0
SY		 3654 "Transfer Descriptor Memory allocation failed\n");
3655 goto out;
3656 }
3657
3658 /*
3659 * Allocate memory for UTP Task Management descriptors
3660 * UFSHCI requires 1024 byte alignment of UTMRD
3661 */
3662 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3663 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3664 utmrdl_size,
3665 &hba->utmrdl_dma_addr,
3666 GFP_KERNEL);
7a3e97b0
SY		 3667 if (!hba->utmrdl_base_addr ||
3668 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3669 dev_err(hba->dev,
7a3e97b0
SY		 3670 "Task Management Descriptor Memory allocation failed\n");
3671 goto out;
3672 }
3673
3674 /* Allocate memory for local reference block */
a86854d0
KC		 3675 hba->lrb = devm_kcalloc(hba->dev,
3676 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3677 GFP_KERNEL);
7a3e97b0 3678 if (!hba->lrb) {
3b1d0580 3679 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3680 goto out;
3681 }
3682 return 0;
3683out:
7a3e97b0
SY		 3684 return -ENOMEM;
3685}
3686
3687/**
3688 * ufshcd_host_memory_configure - configure local reference block with
3689 * memory offsets
3690 * @hba: per adapter instance
3691 *
3692 * Configure Host memory space
3693 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3694 * address.
3695 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3696 * and PRDT offset.
3697 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3698 * into local reference block.
3699 */
3700static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3701{
7a3e97b0
SY		 3702 struct utp_transfer_req_desc *utrdlp;
3703 dma_addr_t cmd_desc_dma_addr;
3704 dma_addr_t cmd_desc_element_addr;
3705 u16 response_offset;
3706 u16 prdt_offset;
3707 int cmd_desc_size;
3708 int i;
3709
3710 utrdlp = hba->utrdl_base_addr;
7a3e97b0
SY		 3711
3712 response_offset =
3713 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3714 prdt_offset =
3715 offsetof(struct utp_transfer_cmd_desc, prd_table);
3716
3717 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3718 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3719
3720 for (i = 0; i < hba->nutrs; i++) {
3721 /* Configure UTRD with command descriptor base address */
3722 cmd_desc_element_addr =
3723 (cmd_desc_dma_addr + (cmd_desc_size * i));
3724 utrdlp[i].command_desc_base_addr_lo =
3725 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3726 utrdlp[i].command_desc_base_addr_hi =
3727 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3728
3729 /* Response upiu and prdt offset should be in double words */
26f968d7
AA		 3730 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3731 utrdlp[i].response_upiu_offset =
3732 cpu_to_le16(response_offset);
3733 utrdlp[i].prd_table_offset =
3734 cpu_to_le16(prdt_offset);
3735 utrdlp[i].response_upiu_length =
3736 cpu_to_le16(ALIGNED_UPIU_SIZE);
3737 } else {
3738 utrdlp[i].response_upiu_offset =
3739 cpu_to_le16(response_offset >> 2);
3740 utrdlp[i].prd_table_offset =
3741 cpu_to_le16(prdt_offset >> 2);
3742 utrdlp[i].response_upiu_length =
3743 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3744 }
7a3e97b0 3745
4d2b8d40 3746 ufshcd_init_lrb(hba, &hba->lrb[i], i);
7a3e97b0
SY		 3747 }
3748}
3749
3750/**
3751 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3752 * @hba: per adapter instance
3753 *
3754 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3755 * in order to initialize the Unipro link startup procedure.
3756 * Once the Unipro links are up, the device connected to the controller
3757 * is detected.
3758 *
3759 * Returns 0 on success, non-zero value on failure
3760 */
3761static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3762{
6ccf44fe
SJ		 3763 struct uic_command uic_cmd = {0};
3764 int ret;
7a3e97b0 3765
6ccf44fe 3766 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3767
6ccf44fe
SJ		 3768 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3769 if (ret)
ff8e20c6 3770 dev_dbg(hba->dev,
6ccf44fe
SJ		 3771 "dme-link-startup: error code %d\n", ret);
3772 return ret;
7a3e97b0 3773}
39bf2d83
AA		 3774/**
3775 * ufshcd_dme_reset - UIC command for DME_RESET
3776 * @hba: per adapter instance
3777 *
3778 * DME_RESET command is issued in order to reset UniPro stack.
3779 * This function now deals with cold reset.
3780 *
3781 * Returns 0 on success, non-zero value on failure
3782 */
3783static int ufshcd_dme_reset(struct ufs_hba *hba)
3784{
3785 struct uic_command uic_cmd = {0};
3786 int ret;
3787
3788 uic_cmd.command = UIC_CMD_DME_RESET;
3789
3790 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3791 if (ret)
3792 dev_err(hba->dev,
3793 "dme-reset: error code %d\n", ret);
3794
3795 return ret;
3796}
3797
fc85a74e
SC		 3798int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3799 int agreed_gear,
3800 int adapt_val)
3801{
3802 int ret;
3803
3804 if (agreed_gear != UFS_HS_G4)
66df79cc 3805 adapt_val = PA_NO_ADAPT;
fc85a74e
SC		 3806
3807 ret = ufshcd_dme_set(hba,
3808 UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3809 adapt_val);
3810 return ret;
3811}
3812EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3813
39bf2d83
AA		 3814/**
3815 * ufshcd_dme_enable - UIC command for DME_ENABLE
3816 * @hba: per adapter instance
3817 *
3818 * DME_ENABLE command is issued in order to enable UniPro stack.
3819 *
3820 * Returns 0 on success, non-zero value on failure
3821 */
3822static int ufshcd_dme_enable(struct ufs_hba *hba)
3823{
3824 struct uic_command uic_cmd = {0};
3825 int ret;
3826
3827 uic_cmd.command = UIC_CMD_DME_ENABLE;
3828
3829 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3830 if (ret)
3831 dev_err(hba->dev,
1fa05700 3832 "dme-enable: error code %d\n", ret);
39bf2d83
AA		 3833
3834 return ret;
3835}
7a3e97b0 3836
cad2e03d
YG		 3837static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3838{
3839 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3840 unsigned long min_sleep_time_us;
3841
3842 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3843 return;
3844
3845 /*
3846 * last_dme_cmd_tstamp will be 0 only for 1st call to
3847 * this function
3848 */
3849 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3850 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3851 } else {
3852 unsigned long delta =
3853 (unsigned long) ktime_to_us(
3854 ktime_sub(ktime_get(),
3855 hba->last_dme_cmd_tstamp));
3856
3857 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3858 min_sleep_time_us =
3859 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3860 else
3861 return; /* no more delay required */
3862 }
3863
3864 /* allow sleep for extra 50us if needed */
3865 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3866}
3867
12b4fdb4
SJ		 3868/**
3869 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3870 * @hba: per adapter instance
3871 * @attr_sel: uic command argument1
3872 * @attr_set: attribute set type as uic command argument2
3873 * @mib_val: setting value as uic command argument3
3874 * @peer: indicate whether peer or local
3875 *
3876 * Returns 0 on success, non-zero value on failure
3877 */
3878int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3879 u8 attr_set, u32 mib_val, u8 peer)
3880{
3881 struct uic_command uic_cmd = {0};
3882 static const char *const action[] = {
3883 "dme-set",
3884 "dme-peer-set"
3885 };
3886 const char *set = action[!!peer];
3887 int ret;
64238fbd 3888 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3889
3890 uic_cmd.command = peer ?
3891 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3892 uic_cmd.argument1 = attr_sel;
3893 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3894 uic_cmd.argument3 = mib_val;
3895
64238fbd
YG		 3896 do {
3897 /* for peer attributes we retry upon failure */
3898 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3899 if (ret)
3900 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3901 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3902 } while (ret && peer && --retries);
3903
f37e9f8c 3904 if (ret)
64238fbd 3905 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3906 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3907 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3908
3909 return ret;
3910}
3911EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3912
3913/**
3914 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3915 * @hba: per adapter instance
3916 * @attr_sel: uic command argument1
3917 * @mib_val: the value of the attribute as returned by the UIC command
3918 * @peer: indicate whether peer or local
3919 *
3920 * Returns 0 on success, non-zero value on failure
3921 */
3922int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3923 u32 *mib_val, u8 peer)
3924{
3925 struct uic_command uic_cmd = {0};
3926 static const char *const action[] = {
3927 "dme-get",
3928 "dme-peer-get"
3929 };
3930 const char *get = action[!!peer];
3931 int ret;
64238fbd 3932 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3933 struct ufs_pa_layer_attr orig_pwr_info;
3934 struct ufs_pa_layer_attr temp_pwr_info;
3935 bool pwr_mode_change = false;
3936
3937 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3938 orig_pwr_info = hba->pwr_info;
3939 temp_pwr_info = orig_pwr_info;
3940
3941 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3942 orig_pwr_info.pwr_rx == FAST_MODE) {
3943 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3944 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3945 pwr_mode_change = true;
3946 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3947 orig_pwr_info.pwr_rx == SLOW_MODE) {
3948 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3949 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3950 pwr_mode_change = true;
3951 }
3952 if (pwr_mode_change) {
3953 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3954 if (ret)
3955 goto out;
3956 }
3957 }
12b4fdb4
SJ		 3958
3959 uic_cmd.command = peer ?
3960 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3961 uic_cmd.argument1 = attr_sel;
3962
64238fbd
YG		 3963 do {
3964 /* for peer attributes we retry upon failure */
3965 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3966 if (ret)
3967 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3968 get, UIC_GET_ATTR_ID(attr_sel), ret);
3969 } while (ret && peer && --retries);
3970
f37e9f8c 3971 if (ret)
64238fbd 3972 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3973 get, UIC_GET_ATTR_ID(attr_sel),
3974 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3975
64238fbd 3976 if (mib_val && !ret)
12b4fdb4 3977 *mib_val = uic_cmd.argument3;
874237f7
YG		 3978
3979 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3980 && pwr_mode_change)
3981 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3982out:
3983 return ret;
3984}
3985EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3986
53b3d9c3 3987/**
57d104c1
SJ		 3988 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3989 * state) and waits for it to take effect.
3990 *
53b3d9c3 3991 * @hba: per adapter instance
57d104c1
SJ		 3992 * @cmd: UIC command to execute
3993 *
3994 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3995 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3996 * and device UniPro link and hence it's final completion would be indicated by
3997 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3998 * addition to normal UIC command completion Status (UCCS). This function only
3999 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 4000 *
4001 * Returns 0 on success, non-zero value on failure
4002 */
57d104c1 4003static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 4004{
8a686f26 4005 DECLARE_COMPLETION_ONSTACK(uic_async_done);
53b3d9c3
SJ		 4006 unsigned long flags;
4007 u8 status;
4008 int ret;
d75f7fe4 4009 bool reenable_intr = false;
53b3d9c3 4010
53b3d9c3 4011 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d 4012 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 4013
4014 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236
CG		 4015 if (ufshcd_is_link_broken(hba)) {
4016 ret = -ENOLINK;
4017 goto out_unlock;
4018 }
57d104c1 4019 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 4020 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4021 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4022 /*
4023 * Make sure UIC command completion interrupt is disabled before
4024 * issuing UIC command.
4025 */
4026 wmb();
4027 reenable_intr = true;
57d104c1 4028 }
d75f7fe4
YG		 4029 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4030 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 4031 if (ret) {
4032 dev_err(hba->dev,
4033 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4034 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 4035 goto out;
4036 }
4037
57d104c1 4038 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 4039 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
4040 dev_err(hba->dev,
57d104c1
SJ		 4041 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4042 cmd->command, cmd->argument3);
0f52fcb9
CG		 4043
4044 if (!cmd->cmd_active) {
4045 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4046 __func__);
4047 goto check_upmcrs;
4048 }
4049
53b3d9c3
SJ		 4050 ret = -ETIMEDOUT;
4051 goto out;
4052 }
4053
0f52fcb9 4054check_upmcrs:
53b3d9c3
SJ		 4055 status = ufshcd_get_upmcrs(hba);
4056 if (status != PWR_LOCAL) {
4057 dev_err(hba->dev,
479da360 4058 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 4059 cmd->command, status);
53b3d9c3
SJ		 4060 ret = (status != PWR_OK) ? status : -1;
4061 }
4062out:
7942f7b5
VG		 4063 if (ret) {
4064 ufshcd_print_host_state(hba);
4065 ufshcd_print_pwr_info(hba);
e965e5e0 4066 ufshcd_print_evt_hist(hba);
7942f7b5
VG		 4067 }
4068
53b3d9c3 4069 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 4070 hba->active_uic_cmd = NULL;
57d104c1 4071 hba->uic_async_done = NULL;
d75f7fe4
YG		 4072 if (reenable_intr)
4073 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4db7a236
CG		 4074 if (ret) {
4075 ufshcd_set_link_broken(hba);
88b09900 4076 ufshcd_schedule_eh_work(hba);
4db7a236
CG		 4077 }
4078out_unlock:
53b3d9c3
SJ		 4079 spin_unlock_irqrestore(hba->host->host_lock, flags);
4080 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 4081
53b3d9c3
SJ		 4082 return ret;
4083}
4084
57d104c1
SJ		 4085/**
4086 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4087 * using DME_SET primitives.
4088 * @hba: per adapter instance
4089 * @mode: powr mode value
4090 *
4091 * Returns 0 on success, non-zero value on failure
4092 */
4093static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4094{
4095 struct uic_command uic_cmd = {0};
1ab27c9c 4096 int ret;
57d104c1 4097
c3a2f9ee
YG		 4098 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4099 ret = ufshcd_dme_set(hba,
4100 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4101 if (ret) {
4102 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4103 __func__, ret);
4104 goto out;
4105 }
4106 }
4107
57d104c1
SJ		 4108 uic_cmd.command = UIC_CMD_DME_SET;
4109 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4110 uic_cmd.argument3 = mode;
1ab27c9c
ST		 4111 ufshcd_hold(hba, false);
4112 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4113 ufshcd_release(hba);
57d104c1 4114
c3a2f9ee 4115out:
1ab27c9c 4116 return ret;
57d104c1
SJ		 4117}
4118
087c5efa 4119int ufshcd_link_recovery(struct ufs_hba *hba)
53c12d0e
YG		 4120{
4121 int ret;
4122 unsigned long flags;
4123
4124 spin_lock_irqsave(hba->host->host_lock, flags);
4125 hba->ufshcd_state = UFSHCD_STATE_RESET;
4126 ufshcd_set_eh_in_progress(hba);
4127 spin_unlock_irqrestore(hba->host->host_lock, flags);
4128
ebdd1dfd 4129 /* Reset the attached device */
31a5d9ca 4130 ufshcd_device_reset(hba);
ebdd1dfd 4131
53c12d0e
YG		 4132 ret = ufshcd_host_reset_and_restore(hba);
4133
4134 spin_lock_irqsave(hba->host->host_lock, flags);
4135 if (ret)
4136 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4137 ufshcd_clear_eh_in_progress(hba);
4138 spin_unlock_irqrestore(hba->host->host_lock, flags);
4139
4140 if (ret)
4141 dev_err(hba->dev, "%s: link recovery failed, err %d",
4142 __func__, ret);
4143
4144 return ret;
4145}
087c5efa 4146EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
53c12d0e 4147
525943a5 4148int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 4149{
87d0b4a6 4150 int ret;
57d104c1 4151 struct uic_command uic_cmd = {0};
911a0771 4152 ktime_t start = ktime_get();
57d104c1 4153
ee32c909
KK		 4154 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4155
57d104c1 4156 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 4157 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 4158 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4159 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 4160
4db7a236 4161 if (ret)
87d0b4a6
YG		 4162 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4163 __func__, ret);
4db7a236 4164 else
ee32c909
KK		 4165 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4166 POST_CHANGE);
53c12d0e 4167
87d0b4a6
YG		 4168 return ret;
4169}
525943a5 4170EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
87d0b4a6 4171
9d19bf7a 4172int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
57d104c1
SJ		 4173{
4174 struct uic_command uic_cmd = {0};
4175 int ret;
911a0771 4176 ktime_t start = ktime_get();
57d104c1 4177
ee32c909
KK		 4178 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4179
57d104c1
SJ		 4180 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4181 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 4182 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4183 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4184
57d104c1 4185 if (ret) {
53c12d0e
YG		 4186 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4187 __func__, ret);
ff8e20c6 4188 } else {
ee32c909
KK		 4189 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4190 POST_CHANGE);
ff8e20c6
DR		 4191 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
4192 hba->ufs_stats.hibern8_exit_cnt++;
4193 }
57d104c1
SJ		 4194
4195 return ret;
4196}
9d19bf7a 4197EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
57d104c1 4198
ba7af5ec
SC		 4199void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4200{
4201 unsigned long flags;
be7594a4 4202 bool update = false;
ba7af5ec 4203
be7594a4 4204 if (!ufshcd_is_auto_hibern8_supported(hba))
ba7af5ec
SC		 4205 return;
4206
4207 spin_lock_irqsave(hba->host->host_lock, flags);
be7594a4
CG		 4208 if (hba->ahit != ahit) {
4209 hba->ahit = ahit;
4210 update = true;
4211 }
ba7af5ec 4212 spin_unlock_irqrestore(hba->host->host_lock, flags);
be7594a4 4213
b294ff3e 4214 if (update &&
e2106584 4215 !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
b294ff3e 4216 ufshcd_rpm_get_sync(hba);
be7594a4
CG		 4217 ufshcd_hold(hba, false);
4218 ufshcd_auto_hibern8_enable(hba);
4219 ufshcd_release(hba);
b294ff3e 4220 ufshcd_rpm_put_sync(hba);
be7594a4 4221 }
ba7af5ec
SC		 4222}
4223EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4224
71d848b8 4225void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378 4226{
499f7a96 4227 if (!ufshcd_is_auto_hibern8_supported(hba))
ad448378
AH		 4228 return;
4229
ad448378 4230 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
ad448378
AH		 4231}
4232
5064636c
YG		 4233 /**
4234 * ufshcd_init_pwr_info - setting the POR (power on reset)
4235 * values in hba power info
4236 * @hba: per-adapter instance
4237 */
4238static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4239{
4240 hba->pwr_info.gear_rx = UFS_PWM_G1;
4241 hba->pwr_info.gear_tx = UFS_PWM_G1;
4242 hba->pwr_info.lane_rx = 1;
4243 hba->pwr_info.lane_tx = 1;
4244 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4245 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4246 hba->pwr_info.hs_rate = 0;
4247}
4248
d3e89bac 4249/**
7eb584db
DR		 4250 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4251 * @hba: per-adapter instance
d3e89bac 4252 */
7eb584db 4253static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 4254{
7eb584db
DR		 4255 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4256
4257 if (hba->max_pwr_info.is_valid)
4258 return 0;
4259
2349b533 4260 pwr_info->pwr_tx = FAST_MODE;
4261 pwr_info->pwr_rx = FAST_MODE;
7eb584db 4262 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 4263
4264 /* Get the connected lane count */
7eb584db
DR		 4265 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4266 &pwr_info->lane_rx);
4267 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4268 &pwr_info->lane_tx);
4269
4270 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4271 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4272 __func__,
4273 pwr_info->lane_rx,
4274 pwr_info->lane_tx);
4275 return -EINVAL;
4276 }
d3e89bac
SJ		 4277
4278 /*
4279 * First, get the maximum gears of HS speed.
4280 * If a zero value, it means there is no HSGEAR capability.
4281 * Then, get the maximum gears of PWM speed.
4282 */
7eb584db
DR		 4283 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4284 if (!pwr_info->gear_rx) {
4285 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4286 &pwr_info->gear_rx);
4287 if (!pwr_info->gear_rx) {
4288 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4289 __func__, pwr_info->gear_rx);
4290 return -EINVAL;
4291 }
2349b533 4292 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 4293 }
4294
7eb584db
DR		 4295 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4296 &pwr_info->gear_tx);
4297 if (!pwr_info->gear_tx) {
d3e89bac 4298 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 4299 &pwr_info->gear_tx);
4300 if (!pwr_info->gear_tx) {
4301 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4302 __func__, pwr_info->gear_tx);
4303 return -EINVAL;
4304 }
2349b533 4305 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 4306 }
4307
4308 hba->max_pwr_info.is_valid = true;
4309 return 0;
4310}
4311
4312static int ufshcd_change_power_mode(struct ufs_hba *hba,
4313 struct ufs_pa_layer_attr *pwr_mode)
4314{
4315 int ret;
4316
4317 /* if already configured to the requested pwr_mode */
2355b66e
CG		 4318 if (!hba->force_pmc &&
4319 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
7eb584db
DR		 4320 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4321 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4322 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4323 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4324 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4325 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4326 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4327 return 0;
d3e89bac
SJ		 4328 }
4329
4330 /*
4331 * Configure attributes for power mode change with below.
4332 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4333 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4334 * - PA_HSSERIES
4335 */
7eb584db
DR		 4336 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4337 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4338 pwr_mode->lane_rx);
4339 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4340 pwr_mode->pwr_rx == FAST_MODE)
21c2e341 4341 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
7eb584db 4342 else
21c2e341 4343 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
d3e89bac 4344
7eb584db
DR		 4345 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4346 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4347 pwr_mode->lane_tx);
4348 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4349 pwr_mode->pwr_tx == FAST_MODE)
21c2e341 4350 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
7eb584db 4351 else
21c2e341 4352 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
d3e89bac 4353
7eb584db
DR		 4354 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4355 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4356 pwr_mode->pwr_rx == FAST_MODE ||
4357 pwr_mode->pwr_tx == FAST_MODE)
4358 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4359 pwr_mode->hs_rate);
d3e89bac 4360
b1d0d2eb
KK		 4361 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4362 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4363 DL_FC0ProtectionTimeOutVal_Default);
4364 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4365 DL_TC0ReplayTimeOutVal_Default);
4366 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4367 DL_AFC0ReqTimeOutVal_Default);
4368 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4369 DL_FC1ProtectionTimeOutVal_Default);
4370 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4371 DL_TC1ReplayTimeOutVal_Default);
4372 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4373 DL_AFC1ReqTimeOutVal_Default);
4374
4375 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4376 DL_FC0ProtectionTimeOutVal_Default);
4377 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4378 DL_TC0ReplayTimeOutVal_Default);
4379 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4380 DL_AFC0ReqTimeOutVal_Default);
4381 }
08342537 4382
7eb584db
DR		 4383 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4384 | pwr_mode->pwr_tx);
4385
4386 if (ret) {
d3e89bac 4387 dev_err(hba->dev,
7eb584db
DR		 4388 "%s: power mode change failed %d\n", __func__, ret);
4389 } else {
0263bcd0
YG		 4390 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4391 pwr_mode);
7eb584db
DR		 4392
4393 memcpy(&hba->pwr_info, pwr_mode,
4394 sizeof(struct ufs_pa_layer_attr));
4395 }
4396
4397 return ret;
4398}
4399
4400/**
4401 * ufshcd_config_pwr_mode - configure a new power mode
4402 * @hba: per-adapter instance
4403 * @desired_pwr_mode: desired power configuration
4404 */
0d846e70 4405int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4406 struct ufs_pa_layer_attr *desired_pwr_mode)
4407{
4408 struct ufs_pa_layer_attr final_params = { 0 };
4409 int ret;
4410
0263bcd0
YG		 4411 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4412 desired_pwr_mode, &final_params);
4413
4414 if (ret)
7eb584db
DR		 4415 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4416
4417 ret = ufshcd_change_power_mode(hba, &final_params);
d3e89bac
SJ		 4418
4419 return ret;
4420}
0d846e70 4421EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4422
68078d5c
DR		 4423/**
4424 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4425 * @hba: per-adapter instance
68078d5c
DR		 4426 *
4427 * Set fDeviceInit flag and poll until device toggles it.
4428 */
4429static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4430{
dc3c8d3a 4431 int err;
7dfdcc39 4432 bool flag_res = true;
29707fab 4433 ktime_t timeout;
68078d5c 4434
dc3c8d3a 4435 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 4436 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
68078d5c
DR		 4437 if (err) {
4438 dev_err(hba->dev,
4439 "%s setting fDeviceInit flag failed with error %d\n",
4440 __func__, err);
4441 goto out;
4442 }
4443
29707fab
KK		 4444 /* Poll fDeviceInit flag to be cleared */
4445 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4446 do {
4447 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4448 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4449 if (!flag_res)
4450 break;
a4e6496f 4451 usleep_range(500, 1000);
29707fab 4452 } while (ktime_before(ktime_get(), timeout));
dc3c8d3a 4453
29707fab 4454 if (err) {
68078d5c 4455 dev_err(hba->dev,
29707fab
KK		 4456 "%s reading fDeviceInit flag failed with error %d\n",
4457 __func__, err);
4458 } else if (flag_res) {
68078d5c 4459 dev_err(hba->dev,
29707fab
KK		 4460 "%s fDeviceInit was not cleared by the device\n",
4461 __func__);
4462 err = -EBUSY;
4463 }
68078d5c
DR		 4464out:
4465 return err;
4466}
4467
7a3e97b0
SY		 4468/**
4469 * ufshcd_make_hba_operational - Make UFS controller operational
4470 * @hba: per adapter instance
4471 *
4472 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4473 * 1. Enable required interrupts
4474 * 2. Configure interrupt aggregation
897efe62 4475 * 3. Program UTRL and UTMRL base address
5c0c28a8 4476 * 4. Configure run-stop-registers
7a3e97b0
SY		 4477 *
4478 * Returns 0 on success, non-zero value on failure
4479 */
9d19bf7a 4480int ufshcd_make_hba_operational(struct ufs_hba *hba)
7a3e97b0
SY		 4481{
4482 int err = 0;
4483 u32 reg;
4484
6ccf44fe
SJ		 4485 /* Enable required interrupts */
4486 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4487
4488 /* Configure interrupt aggregation */
b852190e
YG		 4489 if (ufshcd_is_intr_aggr_allowed(hba))
4490 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4491 else
4492 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4493
4494 /* Configure UTRL and UTMRL base address registers */
4495 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4496 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4497 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4498 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4499 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4500 REG_UTP_TASK_REQ_LIST_BASE_L);
4501 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4502 REG_UTP_TASK_REQ_LIST_BASE_H);
4503
897efe62
YG		 4504 /*
4505 * Make sure base address and interrupt setup are updated before
4506 * enabling the run/stop registers below.
4507 */
4508 wmb();
4509
7a3e97b0
SY		 4510 /*
4511 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4512 */
5c0c28a8 4513 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4514 if (!(ufshcd_get_lists_status(reg))) {
4515 ufshcd_enable_run_stop_reg(hba);
4516 } else {
3b1d0580 4517 dev_err(hba->dev,
7a3e97b0
SY		 4518 "Host controller not ready to process requests");
4519 err = -EIO;
7a3e97b0
SY		 4520 }
4521
7a3e97b0
SY		 4522 return err;
4523}
9d19bf7a 4524EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
7a3e97b0 4525
596585a2
YG		 4526/**
4527 * ufshcd_hba_stop - Send controller to reset state
4528 * @hba: per adapter instance
596585a2 4529 */
3a95f5b3 4530void ufshcd_hba_stop(struct ufs_hba *hba)
596585a2 4531{
5cac1095 4532 unsigned long flags;
596585a2
YG		 4533 int err;
4534
5cac1095
BVA		 4535 /*
4536 * Obtain the host lock to prevent that the controller is disabled
4537 * while the UFS interrupt handler is active on another CPU.
4538 */
4539 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 4540 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
5cac1095
BVA		 4541 spin_unlock_irqrestore(hba->host->host_lock, flags);
4542
596585a2
YG		 4543 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4544 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
5cac1095 4545 10, 1);
596585a2
YG		 4546 if (err)
4547 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4548}
3a95f5b3 4549EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
596585a2 4550
7a3e97b0 4551/**
39bf2d83 4552 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4553 * @hba: per adapter instance
4554 *
4555 * The controller resets itself and controller firmware initialization
4556 * sequence kicks off. When controller is ready it will set
4557 * the Host Controller Enable bit to 1.
4558 *
4559 * Returns 0 on success, non-zero value on failure
4560 */
39bf2d83 4561static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0 4562{
6081b12c
SC		 4563 int retry_outer = 3;
4564 int retry_inner;
7a3e97b0 4565
6081b12c 4566start:
acbbfe48 4567 if (ufshcd_is_hba_active(hba))
7a3e97b0 4568 /* change controller state to "reset state" */
5cac1095 4569 ufshcd_hba_stop(hba);
7a3e97b0 4570
57d104c1
SJ		 4571 /* UniPro link is disabled at this point */
4572 ufshcd_set_link_off(hba);
4573
0263bcd0 4574 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4575
7a3e97b0
SY		 4576 /* start controller initialization sequence */
4577 ufshcd_hba_start(hba);
4578
4579 /*
4580 * To initialize a UFS host controller HCE bit must be set to 1.
4581 * During initialization the HCE bit value changes from 1->0->1.
4582 * When the host controller completes initialization sequence
4583 * it sets the value of HCE bit to 1. The same HCE bit is read back
4584 * to check if the controller has completed initialization sequence.
4585 * So without this delay the value HCE = 1, set in the previous
4586 * instruction might be read back.
4587 * This delay can be changed based on the controller.
4588 */
90b8491c 4589 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
7a3e97b0
SY		 4590
4591 /* wait for the host controller to complete initialization */
6081b12c 4592 retry_inner = 50;
acbbfe48 4593 while (!ufshcd_is_hba_active(hba)) {
6081b12c
SC		 4594 if (retry_inner) {
4595 retry_inner--;
7a3e97b0 4596 } else {
3b1d0580 4597 dev_err(hba->dev,
7a3e97b0 4598 "Controller enable failed\n");
6081b12c
SC		 4599 if (retry_outer) {
4600 retry_outer--;
4601 goto start;
4602 }
7a3e97b0
SY		 4603 return -EIO;
4604 }
9fc305ef 4605 usleep_range(1000, 1100);
7a3e97b0 4606 }
5c0c28a8 4607
1d337ec2 4608 /* enable UIC related interrupts */
57d104c1 4609 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4610
0263bcd0 4611 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4612
7a3e97b0
SY		 4613 return 0;
4614}
39bf2d83
AA		 4615
4616int ufshcd_hba_enable(struct ufs_hba *hba)
4617{
4618 int ret;
4619
4620 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4621 ufshcd_set_link_off(hba);
4622 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4623
4624 /* enable UIC related interrupts */
4625 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4626 ret = ufshcd_dme_reset(hba);
4627 if (!ret) {
4628 ret = ufshcd_dme_enable(hba);
4629 if (!ret)
4630 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4631 if (ret)
4632 dev_err(hba->dev,
4633 "Host controller enable failed with non-hce\n");
4634 }
4635 } else {
4636 ret = ufshcd_hba_execute_hce(hba);
4637 }
4638
4639 return ret;
4640}
9d19bf7a
SC		 4641EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4642
7ca38cf3
YG		 4643static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4644{
ba0320fb 4645 int tx_lanes = 0, i, err = 0;
7ca38cf3
YG		 4646
4647 if (!peer)
4648 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4649 &tx_lanes);
4650 else
4651 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4652 &tx_lanes);
4653 for (i = 0; i < tx_lanes; i++) {
4654 if (!peer)
4655 err = ufshcd_dme_set(hba,
4656 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4657 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4658 0);
4659 else
4660 err = ufshcd_dme_peer_set(hba,
4661 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4662 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4663 0);
4664 if (err) {
4665 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4666 __func__, peer, i, err);
4667 break;
4668 }
4669 }
4670
4671 return err;
4672}
4673
4674static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4675{
4676 return ufshcd_disable_tx_lcc(hba, true);
4677}
4678
e965e5e0 4679void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
8808b4e9 4680{
e965e5e0
SC		 4681 struct ufs_event_hist *e;
4682
4683 if (id >= UFS_EVT_CNT)
4684 return;
4685
4686 e = &hba->ufs_stats.event[id];
4687 e->val[e->pos] = val;
4688 e->tstamp[e->pos] = ktime_get();
b6cacaf2 4689 e->cnt += 1;
e965e5e0 4690 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
172614a9
SC		 4691
4692 ufshcd_vops_event_notify(hba, id, &val);
8808b4e9 4693}
e965e5e0 4694EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
8808b4e9 4695
7a3e97b0 4696/**
6ccf44fe 4697 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4698 * @hba: per adapter instance
4699 *
6ccf44fe 4700 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4701 */
6ccf44fe 4702static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4703{
6ccf44fe 4704 int ret;
1d337ec2 4705 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4706 bool link_startup_again = false;
7a3e97b0 4707
7caf489b 4708 /*
4709 * If UFS device isn't active then we will have to issue link startup
4710 * 2 times to make sure the device state move to active.
4711 */
4712 if (!ufshcd_is_ufs_dev_active(hba))
4713 link_startup_again = true;
7a3e97b0 4714
7caf489b 4715link_startup:
1d337ec2 4716 do {
0263bcd0 4717 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4718
1d337ec2 4719 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4720
1d337ec2
SRT		 4721 /* check if device is detected by inter-connect layer */
4722 if (!ret && !ufshcd_is_device_present(hba)) {
e965e5e0
SC		 4723 ufshcd_update_evt_hist(hba,
4724 UFS_EVT_LINK_STARTUP_FAIL,
8808b4e9 4725 0);
1d337ec2
SRT		 4726 dev_err(hba->dev, "%s: Device not present\n", __func__);
4727 ret = -ENXIO;
4728 goto out;
4729 }
6ccf44fe 4730
1d337ec2
SRT		 4731 /*
4732 * DME link lost indication is only received when link is up,
4733 * but we can't be sure if the link is up until link startup
4734 * succeeds. So reset the local Uni-Pro and try again.
4735 */
8808b4e9 4736 if (ret && ufshcd_hba_enable(hba)) {
e965e5e0
SC		 4737 ufshcd_update_evt_hist(hba,
4738 UFS_EVT_LINK_STARTUP_FAIL,
8808b4e9 4739 (u32)ret);
1d337ec2 4740 goto out;
8808b4e9 4741 }
1d337ec2
SRT		 4742 } while (ret && retries--);
4743
8808b4e9 4744 if (ret) {
1d337ec2 4745 /* failed to get the link up... retire */
e965e5e0
SC		 4746 ufshcd_update_evt_hist(hba,
4747 UFS_EVT_LINK_STARTUP_FAIL,
8808b4e9 4748 (u32)ret);
5c0c28a8 4749 goto out;
8808b4e9 4750 }
5c0c28a8 4751
7caf489b 4752 if (link_startup_again) {
4753 link_startup_again = false;
4754 retries = DME_LINKSTARTUP_RETRIES;
4755 goto link_startup;
4756 }
4757
d2aebb9b 4758 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4759 ufshcd_init_pwr_info(hba);
4760 ufshcd_print_pwr_info(hba);
4761
7ca38cf3
YG		 4762 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4763 ret = ufshcd_disable_device_tx_lcc(hba);
4764 if (ret)
4765 goto out;
4766 }
4767
5c0c28a8 4768 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4769 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4770 if (ret)
4771 goto out;
7a3e97b0 4772
2355b66e
CG		 4773 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4774 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5c0c28a8 4775 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4776out:
7942f7b5 4777 if (ret) {
6ccf44fe 4778 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4779 ufshcd_print_host_state(hba);
4780 ufshcd_print_pwr_info(hba);
e965e5e0 4781 ufshcd_print_evt_hist(hba);
7942f7b5 4782 }
6ccf44fe 4783 return ret;
7a3e97b0
SY		 4784}
4785
5a0b0cb9
SRT		 4786/**
4787 * ufshcd_verify_dev_init() - Verify device initialization
4788 * @hba: per-adapter instance
4789 *
4790 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4791 * device Transport Protocol (UTP) layer is ready after a reset.
4792 * If the UTP layer at the device side is not initialized, it may
4793 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4794 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4795 */
4796static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4797{
4798 int err = 0;
4799 int retries;
4800
1ab27c9c 4801 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4802 mutex_lock(&hba->dev_cmd.lock);
4803 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4804 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
1cbc9ad3 4805 hba->nop_out_timeout);
5a0b0cb9
SRT		 4806
4807 if (!err || err == -ETIMEDOUT)
4808 break;
4809
4810 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4811 }
4812 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4813 ufshcd_release(hba);
5a0b0cb9
SRT		 4814
4815 if (err)
4816 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4817 return err;
4818}
4819
0ce147d4
SJ		 4820/**
4821 * ufshcd_set_queue_depth - set lun queue depth
4822 * @sdev: pointer to SCSI device
4823 *
4824 * Read bLUQueueDepth value and activate scsi tagged command
4825 * queueing. For WLUN, queue depth is set to 1. For best-effort
4826 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4827 * value that host can queue.
4828 */
4829static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4830{
4831 int ret = 0;
4832 u8 lun_qdepth;
4833 struct ufs_hba *hba;
4834
4835 hba = shost_priv(sdev->host);
4836
4837 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4838 ret = ufshcd_read_unit_desc_param(hba,
4839 ufshcd_scsi_to_upiu_lun(sdev->lun),
4840 UNIT_DESC_PARAM_LU_Q_DEPTH,
4841 &lun_qdepth,
4842 sizeof(lun_qdepth));
0ce147d4
SJ		 4843
4844 /* Some WLUN doesn't support unit descriptor */
4845 if (ret == -EOPNOTSUPP)
4846 lun_qdepth = 1;
4847 else if (!lun_qdepth)
4848 /* eventually, we can figure out the real queue depth */
4849 lun_qdepth = hba->nutrs;
4850 else
4851 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4852
4853 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4854 __func__, lun_qdepth);
db5ed4df 4855 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4856}
4857
57d104c1
SJ		 4858/*
4859 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4860 * @hba: per-adapter instance
4861 * @lun: UFS device lun id
4862 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4863 *
4864 * Returns 0 in case of success and b_lu_write_protect status would be returned
4865 * @b_lu_write_protect parameter.
4866 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4867 * Returns -EINVAL in case of invalid parameters passed to this function.
4868 */
4869static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4870 u8 lun,
4871 u8 *b_lu_write_protect)
4872{
4873 int ret;
4874
4875 if (!b_lu_write_protect)
4876 ret = -EINVAL;
4877 /*
4878 * According to UFS device spec, RPMB LU can't be write
4879 * protected so skip reading bLUWriteProtect parameter for
4880 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4881 */
1baa8011 4882 else if (lun >= hba->dev_info.max_lu_supported)
57d104c1
SJ		 4883 ret = -ENOTSUPP;
4884 else
4885 ret = ufshcd_read_unit_desc_param(hba,
4886 lun,
4887 UNIT_DESC_PARAM_LU_WR_PROTECT,
4888 b_lu_write_protect,
4889 sizeof(*b_lu_write_protect));
4890 return ret;
4891}
4892
4893/**
4894 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4895 * status
4896 * @hba: per-adapter instance
4897 * @sdev: pointer to SCSI device
4898 *
4899 */
4900static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4901 struct scsi_device *sdev)
4902{
4903 if (hba->dev_info.f_power_on_wp_en &&
4904 !hba->dev_info.is_lu_power_on_wp) {
4905 u8 b_lu_write_protect;
4906
4907 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4908 &b_lu_write_protect) &&
4909 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4910 hba->dev_info.is_lu_power_on_wp = true;
4911 }
4912}
4913
b294ff3e
AD		 4914/**
4915 * ufshcd_setup_links - associate link b/w device wlun and other luns
4916 * @sdev: pointer to SCSI device
4917 * @hba: pointer to ufs hba
4918 */
4919static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
4920{
4921 struct device_link *link;
4922
4923 /*
4924 * Device wlun is the supplier & rest of the luns are consumers.
4925 * This ensures that device wlun suspends after all other luns.
4926 */
e2106584 4927 if (hba->ufs_device_wlun) {
b294ff3e 4928 link = device_link_add(&sdev->sdev_gendev,
e2106584 4929 &hba->ufs_device_wlun->sdev_gendev,
b294ff3e
AD		 4930 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
4931 if (!link) {
4932 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
e2106584 4933 dev_name(&hba->ufs_device_wlun->sdev_gendev));
b294ff3e
AD		 4934 return;
4935 }
4936 hba->luns_avail--;
4937 /* Ignore REPORT_LUN wlun probing */
4938 if (hba->luns_avail == 1) {
4939 ufshcd_rpm_put(hba);
4940 return;
4941 }
4942 } else {
4943 /*
4944 * Device wlun is probed. The assumption is that WLUNs are
4945 * scanned before other LUNs.
4946 */
4947 hba->luns_avail--;
4948 }
4949}
4950
7a3e97b0
SY		 4951/**
4952 * ufshcd_slave_alloc - handle initial SCSI device configurations
4953 * @sdev: pointer to SCSI device
4954 *
4955 * Returns success
4956 */
4957static int ufshcd_slave_alloc(struct scsi_device *sdev)
4958{
4959 struct ufs_hba *hba;
4960
4961 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4962
4963 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4964 sdev->use_10_for_ms = 1;
a3a76391
CG		 4965
4966 /* DBD field should be set to 1 in mode sense(10) */
4967 sdev->set_dbd_for_ms = 1;
7a3e97b0 4968
e8e7f271
SRT		 4969 /* allow SCSI layer to restart the device in case of errors */
4970 sdev->allow_restart = 1;
4264fd61 4971
b2a6c522
SRT		 4972 /* REPORT SUPPORTED OPERATION CODES is not supported */
4973 sdev->no_report_opcodes = 1;
4974
84af7e8b
SRT		 4975 /* WRITE_SAME command is not supported */
4976 sdev->no_write_same = 1;
e8e7f271 4977
0ce147d4 4978 ufshcd_set_queue_depth(sdev);
4264fd61 4979
57d104c1
SJ		 4980 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4981
b294ff3e
AD		 4982 ufshcd_setup_links(hba, sdev);
4983
7a3e97b0
SY		 4984 return 0;
4985}
4986
4264fd61
SRT		 4987/**
4988 * ufshcd_change_queue_depth - change queue depth
4989 * @sdev: pointer to SCSI device
4990 * @depth: required depth to set
4264fd61 4991 *
db5ed4df 4992 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4993 */
db5ed4df 4994static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61 4995{
fc21da8a 4996 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
4264fd61
SRT		 4997}
4998
f02bc975
DP		 4999static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev)
5000{
5001 /* skip well-known LU */
41d8a933
DP		 5002 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
5003 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
f02bc975
DP		 5004 return;
5005
5006 ufshpb_destroy_lu(hba, sdev);
5007}
5008
5009static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev)
5010{
5011 /* skip well-known LU */
5012 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
5013 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
5014 return;
5015
5016 ufshpb_init_hpb_lu(hba, sdev);
5017}
5018
eeda4749
AM		 5019/**
5020 * ufshcd_slave_configure - adjust SCSI device configurations
5021 * @sdev: pointer to SCSI device
5022 */
5023static int ufshcd_slave_configure(struct scsi_device *sdev)
5024{
49615ba1 5025 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 5026 struct request_queue *q = sdev->request_queue;
5027
f02bc975
DP		 5028 ufshcd_hpb_configure(hba, sdev);
5029
eeda4749 5030 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
2b2bfc8a
KK		 5031 if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
5032 blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
b294ff3e
AD		 5033 /*
5034 * Block runtime-pm until all consumers are added.
5035 * Refer ufshcd_setup_links().
5036 */
5037 if (is_device_wlun(sdev))
5038 pm_runtime_get_noresume(&sdev->sdev_gendev);
5039 else if (ufshcd_is_rpm_autosuspend_allowed(hba))
49615ba1 5040 sdev->rpm_autosuspend = 1;
71bb9ab6
AH		 5041 /*
5042 * Do not print messages during runtime PM to avoid never-ending cycles
5043 * of messages written back to storage by user space causing runtime
5044 * resume, causing more messages and so on.
5045 */
5046 sdev->silence_suspend = 1;
49615ba1 5047
cb77cb5a 5048 ufshcd_crypto_register(hba, q);
df043c74 5049
eeda4749
AM		 5050 return 0;
5051}
5052
7a3e97b0
SY		 5053/**
5054 * ufshcd_slave_destroy - remove SCSI device configurations
5055 * @sdev: pointer to SCSI device
5056 */
5057static void ufshcd_slave_destroy(struct scsi_device *sdev)
5058{
5059 struct ufs_hba *hba;
bf25967a 5060 unsigned long flags;
7a3e97b0
SY		 5061
5062 hba = shost_priv(sdev->host);
f02bc975
DP		 5063
5064 ufshcd_hpb_destroy(hba, sdev);
5065
0ce147d4 5066 /* Drop the reference as it won't be needed anymore */
7c48bfd0 5067 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
7c48bfd0 5068 spin_lock_irqsave(hba->host->host_lock, flags);
e2106584 5069 hba->ufs_device_wlun = NULL;
7c48bfd0 5070 spin_unlock_irqrestore(hba->host->host_lock, flags);
e2106584 5071 } else if (hba->ufs_device_wlun) {
bf25967a
AH		 5072 struct device *supplier = NULL;
5073
5074 /* Ensure UFS Device WLUN exists and does not disappear */
5075 spin_lock_irqsave(hba->host->host_lock, flags);
e2106584
BVA		 5076 if (hba->ufs_device_wlun) {
5077 supplier = &hba->ufs_device_wlun->sdev_gendev;
bf25967a
AH		 5078 get_device(supplier);
5079 }
5080 spin_unlock_irqrestore(hba->host->host_lock, flags);
5081
5082 if (supplier) {
5083 /*
5084 * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5085 * device will not have been registered but can still
5086 * have a device link holding a reference to the device.
5087 */
5088 device_link_remove(&sdev->sdev_gendev, supplier);
5089 put_device(supplier);
5090 }
7c48bfd0 5091 }
7a3e97b0
SY		 5092}
5093
7a3e97b0
SY		 5094/**
5095 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 5096 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 5097 * @scsi_status: SCSI command status
5098 *
5099 * Returns value base on SCSI command status
5100 */
5101static inline int
5102ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5103{
5104 int result = 0;
5105
5106 switch (scsi_status) {
7a3e97b0 5107 case SAM_STAT_CHECK_CONDITION:
1c2623c5 5108 ufshcd_copy_sense_data(lrbp);
df561f66 5109 fallthrough;
1c2623c5 5110 case SAM_STAT_GOOD:
db83d8a5 5111 result |= DID_OK << 16 | scsi_status;
7a3e97b0
SY		 5112 break;
5113 case SAM_STAT_TASK_SET_FULL:
1c2623c5 5114 case SAM_STAT_BUSY:
7a3e97b0 5115 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 5116 ufshcd_copy_sense_data(lrbp);
5117 result |= scsi_status;
7a3e97b0
SY		 5118 break;
5119 default:
5120 result |= DID_ERROR << 16;
5121 break;
5122 } /* end of switch */
5123
5124 return result;
5125}
5126
5127/**
5128 * ufshcd_transfer_rsp_status - Get overall status of the response
5129 * @hba: per adapter instance
8aa29f19 5130 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 5131 *
5132 * Returns result of the command to notify SCSI midlayer
5133 */
5134static inline int
5135ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
5136{
5137 int result = 0;
5138 int scsi_status;
957d63e7 5139 enum utp_ocs ocs;
7a3e97b0
SY		 5140
5141 /* overall command status of utrd */
5142 ocs = ufshcd_get_tr_ocs(lrbp);
5143
d779a6e9
KK		 5144 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5145 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
5146 MASK_RSP_UPIU_RESULT)
5147 ocs = OCS_SUCCESS;
5148 }
5149
7a3e97b0
SY		 5150 switch (ocs) {
5151 case OCS_SUCCESS:
5a0b0cb9 5152 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 5153 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 5154 switch (result) {
5155 case UPIU_TRANSACTION_RESPONSE:
5156 /*
5157 * get the response UPIU result to extract
5158 * the SCSI command status
5159 */
5160 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
5161
5162 /*
5163 * get the result based on SCSI status response
5164 * to notify the SCSI midlayer of the command status
5165 */
5166 scsi_status = result & MASK_SCSI_STATUS;
5167 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 5168
f05ac2e5
YG		 5169 /*
5170 * Currently we are only supporting BKOPs exception
5171 * events hence we can ignore BKOPs exception event
5172 * during power management callbacks. BKOPs exception
5173 * event is not expected to be raised in runtime suspend
5174 * callback as it allows the urgent bkops.
5175 * During system suspend, we are anyway forcefully
5176 * disabling the bkops and if urgent bkops is needed
5177 * it will be enabled on system resume. Long term
5178 * solution could be to abort the system suspend if
5179 * UFS device needs urgent BKOPs.
5180 */
5181 if (!hba->pm_op_in_progress &&
aa53f580 5182 !ufshcd_eh_in_progress(hba) &&
b294ff3e
AD		 5183 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5184 /* Flushed in suspend */
5185 schedule_work(&hba->eeh_work);
4b5f4907
DP		 5186
5187 if (scsi_status == SAM_STAT_GOOD)
5188 ufshpb_rsp_upiu(hba, lrbp);
5a0b0cb9
SRT		 5189 break;
5190 case UPIU_TRANSACTION_REJECT_UPIU:
5191 /* TODO: handle Reject UPIU Response */
5192 result = DID_ERROR << 16;
3b1d0580 5193 dev_err(hba->dev,
5a0b0cb9
SRT		 5194 "Reject UPIU not fully implemented\n");
5195 break;
5196 default:
5a0b0cb9
SRT		 5197 dev_err(hba->dev,
5198 "Unexpected request response code = %x\n",
5199 result);
e0347d89 5200 result = DID_ERROR << 16;
7a3e97b0
SY		 5201 break;
5202 }
7a3e97b0
SY		 5203 break;
5204 case OCS_ABORTED:
5205 result |= DID_ABORT << 16;
5206 break;
e8e7f271
SRT		 5207 case OCS_INVALID_COMMAND_STATUS:
5208 result |= DID_REQUEUE << 16;
5209 break;
7a3e97b0
SY		 5210 case OCS_INVALID_CMD_TABLE_ATTR:
5211 case OCS_INVALID_PRDT_ATTR:
5212 case OCS_MISMATCH_DATA_BUF_SIZE:
5213 case OCS_MISMATCH_RESP_UPIU_SIZE:
5214 case OCS_PEER_COMM_FAILURE:
5215 case OCS_FATAL_ERROR:
5e7341e1
ST		 5216 case OCS_DEVICE_FATAL_ERROR:
5217 case OCS_INVALID_CRYPTO_CONFIG:
5218 case OCS_GENERAL_CRYPTO_ERROR:
7a3e97b0
SY		 5219 default:
5220 result |= DID_ERROR << 16;
3b1d0580 5221 dev_err(hba->dev,
ff8e20c6
DR		 5222 "OCS error from controller = %x for tag %d\n",
5223 ocs, lrbp->task_tag);
e965e5e0 5224 ufshcd_print_evt_hist(hba);
6ba65588 5225 ufshcd_print_host_state(hba);
7a3e97b0
SY		 5226 break;
5227 } /* end of switch */
5228
eeb1b55b
JK		 5229 if ((host_byte(result) != DID_OK) &&
5230 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
66cc820f 5231 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 5232 return result;
5233}
5234
a45f9371
CG		 5235static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5236 u32 intr_mask)
5237{
5238 if (!ufshcd_is_auto_hibern8_supported(hba) ||
5239 !ufshcd_is_auto_hibern8_enabled(hba))
5240 return false;
5241
5242 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5243 return false;
5244
5245 if (hba->active_uic_cmd &&
5246 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5247 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5248 return false;
5249
5250 return true;
5251}
5252
6ccf44fe
SJ		 5253/**
5254 * ufshcd_uic_cmd_compl - handle completion of uic command
5255 * @hba: per adapter instance
53b3d9c3 5256 * @intr_status: interrupt status generated by the controller
9333d775
VG		 5257 *
5258 * Returns
5259 * IRQ_HANDLED - If interrupt is valid
5260 * IRQ_NONE - If invalid interrupt
6ccf44fe 5261 */
9333d775 5262static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 5263{
9333d775
VG		 5264 irqreturn_t retval = IRQ_NONE;
5265
a45f9371
CG		 5266 spin_lock(hba->host->host_lock);
5267 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5268 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5269
53b3d9c3 5270 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 5271 hba->active_uic_cmd->argument2 |=
5272 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 5273 hba->active_uic_cmd->argument3 =
5274 ufshcd_get_dme_attr_val(hba);
0f52fcb9
CG		 5275 if (!hba->uic_async_done)
5276 hba->active_uic_cmd->cmd_active = 0;
6ccf44fe 5277 complete(&hba->active_uic_cmd->done);
9333d775 5278 retval = IRQ_HANDLED;
6ccf44fe 5279 }
53b3d9c3 5280
9333d775 5281 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
0f52fcb9 5282 hba->active_uic_cmd->cmd_active = 0;
57d104c1 5283 complete(hba->uic_async_done);
9333d775
VG		 5284 retval = IRQ_HANDLED;
5285 }
aa5c6979
SC		 5286
5287 if (retval == IRQ_HANDLED)
5288 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
28fa68fc 5289 UFS_CMD_COMP);
a45f9371 5290 spin_unlock(hba->host->host_lock);
9333d775 5291 return retval;
6ccf44fe
SJ		 5292}
5293
6f8dafde
BVA		 5294/* Release the resources allocated for processing a SCSI command. */
5295static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5296 struct ufshcd_lrb *lrbp)
5297{
5298 struct scsi_cmnd *cmd = lrbp->cmd;
5299
5300 scsi_dma_unmap(cmd);
5301 lrbp->cmd = NULL; /* Mark the command as completed. */
5302 ufshcd_release(hba);
5303 ufshcd_clk_scaling_update_busy(hba);
5304}
5305
7a3e97b0 5306/**
9a47ec7c 5307 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 5308 * @hba: per adapter instance
73dc3c4a 5309 * @completed_reqs: bitmask that indicates which requests to complete
7a3e97b0 5310 */
9a47ec7c 5311static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
11682523 5312 unsigned long completed_reqs)
7a3e97b0 5313{
5a0b0cb9
SRT		 5314 struct ufshcd_lrb *lrbp;
5315 struct scsi_cmnd *cmd;
7a3e97b0 5316 int index;
e9d501b1 5317
e9d501b1
DR		 5318 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5319 lrbp = &hba->lrb[index];
a3170376 5320 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 5321 cmd = lrbp->cmd;
5322 if (cmd) {
1d8613a2
CG		 5323 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5324 ufshcd_update_monitor(hba, lrbp);
28fa68fc 5325 ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
6f8dafde
BVA		 5326 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp);
5327 ufshcd_release_scsi_cmd(hba, lrbp);
e9d501b1 5328 /* Do not touch lrbp after scsi done */
35c3730a 5329 scsi_done(cmd);
300bb13f
JP		 5330 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5331 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
1a07f2d9
LS		 5332 if (hba->dev_cmd.complete) {
5333 ufshcd_add_command_trace(hba, index,
28fa68fc 5334 UFS_DEV_COMP);
e9d501b1 5335 complete(hba->dev_cmd.complete);
3eb9dcc0 5336 ufshcd_clk_scaling_update_busy(hba);
1a07f2d9 5337 }
e9d501b1
DR		 5338 }
5339 }
7a3e97b0
SY		 5340}
5341
eaab9b57
BVA		 5342/*
5343 * Returns > 0 if one or more commands have been completed or 0 if no
5344 * requests have been completed.
5345 */
5346static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5347{
5348 struct ufs_hba *hba = shost_priv(shost);
5349 unsigned long completed_reqs, flags;
5350 u32 tr_doorbell;
5351
5352 spin_lock_irqsave(&hba->outstanding_lock, flags);
5353 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5354 completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5355 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5356 "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5357 hba->outstanding_reqs);
5358 hba->outstanding_reqs &= ~completed_reqs;
5359 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5360
5361 if (completed_reqs)
5362 __ufshcd_transfer_req_compl(hba, completed_reqs);
5363
5364 return completed_reqs;
5365}
5366
9a47ec7c 5367/**
1f522c50 5368 * ufshcd_transfer_req_compl - handle SCSI and query command completion
9a47ec7c 5369 * @hba: per adapter instance
9333d775
VG		 5370 *
5371 * Returns
5372 * IRQ_HANDLED - If interrupt is valid
5373 * IRQ_NONE - If invalid interrupt
9a47ec7c 5374 */
11682523 5375static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c 5376{
9a47ec7c
YG		 5377 /* Resetting interrupt aggregation counters first and reading the
5378 * DOOR_BELL afterward allows us to handle all the completed requests.
5379 * In order to prevent other interrupts starvation the DB is read once
5380 * after reset. The down side of this solution is the possibility of
5381 * false interrupt if device completes another request after resetting
5382 * aggregation and before reading the DB.
5383 */
b638b5eb
AA		 5384 if (ufshcd_is_intr_aggr_allowed(hba) &&
5385 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 5386 ufshcd_reset_intr_aggr(hba);
5387
c11a1ae9
BVA		 5388 if (ufs_fail_completion())
5389 return IRQ_HANDLED;
5390
eaab9b57
BVA		 5391 /*
5392 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5393 * do not want polling to trigger spurious interrupt complaints.
5394 */
5395 ufshcd_poll(hba->host, 0);
9a47ec7c 5396
eaab9b57 5397 return IRQ_HANDLED;
9a47ec7c
YG		 5398}
5399
7deedfda 5400int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
cd469475
AH		 5401{
5402 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5403 QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5404 &ee_ctrl_mask);
5405}
5406
7deedfda 5407int ufshcd_write_ee_control(struct ufs_hba *hba)
cd469475
AH		 5408{
5409 int err;
5410
5411 mutex_lock(&hba->ee_ctrl_mutex);
5412 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5413 mutex_unlock(&hba->ee_ctrl_mutex);
5414 if (err)
5415 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5416 __func__, err);
5417 return err;
5418}
5419
5420int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, u16 *other_mask,
5421 u16 set, u16 clr)
5422{
5423 u16 new_mask, ee_ctrl_mask;
5424 int err = 0;
5425
5426 mutex_lock(&hba->ee_ctrl_mutex);
5427 new_mask = (*mask & ~clr) | set;
5428 ee_ctrl_mask = new_mask | *other_mask;
5429 if (ee_ctrl_mask != hba->ee_ctrl_mask)
5430 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5431 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5432 if (!err) {
5433 hba->ee_ctrl_mask = ee_ctrl_mask;
5434 *mask = new_mask;
5435 }
5436 mutex_unlock(&hba->ee_ctrl_mutex);
5437 return err;
5438}
5439
66ec6d59
SRT		 5440/**
5441 * ufshcd_disable_ee - disable exception event
5442 * @hba: per-adapter instance
5443 * @mask: exception event to disable
5444 *
5445 * Disables exception event in the device so that the EVENT_ALERT
5446 * bit is not set.
5447 *
5448 * Returns zero on success, non-zero error value on failure.
5449 */
cd469475 5450static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
66ec6d59 5451{
cd469475 5452 return ufshcd_update_ee_drv_mask(hba, 0, mask);
66ec6d59
SRT		 5453}
5454
5455/**
5456 * ufshcd_enable_ee - enable exception event
5457 * @hba: per-adapter instance
5458 * @mask: exception event to enable
5459 *
5460 * Enable corresponding exception event in the device to allow
5461 * device to alert host in critical scenarios.
5462 *
5463 * Returns zero on success, non-zero error value on failure.
5464 */
cd469475 5465static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
66ec6d59 5466{
cd469475 5467 return ufshcd_update_ee_drv_mask(hba, mask, 0);
66ec6d59
SRT		 5468}
5469
5470/**
5471 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5472 * @hba: per-adapter instance
5473 *
5474 * Allow device to manage background operations on its own. Enabling
5475 * this might lead to inconsistent latencies during normal data transfers
5476 * as the device is allowed to manage its own way of handling background
5477 * operations.
5478 *
5479 * Returns zero on success, non-zero on failure.
5480 */
5481static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5482{
5483 int err = 0;
5484
5485 if (hba->auto_bkops_enabled)
5486 goto out;
5487
dc3c8d3a 5488 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 5489 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5490 if (err) {
5491 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5492 __func__, err);
5493 goto out;
5494 }
5495
5496 hba->auto_bkops_enabled = true;
7ff5ab47 5497 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 5498
5499 /* No need of URGENT_BKOPS exception from the device */
5500 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5501 if (err)
5502 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5503 __func__, err);
5504out:
5505 return err;
5506}
5507
5508/**
5509 * ufshcd_disable_auto_bkops - block device in doing background operations
5510 * @hba: per-adapter instance
5511 *
5512 * Disabling background operations improves command response latency but
5513 * has drawback of device moving into critical state where the device is
5514 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5515 * host is idle so that BKOPS are managed effectively without any negative
5516 * impacts.
5517 *
5518 * Returns zero on success, non-zero on failure.
5519 */
5520static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5521{
5522 int err = 0;
5523
5524 if (!hba->auto_bkops_enabled)
5525 goto out;
5526
5527 /*
5528 * If host assisted BKOPs is to be enabled, make sure
5529 * urgent bkops exception is allowed.
5530 */
5531 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5532 if (err) {
5533 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5534 __func__, err);
5535 goto out;
5536 }
5537
dc3c8d3a 5538 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
1f34eedf 5539 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5540 if (err) {
5541 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5542 __func__, err);
5543 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5544 goto out;
5545 }
5546
5547 hba->auto_bkops_enabled = false;
7ff5ab47 5548 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 5549 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5550out:
5551 return err;
5552}
5553
5554/**
4e768e76 5555 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 5556 * @hba: per adapter instance
5557 *
5558 * After a device reset the device may toggle the BKOPS_EN flag
5559 * to default value. The s/w tracking variables should be updated
4e768e76 5560 * as well. This function would change the auto-bkops state based on
5561 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5562 */
4e768e76 5563static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5564{
4e768e76 5565 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5566 hba->auto_bkops_enabled = false;
5567 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5568 ufshcd_enable_auto_bkops(hba);
5569 } else {
5570 hba->auto_bkops_enabled = true;
5571 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5572 ufshcd_disable_auto_bkops(hba);
5573 }
7b6668d8 5574 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
24366c2a 5575 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5576}
5577
5578static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5579{
5e86ae44 5580 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5581 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5582}
5583
5584/**
57d104c1 5585 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5586 * @hba: per-adapter instance
57d104c1 5587 * @status: bkops_status value
66ec6d59 5588 *
57d104c1
SJ		 5589 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5590 * flag in the device to permit background operations if the device
5591 * bkops_status is greater than or equal to "status" argument passed to
5592 * this function, disable otherwise.
5593 *
5594 * Returns 0 for success, non-zero in case of failure.
5595 *
5596 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5597 * to know whether auto bkops is enabled or disabled after this function
5598 * returns control to it.
66ec6d59 5599 */
57d104c1
SJ		 5600static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5601 enum bkops_status status)
66ec6d59
SRT		 5602{
5603 int err;
57d104c1 5604 u32 curr_status = 0;
66ec6d59 5605
57d104c1 5606 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5607 if (err) {
5608 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5609 __func__, err);
5610 goto out;
57d104c1
SJ		 5611 } else if (curr_status > BKOPS_STATUS_MAX) {
5612 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5613 __func__, curr_status);
5614 err = -EINVAL;
5615 goto out;
66ec6d59
SRT		 5616 }
5617
57d104c1 5618 if (curr_status >= status)
66ec6d59 5619 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5620 else
5621 err = ufshcd_disable_auto_bkops(hba);
66ec6d59
SRT		 5622out:
5623 return err;
5624}
5625
57d104c1
SJ		 5626/**
5627 * ufshcd_urgent_bkops - handle urgent bkops exception event
5628 * @hba: per-adapter instance
5629 *
5630 * Enable fBackgroundOpsEn flag in the device to permit background
5631 * operations.
5632 *
5633 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5634 * and negative error value for any other failure.
5635 */
5636static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5637{
afdfff59 5638 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5639}
5640
66ec6d59
SRT		 5641static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5642{
5e86ae44 5643 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5644 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5645}
5646
afdfff59
YG		 5647static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5648{
5649 int err;
5650 u32 curr_status = 0;
5651
5652 if (hba->is_urgent_bkops_lvl_checked)
5653 goto enable_auto_bkops;
5654
5655 err = ufshcd_get_bkops_status(hba, &curr_status);
5656 if (err) {
5657 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5658 __func__, err);
5659 goto out;
5660 }
5661
5662 /*
5663 * We are seeing that some devices are raising the urgent bkops
5664 * exception events even when BKOPS status doesn't indicate performace
5665 * impacted or critical. Handle these device by determining their urgent
5666 * bkops status at runtime.
5667 */
5668 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5669 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5670 __func__, curr_status);
5671 /* update the current status as the urgent bkops level */
5672 hba->urgent_bkops_lvl = curr_status;
5673 hba->is_urgent_bkops_lvl_checked = true;
5674 }
5675
5676enable_auto_bkops:
5677 err = ufshcd_enable_auto_bkops(hba);
5678out:
5679 if (err < 0)
5680 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5681 __func__, err);
5682}
5683
322c4b29
AA		 5684static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5685{
5686 u32 value;
5687
5688 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5689 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
5690 return;
5691
5692 dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5693
5694 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
5695
5696 /*
5697 * A placeholder for the platform vendors to add whatever additional
5698 * steps required
5699 */
5700}
5701
3b5f3c0d 5702static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
3d17b9b5 5703{
6f8d5a6a 5704 u8 index;
3b5f3c0d
YH		 5705 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5706 UPIU_QUERY_OPCODE_CLEAR_FLAG;
5707
5708 index = ufshcd_wb_get_query_index(hba);
5709 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
5710}
5711
5712int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
5713{
5714 int ret;
3d17b9b5 5715
79e3520f 5716 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5717 return 0;
5718
4cd48995 5719 if (!(enable ^ hba->dev_info.wb_enabled))
3d17b9b5 5720 return 0;
3d17b9b5 5721
3b5f3c0d 5722 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
3d17b9b5 5723 if (ret) {
3b5f3c0d 5724 dev_err(hba->dev, "%s Write Booster %s failed %d\n",
3d17b9b5
AD		 5725 __func__, enable ? "enable" : "disable", ret);
5726 return ret;
5727 }
5728
4cd48995 5729 hba->dev_info.wb_enabled = enable;
3b5f3c0d
YH		 5730 dev_info(hba->dev, "%s Write Booster %s\n",
5731 __func__, enable ? "enabled" : "disabled");
3d17b9b5
AD		 5732
5733 return ret;
5734}
5735
3b5f3c0d 5736static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
3d17b9b5 5737{
3b5f3c0d 5738 int ret;
3d17b9b5 5739
3b5f3c0d
YH		 5740 ret = __ufshcd_wb_toggle(hba, set,
5741 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
5742 if (ret) {
5743 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed: %d\n",
5744 __func__, set ? "enable" : "disable", ret);
5745 return;
5746 }
5747 dev_dbg(hba->dev, "%s WB-Buf Flush during H8 %s\n",
5748 __func__, set ? "enabled" : "disabled");
3d17b9b5
AD		 5749}
5750
3b5f3c0d 5751static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
3d17b9b5
AD		 5752{
5753 int ret;
5754
d3ba622d
BH		 5755 if (!ufshcd_is_wb_allowed(hba) ||
5756 hba->dev_info.wb_buf_flush_enabled == enable)
3b5f3c0d 5757 return;
3d17b9b5 5758
3b5f3c0d 5759 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
3d17b9b5 5760 if (ret) {
d3ba622d
BH		 5761 dev_err(hba->dev, "%s WB-Buf Flush %s failed %d\n", __func__,
5762 enable ? "enable" : "disable", ret);
3b5f3c0d 5763 return;
3d17b9b5
AD		 5764 }
5765
d3ba622d
BH		 5766 hba->dev_info.wb_buf_flush_enabled = enable;
5767
3b5f3c0d
YH		 5768 dev_dbg(hba->dev, "%s WB-Buf Flush %s\n",
5769 __func__, enable ? "enabled" : "disabled");
3d17b9b5
AD		 5770}
5771
5772static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5773 u32 avail_buf)
5774{
5775 u32 cur_buf;
5776 int ret;
e31011ab 5777 u8 index;
3d17b9b5 5778
e31011ab 5779 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5780 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5781 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
e31011ab 5782 index, 0, &cur_buf);
3d17b9b5
AD		 5783 if (ret) {
5784 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5785 __func__, ret);
5786 return false;
5787 }
5788
5789 if (!cur_buf) {
5790 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5791 cur_buf);
5792 return false;
5793 }
d14734ae 5794 /* Let it continue to flush when available buffer exceeds threshold */
a858af9a 5795 return avail_buf < hba->vps->wb_flush_threshold;
3d17b9b5
AD		 5796}
5797
f681d107
JC		 5798static void ufshcd_wb_force_disable(struct ufs_hba *hba)
5799{
5800 if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
5801 ufshcd_wb_toggle_flush(hba, false);
5802
5803 ufshcd_wb_toggle_flush_during_h8(hba, false);
5804 ufshcd_wb_toggle(hba, false);
5805 hba->caps &= ~UFSHCD_CAP_WB_EN;
5806
5807 dev_info(hba->dev, "%s: WB force disabled\n", __func__);
5808}
5809
5810static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
5811{
5812 u32 lifetime;
5813 int ret;
5814 u8 index;
5815
5816 index = ufshcd_wb_get_query_index(hba);
5817 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5818 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
5819 index, 0, &lifetime);
5820 if (ret) {
5821 dev_err(hba->dev,
5822 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
5823 __func__, ret);
5824 return false;
5825 }
5826
5827 if (lifetime == UFS_WB_EXCEED_LIFETIME) {
5828 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
5829 __func__, lifetime);
5830 return false;
5831 }
5832
5833 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
5834 __func__, lifetime);
5835
5836 return true;
5837}
5838
51dd905b 5839static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
3d17b9b5
AD		 5840{
5841 int ret;
5842 u32 avail_buf;
e31011ab 5843 u8 index;
3d17b9b5 5844
79e3520f 5845 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5 5846 return false;
f681d107
JC		 5847
5848 if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
5849 ufshcd_wb_force_disable(hba);
5850 return false;
5851 }
5852
3d17b9b5
AD		 5853 /*
5854 * The ufs device needs the vcc to be ON to flush.
5855 * With user-space reduction enabled, it's enough to enable flush
5856 * by checking only the available buffer. The threshold
5857 * defined here is > 90% full.
5858 * With user-space preserved enabled, the current-buffer
5859 * should be checked too because the wb buffer size can reduce
5860 * when disk tends to be full. This info is provided by current
5861 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5862 * keeping vcc on when current buffer is empty.
5863 */
e31011ab 5864 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5865 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5866 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
e31011ab 5867 index, 0, &avail_buf);
3d17b9b5
AD		 5868 if (ret) {
5869 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5870 __func__, ret);
5871 return false;
5872 }
5873
a858af9a
BVA		 5874 if (!hba->dev_info.b_presrv_uspc_en)
5875 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
3d17b9b5
AD		 5876
5877 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5878}
5879
51dd905b
SC		 5880static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5881{
5882 struct ufs_hba *hba = container_of(to_delayed_work(work),
5883 struct ufs_hba,
5884 rpm_dev_flush_recheck_work);
5885 /*
5886 * To prevent unnecessary VCC power drain after device finishes
5887 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5888 * after a certain delay to recheck the threshold by next runtime
5889 * suspend.
5890 */
b294ff3e
AD		 5891 ufshcd_rpm_get_sync(hba);
5892 ufshcd_rpm_put_sync(hba);
51dd905b
SC		 5893}
5894
66ec6d59
SRT		 5895/**
5896 * ufshcd_exception_event_handler - handle exceptions raised by device
5897 * @work: pointer to work data
5898 *
5899 * Read bExceptionEventStatus attribute from the device and handle the
5900 * exception event accordingly.
5901 */
5902static void ufshcd_exception_event_handler(struct work_struct *work)
5903{
5904 struct ufs_hba *hba;
5905 int err;
5906 u32 status = 0;
5907 hba = container_of(work, struct ufs_hba, eeh_work);
5908
03e1d28e 5909 ufshcd_scsi_block_requests(hba);
66ec6d59
SRT		 5910 err = ufshcd_get_ee_status(hba, &status);
5911 if (err) {
5912 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5913 __func__, err);
5914 goto out;
5915 }
5916
f7733625
AH		 5917 trace_ufshcd_exception_event(dev_name(hba->dev), status);
5918
cd469475 5919 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
afdfff59
YG		 5920 ufshcd_bkops_exception_event_handler(hba);
5921
322c4b29
AA		 5922 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
5923 ufshcd_temp_exception_event_handler(hba, status);
5924
7deedfda 5925 ufs_debugfs_exception_event(hba, status);
66ec6d59 5926out:
03e1d28e 5927 ufshcd_scsi_unblock_requests(hba);
66ec6d59
SRT		 5928}
5929
9a47ec7c
YG		 5930/* Complete requests that have door-bell cleared */
5931static void ufshcd_complete_requests(struct ufs_hba *hba)
5932{
11682523 5933 ufshcd_transfer_req_compl(hba);
9a47ec7c
YG		 5934 ufshcd_tmc_handler(hba);
5935}
5936
583fa62d
YG		 5937/**
5938 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5939 * to recover from the DL NAC errors or not.
5940 * @hba: per-adapter instance
5941 *
5942 * Returns true if error handling is required, false otherwise
5943 */
5944static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5945{
5946 unsigned long flags;
5947 bool err_handling = true;
5948
5949 spin_lock_irqsave(hba->host->host_lock, flags);
5950 /*
5951 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5952 * device fatal error and/or DL NAC & REPLAY timeout errors.
5953 */
5954 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5955 goto out;
5956
5957 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5958 ((hba->saved_err & UIC_ERROR) &&
5959 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5960 goto out;
5961
5962 if ((hba->saved_err & UIC_ERROR) &&
5963 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5964 int err;
5965 /*
5966 * wait for 50ms to see if we can get any other errors or not.
5967 */
5968 spin_unlock_irqrestore(hba->host->host_lock, flags);
5969 msleep(50);
5970 spin_lock_irqsave(hba->host->host_lock, flags);
5971
5972 /*
5973 * now check if we have got any other severe errors other than
5974 * DL NAC error?
5975 */
5976 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5977 ((hba->saved_err & UIC_ERROR) &&
5978 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5979 goto out;
5980
5981 /*
5982 * As DL NAC is the only error received so far, send out NOP
5983 * command to confirm if link is still active or not.
5984 * - If we don't get any response then do error recovery.
5985 * - If we get response then clear the DL NAC error bit.
5986 */
5987
5988 spin_unlock_irqrestore(hba->host->host_lock, flags);
5989 err = ufshcd_verify_dev_init(hba);
5990 spin_lock_irqsave(hba->host->host_lock, flags);
5991
5992 if (err)
5993 goto out;
5994
5995 /* Link seems to be alive hence ignore the DL NAC errors */
5996 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5997 hba->saved_err &= ~UIC_ERROR;
5998 /* clear NAC error */
5999 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
b0008625 6000 if (!hba->saved_uic_err)
583fa62d 6001 err_handling = false;
583fa62d
YG		 6002 }
6003out:
6004 spin_unlock_irqrestore(hba->host->host_lock, flags);
6005 return err_handling;
6006}
6007
88b09900
AH		 6008/* host lock must be held before calling this func */
6009static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6010{
6011 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6012 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6013}
6014
267a59f6 6015void ufshcd_schedule_eh_work(struct ufs_hba *hba)
88b09900 6016{
267a59f6
BVA		 6017 lockdep_assert_held(hba->host->host_lock);
6018
88b09900
AH		 6019 /* handle fatal errors only when link is not in error state */
6020 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6021 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6022 ufshcd_is_saved_err_fatal(hba))
6023 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6024 else
6025 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6026 queue_work(hba->eh_wq, &hba->eh_work);
6027 }
6028}
6029
348e1bc5
SC		 6030static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6031{
6032 down_write(&hba->clk_scaling_lock);
6033 hba->clk_scaling.is_allowed = allow;
6034 up_write(&hba->clk_scaling_lock);
6035}
6036
6037static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6038{
6039 if (suspend) {
6040 if (hba->clk_scaling.is_enabled)
6041 ufshcd_suspend_clkscaling(hba);
6042 ufshcd_clk_scaling_allow(hba, false);
6043 } else {
6044 ufshcd_clk_scaling_allow(hba, true);
6045 if (hba->clk_scaling.is_enabled)
6046 ufshcd_resume_clkscaling(hba);
6047 }
6048}
6049
c72e79c0
CG		 6050static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6051{
b294ff3e 6052 ufshcd_rpm_get_sync(hba);
e2106584 6053 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
b294ff3e 6054 hba->is_sys_suspended) {
88a92d6a
CG		 6055 enum ufs_pm_op pm_op;
6056
c72e79c0 6057 /*
b294ff3e 6058 * Don't assume anything of resume, if
c72e79c0
CG		 6059 * resume fails, irq and clocks can be OFF, and powers
6060 * can be OFF or in LPM.
6061 */
6062 ufshcd_setup_hba_vreg(hba, true);
6063 ufshcd_enable_irq(hba);
6064 ufshcd_setup_vreg(hba, true);
6065 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6066 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6067 ufshcd_hold(hba, false);
6068 if (!ufshcd_is_clkgating_allowed(hba))
6069 ufshcd_setup_clocks(hba, true);
6070 ufshcd_release(hba);
88a92d6a
CG		 6071 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6072 ufshcd_vops_resume(hba, pm_op);
c72e79c0
CG		 6073 } else {
6074 ufshcd_hold(hba, false);
348e1bc5
SC		 6075 if (ufshcd_is_clkscaling_supported(hba) &&
6076 hba->clk_scaling.is_enabled)
c72e79c0 6077 ufshcd_suspend_clkscaling(hba);
348e1bc5 6078 ufshcd_clk_scaling_allow(hba, false);
c72e79c0 6079 }
aa53f580
CG		 6080 ufshcd_scsi_block_requests(hba);
6081 /* Drain ufshcd_queuecommand() */
5675c381 6082 synchronize_rcu();
aa53f580 6083 cancel_work_sync(&hba->eeh_work);
c72e79c0
CG		 6084}
6085
6086static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6087{
aa53f580 6088 ufshcd_scsi_unblock_requests(hba);
c72e79c0 6089 ufshcd_release(hba);
348e1bc5
SC		 6090 if (ufshcd_is_clkscaling_supported(hba))
6091 ufshcd_clk_scaling_suspend(hba, false);
b294ff3e 6092 ufshcd_rpm_put(hba);
c72e79c0
CG		 6093}
6094
6095static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6096{
9cd20d3f 6097 return (!hba->is_powered || hba->shutting_down ||
e2106584 6098 !hba->ufs_device_wlun ||
9cd20d3f 6099 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
c72e79c0 6100 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
9cd20d3f 6101 ufshcd_is_link_broken(hba))));
c72e79c0
CG		 6102}
6103
6104#ifdef CONFIG_PM
6105static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6106{
6107 struct Scsi_Host *shost = hba->host;
6108 struct scsi_device *sdev;
6109 struct request_queue *q;
6110 int ret;
6111
88a92d6a 6112 hba->is_sys_suspended = false;
c72e79c0 6113 /*
b294ff3e 6114 * Set RPM status of wlun device to RPM_ACTIVE,
c72e79c0
CG		 6115 * this also clears its runtime error.
6116 */
e2106584 6117 ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
b294ff3e
AD		 6118
6119 /* hba device might have a runtime error otherwise */
6120 if (ret)
6121 ret = pm_runtime_set_active(hba->dev);
c72e79c0 6122 /*
b294ff3e
AD		 6123 * If wlun device had runtime error, we also need to resume those
6124 * consumer scsi devices in case any of them has failed to be
6125 * resumed due to supplier runtime resume failure. This is to unblock
c72e79c0
CG		 6126 * blk_queue_enter in case there are bios waiting inside it.
6127 */
6128 if (!ret) {
6129 shost_for_each_device(sdev, shost) {
6130 q = sdev->request_queue;
6131 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6132 q->rpm_status == RPM_SUSPENDING))
6133 pm_request_resume(q->dev);
6134 }
6135 }
6136}
6137#else
6138static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6139{
6140}
6141#endif
6142
2355b66e
CG		 6143static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6144{
6145 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6146 u32 mode;
6147
6148 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6149
6150 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6151 return true;
6152
6153 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6154 return true;
6155
6156 return false;
6157}
6158
7a3e97b0 6159/**
e8e7f271 6160 * ufshcd_err_handler - handle UFS errors that require s/w attention
88b09900 6161 * @work: pointer to work structure
7a3e97b0 6162 */
88b09900 6163static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0 6164{
87bf6a6b 6165 int retries = MAX_ERR_HANDLER_RETRIES;
88b09900 6166 struct ufs_hba *hba;
e8e7f271 6167 unsigned long flags;
87bf6a6b
AH		 6168 bool needs_restore;
6169 bool needs_reset;
6170 bool err_xfer;
6171 bool err_tm;
6172 int pmc_err;
e8e7f271
SRT		 6173 int tag;
6174
88b09900
AH		 6175 hba = container_of(work, struct ufs_hba, eh_work);
6176
4693fad7
BVA		 6177 dev_info(hba->dev,
6178 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6179 __func__, ufshcd_state_name[hba->ufshcd_state],
6180 hba->is_powered, hba->shutting_down, hba->saved_err,
6181 hba->saved_uic_err, hba->force_reset,
6182 ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6183
9cd20d3f 6184 down(&hba->host_sem);
e8e7f271 6185 spin_lock_irqsave(hba->host->host_lock, flags);
c72e79c0 6186 if (ufshcd_err_handling_should_stop(hba)) {
4db7a236
CG		 6187 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6188 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6189 spin_unlock_irqrestore(hba->host->host_lock, flags);
9cd20d3f 6190 up(&hba->host_sem);
4db7a236
CG		 6191 return;
6192 }
6193 ufshcd_set_eh_in_progress(hba);
6194 spin_unlock_irqrestore(hba->host->host_lock, flags);
c72e79c0 6195 ufshcd_err_handling_prepare(hba);
a45f9371
CG		 6196 /* Complete requests that have door-bell cleared by h/w */
6197 ufshcd_complete_requests(hba);
e8e7f271 6198 spin_lock_irqsave(hba->host->host_lock, flags);
87bf6a6b
AH		 6199again:
6200 needs_restore = false;
6201 needs_reset = false;
6202 err_xfer = false;
6203 err_tm = false;
6204
aa53f580
CG		 6205 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6206 hba->ufshcd_state = UFSHCD_STATE_RESET;
88a92d6a
CG		 6207 /*
6208 * A full reset and restore might have happened after preparation
6209 * is finished, double check whether we should stop.
6210 */
6211 if (ufshcd_err_handling_should_stop(hba))
6212 goto skip_err_handling;
6213
583fa62d
YG		 6214 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6215 bool ret;
6216
6217 spin_unlock_irqrestore(hba->host->host_lock, flags);
6218 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6219 ret = ufshcd_quirk_dl_nac_errors(hba);
6220 spin_lock_irqsave(hba->host->host_lock, flags);
88a92d6a 6221 if (!ret && ufshcd_err_handling_should_stop(hba))
583fa62d
YG		 6222 goto skip_err_handling;
6223 }
4db7a236 6224
2355b66e
CG		 6225 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6226 (hba->saved_uic_err &&
6227 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
c3be8d1e
CG		 6228 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6229
6230 spin_unlock_irqrestore(hba->host->host_lock, flags);
6231 ufshcd_print_host_state(hba);
6232 ufshcd_print_pwr_info(hba);
e965e5e0 6233 ufshcd_print_evt_hist(hba);
c3be8d1e
CG		 6234 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6235 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
6236 spin_lock_irqsave(hba->host->host_lock, flags);
6237 }
6238
9a47ec7c
YG		 6239 /*
6240 * if host reset is required then skip clearing the pending
2df74b69
CG		 6241 * transfers forcefully because they will get cleared during
6242 * host reset and restore
9a47ec7c 6243 */
88a92d6a
CG		 6244 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6245 ufshcd_is_saved_err_fatal(hba) ||
6246 ((hba->saved_err & UIC_ERROR) &&
6247 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6248 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6249 needs_reset = true;
2355b66e 6250 goto do_reset;
88a92d6a 6251 }
9a47ec7c 6252
2355b66e
CG		 6253 /*
6254 * If LINERESET was caught, UFS might have been put to PWM mode,
6255 * check if power mode restore is needed.
6256 */
6257 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6258 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6259 if (!hba->saved_uic_err)
6260 hba->saved_err &= ~UIC_ERROR;
6261 spin_unlock_irqrestore(hba->host->host_lock, flags);
6262 if (ufshcd_is_pwr_mode_restore_needed(hba))
6263 needs_restore = true;
6264 spin_lock_irqsave(hba->host->host_lock, flags);
6265 if (!hba->saved_err && !needs_restore)
6266 goto skip_err_handling;
6267 }
9a47ec7c 6268
2355b66e 6269 hba->silence_err_logs = true;
9a47ec7c
YG		 6270 /* release lock as clear command might sleep */
6271 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 6272 /* Clear pending transfer requests */
9a47ec7c 6273 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
307348f6 6274 if (ufshcd_try_to_abort_task(hba, tag)) {
9a47ec7c
YG		 6275 err_xfer = true;
6276 goto lock_skip_pending_xfer_clear;
6277 }
4693fad7
BVA		 6278 dev_err(hba->dev, "Aborted tag %d / CDB %#02x\n", tag,
6279 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1);
9a47ec7c 6280 }
e8e7f271
SRT		 6281
6282 /* Clear pending task management requests */
9a47ec7c
YG		 6283 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6284 if (ufshcd_clear_tm_cmd(hba, tag)) {
6285 err_tm = true;
6286 goto lock_skip_pending_xfer_clear;
6287 }
6288 }
e8e7f271 6289
9a47ec7c 6290lock_skip_pending_xfer_clear:
11682523
BVA		 6291 /* Complete the requests that are cleared by s/w */
6292 ufshcd_complete_requests(hba);
9a47ec7c 6293
a45f9371
CG		 6294 spin_lock_irqsave(hba->host->host_lock, flags);
6295 hba->silence_err_logs = false;
2355b66e 6296 if (err_xfer || err_tm) {
9a47ec7c 6297 needs_reset = true;
2355b66e
CG		 6298 goto do_reset;
6299 }
9a47ec7c 6300
2355b66e
CG		 6301 /*
6302 * After all reqs and tasks are cleared from doorbell,
6303 * now it is safe to retore power mode.
6304 */
6305 if (needs_restore) {
6306 spin_unlock_irqrestore(hba->host->host_lock, flags);
6307 /*
6308 * Hold the scaling lock just in case dev cmds
6309 * are sent via bsg and/or sysfs.
6310 */
6311 down_write(&hba->clk_scaling_lock);
6312 hba->force_pmc = true;
6313 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6314 if (pmc_err) {
6315 needs_reset = true;
6316 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6317 __func__, pmc_err);
6318 }
6319 hba->force_pmc = false;
6320 ufshcd_print_pwr_info(hba);
6321 up_write(&hba->clk_scaling_lock);
6322 spin_lock_irqsave(hba->host->host_lock, flags);
6323 }
9a47ec7c 6324
2355b66e 6325do_reset:
e8e7f271 6326 /* Fatal errors need reset */
9a47ec7c 6327 if (needs_reset) {
87bf6a6b
AH		 6328 int err;
6329
4db7a236 6330 hba->force_reset = false;
9a47ec7c 6331 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 6332 err = ufshcd_reset_and_restore(hba);
4db7a236
CG		 6333 if (err)
6334 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6335 __func__, err);
c72e79c0
CG		 6336 else
6337 ufshcd_recover_pm_error(hba);
9a47ec7c 6338 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 6339 }
9a47ec7c 6340
583fa62d 6341skip_err_handling:
9a47ec7c 6342 if (!needs_reset) {
4db7a236
CG		 6343 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6344 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
9a47ec7c
YG		 6345 if (hba->saved_err || hba->saved_uic_err)
6346 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6347 __func__, hba->saved_err, hba->saved_uic_err);
6348 }
87bf6a6b
AH		 6349 /* Exit in an operational state or dead */
6350 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6351 hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6352 if (--retries)
6353 goto again;
6354 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6355 }
e8e7f271 6356 ufshcd_clear_eh_in_progress(hba);
9a47ec7c 6357 spin_unlock_irqrestore(hba->host->host_lock, flags);
c72e79c0 6358 ufshcd_err_handling_unprepare(hba);
9cd20d3f 6359 up(&hba->host_sem);
4693fad7
BVA		 6360
6361 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6362 ufshcd_state_name[hba->ufshcd_state]);
7a3e97b0
SY		 6363}
6364
6365/**
e8e7f271
SRT		 6366 * ufshcd_update_uic_error - check and set fatal UIC error flags.
6367 * @hba: per-adapter instance
9333d775
VG		 6368 *
6369 * Returns
6370 * IRQ_HANDLED - If interrupt is valid
6371 * IRQ_NONE - If invalid interrupt
7a3e97b0 6372 */
9333d775 6373static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 6374{
6375 u32 reg;
9333d775 6376 irqreturn_t retval = IRQ_NONE;
7a3e97b0 6377
2355b66e 6378 /* PHY layer error */
fb7b45f0 6379 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
fb7b45f0 6380 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
2355b66e 6381 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6382 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
fb7b45f0
DR		 6383 /*
6384 * To know whether this error is fatal or not, DB timeout
6385 * must be checked but this error is handled separately.
6386 */
2355b66e
CG		 6387 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6388 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6389 __func__);
6390
6391 /* Got a LINERESET indication. */
6392 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6393 struct uic_command *cmd = NULL;
6394
6395 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6396 if (hba->uic_async_done && hba->active_uic_cmd)
6397 cmd = hba->active_uic_cmd;
6398 /*
6399 * Ignore the LINERESET during power mode change
6400 * operation via DME_SET command.
6401 */
6402 if (cmd && (cmd->command == UIC_CMD_DME_SET))
6403 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6404 }
9333d775 6405 retval |= IRQ_HANDLED;
ff8e20c6 6406 }
fb7b45f0 6407
e8e7f271
SRT		 6408 /* PA_INIT_ERROR is fatal and needs UIC reset */
6409 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 6410 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6411 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6412 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
ff8e20c6 6413
9333d775
VG		 6414 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6415 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6416 else if (hba->dev_quirks &
6417 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6418 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6419 hba->uic_error |=
6420 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6421 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6422 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6423 }
6424 retval |= IRQ_HANDLED;
583fa62d 6425 }
e8e7f271
SRT		 6426
6427 /* UIC NL/TL/DME errors needs software retry */
6428 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 6429 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6430 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6431 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
e8e7f271 6432 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 6433 retval |= IRQ_HANDLED;
ff8e20c6 6434 }
e8e7f271
SRT		 6435
6436 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 6437 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6438 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
e965e5e0 6439 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
e8e7f271 6440 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 6441 retval |= IRQ_HANDLED;
ff8e20c6 6442 }
e8e7f271
SRT		 6443
6444 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 6445 if ((reg & UIC_DME_ERROR) &&
6446 (reg & UIC_DME_ERROR_CODE_MASK)) {
e965e5e0 6447 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
e8e7f271 6448 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 6449 retval |= IRQ_HANDLED;
ff8e20c6 6450 }
e8e7f271
SRT		 6451
6452 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6453 __func__, hba->uic_error);
9333d775 6454 return retval;
e8e7f271
SRT		 6455}
6456
6457/**
6458 * ufshcd_check_errors - Check for errors that need s/w attention
6459 * @hba: per-adapter instance
a45f9371 6460 * @intr_status: interrupt status generated by the controller
9333d775
VG		 6461 *
6462 * Returns
6463 * IRQ_HANDLED - If interrupt is valid
6464 * IRQ_NONE - If invalid interrupt
e8e7f271 6465 */
a45f9371 6466static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
e8e7f271
SRT		 6467{
6468 bool queue_eh_work = false;
9333d775 6469 irqreturn_t retval = IRQ_NONE;
e8e7f271 6470
a45f9371
CG		 6471 spin_lock(hba->host->host_lock);
6472 hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6473
d3c615bf 6474 if (hba->errors & INT_FATAL_ERRORS) {
e965e5e0
SC		 6475 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6476 hba->errors);
e8e7f271 6477 queue_eh_work = true;
d3c615bf 6478 }
7a3e97b0
SY		 6479
6480 if (hba->errors & UIC_ERROR) {
e8e7f271 6481 hba->uic_error = 0;
9333d775 6482 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 6483 if (hba->uic_error)
6484 queue_eh_work = true;
7a3e97b0 6485 }
e8e7f271 6486
82174440
SC		 6487 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6488 dev_err(hba->dev,
6489 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6490 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6491 "Enter" : "Exit",
6492 hba->errors, ufshcd_get_upmcrs(hba));
e965e5e0 6493 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
d3c615bf 6494 hba->errors);
4db7a236 6495 ufshcd_set_link_broken(hba);
82174440
SC		 6496 queue_eh_work = true;
6497 }
6498
e8e7f271 6499 if (queue_eh_work) {
9a47ec7c
YG		 6500 /*
6501 * update the transfer error masks to sticky bits, let's do this
6502 * irrespective of current ufshcd_state.
6503 */
6504 hba->saved_err |= hba->errors;
6505 hba->saved_uic_err |= hba->uic_error;
6506
4db7a236 6507 /* dump controller state before resetting */
ace3804b
CG		 6508 if ((hba->saved_err &
6509 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
2355b66e
CG		 6510 (hba->saved_uic_err &&
6511 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
4db7a236 6512 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
66cc820f
DR		 6513 __func__, hba->saved_err,
6514 hba->saved_uic_err);
c3be8d1e
CG		 6515 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6516 "host_regs: ");
4db7a236 6517 ufshcd_print_pwr_info(hba);
e8e7f271 6518 }
88b09900 6519 ufshcd_schedule_eh_work(hba);
9333d775 6520 retval |= IRQ_HANDLED;
3441da7d 6521 }
e8e7f271
SRT		 6522 /*
6523 * if (!queue_eh_work) -
6524 * Other errors are either non-fatal where host recovers
6525 * itself without s/w intervention or errors that will be
6526 * handled by the SCSI core layer.
6527 */
a45f9371
CG		 6528 hba->errors = 0;
6529 hba->uic_error = 0;
6530 spin_unlock(hba->host->host_lock);
9333d775 6531 return retval;
7a3e97b0
SY		 6532}
6533
6534/**
6535 * ufshcd_tmc_handler - handle task management function completion
6536 * @hba: per adapter instance
9333d775
VG		 6537 *
6538 * Returns
6539 * IRQ_HANDLED - If interrupt is valid
6540 * IRQ_NONE - If invalid interrupt
7a3e97b0 6541 */
9333d775 6542static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 6543{
f5ef336f
AH		 6544 unsigned long flags, pending, issued;
6545 irqreturn_t ret = IRQ_NONE;
6546 int tag;
6547
a45f9371 6548 spin_lock_irqsave(hba->host->host_lock, flags);
5cb37a26 6549 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
f5ef336f
AH		 6550 issued = hba->outstanding_tasks & ~pending;
6551 for_each_set_bit(tag, &issued, hba->nutmrs) {
6552 struct request *req = hba->tmf_rqs[tag];
6553 struct completion *c = req->end_io_data;
6554
6555 complete(c);
6556 ret = IRQ_HANDLED;
6557 }
a45f9371
CG		 6558 spin_unlock_irqrestore(hba->host->host_lock, flags);
6559
f5ef336f 6560 return ret;
7a3e97b0
SY		 6561}
6562
6563/**
6564 * ufshcd_sl_intr - Interrupt service routine
6565 * @hba: per adapter instance
6566 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 6567 *
6568 * Returns
6569 * IRQ_HANDLED - If interrupt is valid
6570 * IRQ_NONE - If invalid interrupt
7a3e97b0 6571 */
9333d775 6572static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 6573{
9333d775
VG		 6574 irqreturn_t retval = IRQ_NONE;
6575
53b3d9c3 6576 if (intr_status & UFSHCD_UIC_MASK)
9333d775 6577 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0 6578
a45f9371
CG		 6579 if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6580 retval |= ufshcd_check_errors(hba, intr_status);
6581
7a3e97b0 6582 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 6583 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 6584
6585 if (intr_status & UTP_TRANSFER_REQ_COMPL)
11682523 6586 retval |= ufshcd_transfer_req_compl(hba);
9333d775
VG		 6587
6588 return retval;
7a3e97b0
SY		 6589}
6590
6591/**
6592 * ufshcd_intr - Main interrupt service routine
6593 * @irq: irq number
6594 * @__hba: pointer to adapter instance
6595 *
9333d775
VG		 6596 * Returns
6597 * IRQ_HANDLED - If interrupt is valid
6598 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 6599 */
6600static irqreturn_t ufshcd_intr(int irq, void *__hba)
6601{
127d5f7c 6602 u32 intr_status, enabled_intr_status = 0;
7a3e97b0
SY		 6603 irqreturn_t retval = IRQ_NONE;
6604 struct ufs_hba *hba = __hba;
7f6ba4f1 6605 int retries = hba->nutrs;
7a3e97b0 6606
b873a275 6607 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
3f8af604
CG		 6608 hba->ufs_stats.last_intr_status = intr_status;
6609 hba->ufs_stats.last_intr_ts = ktime_get();
7a3e97b0 6610
7f6ba4f1
VG		 6611 /*
6612 * There could be max of hba->nutrs reqs in flight and in worst case
6613 * if the reqs get finished 1 by 1 after the interrupt status is
6614 * read, make sure we handle them by checking the interrupt status
6615 * again in a loop until we process all of the reqs before returning.
6616 */
127d5f7c 6617 while (intr_status && retries--) {
7f6ba4f1
VG		 6618 enabled_intr_status =
6619 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
60ec3755 6620 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 6621 if (enabled_intr_status)
6622 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 6623
6624 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
127d5f7c 6625 }
d75f7fe4 6626
eeb1b55b 6627 if (enabled_intr_status && retval == IRQ_NONE &&
40d2fd05
BVA		 6628 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6629 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
eeb1b55b
JK		 6630 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6631 __func__,
6632 intr_status,
6633 hba->ufs_stats.last_intr_status,
6634 enabled_intr_status);
9333d775
VG		 6635 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6636 }
6637
7a3e97b0
SY		 6638 return retval;
6639}
6640
e2933132
SRT		 6641static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6642{
6643 int err = 0;
6644 u32 mask = 1 << tag;
6645 unsigned long flags;
6646
6647 if (!test_bit(tag, &hba->outstanding_tasks))
6648 goto out;
6649
6650 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 6651 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 6652 spin_unlock_irqrestore(hba->host->host_lock, flags);
6653
6654 /* poll for max. 1 sec to clear door bell register by h/w */
6655 err = ufshcd_wait_for_register(hba,
6656 REG_UTP_TASK_REQ_DOOR_BELL,
5cac1095 6657 mask, 0, 1000, 1000);
4693fad7
BVA		 6658
6659 dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
6660 tag, err ? "succeeded" : "failed");
6661
e2933132
SRT		 6662out:
6663 return err;
6664}
6665
c6049cd9
CH		 6666static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6667 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 6668{
69a6c269 6669 struct request_queue *q = hba->tmf_queue;
c6049cd9 6670 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 6671 DECLARE_COMPLETION_ONSTACK(wait);
6672 struct request *req;
7a3e97b0 6673 unsigned long flags;
4b42d557 6674 int task_tag, err;
7a3e97b0 6675
e2933132 6676 /*
0bf6d96c 6677 * blk_mq_alloc_request() is used here only to get a free tag.
e2933132 6678 */
0bf6d96c 6679 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
eeb1b55b
JK		 6680 if (IS_ERR(req))
6681 return PTR_ERR(req);
6682
69a6c269 6683 req->end_io_data = &wait;
1ab27c9c 6684 ufshcd_hold(hba, false);
7a3e97b0 6685
e2933132 6686 spin_lock_irqsave(host->host_lock, flags);
7a3e97b0 6687
4b42d557 6688 task_tag = req->tag;
eaab9b57
BVA		 6689 WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n",
6690 task_tag);
f5ef336f 6691 hba->tmf_rqs[req->tag] = req;
1352eec8 6692 treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag);
c6049cd9 6693
4b42d557
CG		 6694 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
6695 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
d2877be4 6696
7a3e97b0 6697 /* send command to the controller */
4b42d557 6698 __set_bit(task_tag, &hba->outstanding_tasks);
897efe62 6699
4b42d557 6700 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 6701 /* Make sure that doorbell is committed immediately */
6702 wmb();
7a3e97b0
SY		 6703
6704 spin_unlock_irqrestore(host->host_lock, flags);
6705
28fa68fc 6706 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
6667e6d9 6707
7a3e97b0 6708 /* wait until the task management command is completed */
69a6c269 6709 err = wait_for_completion_io_timeout(&wait,
e2933132 6710 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 6711 if (!err) {
28fa68fc 6712 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
e2933132
SRT		 6713 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6714 __func__, tm_function);
4b42d557
CG		 6715 if (ufshcd_clear_tm_cmd(hba, task_tag))
6716 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
6717 __func__, task_tag);
e2933132
SRT		 6718 err = -ETIMEDOUT;
6719 } else {
c6049cd9 6720 err = 0;
4b42d557 6721 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
c6049cd9 6722
28fa68fc 6723 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
7a3e97b0 6724 }
e2933132 6725
b557217c 6726 spin_lock_irqsave(hba->host->host_lock, flags);
f5ef336f 6727 hba->tmf_rqs[req->tag] = NULL;
4b42d557 6728 __clear_bit(task_tag, &hba->outstanding_tasks);
b557217c
SC		 6729 spin_unlock_irqrestore(hba->host->host_lock, flags);
6730
4b42d557 6731 ufshcd_release(hba);
0bf6d96c 6732 blk_mq_free_request(req);
e2933132 6733
7a3e97b0
SY		 6734 return err;
6735}
6736
c6049cd9
CH		 6737/**
6738 * ufshcd_issue_tm_cmd - issues task management commands to controller
6739 * @hba: per adapter instance
6740 * @lun_id: LUN ID to which TM command is sent
6741 * @task_id: task ID to which the TM command is applicable
6742 * @tm_function: task management function opcode
6743 * @tm_response: task management service response return value
6744 *
6745 * Returns non-zero value on error, zero on success.
6746 */
6747static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6748 u8 tm_function, u8 *tm_response)
6749{
6750 struct utp_task_req_desc treq = { { 0 }, };
957d63e7
BVA		 6751 enum utp_ocs ocs_value;
6752 int err;
c6049cd9
CH		 6753
6754 /* Configure task request descriptor */
6755 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6756 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6757
6758 /* Configure task request UPIU */
1352eec8 6759 treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
c6049cd9 6760 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
1352eec8 6761 treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16);
c6049cd9
CH		 6762
6763 /*
6764 * The host shall provide the same value for LUN field in the basic
6765 * header and for Input Parameter.
6766 */
1352eec8
GS		 6767 treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
6768 treq.upiu_req.input_param2 = cpu_to_be32(task_id);
c6049cd9
CH		 6769
6770 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6771 if (err == -ETIMEDOUT)
6772 return err;
6773
6774 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6775 if (ocs_value != OCS_SUCCESS)
6776 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6777 __func__, ocs_value);
6778 else if (tm_response)
1352eec8 6779 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
c6049cd9
CH		 6780 MASK_TM_SERVICE_RESP;
6781 return err;
6782}
6783
5e0a86ee
AA		 6784/**
6785 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6786 * @hba: per-adapter instance
6787 * @req_upiu: upiu request
6788 * @rsp_upiu: upiu reply
5e0a86ee
AA		 6789 * @desc_buff: pointer to descriptor buffer, NULL if NA
6790 * @buff_len: descriptor size, 0 if NA
d0e9760d 6791 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 6792 * @desc_op: descriptor operation
6793 *
6794 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6795 * Therefore, it "rides" the device management infrastructure: uses its tag and
6796 * tasks work queues.
6797 *
6798 * Since there is only one available tag for device management commands,
6799 * the caller is expected to hold the hba->dev_cmd.lock mutex.
6800 */
6801static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6802 struct utp_upiu_req *req_upiu,
6803 struct utp_upiu_req *rsp_upiu,
6804 u8 *desc_buff, int *buff_len,
7f674c38 6805 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 6806 enum query_opcode desc_op)
6807{
8a686f26 6808 DECLARE_COMPLETION_ONSTACK(wait);
945c3cca 6809 const u32 tag = hba->reserved_slot;
5e0a86ee
AA		 6810 struct ufshcd_lrb *lrbp;
6811 int err = 0;
a23064c4 6812 u8 upiu_flags;
5e0a86ee 6813
945c3cca
BVA		 6814 /* Protects use of hba->reserved_slot. */
6815 lockdep_assert_held(&hba->dev_cmd.lock);
5e0a86ee 6816
945c3cca 6817 down_read(&hba->clk_scaling_lock);
5e0a86ee 6818
a45f9371 6819 lrbp = &hba->lrb[tag];
7a7e66c6 6820 WARN_ON(lrbp->cmd);
5e0a86ee 6821 lrbp->cmd = NULL;
5e0a86ee
AA		 6822 lrbp->task_tag = tag;
6823 lrbp->lun = 0;
6824 lrbp->intr_cmd = true;
df043c74 6825 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5e0a86ee
AA		 6826 hba->dev_cmd.type = cmd_type;
6827
51428818 6828 if (hba->ufs_version <= ufshci_version(1, 1))
5e0a86ee 6829 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
51428818 6830 else
5e0a86ee 6831 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
5e0a86ee
AA		 6832
6833 /* update the task tag in the request upiu */
6834 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6835
6836 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6837
6838 /* just copy the upiu request as it is */
6839 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6840 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6841 /* The Data Segment Area is optional depending upon the query
6842 * function value. for WRITE DESCRIPTOR, the data segment
6843 * follows right after the tsf.
6844 */
6845 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6846 *buff_len = 0;
6847 }
6848
6849 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6850
6851 hba->dev_cmd.complete = &wait;
6852
10542489 6853 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
5e0a86ee 6854
a45f9371 6855 ufshcd_send_command(hba, tag);
5e0a86ee
AA		 6856 /*
6857 * ignore the returning value here - ufshcd_check_query_response is
6858 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6859 * read the response directly ignoring all errors.
6860 */
6861 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6862
6863 /* just copy the upiu response as it is */
6864 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 6865 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6866 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6867 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6868 MASK_QUERY_DATA_SEG_LEN;
6869
6870 if (*buff_len >= resp_len) {
6871 memcpy(desc_buff, descp, resp_len);
6872 *buff_len = resp_len;
6873 } else {
3d4881d1
BH		 6874 dev_warn(hba->dev,
6875 "%s: rsp size %d is bigger than buffer size %d",
6876 __func__, resp_len, *buff_len);
4bbbe242
AA		 6877 *buff_len = 0;
6878 err = -EINVAL;
6879 }
6880 }
10542489
BH		 6881 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
6882 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
5e0a86ee 6883
5e0a86ee
AA		 6884 up_read(&hba->clk_scaling_lock);
6885 return err;
6886}
6887
6888/**
6889 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6890 * @hba: per-adapter instance
6891 * @req_upiu: upiu request
6892 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
6893 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
6894 * @desc_buff: pointer to descriptor buffer, NULL if NA
6895 * @buff_len: descriptor size, 0 if NA
6896 * @desc_op: descriptor operation
6897 *
6898 * Supports UTP Transfer requests (nop and query), and UTP Task
6899 * Management requests.
6900 * It is up to the caller to fill the upiu conent properly, as it will
6901 * be copied without any further input validations.
6902 */
6903int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6904 struct utp_upiu_req *req_upiu,
6905 struct utp_upiu_req *rsp_upiu,
6906 int msgcode,
6907 u8 *desc_buff, int *buff_len,
6908 enum query_opcode desc_op)
6909{
6910 int err;
7f674c38 6911 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee 6912 struct utp_task_req_desc treq = { { 0 }, };
957d63e7 6913 enum utp_ocs ocs_value;
5e0a86ee
AA		 6914 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6915
5e0a86ee
AA		 6916 switch (msgcode) {
6917 case UPIU_TRANSACTION_NOP_OUT:
6918 cmd_type = DEV_CMD_TYPE_NOP;
df561f66 6919 fallthrough;
5e0a86ee
AA		 6920 case UPIU_TRANSACTION_QUERY_REQ:
6921 ufshcd_hold(hba, false);
6922 mutex_lock(&hba->dev_cmd.lock);
6923 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6924 desc_buff, buff_len,
6925 cmd_type, desc_op);
6926 mutex_unlock(&hba->dev_cmd.lock);
6927 ufshcd_release(hba);
6928
6929 break;
6930 case UPIU_TRANSACTION_TASK_REQ:
6931 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6932 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6933
1352eec8 6934 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
5e0a86ee
AA		 6935
6936 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6937 if (err == -ETIMEDOUT)
6938 break;
6939
6940 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6941 if (ocs_value != OCS_SUCCESS) {
6942 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6943 ocs_value);
6944 break;
6945 }
6946
1352eec8 6947 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
5e0a86ee
AA		 6948
6949 break;
6950 default:
6951 err = -EINVAL;
6952
6953 break;
6954 }
6955
5e0a86ee
AA		 6956 return err;
6957}
6958
7a3e97b0 6959/**
3441da7d
SRT		 6960 * ufshcd_eh_device_reset_handler - device reset handler registered to
6961 * scsi layer.
7a3e97b0
SY		 6962 * @cmd: SCSI command pointer
6963 *
6964 * Returns SUCCESS/FAILED
6965 */
3441da7d 6966static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6967{
6968 struct Scsi_Host *host;
6969 struct ufs_hba *hba;
7a3e97b0
SY		 6970 u32 pos;
6971 int err;
35fc4cd3 6972 u8 resp = 0xF, lun;
7a3e97b0
SY		 6973
6974 host = cmd->device->host;
6975 hba = shost_priv(host);
7a3e97b0 6976
35fc4cd3
CG		 6977 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
6978 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
e2933132 6979 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6980 if (!err)
6981 err = resp;
7a3e97b0 6982 goto out;
e2933132 6983 }
7a3e97b0 6984
3441da7d
SRT		 6985 /* clear the commands that were pending for corresponding LUN */
6986 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
35fc4cd3 6987 if (hba->lrb[pos].lun == lun) {
3441da7d
SRT		 6988 err = ufshcd_clear_cmd(hba, pos);
6989 if (err)
6990 break;
11682523 6991 __ufshcd_transfer_req_compl(hba, 1U << pos);
7a3e97b0 6992 }
3441da7d 6993 }
7fabb77b 6994
7a3e97b0 6995out:
7fabb77b 6996 hba->req_abort_count = 0;
e965e5e0 6997 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
3441da7d
SRT		 6998 if (!err) {
6999 err = SUCCESS;
7000 } else {
7001 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7002 err = FAILED;
7003 }
7a3e97b0
SY		 7004 return err;
7005}
7006
e0b299e3
GB		 7007static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7008{
7009 struct ufshcd_lrb *lrbp;
7010 int tag;
7011
7012 for_each_set_bit(tag, &bitmap, hba->nutrs) {
7013 lrbp = &hba->lrb[tag];
7014 lrbp->req_abort_skip = true;
7015 }
7016}
7017
7a3e97b0 7018/**
307348f6 7019 * ufshcd_try_to_abort_task - abort a specific task
d23ec0b6
LJ		 7020 * @hba: Pointer to adapter instance
7021 * @tag: Task tag/index to be aborted
7a3e97b0 7022 *
f20810d8
SRT		 7023 * Abort the pending command in device by sending UFS_ABORT_TASK task management
7024 * command, and in host controller by clearing the door-bell register. There can
7025 * be race between controller sending the command to the device while abort is
7026 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7027 * really issued and then try to abort it.
7028 *
307348f6
CG		 7029 * Returns zero on success, non-zero on failure
7030 */
7031static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7032{
7033 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7034 int err = 0;
7035 int poll_cnt;
7036 u8 resp = 0xF;
7037 u32 reg;
7038
7039 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7040 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7041 UFS_QUERY_TASK, &resp);
7042 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7043 /* cmd pending in the device */
7044 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7045 __func__, tag);
7046 break;
7047 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7048 /*
7049 * cmd not pending in the device, check if it is
7050 * in transition.
7051 */
7052 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7053 __func__, tag);
7054 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7055 if (reg & (1 << tag)) {
7056 /* sleep for max. 200us to stabilize */
7057 usleep_range(100, 200);
7058 continue;
7059 }
7060 /* command completed already */
7061 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7062 __func__, tag);
7063 goto out;
7064 } else {
7065 dev_err(hba->dev,
7066 "%s: no response from device. tag = %d, err %d\n",
7067 __func__, tag, err);
7068 if (!err)
7069 err = resp; /* service response error */
7070 goto out;
7071 }
7072 }
7073
7074 if (!poll_cnt) {
7075 err = -EBUSY;
7076 goto out;
7077 }
7078
7079 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7080 UFS_ABORT_TASK, &resp);
7081 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7082 if (!err) {
7083 err = resp; /* service response error */
7084 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7085 __func__, tag, err);
7086 }
7087 goto out;
7088 }
7089
7090 err = ufshcd_clear_cmd(hba, tag);
7091 if (err)
7092 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7093 __func__, tag, err);
7094
7095out:
7096 return err;
7097}
7098
7099/**
7100 * ufshcd_abort - scsi host template eh_abort_handler callback
7101 * @cmd: SCSI command pointer
7102 *
7a3e97b0
SY		 7103 * Returns SUCCESS/FAILED
7104 */
7105static int ufshcd_abort(struct scsi_cmnd *cmd)
7106{
4728ab4a
BVA		 7107 struct Scsi_Host *host = cmd->device->host;
7108 struct ufs_hba *hba = shost_priv(host);
3f2c1002 7109 int tag = scsi_cmd_to_rq(cmd)->tag;
4728ab4a 7110 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7a3e97b0 7111 unsigned long flags;
64180742 7112 int err = FAILED;
1fbaa02d 7113 bool outstanding;
e9d501b1 7114 u32 reg;
7a3e97b0 7115
4728ab4a 7116 WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
7a3e97b0 7117
1ab27c9c 7118 ufshcd_hold(hba, false);
14497328 7119 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
64180742 7120 /* If command is already aborted/completed, return FAILED. */
14497328
YG		 7121 if (!(test_bit(tag, &hba->outstanding_reqs))) {
7122 dev_err(hba->dev,
7123 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7124 __func__, tag, hba->outstanding_reqs, reg);
64180742 7125 goto release;
14497328 7126 }
7a3e97b0 7127
66cc820f 7128 /* Print Transfer Request of aborted task */
d87a1f6d 7129 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 7130
7fabb77b
GB		 7131 /*
7132 * Print detailed info about aborted request.
7133 * As more than one request might get aborted at the same time,
7134 * print full information only for the first aborted request in order
7135 * to reduce repeated printouts. For other aborted requests only print
7136 * basic details.
7137 */
7a7e66c6 7138 scsi_print_command(cmd);
7fabb77b 7139 if (!hba->req_abort_count) {
e965e5e0
SC		 7140 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7141 ufshcd_print_evt_hist(hba);
6ba65588 7142 ufshcd_print_host_state(hba);
7fabb77b
GB		 7143 ufshcd_print_pwr_info(hba);
7144 ufshcd_print_trs(hba, 1 << tag, true);
7145 } else {
7146 ufshcd_print_trs(hba, 1 << tag, false);
7147 }
7148 hba->req_abort_count++;
e0b299e3 7149
d87a1f6d
BH		 7150 if (!(reg & (1 << tag))) {
7151 dev_err(hba->dev,
7152 "%s: cmd was completed, but without a notifying intr, tag = %d",
7153 __func__, tag);
11682523 7154 __ufshcd_transfer_req_compl(hba, 1UL << tag);
64180742 7155 goto release;
d87a1f6d
BH		 7156 }
7157
7a7e66c6
CG		 7158 /*
7159 * Task abort to the device W-LUN is illegal. When this command
7160 * will fail, due to spec violation, scsi err handling next step
7161 * will be to send LU reset which, again, is a spec violation.
7162 * To avoid these unnecessary/illegal steps, first we clean up
a45f9371 7163 * the lrb taken by this cmd and re-set it in outstanding_reqs,
88b09900 7164 * then queue the eh_work and bail.
7a7e66c6
CG		 7165 */
7166 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7167 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
64180742 7168
7a7e66c6 7169 spin_lock_irqsave(host->host_lock, flags);
a45f9371 7170 hba->force_reset = true;
88b09900 7171 ufshcd_schedule_eh_work(hba);
7a7e66c6 7172 spin_unlock_irqrestore(host->host_lock, flags);
64180742 7173 goto release;
7a7e66c6
CG		 7174 }
7175
e0b299e3 7176 /* Skip task abort in case previous aborts failed and report failure */
64180742
BVA		 7177 if (lrbp->req_abort_skip) {
7178 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7179 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7180 goto release;
7181 }
f20810d8 7182
64180742
BVA		 7183 err = ufshcd_try_to_abort_task(hba, tag);
7184 if (err) {
f20810d8 7185 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 7186 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8 7187 err = FAILED;
64180742 7188 goto release;
f20810d8
SRT		 7189 }
7190
1fbaa02d
BVA		 7191 /*
7192 * Clear the corresponding bit from outstanding_reqs since the command
7193 * has been aborted successfully.
7194 */
7195 spin_lock_irqsave(&hba->outstanding_lock, flags);
7196 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7197 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7198
7199 if (outstanding)
7200 ufshcd_release_scsi_cmd(hba, lrbp);
7201
64180742
BVA		 7202 err = SUCCESS;
7203
7204release:
7205 /* Matches the ufshcd_hold() call at the start of this function. */
1ab27c9c 7206 ufshcd_release(hba);
7a3e97b0
SY		 7207 return err;
7208}
7209
3441da7d
SRT		 7210/**
7211 * ufshcd_host_reset_and_restore - reset and restore host controller
7212 * @hba: per-adapter instance
7213 *
7214 * Note that host controller reset may issue DME_RESET to
7215 * local and remote (device) Uni-Pro stack and the attributes
7216 * are reset to default state.
7217 *
7218 * Returns zero on success, non-zero on failure
7219 */
7220static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7221{
7222 int err;
3441da7d 7223
2df74b69
CG		 7224 /*
7225 * Stop the host controller and complete the requests
7226 * cleared by h/w
7227 */
f02bc975 7228 ufshpb_reset_host(hba);
5cac1095 7229 ufshcd_hba_stop(hba);
2df74b69 7230 hba->silence_err_logs = true;
11682523 7231 ufshcd_complete_requests(hba);
2df74b69 7232 hba->silence_err_logs = false;
3441da7d 7233
a3cd5ec5 7234 /* scale up clocks to max frequency before full reinitialization */
394b949f 7235 ufshcd_set_clk_freq(hba, true);
a3cd5ec5 7236
3441da7d 7237 err = ufshcd_hba_enable(hba);
3441da7d
SRT		 7238
7239 /* Establish the link again and restore the device */
1918651f 7240 if (!err)
4ee7ee53
JK		 7241 err = ufshcd_probe_hba(hba, false);
7242
3441da7d
SRT		 7243 if (err)
7244 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
e965e5e0 7245 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
3441da7d
SRT		 7246 return err;
7247}
7248
7249/**
7250 * ufshcd_reset_and_restore - reset and re-initialize host/device
7251 * @hba: per-adapter instance
7252 *
7253 * Reset and recover device, host and re-establish link. This
7254 * is helpful to recover the communication in fatal error conditions.
7255 *
7256 * Returns zero on success, non-zero on failure
7257 */
7258static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7259{
54a40453
AH		 7260 u32 saved_err = 0;
7261 u32 saved_uic_err = 0;
3441da7d 7262 int err = 0;
4db7a236 7263 unsigned long flags;
1d337ec2 7264 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 7265
4db7a236 7266 spin_lock_irqsave(hba->host->host_lock, flags);
1d337ec2 7267 do {
54a40453
AH		 7268 /*
7269 * This is a fresh start, cache and clear saved error first,
7270 * in case new error generated during reset and restore.
7271 */
7272 saved_err |= hba->saved_err;
7273 saved_uic_err |= hba->saved_uic_err;
7274 hba->saved_err = 0;
7275 hba->saved_uic_err = 0;
7276 hba->force_reset = false;
7277 hba->ufshcd_state = UFSHCD_STATE_RESET;
7278 spin_unlock_irqrestore(hba->host->host_lock, flags);
7279
d8d9f793 7280 /* Reset the attached device */
31a5d9ca 7281 ufshcd_device_reset(hba);
d8d9f793 7282
1d337ec2 7283 err = ufshcd_host_reset_and_restore(hba);
54a40453
AH		 7284
7285 spin_lock_irqsave(hba->host->host_lock, flags);
7286 if (err)
7287 continue;
7288 /* Do not exit unless operational or dead */
7289 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7290 hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7291 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7292 err = -EAGAIN;
1d337ec2 7293 } while (err && --retries);
3441da7d 7294
4db7a236
CG		 7295 /*
7296 * Inform scsi mid-layer that we did reset and allow to handle
7297 * Unit Attention properly.
7298 */
7299 scsi_report_bus_reset(hba->host, 0);
7300 if (err) {
88a92d6a 7301 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4db7a236
CG		 7302 hba->saved_err |= saved_err;
7303 hba->saved_uic_err |= saved_uic_err;
7304 }
7305 spin_unlock_irqrestore(hba->host->host_lock, flags);
7306
3441da7d
SRT		 7307 return err;
7308}
7309
7310/**
7311 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 7312 * @cmd: SCSI command pointer
3441da7d
SRT		 7313 *
7314 * Returns SUCCESS/FAILED
7315 */
7316static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7317{
4db7a236 7318 int err = SUCCESS;
3441da7d
SRT		 7319 unsigned long flags;
7320 struct ufs_hba *hba;
7321
7322 hba = shost_priv(cmd->device->host);
7323
4db7a236
CG		 7324 spin_lock_irqsave(hba->host->host_lock, flags);
7325 hba->force_reset = true;
88b09900 7326 ufshcd_schedule_eh_work(hba);
4db7a236 7327 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
3441da7d
SRT		 7328 spin_unlock_irqrestore(hba->host->host_lock, flags);
7329
88b09900 7330 flush_work(&hba->eh_work);
3441da7d
SRT		 7331
7332 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236 7333 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
3441da7d 7334 err = FAILED;
3441da7d
SRT		 7335 spin_unlock_irqrestore(hba->host->host_lock, flags);
7336
7337 return err;
7338}
7339
3a4bf06d
YG		 7340/**
7341 * ufshcd_get_max_icc_level - calculate the ICC level
7342 * @sup_curr_uA: max. current supported by the regulator
7343 * @start_scan: row at the desc table to start scan from
7344 * @buff: power descriptor buffer
7345 *
7346 * Returns calculated max ICC level for specific regulator
7347 */
7348static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
7349{
7350 int i;
7351 int curr_uA;
7352 u16 data;
7353 u16 unit;
7354
7355 for (i = start_scan; i >= 0; i--) {
9d3ab17e 7356 data = get_unaligned_be16(&buff[2 * i]);
3a4bf06d
YG		 7357 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7358 ATTR_ICC_LVL_UNIT_OFFSET;
7359 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7360 switch (unit) {
7361 case UFSHCD_NANO_AMP:
7362 curr_uA = curr_uA / 1000;
7363 break;
7364 case UFSHCD_MILI_AMP:
7365 curr_uA = curr_uA * 1000;
7366 break;
7367 case UFSHCD_AMP:
7368 curr_uA = curr_uA * 1000 * 1000;
7369 break;
7370 case UFSHCD_MICRO_AMP:
7371 default:
7372 break;
7373 }
7374 if (sup_curr_uA >= curr_uA)
7375 break;
7376 }
7377 if (i < 0) {
7378 i = 0;
7379 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7380 }
7381
7382 return (u32)i;
7383}
7384
7385/**
11eea9b3 7386 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
3a4bf06d
YG		 7387 * In case regulators are not initialized we'll return 0
7388 * @hba: per-adapter instance
7389 * @desc_buf: power descriptor buffer to extract ICC levels from.
7390 * @len: length of desc_buff
7391 *
7392 * Returns calculated ICC level
7393 */
7394static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7395 u8 *desc_buf, int len)
7396{
7397 u32 icc_level = 0;
7398
7399 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7400 !hba->vreg_info.vccq2) {
71bb9ab6
AH		 7401 /*
7402 * Using dev_dbg to avoid messages during runtime PM to avoid
7403 * never-ending cycles of messages written back to storage by
7404 * user space causing runtime resume, causing more messages and
7405 * so on.
7406 */
7407 dev_dbg(hba->dev,
3a4bf06d
YG		 7408 "%s: Regulator capability was not set, actvIccLevel=%d",
7409 __func__, icc_level);
7410 goto out;
7411 }
7412
0873045f 7413 if (hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 7414 icc_level = ufshcd_get_max_icc_level(
7415 hba->vreg_info.vcc->max_uA,
7416 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7417 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7418
0873045f 7419 if (hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 7420 icc_level = ufshcd_get_max_icc_level(
7421 hba->vreg_info.vccq->max_uA,
7422 icc_level,
7423 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7424
0873045f 7425 if (hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 7426 icc_level = ufshcd_get_max_icc_level(
7427 hba->vreg_info.vccq2->max_uA,
7428 icc_level,
7429 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7430out:
7431 return icc_level;
7432}
7433
e89860f1 7434static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
3a4bf06d
YG		 7435{
7436 int ret;
7a0bf85b 7437 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
bbe21d7a 7438 u8 *desc_buf;
e89860f1 7439 u32 icc_level;
bbe21d7a
KC		 7440
7441 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7442 if (!desc_buf)
7443 return;
3a4bf06d 7444
c4607a09
BH		 7445 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7446 desc_buf, buff_len);
3a4bf06d
YG		 7447 if (ret) {
7448 dev_err(hba->dev,
7449 "%s: Failed reading power descriptor.len = %d ret = %d",
7450 __func__, buff_len, ret);
bbe21d7a 7451 goto out;
3a4bf06d
YG		 7452 }
7453
e89860f1
CG		 7454 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7455 buff_len);
7456 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
3a4bf06d 7457
dbd34a61 7458 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
e89860f1 7459 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
3a4bf06d
YG		 7460
7461 if (ret)
7462 dev_err(hba->dev,
7463 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
e89860f1 7464 __func__, icc_level, ret);
3a4bf06d 7465
bbe21d7a
KC		 7466out:
7467 kfree(desc_buf);
3a4bf06d
YG		 7468}
7469
fb276f77
CG		 7470static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7471{
7472 scsi_autopm_get_device(sdev);
7473 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7474 if (sdev->rpm_autosuspend)
7475 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7476 RPM_AUTOSUSPEND_DELAY_MS);
7477 scsi_autopm_put_device(sdev);
7478}
7479
2a8fa600
SJ		 7480/**
7481 * ufshcd_scsi_add_wlus - Adds required W-LUs
7482 * @hba: per-adapter instance
7483 *
7484 * UFS device specification requires the UFS devices to support 4 well known
7485 * logical units:
7486 * "REPORT_LUNS" (address: 01h)
7487 * "UFS Device" (address: 50h)
7488 * "RPMB" (address: 44h)
7489 * "BOOT" (address: 30h)
7490 * UFS device's power management needs to be controlled by "POWER CONDITION"
7491 * field of SSU (START STOP UNIT) command. But this "power condition" field
7492 * will take effect only when its sent to "UFS device" well known logical unit
7493 * hence we require the scsi_device instance to represent this logical unit in
7494 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 7495 *
2a8fa600
SJ		 7496 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7497 * Block) LU so user space process can control this LU. User space may also
7498 * want to have access to BOOT LU.
8aa29f19 7499 *
2a8fa600
SJ		 7500 * This function adds scsi device instances for each of all well known LUs
7501 * (except "REPORT LUNS" LU).
7502 *
7503 * Returns zero on success (all required W-LUs are added successfully),
7504 * non-zero error value on failure (if failed to add any of the required W-LU).
7505 */
7506static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7507{
7508 int ret = 0;
59830c09 7509 struct scsi_device *sdev_boot, *sdev_rpmb;
2a8fa600 7510
e2106584 7511 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
2a8fa600 7512 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
e2106584
BVA		 7513 if (IS_ERR(hba->ufs_device_wlun)) {
7514 ret = PTR_ERR(hba->ufs_device_wlun);
7515 hba->ufs_device_wlun = NULL;
2a8fa600
SJ		 7516 goto out;
7517 }
e2106584 7518 scsi_device_put(hba->ufs_device_wlun);
2a8fa600 7519
59830c09 7520 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 7521 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
59830c09
BVA		 7522 if (IS_ERR(sdev_rpmb)) {
7523 ret = PTR_ERR(sdev_rpmb);
e2106584 7524 goto remove_ufs_device_wlun;
2a8fa600 7525 }
59830c09
BVA		 7526 ufshcd_blk_pm_runtime_init(sdev_rpmb);
7527 scsi_device_put(sdev_rpmb);
3d21fbde
HK		 7528
7529 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7530 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
fb276f77 7531 if (IS_ERR(sdev_boot)) {
3d21fbde 7532 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
fb276f77
CG		 7533 } else {
7534 ufshcd_blk_pm_runtime_init(sdev_boot);
3d21fbde 7535 scsi_device_put(sdev_boot);
fb276f77 7536 }
2a8fa600
SJ		 7537 goto out;
7538
e2106584
BVA		 7539remove_ufs_device_wlun:
7540 scsi_remove_device(hba->ufs_device_wlun);
2a8fa600
SJ		 7541out:
7542 return ret;
7543}
7544
3d17b9b5
AD		 7545static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7546{
a7f1e69d 7547 struct ufs_dev_info *dev_info = &hba->dev_info;
6f8d5a6a
SC		 7548 u8 lun;
7549 u32 d_lu_wb_buf_alloc;
e8d03813 7550 u32 ext_ufs_feature;
6f8d5a6a 7551
817d7e14
SC		 7552 if (!ufshcd_is_wb_allowed(hba))
7553 return;
f681d107 7554
a7f1e69d
SC		 7555 /*
7556 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7557 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7558 * enabled
7559 */
7560 if (!(dev_info->wspecversion >= 0x310 ||
7561 dev_info->wspecversion == 0x220 ||
7562 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7563 goto wb_disabled;
817d7e14 7564
7a0bf85b
BH		 7565 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7566 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
817d7e14
SC		 7567 goto wb_disabled;
7568
e8d03813
BH		 7569 ext_ufs_feature = get_unaligned_be32(desc_buf +
7570 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
817d7e14 7571
e8d03813 7572 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
817d7e14
SC		 7573 goto wb_disabled;
7574
3d17b9b5 7575 /*
ae1ce1fc
BH		 7576 * WB may be supported but not configured while provisioning. The spec
7577 * says, in dedicated wb buffer mode, a max of 1 lun would have wb
7578 * buffer configured.
3d17b9b5 7579 */
4cd48995 7580 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
3d17b9b5 7581
a7f1e69d 7582 dev_info->b_presrv_uspc_en =
3d17b9b5
AD		 7583 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7584
4cd48995 7585 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
e8d03813
BH		 7586 if (!get_unaligned_be32(desc_buf +
7587 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
6f8d5a6a
SC		 7588 goto wb_disabled;
7589 } else {
7590 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7591 d_lu_wb_buf_alloc = 0;
7592 ufshcd_read_unit_desc_param(hba,
7593 lun,
7594 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7595 (u8 *)&d_lu_wb_buf_alloc,
7596 sizeof(d_lu_wb_buf_alloc));
7597 if (d_lu_wb_buf_alloc) {
a7f1e69d 7598 dev_info->wb_dedicated_lu = lun;
6f8d5a6a
SC		 7599 break;
7600 }
7601 }
817d7e14 7602
6f8d5a6a
SC		 7603 if (!d_lu_wb_buf_alloc)
7604 goto wb_disabled;
7605 }
f681d107
JC		 7606
7607 if (!ufshcd_is_wb_buf_lifetime_available(hba))
7608 goto wb_disabled;
7609
817d7e14
SC		 7610 return;
7611
7612wb_disabled:
7613 hba->caps &= ~UFSHCD_CAP_WB_EN;
7614}
7615
e88e2d32
AA		 7616static void ufshcd_temp_notif_probe(struct ufs_hba *hba, u8 *desc_buf)
7617{
7618 struct ufs_dev_info *dev_info = &hba->dev_info;
7619 u32 ext_ufs_feature;
7620 u8 mask = 0;
7621
7622 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
7623 return;
7624
7625 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7626
7627 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
7628 mask |= MASK_EE_TOO_LOW_TEMP;
7629
7630 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
7631 mask |= MASK_EE_TOO_HIGH_TEMP;
7632
7633 if (mask) {
7634 ufshcd_enable_ee(hba, mask);
7635 ufs_hwmon_probe(hba, mask);
7636 }
7637}
7638
aead21f3
BVA		 7639void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
7640 const struct ufs_dev_quirk *fixups)
817d7e14 7641{
aead21f3 7642 const struct ufs_dev_quirk *f;
817d7e14
SC		 7643 struct ufs_dev_info *dev_info = &hba->dev_info;
7644
8db269a5
SC		 7645 if (!fixups)
7646 return;
7647
7648 for (f = fixups; f->quirk; f++) {
817d7e14
SC		 7649 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7650 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7651 ((dev_info->model &&
7652 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7653 !strcmp(f->model, UFS_ANY_MODEL)))
7654 hba->dev_quirks |= f->quirk;
7655 }
3d17b9b5 7656}
8db269a5 7657EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
3d17b9b5 7658
c28c00ba
SC		 7659static void ufs_fixup_device_setup(struct ufs_hba *hba)
7660{
7661 /* fix by general quirk table */
8db269a5 7662 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
c28c00ba
SC		 7663
7664 /* allow vendors to fix quirks */
7665 ufshcd_vops_fixup_dev_quirks(hba);
7666}
7667
09750066 7668static int ufs_get_device_desc(struct ufs_hba *hba)
c58ab7aa
YG		 7669{
7670 int err;
7671 u8 model_index;
f02bc975 7672 u8 b_ufs_feature_sup;
bbe21d7a 7673 u8 *desc_buf;
09750066 7674 struct ufs_dev_info *dev_info = &hba->dev_info;
4b828fe1 7675
458a45f5 7676 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
bbe21d7a
KC		 7677 if (!desc_buf) {
7678 err = -ENOMEM;
7679 goto out;
7680 }
c58ab7aa 7681
c4607a09 7682 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7a0bf85b 7683 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
c58ab7aa
YG		 7684 if (err) {
7685 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7686 __func__, err);
7687 goto out;
7688 }
7689
7690 /*
7691 * getting vendor (manufacturerID) and Bank Index in big endian
7692 * format
7693 */
09750066 7694 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 7695 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7696
09f17791
CG		 7697 /* getting Specification Version in big endian format */
7698 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7699 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
f02bc975 7700 b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT];
09f17791 7701
c58ab7aa 7702 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
3d17b9b5 7703
f02bc975
DP		 7704 if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION &&
7705 (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) {
41d8a933
DP		 7706 bool hpb_en = false;
7707
f02bc975 7708 ufshpb_get_dev_info(hba, desc_buf);
41d8a933
DP		 7709
7710 if (!ufshpb_is_legacy(hba))
7711 err = ufshcd_query_flag_retry(hba,
7712 UPIU_QUERY_OPCODE_READ_FLAG,
7713 QUERY_FLAG_IDN_HPB_EN, 0,
7714 &hpb_en);
7715
7716 if (ufshpb_is_legacy(hba) || (!err && hpb_en))
7717 dev_info->hpb_enabled = true;
f02bc975
DP		 7718 }
7719
4b828fe1 7720 err = ufshcd_read_string_desc(hba, model_index,
09750066 7721 &dev_info->model, SD_ASCII_STD);
4b828fe1 7722 if (err < 0) {
c58ab7aa
YG		 7723 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7724 __func__, err);
7725 goto out;
7726 }
7727
b294ff3e
AD		 7728 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
7729 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
7730
817d7e14
SC		 7731 ufs_fixup_device_setup(hba);
7732
a7f1e69d 7733 ufshcd_wb_probe(hba, desc_buf);
817d7e14 7734
e88e2d32
AA		 7735 ufshcd_temp_notif_probe(hba, desc_buf);
7736
4b828fe1
TW		 7737 /*
7738 * ufshcd_read_string_desc returns size of the string
7739 * reset the error value
7740 */
7741 err = 0;
c58ab7aa
YG		 7742
7743out:
bbe21d7a 7744 kfree(desc_buf);
c58ab7aa
YG		 7745 return err;
7746}
7747
09750066 7748static void ufs_put_device_desc(struct ufs_hba *hba)
4b828fe1 7749{
09750066
BH		 7750 struct ufs_dev_info *dev_info = &hba->dev_info;
7751
7752 kfree(dev_info->model);
7753 dev_info->model = NULL;
4b828fe1
TW		 7754}
7755
37113106
YG		 7756/**
7757 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7758 * @hba: per-adapter instance
7759 *
7760 * PA_TActivate parameter can be tuned manually if UniPro version is less than
7761 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7762 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7763 * the hibern8 exit latency.
7764 *
7765 * Returns zero on success, non-zero error value on failure.
7766 */
7767static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7768{
7769 int ret = 0;
7770 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7771
7772 ret = ufshcd_dme_peer_get(hba,
7773 UIC_ARG_MIB_SEL(
7774 RX_MIN_ACTIVATETIME_CAPABILITY,
7775 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7776 &peer_rx_min_activatetime);
7777 if (ret)
7778 goto out;
7779
7780 /* make sure proper unit conversion is applied */
7781 tuned_pa_tactivate =
7782 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7783 / PA_TACTIVATE_TIME_UNIT_US);
7784 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7785 tuned_pa_tactivate);
7786
7787out:
7788 return ret;
7789}
7790
7791/**
7792 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7793 * @hba: per-adapter instance
7794 *
7795 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7796 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7797 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7798 * This optimal value can help reduce the hibern8 exit latency.
7799 *
7800 * Returns zero on success, non-zero error value on failure.
7801 */
7802static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7803{
7804 int ret = 0;
7805 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7806 u32 max_hibern8_time, tuned_pa_hibern8time;
7807
7808 ret = ufshcd_dme_get(hba,
7809 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7810 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7811 &local_tx_hibern8_time_cap);
7812 if (ret)
7813 goto out;
7814
7815 ret = ufshcd_dme_peer_get(hba,
7816 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7817 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7818 &peer_rx_hibern8_time_cap);
7819 if (ret)
7820 goto out;
7821
7822 max_hibern8_time = max(local_tx_hibern8_time_cap,
7823 peer_rx_hibern8_time_cap);
7824 /* make sure proper unit conversion is applied */
7825 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7826 / PA_HIBERN8_TIME_UNIT_US);
7827 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7828 tuned_pa_hibern8time);
7829out:
7830 return ret;
7831}
7832
c6a6db43 7833/**
7834 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7835 * less than device PA_TACTIVATE time.
7836 * @hba: per-adapter instance
7837 *
7838 * Some UFS devices require host PA_TACTIVATE to be lower than device
7839 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7840 * for such devices.
7841 *
7842 * Returns zero on success, non-zero error value on failure.
7843 */
7844static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7845{
7846 int ret = 0;
7847 u32 granularity, peer_granularity;
7848 u32 pa_tactivate, peer_pa_tactivate;
7849 u32 pa_tactivate_us, peer_pa_tactivate_us;
7850 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7851
7852 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7853 &granularity);
7854 if (ret)
7855 goto out;
7856
7857 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7858 &peer_granularity);
7859 if (ret)
7860 goto out;
7861
7862 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7863 (granularity > PA_GRANULARITY_MAX_VAL)) {
7864 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7865 __func__, granularity);
7866 return -EINVAL;
7867 }
7868
7869 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7870 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7871 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7872 __func__, peer_granularity);
7873 return -EINVAL;
7874 }
7875
7876 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7877 if (ret)
7878 goto out;
7879
7880 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7881 &peer_pa_tactivate);
7882 if (ret)
7883 goto out;
7884
7885 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7886 peer_pa_tactivate_us = peer_pa_tactivate *
7887 gran_to_us_table[peer_granularity - 1];
7888
9008661e 7889 if (pa_tactivate_us >= peer_pa_tactivate_us) {
c6a6db43 7890 u32 new_peer_pa_tactivate;
7891
7892 new_peer_pa_tactivate = pa_tactivate_us /
7893 gran_to_us_table[peer_granularity - 1];
7894 new_peer_pa_tactivate++;
7895 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7896 new_peer_pa_tactivate);
7897 }
7898
7899out:
7900 return ret;
7901}
7902
09750066 7903static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
37113106
YG		 7904{
7905 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7906 ufshcd_tune_pa_tactivate(hba);
7907 ufshcd_tune_pa_hibern8time(hba);
7908 }
7909
e91ed9e0
CG		 7910 ufshcd_vops_apply_dev_quirks(hba);
7911
37113106
YG		 7912 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7913 /* set 1ms timeout for PA_TACTIVATE */
7914 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43 7915
7916 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7917 ufshcd_quirk_tune_host_pa_tactivate(hba);
37113106
YG		 7918}
7919
ff8e20c6
DR		 7920static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7921{
ff8e20c6
DR		 7922 hba->ufs_stats.hibern8_exit_cnt = 0;
7923 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 7924 hba->req_abort_count = 0;
ff8e20c6
DR		 7925}
7926
731f0621
BH		 7927static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7928{
7929 int err;
7930 size_t buff_len;
7931 u8 *desc_buf;
7932
7a0bf85b 7933 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
731f0621
BH		 7934 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7935 if (!desc_buf) {
7936 err = -ENOMEM;
7937 goto out;
7938 }
7939
c4607a09
BH		 7940 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7941 desc_buf, buff_len);
731f0621
BH		 7942 if (err) {
7943 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7944 __func__, err);
7945 goto out;
7946 }
7947
7948 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7949 hba->dev_info.max_lu_supported = 32;
7950 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7951 hba->dev_info.max_lu_supported = 8;
7952
f02bc975
DP		 7953 if (hba->desc_size[QUERY_DESC_IDN_GEOMETRY] >=
7954 GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS)
7955 ufshpb_get_geo_info(hba, desc_buf);
7956
731f0621
BH		 7957out:
7958 kfree(desc_buf);
7959 return err;
7960}
7961
743b09d8
BVA		 7962struct ufs_ref_clk {
7963 unsigned long freq_hz;
7964 enum ufs_ref_clk_freq val;
7965};
7966
9e1e8a75
SJ		 7967static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7968 {19200000, REF_CLK_FREQ_19_2_MHZ},
7969 {26000000, REF_CLK_FREQ_26_MHZ},
7970 {38400000, REF_CLK_FREQ_38_4_MHZ},
7971 {52000000, REF_CLK_FREQ_52_MHZ},
7972 {0, REF_CLK_FREQ_INVAL},
7973};
7974
7975static enum ufs_ref_clk_freq
7976ufs_get_bref_clk_from_hz(unsigned long freq)
7977{
7978 int i;
7979
7980 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7981 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7982 return ufs_ref_clk_freqs[i].val;
7983
7984 return REF_CLK_FREQ_INVAL;
7985}
7986
7987void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7988{
7989 unsigned long freq;
7990
7991 freq = clk_get_rate(refclk);
7992
7993 hba->dev_ref_clk_freq =
7994 ufs_get_bref_clk_from_hz(freq);
7995
7996 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7997 dev_err(hba->dev,
7998 "invalid ref_clk setting = %ld\n", freq);
7999}
8000
8001static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8002{
8003 int err;
8004 u32 ref_clk;
8005 u32 freq = hba->dev_ref_clk_freq;
8006
8007 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8008 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8009
8010 if (err) {
8011 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8012 err);
8013 goto out;
8014 }
8015
8016 if (ref_clk == freq)
8017 goto out; /* nothing to update */
8018
8019 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8020 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8021
8022 if (err) {
8023 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8024 ufs_ref_clk_freqs[freq].freq_hz);
8025 goto out;
8026 }
8027
8028 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8029 ufs_ref_clk_freqs[freq].freq_hz);
8030
8031out:
8032 return err;
8033}
8034
1b9e2141
BH		 8035static int ufshcd_device_params_init(struct ufs_hba *hba)
8036{
8037 bool flag;
7a0bf85b 8038 int ret, i;
1b9e2141 8039
7a0bf85b
BH		 8040 /* Init device descriptor sizes */
8041 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
8042 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
1b9e2141 8043
731f0621
BH		 8044 /* Init UFS geometry descriptor related parameters */
8045 ret = ufshcd_device_geo_params_init(hba);
8046 if (ret)
8047 goto out;
8048
1b9e2141
BH		 8049 /* Check and apply UFS device quirks */
8050 ret = ufs_get_device_desc(hba);
8051 if (ret) {
8052 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8053 __func__, ret);
8054 goto out;
8055 }
8056
09f17791
CG		 8057 ufshcd_get_ref_clk_gating_wait(hba);
8058
1b9e2141 8059 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
1f34eedf 8060 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
1b9e2141
BH		 8061 hba->dev_info.f_power_on_wp_en = flag;
8062
2b35b2ad
BH		 8063 /* Probe maximum power mode co-supported by both UFS host and device */
8064 if (ufshcd_get_max_pwr_mode(hba))
8065 dev_err(hba->dev,
8066 "%s: Failed getting max supported power mode\n",
8067 __func__);
1b9e2141
BH		 8068out:
8069 return ret;
8070}
8071
8072/**
8073 * ufshcd_add_lus - probe and add UFS logical units
8074 * @hba: per-adapter instance
8075 */
8076static int ufshcd_add_lus(struct ufs_hba *hba)
8077{
8078 int ret;
8079
1b9e2141
BH		 8080 /* Add required well known logical units to scsi mid layer */
8081 ret = ufshcd_scsi_add_wlus(hba);
8082 if (ret)
8083 goto out;
8084
8085 /* Initialize devfreq after UFS device is detected */
8086 if (ufshcd_is_clkscaling_supported(hba)) {
8087 memcpy(&hba->clk_scaling.saved_pwr_info.info,
8088 &hba->pwr_info,
8089 sizeof(struct ufs_pa_layer_attr));
8090 hba->clk_scaling.saved_pwr_info.is_valid = true;
1b9e2141 8091 hba->clk_scaling.is_allowed = true;
1b9e2141 8092
b058fa86
SC		 8093 ret = ufshcd_devfreq_init(hba);
8094 if (ret)
8095 goto out;
8096
8097 hba->clk_scaling.is_enabled = true;
8098 ufshcd_init_clk_scaling_sysfs(hba);
1b9e2141
BH		 8099 }
8100
8101 ufs_bsg_probe(hba);
f02bc975 8102 ufshpb_init(hba);
1b9e2141
BH		 8103 scsi_scan_host(hba->host);
8104 pm_runtime_put_sync(hba->dev);
8105
1b9e2141
BH		 8106out:
8107 return ret;
8108}
8109
6ccf44fe 8110/**
568dd995 8111 * ufshcd_probe_hba - probe hba to detect device and initialize it
1d337ec2 8112 * @hba: per-adapter instance
568dd995 8113 * @init_dev_params: whether or not to call ufshcd_device_params_init().
1d337ec2
SRT		 8114 *
8115 * Execute link-startup and verify device initialization
6ccf44fe 8116 */
568dd995 8117static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
6ccf44fe 8118{
6ccf44fe 8119 int ret;
4db7a236 8120 unsigned long flags;
7ff5ab47 8121 ktime_t start = ktime_get();
6ccf44fe 8122
aa53f580
CG		 8123 hba->ufshcd_state = UFSHCD_STATE_RESET;
8124
6ccf44fe 8125 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 8126 if (ret)
8127 goto out;
8128
10fb4f87 8129 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8130 goto out;
8131
ff8e20c6
DR		 8132 /* Debug counters initialization */
8133 ufshcd_clear_dbg_ufs_stats(hba);
8134
57d104c1
SJ		 8135 /* UniPro link is active now */
8136 ufshcd_set_link_active(hba);
d3e89bac 8137
1b9e2141 8138 /* Verify device initialization by sending NOP OUT UPIU */
5a0b0cb9
SRT		 8139 ret = ufshcd_verify_dev_init(hba);
8140 if (ret)
8141 goto out;
68078d5c 8142
1b9e2141 8143 /* Initiate UFS initialization, and waiting until completion */
68078d5c
DR		 8144 ret = ufshcd_complete_dev_init(hba);
8145 if (ret)
8146 goto out;
5a0b0cb9 8147
1b9e2141
BH		 8148 /*
8149 * Initialize UFS device parameters used by driver, these
8150 * parameters are associated with UFS descriptors.
8151 */
568dd995 8152 if (init_dev_params) {
1b9e2141
BH		 8153 ret = ufshcd_device_params_init(hba);
8154 if (ret)
8155 goto out;
93fdd5ac
TW		 8156 }
8157
09750066 8158 ufshcd_tune_unipro_params(hba);
4b828fe1 8159
57d104c1
SJ		 8160 /* UFS device is also active now */
8161 ufshcd_set_ufs_dev_active(hba);
66ec6d59 8162 ufshcd_force_reset_auto_bkops(hba);
57d104c1 8163
2b35b2ad
BH		 8164 /* Gear up to HS gear if supported */
8165 if (hba->max_pwr_info.is_valid) {
9e1e8a75
SJ		 8166 /*
8167 * Set the right value to bRefClkFreq before attempting to
8168 * switch to HS gears.
8169 */
8170 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8171 ufshcd_set_dev_ref_clk(hba);
7eb584db 8172 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 8173 if (ret) {
7eb584db
DR		 8174 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8175 __func__, ret);
8643ae66
DL		 8176 goto out;
8177 }
6a9df818 8178 ufshcd_print_pwr_info(hba);
7eb584db 8179 }
57d104c1 8180
e89860f1
CG		 8181 /*
8182 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8183 * and for removable UFS card as well, hence always set the parameter.
8184 * Note: Error handler may issue the device reset hence resetting
8185 * bActiveICCLevel as well so it is always safe to set this here.
8186 */
8187 ufshcd_set_active_icc_lvl(hba);
8188
3d17b9b5 8189 ufshcd_wb_config(hba);
cd469475
AH		 8190 if (hba->ee_usr_mask)
8191 ufshcd_write_ee_control(hba);
71d848b8
CG		 8192 /* Enable Auto-Hibernate if configured */
8193 ufshcd_auto_hibern8_enable(hba);
8194
f02bc975 8195 ufshpb_reset(hba);
5a0b0cb9 8196out:
4db7a236
CG		 8197 spin_lock_irqsave(hba->host->host_lock, flags);
8198 if (ret)
8199 hba->ufshcd_state = UFSHCD_STATE_ERROR;
8200 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8201 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8202 spin_unlock_irqrestore(hba->host->host_lock, flags);
1d337ec2 8203
7ff5ab47 8204 trace_ufshcd_init(dev_name(hba->dev), ret,
8205 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8206 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 8207 return ret;
8208}
8209
8210/**
8211 * ufshcd_async_scan - asynchronous execution for probing hba
8212 * @data: data pointer to pass to this function
8213 * @cookie: cookie data
8214 */
8215static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8216{
8217 struct ufs_hba *hba = (struct ufs_hba *)data;
1b9e2141 8218 int ret;
1d337ec2 8219
9cd20d3f 8220 down(&hba->host_sem);
1b9e2141
BH		 8221 /* Initialize hba, detect and initialize UFS device */
8222 ret = ufshcd_probe_hba(hba, true);
9cd20d3f 8223 up(&hba->host_sem);
1b9e2141
BH		 8224 if (ret)
8225 goto out;
8226
8227 /* Probe and add UFS logical units */
8228 ret = ufshcd_add_lus(hba);
8229out:
8230 /*
8231 * If we failed to initialize the device or the device is not
8232 * present, turn off the power/clocks etc.
8233 */
8234 if (ret) {
8235 pm_runtime_put_sync(hba->dev);
1b9e2141
BH		 8236 ufshcd_hba_exit(hba);
8237 }
6ccf44fe
SJ		 8238}
8239
d829fc8a
SN		 8240static const struct attribute_group *ufshcd_driver_groups[] = {
8241 &ufs_sysfs_unit_descriptor_group,
ec92b59c 8242 &ufs_sysfs_lun_attributes_group,
f02bc975
DP		 8243#ifdef CONFIG_SCSI_UFS_HPB
8244 &ufs_sysfs_hpb_stat_group,
41d8a933 8245 &ufs_sysfs_hpb_param_group,
f02bc975 8246#endif
d829fc8a
SN		 8247 NULL,
8248};
8249
90b8491c
SC		 8250static struct ufs_hba_variant_params ufs_hba_vps = {
8251 .hba_enable_delay_us = 1000,
d14734ae 8252 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
90b8491c
SC		 8253 .devfreq_profile.polling_ms = 100,
8254 .devfreq_profile.target = ufshcd_devfreq_target,
8255 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8256 .ondemand_data.upthreshold = 70,
8257 .ondemand_data.downdifferential = 5,
8258};
8259
7a3e97b0
SY		 8260static struct scsi_host_template ufshcd_driver_template = {
8261 .module = THIS_MODULE,
8262 .name = UFSHCD,
8263 .proc_name = UFSHCD,
eaab9b57 8264 .map_queues = ufshcd_map_queues,
7a3e97b0 8265 .queuecommand = ufshcd_queuecommand,
eaab9b57 8266 .mq_poll = ufshcd_poll,
7a3e97b0 8267 .slave_alloc = ufshcd_slave_alloc,
eeda4749 8268 .slave_configure = ufshcd_slave_configure,
7a3e97b0 8269 .slave_destroy = ufshcd_slave_destroy,
4264fd61 8270 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 8271 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 8272 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8273 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
7a3e97b0
SY		 8274 .this_id = -1,
8275 .sg_tablesize = SG_ALL,
8276 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
8277 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 8278 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 8279 .max_host_blocked = 1,
c40ecc12 8280 .track_queue_depth = 1,
d829fc8a 8281 .sdev_groups = ufshcd_driver_groups,
4af14d11 8282 .dma_boundary = PAGE_SIZE - 1,
49615ba1 8283 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 8284};
8285
57d104c1
SJ		 8286static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8287 int ua)
8288{
7b16a07c 8289 int ret;
57d104c1 8290
7b16a07c
BA		 8291 if (!vreg)
8292 return 0;
57d104c1 8293
0487fff7
SC		 8294 /*
8295 * "set_load" operation shall be required on those regulators
8296 * which specifically configured current limitation. Otherwise
8297 * zero max_uA may cause unexpected behavior when regulator is
8298 * enabled or set as high power mode.
8299 */
8300 if (!vreg->max_uA)
8301 return 0;
8302
7b16a07c
BA		 8303 ret = regulator_set_load(vreg->reg, ua);
8304 if (ret < 0) {
8305 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
8306 __func__, vreg->name, ua, ret);
57d104c1
SJ		 8307 }
8308
8309 return ret;
8310}
8311
8312static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
8313 struct ufs_vreg *vreg)
8314{
73067981 8315 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 8316}
8317
8318static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
8319 struct ufs_vreg *vreg)
8320{
7c7cfdcf
AH		 8321 if (!vreg)
8322 return 0;
8323
73067981 8324 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 8325}
8326
aa497613
SRT		 8327static int ufshcd_config_vreg(struct device *dev,
8328 struct ufs_vreg *vreg, bool on)
8329{
9474c64e
BVA		 8330 if (regulator_count_voltages(vreg->reg) <= 0)
8331 return 0;
90d88f47 8332
9474c64e 8333 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
aa497613
SRT		 8334}
8335
8336static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
8337{
8338 int ret = 0;
8339
73067981 8340 if (!vreg || vreg->enabled)
aa497613
SRT		 8341 goto out;
8342
8343 ret = ufshcd_config_vreg(dev, vreg, true);
8344 if (!ret)
8345 ret = regulator_enable(vreg->reg);
8346
8347 if (!ret)
8348 vreg->enabled = true;
8349 else
8350 dev_err(dev, "%s: %s enable failed, err=%d\n",
8351 __func__, vreg->name, ret);
8352out:
8353 return ret;
8354}
8355
8356static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
8357{
8358 int ret = 0;
8359
f8162ac7 8360 if (!vreg || !vreg->enabled || vreg->always_on)
aa497613
SRT		 8361 goto out;
8362
8363 ret = regulator_disable(vreg->reg);
8364
8365 if (!ret) {
8366 /* ignore errors on applying disable config */
8367 ufshcd_config_vreg(dev, vreg, false);
8368 vreg->enabled = false;
8369 } else {
8370 dev_err(dev, "%s: %s disable failed, err=%d\n",
8371 __func__, vreg->name, ret);
8372 }
8373out:
8374 return ret;
8375}
8376
8377static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
8378{
8379 int ret = 0;
8380 struct device *dev = hba->dev;
8381 struct ufs_vreg_info *info = &hba->vreg_info;
8382
aa497613
SRT		 8383 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
8384 if (ret)
8385 goto out;
8386
8387 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
8388 if (ret)
8389 goto out;
8390
8391 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
aa497613
SRT		 8392
8393out:
8394 if (ret) {
8395 ufshcd_toggle_vreg(dev, info->vccq2, false);
8396 ufshcd_toggle_vreg(dev, info->vccq, false);
8397 ufshcd_toggle_vreg(dev, info->vcc, false);
8398 }
8399 return ret;
8400}
8401
6a771a65
RS		 8402static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8403{
8404 struct ufs_vreg_info *info = &hba->vreg_info;
8405
60b7b823 8406 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 8407}
8408
aa497613
SRT		 8409static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8410{
8411 int ret = 0;
8412
8413 if (!vreg)
8414 goto out;
8415
8416 vreg->reg = devm_regulator_get(dev, vreg->name);
8417 if (IS_ERR(vreg->reg)) {
8418 ret = PTR_ERR(vreg->reg);
8419 dev_err(dev, "%s: %s get failed, err=%d\n",
8420 __func__, vreg->name, ret);
8421 }
8422out:
8423 return ret;
8424}
8425
8426static int ufshcd_init_vreg(struct ufs_hba *hba)
8427{
8428 int ret = 0;
8429 struct device *dev = hba->dev;
8430 struct ufs_vreg_info *info = &hba->vreg_info;
8431
aa497613
SRT		 8432 ret = ufshcd_get_vreg(dev, info->vcc);
8433 if (ret)
8434 goto out;
8435
8436 ret = ufshcd_get_vreg(dev, info->vccq);
b0008625
BH		 8437 if (!ret)
8438 ret = ufshcd_get_vreg(dev, info->vccq2);
aa497613
SRT		 8439out:
8440 return ret;
8441}
8442
6a771a65
RS		 8443static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8444{
8445 struct ufs_vreg_info *info = &hba->vreg_info;
8446
8447 if (info)
8448 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8449
8450 return 0;
8451}
8452
81309c24 8453static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
c6e79dac
SRT		 8454{
8455 int ret = 0;
8456 struct ufs_clk_info *clki;
8457 struct list_head *head = &hba->clk_list_head;
1ab27c9c 8458 unsigned long flags;
911a0771 8459 ktime_t start = ktime_get();
8460 bool clk_state_changed = false;
c6e79dac 8461
566ec9ad 8462 if (list_empty(head))
c6e79dac
SRT		 8463 goto out;
8464
38f3242e
CG		 8465 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8466 if (ret)
8467 return ret;
1e879e8f 8468
c6e79dac
SRT		 8469 list_for_each_entry(clki, head, list) {
8470 if (!IS_ERR_OR_NULL(clki->clk)) {
81309c24
CG		 8471 /*
8472 * Don't disable clocks which are needed
8473 * to keep the link active.
8474 */
8475 if (ufshcd_is_link_active(hba) &&
8476 clki->keep_link_active)
57d104c1
SJ		 8477 continue;
8478
911a0771 8479 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 8480 if (on && !clki->enabled) {
8481 ret = clk_prepare_enable(clki->clk);
8482 if (ret) {
8483 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8484 __func__, clki->name, ret);
8485 goto out;
8486 }
8487 } else if (!on && clki->enabled) {
8488 clk_disable_unprepare(clki->clk);
8489 }
8490 clki->enabled = on;
8491 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8492 clki->name, on ? "en" : "dis");
8493 }
8494 }
1ab27c9c 8495
38f3242e
CG		 8496 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8497 if (ret)
8498 return ret;
1e879e8f 8499
c6e79dac
SRT		 8500out:
8501 if (ret) {
8502 list_for_each_entry(clki, head, list) {
8503 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8504 clk_disable_unprepare(clki->clk);
8505 }
7ff5ab47 8506 } else if (!ret && on) {
1ab27c9c
ST		 8507 spin_lock_irqsave(hba->host->host_lock, flags);
8508 hba->clk_gating.state = CLKS_ON;
7ff5ab47 8509 trace_ufshcd_clk_gating(dev_name(hba->dev),
8510 hba->clk_gating.state);
1ab27c9c 8511 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 8512 }
7ff5ab47 8513
911a0771 8514 if (clk_state_changed)
8515 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8516 (on ? "on" : "off"),
8517 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 8518 return ret;
8519}
8520
8521static int ufshcd_init_clocks(struct ufs_hba *hba)
8522{
8523 int ret = 0;
8524 struct ufs_clk_info *clki;
8525 struct device *dev = hba->dev;
8526 struct list_head *head = &hba->clk_list_head;
8527
566ec9ad 8528 if (list_empty(head))
c6e79dac
SRT		 8529 goto out;
8530
8531 list_for_each_entry(clki, head, list) {
8532 if (!clki->name)
8533 continue;
8534
8535 clki->clk = devm_clk_get(dev, clki->name);
8536 if (IS_ERR(clki->clk)) {
8537 ret = PTR_ERR(clki->clk);
8538 dev_err(dev, "%s: %s clk get failed, %d\n",
8539 __func__, clki->name, ret);
8540 goto out;
8541 }
8542
9e1e8a75
SJ		 8543 /*
8544 * Parse device ref clk freq as per device tree "ref_clk".
8545 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8546 * in ufshcd_alloc_host().
8547 */
8548 if (!strcmp(clki->name, "ref_clk"))
8549 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8550
c6e79dac
SRT		 8551 if (clki->max_freq) {
8552 ret = clk_set_rate(clki->clk, clki->max_freq);
8553 if (ret) {
8554 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8555 __func__, clki->name,
8556 clki->max_freq, ret);
8557 goto out;
8558 }
856b3483 8559 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 8560 }
8561 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8562 clki->name, clk_get_rate(clki->clk));
8563 }
8564out:
8565 return ret;
8566}
8567
5c0c28a8
SRT		 8568static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8569{
8570 int err = 0;
8571
8572 if (!hba->vops)
8573 goto out;
8574
0263bcd0 8575 err = ufshcd_vops_init(hba);
5c0c28a8
SRT		 8576 if (err)
8577 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 8578 __func__, ufshcd_get_var_name(hba), err);
ade921a8 8579out:
5c0c28a8
SRT		 8580 return err;
8581}
8582
8583static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8584{
8585 if (!hba->vops)
8586 return;
8587
0263bcd0 8588 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 8589}
8590
aa497613
SRT		 8591static int ufshcd_hba_init(struct ufs_hba *hba)
8592{
8593 int err;
8594
6a771a65
RS		 8595 /*
8596 * Handle host controller power separately from the UFS device power
8597 * rails as it will help controlling the UFS host controller power
8598 * collapse easily which is different than UFS device power collapse.
8599 * Also, enable the host controller power before we go ahead with rest
8600 * of the initialization here.
8601 */
8602 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 8603 if (err)
8604 goto out;
8605
6a771a65 8606 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 8607 if (err)
8608 goto out;
8609
6a771a65
RS		 8610 err = ufshcd_init_clocks(hba);
8611 if (err)
8612 goto out_disable_hba_vreg;
8613
8614 err = ufshcd_setup_clocks(hba, true);
8615 if (err)
8616 goto out_disable_hba_vreg;
8617
c6e79dac
SRT		 8618 err = ufshcd_init_vreg(hba);
8619 if (err)
8620 goto out_disable_clks;
8621
8622 err = ufshcd_setup_vreg(hba, true);
8623 if (err)
8624 goto out_disable_clks;
8625
aa497613
SRT		 8626 err = ufshcd_variant_hba_init(hba);
8627 if (err)
8628 goto out_disable_vreg;
8629
b6cacaf2
AH		 8630 ufs_debugfs_hba_init(hba);
8631
1d337ec2 8632 hba->is_powered = true;
aa497613
SRT		 8633 goto out;
8634
8635out_disable_vreg:
8636 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 8637out_disable_clks:
8638 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 8639out_disable_hba_vreg:
8640 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 8641out:
8642 return err;
8643}
8644
8645static void ufshcd_hba_exit(struct ufs_hba *hba)
8646{
1d337ec2 8647 if (hba->is_powered) {
4543d9d7
CG		 8648 ufshcd_exit_clk_scaling(hba);
8649 ufshcd_exit_clk_gating(hba);
88b09900
AH		 8650 if (hba->eh_wq)
8651 destroy_workqueue(hba->eh_wq);
b6cacaf2 8652 ufs_debugfs_hba_exit(hba);
1d337ec2
SRT		 8653 ufshcd_variant_hba_exit(hba);
8654 ufshcd_setup_vreg(hba, false);
8655 ufshcd_setup_clocks(hba, false);
8656 ufshcd_setup_hba_vreg(hba, false);
8657 hba->is_powered = false;
09750066 8658 ufs_put_device_desc(hba);
1d337ec2 8659 }
aa497613
SRT		 8660}
8661
57d104c1
SJ		 8662/**
8663 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8664 * power mode
8665 * @hba: per adapter instance
8666 * @pwr_mode: device power mode to set
8667 *
8668 * Returns 0 if requested power mode is set successfully
ad6c8a42 8669 * Returns < 0 if failed to set the requested power mode
57d104c1
SJ		 8670 */
8671static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8672 enum ufs_dev_pwr_mode pwr_mode)
8673{
8674 unsigned char cmd[6] = { START_STOP };
8675 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 8676 struct scsi_device *sdp;
8677 unsigned long flags;
af21c3fd 8678 int ret, retries;
57d104c1 8679
7c48bfd0 8680 spin_lock_irqsave(hba->host->host_lock, flags);
e2106584 8681 sdp = hba->ufs_device_wlun;
7c48bfd0
AM		 8682 if (sdp) {
8683 ret = scsi_device_get(sdp);
8684 if (!ret && !scsi_device_online(sdp)) {
8685 ret = -ENODEV;
8686 scsi_device_put(sdp);
8687 }
8688 } else {
8689 ret = -ENODEV;
8690 }
8691 spin_unlock_irqrestore(hba->host->host_lock, flags);
8692
8693 if (ret)
8694 return ret;
57d104c1
SJ		 8695
8696 /*
8697 * If scsi commands fail, the scsi mid-layer schedules scsi error-
8698 * handling, which would wait for host to be resumed. Since we know
8699 * we are functional while we are here, skip host resume in error
8700 * handling context.
8701 */
8702 hba->host->eh_noresume = 1;
57d104c1
SJ		 8703
8704 cmd[4] = pwr_mode << 4;
8705
8706 /*
8707 * Current function would be generally called from the power management
e8064021 8708 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 8709 * already suspended childs.
8710 */
af21c3fd
JK		 8711 for (retries = 3; retries > 0; --retries) {
8712 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8713 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
8714 if (!scsi_status_is_check_condition(ret) ||
8715 !scsi_sense_valid(&sshdr) ||
8716 sshdr.sense_key != UNIT_ATTENTION)
8717 break;
8718 }
57d104c1
SJ		 8719 if (ret) {
8720 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 8721 "START_STOP failed for power mode: %d, result %x\n",
8722 pwr_mode, ret);
ad6c8a42
KK		 8723 if (ret > 0) {
8724 if (scsi_sense_valid(&sshdr))
8725 scsi_print_sense_hdr(sdp, NULL, &sshdr);
8726 ret = -EIO;
8727 }
57d104c1
SJ		 8728 }
8729
8730 if (!ret)
8731 hba->curr_dev_pwr_mode = pwr_mode;
1918651f 8732
7c48bfd0 8733 scsi_device_put(sdp);
57d104c1
SJ		 8734 hba->host->eh_noresume = 0;
8735 return ret;
8736}
8737
8738static int ufshcd_link_state_transition(struct ufs_hba *hba,
8739 enum uic_link_state req_link_state,
8740 int check_for_bkops)
8741{
8742 int ret = 0;
8743
8744 if (req_link_state == hba->uic_link_state)
8745 return 0;
8746
8747 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8748 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236 8749 if (!ret) {
57d104c1 8750 ufshcd_set_link_hibern8(hba);
4db7a236
CG		 8751 } else {
8752 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8753 __func__, ret);
57d104c1 8754 goto out;
4db7a236 8755 }
57d104c1
SJ		 8756 }
8757 /*
8758 * If autobkops is enabled, link can't be turned off because
fe1d4c2e
AH		 8759 * turning off the link would also turn off the device, except in the
8760 * case of DeepSleep where the device is expected to remain powered.
57d104c1
SJ		 8761 */
8762 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
dc30c9e6 8763 (!check_for_bkops || !hba->auto_bkops_enabled)) {
f3099fbd
YG		 8764 /*
8765 * Let's make sure that link is in low power mode, we are doing
8766 * this currently by putting the link in Hibern8. Otherway to
8767 * put the link in low power mode is to send the DME end point
8768 * to device and then send the DME reset command to local
8769 * unipro. But putting the link in hibern8 is much faster.
fe1d4c2e
AH		 8770 *
8771 * Note also that putting the link in Hibern8 is a requirement
8772 * for entering DeepSleep.
f3099fbd
YG		 8773 */
8774 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236
CG		 8775 if (ret) {
8776 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8777 __func__, ret);
f3099fbd 8778 goto out;
4db7a236 8779 }
57d104c1
SJ		 8780 /*
8781 * Change controller state to "reset state" which
8782 * should also put the link in off/reset state
8783 */
5cac1095 8784 ufshcd_hba_stop(hba);
57d104c1
SJ		 8785 /*
8786 * TODO: Check if we need any delay to make sure that
8787 * controller is reset
8788 */
8789 ufshcd_set_link_off(hba);
8790 }
8791
8792out:
8793 return ret;
8794}
8795
8796static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8797{
c4df6eed
SC		 8798 bool vcc_off = false;
8799
b799fdf7
YG		 8800 /*
8801 * It seems some UFS devices may keep drawing more than sleep current
8802 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8803 * To avoid this situation, add 2ms delay before putting these UFS
8804 * rails in LPM mode.
8805 */
8806 if (!ufshcd_is_link_active(hba) &&
8807 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8808 usleep_range(2000, 2100);
8809
57d104c1
SJ		 8810 /*
8811 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8812 * power.
8813 *
8814 * If UFS device and link is in OFF state, all power supplies (VCC,
8815 * VCCQ, VCCQ2) can be turned off if power on write protect is not
8816 * required. If UFS link is inactive (Hibern8 or OFF state) and device
8817 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8818 *
8819 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8820 * in low power state which would save some power.
3d17b9b5
AD		 8821 *
8822 * If Write Booster is enabled and the device needs to flush the WB
8823 * buffer OR if bkops status is urgent for WB, keep Vcc on.
57d104c1
SJ		 8824 */
8825 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8826 !hba->dev_info.is_lu_power_on_wp) {
8827 ufshcd_setup_vreg(hba, false);
c4df6eed 8828 vcc_off = true;
57d104c1 8829 } else if (!ufshcd_is_ufs_dev_active(hba)) {
51dd905b 8830 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
c4df6eed 8831 vcc_off = true;
23043dd8 8832 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
57d104c1
SJ		 8833 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8834 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8835 }
8836 }
c4df6eed
SC		 8837
8838 /*
8839 * Some UFS devices require delay after VCC power rail is turned-off.
8840 */
8841 if (vcc_off && hba->vreg_info.vcc &&
8842 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8843 usleep_range(5000, 5100);
57d104c1
SJ		 8844}
8845
9bb25e5d 8846#ifdef CONFIG_PM
57d104c1
SJ		 8847static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8848{
8849 int ret = 0;
8850
8851 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8852 !hba->dev_info.is_lu_power_on_wp) {
8853 ret = ufshcd_setup_vreg(hba, true);
8854 } else if (!ufshcd_is_ufs_dev_active(hba)) {
23043dd8 8855 if (!ufshcd_is_link_active(hba)) {
57d104c1
SJ		 8856 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8857 if (ret)
8858 goto vcc_disable;
8859 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8860 if (ret)
8861 goto vccq_lpm;
8862 }
69d72ac8 8863 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 8864 }
8865 goto out;
8866
8867vccq_lpm:
8868 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8869vcc_disable:
8870 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8871out:
8872 return ret;
8873}
9bb25e5d 8874#endif /* CONFIG_PM */
57d104c1
SJ		 8875
8876static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8877{
dd7143e2 8878 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
57d104c1
SJ		 8879 ufshcd_setup_hba_vreg(hba, false);
8880}
8881
8882static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8883{
dd7143e2 8884 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
57d104c1
SJ		 8885 ufshcd_setup_hba_vreg(hba, true);
8886}
8887
b294ff3e 8888static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8889{
57d104c1 8890 int ret = 0;
fe1d4c2e 8891 int check_for_bkops;
57d104c1
SJ		 8892 enum ufs_pm_level pm_lvl;
8893 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8894 enum uic_link_state req_link_state;
8895
b294ff3e 8896 hba->pm_op_in_progress = true;
4c6cb9ed
BVA		 8897 if (pm_op != UFS_SHUTDOWN_PM) {
8898 pm_lvl = pm_op == UFS_RUNTIME_PM ?
57d104c1
SJ		 8899 hba->rpm_lvl : hba->spm_lvl;
8900 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8901 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8902 } else {
8903 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8904 req_link_state = UIC_LINK_OFF_STATE;
8905 }
8906
f02bc975
DP		 8907 ufshpb_suspend(hba);
8908
7a3e97b0 8909 /*
57d104c1
SJ		 8910 * If we can't transition into any of the low power modes
8911 * just gate the clocks.
7a3e97b0 8912 */
1ab27c9c
ST		 8913 ufshcd_hold(hba, false);
8914 hba->clk_gating.is_suspended = true;
8915
348e1bc5
SC		 8916 if (ufshcd_is_clkscaling_supported(hba))
8917 ufshcd_clk_scaling_suspend(hba, true);
d6fcf81a 8918
57d104c1
SJ		 8919 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8920 req_link_state == UIC_LINK_ACTIVE_STATE) {
b294ff3e 8921 goto vops_suspend;
57d104c1 8922 }
7a3e97b0 8923
57d104c1
SJ		 8924 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8925 (req_link_state == hba->uic_link_state))
b294ff3e 8926 goto enable_scaling;
57d104c1
SJ		 8927
8928 /* UFS device & link must be active before we enter in this function */
8929 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8930 ret = -EINVAL;
b294ff3e 8931 goto enable_scaling;
57d104c1
SJ		 8932 }
8933
4c6cb9ed 8934 if (pm_op == UFS_RUNTIME_PM) {
374a246e
SJ		 8935 if (ufshcd_can_autobkops_during_suspend(hba)) {
8936 /*
8937 * The device is idle with no requests in the queue,
8938 * allow background operations if bkops status shows
8939 * that performance might be impacted.
8940 */
8941 ret = ufshcd_urgent_bkops(hba);
8942 if (ret)
b294ff3e 8943 goto enable_scaling;
374a246e
SJ		 8944 } else {
8945 /* make sure that auto bkops is disabled */
8946 ufshcd_disable_auto_bkops(hba);
8947 }
3d17b9b5 8948 /*
51dd905b
SC		 8949 * If device needs to do BKOP or WB buffer flush during
8950 * Hibern8, keep device power mode as "active power mode"
8951 * and VCC supply.
3d17b9b5 8952 */
51dd905b
SC		 8953 hba->dev_info.b_rpm_dev_flush_capable =
8954 hba->auto_bkops_enabled ||
8955 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8956 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8957 ufshcd_is_auto_hibern8_enabled(hba))) &&
8958 ufshcd_wb_need_flush(hba));
8959 }
8960
6948a96a
KK		 8961 flush_work(&hba->eeh_work);
8962
9561f584
PW		 8963 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
8964 if (ret)
8965 goto enable_scaling;
8966
51dd905b 8967 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
4c6cb9ed 8968 if (pm_op != UFS_RUNTIME_PM)
51dd905b
SC		 8969 /* ensure that bkops is disabled */
8970 ufshcd_disable_auto_bkops(hba);
57d104c1 8971
51dd905b
SC		 8972 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8973 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8974 if (ret)
b294ff3e 8975 goto enable_scaling;
51dd905b 8976 }
57d104c1
SJ		 8977 }
8978
fe1d4c2e
AH		 8979 /*
8980 * In the case of DeepSleep, the device is expected to remain powered
8981 * with the link off, so do not check for bkops.
8982 */
8983 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
8984 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
57d104c1
SJ		 8985 if (ret)
8986 goto set_dev_active;
8987
b294ff3e 8988vops_suspend:
57d104c1
SJ		 8989 /*
8990 * Call vendor specific suspend callback. As these callbacks may access
8991 * vendor specific host controller register space call them before the
8992 * host clocks are ON.
8993 */
9561f584 8994 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
0263bcd0
YG		 8995 if (ret)
8996 goto set_link_active;
57d104c1
SJ		 8997 goto out;
8998
57d104c1 8999set_link_active:
fe1d4c2e
AH		 9000 /*
9001 * Device hardware reset is required to exit DeepSleep. Also, for
9002 * DeepSleep, the link is off so host reset and restore will be done
9003 * further below.
9004 */
9005 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
31a5d9ca 9006 ufshcd_device_reset(hba);
fe1d4c2e
AH		 9007 WARN_ON(!ufshcd_is_link_off(hba));
9008 }
57d104c1
SJ		 9009 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9010 ufshcd_set_link_active(hba);
9011 else if (ufshcd_is_link_off(hba))
9012 ufshcd_host_reset_and_restore(hba);
9013set_dev_active:
fe1d4c2e
AH		 9014 /* Can also get here needing to exit DeepSleep */
9015 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
31a5d9ca 9016 ufshcd_device_reset(hba);
fe1d4c2e
AH		 9017 ufshcd_host_reset_and_restore(hba);
9018 }
57d104c1
SJ		 9019 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9020 ufshcd_disable_auto_bkops(hba);
b294ff3e 9021enable_scaling:
348e1bc5
SC		 9022 if (ufshcd_is_clkscaling_supported(hba))
9023 ufshcd_clk_scaling_suspend(hba, false);
9024
51dd905b 9025 hba->dev_info.b_rpm_dev_flush_capable = false;
57d104c1 9026out:
51dd905b
SC		 9027 if (hba->dev_info.b_rpm_dev_flush_capable) {
9028 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9029 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9030 }
9031
b294ff3e
AD		 9032 if (ret) {
9033 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9034 hba->clk_gating.is_suspended = false;
9035 ufshcd_release(hba);
f02bc975 9036 ufshpb_resume(hba);
b294ff3e
AD		 9037 }
9038 hba->pm_op_in_progress = false;
57d104c1 9039 return ret;
7a3e97b0
SY		 9040}
9041
75d645a6 9042#ifdef CONFIG_PM
b294ff3e 9043static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 9044{
57d104c1 9045 int ret;
b294ff3e 9046 enum uic_link_state old_link_state = hba->uic_link_state;
57d104c1 9047
b294ff3e 9048 hba->pm_op_in_progress = true;
57d104c1 9049
7a3e97b0 9050 /*
57d104c1
SJ		 9051 * Call vendor specific resume callback. As these callbacks may access
9052 * vendor specific host controller register space call them when the
9053 * host clocks are ON.
7a3e97b0 9054 */
0263bcd0
YG		 9055 ret = ufshcd_vops_resume(hba, pm_op);
9056 if (ret)
b294ff3e 9057 goto out;
57d104c1 9058
fe1d4c2e
AH		 9059 /* For DeepSleep, the only supported option is to have the link off */
9060 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9061
57d104c1
SJ		 9062 if (ufshcd_is_link_hibern8(hba)) {
9063 ret = ufshcd_uic_hibern8_exit(hba);
4db7a236 9064 if (!ret) {
57d104c1 9065 ufshcd_set_link_active(hba);
4db7a236
CG		 9066 } else {
9067 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9068 __func__, ret);
57d104c1 9069 goto vendor_suspend;
4db7a236 9070 }
57d104c1 9071 } else if (ufshcd_is_link_off(hba)) {
57d104c1 9072 /*
089f5b64
AD		 9073 * A full initialization of the host and the device is
9074 * required since the link was put to off during suspend.
fe1d4c2e
AH		 9075 * Note, in the case of DeepSleep, the device will exit
9076 * DeepSleep due to device reset.
089f5b64
AD		 9077 */
9078 ret = ufshcd_reset_and_restore(hba);
9079 /*
9080 * ufshcd_reset_and_restore() should have already
57d104c1
SJ		 9081 * set the link state as active
9082 */
9083 if (ret || !ufshcd_is_link_active(hba))
9084 goto vendor_suspend;
9085 }
9086
9087 if (!ufshcd_is_ufs_dev_active(hba)) {
9088 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9089 if (ret)
9090 goto set_old_link_state;
9091 }
9092
4e768e76 9093 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9094 ufshcd_enable_auto_bkops(hba);
9095 else
9096 /*
9097 * If BKOPs operations are urgently needed at this moment then
9098 * keep auto-bkops enabled or else disable it.
9099 */
9100 ufshcd_urgent_bkops(hba);
9101
cd469475
AH		 9102 if (hba->ee_usr_mask)
9103 ufshcd_write_ee_control(hba);
9104
348e1bc5
SC		 9105 if (ufshcd_is_clkscaling_supported(hba))
9106 ufshcd_clk_scaling_suspend(hba, false);
856b3483 9107
51dd905b
SC		 9108 if (hba->dev_info.b_rpm_dev_flush_capable) {
9109 hba->dev_info.b_rpm_dev_flush_capable = false;
9110 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9111 }
9112
b294ff3e
AD		 9113 /* Enable Auto-Hibernate if configured */
9114 ufshcd_auto_hibern8_enable(hba);
f02bc975
DP		 9115
9116 ufshpb_resume(hba);
57d104c1
SJ		 9117 goto out;
9118
9119set_old_link_state:
9120 ufshcd_link_state_transition(hba, old_link_state, 0);
9121vendor_suspend:
9561f584
PW		 9122 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9123 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
b294ff3e
AD		 9124out:
9125 if (ret)
9126 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9127 hba->clk_gating.is_suspended = false;
9128 ufshcd_release(hba);
9129 hba->pm_op_in_progress = false;
9130 return ret;
9131}
9132
9133static int ufshcd_wl_runtime_suspend(struct device *dev)
9134{
9135 struct scsi_device *sdev = to_scsi_device(dev);
9136 struct ufs_hba *hba;
9137 int ret;
9138 ktime_t start = ktime_get();
9139
9140 hba = shost_priv(sdev->host);
9141
9142 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9143 if (ret)
9144 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9145
9146 trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9147 ktime_to_us(ktime_sub(ktime_get(), start)),
9148 hba->curr_dev_pwr_mode, hba->uic_link_state);
9149
9150 return ret;
9151}
9152
9153static int ufshcd_wl_runtime_resume(struct device *dev)
9154{
9155 struct scsi_device *sdev = to_scsi_device(dev);
9156 struct ufs_hba *hba;
9157 int ret = 0;
9158 ktime_t start = ktime_get();
9159
9160 hba = shost_priv(sdev->host);
9161
9162 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9163 if (ret)
9164 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9165
9166 trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9167 ktime_to_us(ktime_sub(ktime_get(), start)),
9168 hba->curr_dev_pwr_mode, hba->uic_link_state);
9169
9170 return ret;
9171}
75d645a6 9172#endif
b294ff3e
AD		 9173
9174#ifdef CONFIG_PM_SLEEP
9175static int ufshcd_wl_suspend(struct device *dev)
9176{
9177 struct scsi_device *sdev = to_scsi_device(dev);
9178 struct ufs_hba *hba;
9179 int ret = 0;
9180 ktime_t start = ktime_get();
9181
9182 hba = shost_priv(sdev->host);
9183 down(&hba->host_sem);
9184
9185 if (pm_runtime_suspended(dev))
9186 goto out;
9187
9188 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9189 if (ret) {
9190 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9191 up(&hba->host_sem);
9192 }
9193
9194out:
9195 if (!ret)
9196 hba->is_sys_suspended = true;
9197 trace_ufshcd_wl_suspend(dev_name(dev), ret,
9198 ktime_to_us(ktime_sub(ktime_get(), start)),
9199 hba->curr_dev_pwr_mode, hba->uic_link_state);
9200
9201 return ret;
9202}
9203
9204static int ufshcd_wl_resume(struct device *dev)
9205{
9206 struct scsi_device *sdev = to_scsi_device(dev);
9207 struct ufs_hba *hba;
9208 int ret = 0;
9209 ktime_t start = ktime_get();
9210
9211 hba = shost_priv(sdev->host);
9212
9213 if (pm_runtime_suspended(dev))
9214 goto out;
9215
9216 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9217 if (ret)
9218 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9219out:
9220 trace_ufshcd_wl_resume(dev_name(dev), ret,
9221 ktime_to_us(ktime_sub(ktime_get(), start)),
9222 hba->curr_dev_pwr_mode, hba->uic_link_state);
9223 if (!ret)
9224 hba->is_sys_suspended = false;
9225 up(&hba->host_sem);
9226 return ret;
9227}
9228#endif
9229
9230static void ufshcd_wl_shutdown(struct device *dev)
9231{
9232 struct scsi_device *sdev = to_scsi_device(dev);
9233 struct ufs_hba *hba;
9234
9235 hba = shost_priv(sdev->host);
9236
9237 down(&hba->host_sem);
9238 hba->shutting_down = true;
9239 up(&hba->host_sem);
9240
9241 /* Turn on everything while shutting down */
9242 ufshcd_rpm_get_sync(hba);
9243 scsi_device_quiesce(sdev);
9244 shost_for_each_device(sdev, hba->host) {
e2106584 9245 if (sdev == hba->ufs_device_wlun)
b294ff3e
AD		 9246 continue;
9247 scsi_device_quiesce(sdev);
9248 }
9249 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9250}
9251
9252/**
9253 * ufshcd_suspend - helper function for suspend operations
9254 * @hba: per adapter instance
9255 *
9256 * This function will put disable irqs, turn off clocks
9257 * and set vreg and hba-vreg in lpm mode.
b294ff3e
AD		 9258 */
9259static int ufshcd_suspend(struct ufs_hba *hba)
9260{
9261 int ret;
9262
9263 if (!hba->is_powered)
9264 return 0;
9265 /*
9266 * Disable the host irq as host controller as there won't be any
9267 * host controller transaction expected till resume.
9268 */
57d104c1 9269 ufshcd_disable_irq(hba);
b294ff3e
AD		 9270 ret = ufshcd_setup_clocks(hba, false);
9271 if (ret) {
9272 ufshcd_enable_irq(hba);
9273 return ret;
9274 }
2dec9475
CG		 9275 if (ufshcd_is_clkgating_allowed(hba)) {
9276 hba->clk_gating.state = CLKS_OFF;
9277 trace_ufshcd_clk_gating(dev_name(hba->dev),
9278 hba->clk_gating.state);
9279 }
b294ff3e
AD		 9280
9281 ufshcd_vreg_set_lpm(hba);
9282 /* Put the host controller in low power mode if possible */
9283 ufshcd_hba_vreg_set_lpm(hba);
9284 return ret;
9285}
9286
9bb25e5d 9287#ifdef CONFIG_PM
b294ff3e
AD		 9288/**
9289 * ufshcd_resume - helper function for resume operations
9290 * @hba: per adapter instance
9291 *
9292 * This function basically turns on the regulators, clocks and
9293 * irqs of the hba.
b294ff3e
AD		 9294 *
9295 * Returns 0 for success and non-zero for failure
9296 */
9297static int ufshcd_resume(struct ufs_hba *hba)
9298{
9299 int ret;
9300
9301 if (!hba->is_powered)
9302 return 0;
9303
9304 ufshcd_hba_vreg_set_hpm(hba);
9305 ret = ufshcd_vreg_set_hpm(hba);
9306 if (ret)
9307 goto out;
9308
9309 /* Make sure clocks are enabled before accessing controller */
9310 ret = ufshcd_setup_clocks(hba, true);
9311 if (ret)
9312 goto disable_vreg;
9313
9314 /* enable the host irq as host controller would be active soon */
9315 ufshcd_enable_irq(hba);
9316 goto out;
9317
528db9e5
ZC		 9318disable_vreg:
9319 ufshcd_vreg_set_lpm(hba);
57d104c1 9320out:
8808b4e9 9321 if (ret)
e965e5e0 9322 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
57d104c1
SJ		 9323 return ret;
9324}
9bb25e5d 9325#endif /* CONFIG_PM */
57d104c1 9326
9bb25e5d 9327#ifdef CONFIG_PM_SLEEP
57d104c1 9328/**
f1ecbe1e
BVA		 9329 * ufshcd_system_suspend - system suspend callback
9330 * @dev: Device associated with the UFS controller.
57d104c1 9331 *
f1ecbe1e
BVA		 9332 * Executed before putting the system into a sleep state in which the contents
9333 * of main memory are preserved.
57d104c1
SJ		 9334 *
9335 * Returns 0 for success and non-zero for failure
9336 */
f1ecbe1e 9337int ufshcd_system_suspend(struct device *dev)
57d104c1 9338{
f1ecbe1e 9339 struct ufs_hba *hba = dev_get_drvdata(dev);
57d104c1 9340 int ret = 0;
7ff5ab47 9341 ktime_t start = ktime_get();
57d104c1 9342
b294ff3e 9343 if (pm_runtime_suspended(hba->dev))
0b257734 9344 goto out;
57d104c1 9345
b294ff3e 9346 ret = ufshcd_suspend(hba);
57d104c1 9347out:
7ff5ab47 9348 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
9349 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9350 hba->curr_dev_pwr_mode, hba->uic_link_state);
57d104c1
SJ		 9351 return ret;
9352}
9353EXPORT_SYMBOL(ufshcd_system_suspend);
9354
9355/**
f1ecbe1e
BVA		 9356 * ufshcd_system_resume - system resume callback
9357 * @dev: Device associated with the UFS controller.
9358 *
9359 * Executed after waking the system up from a sleep state in which the contents
9360 * of main memory were preserved.
57d104c1
SJ		 9361 *
9362 * Returns 0 for success and non-zero for failure
9363 */
f1ecbe1e 9364int ufshcd_system_resume(struct device *dev)
57d104c1 9365{
f1ecbe1e 9366 struct ufs_hba *hba = dev_get_drvdata(dev);
7ff5ab47 9367 ktime_t start = ktime_get();
f1ecbe1e 9368 int ret = 0;
7ff5ab47 9369
b294ff3e 9370 if (pm_runtime_suspended(hba->dev))
7ff5ab47 9371 goto out;
b294ff3e
AD		 9372
9373 ret = ufshcd_resume(hba);
9374
7ff5ab47 9375out:
9376 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
9377 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9378 hba->curr_dev_pwr_mode, hba->uic_link_state);
b294ff3e 9379
7ff5ab47 9380 return ret;
7a3e97b0 9381}
57d104c1 9382EXPORT_SYMBOL(ufshcd_system_resume);
9bb25e5d 9383#endif /* CONFIG_PM_SLEEP */
3b1d0580 9384
9bb25e5d 9385#ifdef CONFIG_PM
57d104c1 9386/**
f1ecbe1e
BVA		 9387 * ufshcd_runtime_suspend - runtime suspend callback
9388 * @dev: Device associated with the UFS controller.
57d104c1
SJ		 9389 *
9390 * Check the description of ufshcd_suspend() function for more details.
9391 *
9392 * Returns 0 for success and non-zero for failure
9393 */
f1ecbe1e 9394int ufshcd_runtime_suspend(struct device *dev)
66ec6d59 9395{
f1ecbe1e 9396 struct ufs_hba *hba = dev_get_drvdata(dev);
b294ff3e 9397 int ret;
7ff5ab47 9398 ktime_t start = ktime_get();
9399
b294ff3e
AD		 9400 ret = ufshcd_suspend(hba);
9401
7ff5ab47 9402 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
9403 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9404 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 9405 return ret;
66ec6d59
SRT		 9406}
9407EXPORT_SYMBOL(ufshcd_runtime_suspend);
9408
57d104c1
SJ		 9409/**
9410 * ufshcd_runtime_resume - runtime resume routine
f1ecbe1e 9411 * @dev: Device associated with the UFS controller.
57d104c1 9412 *
b294ff3e 9413 * This function basically brings controller
57d104c1
SJ		 9414 * to active state. Following operations are done in this function:
9415 *
9416 * 1. Turn on all the controller related clocks
b294ff3e 9417 * 2. Turn ON VCC rail
57d104c1 9418 */
f1ecbe1e 9419int ufshcd_runtime_resume(struct device *dev)
66ec6d59 9420{
f1ecbe1e 9421 struct ufs_hba *hba = dev_get_drvdata(dev);
b294ff3e 9422 int ret;
7ff5ab47 9423 ktime_t start = ktime_get();
9424
b294ff3e
AD		 9425 ret = ufshcd_resume(hba);
9426
7ff5ab47 9427 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
9428 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 9429 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 9430 return ret;
66ec6d59
SRT		 9431}
9432EXPORT_SYMBOL(ufshcd_runtime_resume);
9bb25e5d 9433#endif /* CONFIG_PM */
66ec6d59 9434
57d104c1
SJ		 9435/**
9436 * ufshcd_shutdown - shutdown routine
9437 * @hba: per adapter instance
9438 *
b294ff3e
AD		 9439 * This function would turn off both UFS device and UFS hba
9440 * regulators. It would also disable clocks.
57d104c1
SJ		 9441 *
9442 * Returns 0 always to allow force shutdown even in case of errors.
9443 */
9444int ufshcd_shutdown(struct ufs_hba *hba)
9445{
57d104c1
SJ		 9446 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9447 goto out;
9448
e92643db 9449 pm_runtime_get_sync(hba->dev);
57d104c1 9450
b294ff3e 9451 ufshcd_suspend(hba);
57d104c1 9452out:
88a92d6a 9453 hba->is_powered = false;
57d104c1
SJ		 9454 /* allow force shutdown even in case of errors */
9455 return 0;
9456}
9457EXPORT_SYMBOL(ufshcd_shutdown);
9458
7a3e97b0 9459/**
3b1d0580 9460 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 9461 * data structure memory
8aa29f19 9462 * @hba: per adapter instance
7a3e97b0 9463 */
3b1d0580 9464void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 9465{
e2106584 9466 if (hba->ufs_device_wlun)
b294ff3e 9467 ufshcd_rpm_get_sync(hba);
e88e2d32 9468 ufs_hwmon_remove(hba);
df032bf2 9469 ufs_bsg_remove(hba);
4b5f4907 9470 ufshpb_remove(hba);
cbb6813e 9471 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 9472 blk_cleanup_queue(hba->tmf_queue);
9473 blk_mq_free_tag_set(&hba->tmf_tag_set);
cfdf9c91 9474 scsi_remove_host(hba->host);
7a3e97b0 9475 /* disable interrupts */
2fbd009b 9476 ufshcd_disable_intr(hba, hba->intr_mask);
5cac1095 9477 ufshcd_hba_stop(hba);
aa497613 9478 ufshcd_hba_exit(hba);
3b1d0580
VH		 9479}
9480EXPORT_SYMBOL_GPL(ufshcd_remove);
9481
47555a5c
YG		 9482/**
9483 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9484 * @hba: pointer to Host Bus Adapter (HBA)
9485 */
9486void ufshcd_dealloc_host(struct ufs_hba *hba)
9487{
9488 scsi_host_put(hba->host);
9489}
9490EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9491
ca3d7bf9
AM		 9492/**
9493 * ufshcd_set_dma_mask - Set dma mask based on the controller
9494 * addressing capability
9495 * @hba: per adapter instance
9496 *
9497 * Returns 0 for success, non-zero for failure
9498 */
9499static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9500{
9501 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9502 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9503 return 0;
9504 }
9505 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9506}
9507
7a3e97b0 9508/**
5c0c28a8 9509 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 9510 * @dev: pointer to device handle
9511 * @hba_handle: driver private handle
7a3e97b0
SY		 9512 * Returns 0 on success, non-zero value on failure
9513 */
5c0c28a8 9514int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 9515{
9516 struct Scsi_Host *host;
9517 struct ufs_hba *hba;
5c0c28a8 9518 int err = 0;
7a3e97b0 9519
3b1d0580
VH		 9520 if (!dev) {
9521 dev_err(dev,
9522 "Invalid memory reference for dev is NULL\n");
9523 err = -ENODEV;
7a3e97b0
SY		 9524 goto out_error;
9525 }
9526
7a3e97b0
SY		 9527 host = scsi_host_alloc(&ufshcd_driver_template,
9528 sizeof(struct ufs_hba));
9529 if (!host) {
3b1d0580 9530 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 9531 err = -ENOMEM;
3b1d0580 9532 goto out_error;
7a3e97b0 9533 }
eaab9b57 9534 host->nr_maps = HCTX_TYPE_POLL + 1;
7a3e97b0 9535 hba = shost_priv(host);
7a3e97b0 9536 hba->host = host;
3b1d0580 9537 hba->dev = dev;
9e1e8a75 9538 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
1cbc9ad3 9539 hba->nop_out_timeout = NOP_OUT_TIMEOUT;
566ec9ad 9540 INIT_LIST_HEAD(&hba->clk_list_head);
169f5eb2
BVA		 9541 spin_lock_init(&hba->outstanding_lock);
9542
9543 *hba_handle = hba;
566ec9ad 9544
5c0c28a8
SRT		 9545out_error:
9546 return err;
9547}
9548EXPORT_SYMBOL(ufshcd_alloc_host);
9549
69a6c269
BVA		 9550/* This function exists because blk_mq_alloc_tag_set() requires this. */
9551static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9552 const struct blk_mq_queue_data *qd)
9553{
9554 WARN_ON_ONCE(true);
9555 return BLK_STS_NOTSUPP;
9556}
9557
9558static const struct blk_mq_ops ufshcd_tmf_ops = {
9559 .queue_rq = ufshcd_queue_tmf,
9560};
9561
5c0c28a8
SRT		 9562/**
9563 * ufshcd_init - Driver initialization routine
9564 * @hba: per-adapter instance
9565 * @mmio_base: base register address
9566 * @irq: Interrupt line of device
9567 * Returns 0 on success, non-zero value on failure
9568 */
9569int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9570{
9571 int err;
9572 struct Scsi_Host *host = hba->host;
9573 struct device *dev = hba->dev;
88b09900 9574 char eh_wq_name[sizeof("ufs_eh_wq_00")];
5c0c28a8 9575
21ad0e49
BVA		 9576 /*
9577 * dev_set_drvdata() must be called before any callbacks are registered
9578 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
9579 * sysfs).
9580 */
9581 dev_set_drvdata(dev, hba);
9582
5c0c28a8
SRT		 9583 if (!mmio_base) {
9584 dev_err(hba->dev,
9585 "Invalid memory reference for mmio_base is NULL\n");
9586 err = -ENODEV;
9587 goto out_error;
9588 }
9589
3b1d0580
VH		 9590 hba->mmio_base = mmio_base;
9591 hba->irq = irq;
90b8491c 9592 hba->vps = &ufs_hba_vps;
7a3e97b0 9593
aa497613 9594 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 9595 if (err)
9596 goto out_error;
9597
7a3e97b0 9598 /* Read capabilities registers */
df043c74
ST		 9599 err = ufshcd_hba_capabilities(hba);
9600 if (err)
9601 goto out_disable;
7a3e97b0
SY		 9602
9603 /* Get UFS version supported by the controller */
9604 hba->ufs_version = ufshcd_get_ufs_version(hba);
9605
2fbd009b
SJ		 9606 /* Get Interrupt bit mask per version */
9607 hba->intr_mask = ufshcd_get_intr_mask(hba);
9608
ca3d7bf9
AM		 9609 err = ufshcd_set_dma_mask(hba);
9610 if (err) {
9611 dev_err(hba->dev, "set dma mask failed\n");
9612 goto out_disable;
9613 }
9614
7a3e97b0
SY		 9615 /* Allocate memory for host memory space */
9616 err = ufshcd_memory_alloc(hba);
9617 if (err) {
3b1d0580
VH		 9618 dev_err(hba->dev, "Memory allocation failed\n");
9619 goto out_disable;
7a3e97b0
SY		 9620 }
9621
9622 /* Configure LRB */
9623 ufshcd_host_memory_configure(hba);
9624
945c3cca
BVA		 9625 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
9626 host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
7a3e97b0 9627 host->max_id = UFSHCD_MAX_ID;
0ce147d4 9628 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 9629 host->max_channel = UFSHCD_MAX_CHANNEL;
9630 host->unique_id = host->host_no;
a851b2bd 9631 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 9632
7eb584db
DR		 9633 hba->max_pwr_info.is_valid = false;
9634
88b09900
AH		 9635 /* Initialize work queues */
9636 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9637 hba->host->host_no);
9638 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9639 if (!hba->eh_wq) {
9640 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9641 __func__);
9642 err = -ENOMEM;
9643 goto out_disable;
9644 }
9645 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 9646 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 9647
9cd20d3f 9648 sema_init(&hba->host_sem, 1);
88a92d6a 9649
6ccf44fe
SJ		 9650 /* Initialize UIC command mutex */
9651 mutex_init(&hba->uic_cmd_mutex);
9652
5a0b0cb9
SRT		 9653 /* Initialize mutex for device management commands */
9654 mutex_init(&hba->dev_cmd.lock);
9655
cd469475
AH		 9656 /* Initialize mutex for exception event control */
9657 mutex_init(&hba->ee_ctrl_mutex);
9658
a3cd5ec5 9659 init_rwsem(&hba->clk_scaling_lock);
9660
1ab27c9c 9661 ufshcd_init_clk_gating(hba);
199ef13c 9662
eebcc196
VG		 9663 ufshcd_init_clk_scaling(hba);
9664
199ef13c
YG		 9665 /*
9666 * In order to avoid any spurious interrupt immediately after
9667 * registering UFS controller interrupt handler, clear any pending UFS
9668 * interrupt status and disable all the UFS interrupts.
9669 */
9670 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9671 REG_INTERRUPT_STATUS);
9672 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9673 /*
9674 * Make sure that UFS interrupts are disabled and any pending interrupt
9675 * status is cleared before registering UFS interrupt handler.
9676 */
9677 mb();
9678
7a3e97b0 9679 /* IRQ registration */
2953f850 9680 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 9681 if (err) {
3b1d0580 9682 dev_err(hba->dev, "request irq failed\n");
4543d9d7 9683 goto out_disable;
57d104c1
SJ		 9684 } else {
9685 hba->is_irq_enabled = true;
7a3e97b0
SY		 9686 }
9687
3b1d0580 9688 err = scsi_add_host(host, hba->dev);
7a3e97b0 9689 if (err) {
3b1d0580 9690 dev_err(hba->dev, "scsi_add_host failed\n");
4543d9d7 9691 goto out_disable;
7a3e97b0
SY		 9692 }
9693
69a6c269
BVA		 9694 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9695 .nr_hw_queues = 1,
9696 .queue_depth = hba->nutmrs,
9697 .ops = &ufshcd_tmf_ops,
9698 .flags = BLK_MQ_F_NO_SCHED,
9699 };
9700 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9701 if (err < 0)
511a083b 9702 goto out_remove_scsi_host;
69a6c269
BVA		 9703 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9704 if (IS_ERR(hba->tmf_queue)) {
9705 err = PTR_ERR(hba->tmf_queue);
9706 goto free_tmf_tag_set;
9707 }
f5ef336f
AH		 9708 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
9709 sizeof(*hba->tmf_rqs), GFP_KERNEL);
9710 if (!hba->tmf_rqs) {
9711 err = -ENOMEM;
9712 goto free_tmf_queue;
9713 }
69a6c269 9714
d8d9f793 9715 /* Reset the attached device */
31a5d9ca 9716 ufshcd_device_reset(hba);
d8d9f793 9717
df043c74
ST		 9718 ufshcd_init_crypto(hba);
9719
6ccf44fe
SJ		 9720 /* Host controller enable */
9721 err = ufshcd_hba_enable(hba);
7a3e97b0 9722 if (err) {
6ccf44fe 9723 dev_err(hba->dev, "Host controller enable failed\n");
e965e5e0 9724 ufshcd_print_evt_hist(hba);
6ba65588 9725 ufshcd_print_host_state(hba);
69a6c269 9726 goto free_tmf_queue;
7a3e97b0 9727 }
6ccf44fe 9728
0c8f7586 9729 /*
9730 * Set the default power management level for runtime and system PM.
9731 * Default power saving mode is to keep UFS link in Hibern8 state
9732 * and UFS device in sleep state.
9733 */
9734 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9735 UFS_SLEEP_PWR_MODE,
9736 UIC_LINK_HIBERN8_STATE);
9737 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9738 UFS_SLEEP_PWR_MODE,
9739 UIC_LINK_HIBERN8_STATE);
9740
51dd905b
SC		 9741 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9742 ufshcd_rpm_dev_flush_recheck_work);
9743
ad448378 9744 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 9745 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 9746 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9747 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9748 }
9749
62694735
SRT		 9750 /* Hold auto suspend until async scan completes */
9751 pm_runtime_get_sync(dev);
38135535 9752 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 9753 /*
7caf489b 9754 * We are assuming that device wasn't put in sleep/power-down
9755 * state exclusively during the boot stage before kernel.
9756 * This assumption helps avoid doing link startup twice during
9757 * ufshcd_probe_hba().
57d104c1 9758 */
7caf489b 9759 ufshcd_set_ufs_dev_active(hba);
57d104c1 9760
6ccf44fe 9761 async_schedule(ufshcd_async_scan, hba);
cbb6813e 9762 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 9763
1084514c 9764 device_enable_async_suspend(dev);
7a3e97b0
SY		 9765 return 0;
9766
69a6c269
BVA		 9767free_tmf_queue:
9768 blk_cleanup_queue(hba->tmf_queue);
9769free_tmf_tag_set:
9770 blk_mq_free_tag_set(&hba->tmf_tag_set);
3b1d0580
VH		 9771out_remove_scsi_host:
9772 scsi_remove_host(hba->host);
3b1d0580 9773out_disable:
57d104c1 9774 hba->is_irq_enabled = false;
aa497613 9775 ufshcd_hba_exit(hba);
3b1d0580
VH		 9776out_error:
9777 return err;
9778}
9779EXPORT_SYMBOL_GPL(ufshcd_init);
9780
b294ff3e
AD		 9781void ufshcd_resume_complete(struct device *dev)
9782{
9783 struct ufs_hba *hba = dev_get_drvdata(dev);
9784
9785 if (hba->complete_put) {
9786 ufshcd_rpm_put(hba);
9787 hba->complete_put = false;
9788 }
b294ff3e
AD		 9789}
9790EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
9791
ddba1cf7
AH		 9792static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
9793{
e2106584 9794 struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
ddba1cf7
AH		 9795 enum ufs_dev_pwr_mode dev_pwr_mode;
9796 enum uic_link_state link_state;
9797 unsigned long flags;
9798 bool res;
9799
9800 spin_lock_irqsave(&dev->power.lock, flags);
9801 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
9802 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
9803 res = pm_runtime_suspended(dev) &&
9804 hba->curr_dev_pwr_mode == dev_pwr_mode &&
9805 hba->uic_link_state == link_state &&
9806 !hba->dev_info.b_rpm_dev_flush_capable;
9807 spin_unlock_irqrestore(&dev->power.lock, flags);
9808
9809 return res;
9810}
9811
9812int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
b294ff3e
AD		 9813{
9814 struct ufs_hba *hba = dev_get_drvdata(dev);
9815 int ret;
9816
9817 /*
9818 * SCSI assumes that runtime-pm and system-pm for scsi drivers
9819 * are same. And it doesn't wake up the device for system-suspend
9820 * if it's runtime suspended. But ufs doesn't follow that.
9821 * Refer ufshcd_resume_complete()
9822 */
e2106584 9823 if (hba->ufs_device_wlun) {
ddba1cf7
AH		 9824 /* Prevent runtime suspend */
9825 ufshcd_rpm_get_noresume(hba);
9826 /*
9827 * Check if already runtime suspended in same state as system
9828 * suspend would be.
9829 */
9830 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
9831 /* RPM state is not ok for SPM, so runtime resume */
9832 ret = ufshcd_rpm_resume(hba);
9833 if (ret < 0 && ret != -EACCES) {
9834 ufshcd_rpm_put(hba);
9835 return ret;
9836 }
b294ff3e
AD		 9837 }
9838 hba->complete_put = true;
9839 }
b294ff3e
AD		 9840 return 0;
9841}
ddba1cf7
AH		 9842EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
9843
9844int ufshcd_suspend_prepare(struct device *dev)
9845{
9846 return __ufshcd_suspend_prepare(dev, true);
9847}
b294ff3e
AD		 9848EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
9849
9850#ifdef CONFIG_PM_SLEEP
9851static int ufshcd_wl_poweroff(struct device *dev)
9852{
9853 struct scsi_device *sdev = to_scsi_device(dev);
9854 struct ufs_hba *hba = shost_priv(sdev->host);
9855
9856 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9857 return 0;
9858}
9859#endif
9860
9861static int ufshcd_wl_probe(struct device *dev)
9862{
9863 struct scsi_device *sdev = to_scsi_device(dev);
9864
9865 if (!is_device_wlun(sdev))
9866 return -ENODEV;
9867
9868 blk_pm_runtime_init(sdev->request_queue, dev);
9869 pm_runtime_set_autosuspend_delay(dev, 0);
9870 pm_runtime_allow(dev);
9871
9872 return 0;
9873}
9874
9875static int ufshcd_wl_remove(struct device *dev)
9876{
9877 pm_runtime_forbid(dev);
9878 return 0;
9879}
9880
9881static const struct dev_pm_ops ufshcd_wl_pm_ops = {
9882#ifdef CONFIG_PM_SLEEP
9883 .suspend = ufshcd_wl_suspend,
9884 .resume = ufshcd_wl_resume,
9885 .freeze = ufshcd_wl_suspend,
9886 .thaw = ufshcd_wl_resume,
9887 .poweroff = ufshcd_wl_poweroff,
9888 .restore = ufshcd_wl_resume,
9889#endif
9890 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
9891};
9892
9893/*
9894 * ufs_dev_wlun_template - describes ufs device wlun
9895 * ufs-device wlun - used to send pm commands
9896 * All luns are consumers of ufs-device wlun.
9897 *
9898 * Currently, no sd driver is present for wluns.
9899 * Hence the no specific pm operations are performed.
9900 * With ufs design, SSU should be sent to ufs-device wlun.
9901 * Hence register a scsi driver for ufs wluns only.
9902 */
9903static struct scsi_driver ufs_dev_wlun_template = {
9904 .gendrv = {
9905 .name = "ufs_device_wlun",
9906 .owner = THIS_MODULE,
9907 .probe = ufshcd_wl_probe,
9908 .remove = ufshcd_wl_remove,
9909 .pm = &ufshcd_wl_pm_ops,
9910 .shutdown = ufshcd_wl_shutdown,
9911 },
9912};
9913
b6cacaf2
AH		 9914static int __init ufshcd_core_init(void)
9915{
b294ff3e
AD		 9916 int ret;
9917
9a868c8a
BVA		 9918 /* Verify that there are no gaps in struct utp_transfer_cmd_desc. */
9919 static_assert(sizeof(struct utp_transfer_cmd_desc) ==
9920 2 * ALIGNED_UPIU_SIZE +
9921 SG_ALL * sizeof(struct ufshcd_sg_entry));
9922
b6cacaf2 9923 ufs_debugfs_init();
b294ff3e
AD		 9924
9925 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
9926 if (ret)
edc0596c 9927 ufs_debugfs_exit();
b294ff3e 9928 return ret;
b6cacaf2
AH		 9929}
9930
9931static void __exit ufshcd_core_exit(void)
9932{
9933 ufs_debugfs_exit();
b294ff3e 9934 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
b6cacaf2
AH		 9935}
9936
9937module_init(ufshcd_core_init);
9938module_exit(ufshcd_core_exit);
9939
3b1d0580
VH		 9940MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9941MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 9942MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0 9943MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)