projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: scsi_debug: Make sdebug_build_parts() respect virtual_gb
[linux-2.6-block.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
67351119 1// SPDX-License-Identifier: GPL-2.0-or-later
7a3e97b0 2/*
e0eca63e 3 * Universal Flash Storage Host controller driver Core
3b1d0580 4 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 6 *
3b1d0580
VH		 7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 10 */
11
6ccf44fe 12#include <linux/async.h>
856b3483 13#include <linux/devfreq.h>
b573d484 14#include <linux/nls.h>
54b879b7 15#include <linux/of.h>
ad448378 16#include <linux/bitfield.h>
fb276f77 17#include <linux/blk-pm.h>
c72e79c0 18#include <linux/blkdev.h>
e0eca63e 19#include "ufshcd.h"
c58ab7aa 20#include "ufs_quirks.h"
53b3d9c3 21#include "unipro.h"
cbb6813e 22#include "ufs-sysfs.h"
df032bf2 23#include "ufs_bsg.h"
df043c74 24#include "ufshcd-crypto.h"
3d17b9b5
AD		 25#include <asm/unaligned.h>
26#include <linux/blkdev.h>
7a3e97b0 27
7ff5ab47 28#define CREATE_TRACE_POINTS
29#include <trace/events/ufs.h>
30
2fbd009b
SJ		 31#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
32 UTP_TASK_REQ_COMPL |\
33 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 34/* UIC command timeout, unit: ms */
35#define UIC_CMD_TIMEOUT 500
2fbd009b 36
5a0b0cb9
SRT		 37/* NOP OUT retries waiting for NOP IN response */
38#define NOP_OUT_RETRIES 10
39/* Timeout after 30 msecs if NOP OUT hangs without response */
40#define NOP_OUT_TIMEOUT 30 /* msecs */
41
68078d5c 42/* Query request retries */
10fe5888 43#define QUERY_REQ_RETRIES 3
68078d5c 44/* Query request timeout */
10fe5888 45#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 46
e2933132
SRT		 47/* Task management command timeout */
48#define TM_CMD_TIMEOUT 100 /* msecs */
49
64238fbd
YG		 50/* maximum number of retries for a general UIC command */
51#define UFS_UIC_COMMAND_RETRIES 3
52
1d337ec2
SRT		 53/* maximum number of link-startup retries */
54#define DME_LINKSTARTUP_RETRIES 3
55
87d0b4a6
YG		 56/* Maximum retries for Hibern8 enter */
57#define UIC_HIBERN8_ENTER_RETRIES 3
58
1d337ec2
SRT		 59/* maximum number of reset retries before giving up */
60#define MAX_HOST_RESET_RETRIES 5
61
68078d5c
DR		 62/* Expose the flag value from utp_upiu_query.value */
63#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
64
7d568652
SJ		 65/* Interrupt aggregation default timeout, unit: 40us */
66#define INT_AGGR_DEF_TO 0x02
67
49615ba1
SC		 68/* default delay of autosuspend: 2000 ms */
69#define RPM_AUTOSUSPEND_DELAY_MS 2000
70
51dd905b
SC		 71/* Default delay of RPM device flush delayed work */
72#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
73
09f17791
CG		 74/* Default value of wait time before gating device ref clock */
75#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
76
29707fab
KK		 77/* Polling time to wait for fDeviceInit */
78#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
79
aa497613
SRT		 80#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
81 ({ \
82 int _ret; \
83 if (_on) \
84 _ret = ufshcd_enable_vreg(_dev, _vreg); \
85 else \
86 _ret = ufshcd_disable_vreg(_dev, _vreg); \
87 _ret; \
88 })
89
ba80917d
TW		 90#define ufshcd_hex_dump(prefix_str, buf, len) do { \
91 size_t __len = (len); \
92 print_hex_dump(KERN_ERR, prefix_str, \
93 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
94 16, 4, buf, __len, false); \
95} while (0)
96
97int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
98 const char *prefix)
99{
d6724756
MG		 100 u32 *regs;
101 size_t pos;
102
103 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
104 return -EINVAL;
ba80917d 105
cddaebaf 106 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 107 if (!regs)
108 return -ENOMEM;
109
d6724756
MG		 110 for (pos = 0; pos < len; pos += 4)
111 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
112
ba80917d
TW		 113 ufshcd_hex_dump(prefix, regs, len);
114 kfree(regs);
115
116 return 0;
117}
118EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 119
7a3e97b0
SY		 120enum {
121 UFSHCD_MAX_CHANNEL = 0,
122 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 123 UFSHCD_CMD_PER_LUN = 32,
124 UFSHCD_CAN_QUEUE = 32,
125};
126
127/* UFSHCD states */
128enum {
7a3e97b0
SY		 129 UFSHCD_STATE_RESET,
130 UFSHCD_STATE_ERROR,
3441da7d 131 UFSHCD_STATE_OPERATIONAL,
5586dd8e
CG		 132 UFSHCD_STATE_EH_SCHEDULED_FATAL,
133 UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
3441da7d
SRT		 134};
135
136/* UFSHCD error handling flags */
137enum {
138 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 139};
140
e8e7f271
SRT		 141/* UFSHCD UIC layer error flags */
142enum {
143 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 144 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
145 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
146 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
147 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
148 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
2355b66e 149 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
e8e7f271
SRT		 150};
151
3441da7d 152#define ufshcd_set_eh_in_progress(h) \
9c490d2d 153 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 154#define ufshcd_eh_in_progress(h) \
9c490d2d 155 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 156#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 157 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 158
cbb6813e 159struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 160 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
161 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
162 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
163 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
164 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
165 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
166};
167
168static inline enum ufs_dev_pwr_mode
169ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
170{
171 return ufs_pm_lvl_states[lvl].dev_state;
172}
173
174static inline enum uic_link_state
175ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
176{
177 return ufs_pm_lvl_states[lvl].link_state;
178}
179
0c8f7586 180static inline enum ufs_pm_level
181ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
182 enum uic_link_state link_state)
183{
184 enum ufs_pm_level lvl;
185
186 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
187 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
188 (ufs_pm_lvl_states[lvl].link_state == link_state))
189 return lvl;
190 }
191
192 /* if no match found, return the level 0 */
193 return UFS_PM_LVL_0;
194}
195
56d4a186
SJ		 196static struct ufs_dev_fix ufs_fixups[] = {
197 /* UFS cards deviations table */
c0a18ee0
SC		 198 UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
199 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186 200 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
ed0b40ff
SC		 201 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
202 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
56d4a186 203 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
ed0b40ff
SC		 204 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
205 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
206 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
207 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 208 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
209 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
210 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
211 UFS_DEVICE_QUIRK_PA_TACTIVATE),
212 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
213 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 214 END_FIX
215};
216
9333d775 217static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 218static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 219static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 220static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 221static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2 222static void ufshcd_hba_exit(struct ufs_hba *hba);
1b9e2141 223static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
1ab27c9c
ST		 224static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
225 bool skip_ref_clk);
226static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
1ab27c9c 227static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 228static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 229static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 230static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
231static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 232static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 233static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 234static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 235static int ufshcd_change_power_mode(struct ufs_hba *hba,
236 struct ufs_pa_layer_attr *pwr_mode);
4db7a236 237static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
c72e79c0
CG		 238static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
239static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
240static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
241 struct ufs_vreg *vreg);
307348f6 242static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
3d17b9b5
AD		 243static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba);
244static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba);
245static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable);
246static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
247static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
248
14497328
YG		 249static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
250{
251 return tag >= 0 && tag < hba->nutrs;
252}
57d104c1 253
5231d38c 254static inline void ufshcd_enable_irq(struct ufs_hba *hba)
57d104c1 255{
57d104c1 256 if (!hba->is_irq_enabled) {
5231d38c 257 enable_irq(hba->irq);
57d104c1
SJ		 258 hba->is_irq_enabled = true;
259 }
57d104c1
SJ		 260}
261
262static inline void ufshcd_disable_irq(struct ufs_hba *hba)
263{
264 if (hba->is_irq_enabled) {
5231d38c 265 disable_irq(hba->irq);
57d104c1
SJ		 266 hba->is_irq_enabled = false;
267 }
268}
3441da7d 269
3d17b9b5
AD		 270static inline void ufshcd_wb_config(struct ufs_hba *hba)
271{
272 int ret;
273
79e3520f 274 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 275 return;
276
277 ret = ufshcd_wb_ctrl(hba, true);
278 if (ret)
279 dev_err(hba->dev, "%s: Enable WB failed: %d\n", __func__, ret);
280 else
281 dev_info(hba->dev, "%s: Write Booster Configured\n", __func__);
282 ret = ufshcd_wb_toggle_flush_during_h8(hba, true);
283 if (ret)
284 dev_err(hba->dev, "%s: En WB flush during H8: failed: %d\n",
285 __func__, ret);
286 ufshcd_wb_toggle_flush(hba, true);
287}
288
38135535
SJ		 289static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
290{
291 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
292 scsi_unblock_requests(hba->host);
293}
294
295static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
296{
297 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
298 scsi_block_requests(hba->host);
299}
300
6667e6d9
OS		 301static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
302 const char *str)
303{
304 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
305
306 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
307}
308
309static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
310 const char *str)
311{
312 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
313
314 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
315}
316
317static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
318 const char *str)
319{
6667e6d9 320 int off = (int)tag - hba->nutrs;
391e388f 321 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 322
391e388f
CH		 323 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
324 &descp->input_param1);
6667e6d9
OS		 325}
326
aa5c6979
SC		 327static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
328 struct uic_command *ucmd,
329 const char *str)
330{
331 u32 cmd;
332
333 if (!trace_ufshcd_uic_command_enabled())
334 return;
335
336 if (!strcmp(str, "send"))
337 cmd = ucmd->command;
338 else
339 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
340
341 trace_ufshcd_uic_command(dev_name(hba->dev), str, cmd,
342 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
343 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
344 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
345}
346
1a07f2d9
LS		 347static void ufshcd_add_command_trace(struct ufs_hba *hba,
348 unsigned int tag, const char *str)
349{
350 sector_t lba = -1;
351 u8 opcode = 0;
352 u32 intr, doorbell;
e7c3b379 353 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
e4d2add7 354 struct scsi_cmnd *cmd = lrbp->cmd;
1a07f2d9
LS		 355 int transfer_len = -1;
356
e7c3b379
OS		 357 if (!trace_ufshcd_command_enabled()) {
358 /* trace UPIU W/O tracing command */
e4d2add7 359 if (cmd)
e7c3b379 360 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9 361 return;
e7c3b379 362 }
1a07f2d9 363
e4d2add7 364 if (cmd) { /* data phase exists */
e7c3b379
OS		 365 /* trace UPIU also */
366 ufshcd_add_cmd_upiu_trace(hba, tag, str);
e4d2add7 367 opcode = cmd->cmnd[0];
1a07f2d9
LS		 368 if ((opcode == READ_10) || (opcode == WRITE_10)) {
369 /*
370 * Currently we only fully trace read(10) and write(10)
371 * commands
372 */
e4d2add7
BVA		 373 if (cmd->request && cmd->request->bio)
374 lba = cmd->request->bio->bi_iter.bi_sector;
1a07f2d9
LS		 375 transfer_len = be32_to_cpu(
376 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
377 }
378 }
379
380 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
381 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
382 trace_ufshcd_command(dev_name(hba->dev), str, tag,
383 doorbell, transfer_len, intr, lba, opcode);
384}
385
ff8e20c6
DR		 386static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
387{
388 struct ufs_clk_info *clki;
389 struct list_head *head = &hba->clk_list_head;
390
566ec9ad 391 if (list_empty(head))
ff8e20c6
DR		 392 return;
393
394 list_for_each_entry(clki, head, list) {
395 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
396 clki->max_freq)
397 dev_err(hba->dev, "clk: %s, rate: %u\n",
398 clki->name, clki->curr_freq);
399 }
400}
401
48d5b973
SC		 402static void ufshcd_print_err_hist(struct ufs_hba *hba,
403 struct ufs_err_reg_hist *err_hist,
404 char *err_name)
ff8e20c6
DR		 405{
406 int i;
27752647 407 bool found = false;
ff8e20c6 408
48d5b973
SC		 409 for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
410 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
ff8e20c6 411
645728a6 412 if (err_hist->tstamp[p] == 0)
ff8e20c6 413 continue;
c5397f13 414 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
ff8e20c6 415 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
27752647 416 found = true;
ff8e20c6 417 }
27752647
SC		 418
419 if (!found)
fd1fb4d5 420 dev_err(hba->dev, "No record of %s\n", err_name);
ff8e20c6
DR		 421}
422
66cc820f
DR		 423static void ufshcd_print_host_regs(struct ufs_hba *hba)
424{
ba80917d 425 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
ff8e20c6 426
48d5b973
SC		 427 ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
428 ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
429 ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
430 ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
431 ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
d3c615bf
SC		 432 ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
433 "auto_hibern8_err");
8808b4e9
SC		 434 ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
435 ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
436 "link_startup_fail");
437 ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
438 ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
439 "suspend_fail");
440 ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
441 ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
442 ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
ff8e20c6 443
7c486d91 444 ufshcd_vops_dbg_register_dump(hba);
66cc820f
DR		 445}
446
447static
448void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
449{
450 struct ufshcd_lrb *lrbp;
7fabb77b 451 int prdt_length;
66cc820f
DR		 452 int tag;
453
454 for_each_set_bit(tag, &bitmap, hba->nutrs) {
455 lrbp = &hba->lrb[tag];
456
ff8e20c6
DR		 457 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
458 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 459 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
460 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 461 dev_err(hba->dev,
462 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
463 tag, (u64)lrbp->utrd_dma_addr);
464
66cc820f
DR		 465 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
466 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 467 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
468 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 469 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
470 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 471 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
472 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 473 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
474 sizeof(struct utp_upiu_rsp));
66cc820f 475
7fabb77b
GB		 476 prdt_length = le16_to_cpu(
477 lrbp->utr_descriptor_ptr->prd_table_length);
cc770ce3
EB		 478 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
479 prdt_length /= sizeof(struct ufshcd_sg_entry);
480
7fabb77b
GB		 481 dev_err(hba->dev,
482 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
483 tag, prdt_length,
484 (u64)lrbp->ucd_prdt_dma_addr);
485
486 if (pr_prdt)
66cc820f 487 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 488 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 489 }
490}
491
492static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
493{
66cc820f
DR		 494 int tag;
495
496 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 497 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
498
66cc820f 499 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 500 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 501 }
502}
503
6ba65588
GB		 504static void ufshcd_print_host_state(struct ufs_hba *hba)
505{
3f8af604
CG		 506 struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
507
6ba65588 508 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 509 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
510 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 511 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
512 hba->saved_err, hba->saved_uic_err);
513 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
514 hba->curr_dev_pwr_mode, hba->uic_link_state);
515 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
516 hba->pm_op_in_progress, hba->is_sys_suspended);
517 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
518 hba->auto_bkops_enabled, hba->host->host_self_blocked);
519 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
3f8af604
CG		 520 dev_err(hba->dev,
521 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
522 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
523 hba->ufs_stats.hibern8_exit_cnt);
524 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
525 ktime_to_us(hba->ufs_stats.last_intr_ts),
526 hba->ufs_stats.last_intr_status);
6ba65588
GB		 527 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
528 hba->eh_flags, hba->req_abort_count);
3f8af604
CG		 529 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
530 hba->ufs_version, hba->capabilities, hba->caps);
6ba65588
GB		 531 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
532 hba->dev_quirks);
3f8af604
CG		 533 if (sdev_ufs)
534 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
535 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
536
537 ufshcd_print_clk_freqs(hba);
6ba65588
GB		 538}
539
ff8e20c6
DR		 540/**
541 * ufshcd_print_pwr_info - print power params as saved in hba
542 * power info
543 * @hba: per-adapter instance
544 */
545static void ufshcd_print_pwr_info(struct ufs_hba *hba)
546{
547 static const char * const names[] = {
548 "INVALID MODE",
549 "FAST MODE",
550 "SLOW_MODE",
551 "INVALID MODE",
552 "FASTAUTO_MODE",
553 "SLOWAUTO_MODE",
554 "INVALID MODE",
555 };
556
557 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
558 __func__,
559 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
560 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
561 names[hba->pwr_info.pwr_rx],
562 names[hba->pwr_info.pwr_tx],
563 hba->pwr_info.hs_rate);
564}
565
5c955c10
SC		 566void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
567{
568 if (!us)
569 return;
570
571 if (us < 10)
572 udelay(us);
573 else
574 usleep_range(us, us + tolerance);
575}
576EXPORT_SYMBOL_GPL(ufshcd_delay_us);
577
5cac1095 578/**
5a0b0cb9 579 * ufshcd_wait_for_register - wait for register value to change
5cac1095
BVA		 580 * @hba: per-adapter interface
581 * @reg: mmio register offset
582 * @mask: mask to apply to the read register value
583 * @val: value to wait for
584 * @interval_us: polling interval in microseconds
585 * @timeout_ms: timeout in milliseconds
5a0b0cb9 586 *
5cac1095
BVA		 587 * Return:
588 * -ETIMEDOUT on error, zero on success.
5a0b0cb9 589 */
596585a2
YG		 590int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
591 u32 val, unsigned long interval_us,
5cac1095 592 unsigned long timeout_ms)
5a0b0cb9
SRT		 593{
594 int err = 0;
595 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
596
597 /* ignore bits that we don't intend to wait on */
598 val = val & mask;
599
600 while ((ufshcd_readl(hba, reg) & mask) != val) {
5cac1095 601 usleep_range(interval_us, interval_us + 50);
5a0b0cb9
SRT		 602 if (time_after(jiffies, timeout)) {
603 if ((ufshcd_readl(hba, reg) & mask) != val)
604 err = -ETIMEDOUT;
605 break;
606 }
607 }
608
609 return err;
610}
611
2fbd009b
SJ		 612/**
613 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 614 * @hba: Pointer to adapter instance
2fbd009b
SJ		 615 *
616 * Returns interrupt bit mask per version
617 */
618static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
619{
c01848c6
YG		 620 u32 intr_mask = 0;
621
622 switch (hba->ufs_version) {
623 case UFSHCI_VERSION_10:
624 intr_mask = INTERRUPT_MASK_ALL_VER_10;
625 break;
c01848c6
YG		 626 case UFSHCI_VERSION_11:
627 case UFSHCI_VERSION_20:
628 intr_mask = INTERRUPT_MASK_ALL_VER_11;
629 break;
c01848c6
YG		 630 case UFSHCI_VERSION_21:
631 default:
632 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 633 break;
c01848c6
YG		 634 }
635
636 return intr_mask;
2fbd009b
SJ		 637}
638
7a3e97b0
SY		 639/**
640 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 641 * @hba: Pointer to adapter instance
7a3e97b0
SY		 642 *
643 * Returns UFSHCI version supported by the controller
644 */
645static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
646{
0263bcd0
YG		 647 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
648 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 649
b873a275 650 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 651}
652
653/**
654 * ufshcd_is_device_present - Check if any device connected to
655 * the host controller
5c0c28a8 656 * @hba: pointer to adapter instance
7a3e97b0 657 *
c9e6010b 658 * Returns true if device present, false if no device detected
7a3e97b0 659 */
c9e6010b 660static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 661{
5c0c28a8 662 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 663 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 664}
665
666/**
667 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 668 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 669 *
670 * This function is used to get the OCS field from UTRD
671 * Returns the OCS field in the UTRD
672 */
673static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
674{
e8c8e82a 675 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 676}
677
7a3e97b0
SY		 678/**
679 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
680 * @hba: per adapter instance
681 * @pos: position of the bit to be cleared
682 */
683static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
684{
87183841
AA		 685 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
686 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
687 else
688 ufshcd_writel(hba, ~(1 << pos),
689 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
1399c5b0
AA		 690}
691
692/**
693 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
694 * @hba: per adapter instance
695 * @pos: position of the bit to be cleared
696 */
697static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
698{
87183841
AA		 699 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
700 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
701 else
702 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 703}
704
a48353f6
YG		 705/**
706 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
707 * @hba: per adapter instance
708 * @tag: position of the bit to be cleared
709 */
710static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
711{
712 __clear_bit(tag, &hba->outstanding_reqs);
713}
714
7a3e97b0
SY		 715/**
716 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
717 * @reg: Register value of host controller status
718 *
719 * Returns integer, 0 on Success and positive value if failed
720 */
721static inline int ufshcd_get_lists_status(u32 reg)
722{
6cf16115 723 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 724}
725
726/**
727 * ufshcd_get_uic_cmd_result - Get the UIC command result
728 * @hba: Pointer to adapter instance
729 *
730 * This function gets the result of UIC command completion
731 * Returns 0 on success, non zero value on error
732 */
733static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
734{
b873a275 735 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 736 MASK_UIC_COMMAND_RESULT;
737}
738
12b4fdb4
SJ		 739/**
740 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
741 * @hba: Pointer to adapter instance
742 *
743 * This function gets UIC command argument3
744 * Returns 0 on success, non zero value on error
745 */
746static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
747{
748 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
749}
750
7a3e97b0 751/**
5a0b0cb9 752 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 753 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 754 */
755static inline int
5a0b0cb9 756ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 757{
5a0b0cb9 758 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 759}
760
761/**
762 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
763 * @ucd_rsp_ptr: pointer to response UPIU
764 *
765 * This function gets the response status and scsi_status from response UPIU
766 * Returns the response result code.
767 */
768static inline int
769ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
770{
771 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
772}
773
1c2623c5
SJ		 774/*
775 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
776 * from response UPIU
777 * @ucd_rsp_ptr: pointer to response UPIU
778 *
779 * Return the data segment length.
780 */
781static inline unsigned int
782ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
783{
784 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
785 MASK_RSP_UPIU_DATA_SEG_LEN;
786}
787
66ec6d59
SRT		 788/**
789 * ufshcd_is_exception_event - Check if the device raised an exception event
790 * @ucd_rsp_ptr: pointer to response UPIU
791 *
792 * The function checks if the device raised an exception event indicated in
793 * the Device Information field of response UPIU.
794 *
795 * Returns true if exception is raised, false otherwise.
796 */
797static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
798{
799 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
800 MASK_RSP_EXCEPTION_EVENT ? true : false;
801}
802
7a3e97b0 803/**
7d568652 804 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 805 * @hba: per adapter instance
7a3e97b0
SY		 806 */
807static inline void
7d568652 808ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 809{
7d568652
SJ		 810 ufshcd_writel(hba, INT_AGGR_ENABLE |
811 INT_AGGR_COUNTER_AND_TIMER_RESET,
812 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
813}
814
815/**
816 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
817 * @hba: per adapter instance
818 * @cnt: Interrupt aggregation counter threshold
819 * @tmout: Interrupt aggregation timeout value
820 */
821static inline void
822ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
823{
824 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
825 INT_AGGR_COUNTER_THLD_VAL(cnt) |
826 INT_AGGR_TIMEOUT_VAL(tmout),
827 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 828}
829
b852190e
YG		 830/**
831 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
832 * @hba: per adapter instance
833 */
834static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
835{
836 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
837}
838
7a3e97b0
SY		 839/**
840 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
841 * When run-stop registers are set to 1, it indicates the
842 * host controller that it can process the requests
843 * @hba: per adapter instance
844 */
845static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
846{
b873a275
SJ		 847 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
848 REG_UTP_TASK_REQ_LIST_RUN_STOP);
849 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
850 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 851}
852
7a3e97b0
SY		 853/**
854 * ufshcd_hba_start - Start controller initialization sequence
855 * @hba: per adapter instance
856 */
857static inline void ufshcd_hba_start(struct ufs_hba *hba)
858{
df043c74
ST		 859 u32 val = CONTROLLER_ENABLE;
860
861 if (ufshcd_crypto_enable(hba))
862 val |= CRYPTO_GENERAL_ENABLE;
863
864 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 865}
866
867/**
868 * ufshcd_is_hba_active - Get controller state
869 * @hba: per adapter instance
870 *
c9e6010b 871 * Returns false if controller is active, true otherwise
7a3e97b0 872 */
c9e6010b 873static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 874{
4a8eec2b
TK		 875 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
876 ? false : true;
7a3e97b0
SY		 877}
878
37113106
YG		 879u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
880{
881 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
882 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
883 (hba->ufs_version == UFSHCI_VERSION_11))
884 return UFS_UNIPRO_VER_1_41;
885 else
886 return UFS_UNIPRO_VER_1_6;
887}
888EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
889
890static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
891{
892 /*
893 * If both host and device support UniPro ver1.6 or later, PA layer
894 * parameters tuning happens during link startup itself.
895 *
896 * We can manually tune PA layer parameters if either host or device
897 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
898 * logic simple, we will only do manual tuning if local unipro version
899 * doesn't support ver1.6 or later.
900 */
901 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
902 return true;
903 else
904 return false;
905}
906
394b949f
SJ		 907/**
908 * ufshcd_set_clk_freq - set UFS controller clock frequencies
909 * @hba: per adapter instance
910 * @scale_up: If True, set max possible frequency othewise set low frequency
911 *
912 * Returns 0 if successful
913 * Returns < 0 for any other errors
914 */
915static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
a3cd5ec5 916{
917 int ret = 0;
918 struct ufs_clk_info *clki;
919 struct list_head *head = &hba->clk_list_head;
a3cd5ec5 920
566ec9ad 921 if (list_empty(head))
a3cd5ec5 922 goto out;
923
a3cd5ec5 924 list_for_each_entry(clki, head, list) {
925 if (!IS_ERR_OR_NULL(clki->clk)) {
926 if (scale_up && clki->max_freq) {
927 if (clki->curr_freq == clki->max_freq)
928 continue;
929
a3cd5ec5 930 ret = clk_set_rate(clki->clk, clki->max_freq);
931 if (ret) {
932 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
933 __func__, clki->name,
934 clki->max_freq, ret);
935 break;
936 }
937 trace_ufshcd_clk_scaling(dev_name(hba->dev),
938 "scaled up", clki->name,
939 clki->curr_freq,
940 clki->max_freq);
941
942 clki->curr_freq = clki->max_freq;
943
944 } else if (!scale_up && clki->min_freq) {
945 if (clki->curr_freq == clki->min_freq)
946 continue;
947
a3cd5ec5 948 ret = clk_set_rate(clki->clk, clki->min_freq);
949 if (ret) {
950 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
951 __func__, clki->name,
952 clki->min_freq, ret);
953 break;
954 }
955 trace_ufshcd_clk_scaling(dev_name(hba->dev),
956 "scaled down", clki->name,
957 clki->curr_freq,
958 clki->min_freq);
959 clki->curr_freq = clki->min_freq;
960 }
961 }
962 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
963 clki->name, clk_get_rate(clki->clk));
964 }
965
394b949f
SJ		 966out:
967 return ret;
968}
969
970/**
971 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
972 * @hba: per adapter instance
973 * @scale_up: True if scaling up and false if scaling down
974 *
975 * Returns 0 if successful
976 * Returns < 0 for any other errors
977 */
978static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
979{
980 int ret = 0;
981 ktime_t start = ktime_get();
982
983 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
984 if (ret)
985 goto out;
986
987 ret = ufshcd_set_clk_freq(hba, scale_up);
988 if (ret)
989 goto out;
990
a3cd5ec5 991 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
394b949f
SJ		 992 if (ret)
993 ufshcd_set_clk_freq(hba, !scale_up);
a3cd5ec5 994
995out:
394b949f 996 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
a3cd5ec5 997 (scale_up ? "up" : "down"),
998 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
999 return ret;
1000}
1001
1002/**
1003 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1004 * @hba: per adapter instance
1005 * @scale_up: True if scaling up and false if scaling down
1006 *
1007 * Returns true if scaling is required, false otherwise.
1008 */
1009static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1010 bool scale_up)
1011{
1012 struct ufs_clk_info *clki;
1013 struct list_head *head = &hba->clk_list_head;
1014
566ec9ad 1015 if (list_empty(head))
a3cd5ec5 1016 return false;
1017
1018 list_for_each_entry(clki, head, list) {
1019 if (!IS_ERR_OR_NULL(clki->clk)) {
1020 if (scale_up && clki->max_freq) {
1021 if (clki->curr_freq == clki->max_freq)
1022 continue;
1023 return true;
1024 } else if (!scale_up && clki->min_freq) {
1025 if (clki->curr_freq == clki->min_freq)
1026 continue;
1027 return true;
1028 }
1029 }
1030 }
1031
1032 return false;
1033}
1034
1035static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1036 u64 wait_timeout_us)
1037{
1038 unsigned long flags;
1039 int ret = 0;
1040 u32 tm_doorbell;
1041 u32 tr_doorbell;
1042 bool timeout = false, do_last_check = false;
1043 ktime_t start;
1044
1045 ufshcd_hold(hba, false);
1046 spin_lock_irqsave(hba->host->host_lock, flags);
1047 /*
1048 * Wait for all the outstanding tasks/transfer requests.
1049 * Verify by checking the doorbell registers are clear.
1050 */
1051 start = ktime_get();
1052 do {
1053 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1054 ret = -EBUSY;
1055 goto out;
1056 }
1057
1058 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1059 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1060 if (!tm_doorbell && !tr_doorbell) {
1061 timeout = false;
1062 break;
1063 } else if (do_last_check) {
1064 break;
1065 }
1066
1067 spin_unlock_irqrestore(hba->host->host_lock, flags);
1068 schedule();
1069 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1070 wait_timeout_us) {
1071 timeout = true;
1072 /*
1073 * We might have scheduled out for long time so make
1074 * sure to check if doorbells are cleared by this time
1075 * or not.
1076 */
1077 do_last_check = true;
1078 }
1079 spin_lock_irqsave(hba->host->host_lock, flags);
1080 } while (tm_doorbell || tr_doorbell);
1081
1082 if (timeout) {
1083 dev_err(hba->dev,
1084 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1085 __func__, tm_doorbell, tr_doorbell);
1086 ret = -EBUSY;
1087 }
1088out:
1089 spin_unlock_irqrestore(hba->host->host_lock, flags);
1090 ufshcd_release(hba);
1091 return ret;
1092}
1093
1094/**
1095 * ufshcd_scale_gear - scale up/down UFS gear
1096 * @hba: per adapter instance
1097 * @scale_up: True for scaling up gear and false for scaling down
1098 *
1099 * Returns 0 for success,
1100 * Returns -EBUSY if scaling can't happen at this time
1101 * Returns non-zero for any other errors
1102 */
1103static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1104{
1105 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1106 int ret = 0;
1107 struct ufs_pa_layer_attr new_pwr_info;
1108
1109 if (scale_up) {
1110 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1111 sizeof(struct ufs_pa_layer_attr));
1112 } else {
1113 memcpy(&new_pwr_info, &hba->pwr_info,
1114 sizeof(struct ufs_pa_layer_attr));
1115
1116 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1117 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1118 /* save the current power mode */
1119 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1120 &hba->pwr_info,
1121 sizeof(struct ufs_pa_layer_attr));
1122
1123 /* scale down gear */
1124 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1125 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1126 }
1127 }
1128
1129 /* check if the power mode needs to be changed or not? */
6a9df818 1130 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
a3cd5ec5 1131 if (ret)
1132 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1133 __func__, ret,
1134 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1135 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1136
1137 return ret;
1138}
1139
1140static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1141{
1142 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1143 int ret = 0;
1144 /*
1145 * make sure that there are no outstanding requests when
1146 * clock scaling is in progress
1147 */
38135535 1148 ufshcd_scsi_block_requests(hba);
a3cd5ec5 1149 down_write(&hba->clk_scaling_lock);
1150 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1151 ret = -EBUSY;
1152 up_write(&hba->clk_scaling_lock);
38135535 1153 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1154 }
1155
1156 return ret;
1157}
1158
1159static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1160{
1161 up_write(&hba->clk_scaling_lock);
38135535 1162 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1163}
1164
1165/**
1166 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1167 * @hba: per adapter instance
1168 * @scale_up: True for scaling up and false for scalin down
1169 *
1170 * Returns 0 for success,
1171 * Returns -EBUSY if scaling can't happen at this time
1172 * Returns non-zero for any other errors
1173 */
1174static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1175{
1176 int ret = 0;
1177
401f1e44 1178 /* let's not get into low power until clock scaling is completed */
1179 ufshcd_hold(hba, false);
1180
a3cd5ec5 1181 ret = ufshcd_clock_scaling_prepare(hba);
1182 if (ret)
394b949f 1183 goto out;
a3cd5ec5 1184
1185 /* scale down the gear before scaling down clocks */
1186 if (!scale_up) {
1187 ret = ufshcd_scale_gear(hba, false);
1188 if (ret)
394b949f 1189 goto out_unprepare;
a3cd5ec5 1190 }
1191
1192 ret = ufshcd_scale_clks(hba, scale_up);
1193 if (ret) {
1194 if (!scale_up)
1195 ufshcd_scale_gear(hba, true);
394b949f 1196 goto out_unprepare;
a3cd5ec5 1197 }
1198
1199 /* scale up the gear after scaling up clocks */
1200 if (scale_up) {
1201 ret = ufshcd_scale_gear(hba, true);
3d17b9b5 1202 if (ret) {
a3cd5ec5 1203 ufshcd_scale_clks(hba, false);
3d17b9b5
AD		 1204 goto out_unprepare;
1205 }
a3cd5ec5 1206 }
1207
3d17b9b5
AD		 1208 /* Enable Write Booster if we have scaled up else disable it */
1209 up_write(&hba->clk_scaling_lock);
1210 ufshcd_wb_ctrl(hba, scale_up);
1211 down_write(&hba->clk_scaling_lock);
1212
394b949f 1213out_unprepare:
a3cd5ec5 1214 ufshcd_clock_scaling_unprepare(hba);
394b949f 1215out:
401f1e44 1216 ufshcd_release(hba);
a3cd5ec5 1217 return ret;
1218}
1219
401f1e44 1220static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1221{
1222 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1223 clk_scaling.suspend_work);
1224 unsigned long irq_flags;
1225
1226 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1227 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1228 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1229 return;
1230 }
1231 hba->clk_scaling.is_suspended = true;
1232 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1233
1234 __ufshcd_suspend_clkscaling(hba);
1235}
1236
1237static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1238{
1239 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1240 clk_scaling.resume_work);
1241 unsigned long irq_flags;
1242
1243 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1244 if (!hba->clk_scaling.is_suspended) {
1245 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1246 return;
1247 }
1248 hba->clk_scaling.is_suspended = false;
1249 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1250
1251 devfreq_resume_device(hba->devfreq);
1252}
1253
a3cd5ec5 1254static int ufshcd_devfreq_target(struct device *dev,
1255 unsigned long *freq, u32 flags)
1256{
1257 int ret = 0;
1258 struct ufs_hba *hba = dev_get_drvdata(dev);
1259 ktime_t start;
401f1e44 1260 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1261 struct list_head *clk_list = &hba->clk_list_head;
1262 struct ufs_clk_info *clki;
a3cd5ec5 1263 unsigned long irq_flags;
1264
1265 if (!ufshcd_is_clkscaling_supported(hba))
1266 return -EINVAL;
1267
91831d33
AD		 1268 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1269 /* Override with the closest supported frequency */
1270 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
a3cd5ec5 1271 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1272 if (ufshcd_eh_in_progress(hba)) {
1273 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1274 return 0;
1275 }
1276
401f1e44 1277 if (!hba->clk_scaling.active_reqs)
1278 sched_clk_scaling_suspend_work = true;
1279
092b4558
BA		 1280 if (list_empty(clk_list)) {
1281 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1282 goto out;
1283 }
1284
91831d33 1285 /* Decide based on the rounded-off frequency and update */
092b4558 1286 scale_up = (*freq == clki->max_freq) ? true : false;
91831d33
AD		 1287 if (!scale_up)
1288 *freq = clki->min_freq;
1289 /* Update the frequency */
401f1e44 1290 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1291 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1292 ret = 0;
1293 goto out; /* no state change required */
a3cd5ec5 1294 }
1295 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1296
1297 start = ktime_get();
a3cd5ec5 1298 ret = ufshcd_devfreq_scale(hba, scale_up);
1299
a3cd5ec5 1300 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1301 (scale_up ? "up" : "down"),
1302 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1303
401f1e44 1304out:
1305 if (sched_clk_scaling_suspend_work)
1306 queue_work(hba->clk_scaling.workq,
1307 &hba->clk_scaling.suspend_work);
1308
a3cd5ec5 1309 return ret;
1310}
1311
7252a360
BVA		 1312static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1313{
1314 int *busy = priv;
1315
1316 WARN_ON_ONCE(reserved);
1317 (*busy)++;
1318 return false;
1319}
1320
1321/* Whether or not any tag is in use by a request that is in progress. */
1322static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1323{
1324 struct request_queue *q = hba->cmd_queue;
1325 int busy = 0;
1326
1327 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1328 return busy;
1329}
a3cd5ec5 1330
1331static int ufshcd_devfreq_get_dev_status(struct device *dev,
1332 struct devfreq_dev_status *stat)
1333{
1334 struct ufs_hba *hba = dev_get_drvdata(dev);
1335 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1336 unsigned long flags;
91831d33
AD		 1337 struct list_head *clk_list = &hba->clk_list_head;
1338 struct ufs_clk_info *clki;
b1bf66d1 1339 ktime_t curr_t;
a3cd5ec5 1340
1341 if (!ufshcd_is_clkscaling_supported(hba))
1342 return -EINVAL;
1343
1344 memset(stat, 0, sizeof(*stat));
1345
1346 spin_lock_irqsave(hba->host->host_lock, flags);
b1bf66d1 1347 curr_t = ktime_get();
a3cd5ec5 1348 if (!scaling->window_start_t)
1349 goto start_window;
1350
91831d33
AD		 1351 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1352 /*
1353 * If current frequency is 0, then the ondemand governor considers
1354 * there's no initial frequency set. And it always requests to set
1355 * to max. frequency.
1356 */
1357 stat->current_frequency = clki->curr_freq;
a3cd5ec5 1358 if (scaling->is_busy_started)
b1bf66d1
SC		 1359 scaling->tot_busy_t += ktime_us_delta(curr_t,
1360 scaling->busy_start_t);
a3cd5ec5 1361
b1bf66d1 1362 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
a3cd5ec5 1363 stat->busy_time = scaling->tot_busy_t;
1364start_window:
b1bf66d1 1365 scaling->window_start_t = curr_t;
a3cd5ec5 1366 scaling->tot_busy_t = 0;
1367
1368 if (hba->outstanding_reqs) {
b1bf66d1 1369 scaling->busy_start_t = curr_t;
a3cd5ec5 1370 scaling->is_busy_started = true;
1371 } else {
1372 scaling->busy_start_t = 0;
1373 scaling->is_busy_started = false;
1374 }
1375 spin_unlock_irqrestore(hba->host->host_lock, flags);
1376 return 0;
1377}
1378
deac444f
BA		 1379static int ufshcd_devfreq_init(struct ufs_hba *hba)
1380{
092b4558
BA		 1381 struct list_head *clk_list = &hba->clk_list_head;
1382 struct ufs_clk_info *clki;
deac444f
BA		 1383 struct devfreq *devfreq;
1384 int ret;
1385
092b4558
BA		 1386 /* Skip devfreq if we don't have any clocks in the list */
1387 if (list_empty(clk_list))
1388 return 0;
1389
1390 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1391 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1392 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1393
90b8491c
SC		 1394 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1395 &hba->vps->ondemand_data);
092b4558 1396 devfreq = devfreq_add_device(hba->dev,
90b8491c 1397 &hba->vps->devfreq_profile,
deac444f 1398 DEVFREQ_GOV_SIMPLE_ONDEMAND,
90b8491c 1399 &hba->vps->ondemand_data);
deac444f
BA		 1400 if (IS_ERR(devfreq)) {
1401 ret = PTR_ERR(devfreq);
1402 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1403
1404 dev_pm_opp_remove(hba->dev, clki->min_freq);
1405 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1406 return ret;
1407 }
1408
1409 hba->devfreq = devfreq;
1410
1411 return 0;
1412}
1413
092b4558
BA		 1414static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1415{
1416 struct list_head *clk_list = &hba->clk_list_head;
1417 struct ufs_clk_info *clki;
1418
1419 if (!hba->devfreq)
1420 return;
1421
1422 devfreq_remove_device(hba->devfreq);
1423 hba->devfreq = NULL;
1424
1425 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1426 dev_pm_opp_remove(hba->dev, clki->min_freq);
1427 dev_pm_opp_remove(hba->dev, clki->max_freq);
1428}
1429
401f1e44 1430static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1431{
1432 unsigned long flags;
1433
1434 devfreq_suspend_device(hba->devfreq);
1435 spin_lock_irqsave(hba->host->host_lock, flags);
1436 hba->clk_scaling.window_start_t = 0;
1437 spin_unlock_irqrestore(hba->host->host_lock, flags);
1438}
a3cd5ec5 1439
a508253d
GB		 1440static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1441{
401f1e44 1442 unsigned long flags;
1443 bool suspend = false;
1444
fcb0c4b0
ST		 1445 if (!ufshcd_is_clkscaling_supported(hba))
1446 return;
1447
401f1e44 1448 spin_lock_irqsave(hba->host->host_lock, flags);
1449 if (!hba->clk_scaling.is_suspended) {
1450 suspend = true;
1451 hba->clk_scaling.is_suspended = true;
1452 }
1453 spin_unlock_irqrestore(hba->host->host_lock, flags);
1454
1455 if (suspend)
1456 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1457}
1458
1459static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1460{
401f1e44 1461 unsigned long flags;
1462 bool resume = false;
1463
1464 if (!ufshcd_is_clkscaling_supported(hba))
1465 return;
1466
1467 spin_lock_irqsave(hba->host->host_lock, flags);
1468 if (hba->clk_scaling.is_suspended) {
1469 resume = true;
1470 hba->clk_scaling.is_suspended = false;
1471 }
1472 spin_unlock_irqrestore(hba->host->host_lock, flags);
1473
1474 if (resume)
1475 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1476}
1477
1478static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1479 struct device_attribute *attr, char *buf)
1480{
1481 struct ufs_hba *hba = dev_get_drvdata(dev);
1482
1483 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1484}
1485
1486static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1487 struct device_attribute *attr, const char *buf, size_t count)
1488{
1489 struct ufs_hba *hba = dev_get_drvdata(dev);
1490 u32 value;
1491 int err;
1492
1493 if (kstrtou32(buf, 0, &value))
1494 return -EINVAL;
1495
1496 value = !!value;
1497 if (value == hba->clk_scaling.is_allowed)
1498 goto out;
1499
1500 pm_runtime_get_sync(hba->dev);
1501 ufshcd_hold(hba, false);
1502
401f1e44 1503 cancel_work_sync(&hba->clk_scaling.suspend_work);
1504 cancel_work_sync(&hba->clk_scaling.resume_work);
1505
1506 hba->clk_scaling.is_allowed = value;
1507
fcb0c4b0
ST		 1508 if (value) {
1509 ufshcd_resume_clkscaling(hba);
1510 } else {
1511 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1512 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1513 if (err)
1514 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1515 __func__, err);
1516 }
fcb0c4b0
ST		 1517
1518 ufshcd_release(hba);
1519 pm_runtime_put_sync(hba->dev);
1520out:
1521 return count;
a508253d
GB		 1522}
1523
a3cd5ec5 1524static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1525{
1526 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1527 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1528 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1529 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1530 hba->clk_scaling.enable_attr.attr.mode = 0644;
1531 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1532 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1533}
1534
1ab27c9c
ST		 1535static void ufshcd_ungate_work(struct work_struct *work)
1536{
1537 int ret;
1538 unsigned long flags;
1539 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1540 clk_gating.ungate_work);
1541
1542 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1543
1544 spin_lock_irqsave(hba->host->host_lock, flags);
1545 if (hba->clk_gating.state == CLKS_ON) {
1546 spin_unlock_irqrestore(hba->host->host_lock, flags);
1547 goto unblock_reqs;
1548 }
1549
1550 spin_unlock_irqrestore(hba->host->host_lock, flags);
1551 ufshcd_setup_clocks(hba, true);
1552
8b0bbf00
SC		 1553 ufshcd_enable_irq(hba);
1554
1ab27c9c
ST		 1555 /* Exit from hibern8 */
1556 if (ufshcd_can_hibern8_during_gating(hba)) {
1557 /* Prevent gating in this path */
1558 hba->clk_gating.is_suspended = true;
1559 if (ufshcd_is_link_hibern8(hba)) {
1560 ret = ufshcd_uic_hibern8_exit(hba);
1561 if (ret)
1562 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1563 __func__, ret);
1564 else
1565 ufshcd_set_link_active(hba);
1566 }
1567 hba->clk_gating.is_suspended = false;
1568 }
1569unblock_reqs:
38135535 1570 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1571}
1572
1573/**
1574 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1575 * Also, exit from hibern8 mode and set the link as active.
1576 * @hba: per adapter instance
1577 * @async: This indicates whether caller should ungate clocks asynchronously.
1578 */
1579int ufshcd_hold(struct ufs_hba *hba, bool async)
1580{
1581 int rc = 0;
1582 unsigned long flags;
1583
1584 if (!ufshcd_is_clkgating_allowed(hba))
1585 goto out;
1ab27c9c
ST		 1586 spin_lock_irqsave(hba->host->host_lock, flags);
1587 hba->clk_gating.active_reqs++;
1588
856b3483 1589start:
1ab27c9c
ST		 1590 switch (hba->clk_gating.state) {
1591 case CLKS_ON:
f2a785ac
VG		 1592 /*
1593 * Wait for the ungate work to complete if in progress.
1594 * Though the clocks may be in ON state, the link could
1595 * still be in hibner8 state if hibern8 is allowed
1596 * during clock gating.
1597 * Make sure we exit hibern8 state also in addition to
1598 * clocks being ON.
1599 */
1600 if (ufshcd_can_hibern8_during_gating(hba) &&
1601 ufshcd_is_link_hibern8(hba)) {
c63d6099
CG		 1602 if (async) {
1603 rc = -EAGAIN;
1604 hba->clk_gating.active_reqs--;
1605 break;
1606 }
f2a785ac
VG		 1607 spin_unlock_irqrestore(hba->host->host_lock, flags);
1608 flush_work(&hba->clk_gating.ungate_work);
1609 spin_lock_irqsave(hba->host->host_lock, flags);
1610 goto start;
1611 }
1ab27c9c
ST		 1612 break;
1613 case REQ_CLKS_OFF:
1614 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1615 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1616 trace_ufshcd_clk_gating(dev_name(hba->dev),
1617 hba->clk_gating.state);
1ab27c9c
ST		 1618 break;
1619 }
1620 /*
9c490d2d 1621 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1622 * currently running. Hence, fall through to cancel gating
1623 * work and to enable clocks.
1624 */
30eb2e4c 1625 /* fallthrough */
1ab27c9c 1626 case CLKS_OFF:
38135535 1627 ufshcd_scsi_block_requests(hba);
1ab27c9c 1628 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47 1629 trace_ufshcd_clk_gating(dev_name(hba->dev),
1630 hba->clk_gating.state);
10e5e375
VV		 1631 queue_work(hba->clk_gating.clk_gating_workq,
1632 &hba->clk_gating.ungate_work);
1ab27c9c
ST		 1633 /*
1634 * fall through to check if we should wait for this
1635 * work to be done or not.
1636 */
30eb2e4c 1637 /* fallthrough */
1ab27c9c
ST		 1638 case REQ_CLKS_ON:
1639 if (async) {
1640 rc = -EAGAIN;
1641 hba->clk_gating.active_reqs--;
1642 break;
1643 }
1644
1645 spin_unlock_irqrestore(hba->host->host_lock, flags);
1646 flush_work(&hba->clk_gating.ungate_work);
1647 /* Make sure state is CLKS_ON before returning */
856b3483 1648 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1649 goto start;
1650 default:
1651 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1652 __func__, hba->clk_gating.state);
1653 break;
1654 }
1655 spin_unlock_irqrestore(hba->host->host_lock, flags);
1656out:
1657 return rc;
1658}
6e3fd44d 1659EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1660
1661static void ufshcd_gate_work(struct work_struct *work)
1662{
1663 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1664 clk_gating.gate_work.work);
1665 unsigned long flags;
4db7a236 1666 int ret;
1ab27c9c
ST		 1667
1668 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1669 /*
1670 * In case you are here to cancel this work the gating state
1671 * would be marked as REQ_CLKS_ON. In this case save time by
1672 * skipping the gating work and exit after changing the clock
1673 * state to CLKS_ON.
1674 */
1675 if (hba->clk_gating.is_suspended ||
18f01374 1676 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1677 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1678 trace_ufshcd_clk_gating(dev_name(hba->dev),
1679 hba->clk_gating.state);
1ab27c9c
ST		 1680 goto rel_lock;
1681 }
1682
1683 if (hba->clk_gating.active_reqs
1684 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1685 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1ab27c9c
ST		 1686 || hba->active_uic_cmd || hba->uic_async_done)
1687 goto rel_lock;
1688
1689 spin_unlock_irqrestore(hba->host->host_lock, flags);
1690
1691 /* put the link into hibern8 mode before turning off clocks */
1692 if (ufshcd_can_hibern8_during_gating(hba)) {
4db7a236
CG		 1693 ret = ufshcd_uic_hibern8_enter(hba);
1694 if (ret) {
1ab27c9c 1695 hba->clk_gating.state = CLKS_ON;
4db7a236
CG		 1696 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1697 __func__, ret);
7ff5ab47 1698 trace_ufshcd_clk_gating(dev_name(hba->dev),
1699 hba->clk_gating.state);
1ab27c9c
ST		 1700 goto out;
1701 }
1702 ufshcd_set_link_hibern8(hba);
1703 }
1704
8b0bbf00
SC		 1705 ufshcd_disable_irq(hba);
1706
1ab27c9c
ST		 1707 if (!ufshcd_is_link_active(hba))
1708 ufshcd_setup_clocks(hba, false);
1709 else
1710 /* If link is active, device ref_clk can't be switched off */
1711 __ufshcd_setup_clocks(hba, false, true);
1712
1713 /*
1714 * In case you are here to cancel this work the gating state
1715 * would be marked as REQ_CLKS_ON. In this case keep the state
1716 * as REQ_CLKS_ON which would anyway imply that clocks are off
1717 * and a request to turn them on is pending. By doing this way,
1718 * we keep the state machine in tact and this would ultimately
1719 * prevent from doing cancel work multiple times when there are
1720 * new requests arriving before the current cancel work is done.
1721 */
1722 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1723 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1724 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 1725 trace_ufshcd_clk_gating(dev_name(hba->dev),
1726 hba->clk_gating.state);
1727 }
1ab27c9c
ST		 1728rel_lock:
1729 spin_unlock_irqrestore(hba->host->host_lock, flags);
1730out:
1731 return;
1732}
1733
1734/* host lock must be held before calling this variant */
1735static void __ufshcd_release(struct ufs_hba *hba)
1736{
1737 if (!ufshcd_is_clkgating_allowed(hba))
1738 return;
1739
1740 hba->clk_gating.active_reqs--;
1741
4db7a236
CG		 1742 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1743 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1744 ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks ||
1745 hba->active_uic_cmd || hba->uic_async_done)
1ab27c9c
ST		 1746 return;
1747
1748 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1749 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1750 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1751 &hba->clk_gating.gate_work,
1752 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1753}
1754
1755void ufshcd_release(struct ufs_hba *hba)
1756{
1757 unsigned long flags;
1758
1759 spin_lock_irqsave(hba->host->host_lock, flags);
1760 __ufshcd_release(hba);
1761 spin_unlock_irqrestore(hba->host->host_lock, flags);
1762}
6e3fd44d 1763EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1764
1765static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1766 struct device_attribute *attr, char *buf)
1767{
1768 struct ufs_hba *hba = dev_get_drvdata(dev);
1769
1770 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1771}
1772
1773static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1774 struct device_attribute *attr, const char *buf, size_t count)
1775{
1776 struct ufs_hba *hba = dev_get_drvdata(dev);
1777 unsigned long flags, value;
1778
1779 if (kstrtoul(buf, 0, &value))
1780 return -EINVAL;
1781
1782 spin_lock_irqsave(hba->host->host_lock, flags);
1783 hba->clk_gating.delay_ms = value;
1784 spin_unlock_irqrestore(hba->host->host_lock, flags);
1785 return count;
1786}
1787
b427411a
ST		 1788static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1789 struct device_attribute *attr, char *buf)
1790{
1791 struct ufs_hba *hba = dev_get_drvdata(dev);
1792
1793 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1794}
1795
1796static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1797 struct device_attribute *attr, const char *buf, size_t count)
1798{
1799 struct ufs_hba *hba = dev_get_drvdata(dev);
1800 unsigned long flags;
1801 u32 value;
1802
1803 if (kstrtou32(buf, 0, &value))
1804 return -EINVAL;
1805
1806 value = !!value;
1807 if (value == hba->clk_gating.is_enabled)
1808 goto out;
1809
1810 if (value) {
1811 ufshcd_release(hba);
1812 } else {
1813 spin_lock_irqsave(hba->host->host_lock, flags);
1814 hba->clk_gating.active_reqs++;
1815 spin_unlock_irqrestore(hba->host->host_lock, flags);
1816 }
1817
1818 hba->clk_gating.is_enabled = value;
1819out:
1820 return count;
1821}
1822
eebcc196
VG		 1823static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1824{
1825 char wq_name[sizeof("ufs_clkscaling_00")];
1826
1827 if (!ufshcd_is_clkscaling_supported(hba))
1828 return;
1829
1830 INIT_WORK(&hba->clk_scaling.suspend_work,
1831 ufshcd_clk_scaling_suspend_work);
1832 INIT_WORK(&hba->clk_scaling.resume_work,
1833 ufshcd_clk_scaling_resume_work);
1834
1835 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1836 hba->host->host_no);
1837 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1838
1839 ufshcd_clkscaling_init_sysfs(hba);
1840}
1841
1842static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1843{
1844 if (!ufshcd_is_clkscaling_supported(hba))
1845 return;
1846
1847 destroy_workqueue(hba->clk_scaling.workq);
1848 ufshcd_devfreq_remove(hba);
1849}
1850
1ab27c9c
ST		 1851static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1852{
10e5e375
VV		 1853 char wq_name[sizeof("ufs_clk_gating_00")];
1854
1ab27c9c
ST		 1855 if (!ufshcd_is_clkgating_allowed(hba))
1856 return;
1857
2dec9475
CG		 1858 hba->clk_gating.state = CLKS_ON;
1859
1ab27c9c
ST		 1860 hba->clk_gating.delay_ms = 150;
1861 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1862 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1863
10e5e375
VV		 1864 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1865 hba->host->host_no);
1866 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1867 WQ_MEM_RECLAIM);
1868
b427411a
ST		 1869 hba->clk_gating.is_enabled = true;
1870
1ab27c9c
ST		 1871 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1872 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1873 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1874 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
b427411a 1875 hba->clk_gating.delay_attr.attr.mode = 0644;
1ab27c9c
ST		 1876 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1877 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
b427411a
ST		 1878
1879 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1880 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1881 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1882 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1883 hba->clk_gating.enable_attr.attr.mode = 0644;
1884 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1885 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1ab27c9c
ST		 1886}
1887
1888static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1889{
1890 if (!ufshcd_is_clkgating_allowed(hba))
1891 return;
1892 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
b427411a 1893 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
97cd6805
AM		 1894 cancel_work_sync(&hba->clk_gating.ungate_work);
1895 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1896 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1897}
1898
856b3483
ST		 1899/* Must be called with host lock acquired */
1900static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1901{
401f1e44 1902 bool queue_resume_work = false;
b1bf66d1 1903 ktime_t curr_t = ktime_get();
401f1e44 1904
fcb0c4b0 1905 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1906 return;
1907
401f1e44 1908 if (!hba->clk_scaling.active_reqs++)
1909 queue_resume_work = true;
1910
1911 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1912 return;
1913
1914 if (queue_resume_work)
1915 queue_work(hba->clk_scaling.workq,
1916 &hba->clk_scaling.resume_work);
1917
1918 if (!hba->clk_scaling.window_start_t) {
b1bf66d1 1919 hba->clk_scaling.window_start_t = curr_t;
401f1e44 1920 hba->clk_scaling.tot_busy_t = 0;
1921 hba->clk_scaling.is_busy_started = false;
1922 }
1923
856b3483 1924 if (!hba->clk_scaling.is_busy_started) {
b1bf66d1 1925 hba->clk_scaling.busy_start_t = curr_t;
856b3483
ST		 1926 hba->clk_scaling.is_busy_started = true;
1927 }
1928}
1929
1930static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1931{
1932 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1933
fcb0c4b0 1934 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1935 return;
1936
1937 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1938 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1939 scaling->busy_start_t));
8b0e1953 1940 scaling->busy_start_t = 0;
856b3483
ST		 1941 scaling->is_busy_started = false;
1942 }
1943}
7a3e97b0
SY		 1944/**
1945 * ufshcd_send_command - Send SCSI or device management commands
1946 * @hba: per adapter instance
1947 * @task_tag: Task tag of the command
1948 */
1949static inline
1950void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1951{
6edfdcfe
SC		 1952 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
1953
1954 lrbp->issue_time_stamp = ktime_get();
1955 lrbp->compl_time_stamp = ktime_set(0, 0);
1956 ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false));
eacf36f5 1957 ufshcd_add_command_trace(hba, task_tag, "send");
856b3483 1958 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 1959 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 1960 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 1961 /* Make sure that doorbell is committed immediately */
1962 wmb();
7a3e97b0
SY		 1963}
1964
1965/**
1966 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 1967 * @lrbp: pointer to local reference block
7a3e97b0
SY		 1968 */
1969static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1970{
1971 int len;
1c2623c5
SJ		 1972 if (lrbp->sense_buffer &&
1973 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 1974 int len_to_copy;
1975
5a0b0cb9 1976 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 1977 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 1978
09a5a24f
AA		 1979 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1980 len_to_copy);
7a3e97b0
SY		 1981 }
1982}
1983
68078d5c
DR		 1984/**
1985 * ufshcd_copy_query_response() - Copy the Query Response and the data
1986 * descriptor
1987 * @hba: per adapter instance
8aa29f19 1988 * @lrbp: pointer to local reference block
68078d5c
DR		 1989 */
1990static
c6d4a831 1991int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 1992{
1993 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1994
68078d5c 1995 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 1996
68078d5c 1997 /* Get the descriptor */
1c90836f
AA		 1998 if (hba->dev_cmd.query.descriptor &&
1999 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 2000 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 2001 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 2002 u16 resp_len;
2003 u16 buf_len;
68078d5c
DR		 2004
2005 /* data segment length */
c6d4a831 2006 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 2007 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 2008 buf_len = be16_to_cpu(
2009 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 2010 if (likely(buf_len >= resp_len)) {
2011 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2012 } else {
2013 dev_warn(hba->dev,
3d4881d1
BH		 2014 "%s: rsp size %d is bigger than buffer size %d",
2015 __func__, resp_len, buf_len);
c6d4a831
DR		 2016 return -EINVAL;
2017 }
68078d5c 2018 }
c6d4a831
DR		 2019
2020 return 0;
68078d5c
DR		 2021}
2022
7a3e97b0
SY		 2023/**
2024 * ufshcd_hba_capabilities - Read controller capabilities
2025 * @hba: per adapter instance
df043c74
ST		 2026 *
2027 * Return: 0 on success, negative on error.
7a3e97b0 2028 */
df043c74 2029static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
7a3e97b0 2030{
df043c74
ST		 2031 int err;
2032
b873a275 2033 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 2034
2035 /* nutrs and nutmrs are 0 based values */
2036 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2037 hba->nutmrs =
2038 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
df043c74
ST		 2039
2040 /* Read crypto capabilities */
2041 err = ufshcd_hba_init_crypto_capabilities(hba);
2042 if (err)
2043 dev_err(hba->dev, "crypto setup failed\n");
2044
2045 return err;
7a3e97b0
SY		 2046}
2047
2048/**
6ccf44fe
SJ		 2049 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2050 * to accept UIC commands
7a3e97b0 2051 * @hba: per adapter instance
6ccf44fe
SJ		 2052 * Return true on success, else false
2053 */
2054static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2055{
2056 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2057 return true;
2058 else
2059 return false;
2060}
2061
53b3d9c3
SJ		 2062/**
2063 * ufshcd_get_upmcrs - Get the power mode change request status
2064 * @hba: Pointer to adapter instance
2065 *
2066 * This function gets the UPMCRS field of HCS register
2067 * Returns value of UPMCRS field
2068 */
2069static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2070{
2071 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2072}
2073
6ccf44fe
SJ		 2074/**
2075 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2076 * @hba: per adapter instance
2077 * @uic_cmd: UIC command
2078 *
2079 * Mutex must be held.
7a3e97b0
SY		 2080 */
2081static inline void
6ccf44fe 2082ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 2083{
6ccf44fe
SJ		 2084 WARN_ON(hba->active_uic_cmd);
2085
2086 hba->active_uic_cmd = uic_cmd;
2087
7a3e97b0 2088 /* Write Args */
6ccf44fe
SJ		 2089 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2090 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2091 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0 2092
aa5c6979
SC		 2093 ufshcd_add_uic_command_trace(hba, uic_cmd, "send");
2094
7a3e97b0 2095 /* Write UIC Cmd */
6ccf44fe 2096 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 2097 REG_UIC_COMMAND);
7a3e97b0
SY		 2098}
2099
6ccf44fe
SJ		 2100/**
2101 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2102 * @hba: per adapter instance
8aa29f19 2103 * @uic_cmd: UIC command
6ccf44fe
SJ		 2104 *
2105 * Must be called with mutex held.
2106 * Returns 0 only if success.
2107 */
2108static int
2109ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2110{
2111 int ret;
2112 unsigned long flags;
2113
2114 if (wait_for_completion_timeout(&uic_cmd->done,
2115 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2116 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2117 else
2118 ret = -ETIMEDOUT;
2119
2120 spin_lock_irqsave(hba->host->host_lock, flags);
2121 hba->active_uic_cmd = NULL;
2122 spin_unlock_irqrestore(hba->host->host_lock, flags);
2123
2124 return ret;
2125}
2126
2127/**
2128 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2129 * @hba: per adapter instance
2130 * @uic_cmd: UIC command
d75f7fe4 2131 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2132 *
2133 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2134 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2135 * Returns 0 only if success.
2136 */
2137static int
d75f7fe4
YG		 2138__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2139 bool completion)
6ccf44fe 2140{
6ccf44fe
SJ		 2141 if (!ufshcd_ready_for_uic_cmd(hba)) {
2142 dev_err(hba->dev,
2143 "Controller not ready to accept UIC commands\n");
2144 return -EIO;
2145 }
2146
d75f7fe4
YG		 2147 if (completion)
2148 init_completion(&uic_cmd->done);
6ccf44fe 2149
6ccf44fe 2150 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2151
57d104c1 2152 return 0;
6ccf44fe
SJ		 2153}
2154
2155/**
2156 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2157 * @hba: per adapter instance
2158 * @uic_cmd: UIC command
2159 *
2160 * Returns 0 only if success.
2161 */
e77044c5 2162int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2163{
2164 int ret;
57d104c1 2165 unsigned long flags;
6ccf44fe 2166
1ab27c9c 2167 ufshcd_hold(hba, false);
6ccf44fe 2168 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2169 ufshcd_add_delay_before_dme_cmd(hba);
2170
57d104c1 2171 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2172 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2173 spin_unlock_irqrestore(hba->host->host_lock, flags);
2174 if (!ret)
2175 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2176
6ccf44fe
SJ		 2177 mutex_unlock(&hba->uic_cmd_mutex);
2178
1ab27c9c 2179 ufshcd_release(hba);
6ccf44fe
SJ		 2180 return ret;
2181}
2182
7a3e97b0
SY		 2183/**
2184 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2185 * @hba: per adapter instance
2186 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2187 *
2188 * Returns 0 in case of success, non-zero value in case of failure
2189 */
75b1cc4a 2190static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2191{
2192 struct ufshcd_sg_entry *prd_table;
2193 struct scatterlist *sg;
2194 struct scsi_cmnd *cmd;
2195 int sg_segments;
2196 int i;
2197
2198 cmd = lrbp->cmd;
2199 sg_segments = scsi_dma_map(cmd);
2200 if (sg_segments < 0)
2201 return sg_segments;
2202
2203 if (sg_segments) {
26f968d7
AA		 2204
2205 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2206 lrbp->utr_descriptor_ptr->prd_table_length =
2207 cpu_to_le16((sg_segments *
2208 sizeof(struct ufshcd_sg_entry)));
2209 else
2210 lrbp->utr_descriptor_ptr->prd_table_length =
2211 cpu_to_le16((u16) (sg_segments));
7a3e97b0
SY		 2212
2213 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2214
2215 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2216 prd_table[i].size =
2217 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2218 prd_table[i].base_addr =
2219 cpu_to_le32(lower_32_bits(sg->dma_address));
2220 prd_table[i].upper_addr =
2221 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2222 prd_table[i].reserved = 0;
7a3e97b0
SY		 2223 }
2224 } else {
2225 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2226 }
2227
2228 return 0;
2229}
2230
2231/**
2fbd009b 2232 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2233 * @hba: per adapter instance
2fbd009b 2234 * @intrs: interrupt bits
7a3e97b0 2235 */
2fbd009b 2236static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2237{
2fbd009b
SJ		 2238 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2239
2240 if (hba->ufs_version == UFSHCI_VERSION_10) {
2241 u32 rw;
2242 rw = set & INTERRUPT_MASK_RW_VER_10;
2243 set = rw | ((set ^ intrs) & intrs);
2244 } else {
2245 set |= intrs;
2246 }
2247
2248 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2249}
2250
2251/**
2252 * ufshcd_disable_intr - disable interrupts
2253 * @hba: per adapter instance
2254 * @intrs: interrupt bits
2255 */
2256static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2257{
2258 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2259
2260 if (hba->ufs_version == UFSHCI_VERSION_10) {
2261 u32 rw;
2262 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2263 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2264 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2265
2266 } else {
2267 set &= ~intrs;
7a3e97b0 2268 }
2fbd009b
SJ		 2269
2270 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2271}
2272
5a0b0cb9
SRT		 2273/**
2274 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2275 * descriptor according to request
2276 * @lrbp: pointer to local reference block
2277 * @upiu_flags: flags required in the header
2278 * @cmd_dir: requests data direction
2279 */
2280static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
a23064c4 2281 u8 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2282{
2283 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2284 u32 data_direction;
2285 u32 dword_0;
df043c74
ST		 2286 u32 dword_1 = 0;
2287 u32 dword_3 = 0;
5a0b0cb9
SRT		 2288
2289 if (cmd_dir == DMA_FROM_DEVICE) {
2290 data_direction = UTP_DEVICE_TO_HOST;
2291 *upiu_flags = UPIU_CMD_FLAGS_READ;
2292 } else if (cmd_dir == DMA_TO_DEVICE) {
2293 data_direction = UTP_HOST_TO_DEVICE;
2294 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2295 } else {
2296 data_direction = UTP_NO_DATA_TRANSFER;
2297 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2298 }
2299
2300 dword_0 = data_direction | (lrbp->command_type
2301 << UPIU_COMMAND_TYPE_OFFSET);
2302 if (lrbp->intr_cmd)
2303 dword_0 |= UTP_REQ_DESC_INT_CMD;
2304
df043c74
ST		 2305 /* Prepare crypto related dwords */
2306 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2307
5a0b0cb9
SRT		 2308 /* Transfer request descriptor header fields */
2309 req_desc->header.dword_0 = cpu_to_le32(dword_0);
df043c74 2310 req_desc->header.dword_1 = cpu_to_le32(dword_1);
5a0b0cb9
SRT		 2311 /*
2312 * assigning invalid value for command status. Controller
2313 * updates OCS on command completion, with the command
2314 * status
2315 */
2316 req_desc->header.dword_2 =
2317 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
df043c74 2318 req_desc->header.dword_3 = cpu_to_le32(dword_3);
51047266
YG		 2319
2320 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2321}
2322
2323/**
2324 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2325 * for scsi commands
8aa29f19
BVA		 2326 * @lrbp: local reference block pointer
2327 * @upiu_flags: flags
5a0b0cb9
SRT		 2328 */
2329static
a23064c4 2330void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
5a0b0cb9 2331{
1b21b8f0 2332 struct scsi_cmnd *cmd = lrbp->cmd;
5a0b0cb9 2333 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2334 unsigned short cdb_len;
5a0b0cb9
SRT		 2335
2336 /* command descriptor fields */
2337 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2338 UPIU_TRANSACTION_COMMAND, upiu_flags,
2339 lrbp->lun, lrbp->task_tag);
2340 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2341 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2342
2343 /* Total EHS length and Data segment length will be zero */
2344 ucd_req_ptr->header.dword_2 = 0;
2345
1b21b8f0 2346 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
5a0b0cb9 2347
1b21b8f0 2348 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
a851b2bd 2349 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1b21b8f0 2350 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
52ac95fe
YG		 2351
2352 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2353}
2354
68078d5c
DR		 2355/**
2356 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2357 * for query requsts
2358 * @hba: UFS hba
2359 * @lrbp: local reference block pointer
2360 * @upiu_flags: flags
2361 */
2362static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
a23064c4 2363 struct ufshcd_lrb *lrbp, u8 upiu_flags)
68078d5c
DR		 2364{
2365 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2366 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2367 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2368
2369 /* Query request header */
2370 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2371 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2372 lrbp->lun, lrbp->task_tag);
2373 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2374 0, query->request.query_func, 0, 0);
2375
6861285c
ZL		 2376 /* Data segment length only need for WRITE_DESC */
2377 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2378 ucd_req_ptr->header.dword_2 =
2379 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2380 else
2381 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2382
2383 /* Copy the Query Request buffer as is */
2384 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2385 QUERY_OSF_SIZE);
68078d5c
DR		 2386
2387 /* Copy the Descriptor */
c6d4a831 2388 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2389 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2390
51047266 2391 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2392}
2393
5a0b0cb9
SRT		 2394static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2395{
2396 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2397
2398 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2399
2400 /* command descriptor fields */
2401 ucd_req_ptr->header.dword_0 =
2402 UPIU_HEADER_DWORD(
2403 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2404 /* clear rest of the fields of basic header */
2405 ucd_req_ptr->header.dword_1 = 0;
2406 ucd_req_ptr->header.dword_2 = 0;
2407
2408 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2409}
2410
7a3e97b0 2411/**
f273c54b 2412 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
300bb13f 2413 * for Device Management Purposes
8aa29f19
BVA		 2414 * @hba: per adapter instance
2415 * @lrbp: pointer to local reference block
7a3e97b0 2416 */
f273c54b
BH		 2417static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2418 struct ufshcd_lrb *lrbp)
7a3e97b0 2419{
a23064c4 2420 u8 upiu_flags;
5a0b0cb9 2421 int ret = 0;
7a3e97b0 2422
83dc7e3d 2423 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2424 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2425 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2426 else
2427 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2428
2429 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2430 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2431 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2432 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2433 ufshcd_prepare_utp_nop_upiu(lrbp);
2434 else
2435 ret = -EINVAL;
2436
2437 return ret;
2438}
2439
2440/**
2441 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2442 * for SCSI Purposes
8aa29f19
BVA		 2443 * @hba: per adapter instance
2444 * @lrbp: pointer to local reference block
300bb13f
JP		 2445 */
2446static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2447{
a23064c4 2448 u8 upiu_flags;
300bb13f
JP		 2449 int ret = 0;
2450
83dc7e3d 2451 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2452 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2453 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2454 else
2455 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2456
2457 if (likely(lrbp->cmd)) {
2458 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2459 lrbp->cmd->sc_data_direction);
2460 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2461 } else {
2462 ret = -EINVAL;
2463 }
5a0b0cb9
SRT		 2464
2465 return ret;
7a3e97b0
SY		 2466}
2467
2a8fa600
SJ		 2468/**
2469 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2470 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2471 *
2472 * Returns SCSI W-LUN id
2473 */
2474static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2475{
2476 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2477}
2478
4d2b8d40
BVA		 2479static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2480{
2481 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2482 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2483 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2484 i * sizeof(struct utp_transfer_cmd_desc);
2485 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2486 response_upiu);
2487 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2488
2489 lrb->utr_descriptor_ptr = utrdlp + i;
2490 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2491 i * sizeof(struct utp_transfer_req_desc);
2492 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2493 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2494 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2495 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2496 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2497 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2498}
2499
7a3e97b0
SY		 2500/**
2501 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2502 * @host: SCSI host pointer
7a3e97b0 2503 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2504 *
2505 * Returns 0 for success, non-zero in case of failure
2506 */
2507static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2508{
2509 struct ufshcd_lrb *lrbp;
2510 struct ufs_hba *hba;
2511 unsigned long flags;
2512 int tag;
2513 int err = 0;
2514
2515 hba = shost_priv(host);
2516
2517 tag = cmd->request->tag;
14497328
YG		 2518 if (!ufshcd_valid_tag(hba, tag)) {
2519 dev_err(hba->dev,
2520 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2521 __func__, tag, cmd, cmd->request);
2522 BUG();
2523 }
7a3e97b0 2524
a3cd5ec5 2525 if (!down_read_trylock(&hba->clk_scaling_lock))
2526 return SCSI_MLQUEUE_HOST_BUSY;
2527
7fabb77b
GB		 2528 hba->req_abort_count = 0;
2529
1ab27c9c
ST		 2530 err = ufshcd_hold(hba, true);
2531 if (err) {
2532 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2533 goto out;
2534 }
2dec9475
CG		 2535 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2536 (hba->clk_gating.state != CLKS_ON));
1ab27c9c 2537
7a3e97b0
SY		 2538 lrbp = &hba->lrb[tag];
2539
5a0b0cb9 2540 WARN_ON(lrbp->cmd);
7a3e97b0 2541 lrbp->cmd = cmd;
09a5a24f 2542 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2543 lrbp->sense_buffer = cmd->sense_buffer;
2544 lrbp->task_tag = tag;
0ce147d4 2545 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2546 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
df043c74
ST		 2547
2548 ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
2549
e0b299e3 2550 lrbp->req_abort_skip = false;
7a3e97b0 2551
300bb13f
JP		 2552 ufshcd_comp_scsi_upiu(hba, lrbp);
2553
75b1cc4a 2554 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2555 if (err) {
2556 lrbp->cmd = NULL;
17c7d35f 2557 ufshcd_release(hba);
7a3e97b0 2558 goto out;
5a0b0cb9 2559 }
ad1a1b9c
GB		 2560 /* Make sure descriptors are ready before ringing the doorbell */
2561 wmb();
7a3e97b0 2562
7a3e97b0 2563 spin_lock_irqsave(hba->host->host_lock, flags);
5586dd8e
CG		 2564 switch (hba->ufshcd_state) {
2565 case UFSHCD_STATE_OPERATIONAL:
2566 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2567 break;
2568 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2569 /*
2570 * pm_runtime_get_sync() is used at error handling preparation
2571 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2572 * PM ops, it can never be finished if we let SCSI layer keep
2573 * retrying it, which gets err handler stuck forever. Neither
2574 * can we let the scsi cmd pass through, because UFS is in bad
2575 * state, the scsi cmd may eventually time out, which will get
2576 * err handler blocked for too long. So, just fail the scsi cmd
2577 * sent from PM ops, err handler can recover PM error anyways.
2578 */
2579 if (hba->pm_op_in_progress) {
2580 hba->force_reset = true;
2581 set_host_byte(cmd, DID_BAD_TARGET);
2582 goto out_compl_cmd;
2583 }
2584 fallthrough;
2585 case UFSHCD_STATE_RESET:
2586 err = SCSI_MLQUEUE_HOST_BUSY;
2587 goto out_compl_cmd;
2588 case UFSHCD_STATE_ERROR:
2589 set_host_byte(cmd, DID_ERROR);
2590 goto out_compl_cmd;
2591 default:
2592 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2593 __func__, hba->ufshcd_state);
2594 set_host_byte(cmd, DID_BAD_TARGET);
2595 goto out_compl_cmd;
2596 }
7a3e97b0
SY		 2597 ufshcd_send_command(hba, tag);
2598 spin_unlock_irqrestore(hba->host->host_lock, flags);
5586dd8e
CG		 2599 goto out;
2600
2601out_compl_cmd:
2602 scsi_dma_unmap(lrbp->cmd);
2603 lrbp->cmd = NULL;
2604 spin_unlock_irqrestore(hba->host->host_lock, flags);
2605 ufshcd_release(hba);
2606 if (!err)
2607 cmd->scsi_done(cmd);
7a3e97b0 2608out:
a3cd5ec5 2609 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2610 return err;
2611}
2612
5a0b0cb9
SRT		 2613static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2614 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2615{
2616 lrbp->cmd = NULL;
2617 lrbp->sense_bufflen = 0;
2618 lrbp->sense_buffer = NULL;
2619 lrbp->task_tag = tag;
2620 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9 2621 lrbp->intr_cmd = true; /* No interrupt aggregation */
df043c74 2622 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5a0b0cb9
SRT		 2623 hba->dev_cmd.type = cmd_type;
2624
f273c54b 2625 return ufshcd_compose_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2626}
2627
2628static int
2629ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2630{
2631 int err = 0;
2632 unsigned long flags;
2633 u32 mask = 1 << tag;
2634
2635 /* clear outstanding transaction before retry */
2636 spin_lock_irqsave(hba->host->host_lock, flags);
2637 ufshcd_utrl_clear(hba, tag);
2638 spin_unlock_irqrestore(hba->host->host_lock, flags);
2639
2640 /*
2641 * wait for for h/w to clear corresponding bit in door-bell.
2642 * max. wait is 1 sec.
2643 */
2644 err = ufshcd_wait_for_register(hba,
2645 REG_UTP_TRANSFER_REQ_DOOR_BELL,
5cac1095 2646 mask, ~mask, 1000, 1000);
5a0b0cb9
SRT		 2647
2648 return err;
2649}
2650
c6d4a831
DR		 2651static int
2652ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2653{
2654 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2655
2656 /* Get the UPIU response */
2657 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2658 UPIU_RSP_CODE_OFFSET;
2659 return query_res->response;
2660}
2661
5a0b0cb9
SRT		 2662/**
2663 * ufshcd_dev_cmd_completion() - handles device management command responses
2664 * @hba: per adapter instance
2665 * @lrbp: pointer to local reference block
2666 */
2667static int
2668ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2669{
2670 int resp;
2671 int err = 0;
2672
ff8e20c6 2673 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2674 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2675
2676 switch (resp) {
2677 case UPIU_TRANSACTION_NOP_IN:
2678 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2679 err = -EINVAL;
2680 dev_err(hba->dev, "%s: unexpected response %x\n",
2681 __func__, resp);
2682 }
2683 break;
68078d5c 2684 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2685 err = ufshcd_check_query_response(hba, lrbp);
2686 if (!err)
2687 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2688 break;
5a0b0cb9
SRT		 2689 case UPIU_TRANSACTION_REJECT_UPIU:
2690 /* TODO: handle Reject UPIU Response */
2691 err = -EPERM;
2692 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2693 __func__);
2694 break;
2695 default:
2696 err = -EINVAL;
2697 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2698 __func__, resp);
2699 break;
2700 }
2701
2702 return err;
2703}
2704
2705static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2706 struct ufshcd_lrb *lrbp, int max_timeout)
2707{
2708 int err = 0;
2709 unsigned long time_left;
2710 unsigned long flags;
2711
2712 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2713 msecs_to_jiffies(max_timeout));
2714
ad1a1b9c
GB		 2715 /* Make sure descriptors are ready before ringing the doorbell */
2716 wmb();
5a0b0cb9
SRT		 2717 spin_lock_irqsave(hba->host->host_lock, flags);
2718 hba->dev_cmd.complete = NULL;
2719 if (likely(time_left)) {
2720 err = ufshcd_get_tr_ocs(lrbp);
2721 if (!err)
2722 err = ufshcd_dev_cmd_completion(hba, lrbp);
2723 }
2724 spin_unlock_irqrestore(hba->host->host_lock, flags);
2725
2726 if (!time_left) {
2727 err = -ETIMEDOUT;
a48353f6
YG		 2728 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2729 __func__, lrbp->task_tag);
5a0b0cb9 2730 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2731 /* successfully cleared the command, retry if needed */
5a0b0cb9 2732 err = -EAGAIN;
a48353f6
YG		 2733 /*
2734 * in case of an error, after clearing the doorbell,
2735 * we also need to clear the outstanding_request
2736 * field in hba
2737 */
2738 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2739 }
2740
2741 return err;
2742}
2743
5a0b0cb9
SRT		 2744/**
2745 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2746 * @hba: UFS hba
2747 * @cmd_type: specifies the type (NOP, Query...)
2748 * @timeout: time in seconds
5a0b0cb9 2749 *
68078d5c
DR		 2750 * NOTE: Since there is only one available tag for device management commands,
2751 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2752 */
2753static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2754 enum dev_cmd_type cmd_type, int timeout)
2755{
7252a360
BVA		 2756 struct request_queue *q = hba->cmd_queue;
2757 struct request *req;
5a0b0cb9
SRT		 2758 struct ufshcd_lrb *lrbp;
2759 int err;
2760 int tag;
2761 struct completion wait;
2762 unsigned long flags;
2763
a3cd5ec5 2764 down_read(&hba->clk_scaling_lock);
2765
5a0b0cb9
SRT		 2766 /*
2767 * Get free slot, sleep if slots are unavailable.
2768 * Even though we use wait_event() which sleeps indefinitely,
2769 * the maximum wait time is bounded by SCSI request timeout.
2770 */
7252a360 2771 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 2772 if (IS_ERR(req)) {
2773 err = PTR_ERR(req);
2774 goto out_unlock;
2775 }
7252a360
BVA		 2776 tag = req->tag;
2777 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5a0b0cb9
SRT		 2778
2779 init_completion(&wait);
2780 lrbp = &hba->lrb[tag];
2781 WARN_ON(lrbp->cmd);
2782 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2783 if (unlikely(err))
2784 goto out_put_tag;
2785
2786 hba->dev_cmd.complete = &wait;
2787
6667e6d9 2788 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
e3dfdc53
YG		 2789 /* Make sure descriptors are ready before ringing the doorbell */
2790 wmb();
5a0b0cb9
SRT		 2791 spin_lock_irqsave(hba->host->host_lock, flags);
2792 ufshcd_send_command(hba, tag);
2793 spin_unlock_irqrestore(hba->host->host_lock, flags);
2794
2795 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2796
6667e6d9
OS		 2797 ufshcd_add_query_upiu_trace(hba, tag,
2798 err ? "query_complete_err" : "query_complete");
2799
5a0b0cb9 2800out_put_tag:
7252a360 2801 blk_put_request(req);
bb14dd15 2802out_unlock:
a3cd5ec5 2803 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2804 return err;
2805}
2806
d44a5f98
DR		 2807/**
2808 * ufshcd_init_query() - init the query response and request parameters
2809 * @hba: per-adapter instance
2810 * @request: address of the request pointer to be initialized
2811 * @response: address of the response pointer to be initialized
2812 * @opcode: operation to perform
2813 * @idn: flag idn to access
2814 * @index: LU number to access
2815 * @selector: query/flag/descriptor further identification
2816 */
2817static inline void ufshcd_init_query(struct ufs_hba *hba,
2818 struct ufs_query_req **request, struct ufs_query_res **response,
2819 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2820{
2821 *request = &hba->dev_cmd.query.request;
2822 *response = &hba->dev_cmd.query.response;
2823 memset(*request, 0, sizeof(struct ufs_query_req));
2824 memset(*response, 0, sizeof(struct ufs_query_res));
2825 (*request)->upiu_req.opcode = opcode;
2826 (*request)->upiu_req.idn = idn;
2827 (*request)->upiu_req.index = index;
2828 (*request)->upiu_req.selector = selector;
2829}
2830
dc3c8d3a 2831static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1f34eedf 2832 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
dc3c8d3a
YG		 2833{
2834 int ret;
2835 int retries;
2836
2837 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1f34eedf 2838 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
dc3c8d3a
YG		 2839 if (ret)
2840 dev_dbg(hba->dev,
2841 "%s: failed with error %d, retries %d\n",
2842 __func__, ret, retries);
2843 else
2844 break;
2845 }
2846
2847 if (ret)
2848 dev_err(hba->dev,
2849 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2850 __func__, opcode, idn, ret, retries);
2851 return ret;
2852}
2853
68078d5c
DR		 2854/**
2855 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2856 * @hba: per-adapter instance
2857 * @opcode: flag query to perform
2858 * @idn: flag idn to access
1f34eedf 2859 * @index: flag index to access
8aa29f19 2860 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2861 *
2862 * Returns 0 for success, non-zero in case of failure
2863 */
dc3c8d3a 2864int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1f34eedf 2865 enum flag_idn idn, u8 index, bool *flag_res)
68078d5c 2866{
d44a5f98
DR		 2867 struct ufs_query_req *request = NULL;
2868 struct ufs_query_res *response = NULL;
1f34eedf 2869 int err, selector = 0;
e5ad406c 2870 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2871
2872 BUG_ON(!hba);
2873
1ab27c9c 2874 ufshcd_hold(hba, false);
68078d5c 2875 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2876 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2877 selector);
68078d5c
DR		 2878
2879 switch (opcode) {
2880 case UPIU_QUERY_OPCODE_SET_FLAG:
2881 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2882 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2883 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2884 break;
2885 case UPIU_QUERY_OPCODE_READ_FLAG:
2886 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2887 if (!flag_res) {
2888 /* No dummy reads */
2889 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2890 __func__);
2891 err = -EINVAL;
2892 goto out_unlock;
2893 }
2894 break;
2895 default:
2896 dev_err(hba->dev,
2897 "%s: Expected query flag opcode but got = %d\n",
2898 __func__, opcode);
2899 err = -EINVAL;
2900 goto out_unlock;
2901 }
68078d5c 2902
e5ad406c 2903 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 2904
2905 if (err) {
2906 dev_err(hba->dev,
2907 "%s: Sending flag query for idn %d failed, err = %d\n",
2908 __func__, idn, err);
2909 goto out_unlock;
2910 }
2911
2912 if (flag_res)
e8c8e82a 2913 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 2914 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2915
2916out_unlock:
2917 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 2918 ufshcd_release(hba);
68078d5c
DR		 2919 return err;
2920}
2921
66ec6d59
SRT		 2922/**
2923 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 2924 * @hba: per-adapter instance
2925 * @opcode: attribute opcode
2926 * @idn: attribute idn to access
2927 * @index: index field
2928 * @selector: selector field
2929 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 2930 *
2931 * Returns 0 for success, non-zero in case of failure
2932*/
ec92b59c
SN		 2933int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2934 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 2935{
d44a5f98
DR		 2936 struct ufs_query_req *request = NULL;
2937 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 2938 int err;
2939
2940 BUG_ON(!hba);
2941
1ab27c9c 2942 ufshcd_hold(hba, false);
66ec6d59
SRT		 2943 if (!attr_val) {
2944 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2945 __func__, opcode);
2946 err = -EINVAL;
2947 goto out;
2948 }
2949
2950 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2951 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2952 selector);
66ec6d59
SRT		 2953
2954 switch (opcode) {
2955 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2956 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 2957 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 2958 break;
2959 case UPIU_QUERY_OPCODE_READ_ATTR:
2960 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2961 break;
2962 default:
2963 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2964 __func__, opcode);
2965 err = -EINVAL;
2966 goto out_unlock;
2967 }
2968
d44a5f98 2969 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 2970
2971 if (err) {
4b761b58
YG		 2972 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2973 __func__, opcode, idn, index, err);
66ec6d59
SRT		 2974 goto out_unlock;
2975 }
2976
e8c8e82a 2977 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 2978
2979out_unlock:
2980 mutex_unlock(&hba->dev_cmd.lock);
2981out:
1ab27c9c 2982 ufshcd_release(hba);
66ec6d59
SRT		 2983 return err;
2984}
2985
5e86ae44
YG		 2986/**
2987 * ufshcd_query_attr_retry() - API function for sending query
2988 * attribute with retries
2989 * @hba: per-adapter instance
2990 * @opcode: attribute opcode
2991 * @idn: attribute idn to access
2992 * @index: index field
2993 * @selector: selector field
2994 * @attr_val: the attribute value after the query request
2995 * completes
2996 *
2997 * Returns 0 for success, non-zero in case of failure
2998*/
2999static int ufshcd_query_attr_retry(struct ufs_hba *hba,
3000 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3001 u32 *attr_val)
3002{
3003 int ret = 0;
3004 u32 retries;
3005
68c9fcfd 3006 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
5e86ae44
YG		 3007 ret = ufshcd_query_attr(hba, opcode, idn, index,
3008 selector, attr_val);
3009 if (ret)
3010 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3011 __func__, ret, retries);
3012 else
3013 break;
3014 }
3015
3016 if (ret)
3017 dev_err(hba->dev,
3018 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3019 __func__, idn, ret, QUERY_REQ_RETRIES);
3020 return ret;
3021}
3022
a70e91b8 3023static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 3024 enum query_opcode opcode, enum desc_idn idn, u8 index,
3025 u8 selector, u8 *desc_buf, int *buf_len)
3026{
3027 struct ufs_query_req *request = NULL;
3028 struct ufs_query_res *response = NULL;
3029 int err;
3030
3031 BUG_ON(!hba);
3032
1ab27c9c 3033 ufshcd_hold(hba, false);
d44a5f98
DR		 3034 if (!desc_buf) {
3035 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3036 __func__, opcode);
3037 err = -EINVAL;
3038 goto out;
3039 }
3040
a4b0e8a4 3041 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 3042 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3043 __func__, *buf_len);
3044 err = -EINVAL;
3045 goto out;
3046 }
3047
3048 mutex_lock(&hba->dev_cmd.lock);
3049 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3050 selector);
3051 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 3052 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 3053
3054 switch (opcode) {
3055 case UPIU_QUERY_OPCODE_WRITE_DESC:
3056 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3057 break;
3058 case UPIU_QUERY_OPCODE_READ_DESC:
3059 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3060 break;
3061 default:
3062 dev_err(hba->dev,
3063 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3064 __func__, opcode);
3065 err = -EINVAL;
3066 goto out_unlock;
3067 }
3068
3069 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3070
3071 if (err) {
4b761b58
YG		 3072 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3073 __func__, opcode, idn, index, err);
d44a5f98
DR		 3074 goto out_unlock;
3075 }
3076
ea2aab24 3077 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 3078
3079out_unlock:
cfcbae38 3080 hba->dev_cmd.query.descriptor = NULL;
d44a5f98
DR		 3081 mutex_unlock(&hba->dev_cmd.lock);
3082out:
1ab27c9c 3083 ufshcd_release(hba);
d44a5f98
DR		 3084 return err;
3085}
3086
a70e91b8 3087/**
8aa29f19
BVA		 3088 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3089 * @hba: per-adapter instance
3090 * @opcode: attribute opcode
3091 * @idn: attribute idn to access
3092 * @index: index field
3093 * @selector: selector field
3094 * @desc_buf: the buffer that contains the descriptor
3095 * @buf_len: length parameter passed to the device
a70e91b8
YG		 3096 *
3097 * Returns 0 for success, non-zero in case of failure.
3098 * The buf_len parameter will contain, on return, the length parameter
3099 * received on the response.
3100 */
2238d31c
SN		 3101int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3102 enum query_opcode opcode,
3103 enum desc_idn idn, u8 index,
3104 u8 selector,
3105 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 3106{
3107 int err;
3108 int retries;
3109
3110 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3111 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3112 selector, desc_buf, buf_len);
3113 if (!err || err == -EINVAL)
3114 break;
3115 }
3116
3117 return err;
3118}
a70e91b8 3119
a4b0e8a4
PM		 3120/**
3121 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3122 * @hba: Pointer to adapter instance
3123 * @desc_id: descriptor idn value
3124 * @desc_len: mapped desc length (out)
a4b0e8a4 3125 */
7a0bf85b
BH		 3126void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3127 int *desc_len)
a4b0e8a4 3128{
7a0bf85b
BH		 3129 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3130 desc_id == QUERY_DESC_IDN_RFU_1)
a4b0e8a4 3131 *desc_len = 0;
7a0bf85b
BH		 3132 else
3133 *desc_len = hba->desc_size[desc_id];
a4b0e8a4
PM		 3134}
3135EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3136
7a0bf85b 3137static void ufshcd_update_desc_length(struct ufs_hba *hba,
72fb690e 3138 enum desc_idn desc_id, int desc_index,
7a0bf85b
BH		 3139 unsigned char desc_len)
3140{
3141 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
72fb690e
BH		 3142 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3143 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3144 * than the RPMB unit, however, both descriptors share the same
3145 * desc_idn, to cover both unit descriptors with one length, we
3146 * choose the normal unit descriptor length by desc_index.
3147 */
7a0bf85b
BH		 3148 hba->desc_size[desc_id] = desc_len;
3149}
3150
da461cec
SJ		 3151/**
3152 * ufshcd_read_desc_param - read the specified descriptor parameter
3153 * @hba: Pointer to adapter instance
3154 * @desc_id: descriptor idn value
3155 * @desc_index: descriptor index
3156 * @param_offset: offset of the parameter to read
3157 * @param_read_buf: pointer to buffer where parameter would be read
3158 * @param_size: sizeof(param_read_buf)
3159 *
3160 * Return 0 in case of success, non-zero otherwise
3161 */
45bced87
SN		 3162int ufshcd_read_desc_param(struct ufs_hba *hba,
3163 enum desc_idn desc_id,
3164 int desc_index,
3165 u8 param_offset,
3166 u8 *param_read_buf,
3167 u8 param_size)
da461cec
SJ		 3168{
3169 int ret;
3170 u8 *desc_buf;
a4b0e8a4 3171 int buff_len;
da461cec
SJ		 3172 bool is_kmalloc = true;
3173
a4b0e8a4
PM		 3174 /* Safety check */
3175 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3176 return -EINVAL;
3177
7a0bf85b
BH		 3178 /* Get the length of descriptor */
3179 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3180 if (!buff_len) {
3181 dev_err(hba->dev, "%s: Failed to get desc length", __func__);
3182 return -EINVAL;
a4b0e8a4
PM		 3183 }
3184
3185 /* Check whether we need temp memory */
3186 if (param_offset != 0 || param_size < buff_len) {
da461cec
SJ		 3187 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3188 if (!desc_buf)
3189 return -ENOMEM;
a4b0e8a4
PM		 3190 } else {
3191 desc_buf = param_read_buf;
3192 is_kmalloc = false;
da461cec
SJ		 3193 }
3194
a4b0e8a4 3195 /* Request for full descriptor */
a70e91b8 3196 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3197 desc_id, desc_index, 0,
3198 desc_buf, &buff_len);
da461cec 3199
bde44bb6 3200 if (ret) {
3201 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3202 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3203 goto out;
3204 }
3205
bde44bb6 3206 /* Sanity check */
3207 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3208 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3209 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3210 ret = -EINVAL;
3211 goto out;
3212 }
3213
7a0bf85b
BH		 3214 /* Update descriptor length */
3215 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
72fb690e 3216 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
7a0bf85b 3217
a4b0e8a4 3218 /* Check wherher we will not copy more data, than available */
cbe193f6
BH		 3219 if (is_kmalloc && (param_offset + param_size) > buff_len)
3220 param_size = buff_len - param_offset;
bde44bb6 3221
da461cec
SJ		 3222 if (is_kmalloc)
3223 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3224out:
3225 if (is_kmalloc)
3226 kfree(desc_buf);
3227 return ret;
3228}
3229
4b828fe1
TW		 3230/**
3231 * struct uc_string_id - unicode string
3232 *
3233 * @len: size of this descriptor inclusive
3234 * @type: descriptor type
3235 * @uc: unicode string character
3236 */
3237struct uc_string_id {
3238 u8 len;
3239 u8 type;
ec38c0ad 3240 wchar_t uc[];
4b828fe1
TW		 3241} __packed;
3242
3243/* replace non-printable or non-ASCII characters with spaces */
3244static inline char ufshcd_remove_non_printable(u8 ch)
3245{
3246 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3247}
3248
b573d484
YG		 3249/**
3250 * ufshcd_read_string_desc - read string descriptor
3251 * @hba: pointer to adapter instance
3252 * @desc_index: descriptor index
4b828fe1
TW		 3253 * @buf: pointer to buffer where descriptor would be read,
3254 * the caller should free the memory.
b573d484 3255 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3256 * null terminated string.
b573d484 3257 *
4b828fe1
TW		 3258 * Return:
3259 * * string size on success.
3260 * * -ENOMEM: on allocation failure
3261 * * -EINVAL: on a wrong parameter
b573d484 3262 */
4b828fe1
TW		 3263int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3264 u8 **buf, bool ascii)
b573d484 3265{
4b828fe1
TW		 3266 struct uc_string_id *uc_str;
3267 u8 *str;
3268 int ret;
b573d484 3269
4b828fe1
TW		 3270 if (!buf)
3271 return -EINVAL;
b573d484 3272
4b828fe1
TW		 3273 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3274 if (!uc_str)
3275 return -ENOMEM;
b573d484 3276
c4607a09
BH		 3277 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3278 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
4b828fe1
TW		 3279 if (ret < 0) {
3280 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3281 QUERY_REQ_RETRIES, ret);
3282 str = NULL;
3283 goto out;
3284 }
3285
3286 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3287 dev_dbg(hba->dev, "String Desc is of zero length\n");
3288 str = NULL;
3289 ret = 0;
b573d484
YG		 3290 goto out;
3291 }
3292
3293 if (ascii) {
4b828fe1 3294 ssize_t ascii_len;
b573d484 3295 int i;
b573d484 3296 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3297 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3298 str = kzalloc(ascii_len, GFP_KERNEL);
3299 if (!str) {
3300 ret = -ENOMEM;
fcbefc3b 3301 goto out;
b573d484
YG		 3302 }
3303
3304 /*
3305 * the descriptor contains string in UTF16 format
3306 * we need to convert to utf-8 so it can be displayed
3307 */
4b828fe1
TW		 3308 ret = utf16s_to_utf8s(uc_str->uc,
3309 uc_str->len - QUERY_DESC_HDR_SIZE,
3310 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3311
3312 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3313 for (i = 0; i < ret; i++)
3314 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3315
4b828fe1
TW		 3316 str[ret++] = '\0';
3317
3318 } else {
5f57704d 3319 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3320 if (!str) {
3321 ret = -ENOMEM;
3322 goto out;
3323 }
4b828fe1 3324 ret = uc_str->len;
b573d484
YG		 3325 }
3326out:
4b828fe1
TW		 3327 *buf = str;
3328 kfree(uc_str);
3329 return ret;
b573d484 3330}
b573d484 3331
da461cec
SJ		 3332/**
3333 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3334 * @hba: Pointer to adapter instance
3335 * @lun: lun id
3336 * @param_offset: offset of the parameter to read
3337 * @param_read_buf: pointer to buffer where parameter would be read
3338 * @param_size: sizeof(param_read_buf)
3339 *
3340 * Return 0 in case of success, non-zero otherwise
3341 */
3342static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3343 int lun,
3344 enum unit_desc_param param_offset,
3345 u8 *param_read_buf,
3346 u32 param_size)
3347{
3348 /*
3349 * Unit descriptors are only available for general purpose LUs (LUN id
3350 * from 0 to 7) and RPMB Well known LU.
3351 */
1baa8011 3352 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
da461cec
SJ		 3353 return -EOPNOTSUPP;
3354
3355 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3356 param_offset, param_read_buf, param_size);
3357}
3358
09f17791
CG		 3359static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3360{
3361 int err = 0;
3362 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3363
3364 if (hba->dev_info.wspecversion >= 0x300) {
3365 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3366 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3367 &gating_wait);
3368 if (err)
3369 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3370 err, gating_wait);
3371
3372 if (gating_wait == 0) {
3373 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3374 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3375 gating_wait);
3376 }
3377
3378 hba->dev_info.clk_gating_wait_us = gating_wait;
3379 }
3380
3381 return err;
3382}
3383
7a3e97b0
SY		 3384/**
3385 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3386 * @hba: per adapter instance
3387 *
3388 * 1. Allocate DMA memory for Command Descriptor array
3389 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3390 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3391 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3392 * (UTMRDL)
3393 * 4. Allocate memory for local reference block(lrb).
3394 *
3395 * Returns 0 for success, non-zero in case of failure
3396 */
3397static int ufshcd_memory_alloc(struct ufs_hba *hba)
3398{
3399 size_t utmrdl_size, utrdl_size, ucdl_size;
3400
3401 /* Allocate memory for UTP command descriptors */
3402 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3403 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3404 ucdl_size,
3405 &hba->ucdl_dma_addr,
3406 GFP_KERNEL);
7a3e97b0
SY		 3407
3408 /*
3409 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3410 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3411 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3412 * be aligned to 128 bytes as well
3413 */
3414 if (!hba->ucdl_base_addr ||
3415 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3416 dev_err(hba->dev,
7a3e97b0
SY		 3417 "Command Descriptor Memory allocation failed\n");
3418 goto out;
3419 }
3420
3421 /*
3422 * Allocate memory for UTP Transfer descriptors
3423 * UFSHCI requires 1024 byte alignment of UTRD
3424 */
3425 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3426 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3427 utrdl_size,
3428 &hba->utrdl_dma_addr,
3429 GFP_KERNEL);
7a3e97b0
SY		 3430 if (!hba->utrdl_base_addr ||
3431 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3432 dev_err(hba->dev,
7a3e97b0
SY		 3433 "Transfer Descriptor Memory allocation failed\n");
3434 goto out;
3435 }
3436
3437 /*
3438 * Allocate memory for UTP Task Management descriptors
3439 * UFSHCI requires 1024 byte alignment of UTMRD
3440 */
3441 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3442 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3443 utmrdl_size,
3444 &hba->utmrdl_dma_addr,
3445 GFP_KERNEL);
7a3e97b0
SY		 3446 if (!hba->utmrdl_base_addr ||
3447 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3448 dev_err(hba->dev,
7a3e97b0
SY		 3449 "Task Management Descriptor Memory allocation failed\n");
3450 goto out;
3451 }
3452
3453 /* Allocate memory for local reference block */
a86854d0
KC		 3454 hba->lrb = devm_kcalloc(hba->dev,
3455 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3456 GFP_KERNEL);
7a3e97b0 3457 if (!hba->lrb) {
3b1d0580 3458 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3459 goto out;
3460 }
3461 return 0;
3462out:
7a3e97b0
SY		 3463 return -ENOMEM;
3464}
3465
3466/**
3467 * ufshcd_host_memory_configure - configure local reference block with
3468 * memory offsets
3469 * @hba: per adapter instance
3470 *
3471 * Configure Host memory space
3472 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3473 * address.
3474 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3475 * and PRDT offset.
3476 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3477 * into local reference block.
3478 */
3479static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3480{
7a3e97b0
SY		 3481 struct utp_transfer_req_desc *utrdlp;
3482 dma_addr_t cmd_desc_dma_addr;
3483 dma_addr_t cmd_desc_element_addr;
3484 u16 response_offset;
3485 u16 prdt_offset;
3486 int cmd_desc_size;
3487 int i;
3488
3489 utrdlp = hba->utrdl_base_addr;
7a3e97b0
SY		 3490
3491 response_offset =
3492 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3493 prdt_offset =
3494 offsetof(struct utp_transfer_cmd_desc, prd_table);
3495
3496 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3497 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3498
3499 for (i = 0; i < hba->nutrs; i++) {
3500 /* Configure UTRD with command descriptor base address */
3501 cmd_desc_element_addr =
3502 (cmd_desc_dma_addr + (cmd_desc_size * i));
3503 utrdlp[i].command_desc_base_addr_lo =
3504 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3505 utrdlp[i].command_desc_base_addr_hi =
3506 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3507
3508 /* Response upiu and prdt offset should be in double words */
26f968d7
AA		 3509 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3510 utrdlp[i].response_upiu_offset =
3511 cpu_to_le16(response_offset);
3512 utrdlp[i].prd_table_offset =
3513 cpu_to_le16(prdt_offset);
3514 utrdlp[i].response_upiu_length =
3515 cpu_to_le16(ALIGNED_UPIU_SIZE);
3516 } else {
3517 utrdlp[i].response_upiu_offset =
3518 cpu_to_le16(response_offset >> 2);
3519 utrdlp[i].prd_table_offset =
3520 cpu_to_le16(prdt_offset >> 2);
3521 utrdlp[i].response_upiu_length =
3522 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3523 }
7a3e97b0 3524
4d2b8d40 3525 ufshcd_init_lrb(hba, &hba->lrb[i], i);
7a3e97b0
SY		 3526 }
3527}
3528
3529/**
3530 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3531 * @hba: per adapter instance
3532 *
3533 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3534 * in order to initialize the Unipro link startup procedure.
3535 * Once the Unipro links are up, the device connected to the controller
3536 * is detected.
3537 *
3538 * Returns 0 on success, non-zero value on failure
3539 */
3540static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3541{
6ccf44fe
SJ		 3542 struct uic_command uic_cmd = {0};
3543 int ret;
7a3e97b0 3544
6ccf44fe 3545 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3546
6ccf44fe
SJ		 3547 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3548 if (ret)
ff8e20c6 3549 dev_dbg(hba->dev,
6ccf44fe
SJ		 3550 "dme-link-startup: error code %d\n", ret);
3551 return ret;
7a3e97b0 3552}
39bf2d83
AA		 3553/**
3554 * ufshcd_dme_reset - UIC command for DME_RESET
3555 * @hba: per adapter instance
3556 *
3557 * DME_RESET command is issued in order to reset UniPro stack.
3558 * This function now deals with cold reset.
3559 *
3560 * Returns 0 on success, non-zero value on failure
3561 */
3562static int ufshcd_dme_reset(struct ufs_hba *hba)
3563{
3564 struct uic_command uic_cmd = {0};
3565 int ret;
3566
3567 uic_cmd.command = UIC_CMD_DME_RESET;
3568
3569 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3570 if (ret)
3571 dev_err(hba->dev,
3572 "dme-reset: error code %d\n", ret);
3573
3574 return ret;
3575}
3576
3577/**
3578 * ufshcd_dme_enable - UIC command for DME_ENABLE
3579 * @hba: per adapter instance
3580 *
3581 * DME_ENABLE command is issued in order to enable UniPro stack.
3582 *
3583 * Returns 0 on success, non-zero value on failure
3584 */
3585static int ufshcd_dme_enable(struct ufs_hba *hba)
3586{
3587 struct uic_command uic_cmd = {0};
3588 int ret;
3589
3590 uic_cmd.command = UIC_CMD_DME_ENABLE;
3591
3592 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3593 if (ret)
3594 dev_err(hba->dev,
3595 "dme-reset: error code %d\n", ret);
3596
3597 return ret;
3598}
7a3e97b0 3599
cad2e03d
YG		 3600static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3601{
3602 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3603 unsigned long min_sleep_time_us;
3604
3605 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3606 return;
3607
3608 /*
3609 * last_dme_cmd_tstamp will be 0 only for 1st call to
3610 * this function
3611 */
3612 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3613 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3614 } else {
3615 unsigned long delta =
3616 (unsigned long) ktime_to_us(
3617 ktime_sub(ktime_get(),
3618 hba->last_dme_cmd_tstamp));
3619
3620 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3621 min_sleep_time_us =
3622 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3623 else
3624 return; /* no more delay required */
3625 }
3626
3627 /* allow sleep for extra 50us if needed */
3628 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3629}
3630
12b4fdb4
SJ		 3631/**
3632 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3633 * @hba: per adapter instance
3634 * @attr_sel: uic command argument1
3635 * @attr_set: attribute set type as uic command argument2
3636 * @mib_val: setting value as uic command argument3
3637 * @peer: indicate whether peer or local
3638 *
3639 * Returns 0 on success, non-zero value on failure
3640 */
3641int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3642 u8 attr_set, u32 mib_val, u8 peer)
3643{
3644 struct uic_command uic_cmd = {0};
3645 static const char *const action[] = {
3646 "dme-set",
3647 "dme-peer-set"
3648 };
3649 const char *set = action[!!peer];
3650 int ret;
64238fbd 3651 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3652
3653 uic_cmd.command = peer ?
3654 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3655 uic_cmd.argument1 = attr_sel;
3656 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3657 uic_cmd.argument3 = mib_val;
3658
64238fbd
YG		 3659 do {
3660 /* for peer attributes we retry upon failure */
3661 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3662 if (ret)
3663 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3664 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3665 } while (ret && peer && --retries);
3666
f37e9f8c 3667 if (ret)
64238fbd 3668 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3669 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3670 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3671
3672 return ret;
3673}
3674EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3675
3676/**
3677 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3678 * @hba: per adapter instance
3679 * @attr_sel: uic command argument1
3680 * @mib_val: the value of the attribute as returned by the UIC command
3681 * @peer: indicate whether peer or local
3682 *
3683 * Returns 0 on success, non-zero value on failure
3684 */
3685int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3686 u32 *mib_val, u8 peer)
3687{
3688 struct uic_command uic_cmd = {0};
3689 static const char *const action[] = {
3690 "dme-get",
3691 "dme-peer-get"
3692 };
3693 const char *get = action[!!peer];
3694 int ret;
64238fbd 3695 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3696 struct ufs_pa_layer_attr orig_pwr_info;
3697 struct ufs_pa_layer_attr temp_pwr_info;
3698 bool pwr_mode_change = false;
3699
3700 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3701 orig_pwr_info = hba->pwr_info;
3702 temp_pwr_info = orig_pwr_info;
3703
3704 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3705 orig_pwr_info.pwr_rx == FAST_MODE) {
3706 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3707 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3708 pwr_mode_change = true;
3709 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3710 orig_pwr_info.pwr_rx == SLOW_MODE) {
3711 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3712 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3713 pwr_mode_change = true;
3714 }
3715 if (pwr_mode_change) {
3716 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3717 if (ret)
3718 goto out;
3719 }
3720 }
12b4fdb4
SJ		 3721
3722 uic_cmd.command = peer ?
3723 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3724 uic_cmd.argument1 = attr_sel;
3725
64238fbd
YG		 3726 do {
3727 /* for peer attributes we retry upon failure */
3728 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3729 if (ret)
3730 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3731 get, UIC_GET_ATTR_ID(attr_sel), ret);
3732 } while (ret && peer && --retries);
3733
f37e9f8c 3734 if (ret)
64238fbd 3735 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3736 get, UIC_GET_ATTR_ID(attr_sel),
3737 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3738
64238fbd 3739 if (mib_val && !ret)
12b4fdb4 3740 *mib_val = uic_cmd.argument3;
874237f7
YG		 3741
3742 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3743 && pwr_mode_change)
3744 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3745out:
3746 return ret;
3747}
3748EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3749
53b3d9c3 3750/**
57d104c1
SJ		 3751 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3752 * state) and waits for it to take effect.
3753 *
53b3d9c3 3754 * @hba: per adapter instance
57d104c1
SJ		 3755 * @cmd: UIC command to execute
3756 *
3757 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3758 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3759 * and device UniPro link and hence it's final completion would be indicated by
3760 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3761 * addition to normal UIC command completion Status (UCCS). This function only
3762 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3763 *
3764 * Returns 0 on success, non-zero value on failure
3765 */
57d104c1 3766static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3767{
57d104c1 3768 struct completion uic_async_done;
53b3d9c3
SJ		 3769 unsigned long flags;
3770 u8 status;
3771 int ret;
d75f7fe4 3772 bool reenable_intr = false;
53b3d9c3 3773
53b3d9c3 3774 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3775 init_completion(&uic_async_done);
cad2e03d 3776 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3777
3778 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236
CG		 3779 if (ufshcd_is_link_broken(hba)) {
3780 ret = -ENOLINK;
3781 goto out_unlock;
3782 }
57d104c1 3783 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3784 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3785 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3786 /*
3787 * Make sure UIC command completion interrupt is disabled before
3788 * issuing UIC command.
3789 */
3790 wmb();
3791 reenable_intr = true;
57d104c1 3792 }
d75f7fe4
YG		 3793 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3794 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3795 if (ret) {
3796 dev_err(hba->dev,
3797 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3798 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3799 goto out;
3800 }
3801
57d104c1 3802 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3803 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3804 dev_err(hba->dev,
57d104c1
SJ		 3805 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3806 cmd->command, cmd->argument3);
53b3d9c3
SJ		 3807 ret = -ETIMEDOUT;
3808 goto out;
3809 }
3810
3811 status = ufshcd_get_upmcrs(hba);
3812 if (status != PWR_LOCAL) {
3813 dev_err(hba->dev,
479da360 3814 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3815 cmd->command, status);
53b3d9c3
SJ		 3816 ret = (status != PWR_OK) ? status : -1;
3817 }
3818out:
7942f7b5
VG		 3819 if (ret) {
3820 ufshcd_print_host_state(hba);
3821 ufshcd_print_pwr_info(hba);
3822 ufshcd_print_host_regs(hba);
3823 }
3824
53b3d9c3 3825 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3826 hba->active_uic_cmd = NULL;
57d104c1 3827 hba->uic_async_done = NULL;
d75f7fe4
YG		 3828 if (reenable_intr)
3829 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4db7a236
CG		 3830 if (ret) {
3831 ufshcd_set_link_broken(hba);
3832 ufshcd_schedule_eh_work(hba);
3833 }
3834out_unlock:
53b3d9c3
SJ		 3835 spin_unlock_irqrestore(hba->host->host_lock, flags);
3836 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3837
53b3d9c3
SJ		 3838 return ret;
3839}
3840
57d104c1
SJ		 3841/**
3842 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3843 * using DME_SET primitives.
3844 * @hba: per adapter instance
3845 * @mode: powr mode value
3846 *
3847 * Returns 0 on success, non-zero value on failure
3848 */
3849static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3850{
3851 struct uic_command uic_cmd = {0};
1ab27c9c 3852 int ret;
57d104c1 3853
c3a2f9ee
YG		 3854 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3855 ret = ufshcd_dme_set(hba,
3856 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3857 if (ret) {
3858 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3859 __func__, ret);
3860 goto out;
3861 }
3862 }
3863
57d104c1
SJ		 3864 uic_cmd.command = UIC_CMD_DME_SET;
3865 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3866 uic_cmd.argument3 = mode;
1ab27c9c
ST		 3867 ufshcd_hold(hba, false);
3868 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3869 ufshcd_release(hba);
57d104c1 3870
c3a2f9ee 3871out:
1ab27c9c 3872 return ret;
57d104c1
SJ		 3873}
3874
087c5efa 3875int ufshcd_link_recovery(struct ufs_hba *hba)
53c12d0e
YG		 3876{
3877 int ret;
3878 unsigned long flags;
3879
3880 spin_lock_irqsave(hba->host->host_lock, flags);
3881 hba->ufshcd_state = UFSHCD_STATE_RESET;
3882 ufshcd_set_eh_in_progress(hba);
3883 spin_unlock_irqrestore(hba->host->host_lock, flags);
3884
ebdd1dfd
CG		 3885 /* Reset the attached device */
3886 ufshcd_vops_device_reset(hba);
3887
53c12d0e
YG		 3888 ret = ufshcd_host_reset_and_restore(hba);
3889
3890 spin_lock_irqsave(hba->host->host_lock, flags);
3891 if (ret)
3892 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3893 ufshcd_clear_eh_in_progress(hba);
3894 spin_unlock_irqrestore(hba->host->host_lock, flags);
3895
3896 if (ret)
3897 dev_err(hba->dev, "%s: link recovery failed, err %d",
3898 __func__, ret);
3899
3900 return ret;
3901}
087c5efa 3902EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
53c12d0e 3903
4db7a236 3904static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 3905{
87d0b4a6 3906 int ret;
57d104c1 3907 struct uic_command uic_cmd = {0};
911a0771 3908 ktime_t start = ktime_get();
57d104c1 3909
ee32c909
KK		 3910 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3911
57d104c1 3912 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 3913 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3914 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3915 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 3916
4db7a236 3917 if (ret)
87d0b4a6
YG		 3918 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3919 __func__, ret);
4db7a236 3920 else
ee32c909
KK		 3921 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3922 POST_CHANGE);
53c12d0e 3923
87d0b4a6
YG		 3924 return ret;
3925}
3926
9d19bf7a 3927int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
57d104c1
SJ		 3928{
3929 struct uic_command uic_cmd = {0};
3930 int ret;
911a0771 3931 ktime_t start = ktime_get();
57d104c1 3932
ee32c909
KK		 3933 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3934
57d104c1
SJ		 3935 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3936 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3937 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3938 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3939
57d104c1 3940 if (ret) {
53c12d0e
YG		 3941 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3942 __func__, ret);
ff8e20c6 3943 } else {
ee32c909
KK		 3944 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3945 POST_CHANGE);
ff8e20c6
DR		 3946 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3947 hba->ufs_stats.hibern8_exit_cnt++;
3948 }
57d104c1
SJ		 3949
3950 return ret;
3951}
9d19bf7a 3952EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
57d104c1 3953
ba7af5ec
SC		 3954void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
3955{
3956 unsigned long flags;
be7594a4 3957 bool update = false;
ba7af5ec 3958
be7594a4 3959 if (!ufshcd_is_auto_hibern8_supported(hba))
ba7af5ec
SC		 3960 return;
3961
3962 spin_lock_irqsave(hba->host->host_lock, flags);
be7594a4
CG		 3963 if (hba->ahit != ahit) {
3964 hba->ahit = ahit;
3965 update = true;
3966 }
ba7af5ec 3967 spin_unlock_irqrestore(hba->host->host_lock, flags);
be7594a4
CG		 3968
3969 if (update && !pm_runtime_suspended(hba->dev)) {
3970 pm_runtime_get_sync(hba->dev);
3971 ufshcd_hold(hba, false);
3972 ufshcd_auto_hibern8_enable(hba);
3973 ufshcd_release(hba);
3974 pm_runtime_put(hba->dev);
3975 }
ba7af5ec
SC		 3976}
3977EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
3978
71d848b8 3979void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378
AH		 3980{
3981 unsigned long flags;
3982
ee5f1042 3983 if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
ad448378
AH		 3984 return;
3985
3986 spin_lock_irqsave(hba->host->host_lock, flags);
3987 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3988 spin_unlock_irqrestore(hba->host->host_lock, flags);
3989}
3990
5064636c
YG		 3991 /**
3992 * ufshcd_init_pwr_info - setting the POR (power on reset)
3993 * values in hba power info
3994 * @hba: per-adapter instance
3995 */
3996static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3997{
3998 hba->pwr_info.gear_rx = UFS_PWM_G1;
3999 hba->pwr_info.gear_tx = UFS_PWM_G1;
4000 hba->pwr_info.lane_rx = 1;
4001 hba->pwr_info.lane_tx = 1;
4002 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4003 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4004 hba->pwr_info.hs_rate = 0;
4005}
4006
d3e89bac 4007/**
7eb584db
DR		 4008 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4009 * @hba: per-adapter instance
d3e89bac 4010 */
7eb584db 4011static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 4012{
7eb584db
DR		 4013 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4014
4015 if (hba->max_pwr_info.is_valid)
4016 return 0;
4017
2349b533 4018 pwr_info->pwr_tx = FAST_MODE;
4019 pwr_info->pwr_rx = FAST_MODE;
7eb584db 4020 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 4021
4022 /* Get the connected lane count */
7eb584db
DR		 4023 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4024 &pwr_info->lane_rx);
4025 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4026 &pwr_info->lane_tx);
4027
4028 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4029 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4030 __func__,
4031 pwr_info->lane_rx,
4032 pwr_info->lane_tx);
4033 return -EINVAL;
4034 }
d3e89bac
SJ		 4035
4036 /*
4037 * First, get the maximum gears of HS speed.
4038 * If a zero value, it means there is no HSGEAR capability.
4039 * Then, get the maximum gears of PWM speed.
4040 */
7eb584db
DR		 4041 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4042 if (!pwr_info->gear_rx) {
4043 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4044 &pwr_info->gear_rx);
4045 if (!pwr_info->gear_rx) {
4046 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4047 __func__, pwr_info->gear_rx);
4048 return -EINVAL;
4049 }
2349b533 4050 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 4051 }
4052
7eb584db
DR		 4053 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4054 &pwr_info->gear_tx);
4055 if (!pwr_info->gear_tx) {
d3e89bac 4056 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 4057 &pwr_info->gear_tx);
4058 if (!pwr_info->gear_tx) {
4059 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4060 __func__, pwr_info->gear_tx);
4061 return -EINVAL;
4062 }
2349b533 4063 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 4064 }
4065
4066 hba->max_pwr_info.is_valid = true;
4067 return 0;
4068}
4069
4070static int ufshcd_change_power_mode(struct ufs_hba *hba,
4071 struct ufs_pa_layer_attr *pwr_mode)
4072{
4073 int ret;
4074
4075 /* if already configured to the requested pwr_mode */
2355b66e
CG		 4076 if (!hba->force_pmc &&
4077 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
7eb584db
DR		 4078 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4079 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4080 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4081 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4082 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4083 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4084 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4085 return 0;
d3e89bac
SJ		 4086 }
4087
4088 /*
4089 * Configure attributes for power mode change with below.
4090 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4091 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4092 * - PA_HSSERIES
4093 */
7eb584db
DR		 4094 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4095 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4096 pwr_mode->lane_rx);
4097 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4098 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4099 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4100 else
4101 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4102
7eb584db
DR		 4103 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4104 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4105 pwr_mode->lane_tx);
4106 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4107 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4108 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4109 else
4110 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4111
7eb584db
DR		 4112 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4113 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4114 pwr_mode->pwr_rx == FAST_MODE ||
4115 pwr_mode->pwr_tx == FAST_MODE)
4116 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4117 pwr_mode->hs_rate);
d3e89bac 4118
08342537
CG		 4119 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4120 DL_FC0ProtectionTimeOutVal_Default);
4121 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4122 DL_TC0ReplayTimeOutVal_Default);
4123 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4124 DL_AFC0ReqTimeOutVal_Default);
4125 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4126 DL_FC1ProtectionTimeOutVal_Default);
4127 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4128 DL_TC1ReplayTimeOutVal_Default);
4129 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4130 DL_AFC1ReqTimeOutVal_Default);
4131
4132 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4133 DL_FC0ProtectionTimeOutVal_Default);
4134 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4135 DL_TC0ReplayTimeOutVal_Default);
4136 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4137 DL_AFC0ReqTimeOutVal_Default);
4138
7eb584db
DR		 4139 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4140 | pwr_mode->pwr_tx);
4141
4142 if (ret) {
d3e89bac 4143 dev_err(hba->dev,
7eb584db
DR		 4144 "%s: power mode change failed %d\n", __func__, ret);
4145 } else {
0263bcd0
YG		 4146 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4147 pwr_mode);
7eb584db
DR		 4148
4149 memcpy(&hba->pwr_info, pwr_mode,
4150 sizeof(struct ufs_pa_layer_attr));
4151 }
4152
4153 return ret;
4154}
4155
4156/**
4157 * ufshcd_config_pwr_mode - configure a new power mode
4158 * @hba: per-adapter instance
4159 * @desired_pwr_mode: desired power configuration
4160 */
0d846e70 4161int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4162 struct ufs_pa_layer_attr *desired_pwr_mode)
4163{
4164 struct ufs_pa_layer_attr final_params = { 0 };
4165 int ret;
4166
0263bcd0
YG		 4167 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4168 desired_pwr_mode, &final_params);
4169
4170 if (ret)
7eb584db
DR		 4171 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4172
4173 ret = ufshcd_change_power_mode(hba, &final_params);
d3e89bac
SJ		 4174
4175 return ret;
4176}
0d846e70 4177EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4178
68078d5c
DR		 4179/**
4180 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4181 * @hba: per-adapter instance
68078d5c
DR		 4182 *
4183 * Set fDeviceInit flag and poll until device toggles it.
4184 */
4185static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4186{
dc3c8d3a 4187 int err;
7dfdcc39 4188 bool flag_res = true;
29707fab 4189 ktime_t timeout;
68078d5c 4190
dc3c8d3a 4191 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 4192 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
68078d5c
DR		 4193 if (err) {
4194 dev_err(hba->dev,
4195 "%s setting fDeviceInit flag failed with error %d\n",
4196 __func__, err);
4197 goto out;
4198 }
4199
29707fab
KK		 4200 /* Poll fDeviceInit flag to be cleared */
4201 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4202 do {
4203 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4204 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4205 if (!flag_res)
4206 break;
4207 usleep_range(5000, 10000);
4208 } while (ktime_before(ktime_get(), timeout));
dc3c8d3a 4209
29707fab 4210 if (err) {
68078d5c 4211 dev_err(hba->dev,
29707fab
KK		 4212 "%s reading fDeviceInit flag failed with error %d\n",
4213 __func__, err);
4214 } else if (flag_res) {
68078d5c 4215 dev_err(hba->dev,
29707fab
KK		 4216 "%s fDeviceInit was not cleared by the device\n",
4217 __func__);
4218 err = -EBUSY;
4219 }
68078d5c
DR		 4220out:
4221 return err;
4222}
4223
7a3e97b0
SY		 4224/**
4225 * ufshcd_make_hba_operational - Make UFS controller operational
4226 * @hba: per adapter instance
4227 *
4228 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4229 * 1. Enable required interrupts
4230 * 2. Configure interrupt aggregation
897efe62 4231 * 3. Program UTRL and UTMRL base address
5c0c28a8 4232 * 4. Configure run-stop-registers
7a3e97b0
SY		 4233 *
4234 * Returns 0 on success, non-zero value on failure
4235 */
9d19bf7a 4236int ufshcd_make_hba_operational(struct ufs_hba *hba)
7a3e97b0
SY		 4237{
4238 int err = 0;
4239 u32 reg;
4240
6ccf44fe
SJ		 4241 /* Enable required interrupts */
4242 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4243
4244 /* Configure interrupt aggregation */
b852190e
YG		 4245 if (ufshcd_is_intr_aggr_allowed(hba))
4246 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4247 else
4248 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4249
4250 /* Configure UTRL and UTMRL base address registers */
4251 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4252 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4253 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4254 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4255 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4256 REG_UTP_TASK_REQ_LIST_BASE_L);
4257 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4258 REG_UTP_TASK_REQ_LIST_BASE_H);
4259
897efe62
YG		 4260 /*
4261 * Make sure base address and interrupt setup are updated before
4262 * enabling the run/stop registers below.
4263 */
4264 wmb();
4265
7a3e97b0
SY		 4266 /*
4267 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4268 */
5c0c28a8 4269 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4270 if (!(ufshcd_get_lists_status(reg))) {
4271 ufshcd_enable_run_stop_reg(hba);
4272 } else {
3b1d0580 4273 dev_err(hba->dev,
7a3e97b0
SY		 4274 "Host controller not ready to process requests");
4275 err = -EIO;
7a3e97b0
SY		 4276 }
4277
7a3e97b0
SY		 4278 return err;
4279}
9d19bf7a 4280EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
7a3e97b0 4281
596585a2
YG		 4282/**
4283 * ufshcd_hba_stop - Send controller to reset state
4284 * @hba: per adapter instance
596585a2 4285 */
5cac1095 4286static inline void ufshcd_hba_stop(struct ufs_hba *hba)
596585a2 4287{
5cac1095 4288 unsigned long flags;
596585a2
YG		 4289 int err;
4290
5cac1095
BVA		 4291 /*
4292 * Obtain the host lock to prevent that the controller is disabled
4293 * while the UFS interrupt handler is active on another CPU.
4294 */
4295 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 4296 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
5cac1095
BVA		 4297 spin_unlock_irqrestore(hba->host->host_lock, flags);
4298
596585a2
YG		 4299 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4300 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
5cac1095 4301 10, 1);
596585a2
YG		 4302 if (err)
4303 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4304}
4305
7a3e97b0 4306/**
39bf2d83 4307 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4308 * @hba: per adapter instance
4309 *
4310 * The controller resets itself and controller firmware initialization
4311 * sequence kicks off. When controller is ready it will set
4312 * the Host Controller Enable bit to 1.
4313 *
4314 * Returns 0 on success, non-zero value on failure
4315 */
39bf2d83 4316static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0
SY		 4317{
4318 int retry;
4319
596585a2 4320 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4321 /* change controller state to "reset state" */
5cac1095 4322 ufshcd_hba_stop(hba);
7a3e97b0 4323
57d104c1
SJ		 4324 /* UniPro link is disabled at this point */
4325 ufshcd_set_link_off(hba);
4326
0263bcd0 4327 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4328
7a3e97b0
SY		 4329 /* start controller initialization sequence */
4330 ufshcd_hba_start(hba);
4331
4332 /*
4333 * To initialize a UFS host controller HCE bit must be set to 1.
4334 * During initialization the HCE bit value changes from 1->0->1.
4335 * When the host controller completes initialization sequence
4336 * it sets the value of HCE bit to 1. The same HCE bit is read back
4337 * to check if the controller has completed initialization sequence.
4338 * So without this delay the value HCE = 1, set in the previous
4339 * instruction might be read back.
4340 * This delay can be changed based on the controller.
4341 */
90b8491c 4342 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
7a3e97b0
SY		 4343
4344 /* wait for the host controller to complete initialization */
9fc305ef 4345 retry = 50;
7a3e97b0
SY		 4346 while (ufshcd_is_hba_active(hba)) {
4347 if (retry) {
4348 retry--;
4349 } else {
3b1d0580 4350 dev_err(hba->dev,
7a3e97b0
SY		 4351 "Controller enable failed\n");
4352 return -EIO;
4353 }
9fc305ef 4354 usleep_range(1000, 1100);
7a3e97b0 4355 }
5c0c28a8 4356
1d337ec2 4357 /* enable UIC related interrupts */
57d104c1 4358 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4359
0263bcd0 4360 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4361
7a3e97b0
SY		 4362 return 0;
4363}
39bf2d83
AA		 4364
4365int ufshcd_hba_enable(struct ufs_hba *hba)
4366{
4367 int ret;
4368
4369 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4370 ufshcd_set_link_off(hba);
4371 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4372
4373 /* enable UIC related interrupts */
4374 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4375 ret = ufshcd_dme_reset(hba);
4376 if (!ret) {
4377 ret = ufshcd_dme_enable(hba);
4378 if (!ret)
4379 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4380 if (ret)
4381 dev_err(hba->dev,
4382 "Host controller enable failed with non-hce\n");
4383 }
4384 } else {
4385 ret = ufshcd_hba_execute_hce(hba);
4386 }
4387
4388 return ret;
4389}
9d19bf7a
SC		 4390EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4391
7ca38cf3
YG		 4392static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4393{
ba0320fb 4394 int tx_lanes = 0, i, err = 0;
7ca38cf3
YG		 4395
4396 if (!peer)
4397 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4398 &tx_lanes);
4399 else
4400 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4401 &tx_lanes);
4402 for (i = 0; i < tx_lanes; i++) {
4403 if (!peer)
4404 err = ufshcd_dme_set(hba,
4405 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4406 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4407 0);
4408 else
4409 err = ufshcd_dme_peer_set(hba,
4410 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4411 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4412 0);
4413 if (err) {
4414 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4415 __func__, peer, i, err);
4416 break;
4417 }
4418 }
4419
4420 return err;
4421}
4422
4423static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4424{
4425 return ufshcd_disable_tx_lcc(hba, true);
4426}
4427
a5fe372d
SC		 4428void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4429 u32 reg)
8808b4e9
SC		 4430{
4431 reg_hist->reg[reg_hist->pos] = reg;
4432 reg_hist->tstamp[reg_hist->pos] = ktime_get();
4433 reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4434}
a5fe372d 4435EXPORT_SYMBOL_GPL(ufshcd_update_reg_hist);
8808b4e9 4436
7a3e97b0 4437/**
6ccf44fe 4438 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4439 * @hba: per adapter instance
4440 *
6ccf44fe 4441 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4442 */
6ccf44fe 4443static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4444{
6ccf44fe 4445 int ret;
1d337ec2 4446 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4447 bool link_startup_again = false;
7a3e97b0 4448
7caf489b 4449 /*
4450 * If UFS device isn't active then we will have to issue link startup
4451 * 2 times to make sure the device state move to active.
4452 */
4453 if (!ufshcd_is_ufs_dev_active(hba))
4454 link_startup_again = true;
7a3e97b0 4455
7caf489b 4456link_startup:
1d337ec2 4457 do {
0263bcd0 4458 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4459
1d337ec2 4460 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4461
1d337ec2
SRT		 4462 /* check if device is detected by inter-connect layer */
4463 if (!ret && !ufshcd_is_device_present(hba)) {
8808b4e9
SC		 4464 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4465 0);
1d337ec2
SRT		 4466 dev_err(hba->dev, "%s: Device not present\n", __func__);
4467 ret = -ENXIO;
4468 goto out;
4469 }
6ccf44fe 4470
1d337ec2
SRT		 4471 /*
4472 * DME link lost indication is only received when link is up,
4473 * but we can't be sure if the link is up until link startup
4474 * succeeds. So reset the local Uni-Pro and try again.
4475 */
8808b4e9
SC		 4476 if (ret && ufshcd_hba_enable(hba)) {
4477 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4478 (u32)ret);
1d337ec2 4479 goto out;
8808b4e9 4480 }
1d337ec2
SRT		 4481 } while (ret && retries--);
4482
8808b4e9 4483 if (ret) {
1d337ec2 4484 /* failed to get the link up... retire */
8808b4e9
SC		 4485 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4486 (u32)ret);
5c0c28a8 4487 goto out;
8808b4e9 4488 }
5c0c28a8 4489
7caf489b 4490 if (link_startup_again) {
4491 link_startup_again = false;
4492 retries = DME_LINKSTARTUP_RETRIES;
4493 goto link_startup;
4494 }
4495
d2aebb9b 4496 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4497 ufshcd_init_pwr_info(hba);
4498 ufshcd_print_pwr_info(hba);
4499
7ca38cf3
YG		 4500 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4501 ret = ufshcd_disable_device_tx_lcc(hba);
4502 if (ret)
4503 goto out;
4504 }
4505
5c0c28a8 4506 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4507 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4508 if (ret)
4509 goto out;
7a3e97b0 4510
2355b66e
CG		 4511 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4512 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5c0c28a8 4513 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4514out:
7942f7b5 4515 if (ret) {
6ccf44fe 4516 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4517 ufshcd_print_host_state(hba);
4518 ufshcd_print_pwr_info(hba);
4519 ufshcd_print_host_regs(hba);
4520 }
6ccf44fe 4521 return ret;
7a3e97b0
SY		 4522}
4523
5a0b0cb9
SRT		 4524/**
4525 * ufshcd_verify_dev_init() - Verify device initialization
4526 * @hba: per-adapter instance
4527 *
4528 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4529 * device Transport Protocol (UTP) layer is ready after a reset.
4530 * If the UTP layer at the device side is not initialized, it may
4531 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4532 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4533 */
4534static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4535{
4536 int err = 0;
4537 int retries;
4538
1ab27c9c 4539 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4540 mutex_lock(&hba->dev_cmd.lock);
4541 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4542 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4543 NOP_OUT_TIMEOUT);
4544
4545 if (!err || err == -ETIMEDOUT)
4546 break;
4547
4548 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4549 }
4550 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4551 ufshcd_release(hba);
5a0b0cb9
SRT		 4552
4553 if (err)
4554 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4555 return err;
4556}
4557
0ce147d4
SJ		 4558/**
4559 * ufshcd_set_queue_depth - set lun queue depth
4560 * @sdev: pointer to SCSI device
4561 *
4562 * Read bLUQueueDepth value and activate scsi tagged command
4563 * queueing. For WLUN, queue depth is set to 1. For best-effort
4564 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4565 * value that host can queue.
4566 */
4567static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4568{
4569 int ret = 0;
4570 u8 lun_qdepth;
4571 struct ufs_hba *hba;
4572
4573 hba = shost_priv(sdev->host);
4574
4575 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4576 ret = ufshcd_read_unit_desc_param(hba,
4577 ufshcd_scsi_to_upiu_lun(sdev->lun),
4578 UNIT_DESC_PARAM_LU_Q_DEPTH,
4579 &lun_qdepth,
4580 sizeof(lun_qdepth));
0ce147d4
SJ		 4581
4582 /* Some WLUN doesn't support unit descriptor */
4583 if (ret == -EOPNOTSUPP)
4584 lun_qdepth = 1;
4585 else if (!lun_qdepth)
4586 /* eventually, we can figure out the real queue depth */
4587 lun_qdepth = hba->nutrs;
4588 else
4589 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4590
4591 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4592 __func__, lun_qdepth);
db5ed4df 4593 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4594}
4595
57d104c1
SJ		 4596/*
4597 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4598 * @hba: per-adapter instance
4599 * @lun: UFS device lun id
4600 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4601 *
4602 * Returns 0 in case of success and b_lu_write_protect status would be returned
4603 * @b_lu_write_protect parameter.
4604 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4605 * Returns -EINVAL in case of invalid parameters passed to this function.
4606 */
4607static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4608 u8 lun,
4609 u8 *b_lu_write_protect)
4610{
4611 int ret;
4612
4613 if (!b_lu_write_protect)
4614 ret = -EINVAL;
4615 /*
4616 * According to UFS device spec, RPMB LU can't be write
4617 * protected so skip reading bLUWriteProtect parameter for
4618 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4619 */
1baa8011 4620 else if (lun >= hba->dev_info.max_lu_supported)
57d104c1
SJ		 4621 ret = -ENOTSUPP;
4622 else
4623 ret = ufshcd_read_unit_desc_param(hba,
4624 lun,
4625 UNIT_DESC_PARAM_LU_WR_PROTECT,
4626 b_lu_write_protect,
4627 sizeof(*b_lu_write_protect));
4628 return ret;
4629}
4630
4631/**
4632 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4633 * status
4634 * @hba: per-adapter instance
4635 * @sdev: pointer to SCSI device
4636 *
4637 */
4638static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4639 struct scsi_device *sdev)
4640{
4641 if (hba->dev_info.f_power_on_wp_en &&
4642 !hba->dev_info.is_lu_power_on_wp) {
4643 u8 b_lu_write_protect;
4644
4645 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4646 &b_lu_write_protect) &&
4647 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4648 hba->dev_info.is_lu_power_on_wp = true;
4649 }
4650}
4651
7a3e97b0
SY		 4652/**
4653 * ufshcd_slave_alloc - handle initial SCSI device configurations
4654 * @sdev: pointer to SCSI device
4655 *
4656 * Returns success
4657 */
4658static int ufshcd_slave_alloc(struct scsi_device *sdev)
4659{
4660 struct ufs_hba *hba;
4661
4662 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4663
4664 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4665 sdev->use_10_for_ms = 1;
a3a76391
CG		 4666
4667 /* DBD field should be set to 1 in mode sense(10) */
4668 sdev->set_dbd_for_ms = 1;
7a3e97b0 4669
e8e7f271
SRT		 4670 /* allow SCSI layer to restart the device in case of errors */
4671 sdev->allow_restart = 1;
4264fd61 4672
b2a6c522
SRT		 4673 /* REPORT SUPPORTED OPERATION CODES is not supported */
4674 sdev->no_report_opcodes = 1;
4675
84af7e8b
SRT		 4676 /* WRITE_SAME command is not supported */
4677 sdev->no_write_same = 1;
e8e7f271 4678
0ce147d4 4679 ufshcd_set_queue_depth(sdev);
4264fd61 4680
57d104c1
SJ		 4681 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4682
7a3e97b0
SY		 4683 return 0;
4684}
4685
4264fd61
SRT		 4686/**
4687 * ufshcd_change_queue_depth - change queue depth
4688 * @sdev: pointer to SCSI device
4689 * @depth: required depth to set
4264fd61 4690 *
db5ed4df 4691 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4692 */
db5ed4df 4693static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4694{
4695 struct ufs_hba *hba = shost_priv(sdev->host);
4696
4697 if (depth > hba->nutrs)
4698 depth = hba->nutrs;
db5ed4df 4699 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4700}
4701
eeda4749
AM		 4702/**
4703 * ufshcd_slave_configure - adjust SCSI device configurations
4704 * @sdev: pointer to SCSI device
4705 */
4706static int ufshcd_slave_configure(struct scsi_device *sdev)
4707{
49615ba1 4708 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 4709 struct request_queue *q = sdev->request_queue;
4710
4711 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
49615ba1
SC		 4712
4713 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4714 sdev->rpm_autosuspend = 1;
4715
df043c74
ST		 4716 ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
4717
eeda4749
AM		 4718 return 0;
4719}
4720
7a3e97b0
SY		 4721/**
4722 * ufshcd_slave_destroy - remove SCSI device configurations
4723 * @sdev: pointer to SCSI device
4724 */
4725static void ufshcd_slave_destroy(struct scsi_device *sdev)
4726{
4727 struct ufs_hba *hba;
4728
4729 hba = shost_priv(sdev->host);
0ce147d4 4730 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4731 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4732 unsigned long flags;
4733
4734 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4735 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4736 spin_unlock_irqrestore(hba->host->host_lock, flags);
4737 }
7a3e97b0
SY		 4738}
4739
7a3e97b0
SY		 4740/**
4741 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4742 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4743 * @scsi_status: SCSI command status
4744 *
4745 * Returns value base on SCSI command status
4746 */
4747static inline int
4748ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4749{
4750 int result = 0;
4751
4752 switch (scsi_status) {
7a3e97b0 4753 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4754 ufshcd_copy_sense_data(lrbp);
30eb2e4c 4755 /* fallthrough */
1c2623c5 4756 case SAM_STAT_GOOD:
7a3e97b0
SY		 4757 result |= DID_OK << 16 |
4758 COMMAND_COMPLETE << 8 |
1c2623c5 4759 scsi_status;
7a3e97b0
SY		 4760 break;
4761 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4762 case SAM_STAT_BUSY:
7a3e97b0 4763 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4764 ufshcd_copy_sense_data(lrbp);
4765 result |= scsi_status;
7a3e97b0
SY		 4766 break;
4767 default:
4768 result |= DID_ERROR << 16;
4769 break;
4770 } /* end of switch */
4771
4772 return result;
4773}
4774
4775/**
4776 * ufshcd_transfer_rsp_status - Get overall status of the response
4777 * @hba: per adapter instance
8aa29f19 4778 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4779 *
4780 * Returns result of the command to notify SCSI midlayer
4781 */
4782static inline int
4783ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4784{
4785 int result = 0;
4786 int scsi_status;
4787 int ocs;
4788
4789 /* overall command status of utrd */
4790 ocs = ufshcd_get_tr_ocs(lrbp);
4791
d779a6e9
KK		 4792 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
4793 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
4794 MASK_RSP_UPIU_RESULT)
4795 ocs = OCS_SUCCESS;
4796 }
4797
7a3e97b0
SY		 4798 switch (ocs) {
4799 case OCS_SUCCESS:
5a0b0cb9 4800 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4801 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4802 switch (result) {
4803 case UPIU_TRANSACTION_RESPONSE:
4804 /*
4805 * get the response UPIU result to extract
4806 * the SCSI command status
4807 */
4808 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4809
4810 /*
4811 * get the result based on SCSI status response
4812 * to notify the SCSI midlayer of the command status
4813 */
4814 scsi_status = result & MASK_SCSI_STATUS;
4815 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4816
f05ac2e5
YG		 4817 /*
4818 * Currently we are only supporting BKOPs exception
4819 * events hence we can ignore BKOPs exception event
4820 * during power management callbacks. BKOPs exception
4821 * event is not expected to be raised in runtime suspend
4822 * callback as it allows the urgent bkops.
4823 * During system suspend, we are anyway forcefully
4824 * disabling the bkops and if urgent bkops is needed
4825 * it will be enabled on system resume. Long term
4826 * solution could be to abort the system suspend if
4827 * UFS device needs urgent BKOPs.
4828 */
4829 if (!hba->pm_op_in_progress &&
2824ec9f
SL		 4830 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr) &&
4831 schedule_work(&hba->eeh_work)) {
4832 /*
4833 * Prevent suspend once eeh_work is scheduled
4834 * to avoid deadlock between ufshcd_suspend
4835 * and exception event handler.
4836 */
4837 pm_runtime_get_noresume(hba->dev);
4838 }
5a0b0cb9
SRT		 4839 break;
4840 case UPIU_TRANSACTION_REJECT_UPIU:
4841 /* TODO: handle Reject UPIU Response */
4842 result = DID_ERROR << 16;
3b1d0580 4843 dev_err(hba->dev,
5a0b0cb9
SRT		 4844 "Reject UPIU not fully implemented\n");
4845 break;
4846 default:
5a0b0cb9
SRT		 4847 dev_err(hba->dev,
4848 "Unexpected request response code = %x\n",
4849 result);
e0347d89 4850 result = DID_ERROR << 16;
7a3e97b0
SY		 4851 break;
4852 }
7a3e97b0
SY		 4853 break;
4854 case OCS_ABORTED:
4855 result |= DID_ABORT << 16;
4856 break;
e8e7f271
SRT		 4857 case OCS_INVALID_COMMAND_STATUS:
4858 result |= DID_REQUEUE << 16;
4859 break;
7a3e97b0
SY		 4860 case OCS_INVALID_CMD_TABLE_ATTR:
4861 case OCS_INVALID_PRDT_ATTR:
4862 case OCS_MISMATCH_DATA_BUF_SIZE:
4863 case OCS_MISMATCH_RESP_UPIU_SIZE:
4864 case OCS_PEER_COMM_FAILURE:
4865 case OCS_FATAL_ERROR:
5e7341e1
ST		 4866 case OCS_DEVICE_FATAL_ERROR:
4867 case OCS_INVALID_CRYPTO_CONFIG:
4868 case OCS_GENERAL_CRYPTO_ERROR:
7a3e97b0
SY		 4869 default:
4870 result |= DID_ERROR << 16;
3b1d0580 4871 dev_err(hba->dev,
ff8e20c6
DR		 4872 "OCS error from controller = %x for tag %d\n",
4873 ocs, lrbp->task_tag);
4874 ufshcd_print_host_regs(hba);
6ba65588 4875 ufshcd_print_host_state(hba);
7a3e97b0
SY		 4876 break;
4877 } /* end of switch */
4878
2df74b69 4879 if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
66cc820f 4880 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 4881 return result;
4882}
4883
6ccf44fe
SJ		 4884/**
4885 * ufshcd_uic_cmd_compl - handle completion of uic command
4886 * @hba: per adapter instance
53b3d9c3 4887 * @intr_status: interrupt status generated by the controller
9333d775
VG		 4888 *
4889 * Returns
4890 * IRQ_HANDLED - If interrupt is valid
4891 * IRQ_NONE - If invalid interrupt
6ccf44fe 4892 */
9333d775 4893static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 4894{
9333d775
VG		 4895 irqreturn_t retval = IRQ_NONE;
4896
53b3d9c3 4897 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 4898 hba->active_uic_cmd->argument2 |=
4899 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 4900 hba->active_uic_cmd->argument3 =
4901 ufshcd_get_dme_attr_val(hba);
6ccf44fe 4902 complete(&hba->active_uic_cmd->done);
9333d775 4903 retval = IRQ_HANDLED;
6ccf44fe 4904 }
53b3d9c3 4905
9333d775 4906 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
57d104c1 4907 complete(hba->uic_async_done);
9333d775
VG		 4908 retval = IRQ_HANDLED;
4909 }
aa5c6979
SC		 4910
4911 if (retval == IRQ_HANDLED)
4912 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
4913 "complete");
9333d775 4914 return retval;
6ccf44fe
SJ		 4915}
4916
7a3e97b0 4917/**
9a47ec7c 4918 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 4919 * @hba: per adapter instance
9a47ec7c 4920 * @completed_reqs: requests to complete
7a3e97b0 4921 */
9a47ec7c
YG		 4922static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4923 unsigned long completed_reqs)
7a3e97b0 4924{
5a0b0cb9
SRT		 4925 struct ufshcd_lrb *lrbp;
4926 struct scsi_cmnd *cmd;
7a3e97b0
SY		 4927 int result;
4928 int index;
e9d501b1 4929
e9d501b1
DR		 4930 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4931 lrbp = &hba->lrb[index];
a3170376 4932 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 4933 cmd = lrbp->cmd;
4934 if (cmd) {
1a07f2d9 4935 ufshcd_add_command_trace(hba, index, "complete");
e9d501b1
DR		 4936 result = ufshcd_transfer_rsp_status(hba, lrbp);
4937 scsi_dma_unmap(cmd);
4938 cmd->result = result;
4939 /* Mark completed command as NULL in LRB */
4940 lrbp->cmd = NULL;
e9d501b1
DR		 4941 /* Do not touch lrbp after scsi done */
4942 cmd->scsi_done(cmd);
1ab27c9c 4943 __ufshcd_release(hba);
300bb13f
JP		 4944 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4945 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
1a07f2d9
LS		 4946 if (hba->dev_cmd.complete) {
4947 ufshcd_add_command_trace(hba, index,
4948 "dev_complete");
e9d501b1 4949 complete(hba->dev_cmd.complete);
1a07f2d9 4950 }
e9d501b1 4951 }
401f1e44 4952 if (ufshcd_is_clkscaling_supported(hba))
4953 hba->clk_scaling.active_reqs--;
e9d501b1 4954 }
7a3e97b0
SY		 4955
4956 /* clear corresponding bits of completed commands */
4957 hba->outstanding_reqs ^= completed_reqs;
4958
856b3483 4959 ufshcd_clk_scaling_update_busy(hba);
7a3e97b0
SY		 4960}
4961
9a47ec7c
YG		 4962/**
4963 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4964 * @hba: per adapter instance
9333d775
VG		 4965 *
4966 * Returns
4967 * IRQ_HANDLED - If interrupt is valid
4968 * IRQ_NONE - If invalid interrupt
9a47ec7c 4969 */
9333d775 4970static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c
YG		 4971{
4972 unsigned long completed_reqs;
4973 u32 tr_doorbell;
4974
4975 /* Resetting interrupt aggregation counters first and reading the
4976 * DOOR_BELL afterward allows us to handle all the completed requests.
4977 * In order to prevent other interrupts starvation the DB is read once
4978 * after reset. The down side of this solution is the possibility of
4979 * false interrupt if device completes another request after resetting
4980 * aggregation and before reading the DB.
4981 */
b638b5eb
AA		 4982 if (ufshcd_is_intr_aggr_allowed(hba) &&
4983 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 4984 ufshcd_reset_intr_aggr(hba);
4985
4986 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4987 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4988
9333d775
VG		 4989 if (completed_reqs) {
4990 __ufshcd_transfer_req_compl(hba, completed_reqs);
4991 return IRQ_HANDLED;
4992 } else {
4993 return IRQ_NONE;
4994 }
9a47ec7c
YG		 4995}
4996
66ec6d59
SRT		 4997/**
4998 * ufshcd_disable_ee - disable exception event
4999 * @hba: per-adapter instance
5000 * @mask: exception event to disable
5001 *
5002 * Disables exception event in the device so that the EVENT_ALERT
5003 * bit is not set.
5004 *
5005 * Returns zero on success, non-zero error value on failure.
5006 */
5007static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5008{
5009 int err = 0;
5010 u32 val;
5011
5012 if (!(hba->ee_ctrl_mask & mask))
5013 goto out;
5014
5015 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 5016 val &= MASK_EE_STATUS;
5e86ae44 5017 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 5018 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5019 if (!err)
5020 hba->ee_ctrl_mask &= ~mask;
5021out:
5022 return err;
5023}
5024
5025/**
5026 * ufshcd_enable_ee - enable exception event
5027 * @hba: per-adapter instance
5028 * @mask: exception event to enable
5029 *
5030 * Enable corresponding exception event in the device to allow
5031 * device to alert host in critical scenarios.
5032 *
5033 * Returns zero on success, non-zero error value on failure.
5034 */
5035static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5036{
5037 int err = 0;
5038 u32 val;
5039
5040 if (hba->ee_ctrl_mask & mask)
5041 goto out;
5042
5043 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 5044 val &= MASK_EE_STATUS;
5e86ae44 5045 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 5046 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5047 if (!err)
5048 hba->ee_ctrl_mask |= mask;
5049out:
5050 return err;
5051}
5052
5053/**
5054 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5055 * @hba: per-adapter instance
5056 *
5057 * Allow device to manage background operations on its own. Enabling
5058 * this might lead to inconsistent latencies during normal data transfers
5059 * as the device is allowed to manage its own way of handling background
5060 * operations.
5061 *
5062 * Returns zero on success, non-zero on failure.
5063 */
5064static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5065{
5066 int err = 0;
5067
5068 if (hba->auto_bkops_enabled)
5069 goto out;
5070
dc3c8d3a 5071 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 5072 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5073 if (err) {
5074 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5075 __func__, err);
5076 goto out;
5077 }
5078
5079 hba->auto_bkops_enabled = true;
7ff5ab47 5080 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 5081
5082 /* No need of URGENT_BKOPS exception from the device */
5083 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5084 if (err)
5085 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5086 __func__, err);
5087out:
5088 return err;
5089}
5090
5091/**
5092 * ufshcd_disable_auto_bkops - block device in doing background operations
5093 * @hba: per-adapter instance
5094 *
5095 * Disabling background operations improves command response latency but
5096 * has drawback of device moving into critical state where the device is
5097 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5098 * host is idle so that BKOPS are managed effectively without any negative
5099 * impacts.
5100 *
5101 * Returns zero on success, non-zero on failure.
5102 */
5103static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5104{
5105 int err = 0;
5106
5107 if (!hba->auto_bkops_enabled)
5108 goto out;
5109
5110 /*
5111 * If host assisted BKOPs is to be enabled, make sure
5112 * urgent bkops exception is allowed.
5113 */
5114 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5115 if (err) {
5116 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5117 __func__, err);
5118 goto out;
5119 }
5120
dc3c8d3a 5121 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
1f34eedf 5122 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
66ec6d59
SRT		 5123 if (err) {
5124 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5125 __func__, err);
5126 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5127 goto out;
5128 }
5129
5130 hba->auto_bkops_enabled = false;
7ff5ab47 5131 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 5132 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5133out:
5134 return err;
5135}
5136
5137/**
4e768e76 5138 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 5139 * @hba: per adapter instance
5140 *
5141 * After a device reset the device may toggle the BKOPS_EN flag
5142 * to default value. The s/w tracking variables should be updated
4e768e76 5143 * as well. This function would change the auto-bkops state based on
5144 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5145 */
4e768e76 5146static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5147{
4e768e76 5148 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5149 hba->auto_bkops_enabled = false;
5150 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5151 ufshcd_enable_auto_bkops(hba);
5152 } else {
5153 hba->auto_bkops_enabled = true;
5154 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5155 ufshcd_disable_auto_bkops(hba);
5156 }
7b6668d8 5157 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
24366c2a 5158 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5159}
5160
5161static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5162{
5e86ae44 5163 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5164 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5165}
5166
5167/**
57d104c1 5168 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5169 * @hba: per-adapter instance
57d104c1 5170 * @status: bkops_status value
66ec6d59 5171 *
57d104c1
SJ		 5172 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5173 * flag in the device to permit background operations if the device
5174 * bkops_status is greater than or equal to "status" argument passed to
5175 * this function, disable otherwise.
5176 *
5177 * Returns 0 for success, non-zero in case of failure.
5178 *
5179 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5180 * to know whether auto bkops is enabled or disabled after this function
5181 * returns control to it.
66ec6d59 5182 */
57d104c1
SJ		 5183static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5184 enum bkops_status status)
66ec6d59
SRT		 5185{
5186 int err;
57d104c1 5187 u32 curr_status = 0;
66ec6d59 5188
57d104c1 5189 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5190 if (err) {
5191 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5192 __func__, err);
5193 goto out;
57d104c1
SJ		 5194 } else if (curr_status > BKOPS_STATUS_MAX) {
5195 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5196 __func__, curr_status);
5197 err = -EINVAL;
5198 goto out;
66ec6d59
SRT		 5199 }
5200
57d104c1 5201 if (curr_status >= status)
66ec6d59 5202 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5203 else
5204 err = ufshcd_disable_auto_bkops(hba);
66ec6d59
SRT		 5205out:
5206 return err;
5207}
5208
57d104c1
SJ		 5209/**
5210 * ufshcd_urgent_bkops - handle urgent bkops exception event
5211 * @hba: per-adapter instance
5212 *
5213 * Enable fBackgroundOpsEn flag in the device to permit background
5214 * operations.
5215 *
5216 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5217 * and negative error value for any other failure.
5218 */
5219static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5220{
afdfff59 5221 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5222}
5223
66ec6d59
SRT		 5224static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5225{
5e86ae44 5226 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5227 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5228}
5229
afdfff59
YG		 5230static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5231{
5232 int err;
5233 u32 curr_status = 0;
5234
5235 if (hba->is_urgent_bkops_lvl_checked)
5236 goto enable_auto_bkops;
5237
5238 err = ufshcd_get_bkops_status(hba, &curr_status);
5239 if (err) {
5240 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5241 __func__, err);
5242 goto out;
5243 }
5244
5245 /*
5246 * We are seeing that some devices are raising the urgent bkops
5247 * exception events even when BKOPS status doesn't indicate performace
5248 * impacted or critical. Handle these device by determining their urgent
5249 * bkops status at runtime.
5250 */
5251 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5252 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5253 __func__, curr_status);
5254 /* update the current status as the urgent bkops level */
5255 hba->urgent_bkops_lvl = curr_status;
5256 hba->is_urgent_bkops_lvl_checked = true;
5257 }
5258
5259enable_auto_bkops:
5260 err = ufshcd_enable_auto_bkops(hba);
5261out:
5262 if (err < 0)
5263 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5264 __func__, err);
5265}
5266
3d17b9b5
AD		 5267static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable)
5268{
5269 int ret;
6f8d5a6a 5270 u8 index;
3d17b9b5
AD		 5271 enum query_opcode opcode;
5272
79e3520f 5273 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5274 return 0;
5275
5276 if (!(enable ^ hba->wb_enabled))
5277 return 0;
5278 if (enable)
5279 opcode = UPIU_QUERY_OPCODE_SET_FLAG;
5280 else
5281 opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5282
e31011ab 5283 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5284 ret = ufshcd_query_flag_retry(hba, opcode,
6f8d5a6a 5285 QUERY_FLAG_IDN_WB_EN, index, NULL);
3d17b9b5
AD		 5286 if (ret) {
5287 dev_err(hba->dev, "%s write booster %s failed %d\n",
5288 __func__, enable ? "enable" : "disable", ret);
5289 return ret;
5290 }
5291
5292 hba->wb_enabled = enable;
5293 dev_dbg(hba->dev, "%s write booster %s %d\n",
5294 __func__, enable ? "enable" : "disable", ret);
5295
5296 return ret;
5297}
5298
5299static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5300{
5301 int val;
6f8d5a6a 5302 u8 index;
3d17b9b5
AD		 5303
5304 if (set)
5305 val = UPIU_QUERY_OPCODE_SET_FLAG;
5306 else
5307 val = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5308
e31011ab 5309 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5310 return ufshcd_query_flag_retry(hba, val,
6f8d5a6a
SC		 5311 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8,
5312 index, NULL);
3d17b9b5
AD		 5313}
5314
5315static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
5316{
5df6f2de
KK		 5317 if (hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL)
5318 return;
5319
3d17b9b5
AD		 5320 if (enable)
5321 ufshcd_wb_buf_flush_enable(hba);
5322 else
5323 ufshcd_wb_buf_flush_disable(hba);
5324
5325}
5326
5327static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba)
5328{
5329 int ret;
6f8d5a6a 5330 u8 index;
3d17b9b5 5331
79e3520f 5332 if (!ufshcd_is_wb_allowed(hba) || hba->wb_buf_flush_enabled)
3d17b9b5
AD		 5333 return 0;
5334
e31011ab 5335 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5336 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1f34eedf 5337 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
6f8d5a6a 5338 index, NULL);
3d17b9b5
AD		 5339 if (ret)
5340 dev_err(hba->dev, "%s WB - buf flush enable failed %d\n",
5341 __func__, ret);
5342 else
5343 hba->wb_buf_flush_enabled = true;
5344
5345 dev_dbg(hba->dev, "WB - Flush enabled: %d\n", ret);
5346 return ret;
5347}
5348
5349static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba)
5350{
5351 int ret;
6f8d5a6a 5352 u8 index;
3d17b9b5 5353
79e3520f 5354 if (!ufshcd_is_wb_allowed(hba) || !hba->wb_buf_flush_enabled)
3d17b9b5
AD		 5355 return 0;
5356
e31011ab 5357 index = ufshcd_wb_get_query_index(hba);
3d17b9b5 5358 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
6f8d5a6a
SC		 5359 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
5360 index, NULL);
3d17b9b5
AD		 5361 if (ret) {
5362 dev_warn(hba->dev, "%s: WB - buf flush disable failed %d\n",
5363 __func__, ret);
5364 } else {
5365 hba->wb_buf_flush_enabled = false;
5366 dev_dbg(hba->dev, "WB - Flush disabled: %d\n", ret);
5367 }
5368
5369 return ret;
5370}
5371
5372static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5373 u32 avail_buf)
5374{
5375 u32 cur_buf;
5376 int ret;
e31011ab 5377 u8 index;
3d17b9b5 5378
e31011ab 5379 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5380 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5381 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
e31011ab 5382 index, 0, &cur_buf);
3d17b9b5
AD		 5383 if (ret) {
5384 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5385 __func__, ret);
5386 return false;
5387 }
5388
5389 if (!cur_buf) {
5390 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5391 cur_buf);
5392 return false;
5393 }
d14734ae
SC		 5394 /* Let it continue to flush when available buffer exceeds threshold */
5395 if (avail_buf < hba->vps->wb_flush_threshold)
3d17b9b5
AD		 5396 return true;
5397
5398 return false;
5399}
5400
51dd905b 5401static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
3d17b9b5
AD		 5402{
5403 int ret;
5404 u32 avail_buf;
e31011ab 5405 u8 index;
3d17b9b5 5406
79e3520f 5407 if (!ufshcd_is_wb_allowed(hba))
3d17b9b5
AD		 5408 return false;
5409 /*
5410 * The ufs device needs the vcc to be ON to flush.
5411 * With user-space reduction enabled, it's enough to enable flush
5412 * by checking only the available buffer. The threshold
5413 * defined here is > 90% full.
5414 * With user-space preserved enabled, the current-buffer
5415 * should be checked too because the wb buffer size can reduce
5416 * when disk tends to be full. This info is provided by current
5417 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5418 * keeping vcc on when current buffer is empty.
5419 */
e31011ab 5420 index = ufshcd_wb_get_query_index(hba);
3d17b9b5
AD		 5421 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5422 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
e31011ab 5423 index, 0, &avail_buf);
3d17b9b5
AD		 5424 if (ret) {
5425 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5426 __func__, ret);
5427 return false;
5428 }
5429
5430 if (!hba->dev_info.b_presrv_uspc_en) {
d14734ae 5431 if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
3d17b9b5
AD		 5432 return true;
5433 return false;
5434 }
5435
5436 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5437}
5438
51dd905b
SC		 5439static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5440{
5441 struct ufs_hba *hba = container_of(to_delayed_work(work),
5442 struct ufs_hba,
5443 rpm_dev_flush_recheck_work);
5444 /*
5445 * To prevent unnecessary VCC power drain after device finishes
5446 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5447 * after a certain delay to recheck the threshold by next runtime
5448 * suspend.
5449 */
5450 pm_runtime_get_sync(hba->dev);
5451 pm_runtime_put_sync(hba->dev);
5452}
5453
66ec6d59
SRT		 5454/**
5455 * ufshcd_exception_event_handler - handle exceptions raised by device
5456 * @work: pointer to work data
5457 *
5458 * Read bExceptionEventStatus attribute from the device and handle the
5459 * exception event accordingly.
5460 */
5461static void ufshcd_exception_event_handler(struct work_struct *work)
5462{
5463 struct ufs_hba *hba;
5464 int err;
5465 u32 status = 0;
5466 hba = container_of(work, struct ufs_hba, eeh_work);
5467
62694735 5468 pm_runtime_get_sync(hba->dev);
03e1d28e 5469 ufshcd_scsi_block_requests(hba);
66ec6d59
SRT		 5470 err = ufshcd_get_ee_status(hba, &status);
5471 if (err) {
5472 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5473 __func__, err);
5474 goto out;
5475 }
5476
5477 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5478
5479 if (status & MASK_EE_URGENT_BKOPS)
5480 ufshcd_bkops_exception_event_handler(hba);
5481
66ec6d59 5482out:
03e1d28e 5483 ufshcd_scsi_unblock_requests(hba);
2824ec9f
SL		 5484 /*
5485 * pm_runtime_get_noresume is called while scheduling
5486 * eeh_work to avoid suspend racing with exception work.
5487 * Hence decrement usage counter using pm_runtime_put_noidle
5488 * to allow suspend on completion of exception event handler.
5489 */
5490 pm_runtime_put_noidle(hba->dev);
5491 pm_runtime_put(hba->dev);
66ec6d59
SRT		 5492 return;
5493}
5494
9a47ec7c
YG		 5495/* Complete requests that have door-bell cleared */
5496static void ufshcd_complete_requests(struct ufs_hba *hba)
5497{
5498 ufshcd_transfer_req_compl(hba);
5499 ufshcd_tmc_handler(hba);
5500}
5501
583fa62d
YG		 5502/**
5503 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5504 * to recover from the DL NAC errors or not.
5505 * @hba: per-adapter instance
5506 *
5507 * Returns true if error handling is required, false otherwise
5508 */
5509static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5510{
5511 unsigned long flags;
5512 bool err_handling = true;
5513
5514 spin_lock_irqsave(hba->host->host_lock, flags);
5515 /*
5516 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5517 * device fatal error and/or DL NAC & REPLAY timeout errors.
5518 */
5519 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5520 goto out;
5521
5522 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5523 ((hba->saved_err & UIC_ERROR) &&
5524 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5525 goto out;
5526
5527 if ((hba->saved_err & UIC_ERROR) &&
5528 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5529 int err;
5530 /*
5531 * wait for 50ms to see if we can get any other errors or not.
5532 */
5533 spin_unlock_irqrestore(hba->host->host_lock, flags);
5534 msleep(50);
5535 spin_lock_irqsave(hba->host->host_lock, flags);
5536
5537 /*
5538 * now check if we have got any other severe errors other than
5539 * DL NAC error?
5540 */
5541 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5542 ((hba->saved_err & UIC_ERROR) &&
5543 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5544 goto out;
5545
5546 /*
5547 * As DL NAC is the only error received so far, send out NOP
5548 * command to confirm if link is still active or not.
5549 * - If we don't get any response then do error recovery.
5550 * - If we get response then clear the DL NAC error bit.
5551 */
5552
5553 spin_unlock_irqrestore(hba->host->host_lock, flags);
5554 err = ufshcd_verify_dev_init(hba);
5555 spin_lock_irqsave(hba->host->host_lock, flags);
5556
5557 if (err)
5558 goto out;
5559
5560 /* Link seems to be alive hence ignore the DL NAC errors */
5561 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5562 hba->saved_err &= ~UIC_ERROR;
5563 /* clear NAC error */
5564 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
b0008625 5565 if (!hba->saved_uic_err)
583fa62d 5566 err_handling = false;
583fa62d
YG		 5567 }
5568out:
5569 spin_unlock_irqrestore(hba->host->host_lock, flags);
5570 return err_handling;
5571}
5572
4db7a236
CG		 5573/* host lock must be held before calling this func */
5574static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
5575{
5576 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
5577 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
5578}
5579
5580/* host lock must be held before calling this func */
5581static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
5582{
5583 /* handle fatal errors only when link is not in error state */
5584 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
5586dd8e
CG		 5585 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5586 ufshcd_is_saved_err_fatal(hba))
5587 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
5588 else
5589 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
5590 queue_work(hba->eh_wq, &hba->eh_work);
4db7a236
CG		 5591 }
5592}
5593
c72e79c0
CG		 5594static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
5595{
5596 pm_runtime_get_sync(hba->dev);
5597 if (pm_runtime_suspended(hba->dev)) {
5598 /*
5599 * Don't assume anything of pm_runtime_get_sync(), if
5600 * resume fails, irq and clocks can be OFF, and powers
5601 * can be OFF or in LPM.
5602 */
5603 ufshcd_setup_hba_vreg(hba, true);
5604 ufshcd_enable_irq(hba);
5605 ufshcd_setup_vreg(hba, true);
5606 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5607 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5608 ufshcd_hold(hba, false);
5609 if (!ufshcd_is_clkgating_allowed(hba))
5610 ufshcd_setup_clocks(hba, true);
5611 ufshcd_release(hba);
5612 ufshcd_vops_resume(hba, UFS_RUNTIME_PM);
5613 } else {
5614 ufshcd_hold(hba, false);
5615 if (hba->clk_scaling.is_allowed) {
5616 cancel_work_sync(&hba->clk_scaling.suspend_work);
5617 cancel_work_sync(&hba->clk_scaling.resume_work);
5618 ufshcd_suspend_clkscaling(hba);
5619 }
5620 }
5621}
5622
5623static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
5624{
5625 ufshcd_release(hba);
5626 if (hba->clk_scaling.is_allowed)
5627 ufshcd_resume_clkscaling(hba);
5628 pm_runtime_put(hba->dev);
5629}
5630
5631static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
5632{
5633 return (hba->ufshcd_state == UFSHCD_STATE_ERROR ||
5634 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
5635 ufshcd_is_link_broken(hba))));
5636}
5637
5638#ifdef CONFIG_PM
5639static void ufshcd_recover_pm_error(struct ufs_hba *hba)
5640{
5641 struct Scsi_Host *shost = hba->host;
5642 struct scsi_device *sdev;
5643 struct request_queue *q;
5644 int ret;
5645
5646 /*
5647 * Set RPM status of hba device to RPM_ACTIVE,
5648 * this also clears its runtime error.
5649 */
5650 ret = pm_runtime_set_active(hba->dev);
5651 /*
5652 * If hba device had runtime error, we also need to resume those
5653 * scsi devices under hba in case any of them has failed to be
5654 * resumed due to hba runtime resume failure. This is to unblock
5655 * blk_queue_enter in case there are bios waiting inside it.
5656 */
5657 if (!ret) {
5658 shost_for_each_device(sdev, shost) {
5659 q = sdev->request_queue;
5660 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
5661 q->rpm_status == RPM_SUSPENDING))
5662 pm_request_resume(q->dev);
5663 }
5664 }
5665}
5666#else
5667static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
5668{
5669}
5670#endif
5671
2355b66e
CG		 5672static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
5673{
5674 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
5675 u32 mode;
5676
5677 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
5678
5679 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
5680 return true;
5681
5682 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
5683 return true;
5684
5685 return false;
5686}
5687
7a3e97b0 5688/**
e8e7f271
SRT		 5689 * ufshcd_err_handler - handle UFS errors that require s/w attention
5690 * @work: pointer to work structure
7a3e97b0 5691 */
e8e7f271 5692static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5693{
5694 struct ufs_hba *hba;
e8e7f271 5695 unsigned long flags;
307348f6
CG		 5696 bool err_xfer = false;
5697 bool err_tm = false;
2355b66e 5698 int err = 0, pmc_err;
e8e7f271 5699 int tag;
2355b66e 5700 bool needs_reset = false, needs_restore = false;
e8e7f271
SRT		 5701
5702 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5703
4db7a236 5704 spin_lock_irqsave(hba->host->host_lock, flags);
c72e79c0 5705 if (ufshcd_err_handling_should_stop(hba)) {
4db7a236
CG		 5706 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5707 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5708 spin_unlock_irqrestore(hba->host->host_lock, flags);
4db7a236
CG		 5709 return;
5710 }
5711 ufshcd_set_eh_in_progress(hba);
5712 spin_unlock_irqrestore(hba->host->host_lock, flags);
c72e79c0 5713 ufshcd_err_handling_prepare(hba);
e8e7f271 5714 spin_lock_irqsave(hba->host->host_lock, flags);
5586dd8e 5715 ufshcd_scsi_block_requests(hba);
c72e79c0
CG		 5716 /*
5717 * A full reset and restore might have happened after preparation
5718 * is finished, double check whether we should stop.
5719 */
5720 if (ufshcd_err_handling_should_stop(hba)) {
5721 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5722 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5723 goto out;
5724 }
e8e7f271 5725 hba->ufshcd_state = UFSHCD_STATE_RESET;
e8e7f271
SRT		 5726
5727 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5728 ufshcd_complete_requests(hba);
583fa62d
YG		 5729
5730 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5731 bool ret;
5732
5733 spin_unlock_irqrestore(hba->host->host_lock, flags);
5734 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5735 ret = ufshcd_quirk_dl_nac_errors(hba);
5736 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236 5737 if (!ret && !hba->force_reset && ufshcd_is_link_active(hba))
583fa62d
YG		 5738 goto skip_err_handling;
5739 }
4db7a236
CG		 5740
5741 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5742 ufshcd_is_saved_err_fatal(hba) ||
9a47ec7c 5743 ((hba->saved_err & UIC_ERROR) &&
4db7a236
CG		 5744 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5745 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
9a47ec7c 5746 needs_reset = true;
e8e7f271 5747
2355b66e
CG		 5748 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
5749 (hba->saved_uic_err &&
5750 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
c3be8d1e
CG		 5751 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
5752
5753 spin_unlock_irqrestore(hba->host->host_lock, flags);
5754 ufshcd_print_host_state(hba);
5755 ufshcd_print_pwr_info(hba);
5756 ufshcd_print_host_regs(hba);
5757 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5758 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
5759 spin_lock_irqsave(hba->host->host_lock, flags);
5760 }
5761
9a47ec7c
YG		 5762 /*
5763 * if host reset is required then skip clearing the pending
2df74b69
CG		 5764 * transfers forcefully because they will get cleared during
5765 * host reset and restore
9a47ec7c
YG		 5766 */
5767 if (needs_reset)
2355b66e
CG		 5768 goto do_reset;
5769
5770 /*
5771 * If LINERESET was caught, UFS might have been put to PWM mode,
5772 * check if power mode restore is needed.
5773 */
5774 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
5775 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5776 if (!hba->saved_uic_err)
5777 hba->saved_err &= ~UIC_ERROR;
5778 spin_unlock_irqrestore(hba->host->host_lock, flags);
5779 if (ufshcd_is_pwr_mode_restore_needed(hba))
5780 needs_restore = true;
5781 spin_lock_irqsave(hba->host->host_lock, flags);
5782 if (!hba->saved_err && !needs_restore)
5783 goto skip_err_handling;
5784 }
9a47ec7c 5785
2355b66e 5786 hba->silence_err_logs = true;
9a47ec7c
YG		 5787 /* release lock as clear command might sleep */
5788 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5789 /* Clear pending transfer requests */
9a47ec7c 5790 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
307348f6 5791 if (ufshcd_try_to_abort_task(hba, tag)) {
9a47ec7c
YG		 5792 err_xfer = true;
5793 goto lock_skip_pending_xfer_clear;
5794 }
5795 }
e8e7f271
SRT		 5796
5797 /* Clear pending task management requests */
9a47ec7c
YG		 5798 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5799 if (ufshcd_clear_tm_cmd(hba, tag)) {
5800 err_tm = true;
5801 goto lock_skip_pending_xfer_clear;
5802 }
5803 }
e8e7f271 5804
9a47ec7c 5805lock_skip_pending_xfer_clear:
e8e7f271 5806 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5807
9a47ec7c
YG		 5808 /* Complete the requests that are cleared by s/w */
5809 ufshcd_complete_requests(hba);
2355b66e 5810 hba->silence_err_logs = false;
9a47ec7c 5811
2355b66e 5812 if (err_xfer || err_tm) {
9a47ec7c 5813 needs_reset = true;
2355b66e
CG		 5814 goto do_reset;
5815 }
5816
5817 /*
5818 * After all reqs and tasks are cleared from doorbell,
5819 * now it is safe to retore power mode.
5820 */
5821 if (needs_restore) {
5822 spin_unlock_irqrestore(hba->host->host_lock, flags);
5823 /*
5824 * Hold the scaling lock just in case dev cmds
5825 * are sent via bsg and/or sysfs.
5826 */
5827 down_write(&hba->clk_scaling_lock);
5828 hba->force_pmc = true;
5829 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
5830 if (pmc_err) {
5831 needs_reset = true;
5832 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
5833 __func__, pmc_err);
5834 }
5835 hba->force_pmc = false;
5836 ufshcd_print_pwr_info(hba);
5837 up_write(&hba->clk_scaling_lock);
5838 spin_lock_irqsave(hba->host->host_lock, flags);
5839 }
9a47ec7c 5840
2355b66e 5841do_reset:
e8e7f271 5842 /* Fatal errors need reset */
9a47ec7c
YG		 5843 if (needs_reset) {
5844 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5845
5846 /*
5847 * ufshcd_reset_and_restore() does the link reinitialization
5848 * which will need atleast one empty doorbell slot to send the
5849 * device management commands (NOP and query commands).
5850 * If there is no slot empty at this moment then free up last
5851 * slot forcefully.
5852 */
5853 if (hba->outstanding_reqs == max_doorbells)
5854 __ufshcd_transfer_req_compl(hba,
5855 (1UL << (hba->nutrs - 1)));
5856
4db7a236 5857 hba->force_reset = false;
9a47ec7c 5858 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5859 err = ufshcd_reset_and_restore(hba);
4db7a236
CG		 5860 if (err)
5861 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
5862 __func__, err);
c72e79c0
CG		 5863 else
5864 ufshcd_recover_pm_error(hba);
5865 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5866 }
9a47ec7c 5867
583fa62d 5868skip_err_handling:
9a47ec7c 5869 if (!needs_reset) {
4db7a236
CG		 5870 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
5871 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
9a47ec7c
YG		 5872 if (hba->saved_err || hba->saved_uic_err)
5873 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5874 __func__, hba->saved_err, hba->saved_uic_err);
5875 }
5876
c72e79c0 5877out:
e8e7f271 5878 ufshcd_clear_eh_in_progress(hba);
9a47ec7c 5879 spin_unlock_irqrestore(hba->host->host_lock, flags);
38135535 5880 ufshcd_scsi_unblock_requests(hba);
c72e79c0 5881 ufshcd_err_handling_unprepare(hba);
7a3e97b0
SY		 5882}
5883
5884/**
e8e7f271
SRT		 5885 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5886 * @hba: per-adapter instance
9333d775
VG		 5887 *
5888 * Returns
5889 * IRQ_HANDLED - If interrupt is valid
5890 * IRQ_NONE - If invalid interrupt
7a3e97b0 5891 */
9333d775 5892static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 5893{
5894 u32 reg;
9333d775 5895 irqreturn_t retval = IRQ_NONE;
7a3e97b0 5896
2355b66e 5897 /* PHY layer error */
fb7b45f0 5898 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
fb7b45f0 5899 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
2355b66e
CG		 5900 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
5901 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
fb7b45f0
DR		 5902 /*
5903 * To know whether this error is fatal or not, DB timeout
5904 * must be checked but this error is handled separately.
5905 */
2355b66e
CG		 5906 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
5907 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
5908 __func__);
5909
5910 /* Got a LINERESET indication. */
5911 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
5912 struct uic_command *cmd = NULL;
5913
5914 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
5915 if (hba->uic_async_done && hba->active_uic_cmd)
5916 cmd = hba->active_uic_cmd;
5917 /*
5918 * Ignore the LINERESET during power mode change
5919 * operation via DME_SET command.
5920 */
5921 if (cmd && (cmd->command == UIC_CMD_DME_SET))
5922 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5923 }
9333d775 5924 retval |= IRQ_HANDLED;
ff8e20c6 5925 }
fb7b45f0 5926
e8e7f271
SRT		 5927 /* PA_INIT_ERROR is fatal and needs UIC reset */
5928 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 5929 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5930 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5931 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
ff8e20c6 5932
9333d775
VG		 5933 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5934 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5935 else if (hba->dev_quirks &
5936 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5937 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5938 hba->uic_error |=
5939 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5940 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5941 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5942 }
5943 retval |= IRQ_HANDLED;
583fa62d 5944 }
e8e7f271
SRT		 5945
5946 /* UIC NL/TL/DME errors needs software retry */
5947 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 5948 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5949 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5950 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
e8e7f271 5951 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 5952 retval |= IRQ_HANDLED;
ff8e20c6 5953 }
e8e7f271
SRT		 5954
5955 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 5956 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
5957 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
48d5b973 5958 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
e8e7f271 5959 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 5960 retval |= IRQ_HANDLED;
ff8e20c6 5961 }
e8e7f271
SRT		 5962
5963 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 5964 if ((reg & UIC_DME_ERROR) &&
5965 (reg & UIC_DME_ERROR_CODE_MASK)) {
48d5b973 5966 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
e8e7f271 5967 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 5968 retval |= IRQ_HANDLED;
ff8e20c6 5969 }
e8e7f271
SRT		 5970
5971 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5972 __func__, hba->uic_error);
9333d775 5973 return retval;
e8e7f271
SRT		 5974}
5975
82174440
SC		 5976static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5977 u32 intr_mask)
5978{
5a244e0e
SC		 5979 if (!ufshcd_is_auto_hibern8_supported(hba) ||
5980 !ufshcd_is_auto_hibern8_enabled(hba))
82174440
SC		 5981 return false;
5982
5983 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5984 return false;
5985
5986 if (hba->active_uic_cmd &&
5987 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5988 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5989 return false;
5990
5991 return true;
5992}
5993
e8e7f271
SRT		 5994/**
5995 * ufshcd_check_errors - Check for errors that need s/w attention
5996 * @hba: per-adapter instance
9333d775
VG		 5997 *
5998 * Returns
5999 * IRQ_HANDLED - If interrupt is valid
6000 * IRQ_NONE - If invalid interrupt
e8e7f271 6001 */
9333d775 6002static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
e8e7f271
SRT		 6003{
6004 bool queue_eh_work = false;
9333d775 6005 irqreturn_t retval = IRQ_NONE;
e8e7f271 6006
d3c615bf
SC		 6007 if (hba->errors & INT_FATAL_ERRORS) {
6008 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
e8e7f271 6009 queue_eh_work = true;
d3c615bf 6010 }
7a3e97b0
SY		 6011
6012 if (hba->errors & UIC_ERROR) {
e8e7f271 6013 hba->uic_error = 0;
9333d775 6014 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 6015 if (hba->uic_error)
6016 queue_eh_work = true;
7a3e97b0 6017 }
e8e7f271 6018
82174440
SC		 6019 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6020 dev_err(hba->dev,
6021 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6022 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6023 "Enter" : "Exit",
6024 hba->errors, ufshcd_get_upmcrs(hba));
d3c615bf
SC		 6025 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
6026 hba->errors);
4db7a236 6027 ufshcd_set_link_broken(hba);
82174440
SC		 6028 queue_eh_work = true;
6029 }
6030
e8e7f271 6031 if (queue_eh_work) {
9a47ec7c
YG		 6032 /*
6033 * update the transfer error masks to sticky bits, let's do this
6034 * irrespective of current ufshcd_state.
6035 */
6036 hba->saved_err |= hba->errors;
6037 hba->saved_uic_err |= hba->uic_error;
6038
4db7a236 6039 /* dump controller state before resetting */
2355b66e
CG		 6040 if ((hba->saved_err & (INT_FATAL_ERRORS)) ||
6041 (hba->saved_uic_err &&
6042 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
4db7a236 6043 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
66cc820f
DR		 6044 __func__, hba->saved_err,
6045 hba->saved_uic_err);
c3be8d1e
CG		 6046 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6047 "host_regs: ");
4db7a236 6048 ufshcd_print_pwr_info(hba);
e8e7f271 6049 }
4db7a236 6050 ufshcd_schedule_eh_work(hba);
9333d775 6051 retval |= IRQ_HANDLED;
3441da7d 6052 }
e8e7f271
SRT		 6053 /*
6054 * if (!queue_eh_work) -
6055 * Other errors are either non-fatal where host recovers
6056 * itself without s/w intervention or errors that will be
6057 * handled by the SCSI core layer.
6058 */
9333d775 6059 return retval;
7a3e97b0
SY		 6060}
6061
69a6c269
BVA		 6062struct ctm_info {
6063 struct ufs_hba *hba;
6064 unsigned long pending;
6065 unsigned int ncpl;
6066};
6067
6068static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
6069{
6070 struct ctm_info *const ci = priv;
6071 struct completion *c;
6072
6073 WARN_ON_ONCE(reserved);
6074 if (test_bit(req->tag, &ci->pending))
6075 return true;
6076 ci->ncpl++;
6077 c = req->end_io_data;
6078 if (c)
6079 complete(c);
6080 return true;
6081}
6082
7a3e97b0
SY		 6083/**
6084 * ufshcd_tmc_handler - handle task management function completion
6085 * @hba: per adapter instance
9333d775
VG		 6086 *
6087 * Returns
6088 * IRQ_HANDLED - If interrupt is valid
6089 * IRQ_NONE - If invalid interrupt
7a3e97b0 6090 */
9333d775 6091static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 6092{
69a6c269
BVA		 6093 struct request_queue *q = hba->tmf_queue;
6094 struct ctm_info ci = {
6095 .hba = hba,
6096 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
6097 };
7a3e97b0 6098
69a6c269
BVA		 6099 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
6100 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
7a3e97b0
SY		 6101}
6102
6103/**
6104 * ufshcd_sl_intr - Interrupt service routine
6105 * @hba: per adapter instance
6106 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 6107 *
6108 * Returns
6109 * IRQ_HANDLED - If interrupt is valid
6110 * IRQ_NONE - If invalid interrupt
7a3e97b0 6111 */
9333d775 6112static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 6113{
9333d775
VG		 6114 irqreturn_t retval = IRQ_NONE;
6115
7a3e97b0 6116 hba->errors = UFSHCD_ERROR_MASK & intr_status;
82174440
SC		 6117
6118 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
6119 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
6120
7a3e97b0 6121 if (hba->errors)
9333d775 6122 retval |= ufshcd_check_errors(hba);
7a3e97b0 6123
53b3d9c3 6124 if (intr_status & UFSHCD_UIC_MASK)
9333d775 6125 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 6126
6127 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 6128 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 6129
6130 if (intr_status & UTP_TRANSFER_REQ_COMPL)
9333d775
VG		 6131 retval |= ufshcd_transfer_req_compl(hba);
6132
6133 return retval;
7a3e97b0
SY		 6134}
6135
6136/**
6137 * ufshcd_intr - Main interrupt service routine
6138 * @irq: irq number
6139 * @__hba: pointer to adapter instance
6140 *
9333d775
VG		 6141 * Returns
6142 * IRQ_HANDLED - If interrupt is valid
6143 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 6144 */
6145static irqreturn_t ufshcd_intr(int irq, void *__hba)
6146{
d75f7fe4 6147 u32 intr_status, enabled_intr_status;
7a3e97b0
SY		 6148 irqreturn_t retval = IRQ_NONE;
6149 struct ufs_hba *hba = __hba;
7f6ba4f1 6150 int retries = hba->nutrs;
7a3e97b0
SY		 6151
6152 spin_lock(hba->host->host_lock);
b873a275 6153 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
3f8af604
CG		 6154 hba->ufs_stats.last_intr_status = intr_status;
6155 hba->ufs_stats.last_intr_ts = ktime_get();
7a3e97b0 6156
7f6ba4f1
VG		 6157 /*
6158 * There could be max of hba->nutrs reqs in flight and in worst case
6159 * if the reqs get finished 1 by 1 after the interrupt status is
6160 * read, make sure we handle them by checking the interrupt status
6161 * again in a loop until we process all of the reqs before returning.
6162 */
6163 do {
6164 enabled_intr_status =
6165 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6166 if (intr_status)
6167 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 6168 if (enabled_intr_status)
6169 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 6170
6171 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6172 } while (intr_status && --retries);
d75f7fe4 6173
9333d775
VG		 6174 if (retval == IRQ_NONE) {
6175 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
6176 __func__, intr_status);
6177 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6178 }
6179
7a3e97b0
SY		 6180 spin_unlock(hba->host->host_lock);
6181 return retval;
6182}
6183
e2933132
SRT		 6184static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6185{
6186 int err = 0;
6187 u32 mask = 1 << tag;
6188 unsigned long flags;
6189
6190 if (!test_bit(tag, &hba->outstanding_tasks))
6191 goto out;
6192
6193 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 6194 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 6195 spin_unlock_irqrestore(hba->host->host_lock, flags);
6196
6197 /* poll for max. 1 sec to clear door bell register by h/w */
6198 err = ufshcd_wait_for_register(hba,
6199 REG_UTP_TASK_REQ_DOOR_BELL,
5cac1095 6200 mask, 0, 1000, 1000);
e2933132
SRT		 6201out:
6202 return err;
6203}
6204
c6049cd9
CH		 6205static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6206 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 6207{
69a6c269 6208 struct request_queue *q = hba->tmf_queue;
c6049cd9 6209 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 6210 DECLARE_COMPLETION_ONSTACK(wait);
6211 struct request *req;
7a3e97b0 6212 unsigned long flags;
c6049cd9 6213 int free_slot, task_tag, err;
7a3e97b0 6214
e2933132
SRT		 6215 /*
6216 * Get free slot, sleep if slots are unavailable.
6217 * Even though we use wait_event() which sleeps indefinitely,
6218 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
6219 */
69a6c269
BVA		 6220 req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
6221 req->end_io_data = &wait;
6222 free_slot = req->tag;
6223 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
1ab27c9c 6224 ufshcd_hold(hba, false);
7a3e97b0 6225
e2933132 6226 spin_lock_irqsave(host->host_lock, flags);
e2933132 6227 task_tag = hba->nutrs + free_slot;
7a3e97b0 6228
c6049cd9
CH		 6229 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
6230
6231 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 6232 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
6233
7a3e97b0
SY		 6234 /* send command to the controller */
6235 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 6236
6237 /* Make sure descriptors are ready before ringing the task doorbell */
6238 wmb();
6239
b873a275 6240 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 6241 /* Make sure that doorbell is committed immediately */
6242 wmb();
7a3e97b0
SY		 6243
6244 spin_unlock_irqrestore(host->host_lock, flags);
6245
6667e6d9
OS		 6246 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
6247
7a3e97b0 6248 /* wait until the task management command is completed */
69a6c269 6249 err = wait_for_completion_io_timeout(&wait,
e2933132 6250 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 6251 if (!err) {
69a6c269
BVA		 6252 /*
6253 * Make sure that ufshcd_compl_tm() does not trigger a
6254 * use-after-free.
6255 */
6256 req->end_io_data = NULL;
6667e6d9 6257 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
e2933132
SRT		 6258 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6259 __func__, tm_function);
6260 if (ufshcd_clear_tm_cmd(hba, free_slot))
6261 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
6262 __func__, free_slot);
6263 err = -ETIMEDOUT;
6264 } else {
c6049cd9
CH		 6265 err = 0;
6266 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
6267
6667e6d9 6268 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
7a3e97b0 6269 }
e2933132 6270
b557217c
SC		 6271 spin_lock_irqsave(hba->host->host_lock, flags);
6272 __clear_bit(free_slot, &hba->outstanding_tasks);
6273 spin_unlock_irqrestore(hba->host->host_lock, flags);
6274
69a6c269 6275 blk_put_request(req);
e2933132 6276
1ab27c9c 6277 ufshcd_release(hba);
7a3e97b0
SY		 6278 return err;
6279}
6280
c6049cd9
CH		 6281/**
6282 * ufshcd_issue_tm_cmd - issues task management commands to controller
6283 * @hba: per adapter instance
6284 * @lun_id: LUN ID to which TM command is sent
6285 * @task_id: task ID to which the TM command is applicable
6286 * @tm_function: task management function opcode
6287 * @tm_response: task management service response return value
6288 *
6289 * Returns non-zero value on error, zero on success.
6290 */
6291static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6292 u8 tm_function, u8 *tm_response)
6293{
6294 struct utp_task_req_desc treq = { { 0 }, };
6295 int ocs_value, err;
6296
6297 /* Configure task request descriptor */
6298 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6299 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6300
6301 /* Configure task request UPIU */
6302 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6303 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6304 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6305
6306 /*
6307 * The host shall provide the same value for LUN field in the basic
6308 * header and for Input Parameter.
6309 */
6310 treq.input_param1 = cpu_to_be32(lun_id);
6311 treq.input_param2 = cpu_to_be32(task_id);
6312
6313 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6314 if (err == -ETIMEDOUT)
6315 return err;
6316
6317 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6318 if (ocs_value != OCS_SUCCESS)
6319 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6320 __func__, ocs_value);
6321 else if (tm_response)
6322 *tm_response = be32_to_cpu(treq.output_param1) &
6323 MASK_TM_SERVICE_RESP;
6324 return err;
6325}
6326
5e0a86ee
AA		 6327/**
6328 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6329 * @hba: per-adapter instance
6330 * @req_upiu: upiu request
6331 * @rsp_upiu: upiu reply
5e0a86ee
AA		 6332 * @desc_buff: pointer to descriptor buffer, NULL if NA
6333 * @buff_len: descriptor size, 0 if NA
d0e9760d 6334 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 6335 * @desc_op: descriptor operation
6336 *
6337 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6338 * Therefore, it "rides" the device management infrastructure: uses its tag and
6339 * tasks work queues.
6340 *
6341 * Since there is only one available tag for device management commands,
6342 * the caller is expected to hold the hba->dev_cmd.lock mutex.
6343 */
6344static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6345 struct utp_upiu_req *req_upiu,
6346 struct utp_upiu_req *rsp_upiu,
6347 u8 *desc_buff, int *buff_len,
7f674c38 6348 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 6349 enum query_opcode desc_op)
6350{
7252a360
BVA		 6351 struct request_queue *q = hba->cmd_queue;
6352 struct request *req;
5e0a86ee
AA		 6353 struct ufshcd_lrb *lrbp;
6354 int err = 0;
6355 int tag;
6356 struct completion wait;
6357 unsigned long flags;
a23064c4 6358 u8 upiu_flags;
5e0a86ee
AA		 6359
6360 down_read(&hba->clk_scaling_lock);
6361
7252a360 6362 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 6363 if (IS_ERR(req)) {
6364 err = PTR_ERR(req);
6365 goto out_unlock;
6366 }
7252a360
BVA		 6367 tag = req->tag;
6368 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5e0a86ee
AA		 6369
6370 init_completion(&wait);
6371 lrbp = &hba->lrb[tag];
6372 WARN_ON(lrbp->cmd);
6373
6374 lrbp->cmd = NULL;
6375 lrbp->sense_bufflen = 0;
6376 lrbp->sense_buffer = NULL;
6377 lrbp->task_tag = tag;
6378 lrbp->lun = 0;
6379 lrbp->intr_cmd = true;
df043c74 6380 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
5e0a86ee
AA		 6381 hba->dev_cmd.type = cmd_type;
6382
6383 switch (hba->ufs_version) {
6384 case UFSHCI_VERSION_10:
6385 case UFSHCI_VERSION_11:
6386 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6387 break;
6388 default:
6389 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6390 break;
6391 }
6392
6393 /* update the task tag in the request upiu */
6394 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6395
6396 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6397
6398 /* just copy the upiu request as it is */
6399 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6400 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6401 /* The Data Segment Area is optional depending upon the query
6402 * function value. for WRITE DESCRIPTOR, the data segment
6403 * follows right after the tsf.
6404 */
6405 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6406 *buff_len = 0;
6407 }
6408
6409 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6410
6411 hba->dev_cmd.complete = &wait;
6412
6413 /* Make sure descriptors are ready before ringing the doorbell */
6414 wmb();
6415 spin_lock_irqsave(hba->host->host_lock, flags);
6416 ufshcd_send_command(hba, tag);
6417 spin_unlock_irqrestore(hba->host->host_lock, flags);
6418
6419 /*
6420 * ignore the returning value here - ufshcd_check_query_response is
6421 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6422 * read the response directly ignoring all errors.
6423 */
6424 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6425
6426 /* just copy the upiu response as it is */
6427 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 6428 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6429 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6430 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6431 MASK_QUERY_DATA_SEG_LEN;
6432
6433 if (*buff_len >= resp_len) {
6434 memcpy(desc_buff, descp, resp_len);
6435 *buff_len = resp_len;
6436 } else {
3d4881d1
BH		 6437 dev_warn(hba->dev,
6438 "%s: rsp size %d is bigger than buffer size %d",
6439 __func__, resp_len, *buff_len);
4bbbe242
AA		 6440 *buff_len = 0;
6441 err = -EINVAL;
6442 }
6443 }
5e0a86ee 6444
7252a360 6445 blk_put_request(req);
bb14dd15 6446out_unlock:
5e0a86ee
AA		 6447 up_read(&hba->clk_scaling_lock);
6448 return err;
6449}
6450
6451/**
6452 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6453 * @hba: per-adapter instance
6454 * @req_upiu: upiu request
6455 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
6456 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
6457 * @desc_buff: pointer to descriptor buffer, NULL if NA
6458 * @buff_len: descriptor size, 0 if NA
6459 * @desc_op: descriptor operation
6460 *
6461 * Supports UTP Transfer requests (nop and query), and UTP Task
6462 * Management requests.
6463 * It is up to the caller to fill the upiu conent properly, as it will
6464 * be copied without any further input validations.
6465 */
6466int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6467 struct utp_upiu_req *req_upiu,
6468 struct utp_upiu_req *rsp_upiu,
6469 int msgcode,
6470 u8 *desc_buff, int *buff_len,
6471 enum query_opcode desc_op)
6472{
6473 int err;
7f674c38 6474 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee
AA		 6475 struct utp_task_req_desc treq = { { 0 }, };
6476 int ocs_value;
6477 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6478
5e0a86ee
AA		 6479 switch (msgcode) {
6480 case UPIU_TRANSACTION_NOP_OUT:
6481 cmd_type = DEV_CMD_TYPE_NOP;
6482 /* fall through */
6483 case UPIU_TRANSACTION_QUERY_REQ:
6484 ufshcd_hold(hba, false);
6485 mutex_lock(&hba->dev_cmd.lock);
6486 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6487 desc_buff, buff_len,
6488 cmd_type, desc_op);
6489 mutex_unlock(&hba->dev_cmd.lock);
6490 ufshcd_release(hba);
6491
6492 break;
6493 case UPIU_TRANSACTION_TASK_REQ:
6494 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6495 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6496
6497 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6498
6499 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6500 if (err == -ETIMEDOUT)
6501 break;
6502
6503 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6504 if (ocs_value != OCS_SUCCESS) {
6505 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6506 ocs_value);
6507 break;
6508 }
6509
6510 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6511
6512 break;
6513 default:
6514 err = -EINVAL;
6515
6516 break;
6517 }
6518
5e0a86ee
AA		 6519 return err;
6520}
6521
7a3e97b0 6522/**
3441da7d
SRT		 6523 * ufshcd_eh_device_reset_handler - device reset handler registered to
6524 * scsi layer.
7a3e97b0
SY		 6525 * @cmd: SCSI command pointer
6526 *
6527 * Returns SUCCESS/FAILED
6528 */
3441da7d 6529static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6530{
6531 struct Scsi_Host *host;
6532 struct ufs_hba *hba;
6533 unsigned int tag;
6534 u32 pos;
6535 int err;
e2933132
SRT		 6536 u8 resp = 0xF;
6537 struct ufshcd_lrb *lrbp;
3441da7d 6538 unsigned long flags;
7a3e97b0
SY		 6539
6540 host = cmd->device->host;
6541 hba = shost_priv(host);
6542 tag = cmd->request->tag;
6543
e2933132
SRT		 6544 lrbp = &hba->lrb[tag];
6545 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6546 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6547 if (!err)
6548 err = resp;
7a3e97b0 6549 goto out;
e2933132 6550 }
7a3e97b0 6551
3441da7d
SRT		 6552 /* clear the commands that were pending for corresponding LUN */
6553 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6554 if (hba->lrb[pos].lun == lrbp->lun) {
6555 err = ufshcd_clear_cmd(hba, pos);
6556 if (err)
6557 break;
7a3e97b0 6558 }
3441da7d
SRT		 6559 }
6560 spin_lock_irqsave(host->host_lock, flags);
6561 ufshcd_transfer_req_compl(hba);
6562 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 6563
7a3e97b0 6564out:
7fabb77b 6565 hba->req_abort_count = 0;
8808b4e9 6566 ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
3441da7d
SRT		 6567 if (!err) {
6568 err = SUCCESS;
6569 } else {
6570 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6571 err = FAILED;
6572 }
7a3e97b0
SY		 6573 return err;
6574}
6575
e0b299e3
GB		 6576static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6577{
6578 struct ufshcd_lrb *lrbp;
6579 int tag;
6580
6581 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6582 lrbp = &hba->lrb[tag];
6583 lrbp->req_abort_skip = true;
6584 }
6585}
6586
7a3e97b0 6587/**
307348f6 6588 * ufshcd_try_to_abort_task - abort a specific task
7a3e97b0
SY		 6589 * @cmd: SCSI command pointer
6590 *
f20810d8
SRT		 6591 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6592 * command, and in host controller by clearing the door-bell register. There can
6593 * be race between controller sending the command to the device while abort is
6594 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6595 * really issued and then try to abort it.
6596 *
307348f6
CG		 6597 * Returns zero on success, non-zero on failure
6598 */
6599static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
6600{
6601 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6602 int err = 0;
6603 int poll_cnt;
6604 u8 resp = 0xF;
6605 u32 reg;
6606
6607 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6608 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6609 UFS_QUERY_TASK, &resp);
6610 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6611 /* cmd pending in the device */
6612 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6613 __func__, tag);
6614 break;
6615 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6616 /*
6617 * cmd not pending in the device, check if it is
6618 * in transition.
6619 */
6620 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6621 __func__, tag);
6622 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6623 if (reg & (1 << tag)) {
6624 /* sleep for max. 200us to stabilize */
6625 usleep_range(100, 200);
6626 continue;
6627 }
6628 /* command completed already */
6629 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6630 __func__, tag);
6631 goto out;
6632 } else {
6633 dev_err(hba->dev,
6634 "%s: no response from device. tag = %d, err %d\n",
6635 __func__, tag, err);
6636 if (!err)
6637 err = resp; /* service response error */
6638 goto out;
6639 }
6640 }
6641
6642 if (!poll_cnt) {
6643 err = -EBUSY;
6644 goto out;
6645 }
6646
6647 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6648 UFS_ABORT_TASK, &resp);
6649 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6650 if (!err) {
6651 err = resp; /* service response error */
6652 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6653 __func__, tag, err);
6654 }
6655 goto out;
6656 }
6657
6658 err = ufshcd_clear_cmd(hba, tag);
6659 if (err)
6660 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6661 __func__, tag, err);
6662
6663out:
6664 return err;
6665}
6666
6667/**
6668 * ufshcd_abort - scsi host template eh_abort_handler callback
6669 * @cmd: SCSI command pointer
6670 *
7a3e97b0
SY		 6671 * Returns SUCCESS/FAILED
6672 */
6673static int ufshcd_abort(struct scsi_cmnd *cmd)
6674{
6675 struct Scsi_Host *host;
6676 struct ufs_hba *hba;
6677 unsigned long flags;
6678 unsigned int tag;
f20810d8 6679 int err = 0;
e2933132 6680 struct ufshcd_lrb *lrbp;
e9d501b1 6681 u32 reg;
7a3e97b0
SY		 6682
6683 host = cmd->device->host;
6684 hba = shost_priv(host);
6685 tag = cmd->request->tag;
e7d38257 6686 lrbp = &hba->lrb[tag];
14497328
YG		 6687 if (!ufshcd_valid_tag(hba, tag)) {
6688 dev_err(hba->dev,
6689 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6690 __func__, tag, cmd, cmd->request);
6691 BUG();
6692 }
7a3e97b0 6693
e7d38257
DR		 6694 /*
6695 * Task abort to the device W-LUN is illegal. When this command
6696 * will fail, due to spec violation, scsi err handling next step
6697 * will be to send LU reset which, again, is a spec violation.
6698 * To avoid these unnecessary/illegal step we skip to the last error
6699 * handling stage: reset and restore.
6700 */
6701 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6702 return ufshcd_eh_host_reset_handler(cmd);
6703
1ab27c9c 6704 ufshcd_hold(hba, false);
14497328 6705 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 6706 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 6707 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6708 dev_err(hba->dev,
6709 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6710 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 6711 goto out;
14497328 6712 }
7a3e97b0 6713
e9d501b1
DR		 6714 if (!(reg & (1 << tag))) {
6715 dev_err(hba->dev,
6716 "%s: cmd was completed, but without a notifying intr, tag = %d",
6717 __func__, tag);
6718 }
6719
66cc820f
DR		 6720 /* Print Transfer Request of aborted task */
6721 dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 6722
7fabb77b
GB		 6723 /*
6724 * Print detailed info about aborted request.
6725 * As more than one request might get aborted at the same time,
6726 * print full information only for the first aborted request in order
6727 * to reduce repeated printouts. For other aborted requests only print
6728 * basic details.
6729 */
6730 scsi_print_command(hba->lrb[tag].cmd);
6731 if (!hba->req_abort_count) {
8808b4e9 6732 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
7fabb77b 6733 ufshcd_print_host_regs(hba);
6ba65588 6734 ufshcd_print_host_state(hba);
7fabb77b
GB		 6735 ufshcd_print_pwr_info(hba);
6736 ufshcd_print_trs(hba, 1 << tag, true);
6737 } else {
6738 ufshcd_print_trs(hba, 1 << tag, false);
6739 }
6740 hba->req_abort_count++;
e0b299e3
GB		 6741
6742 /* Skip task abort in case previous aborts failed and report failure */
6743 if (lrbp->req_abort_skip) {
6744 err = -EIO;
6745 goto out;
6746 }
6747
307348f6
CG		 6748 err = ufshcd_try_to_abort_task(hba, tag);
6749 if (err)
f20810d8
SRT		 6750 goto out;
6751
7a3e97b0 6752 spin_lock_irqsave(host->host_lock, flags);
8bb2dde0 6753 __ufshcd_transfer_req_compl(hba, (1UL << tag));
7a3e97b0 6754 spin_unlock_irqrestore(host->host_lock, flags);
5a0b0cb9 6755
7a3e97b0 6756out:
f20810d8
SRT		 6757 if (!err) {
6758 err = SUCCESS;
6759 } else {
6760 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6761 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6762 err = FAILED;
6763 }
6764
1ab27c9c
ST		 6765 /*
6766 * This ufshcd_release() corresponds to the original scsi cmd that got
6767 * aborted here (as we won't get any IRQ for it).
6768 */
6769 ufshcd_release(hba);
7a3e97b0
SY		 6770 return err;
6771}
6772
3441da7d
SRT		 6773/**
6774 * ufshcd_host_reset_and_restore - reset and restore host controller
6775 * @hba: per-adapter instance
6776 *
6777 * Note that host controller reset may issue DME_RESET to
6778 * local and remote (device) Uni-Pro stack and the attributes
6779 * are reset to default state.
6780 *
6781 * Returns zero on success, non-zero on failure
6782 */
6783static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6784{
6785 int err;
3441da7d
SRT		 6786 unsigned long flags;
6787
2df74b69
CG		 6788 /*
6789 * Stop the host controller and complete the requests
6790 * cleared by h/w
6791 */
5cac1095
BVA		 6792 ufshcd_hba_stop(hba);
6793
3441da7d 6794 spin_lock_irqsave(hba->host->host_lock, flags);
2df74b69
CG		 6795 hba->silence_err_logs = true;
6796 ufshcd_complete_requests(hba);
6797 hba->silence_err_logs = false;
3441da7d
SRT		 6798 spin_unlock_irqrestore(hba->host->host_lock, flags);
6799
a3cd5ec5 6800 /* scale up clocks to max frequency before full reinitialization */
394b949f 6801 ufshcd_set_clk_freq(hba, true);
a3cd5ec5 6802
3441da7d
SRT		 6803 err = ufshcd_hba_enable(hba);
6804 if (err)
6805 goto out;
6806
6807 /* Establish the link again and restore the device */
1b9e2141 6808 err = ufshcd_probe_hba(hba, false);
1d337ec2 6809
3441da7d
SRT		 6810out:
6811 if (err)
6812 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
8808b4e9 6813 ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
3441da7d
SRT		 6814 return err;
6815}
6816
6817/**
6818 * ufshcd_reset_and_restore - reset and re-initialize host/device
6819 * @hba: per-adapter instance
6820 *
6821 * Reset and recover device, host and re-establish link. This
6822 * is helpful to recover the communication in fatal error conditions.
6823 *
6824 * Returns zero on success, non-zero on failure
6825 */
6826static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6827{
4db7a236
CG		 6828 u32 saved_err;
6829 u32 saved_uic_err;
3441da7d 6830 int err = 0;
4db7a236 6831 unsigned long flags;
1d337ec2 6832 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 6833
4db7a236
CG		 6834 /*
6835 * This is a fresh start, cache and clear saved error first,
6836 * in case new error generated during reset and restore.
6837 */
6838 spin_lock_irqsave(hba->host->host_lock, flags);
6839 saved_err = hba->saved_err;
6840 saved_uic_err = hba->saved_uic_err;
6841 hba->saved_err = 0;
6842 hba->saved_uic_err = 0;
6843 spin_unlock_irqrestore(hba->host->host_lock, flags);
6844
1d337ec2 6845 do {
d8d9f793
BA		 6846 /* Reset the attached device */
6847 ufshcd_vops_device_reset(hba);
6848
1d337ec2
SRT		 6849 err = ufshcd_host_reset_and_restore(hba);
6850 } while (err && --retries);
3441da7d 6851
4db7a236
CG		 6852 spin_lock_irqsave(hba->host->host_lock, flags);
6853 /*
6854 * Inform scsi mid-layer that we did reset and allow to handle
6855 * Unit Attention properly.
6856 */
6857 scsi_report_bus_reset(hba->host, 0);
6858 if (err) {
6859 hba->saved_err |= saved_err;
6860 hba->saved_uic_err |= saved_uic_err;
6861 }
6862 spin_unlock_irqrestore(hba->host->host_lock, flags);
6863
3441da7d
SRT		 6864 return err;
6865}
6866
6867/**
6868 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 6869 * @cmd: SCSI command pointer
3441da7d
SRT		 6870 *
6871 * Returns SUCCESS/FAILED
6872 */
6873static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6874{
4db7a236 6875 int err = SUCCESS;
3441da7d
SRT		 6876 unsigned long flags;
6877 struct ufs_hba *hba;
6878
6879 hba = shost_priv(cmd->device->host);
6880
4db7a236
CG		 6881 spin_lock_irqsave(hba->host->host_lock, flags);
6882 hba->force_reset = true;
6883 ufshcd_schedule_eh_work(hba);
6884 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
3441da7d
SRT		 6885 spin_unlock_irqrestore(hba->host->host_lock, flags);
6886
4db7a236 6887 flush_work(&hba->eh_work);
3441da7d
SRT		 6888
6889 spin_lock_irqsave(hba->host->host_lock, flags);
4db7a236 6890 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
3441da7d 6891 err = FAILED;
3441da7d
SRT		 6892 spin_unlock_irqrestore(hba->host->host_lock, flags);
6893
6894 return err;
6895}
6896
3a4bf06d
YG		 6897/**
6898 * ufshcd_get_max_icc_level - calculate the ICC level
6899 * @sup_curr_uA: max. current supported by the regulator
6900 * @start_scan: row at the desc table to start scan from
6901 * @buff: power descriptor buffer
6902 *
6903 * Returns calculated max ICC level for specific regulator
6904 */
6905static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6906{
6907 int i;
6908 int curr_uA;
6909 u16 data;
6910 u16 unit;
6911
6912 for (i = start_scan; i >= 0; i--) {
d79713f9 6913 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 6914 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6915 ATTR_ICC_LVL_UNIT_OFFSET;
6916 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6917 switch (unit) {
6918 case UFSHCD_NANO_AMP:
6919 curr_uA = curr_uA / 1000;
6920 break;
6921 case UFSHCD_MILI_AMP:
6922 curr_uA = curr_uA * 1000;
6923 break;
6924 case UFSHCD_AMP:
6925 curr_uA = curr_uA * 1000 * 1000;
6926 break;
6927 case UFSHCD_MICRO_AMP:
6928 default:
6929 break;
6930 }
6931 if (sup_curr_uA >= curr_uA)
6932 break;
6933 }
6934 if (i < 0) {
6935 i = 0;
6936 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6937 }
6938
6939 return (u32)i;
6940}
6941
6942/**
6943 * ufshcd_calc_icc_level - calculate the max ICC level
6944 * In case regulators are not initialized we'll return 0
6945 * @hba: per-adapter instance
6946 * @desc_buf: power descriptor buffer to extract ICC levels from.
6947 * @len: length of desc_buff
6948 *
6949 * Returns calculated ICC level
6950 */
6951static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6952 u8 *desc_buf, int len)
6953{
6954 u32 icc_level = 0;
6955
6956 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6957 !hba->vreg_info.vccq2) {
6958 dev_err(hba->dev,
6959 "%s: Regulator capability was not set, actvIccLevel=%d",
6960 __func__, icc_level);
6961 goto out;
6962 }
6963
0487fff7 6964 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 6965 icc_level = ufshcd_get_max_icc_level(
6966 hba->vreg_info.vcc->max_uA,
6967 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6968 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6969
0487fff7 6970 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 6971 icc_level = ufshcd_get_max_icc_level(
6972 hba->vreg_info.vccq->max_uA,
6973 icc_level,
6974 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6975
0487fff7 6976 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 6977 icc_level = ufshcd_get_max_icc_level(
6978 hba->vreg_info.vccq2->max_uA,
6979 icc_level,
6980 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6981out:
6982 return icc_level;
6983}
6984
e89860f1 6985static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
3a4bf06d
YG		 6986{
6987 int ret;
7a0bf85b 6988 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
bbe21d7a 6989 u8 *desc_buf;
e89860f1 6990 u32 icc_level;
bbe21d7a
KC		 6991
6992 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6993 if (!desc_buf)
6994 return;
3a4bf06d 6995
c4607a09
BH		 6996 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
6997 desc_buf, buff_len);
3a4bf06d
YG		 6998 if (ret) {
6999 dev_err(hba->dev,
7000 "%s: Failed reading power descriptor.len = %d ret = %d",
7001 __func__, buff_len, ret);
bbe21d7a 7002 goto out;
3a4bf06d
YG		 7003 }
7004
e89860f1
CG		 7005 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7006 buff_len);
7007 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
3a4bf06d 7008
dbd34a61 7009 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
e89860f1 7010 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
3a4bf06d
YG		 7011
7012 if (ret)
7013 dev_err(hba->dev,
7014 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
e89860f1 7015 __func__, icc_level, ret);
3a4bf06d 7016
bbe21d7a
KC		 7017out:
7018 kfree(desc_buf);
3a4bf06d
YG		 7019}
7020
fb276f77
CG		 7021static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7022{
7023 scsi_autopm_get_device(sdev);
7024 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7025 if (sdev->rpm_autosuspend)
7026 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7027 RPM_AUTOSUSPEND_DELAY_MS);
7028 scsi_autopm_put_device(sdev);
7029}
7030
2a8fa600
SJ		 7031/**
7032 * ufshcd_scsi_add_wlus - Adds required W-LUs
7033 * @hba: per-adapter instance
7034 *
7035 * UFS device specification requires the UFS devices to support 4 well known
7036 * logical units:
7037 * "REPORT_LUNS" (address: 01h)
7038 * "UFS Device" (address: 50h)
7039 * "RPMB" (address: 44h)
7040 * "BOOT" (address: 30h)
7041 * UFS device's power management needs to be controlled by "POWER CONDITION"
7042 * field of SSU (START STOP UNIT) command. But this "power condition" field
7043 * will take effect only when its sent to "UFS device" well known logical unit
7044 * hence we require the scsi_device instance to represent this logical unit in
7045 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 7046 *
2a8fa600
SJ		 7047 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7048 * Block) LU so user space process can control this LU. User space may also
7049 * want to have access to BOOT LU.
8aa29f19 7050 *
2a8fa600
SJ		 7051 * This function adds scsi device instances for each of all well known LUs
7052 * (except "REPORT LUNS" LU).
7053 *
7054 * Returns zero on success (all required W-LUs are added successfully),
7055 * non-zero error value on failure (if failed to add any of the required W-LU).
7056 */
7057static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7058{
7059 int ret = 0;
7c48bfd0
AM		 7060 struct scsi_device *sdev_rpmb;
7061 struct scsi_device *sdev_boot;
2a8fa600
SJ		 7062
7063 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
7064 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7065 if (IS_ERR(hba->sdev_ufs_device)) {
7066 ret = PTR_ERR(hba->sdev_ufs_device);
7067 hba->sdev_ufs_device = NULL;
7068 goto out;
7069 }
fb276f77 7070 ufshcd_blk_pm_runtime_init(hba->sdev_ufs_device);
7c48bfd0 7071 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 7072
7c48bfd0 7073 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 7074 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7c48bfd0
AM		 7075 if (IS_ERR(sdev_rpmb)) {
7076 ret = PTR_ERR(sdev_rpmb);
3d21fbde 7077 goto remove_sdev_ufs_device;
2a8fa600 7078 }
fb276f77 7079 ufshcd_blk_pm_runtime_init(sdev_rpmb);
7c48bfd0 7080 scsi_device_put(sdev_rpmb);
3d21fbde
HK		 7081
7082 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7083 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
fb276f77 7084 if (IS_ERR(sdev_boot)) {
3d21fbde 7085 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
fb276f77
CG		 7086 } else {
7087 ufshcd_blk_pm_runtime_init(sdev_boot);
3d21fbde 7088 scsi_device_put(sdev_boot);
fb276f77 7089 }
2a8fa600
SJ		 7090 goto out;
7091
2a8fa600
SJ		 7092remove_sdev_ufs_device:
7093 scsi_remove_device(hba->sdev_ufs_device);
7094out:
7095 return ret;
7096}
7097
3d17b9b5
AD		 7098static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7099{
a7f1e69d 7100 struct ufs_dev_info *dev_info = &hba->dev_info;
6f8d5a6a
SC		 7101 u8 lun;
7102 u32 d_lu_wb_buf_alloc;
7103
817d7e14
SC		 7104 if (!ufshcd_is_wb_allowed(hba))
7105 return;
a7f1e69d
SC		 7106 /*
7107 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7108 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7109 * enabled
7110 */
7111 if (!(dev_info->wspecversion >= 0x310 ||
7112 dev_info->wspecversion == 0x220 ||
7113 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7114 goto wb_disabled;
817d7e14 7115
7a0bf85b
BH		 7116 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7117 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
817d7e14
SC		 7118 goto wb_disabled;
7119
a7f1e69d 7120 dev_info->d_ext_ufs_feature_sup =
3d17b9b5
AD		 7121 get_unaligned_be32(desc_buf +
7122 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
817d7e14 7123
a7f1e69d 7124 if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
817d7e14
SC		 7125 goto wb_disabled;
7126
3d17b9b5
AD		 7127 /*
7128 * WB may be supported but not configured while provisioning.
7129 * The spec says, in dedicated wb buffer mode,
7130 * a max of 1 lun would have wb buffer configured.
7131 * Now only shared buffer mode is supported.
7132 */
a7f1e69d 7133 dev_info->b_wb_buffer_type =
3d17b9b5
AD		 7134 desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7135
a7f1e69d 7136 dev_info->b_presrv_uspc_en =
3d17b9b5
AD		 7137 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7138
a7f1e69d
SC		 7139 if (dev_info->b_wb_buffer_type == WB_BUF_MODE_SHARED) {
7140 dev_info->d_wb_alloc_units =
6f8d5a6a
SC		 7141 get_unaligned_be32(desc_buf +
7142 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS);
a7f1e69d 7143 if (!dev_info->d_wb_alloc_units)
6f8d5a6a
SC		 7144 goto wb_disabled;
7145 } else {
7146 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7147 d_lu_wb_buf_alloc = 0;
7148 ufshcd_read_unit_desc_param(hba,
7149 lun,
7150 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7151 (u8 *)&d_lu_wb_buf_alloc,
7152 sizeof(d_lu_wb_buf_alloc));
7153 if (d_lu_wb_buf_alloc) {
a7f1e69d 7154 dev_info->wb_dedicated_lu = lun;
6f8d5a6a
SC		 7155 break;
7156 }
7157 }
817d7e14 7158
6f8d5a6a
SC		 7159 if (!d_lu_wb_buf_alloc)
7160 goto wb_disabled;
7161 }
817d7e14
SC		 7162 return;
7163
7164wb_disabled:
7165 hba->caps &= ~UFSHCD_CAP_WB_EN;
7166}
7167
8db269a5 7168void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
817d7e14
SC		 7169{
7170 struct ufs_dev_fix *f;
7171 struct ufs_dev_info *dev_info = &hba->dev_info;
7172
8db269a5
SC		 7173 if (!fixups)
7174 return;
7175
7176 for (f = fixups; f->quirk; f++) {
817d7e14
SC		 7177 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7178 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7179 ((dev_info->model &&
7180 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7181 !strcmp(f->model, UFS_ANY_MODEL)))
7182 hba->dev_quirks |= f->quirk;
7183 }
3d17b9b5 7184}
8db269a5 7185EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
3d17b9b5 7186
c28c00ba
SC		 7187static void ufs_fixup_device_setup(struct ufs_hba *hba)
7188{
7189 /* fix by general quirk table */
8db269a5 7190 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
c28c00ba
SC		 7191
7192 /* allow vendors to fix quirks */
7193 ufshcd_vops_fixup_dev_quirks(hba);
7194}
7195
09750066 7196static int ufs_get_device_desc(struct ufs_hba *hba)
c58ab7aa
YG		 7197{
7198 int err;
7199 u8 model_index;
bbe21d7a 7200 u8 *desc_buf;
09750066 7201 struct ufs_dev_info *dev_info = &hba->dev_info;
4b828fe1 7202
458a45f5 7203 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
bbe21d7a
KC		 7204 if (!desc_buf) {
7205 err = -ENOMEM;
7206 goto out;
7207 }
c58ab7aa 7208
c4607a09 7209 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7a0bf85b 7210 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
c58ab7aa
YG		 7211 if (err) {
7212 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7213 __func__, err);
7214 goto out;
7215 }
7216
7217 /*
7218 * getting vendor (manufacturerID) and Bank Index in big endian
7219 * format
7220 */
09750066 7221 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 7222 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7223
09f17791
CG		 7224 /* getting Specification Version in big endian format */
7225 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7226 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7227
c58ab7aa 7228 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
3d17b9b5 7229
4b828fe1 7230 err = ufshcd_read_string_desc(hba, model_index,
09750066 7231 &dev_info->model, SD_ASCII_STD);
4b828fe1 7232 if (err < 0) {
c58ab7aa
YG		 7233 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7234 __func__, err);
7235 goto out;
7236 }
7237
817d7e14
SC		 7238 ufs_fixup_device_setup(hba);
7239
a7f1e69d 7240 ufshcd_wb_probe(hba, desc_buf);
817d7e14 7241
4b828fe1
TW		 7242 /*
7243 * ufshcd_read_string_desc returns size of the string
7244 * reset the error value
7245 */
7246 err = 0;
c58ab7aa
YG		 7247
7248out:
bbe21d7a 7249 kfree(desc_buf);
c58ab7aa
YG		 7250 return err;
7251}
7252
09750066 7253static void ufs_put_device_desc(struct ufs_hba *hba)
4b828fe1 7254{
09750066
BH		 7255 struct ufs_dev_info *dev_info = &hba->dev_info;
7256
7257 kfree(dev_info->model);
7258 dev_info->model = NULL;
4b828fe1
TW		 7259}
7260
37113106
YG		 7261/**
7262 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7263 * @hba: per-adapter instance
7264 *
7265 * PA_TActivate parameter can be tuned manually if UniPro version is less than
7266 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7267 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7268 * the hibern8 exit latency.
7269 *
7270 * Returns zero on success, non-zero error value on failure.
7271 */
7272static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7273{
7274 int ret = 0;
7275 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7276
7277 ret = ufshcd_dme_peer_get(hba,
7278 UIC_ARG_MIB_SEL(
7279 RX_MIN_ACTIVATETIME_CAPABILITY,
7280 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7281 &peer_rx_min_activatetime);
7282 if (ret)
7283 goto out;
7284
7285 /* make sure proper unit conversion is applied */
7286 tuned_pa_tactivate =
7287 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7288 / PA_TACTIVATE_TIME_UNIT_US);
7289 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7290 tuned_pa_tactivate);
7291
7292out:
7293 return ret;
7294}
7295
7296/**
7297 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7298 * @hba: per-adapter instance
7299 *
7300 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7301 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7302 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7303 * This optimal value can help reduce the hibern8 exit latency.
7304 *
7305 * Returns zero on success, non-zero error value on failure.
7306 */
7307static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7308{
7309 int ret = 0;
7310 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7311 u32 max_hibern8_time, tuned_pa_hibern8time;
7312
7313 ret = ufshcd_dme_get(hba,
7314 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7315 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7316 &local_tx_hibern8_time_cap);
7317 if (ret)
7318 goto out;
7319
7320 ret = ufshcd_dme_peer_get(hba,
7321 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7322 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7323 &peer_rx_hibern8_time_cap);
7324 if (ret)
7325 goto out;
7326
7327 max_hibern8_time = max(local_tx_hibern8_time_cap,
7328 peer_rx_hibern8_time_cap);
7329 /* make sure proper unit conversion is applied */
7330 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7331 / PA_HIBERN8_TIME_UNIT_US);
7332 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7333 tuned_pa_hibern8time);
7334out:
7335 return ret;
7336}
7337
c6a6db43 7338/**
7339 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7340 * less than device PA_TACTIVATE time.
7341 * @hba: per-adapter instance
7342 *
7343 * Some UFS devices require host PA_TACTIVATE to be lower than device
7344 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7345 * for such devices.
7346 *
7347 * Returns zero on success, non-zero error value on failure.
7348 */
7349static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7350{
7351 int ret = 0;
7352 u32 granularity, peer_granularity;
7353 u32 pa_tactivate, peer_pa_tactivate;
7354 u32 pa_tactivate_us, peer_pa_tactivate_us;
7355 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7356
7357 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7358 &granularity);
7359 if (ret)
7360 goto out;
7361
7362 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7363 &peer_granularity);
7364 if (ret)
7365 goto out;
7366
7367 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7368 (granularity > PA_GRANULARITY_MAX_VAL)) {
7369 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7370 __func__, granularity);
7371 return -EINVAL;
7372 }
7373
7374 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7375 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7376 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7377 __func__, peer_granularity);
7378 return -EINVAL;
7379 }
7380
7381 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7382 if (ret)
7383 goto out;
7384
7385 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7386 &peer_pa_tactivate);
7387 if (ret)
7388 goto out;
7389
7390 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7391 peer_pa_tactivate_us = peer_pa_tactivate *
7392 gran_to_us_table[peer_granularity - 1];
7393
7394 if (pa_tactivate_us > peer_pa_tactivate_us) {
7395 u32 new_peer_pa_tactivate;
7396
7397 new_peer_pa_tactivate = pa_tactivate_us /
7398 gran_to_us_table[peer_granularity - 1];
7399 new_peer_pa_tactivate++;
7400 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7401 new_peer_pa_tactivate);
7402 }
7403
7404out:
7405 return ret;
7406}
7407
09750066 7408static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
37113106
YG		 7409{
7410 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7411 ufshcd_tune_pa_tactivate(hba);
7412 ufshcd_tune_pa_hibern8time(hba);
7413 }
7414
e91ed9e0
CG		 7415 ufshcd_vops_apply_dev_quirks(hba);
7416
37113106
YG		 7417 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7418 /* set 1ms timeout for PA_TACTIVATE */
7419 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43 7420
7421 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7422 ufshcd_quirk_tune_host_pa_tactivate(hba);
37113106
YG		 7423}
7424
ff8e20c6
DR		 7425static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7426{
ff8e20c6
DR		 7427 hba->ufs_stats.hibern8_exit_cnt = 0;
7428 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 7429 hba->req_abort_count = 0;
ff8e20c6
DR		 7430}
7431
731f0621
BH		 7432static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7433{
7434 int err;
7435 size_t buff_len;
7436 u8 *desc_buf;
7437
7a0bf85b 7438 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
731f0621
BH		 7439 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7440 if (!desc_buf) {
7441 err = -ENOMEM;
7442 goto out;
7443 }
7444
c4607a09
BH		 7445 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7446 desc_buf, buff_len);
731f0621
BH		 7447 if (err) {
7448 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7449 __func__, err);
7450 goto out;
7451 }
7452
7453 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7454 hba->dev_info.max_lu_supported = 32;
7455 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7456 hba->dev_info.max_lu_supported = 8;
7457
7458out:
7459 kfree(desc_buf);
7460 return err;
7461}
7462
9e1e8a75
SJ		 7463static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7464 {19200000, REF_CLK_FREQ_19_2_MHZ},
7465 {26000000, REF_CLK_FREQ_26_MHZ},
7466 {38400000, REF_CLK_FREQ_38_4_MHZ},
7467 {52000000, REF_CLK_FREQ_52_MHZ},
7468 {0, REF_CLK_FREQ_INVAL},
7469};
7470
7471static enum ufs_ref_clk_freq
7472ufs_get_bref_clk_from_hz(unsigned long freq)
7473{
7474 int i;
7475
7476 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7477 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7478 return ufs_ref_clk_freqs[i].val;
7479
7480 return REF_CLK_FREQ_INVAL;
7481}
7482
7483void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7484{
7485 unsigned long freq;
7486
7487 freq = clk_get_rate(refclk);
7488
7489 hba->dev_ref_clk_freq =
7490 ufs_get_bref_clk_from_hz(freq);
7491
7492 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7493 dev_err(hba->dev,
7494 "invalid ref_clk setting = %ld\n", freq);
7495}
7496
7497static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
7498{
7499 int err;
7500 u32 ref_clk;
7501 u32 freq = hba->dev_ref_clk_freq;
7502
7503 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
7504 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
7505
7506 if (err) {
7507 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
7508 err);
7509 goto out;
7510 }
7511
7512 if (ref_clk == freq)
7513 goto out; /* nothing to update */
7514
7515 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7516 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
7517
7518 if (err) {
7519 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
7520 ufs_ref_clk_freqs[freq].freq_hz);
7521 goto out;
7522 }
7523
7524 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
7525 ufs_ref_clk_freqs[freq].freq_hz);
7526
7527out:
7528 return err;
7529}
7530
1b9e2141
BH		 7531static int ufshcd_device_params_init(struct ufs_hba *hba)
7532{
7533 bool flag;
7a0bf85b 7534 int ret, i;
1b9e2141 7535
7a0bf85b
BH		 7536 /* Init device descriptor sizes */
7537 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
7538 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
1b9e2141 7539
731f0621
BH		 7540 /* Init UFS geometry descriptor related parameters */
7541 ret = ufshcd_device_geo_params_init(hba);
7542 if (ret)
7543 goto out;
7544
1b9e2141
BH		 7545 /* Check and apply UFS device quirks */
7546 ret = ufs_get_device_desc(hba);
7547 if (ret) {
7548 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
7549 __func__, ret);
7550 goto out;
7551 }
7552
09f17791
CG		 7553 ufshcd_get_ref_clk_gating_wait(hba);
7554
1b9e2141 7555 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
1f34eedf 7556 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
1b9e2141
BH		 7557 hba->dev_info.f_power_on_wp_en = flag;
7558
2b35b2ad
BH		 7559 /* Probe maximum power mode co-supported by both UFS host and device */
7560 if (ufshcd_get_max_pwr_mode(hba))
7561 dev_err(hba->dev,
7562 "%s: Failed getting max supported power mode\n",
7563 __func__);
1b9e2141
BH		 7564out:
7565 return ret;
7566}
7567
7568/**
7569 * ufshcd_add_lus - probe and add UFS logical units
7570 * @hba: per-adapter instance
7571 */
7572static int ufshcd_add_lus(struct ufs_hba *hba)
7573{
7574 int ret;
7575
1b9e2141
BH		 7576 /* Add required well known logical units to scsi mid layer */
7577 ret = ufshcd_scsi_add_wlus(hba);
7578 if (ret)
7579 goto out;
7580
7581 /* Initialize devfreq after UFS device is detected */
7582 if (ufshcd_is_clkscaling_supported(hba)) {
7583 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7584 &hba->pwr_info,
7585 sizeof(struct ufs_pa_layer_attr));
7586 hba->clk_scaling.saved_pwr_info.is_valid = true;
7587 if (!hba->devfreq) {
7588 ret = ufshcd_devfreq_init(hba);
7589 if (ret)
7590 goto out;
7591 }
7592
7593 hba->clk_scaling.is_allowed = true;
7594 }
7595
7596 ufs_bsg_probe(hba);
7597 scsi_scan_host(hba->host);
7598 pm_runtime_put_sync(hba->dev);
7599
1b9e2141
BH		 7600out:
7601 return ret;
7602}
7603
6ccf44fe 7604/**
1d337ec2
SRT		 7605 * ufshcd_probe_hba - probe hba to detect device and initialize
7606 * @hba: per-adapter instance
1b9e2141 7607 * @async: asynchronous execution or not
1d337ec2
SRT		 7608 *
7609 * Execute link-startup and verify device initialization
6ccf44fe 7610 */
1b9e2141 7611static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
6ccf44fe 7612{
6ccf44fe 7613 int ret;
4db7a236 7614 unsigned long flags;
7ff5ab47 7615 ktime_t start = ktime_get();
6ccf44fe
SJ		 7616
7617 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 7618 if (ret)
7619 goto out;
7620
ff8e20c6
DR		 7621 /* Debug counters initialization */
7622 ufshcd_clear_dbg_ufs_stats(hba);
7623
57d104c1
SJ		 7624 /* UniPro link is active now */
7625 ufshcd_set_link_active(hba);
d3e89bac 7626
1b9e2141 7627 /* Verify device initialization by sending NOP OUT UPIU */
5a0b0cb9
SRT		 7628 ret = ufshcd_verify_dev_init(hba);
7629 if (ret)
7630 goto out;
68078d5c 7631
1b9e2141 7632 /* Initiate UFS initialization, and waiting until completion */
68078d5c
DR		 7633 ret = ufshcd_complete_dev_init(hba);
7634 if (ret)
7635 goto out;
5a0b0cb9 7636
1b9e2141
BH		 7637 /*
7638 * Initialize UFS device parameters used by driver, these
7639 * parameters are associated with UFS descriptors.
7640 */
7641 if (async) {
7642 ret = ufshcd_device_params_init(hba);
7643 if (ret)
7644 goto out;
93fdd5ac
TW		 7645 }
7646
09750066 7647 ufshcd_tune_unipro_params(hba);
4b828fe1 7648
57d104c1
SJ		 7649 /* UFS device is also active now */
7650 ufshcd_set_ufs_dev_active(hba);
66ec6d59 7651 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 7652 hba->wlun_dev_clr_ua = true;
7653
2b35b2ad
BH		 7654 /* Gear up to HS gear if supported */
7655 if (hba->max_pwr_info.is_valid) {
9e1e8a75
SJ		 7656 /*
7657 * Set the right value to bRefClkFreq before attempting to
7658 * switch to HS gears.
7659 */
7660 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7661 ufshcd_set_dev_ref_clk(hba);
7eb584db 7662 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 7663 if (ret) {
7eb584db
DR		 7664 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7665 __func__, ret);
8643ae66
DL		 7666 goto out;
7667 }
6a9df818 7668 ufshcd_print_pwr_info(hba);
7eb584db 7669 }
57d104c1 7670
e89860f1
CG		 7671 /*
7672 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
7673 * and for removable UFS card as well, hence always set the parameter.
7674 * Note: Error handler may issue the device reset hence resetting
7675 * bActiveICCLevel as well so it is always safe to set this here.
7676 */
7677 ufshcd_set_active_icc_lvl(hba);
7678
3d17b9b5 7679 ufshcd_wb_config(hba);
71d848b8
CG		 7680 /* Enable Auto-Hibernate if configured */
7681 ufshcd_auto_hibern8_enable(hba);
7682
5a0b0cb9 7683out:
4db7a236
CG		 7684 spin_lock_irqsave(hba->host->host_lock, flags);
7685 if (ret)
7686 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7687 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
7688 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7689 spin_unlock_irqrestore(hba->host->host_lock, flags);
1d337ec2 7690
7ff5ab47 7691 trace_ufshcd_init(dev_name(hba->dev), ret,
7692 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7693 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 7694 return ret;
7695}
7696
7697/**
7698 * ufshcd_async_scan - asynchronous execution for probing hba
7699 * @data: data pointer to pass to this function
7700 * @cookie: cookie data
7701 */
7702static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7703{
7704 struct ufs_hba *hba = (struct ufs_hba *)data;
1b9e2141 7705 int ret;
1d337ec2 7706
1b9e2141
BH		 7707 /* Initialize hba, detect and initialize UFS device */
7708 ret = ufshcd_probe_hba(hba, true);
7709 if (ret)
7710 goto out;
7711
7712 /* Probe and add UFS logical units */
7713 ret = ufshcd_add_lus(hba);
7714out:
7715 /*
7716 * If we failed to initialize the device or the device is not
7717 * present, turn off the power/clocks etc.
7718 */
7719 if (ret) {
7720 pm_runtime_put_sync(hba->dev);
7721 ufshcd_exit_clk_scaling(hba);
7722 ufshcd_hba_exit(hba);
7723 }
6ccf44fe
SJ		 7724}
7725
d829fc8a
SN		 7726static const struct attribute_group *ufshcd_driver_groups[] = {
7727 &ufs_sysfs_unit_descriptor_group,
ec92b59c 7728 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 7729 NULL,
7730};
7731
90b8491c
SC		 7732static struct ufs_hba_variant_params ufs_hba_vps = {
7733 .hba_enable_delay_us = 1000,
d14734ae 7734 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
90b8491c
SC		 7735 .devfreq_profile.polling_ms = 100,
7736 .devfreq_profile.target = ufshcd_devfreq_target,
7737 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
7738 .ondemand_data.upthreshold = 70,
7739 .ondemand_data.downdifferential = 5,
7740};
7741
7a3e97b0
SY		 7742static struct scsi_host_template ufshcd_driver_template = {
7743 .module = THIS_MODULE,
7744 .name = UFSHCD,
7745 .proc_name = UFSHCD,
7746 .queuecommand = ufshcd_queuecommand,
7747 .slave_alloc = ufshcd_slave_alloc,
eeda4749 7748 .slave_configure = ufshcd_slave_configure,
7a3e97b0 7749 .slave_destroy = ufshcd_slave_destroy,
4264fd61 7750 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 7751 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 7752 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7753 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
7a3e97b0
SY		 7754 .this_id = -1,
7755 .sg_tablesize = SG_ALL,
7756 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
7757 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 7758 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 7759 .max_host_blocked = 1,
c40ecc12 7760 .track_queue_depth = 1,
d829fc8a 7761 .sdev_groups = ufshcd_driver_groups,
4af14d11 7762 .dma_boundary = PAGE_SIZE - 1,
49615ba1 7763 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 7764};
7765
57d104c1
SJ		 7766static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7767 int ua)
7768{
7b16a07c 7769 int ret;
57d104c1 7770
7b16a07c
BA		 7771 if (!vreg)
7772 return 0;
57d104c1 7773
0487fff7
SC		 7774 /*
7775 * "set_load" operation shall be required on those regulators
7776 * which specifically configured current limitation. Otherwise
7777 * zero max_uA may cause unexpected behavior when regulator is
7778 * enabled or set as high power mode.
7779 */
7780 if (!vreg->max_uA)
7781 return 0;
7782
7b16a07c
BA		 7783 ret = regulator_set_load(vreg->reg, ua);
7784 if (ret < 0) {
7785 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7786 __func__, vreg->name, ua, ret);
57d104c1
SJ		 7787 }
7788
7789 return ret;
7790}
7791
7792static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7793 struct ufs_vreg *vreg)
7794{
73067981 7795 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 7796}
7797
7798static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7799 struct ufs_vreg *vreg)
7800{
7c7cfdcf
AH		 7801 if (!vreg)
7802 return 0;
7803
73067981 7804 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 7805}
7806
aa497613
SRT		 7807static int ufshcd_config_vreg(struct device *dev,
7808 struct ufs_vreg *vreg, bool on)
7809{
7810 int ret = 0;
72753590
GS		 7811 struct regulator *reg;
7812 const char *name;
aa497613
SRT		 7813 int min_uV, uA_load;
7814
7815 BUG_ON(!vreg);
7816
72753590
GS		 7817 reg = vreg->reg;
7818 name = vreg->name;
7819
aa497613 7820 if (regulator_count_voltages(reg) > 0) {
90d88f47
AD		 7821 uA_load = on ? vreg->max_uA : 0;
7822 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7823 if (ret)
7824 goto out;
7825
3b141e8c
SC		 7826 if (vreg->min_uV && vreg->max_uV) {
7827 min_uV = on ? vreg->min_uV : 0;
7828 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
b0008625 7829 if (ret)
3b141e8c
SC		 7830 dev_err(dev,
7831 "%s: %s set voltage failed, err=%d\n",
aa497613 7832 __func__, name, ret);
aa497613 7833 }
aa497613
SRT		 7834 }
7835out:
7836 return ret;
7837}
7838
7839static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7840{
7841 int ret = 0;
7842
73067981 7843 if (!vreg || vreg->enabled)
aa497613
SRT		 7844 goto out;
7845
7846 ret = ufshcd_config_vreg(dev, vreg, true);
7847 if (!ret)
7848 ret = regulator_enable(vreg->reg);
7849
7850 if (!ret)
7851 vreg->enabled = true;
7852 else
7853 dev_err(dev, "%s: %s enable failed, err=%d\n",
7854 __func__, vreg->name, ret);
7855out:
7856 return ret;
7857}
7858
7859static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7860{
7861 int ret = 0;
7862
73067981 7863 if (!vreg || !vreg->enabled)
aa497613
SRT		 7864 goto out;
7865
7866 ret = regulator_disable(vreg->reg);
7867
7868 if (!ret) {
7869 /* ignore errors on applying disable config */
7870 ufshcd_config_vreg(dev, vreg, false);
7871 vreg->enabled = false;
7872 } else {
7873 dev_err(dev, "%s: %s disable failed, err=%d\n",
7874 __func__, vreg->name, ret);
7875 }
7876out:
7877 return ret;
7878}
7879
7880static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7881{
7882 int ret = 0;
7883 struct device *dev = hba->dev;
7884 struct ufs_vreg_info *info = &hba->vreg_info;
7885
aa497613
SRT		 7886 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7887 if (ret)
7888 goto out;
7889
7890 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7891 if (ret)
7892 goto out;
7893
7894 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
aa497613
SRT		 7895
7896out:
7897 if (ret) {
7898 ufshcd_toggle_vreg(dev, info->vccq2, false);
7899 ufshcd_toggle_vreg(dev, info->vccq, false);
7900 ufshcd_toggle_vreg(dev, info->vcc, false);
7901 }
7902 return ret;
7903}
7904
6a771a65
RS		 7905static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7906{
7907 struct ufs_vreg_info *info = &hba->vreg_info;
7908
60b7b823 7909 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 7910}
7911
aa497613
SRT		 7912static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7913{
7914 int ret = 0;
7915
7916 if (!vreg)
7917 goto out;
7918
7919 vreg->reg = devm_regulator_get(dev, vreg->name);
7920 if (IS_ERR(vreg->reg)) {
7921 ret = PTR_ERR(vreg->reg);
7922 dev_err(dev, "%s: %s get failed, err=%d\n",
7923 __func__, vreg->name, ret);
7924 }
7925out:
7926 return ret;
7927}
7928
7929static int ufshcd_init_vreg(struct ufs_hba *hba)
7930{
7931 int ret = 0;
7932 struct device *dev = hba->dev;
7933 struct ufs_vreg_info *info = &hba->vreg_info;
7934
aa497613
SRT		 7935 ret = ufshcd_get_vreg(dev, info->vcc);
7936 if (ret)
7937 goto out;
7938
7939 ret = ufshcd_get_vreg(dev, info->vccq);
b0008625
BH		 7940 if (!ret)
7941 ret = ufshcd_get_vreg(dev, info->vccq2);
aa497613
SRT		 7942out:
7943 return ret;
7944}
7945
6a771a65
RS		 7946static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7947{
7948 struct ufs_vreg_info *info = &hba->vreg_info;
7949
7950 if (info)
7951 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7952
7953 return 0;
7954}
7955
57d104c1
SJ		 7956static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7957 bool skip_ref_clk)
c6e79dac
SRT		 7958{
7959 int ret = 0;
7960 struct ufs_clk_info *clki;
7961 struct list_head *head = &hba->clk_list_head;
1ab27c9c 7962 unsigned long flags;
911a0771 7963 ktime_t start = ktime_get();
7964 bool clk_state_changed = false;
c6e79dac 7965
566ec9ad 7966 if (list_empty(head))
c6e79dac
SRT		 7967 goto out;
7968
38f3242e
CG		 7969 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7970 if (ret)
7971 return ret;
1e879e8f 7972
c6e79dac
SRT		 7973 list_for_each_entry(clki, head, list) {
7974 if (!IS_ERR_OR_NULL(clki->clk)) {
57d104c1
SJ		 7975 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7976 continue;
7977
911a0771 7978 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 7979 if (on && !clki->enabled) {
7980 ret = clk_prepare_enable(clki->clk);
7981 if (ret) {
7982 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7983 __func__, clki->name, ret);
7984 goto out;
7985 }
7986 } else if (!on && clki->enabled) {
7987 clk_disable_unprepare(clki->clk);
7988 }
7989 clki->enabled = on;
7990 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7991 clki->name, on ? "en" : "dis");
7992 }
7993 }
1ab27c9c 7994
38f3242e
CG		 7995 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7996 if (ret)
7997 return ret;
1e879e8f 7998
c6e79dac
SRT		 7999out:
8000 if (ret) {
8001 list_for_each_entry(clki, head, list) {
8002 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8003 clk_disable_unprepare(clki->clk);
8004 }
7ff5ab47 8005 } else if (!ret && on) {
1ab27c9c
ST		 8006 spin_lock_irqsave(hba->host->host_lock, flags);
8007 hba->clk_gating.state = CLKS_ON;
7ff5ab47 8008 trace_ufshcd_clk_gating(dev_name(hba->dev),
8009 hba->clk_gating.state);
1ab27c9c 8010 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 8011 }
7ff5ab47 8012
911a0771 8013 if (clk_state_changed)
8014 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8015 (on ? "on" : "off"),
8016 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 8017 return ret;
8018}
8019
57d104c1
SJ		 8020static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8021{
8022 return __ufshcd_setup_clocks(hba, on, false);
8023}
8024
c6e79dac
SRT		 8025static int ufshcd_init_clocks(struct ufs_hba *hba)
8026{
8027 int ret = 0;
8028 struct ufs_clk_info *clki;
8029 struct device *dev = hba->dev;
8030 struct list_head *head = &hba->clk_list_head;
8031
566ec9ad 8032 if (list_empty(head))
c6e79dac
SRT		 8033 goto out;
8034
8035 list_for_each_entry(clki, head, list) {
8036 if (!clki->name)
8037 continue;
8038
8039 clki->clk = devm_clk_get(dev, clki->name);
8040 if (IS_ERR(clki->clk)) {
8041 ret = PTR_ERR(clki->clk);
8042 dev_err(dev, "%s: %s clk get failed, %d\n",
8043 __func__, clki->name, ret);
8044 goto out;
8045 }
8046
9e1e8a75
SJ		 8047 /*
8048 * Parse device ref clk freq as per device tree "ref_clk".
8049 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8050 * in ufshcd_alloc_host().
8051 */
8052 if (!strcmp(clki->name, "ref_clk"))
8053 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8054
c6e79dac
SRT		 8055 if (clki->max_freq) {
8056 ret = clk_set_rate(clki->clk, clki->max_freq);
8057 if (ret) {
8058 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8059 __func__, clki->name,
8060 clki->max_freq, ret);
8061 goto out;
8062 }
856b3483 8063 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 8064 }
8065 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8066 clki->name, clk_get_rate(clki->clk));
8067 }
8068out:
8069 return ret;
8070}
8071
5c0c28a8
SRT		 8072static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8073{
8074 int err = 0;
8075
8076 if (!hba->vops)
8077 goto out;
8078
0263bcd0
YG		 8079 err = ufshcd_vops_init(hba);
8080 if (err)
8081 goto out;
5c0c28a8 8082
0263bcd0
YG		 8083 err = ufshcd_vops_setup_regulators(hba, true);
8084 if (err)
b0008625 8085 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 8086out:
8087 if (err)
8088 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 8089 __func__, ufshcd_get_var_name(hba), err);
5c0c28a8
SRT		 8090 return err;
8091}
8092
8093static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8094{
8095 if (!hba->vops)
8096 return;
8097
0263bcd0 8098 ufshcd_vops_setup_regulators(hba, false);
5c0c28a8 8099
0263bcd0 8100 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 8101}
8102
aa497613
SRT		 8103static int ufshcd_hba_init(struct ufs_hba *hba)
8104{
8105 int err;
8106
6a771a65
RS		 8107 /*
8108 * Handle host controller power separately from the UFS device power
8109 * rails as it will help controlling the UFS host controller power
8110 * collapse easily which is different than UFS device power collapse.
8111 * Also, enable the host controller power before we go ahead with rest
8112 * of the initialization here.
8113 */
8114 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 8115 if (err)
8116 goto out;
8117
6a771a65 8118 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 8119 if (err)
8120 goto out;
8121
6a771a65
RS		 8122 err = ufshcd_init_clocks(hba);
8123 if (err)
8124 goto out_disable_hba_vreg;
8125
8126 err = ufshcd_setup_clocks(hba, true);
8127 if (err)
8128 goto out_disable_hba_vreg;
8129
c6e79dac
SRT		 8130 err = ufshcd_init_vreg(hba);
8131 if (err)
8132 goto out_disable_clks;
8133
8134 err = ufshcd_setup_vreg(hba, true);
8135 if (err)
8136 goto out_disable_clks;
8137
aa497613
SRT		 8138 err = ufshcd_variant_hba_init(hba);
8139 if (err)
8140 goto out_disable_vreg;
8141
1d337ec2 8142 hba->is_powered = true;
aa497613
SRT		 8143 goto out;
8144
8145out_disable_vreg:
8146 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 8147out_disable_clks:
8148 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 8149out_disable_hba_vreg:
8150 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 8151out:
8152 return err;
8153}
8154
8155static void ufshcd_hba_exit(struct ufs_hba *hba)
8156{
1d337ec2
SRT		 8157 if (hba->is_powered) {
8158 ufshcd_variant_hba_exit(hba);
8159 ufshcd_setup_vreg(hba, false);
a508253d 8160 ufshcd_suspend_clkscaling(hba);
eebcc196 8161 if (ufshcd_is_clkscaling_supported(hba))
0701e49d 8162 if (hba->devfreq)
8163 ufshcd_suspend_clkscaling(hba);
1d337ec2
SRT		 8164 ufshcd_setup_clocks(hba, false);
8165 ufshcd_setup_hba_vreg(hba, false);
8166 hba->is_powered = false;
09750066 8167 ufs_put_device_desc(hba);
1d337ec2 8168 }
aa497613
SRT		 8169}
8170
57d104c1
SJ		 8171static int
8172ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
8173{
8174 unsigned char cmd[6] = {REQUEST_SENSE,
8175 0,
8176 0,
8177 0,
09a5a24f 8178 UFS_SENSE_SIZE,
57d104c1
SJ		 8179 0};
8180 char *buffer;
8181 int ret;
8182
09a5a24f 8183 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 8184 if (!buffer) {
8185 ret = -ENOMEM;
8186 goto out;
8187 }
8188
fcbfffe2 8189 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 8190 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 8191 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 8192 if (ret)
8193 pr_err("%s: failed with err %d\n", __func__, ret);
8194
8195 kfree(buffer);
8196out:
8197 return ret;
8198}
8199
8200/**
8201 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8202 * power mode
8203 * @hba: per adapter instance
8204 * @pwr_mode: device power mode to set
8205 *
8206 * Returns 0 if requested power mode is set successfully
8207 * Returns non-zero if failed to set the requested power mode
8208 */
8209static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8210 enum ufs_dev_pwr_mode pwr_mode)
8211{
8212 unsigned char cmd[6] = { START_STOP };
8213 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 8214 struct scsi_device *sdp;
8215 unsigned long flags;
57d104c1
SJ		 8216 int ret;
8217
7c48bfd0
AM		 8218 spin_lock_irqsave(hba->host->host_lock, flags);
8219 sdp = hba->sdev_ufs_device;
8220 if (sdp) {
8221 ret = scsi_device_get(sdp);
8222 if (!ret && !scsi_device_online(sdp)) {
8223 ret = -ENODEV;
8224 scsi_device_put(sdp);
8225 }
8226 } else {
8227 ret = -ENODEV;
8228 }
8229 spin_unlock_irqrestore(hba->host->host_lock, flags);
8230
8231 if (ret)
8232 return ret;
57d104c1
SJ		 8233
8234 /*
8235 * If scsi commands fail, the scsi mid-layer schedules scsi error-
8236 * handling, which would wait for host to be resumed. Since we know
8237 * we are functional while we are here, skip host resume in error
8238 * handling context.
8239 */
8240 hba->host->eh_noresume = 1;
8241 if (hba->wlun_dev_clr_ua) {
8242 ret = ufshcd_send_request_sense(hba, sdp);
8243 if (ret)
8244 goto out;
8245 /* Unit attention condition is cleared now */
8246 hba->wlun_dev_clr_ua = false;
8247 }
8248
8249 cmd[4] = pwr_mode << 4;
8250
8251 /*
8252 * Current function would be generally called from the power management
e8064021 8253 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 8254 * already suspended childs.
8255 */
fcbfffe2
CH		 8256 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8257 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 8258 if (ret) {
8259 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 8260 "START_STOP failed for power mode: %d, result %x\n",
8261 pwr_mode, ret);
c65be1a6 8262 if (driver_byte(ret) == DRIVER_SENSE)
21045519 8263 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 8264 }
8265
8266 if (!ret)
8267 hba->curr_dev_pwr_mode = pwr_mode;
8268out:
7c48bfd0 8269 scsi_device_put(sdp);
57d104c1
SJ		 8270 hba->host->eh_noresume = 0;
8271 return ret;
8272}
8273
8274static int ufshcd_link_state_transition(struct ufs_hba *hba,
8275 enum uic_link_state req_link_state,
8276 int check_for_bkops)
8277{
8278 int ret = 0;
8279
8280 if (req_link_state == hba->uic_link_state)
8281 return 0;
8282
8283 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8284 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236 8285 if (!ret) {
57d104c1 8286 ufshcd_set_link_hibern8(hba);
4db7a236
CG		 8287 } else {
8288 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8289 __func__, ret);
57d104c1 8290 goto out;
4db7a236 8291 }
57d104c1
SJ		 8292 }
8293 /*
8294 * If autobkops is enabled, link can't be turned off because
8295 * turning off the link would also turn off the device.
8296 */
8297 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
dc30c9e6 8298 (!check_for_bkops || !hba->auto_bkops_enabled)) {
f3099fbd
YG		 8299 /*
8300 * Let's make sure that link is in low power mode, we are doing
8301 * this currently by putting the link in Hibern8. Otherway to
8302 * put the link in low power mode is to send the DME end point
8303 * to device and then send the DME reset command to local
8304 * unipro. But putting the link in hibern8 is much faster.
8305 */
8306 ret = ufshcd_uic_hibern8_enter(hba);
4db7a236
CG		 8307 if (ret) {
8308 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8309 __func__, ret);
f3099fbd 8310 goto out;
4db7a236 8311 }
57d104c1
SJ		 8312 /*
8313 * Change controller state to "reset state" which
8314 * should also put the link in off/reset state
8315 */
5cac1095 8316 ufshcd_hba_stop(hba);
57d104c1
SJ		 8317 /*
8318 * TODO: Check if we need any delay to make sure that
8319 * controller is reset
8320 */
8321 ufshcd_set_link_off(hba);
8322 }
8323
8324out:
8325 return ret;
8326}
8327
8328static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8329{
c4df6eed
SC		 8330 bool vcc_off = false;
8331
b799fdf7
YG		 8332 /*
8333 * It seems some UFS devices may keep drawing more than sleep current
8334 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8335 * To avoid this situation, add 2ms delay before putting these UFS
8336 * rails in LPM mode.
8337 */
8338 if (!ufshcd_is_link_active(hba) &&
8339 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8340 usleep_range(2000, 2100);
8341
57d104c1
SJ		 8342 /*
8343 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8344 * power.
8345 *
8346 * If UFS device and link is in OFF state, all power supplies (VCC,
8347 * VCCQ, VCCQ2) can be turned off if power on write protect is not
8348 * required. If UFS link is inactive (Hibern8 or OFF state) and device
8349 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8350 *
8351 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8352 * in low power state which would save some power.
3d17b9b5
AD		 8353 *
8354 * If Write Booster is enabled and the device needs to flush the WB
8355 * buffer OR if bkops status is urgent for WB, keep Vcc on.
57d104c1
SJ		 8356 */
8357 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8358 !hba->dev_info.is_lu_power_on_wp) {
8359 ufshcd_setup_vreg(hba, false);
c4df6eed 8360 vcc_off = true;
57d104c1 8361 } else if (!ufshcd_is_ufs_dev_active(hba)) {
51dd905b 8362 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
c4df6eed 8363 vcc_off = true;
57d104c1
SJ		 8364 if (!ufshcd_is_link_active(hba)) {
8365 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8366 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8367 }
8368 }
c4df6eed
SC		 8369
8370 /*
8371 * Some UFS devices require delay after VCC power rail is turned-off.
8372 */
8373 if (vcc_off && hba->vreg_info.vcc &&
8374 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8375 usleep_range(5000, 5100);
57d104c1
SJ		 8376}
8377
8378static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8379{
8380 int ret = 0;
8381
8382 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8383 !hba->dev_info.is_lu_power_on_wp) {
8384 ret = ufshcd_setup_vreg(hba, true);
8385 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 8386 if (!ret && !ufshcd_is_link_active(hba)) {
8387 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8388 if (ret)
8389 goto vcc_disable;
8390 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8391 if (ret)
8392 goto vccq_lpm;
8393 }
69d72ac8 8394 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 8395 }
8396 goto out;
8397
8398vccq_lpm:
8399 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8400vcc_disable:
8401 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8402out:
8403 return ret;
8404}
8405
8406static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8407{
8408 if (ufshcd_is_link_off(hba))
8409 ufshcd_setup_hba_vreg(hba, false);
8410}
8411
8412static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8413{
8414 if (ufshcd_is_link_off(hba))
8415 ufshcd_setup_hba_vreg(hba, true);
8416}
8417
7a3e97b0 8418/**
57d104c1 8419 * ufshcd_suspend - helper function for suspend operations
3b1d0580 8420 * @hba: per adapter instance
57d104c1
SJ		 8421 * @pm_op: desired low power operation type
8422 *
8423 * This function will try to put the UFS device and link into low power
8424 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
8425 * (System PM level).
8426 *
8427 * If this function is called during shutdown, it will make sure that
8428 * both UFS device and UFS link is powered off.
7a3e97b0 8429 *
57d104c1
SJ		 8430 * NOTE: UFS device & link must be active before we enter in this function.
8431 *
8432 * Returns 0 for success and non-zero for failure
7a3e97b0 8433 */
57d104c1 8434static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8435{
57d104c1
SJ		 8436 int ret = 0;
8437 enum ufs_pm_level pm_lvl;
8438 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8439 enum uic_link_state req_link_state;
8440
8441 hba->pm_op_in_progress = 1;
8442 if (!ufshcd_is_shutdown_pm(pm_op)) {
8443 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
8444 hba->rpm_lvl : hba->spm_lvl;
8445 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8446 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8447 } else {
8448 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8449 req_link_state = UIC_LINK_OFF_STATE;
8450 }
8451
7a3e97b0 8452 /*
57d104c1
SJ		 8453 * If we can't transition into any of the low power modes
8454 * just gate the clocks.
7a3e97b0 8455 */
1ab27c9c
ST		 8456 ufshcd_hold(hba, false);
8457 hba->clk_gating.is_suspended = true;
8458
401f1e44 8459 if (hba->clk_scaling.is_allowed) {
8460 cancel_work_sync(&hba->clk_scaling.suspend_work);
8461 cancel_work_sync(&hba->clk_scaling.resume_work);
8462 ufshcd_suspend_clkscaling(hba);
8463 }
d6fcf81a 8464
57d104c1
SJ		 8465 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8466 req_link_state == UIC_LINK_ACTIVE_STATE) {
8467 goto disable_clks;
8468 }
7a3e97b0 8469
57d104c1
SJ		 8470 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8471 (req_link_state == hba->uic_link_state))
d6fcf81a 8472 goto enable_gating;
57d104c1
SJ		 8473
8474 /* UFS device & link must be active before we enter in this function */
8475 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8476 ret = -EINVAL;
d6fcf81a 8477 goto enable_gating;
57d104c1
SJ		 8478 }
8479
8480 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 8481 if (ufshcd_can_autobkops_during_suspend(hba)) {
8482 /*
8483 * The device is idle with no requests in the queue,
8484 * allow background operations if bkops status shows
8485 * that performance might be impacted.
8486 */
8487 ret = ufshcd_urgent_bkops(hba);
8488 if (ret)
8489 goto enable_gating;
8490 } else {
8491 /* make sure that auto bkops is disabled */
8492 ufshcd_disable_auto_bkops(hba);
8493 }
3d17b9b5 8494 /*
51dd905b
SC		 8495 * If device needs to do BKOP or WB buffer flush during
8496 * Hibern8, keep device power mode as "active power mode"
8497 * and VCC supply.
3d17b9b5 8498 */
51dd905b
SC		 8499 hba->dev_info.b_rpm_dev_flush_capable =
8500 hba->auto_bkops_enabled ||
8501 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8502 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8503 ufshcd_is_auto_hibern8_enabled(hba))) &&
8504 ufshcd_wb_need_flush(hba));
8505 }
8506
8507 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
8508 if ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
8509 !ufshcd_is_runtime_pm(pm_op)) {
8510 /* ensure that bkops is disabled */
8511 ufshcd_disable_auto_bkops(hba);
8512 }
57d104c1 8513
51dd905b
SC		 8514 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8515 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8516 if (ret)
8517 goto enable_gating;
8518 }
57d104c1
SJ		 8519 }
8520
2824ec9f 8521 flush_work(&hba->eeh_work);
57d104c1
SJ		 8522 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
8523 if (ret)
8524 goto set_dev_active;
8525
8526 ufshcd_vreg_set_lpm(hba);
8527
8528disable_clks:
8529 /*
8530 * Call vendor specific suspend callback. As these callbacks may access
8531 * vendor specific host controller register space call them before the
8532 * host clocks are ON.
8533 */
0263bcd0
YG		 8534 ret = ufshcd_vops_suspend(hba, pm_op);
8535 if (ret)
8536 goto set_link_active;
dcb6cec5
SC		 8537 /*
8538 * Disable the host irq as host controller as there won't be any
8539 * host controller transaction expected till resume.
8540 */
8541 ufshcd_disable_irq(hba);
57d104c1 8542
57d104c1
SJ		 8543 if (!ufshcd_is_link_active(hba))
8544 ufshcd_setup_clocks(hba, false);
8545 else
8546 /* If link is active, device ref_clk can't be switched off */
8547 __ufshcd_setup_clocks(hba, false, true);
8548
2dec9475
CG		 8549 if (ufshcd_is_clkgating_allowed(hba)) {
8550 hba->clk_gating.state = CLKS_OFF;
8551 trace_ufshcd_clk_gating(dev_name(hba->dev),
8552 hba->clk_gating.state);
8553 }
dcb6cec5 8554
57d104c1
SJ		 8555 /* Put the host controller in low power mode if possible */
8556 ufshcd_hba_vreg_set_lpm(hba);
8557 goto out;
8558
57d104c1 8559set_link_active:
401f1e44 8560 if (hba->clk_scaling.is_allowed)
8561 ufshcd_resume_clkscaling(hba);
57d104c1
SJ		 8562 ufshcd_vreg_set_hpm(hba);
8563 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
8564 ufshcd_set_link_active(hba);
8565 else if (ufshcd_is_link_off(hba))
8566 ufshcd_host_reset_and_restore(hba);
8567set_dev_active:
8568 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
8569 ufshcd_disable_auto_bkops(hba);
1ab27c9c 8570enable_gating:
401f1e44 8571 if (hba->clk_scaling.is_allowed)
8572 ufshcd_resume_clkscaling(hba);
1ab27c9c 8573 hba->clk_gating.is_suspended = false;
51dd905b 8574 hba->dev_info.b_rpm_dev_flush_capable = false;
1ab27c9c 8575 ufshcd_release(hba);
57d104c1 8576out:
51dd905b
SC		 8577 if (hba->dev_info.b_rpm_dev_flush_capable) {
8578 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
8579 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
8580 }
8581
57d104c1 8582 hba->pm_op_in_progress = 0;
51dd905b 8583
8808b4e9
SC		 8584 if (ret)
8585 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
57d104c1 8586 return ret;
7a3e97b0
SY		 8587}
8588
8589/**
57d104c1 8590 * ufshcd_resume - helper function for resume operations
3b1d0580 8591 * @hba: per adapter instance
57d104c1 8592 * @pm_op: runtime PM or system PM
7a3e97b0 8593 *
57d104c1
SJ		 8594 * This function basically brings the UFS device, UniPro link and controller
8595 * to active state.
8596 *
8597 * Returns 0 for success and non-zero for failure
7a3e97b0 8598 */
57d104c1 8599static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8600{
57d104c1
SJ		 8601 int ret;
8602 enum uic_link_state old_link_state;
8603
8604 hba->pm_op_in_progress = 1;
8605 old_link_state = hba->uic_link_state;
8606
8607 ufshcd_hba_vreg_set_hpm(hba);
8608 /* Make sure clocks are enabled before accessing controller */
8609 ret = ufshcd_setup_clocks(hba, true);
8610 if (ret)
8611 goto out;
8612
57d104c1 8613 /* enable the host irq as host controller would be active soon */
5231d38c 8614 ufshcd_enable_irq(hba);
57d104c1
SJ		 8615
8616 ret = ufshcd_vreg_set_hpm(hba);
8617 if (ret)
8618 goto disable_irq_and_vops_clks;
8619
7a3e97b0 8620 /*
57d104c1
SJ		 8621 * Call vendor specific resume callback. As these callbacks may access
8622 * vendor specific host controller register space call them when the
8623 * host clocks are ON.
7a3e97b0 8624 */
0263bcd0
YG		 8625 ret = ufshcd_vops_resume(hba, pm_op);
8626 if (ret)
8627 goto disable_vreg;
57d104c1
SJ		 8628
8629 if (ufshcd_is_link_hibern8(hba)) {
8630 ret = ufshcd_uic_hibern8_exit(hba);
4db7a236 8631 if (!ret) {
57d104c1 8632 ufshcd_set_link_active(hba);
4db7a236
CG		 8633 } else {
8634 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
8635 __func__, ret);
57d104c1 8636 goto vendor_suspend;
4db7a236 8637 }
57d104c1 8638 } else if (ufshcd_is_link_off(hba)) {
57d104c1 8639 /*
089f5b64
AD		 8640 * A full initialization of the host and the device is
8641 * required since the link was put to off during suspend.
8642 */
8643 ret = ufshcd_reset_and_restore(hba);
8644 /*
8645 * ufshcd_reset_and_restore() should have already
57d104c1
SJ		 8646 * set the link state as active
8647 */
8648 if (ret || !ufshcd_is_link_active(hba))
8649 goto vendor_suspend;
8650 }
8651
8652 if (!ufshcd_is_ufs_dev_active(hba)) {
8653 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8654 if (ret)
8655 goto set_old_link_state;
8656 }
8657
4e768e76 8658 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8659 ufshcd_enable_auto_bkops(hba);
8660 else
8661 /*
8662 * If BKOPs operations are urgently needed at this moment then
8663 * keep auto-bkops enabled or else disable it.
8664 */
8665 ufshcd_urgent_bkops(hba);
8666
1ab27c9c
ST		 8667 hba->clk_gating.is_suspended = false;
8668
fcb0c4b0
ST		 8669 if (hba->clk_scaling.is_allowed)
8670 ufshcd_resume_clkscaling(hba);
856b3483 8671
ad448378
AH		 8672 /* Enable Auto-Hibernate if configured */
8673 ufshcd_auto_hibern8_enable(hba);
8674
51dd905b
SC		 8675 if (hba->dev_info.b_rpm_dev_flush_capable) {
8676 hba->dev_info.b_rpm_dev_flush_capable = false;
8677 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
8678 }
8679
71d848b8
CG		 8680 /* Schedule clock gating in case of no access to UFS device yet */
8681 ufshcd_release(hba);
8682
57d104c1
SJ		 8683 goto out;
8684
8685set_old_link_state:
8686 ufshcd_link_state_transition(hba, old_link_state, 0);
8687vendor_suspend:
0263bcd0 8688 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 8689disable_vreg:
8690 ufshcd_vreg_set_lpm(hba);
8691disable_irq_and_vops_clks:
8692 ufshcd_disable_irq(hba);
401f1e44 8693 if (hba->clk_scaling.is_allowed)
8694 ufshcd_suspend_clkscaling(hba);
57d104c1 8695 ufshcd_setup_clocks(hba, false);
2dec9475
CG		 8696 if (ufshcd_is_clkgating_allowed(hba)) {
8697 hba->clk_gating.state = CLKS_OFF;
8698 trace_ufshcd_clk_gating(dev_name(hba->dev),
8699 hba->clk_gating.state);
8700 }
57d104c1
SJ		 8701out:
8702 hba->pm_op_in_progress = 0;
8808b4e9
SC		 8703 if (ret)
8704 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
57d104c1
SJ		 8705 return ret;
8706}
8707
8708/**
8709 * ufshcd_system_suspend - system suspend routine
8710 * @hba: per adapter instance
57d104c1
SJ		 8711 *
8712 * Check the description of ufshcd_suspend() function for more details.
8713 *
8714 * Returns 0 for success and non-zero for failure
8715 */
8716int ufshcd_system_suspend(struct ufs_hba *hba)
8717{
8718 int ret = 0;
7ff5ab47 8719 ktime_t start = ktime_get();
57d104c1
SJ		 8720
8721 if (!hba || !hba->is_powered)
233b594b 8722 return 0;
57d104c1 8723
0b257734 8724 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8725 hba->curr_dev_pwr_mode) &&
8726 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8727 hba->uic_link_state))
8728 goto out;
57d104c1 8729
0b257734 8730 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 8731 /*
8732 * UFS device and/or UFS link low power states during runtime
8733 * suspend seems to be different than what is expected during
8734 * system suspend. Hence runtime resume the devic & link and
8735 * let the system suspend low power states to take effect.
8736 * TODO: If resume takes longer time, we might have optimize
8737 * it in future by not resuming everything if possible.
8738 */
8739 ret = ufshcd_runtime_resume(hba);
8740 if (ret)
8741 goto out;
8742 }
8743
8744 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8745out:
7ff5ab47 8746 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8747 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8748 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 8749 if (!ret)
8750 hba->is_sys_suspended = true;
57d104c1
SJ		 8751 return ret;
8752}
8753EXPORT_SYMBOL(ufshcd_system_suspend);
8754
8755/**
8756 * ufshcd_system_resume - system resume routine
8757 * @hba: per adapter instance
8758 *
8759 * Returns 0 for success and non-zero for failure
8760 */
7a3e97b0 8761
57d104c1
SJ		 8762int ufshcd_system_resume(struct ufs_hba *hba)
8763{
7ff5ab47 8764 int ret = 0;
8765 ktime_t start = ktime_get();
8766
e3ce73d6
YG		 8767 if (!hba)
8768 return -EINVAL;
8769
8770 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 8771 /*
8772 * Let the runtime resume take care of resuming
8773 * if runtime suspended.
8774 */
7ff5ab47 8775 goto out;
8776 else
8777 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8778out:
8779 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8780 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8781 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 8782 if (!ret)
8783 hba->is_sys_suspended = false;
7ff5ab47 8784 return ret;
7a3e97b0 8785}
57d104c1 8786EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 8787
57d104c1
SJ		 8788/**
8789 * ufshcd_runtime_suspend - runtime suspend routine
8790 * @hba: per adapter instance
8791 *
8792 * Check the description of ufshcd_suspend() function for more details.
8793 *
8794 * Returns 0 for success and non-zero for failure
8795 */
66ec6d59
SRT		 8796int ufshcd_runtime_suspend(struct ufs_hba *hba)
8797{
7ff5ab47 8798 int ret = 0;
8799 ktime_t start = ktime_get();
8800
e3ce73d6
YG		 8801 if (!hba)
8802 return -EINVAL;
8803
8804 if (!hba->is_powered)
7ff5ab47 8805 goto out;
8806 else
8807 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8808out:
8809 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8810 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8811 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8812 return ret;
66ec6d59
SRT		 8813}
8814EXPORT_SYMBOL(ufshcd_runtime_suspend);
8815
57d104c1
SJ		 8816/**
8817 * ufshcd_runtime_resume - runtime resume routine
8818 * @hba: per adapter instance
8819 *
8820 * This function basically brings the UFS device, UniPro link and controller
8821 * to active state. Following operations are done in this function:
8822 *
8823 * 1. Turn on all the controller related clocks
8824 * 2. Bring the UniPro link out of Hibernate state
8825 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8826 * to active state.
8827 * 4. If auto-bkops is enabled on the device, disable it.
8828 *
8829 * So following would be the possible power state after this function return
8830 * successfully:
8831 * S1: UFS device in Active state with VCC rail ON
8832 * UniPro link in Active state
8833 * All the UFS/UniPro controller clocks are ON
8834 *
8835 * Returns 0 for success and non-zero for failure
8836 */
66ec6d59
SRT		 8837int ufshcd_runtime_resume(struct ufs_hba *hba)
8838{
7ff5ab47 8839 int ret = 0;
8840 ktime_t start = ktime_get();
8841
e3ce73d6
YG		 8842 if (!hba)
8843 return -EINVAL;
8844
8845 if (!hba->is_powered)
7ff5ab47 8846 goto out;
8847 else
8848 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8849out:
8850 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8851 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8852 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8853 return ret;
66ec6d59
SRT		 8854}
8855EXPORT_SYMBOL(ufshcd_runtime_resume);
8856
8857int ufshcd_runtime_idle(struct ufs_hba *hba)
8858{
8859 return 0;
8860}
8861EXPORT_SYMBOL(ufshcd_runtime_idle);
8862
57d104c1
SJ		 8863/**
8864 * ufshcd_shutdown - shutdown routine
8865 * @hba: per adapter instance
8866 *
8867 * This function would power off both UFS device and UFS link.
8868 *
8869 * Returns 0 always to allow force shutdown even in case of errors.
8870 */
8871int ufshcd_shutdown(struct ufs_hba *hba)
8872{
8873 int ret = 0;
8874
f51913ee
SC		 8875 if (!hba->is_powered)
8876 goto out;
8877
57d104c1
SJ		 8878 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8879 goto out;
8880
8881 if (pm_runtime_suspended(hba->dev)) {
8882 ret = ufshcd_runtime_resume(hba);
8883 if (ret)
8884 goto out;
8885 }
8886
8887 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8888out:
8889 if (ret)
8890 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8891 /* allow force shutdown even in case of errors */
8892 return 0;
8893}
8894EXPORT_SYMBOL(ufshcd_shutdown);
8895
7a3e97b0 8896/**
3b1d0580 8897 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 8898 * data structure memory
8aa29f19 8899 * @hba: per adapter instance
7a3e97b0 8900 */
3b1d0580 8901void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 8902{
df032bf2 8903 ufs_bsg_remove(hba);
cbb6813e 8904 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 8905 blk_cleanup_queue(hba->tmf_queue);
8906 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360 8907 blk_cleanup_queue(hba->cmd_queue);
cfdf9c91 8908 scsi_remove_host(hba->host);
7a3e97b0 8909 /* disable interrupts */
2fbd009b 8910 ufshcd_disable_intr(hba, hba->intr_mask);
5cac1095 8911 ufshcd_hba_stop(hba);
7a3e97b0 8912
eebcc196 8913 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8914 ufshcd_exit_clk_gating(hba);
fcb0c4b0
ST		 8915 if (ufshcd_is_clkscaling_supported(hba))
8916 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
aa497613 8917 ufshcd_hba_exit(hba);
3b1d0580
VH		 8918}
8919EXPORT_SYMBOL_GPL(ufshcd_remove);
8920
47555a5c
YG		 8921/**
8922 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8923 * @hba: pointer to Host Bus Adapter (HBA)
8924 */
8925void ufshcd_dealloc_host(struct ufs_hba *hba)
8926{
df043c74 8927 ufshcd_crypto_destroy_keyslot_manager(hba);
47555a5c
YG		 8928 scsi_host_put(hba->host);
8929}
8930EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8931
ca3d7bf9
AM		 8932/**
8933 * ufshcd_set_dma_mask - Set dma mask based on the controller
8934 * addressing capability
8935 * @hba: per adapter instance
8936 *
8937 * Returns 0 for success, non-zero for failure
8938 */
8939static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8940{
8941 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8942 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8943 return 0;
8944 }
8945 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8946}
8947
7a3e97b0 8948/**
5c0c28a8 8949 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 8950 * @dev: pointer to device handle
8951 * @hba_handle: driver private handle
7a3e97b0
SY		 8952 * Returns 0 on success, non-zero value on failure
8953 */
5c0c28a8 8954int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 8955{
8956 struct Scsi_Host *host;
8957 struct ufs_hba *hba;
5c0c28a8 8958 int err = 0;
7a3e97b0 8959
3b1d0580
VH		 8960 if (!dev) {
8961 dev_err(dev,
8962 "Invalid memory reference for dev is NULL\n");
8963 err = -ENODEV;
7a3e97b0
SY		 8964 goto out_error;
8965 }
8966
7a3e97b0
SY		 8967 host = scsi_host_alloc(&ufshcd_driver_template,
8968 sizeof(struct ufs_hba));
8969 if (!host) {
3b1d0580 8970 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 8971 err = -ENOMEM;
3b1d0580 8972 goto out_error;
7a3e97b0
SY		 8973 }
8974 hba = shost_priv(host);
7a3e97b0 8975 hba->host = host;
3b1d0580 8976 hba->dev = dev;
5c0c28a8 8977 *hba_handle = hba;
9e1e8a75 8978 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 8979
566ec9ad
SM		 8980 INIT_LIST_HEAD(&hba->clk_list_head);
8981
5c0c28a8
SRT		 8982out_error:
8983 return err;
8984}
8985EXPORT_SYMBOL(ufshcd_alloc_host);
8986
69a6c269
BVA		 8987/* This function exists because blk_mq_alloc_tag_set() requires this. */
8988static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
8989 const struct blk_mq_queue_data *qd)
8990{
8991 WARN_ON_ONCE(true);
8992 return BLK_STS_NOTSUPP;
8993}
8994
8995static const struct blk_mq_ops ufshcd_tmf_ops = {
8996 .queue_rq = ufshcd_queue_tmf,
8997};
8998
5c0c28a8
SRT		 8999/**
9000 * ufshcd_init - Driver initialization routine
9001 * @hba: per-adapter instance
9002 * @mmio_base: base register address
9003 * @irq: Interrupt line of device
9004 * Returns 0 on success, non-zero value on failure
9005 */
9006int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9007{
9008 int err;
9009 struct Scsi_Host *host = hba->host;
9010 struct device *dev = hba->dev;
4db7a236 9011 char eh_wq_name[sizeof("ufs_eh_wq_00")];
5c0c28a8
SRT		 9012
9013 if (!mmio_base) {
9014 dev_err(hba->dev,
9015 "Invalid memory reference for mmio_base is NULL\n");
9016 err = -ENODEV;
9017 goto out_error;
9018 }
9019
3b1d0580
VH		 9020 hba->mmio_base = mmio_base;
9021 hba->irq = irq;
90b8491c 9022 hba->vps = &ufs_hba_vps;
7a3e97b0 9023
aa497613 9024 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 9025 if (err)
9026 goto out_error;
9027
7a3e97b0 9028 /* Read capabilities registers */
df043c74
ST		 9029 err = ufshcd_hba_capabilities(hba);
9030 if (err)
9031 goto out_disable;
7a3e97b0
SY		 9032
9033 /* Get UFS version supported by the controller */
9034 hba->ufs_version = ufshcd_get_ufs_version(hba);
9035
c01848c6
YG		 9036 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
9037 (hba->ufs_version != UFSHCI_VERSION_11) &&
9038 (hba->ufs_version != UFSHCI_VERSION_20) &&
9039 (hba->ufs_version != UFSHCI_VERSION_21))
9040 dev_err(hba->dev, "invalid UFS version 0x%x\n",
9041 hba->ufs_version);
9042
2fbd009b
SJ		 9043 /* Get Interrupt bit mask per version */
9044 hba->intr_mask = ufshcd_get_intr_mask(hba);
9045
ca3d7bf9
AM		 9046 err = ufshcd_set_dma_mask(hba);
9047 if (err) {
9048 dev_err(hba->dev, "set dma mask failed\n");
9049 goto out_disable;
9050 }
9051
7a3e97b0
SY		 9052 /* Allocate memory for host memory space */
9053 err = ufshcd_memory_alloc(hba);
9054 if (err) {
3b1d0580
VH		 9055 dev_err(hba->dev, "Memory allocation failed\n");
9056 goto out_disable;
7a3e97b0
SY		 9057 }
9058
9059 /* Configure LRB */
9060 ufshcd_host_memory_configure(hba);
9061
9062 host->can_queue = hba->nutrs;
9063 host->cmd_per_lun = hba->nutrs;
9064 host->max_id = UFSHCD_MAX_ID;
0ce147d4 9065 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 9066 host->max_channel = UFSHCD_MAX_CHANNEL;
9067 host->unique_id = host->host_no;
a851b2bd 9068 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 9069
7eb584db
DR		 9070 hba->max_pwr_info.is_valid = false;
9071
7a3e97b0 9072 /* Initialize work queues */
4db7a236
CG		 9073 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9074 hba->host->host_no);
9075 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9076 if (!hba->eh_wq) {
9077 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9078 __func__);
9079 err = -ENOMEM;
9080 goto out_disable;
9081 }
e8e7f271 9082 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 9083 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 9084
6ccf44fe
SJ		 9085 /* Initialize UIC command mutex */
9086 mutex_init(&hba->uic_cmd_mutex);
9087
5a0b0cb9
SRT		 9088 /* Initialize mutex for device management commands */
9089 mutex_init(&hba->dev_cmd.lock);
9090
a3cd5ec5 9091 init_rwsem(&hba->clk_scaling_lock);
9092
1ab27c9c 9093 ufshcd_init_clk_gating(hba);
199ef13c 9094
eebcc196
VG		 9095 ufshcd_init_clk_scaling(hba);
9096
199ef13c
YG		 9097 /*
9098 * In order to avoid any spurious interrupt immediately after
9099 * registering UFS controller interrupt handler, clear any pending UFS
9100 * interrupt status and disable all the UFS interrupts.
9101 */
9102 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9103 REG_INTERRUPT_STATUS);
9104 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9105 /*
9106 * Make sure that UFS interrupts are disabled and any pending interrupt
9107 * status is cleared before registering UFS interrupt handler.
9108 */
9109 mb();
9110
7a3e97b0 9111 /* IRQ registration */
2953f850 9112 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 9113 if (err) {
3b1d0580 9114 dev_err(hba->dev, "request irq failed\n");
1ab27c9c 9115 goto exit_gating;
57d104c1
SJ		 9116 } else {
9117 hba->is_irq_enabled = true;
7a3e97b0
SY		 9118 }
9119
3b1d0580 9120 err = scsi_add_host(host, hba->dev);
7a3e97b0 9121 if (err) {
3b1d0580 9122 dev_err(hba->dev, "scsi_add_host failed\n");
1ab27c9c 9123 goto exit_gating;
7a3e97b0
SY		 9124 }
9125
7252a360
BVA		 9126 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
9127 if (IS_ERR(hba->cmd_queue)) {
9128 err = PTR_ERR(hba->cmd_queue);
9129 goto out_remove_scsi_host;
9130 }
9131
69a6c269
BVA		 9132 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9133 .nr_hw_queues = 1,
9134 .queue_depth = hba->nutmrs,
9135 .ops = &ufshcd_tmf_ops,
9136 .flags = BLK_MQ_F_NO_SCHED,
9137 };
9138 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9139 if (err < 0)
9140 goto free_cmd_queue;
9141 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9142 if (IS_ERR(hba->tmf_queue)) {
9143 err = PTR_ERR(hba->tmf_queue);
9144 goto free_tmf_tag_set;
9145 }
9146
d8d9f793
BA		 9147 /* Reset the attached device */
9148 ufshcd_vops_device_reset(hba);
9149
df043c74
ST		 9150 ufshcd_init_crypto(hba);
9151
6ccf44fe
SJ		 9152 /* Host controller enable */
9153 err = ufshcd_hba_enable(hba);
7a3e97b0 9154 if (err) {
6ccf44fe 9155 dev_err(hba->dev, "Host controller enable failed\n");
66cc820f 9156 ufshcd_print_host_regs(hba);
6ba65588 9157 ufshcd_print_host_state(hba);
69a6c269 9158 goto free_tmf_queue;
7a3e97b0 9159 }
6ccf44fe 9160
0c8f7586 9161 /*
9162 * Set the default power management level for runtime and system PM.
9163 * Default power saving mode is to keep UFS link in Hibern8 state
9164 * and UFS device in sleep state.
9165 */
9166 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9167 UFS_SLEEP_PWR_MODE,
9168 UIC_LINK_HIBERN8_STATE);
9169 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9170 UFS_SLEEP_PWR_MODE,
9171 UIC_LINK_HIBERN8_STATE);
9172
51dd905b
SC		 9173 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9174 ufshcd_rpm_dev_flush_recheck_work);
9175
ad448378 9176 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 9177 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 9178 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9179 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9180 }
9181
62694735
SRT		 9182 /* Hold auto suspend until async scan completes */
9183 pm_runtime_get_sync(dev);
38135535 9184 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 9185 /*
7caf489b 9186 * We are assuming that device wasn't put in sleep/power-down
9187 * state exclusively during the boot stage before kernel.
9188 * This assumption helps avoid doing link startup twice during
9189 * ufshcd_probe_hba().
57d104c1 9190 */
7caf489b 9191 ufshcd_set_ufs_dev_active(hba);
57d104c1 9192
6ccf44fe 9193 async_schedule(ufshcd_async_scan, hba);
cbb6813e 9194 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 9195
7a3e97b0
SY		 9196 return 0;
9197
69a6c269
BVA		 9198free_tmf_queue:
9199 blk_cleanup_queue(hba->tmf_queue);
9200free_tmf_tag_set:
9201 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360
BVA		 9202free_cmd_queue:
9203 blk_cleanup_queue(hba->cmd_queue);
3b1d0580
VH		 9204out_remove_scsi_host:
9205 scsi_remove_host(hba->host);
1ab27c9c 9206exit_gating:
eebcc196 9207 ufshcd_exit_clk_scaling(hba);
1ab27c9c 9208 ufshcd_exit_clk_gating(hba);
3b1d0580 9209out_disable:
57d104c1 9210 hba->is_irq_enabled = false;
aa497613 9211 ufshcd_hba_exit(hba);
3b1d0580
VH		 9212out_error:
9213 return err;
9214}
9215EXPORT_SYMBOL_GPL(ufshcd_init);
9216
3b1d0580
VH		 9217MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9218MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 9219MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 9220MODULE_LICENSE("GPL");
9221MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Linux 6.x block layer and io_uring tree(s)