projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: core: Adjust DBD setting in MODE SENSE for caching mode page per LLD
[linux-2.6-block.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
7a3e97b0 1/*
e0eca63e 2 * Universal Flash Storage Host controller driver Core
7a3e97b0
SY		 3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
3b1d0580 5 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 7 *
3b1d0580
VH		 8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
3b1d0580
VH		 16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
7a3e97b0
SY		 18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
3b1d0580
VH		 24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
5c0c28a8
SRT		 35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
7a3e97b0
SY		 38 */
39
6ccf44fe 40#include <linux/async.h>
856b3483 41#include <linux/devfreq.h>
b573d484 42#include <linux/nls.h>
54b879b7 43#include <linux/of.h>
ad448378 44#include <linux/bitfield.h>
e0eca63e 45#include "ufshcd.h"
c58ab7aa 46#include "ufs_quirks.h"
53b3d9c3 47#include "unipro.h"
cbb6813e 48#include "ufs-sysfs.h"
df032bf2 49#include "ufs_bsg.h"
7a3e97b0 50
7ff5ab47 51#define CREATE_TRACE_POINTS
52#include <trace/events/ufs.h>
53
2fbd009b
SJ		 54#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
55 UTP_TASK_REQ_COMPL |\
56 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 57/* UIC command timeout, unit: ms */
58#define UIC_CMD_TIMEOUT 500
2fbd009b 59
5a0b0cb9
SRT		 60/* NOP OUT retries waiting for NOP IN response */
61#define NOP_OUT_RETRIES 10
62/* Timeout after 30 msecs if NOP OUT hangs without response */
63#define NOP_OUT_TIMEOUT 30 /* msecs */
64
68078d5c 65/* Query request retries */
10fe5888 66#define QUERY_REQ_RETRIES 3
68078d5c 67/* Query request timeout */
10fe5888 68#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 69
e2933132
SRT		 70/* Task management command timeout */
71#define TM_CMD_TIMEOUT 100 /* msecs */
72
64238fbd
YG		 73/* maximum number of retries for a general UIC command */
74#define UFS_UIC_COMMAND_RETRIES 3
75
1d337ec2
SRT		 76/* maximum number of link-startup retries */
77#define DME_LINKSTARTUP_RETRIES 3
78
87d0b4a6
YG		 79/* Maximum retries for Hibern8 enter */
80#define UIC_HIBERN8_ENTER_RETRIES 3
81
1d337ec2
SRT		 82/* maximum number of reset retries before giving up */
83#define MAX_HOST_RESET_RETRIES 5
84
68078d5c
DR		 85/* Expose the flag value from utp_upiu_query.value */
86#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87
7d568652
SJ		 88/* Interrupt aggregation default timeout, unit: 40us */
89#define INT_AGGR_DEF_TO 0x02
90
49615ba1
SC		 91/* default delay of autosuspend: 2000 ms */
92#define RPM_AUTOSUSPEND_DELAY_MS 2000
93
aa497613
SRT		 94#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
95 ({ \
96 int _ret; \
97 if (_on) \
98 _ret = ufshcd_enable_vreg(_dev, _vreg); \
99 else \
100 _ret = ufshcd_disable_vreg(_dev, _vreg); \
101 _ret; \
102 })
103
ba80917d
TW		 104#define ufshcd_hex_dump(prefix_str, buf, len) do { \
105 size_t __len = (len); \
106 print_hex_dump(KERN_ERR, prefix_str, \
107 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
108 16, 4, buf, __len, false); \
109} while (0)
110
111int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
112 const char *prefix)
113{
d6724756
MG		 114 u32 *regs;
115 size_t pos;
116
117 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
118 return -EINVAL;
ba80917d 119
cddaebaf 120 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 121 if (!regs)
122 return -ENOMEM;
123
d6724756
MG		 124 for (pos = 0; pos < len; pos += 4)
125 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
126
ba80917d
TW		 127 ufshcd_hex_dump(prefix, regs, len);
128 kfree(regs);
129
130 return 0;
131}
132EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 133
7a3e97b0
SY		 134enum {
135 UFSHCD_MAX_CHANNEL = 0,
136 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 137 UFSHCD_CMD_PER_LUN = 32,
138 UFSHCD_CAN_QUEUE = 32,
139};
140
141/* UFSHCD states */
142enum {
7a3e97b0
SY		 143 UFSHCD_STATE_RESET,
144 UFSHCD_STATE_ERROR,
3441da7d 145 UFSHCD_STATE_OPERATIONAL,
141f8165 146 UFSHCD_STATE_EH_SCHEDULED,
3441da7d
SRT		 147};
148
149/* UFSHCD error handling flags */
150enum {
151 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 152};
153
e8e7f271
SRT		 154/* UFSHCD UIC layer error flags */
155enum {
156 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 157 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
158 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
159 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
160 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
161 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
e8e7f271
SRT		 162};
163
3441da7d 164#define ufshcd_set_eh_in_progress(h) \
9c490d2d 165 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 166#define ufshcd_eh_in_progress(h) \
9c490d2d 167 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 168#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 169 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 170
57d104c1
SJ		 171#define ufshcd_set_ufs_dev_active(h) \
172 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
173#define ufshcd_set_ufs_dev_sleep(h) \
174 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
175#define ufshcd_set_ufs_dev_poweroff(h) \
176 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
177#define ufshcd_is_ufs_dev_active(h) \
178 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
179#define ufshcd_is_ufs_dev_sleep(h) \
180 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
181#define ufshcd_is_ufs_dev_poweroff(h) \
182 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
183
cbb6813e 184struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 185 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
186 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
187 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
188 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
189 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
190 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
191};
192
193static inline enum ufs_dev_pwr_mode
194ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
195{
196 return ufs_pm_lvl_states[lvl].dev_state;
197}
198
199static inline enum uic_link_state
200ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
201{
202 return ufs_pm_lvl_states[lvl].link_state;
203}
204
0c8f7586 205static inline enum ufs_pm_level
206ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
207 enum uic_link_state link_state)
208{
209 enum ufs_pm_level lvl;
210
211 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
212 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
213 (ufs_pm_lvl_states[lvl].link_state == link_state))
214 return lvl;
215 }
216
217 /* if no match found, return the level 0 */
218 return UFS_PM_LVL_0;
219}
220
56d4a186
SJ		 221static struct ufs_dev_fix ufs_fixups[] = {
222 /* UFS cards deviations table */
223 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
224 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186
SJ		 225 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
226 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
56d4a186
SJ		 227 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
228 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
229 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
230 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
231 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
232 UFS_DEVICE_QUIRK_PA_TACTIVATE),
233 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
234 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 235 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
236 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
8e4829c6
WL		 237 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
238 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 239
240 END_FIX
241};
242
9333d775 243static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 244static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 245static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 246static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 247static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2
SRT		 248static void ufshcd_hba_exit(struct ufs_hba *hba);
249static int ufshcd_probe_hba(struct ufs_hba *hba);
1ab27c9c
ST		 250static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
251 bool skip_ref_clk);
252static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
253static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
254static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 255static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 256static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 257static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
258static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 259static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 260static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 261static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 262static int ufshcd_change_power_mode(struct ufs_hba *hba,
263 struct ufs_pa_layer_attr *pwr_mode);
14497328
YG		 264static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
265{
266 return tag >= 0 && tag < hba->nutrs;
267}
57d104c1
SJ		 268
269static inline int ufshcd_enable_irq(struct ufs_hba *hba)
270{
271 int ret = 0;
272
273 if (!hba->is_irq_enabled) {
274 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
275 hba);
276 if (ret)
277 dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
278 __func__, ret);
279 hba->is_irq_enabled = true;
280 }
281
282 return ret;
283}
284
285static inline void ufshcd_disable_irq(struct ufs_hba *hba)
286{
287 if (hba->is_irq_enabled) {
288 free_irq(hba->irq, hba);
289 hba->is_irq_enabled = false;
290 }
291}
3441da7d 292
38135535
SJ		 293static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
294{
295 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
296 scsi_unblock_requests(hba->host);
297}
298
299static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
300{
301 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
302 scsi_block_requests(hba->host);
303}
304
6667e6d9
OS		 305static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
306 const char *str)
307{
308 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
309
310 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
311}
312
313static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
314 const char *str)
315{
316 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
317
318 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
319}
320
321static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
322 const char *str)
323{
6667e6d9 324 int off = (int)tag - hba->nutrs;
391e388f 325 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 326
391e388f
CH		 327 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
328 &descp->input_param1);
6667e6d9
OS		 329}
330
1a07f2d9
LS		 331static void ufshcd_add_command_trace(struct ufs_hba *hba,
332 unsigned int tag, const char *str)
333{
334 sector_t lba = -1;
335 u8 opcode = 0;
336 u32 intr, doorbell;
e7c3b379 337 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
1a07f2d9
LS		 338 int transfer_len = -1;
339
e7c3b379
OS		 340 if (!trace_ufshcd_command_enabled()) {
341 /* trace UPIU W/O tracing command */
342 if (lrbp->cmd)
343 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9 344 return;
e7c3b379 345 }
1a07f2d9
LS		 346
347 if (lrbp->cmd) { /* data phase exists */
e7c3b379
OS		 348 /* trace UPIU also */
349 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9
LS		 350 opcode = (u8)(*lrbp->cmd->cmnd);
351 if ((opcode == READ_10) || (opcode == WRITE_10)) {
352 /*
353 * Currently we only fully trace read(10) and write(10)
354 * commands
355 */
356 if (lrbp->cmd->request && lrbp->cmd->request->bio)
357 lba =
358 lrbp->cmd->request->bio->bi_iter.bi_sector;
359 transfer_len = be32_to_cpu(
360 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
361 }
362 }
363
364 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
365 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
366 trace_ufshcd_command(dev_name(hba->dev), str, tag,
367 doorbell, transfer_len, intr, lba, opcode);
368}
369
ff8e20c6
DR		 370static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
371{
372 struct ufs_clk_info *clki;
373 struct list_head *head = &hba->clk_list_head;
374
566ec9ad 375 if (list_empty(head))
ff8e20c6
DR		 376 return;
377
378 list_for_each_entry(clki, head, list) {
379 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
380 clki->max_freq)
381 dev_err(hba->dev, "clk: %s, rate: %u\n",
382 clki->name, clki->curr_freq);
383 }
384}
385
48d5b973
SC		 386static void ufshcd_print_err_hist(struct ufs_hba *hba,
387 struct ufs_err_reg_hist *err_hist,
388 char *err_name)
ff8e20c6
DR		 389{
390 int i;
27752647 391 bool found = false;
ff8e20c6 392
48d5b973
SC		 393 for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
394 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
ff8e20c6
DR		 395
396 if (err_hist->reg[p] == 0)
397 continue;
c5397f13 398 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
ff8e20c6 399 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
27752647 400 found = true;
ff8e20c6 401 }
27752647
SC		 402
403 if (!found)
48d5b973 404 dev_err(hba->dev, "No record of %s errors\n", err_name);
ff8e20c6
DR		 405}
406
66cc820f
DR		 407static void ufshcd_print_host_regs(struct ufs_hba *hba)
408{
ba80917d 409 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
66cc820f
DR		 410 dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
411 hba->ufs_version, hba->capabilities);
412 dev_err(hba->dev,
413 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
414 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
ff8e20c6
DR		 415 dev_err(hba->dev,
416 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
417 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
418 hba->ufs_stats.hibern8_exit_cnt);
419
48d5b973
SC		 420 ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
421 ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
422 ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
423 ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
424 ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
d3c615bf
SC		 425 ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
426 "auto_hibern8_err");
8808b4e9
SC		 427 ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
428 ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
429 "link_startup_fail");
430 ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
431 ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
432 "suspend_fail");
433 ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
434 ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
435 ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
ff8e20c6
DR		 436
437 ufshcd_print_clk_freqs(hba);
438
439 if (hba->vops && hba->vops->dbg_register_dump)
440 hba->vops->dbg_register_dump(hba);
66cc820f
DR		 441}
442
443static
444void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
445{
446 struct ufshcd_lrb *lrbp;
7fabb77b 447 int prdt_length;
66cc820f
DR		 448 int tag;
449
450 for_each_set_bit(tag, &bitmap, hba->nutrs) {
451 lrbp = &hba->lrb[tag];
452
ff8e20c6
DR		 453 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
454 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 455 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
456 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 457 dev_err(hba->dev,
458 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
459 tag, (u64)lrbp->utrd_dma_addr);
460
66cc820f
DR		 461 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
462 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 463 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
464 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 465 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
466 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 467 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
468 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 469 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
470 sizeof(struct utp_upiu_rsp));
66cc820f 471
7fabb77b
GB		 472 prdt_length = le16_to_cpu(
473 lrbp->utr_descriptor_ptr->prd_table_length);
474 dev_err(hba->dev,
475 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
476 tag, prdt_length,
477 (u64)lrbp->ucd_prdt_dma_addr);
478
479 if (pr_prdt)
66cc820f 480 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 481 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 482 }
483}
484
485static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
486{
66cc820f
DR		 487 int tag;
488
489 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 490 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
491
66cc820f 492 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 493 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 494 }
495}
496
6ba65588
GB		 497static void ufshcd_print_host_state(struct ufs_hba *hba)
498{
499 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 500 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
501 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 502 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
503 hba->saved_err, hba->saved_uic_err);
504 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
505 hba->curr_dev_pwr_mode, hba->uic_link_state);
506 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
507 hba->pm_op_in_progress, hba->is_sys_suspended);
508 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
509 hba->auto_bkops_enabled, hba->host->host_self_blocked);
510 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
511 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
512 hba->eh_flags, hba->req_abort_count);
513 dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
514 hba->capabilities, hba->caps);
515 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
516 hba->dev_quirks);
517}
518
ff8e20c6
DR		 519/**
520 * ufshcd_print_pwr_info - print power params as saved in hba
521 * power info
522 * @hba: per-adapter instance
523 */
524static void ufshcd_print_pwr_info(struct ufs_hba *hba)
525{
526 static const char * const names[] = {
527 "INVALID MODE",
528 "FAST MODE",
529 "SLOW_MODE",
530 "INVALID MODE",
531 "FASTAUTO_MODE",
532 "SLOWAUTO_MODE",
533 "INVALID MODE",
534 };
535
536 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
537 __func__,
538 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
539 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
540 names[hba->pwr_info.pwr_rx],
541 names[hba->pwr_info.pwr_tx],
542 hba->pwr_info.hs_rate);
543}
544
5a0b0cb9
SRT		 545/*
546 * ufshcd_wait_for_register - wait for register value to change
547 * @hba - per-adapter interface
548 * @reg - mmio register offset
549 * @mask - mask to apply to read register value
550 * @val - wait condition
551 * @interval_us - polling interval in microsecs
552 * @timeout_ms - timeout in millisecs
596585a2 553 * @can_sleep - perform sleep or just spin
5a0b0cb9
SRT		 554 *
555 * Returns -ETIMEDOUT on error, zero on success
556 */
596585a2
YG		 557int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
558 u32 val, unsigned long interval_us,
559 unsigned long timeout_ms, bool can_sleep)
5a0b0cb9
SRT		 560{
561 int err = 0;
562 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
563
564 /* ignore bits that we don't intend to wait on */
565 val = val & mask;
566
567 while ((ufshcd_readl(hba, reg) & mask) != val) {
596585a2
YG		 568 if (can_sleep)
569 usleep_range(interval_us, interval_us + 50);
570 else
571 udelay(interval_us);
5a0b0cb9
SRT		 572 if (time_after(jiffies, timeout)) {
573 if ((ufshcd_readl(hba, reg) & mask) != val)
574 err = -ETIMEDOUT;
575 break;
576 }
577 }
578
579 return err;
580}
581
2fbd009b
SJ		 582/**
583 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 584 * @hba: Pointer to adapter instance
2fbd009b
SJ		 585 *
586 * Returns interrupt bit mask per version
587 */
588static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
589{
c01848c6
YG		 590 u32 intr_mask = 0;
591
592 switch (hba->ufs_version) {
593 case UFSHCI_VERSION_10:
594 intr_mask = INTERRUPT_MASK_ALL_VER_10;
595 break;
c01848c6
YG		 596 case UFSHCI_VERSION_11:
597 case UFSHCI_VERSION_20:
598 intr_mask = INTERRUPT_MASK_ALL_VER_11;
599 break;
c01848c6
YG		 600 case UFSHCI_VERSION_21:
601 default:
602 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 603 break;
c01848c6
YG		 604 }
605
606 return intr_mask;
2fbd009b
SJ		 607}
608
7a3e97b0
SY		 609/**
610 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 611 * @hba: Pointer to adapter instance
7a3e97b0
SY		 612 *
613 * Returns UFSHCI version supported by the controller
614 */
615static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
616{
0263bcd0
YG		 617 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
618 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 619
b873a275 620 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 621}
622
623/**
624 * ufshcd_is_device_present - Check if any device connected to
625 * the host controller
5c0c28a8 626 * @hba: pointer to adapter instance
7a3e97b0 627 *
c9e6010b 628 * Returns true if device present, false if no device detected
7a3e97b0 629 */
c9e6010b 630static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 631{
5c0c28a8 632 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 633 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 634}
635
636/**
637 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 638 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 639 *
640 * This function is used to get the OCS field from UTRD
641 * Returns the OCS field in the UTRD
642 */
643static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
644{
e8c8e82a 645 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 646}
647
7a3e97b0
SY		 648/**
649 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
650 * @hba: per adapter instance
651 * @pos: position of the bit to be cleared
652 */
653static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
654{
1399c5b0
AA		 655 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
656 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
657 else
658 ufshcd_writel(hba, ~(1 << pos),
659 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
660}
661
662/**
663 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
664 * @hba: per adapter instance
665 * @pos: position of the bit to be cleared
666 */
667static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
668{
669 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
670 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
671 else
672 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 673}
674
a48353f6
YG		 675/**
676 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
677 * @hba: per adapter instance
678 * @tag: position of the bit to be cleared
679 */
680static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
681{
682 __clear_bit(tag, &hba->outstanding_reqs);
683}
684
7a3e97b0
SY		 685/**
686 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
687 * @reg: Register value of host controller status
688 *
689 * Returns integer, 0 on Success and positive value if failed
690 */
691static inline int ufshcd_get_lists_status(u32 reg)
692{
6cf16115 693 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 694}
695
696/**
697 * ufshcd_get_uic_cmd_result - Get the UIC command result
698 * @hba: Pointer to adapter instance
699 *
700 * This function gets the result of UIC command completion
701 * Returns 0 on success, non zero value on error
702 */
703static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
704{
b873a275 705 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 706 MASK_UIC_COMMAND_RESULT;
707}
708
12b4fdb4
SJ		 709/**
710 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
711 * @hba: Pointer to adapter instance
712 *
713 * This function gets UIC command argument3
714 * Returns 0 on success, non zero value on error
715 */
716static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
717{
718 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
719}
720
7a3e97b0 721/**
5a0b0cb9 722 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 723 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 724 */
725static inline int
5a0b0cb9 726ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 727{
5a0b0cb9 728 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 729}
730
731/**
732 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
733 * @ucd_rsp_ptr: pointer to response UPIU
734 *
735 * This function gets the response status and scsi_status from response UPIU
736 * Returns the response result code.
737 */
738static inline int
739ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
740{
741 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
742}
743
1c2623c5
SJ		 744/*
745 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
746 * from response UPIU
747 * @ucd_rsp_ptr: pointer to response UPIU
748 *
749 * Return the data segment length.
750 */
751static inline unsigned int
752ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
753{
754 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
755 MASK_RSP_UPIU_DATA_SEG_LEN;
756}
757
66ec6d59
SRT		 758/**
759 * ufshcd_is_exception_event - Check if the device raised an exception event
760 * @ucd_rsp_ptr: pointer to response UPIU
761 *
762 * The function checks if the device raised an exception event indicated in
763 * the Device Information field of response UPIU.
764 *
765 * Returns true if exception is raised, false otherwise.
766 */
767static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
768{
769 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
770 MASK_RSP_EXCEPTION_EVENT ? true : false;
771}
772
7a3e97b0 773/**
7d568652 774 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 775 * @hba: per adapter instance
7a3e97b0
SY		 776 */
777static inline void
7d568652 778ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 779{
7d568652
SJ		 780 ufshcd_writel(hba, INT_AGGR_ENABLE |
781 INT_AGGR_COUNTER_AND_TIMER_RESET,
782 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
783}
784
785/**
786 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
787 * @hba: per adapter instance
788 * @cnt: Interrupt aggregation counter threshold
789 * @tmout: Interrupt aggregation timeout value
790 */
791static inline void
792ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
793{
794 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
795 INT_AGGR_COUNTER_THLD_VAL(cnt) |
796 INT_AGGR_TIMEOUT_VAL(tmout),
797 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 798}
799
b852190e
YG		 800/**
801 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
802 * @hba: per adapter instance
803 */
804static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
805{
806 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
807}
808
7a3e97b0
SY		 809/**
810 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
811 * When run-stop registers are set to 1, it indicates the
812 * host controller that it can process the requests
813 * @hba: per adapter instance
814 */
815static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
816{
b873a275
SJ		 817 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
818 REG_UTP_TASK_REQ_LIST_RUN_STOP);
819 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
820 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 821}
822
7a3e97b0
SY		 823/**
824 * ufshcd_hba_start - Start controller initialization sequence
825 * @hba: per adapter instance
826 */
827static inline void ufshcd_hba_start(struct ufs_hba *hba)
828{
b873a275 829 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 830}
831
832/**
833 * ufshcd_is_hba_active - Get controller state
834 * @hba: per adapter instance
835 *
c9e6010b 836 * Returns false if controller is active, true otherwise
7a3e97b0 837 */
c9e6010b 838static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 839{
4a8eec2b
TK		 840 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
841 ? false : true;
7a3e97b0
SY		 842}
843
37113106
YG		 844u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
845{
846 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
847 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
848 (hba->ufs_version == UFSHCI_VERSION_11))
849 return UFS_UNIPRO_VER_1_41;
850 else
851 return UFS_UNIPRO_VER_1_6;
852}
853EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
854
855static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
856{
857 /*
858 * If both host and device support UniPro ver1.6 or later, PA layer
859 * parameters tuning happens during link startup itself.
860 *
861 * We can manually tune PA layer parameters if either host or device
862 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
863 * logic simple, we will only do manual tuning if local unipro version
864 * doesn't support ver1.6 or later.
865 */
866 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
867 return true;
868 else
869 return false;
870}
871
a3cd5ec5 872static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
873{
874 int ret = 0;
875 struct ufs_clk_info *clki;
876 struct list_head *head = &hba->clk_list_head;
877 ktime_t start = ktime_get();
878 bool clk_state_changed = false;
879
566ec9ad 880 if (list_empty(head))
a3cd5ec5 881 goto out;
882
883 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
884 if (ret)
885 return ret;
886
887 list_for_each_entry(clki, head, list) {
888 if (!IS_ERR_OR_NULL(clki->clk)) {
889 if (scale_up && clki->max_freq) {
890 if (clki->curr_freq == clki->max_freq)
891 continue;
892
893 clk_state_changed = true;
894 ret = clk_set_rate(clki->clk, clki->max_freq);
895 if (ret) {
896 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
897 __func__, clki->name,
898 clki->max_freq, ret);
899 break;
900 }
901 trace_ufshcd_clk_scaling(dev_name(hba->dev),
902 "scaled up", clki->name,
903 clki->curr_freq,
904 clki->max_freq);
905
906 clki->curr_freq = clki->max_freq;
907
908 } else if (!scale_up && clki->min_freq) {
909 if (clki->curr_freq == clki->min_freq)
910 continue;
911
912 clk_state_changed = true;
913 ret = clk_set_rate(clki->clk, clki->min_freq);
914 if (ret) {
915 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
916 __func__, clki->name,
917 clki->min_freq, ret);
918 break;
919 }
920 trace_ufshcd_clk_scaling(dev_name(hba->dev),
921 "scaled down", clki->name,
922 clki->curr_freq,
923 clki->min_freq);
924 clki->curr_freq = clki->min_freq;
925 }
926 }
927 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
928 clki->name, clk_get_rate(clki->clk));
929 }
930
931 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
932
933out:
934 if (clk_state_changed)
935 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
936 (scale_up ? "up" : "down"),
937 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
938 return ret;
939}
940
941/**
942 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
943 * @hba: per adapter instance
944 * @scale_up: True if scaling up and false if scaling down
945 *
946 * Returns true if scaling is required, false otherwise.
947 */
948static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
949 bool scale_up)
950{
951 struct ufs_clk_info *clki;
952 struct list_head *head = &hba->clk_list_head;
953
566ec9ad 954 if (list_empty(head))
a3cd5ec5 955 return false;
956
957 list_for_each_entry(clki, head, list) {
958 if (!IS_ERR_OR_NULL(clki->clk)) {
959 if (scale_up && clki->max_freq) {
960 if (clki->curr_freq == clki->max_freq)
961 continue;
962 return true;
963 } else if (!scale_up && clki->min_freq) {
964 if (clki->curr_freq == clki->min_freq)
965 continue;
966 return true;
967 }
968 }
969 }
970
971 return false;
972}
973
974static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
975 u64 wait_timeout_us)
976{
977 unsigned long flags;
978 int ret = 0;
979 u32 tm_doorbell;
980 u32 tr_doorbell;
981 bool timeout = false, do_last_check = false;
982 ktime_t start;
983
984 ufshcd_hold(hba, false);
985 spin_lock_irqsave(hba->host->host_lock, flags);
986 /*
987 * Wait for all the outstanding tasks/transfer requests.
988 * Verify by checking the doorbell registers are clear.
989 */
990 start = ktime_get();
991 do {
992 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
993 ret = -EBUSY;
994 goto out;
995 }
996
997 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
998 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
999 if (!tm_doorbell && !tr_doorbell) {
1000 timeout = false;
1001 break;
1002 } else if (do_last_check) {
1003 break;
1004 }
1005
1006 spin_unlock_irqrestore(hba->host->host_lock, flags);
1007 schedule();
1008 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1009 wait_timeout_us) {
1010 timeout = true;
1011 /*
1012 * We might have scheduled out for long time so make
1013 * sure to check if doorbells are cleared by this time
1014 * or not.
1015 */
1016 do_last_check = true;
1017 }
1018 spin_lock_irqsave(hba->host->host_lock, flags);
1019 } while (tm_doorbell || tr_doorbell);
1020
1021 if (timeout) {
1022 dev_err(hba->dev,
1023 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1024 __func__, tm_doorbell, tr_doorbell);
1025 ret = -EBUSY;
1026 }
1027out:
1028 spin_unlock_irqrestore(hba->host->host_lock, flags);
1029 ufshcd_release(hba);
1030 return ret;
1031}
1032
1033/**
1034 * ufshcd_scale_gear - scale up/down UFS gear
1035 * @hba: per adapter instance
1036 * @scale_up: True for scaling up gear and false for scaling down
1037 *
1038 * Returns 0 for success,
1039 * Returns -EBUSY if scaling can't happen at this time
1040 * Returns non-zero for any other errors
1041 */
1042static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1043{
1044 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1045 int ret = 0;
1046 struct ufs_pa_layer_attr new_pwr_info;
1047
1048 if (scale_up) {
1049 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1050 sizeof(struct ufs_pa_layer_attr));
1051 } else {
1052 memcpy(&new_pwr_info, &hba->pwr_info,
1053 sizeof(struct ufs_pa_layer_attr));
1054
1055 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1056 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1057 /* save the current power mode */
1058 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1059 &hba->pwr_info,
1060 sizeof(struct ufs_pa_layer_attr));
1061
1062 /* scale down gear */
1063 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1064 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1065 }
1066 }
1067
1068 /* check if the power mode needs to be changed or not? */
1069 ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1070
1071 if (ret)
1072 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1073 __func__, ret,
1074 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1075 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1076
1077 return ret;
1078}
1079
1080static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1081{
1082 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1083 int ret = 0;
1084 /*
1085 * make sure that there are no outstanding requests when
1086 * clock scaling is in progress
1087 */
38135535 1088 ufshcd_scsi_block_requests(hba);
a3cd5ec5 1089 down_write(&hba->clk_scaling_lock);
1090 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1091 ret = -EBUSY;
1092 up_write(&hba->clk_scaling_lock);
38135535 1093 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1094 }
1095
1096 return ret;
1097}
1098
1099static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1100{
1101 up_write(&hba->clk_scaling_lock);
38135535 1102 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1103}
1104
1105/**
1106 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1107 * @hba: per adapter instance
1108 * @scale_up: True for scaling up and false for scalin down
1109 *
1110 * Returns 0 for success,
1111 * Returns -EBUSY if scaling can't happen at this time
1112 * Returns non-zero for any other errors
1113 */
1114static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1115{
1116 int ret = 0;
1117
401f1e44 1118 /* let's not get into low power until clock scaling is completed */
1119 ufshcd_hold(hba, false);
1120
a3cd5ec5 1121 ret = ufshcd_clock_scaling_prepare(hba);
1122 if (ret)
1123 return ret;
1124
1125 /* scale down the gear before scaling down clocks */
1126 if (!scale_up) {
1127 ret = ufshcd_scale_gear(hba, false);
1128 if (ret)
1129 goto out;
1130 }
1131
1132 ret = ufshcd_scale_clks(hba, scale_up);
1133 if (ret) {
1134 if (!scale_up)
1135 ufshcd_scale_gear(hba, true);
1136 goto out;
1137 }
1138
1139 /* scale up the gear after scaling up clocks */
1140 if (scale_up) {
1141 ret = ufshcd_scale_gear(hba, true);
1142 if (ret) {
1143 ufshcd_scale_clks(hba, false);
1144 goto out;
1145 }
1146 }
1147
1148 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1149
1150out:
1151 ufshcd_clock_scaling_unprepare(hba);
401f1e44 1152 ufshcd_release(hba);
a3cd5ec5 1153 return ret;
1154}
1155
401f1e44 1156static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1157{
1158 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1159 clk_scaling.suspend_work);
1160 unsigned long irq_flags;
1161
1162 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1163 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1164 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1165 return;
1166 }
1167 hba->clk_scaling.is_suspended = true;
1168 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1169
1170 __ufshcd_suspend_clkscaling(hba);
1171}
1172
1173static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1174{
1175 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1176 clk_scaling.resume_work);
1177 unsigned long irq_flags;
1178
1179 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1180 if (!hba->clk_scaling.is_suspended) {
1181 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1182 return;
1183 }
1184 hba->clk_scaling.is_suspended = false;
1185 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1186
1187 devfreq_resume_device(hba->devfreq);
1188}
1189
a3cd5ec5 1190static int ufshcd_devfreq_target(struct device *dev,
1191 unsigned long *freq, u32 flags)
1192{
1193 int ret = 0;
1194 struct ufs_hba *hba = dev_get_drvdata(dev);
1195 ktime_t start;
401f1e44 1196 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1197 struct list_head *clk_list = &hba->clk_list_head;
1198 struct ufs_clk_info *clki;
a3cd5ec5 1199 unsigned long irq_flags;
1200
1201 if (!ufshcd_is_clkscaling_supported(hba))
1202 return -EINVAL;
1203
a3cd5ec5 1204 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1205 if (ufshcd_eh_in_progress(hba)) {
1206 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1207 return 0;
1208 }
1209
401f1e44 1210 if (!hba->clk_scaling.active_reqs)
1211 sched_clk_scaling_suspend_work = true;
1212
092b4558
BA		 1213 if (list_empty(clk_list)) {
1214 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1215 goto out;
1216 }
1217
1218 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1219 scale_up = (*freq == clki->max_freq) ? true : false;
401f1e44 1220 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1221 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1222 ret = 0;
1223 goto out; /* no state change required */
a3cd5ec5 1224 }
1225 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1226
1227 start = ktime_get();
a3cd5ec5 1228 ret = ufshcd_devfreq_scale(hba, scale_up);
1229
a3cd5ec5 1230 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1231 (scale_up ? "up" : "down"),
1232 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1233
401f1e44 1234out:
1235 if (sched_clk_scaling_suspend_work)
1236 queue_work(hba->clk_scaling.workq,
1237 &hba->clk_scaling.suspend_work);
1238
a3cd5ec5 1239 return ret;
1240}
1241
7252a360
BVA		 1242static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1243{
1244 int *busy = priv;
1245
1246 WARN_ON_ONCE(reserved);
1247 (*busy)++;
1248 return false;
1249}
1250
1251/* Whether or not any tag is in use by a request that is in progress. */
1252static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1253{
1254 struct request_queue *q = hba->cmd_queue;
1255 int busy = 0;
1256
1257 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1258 return busy;
1259}
a3cd5ec5 1260
1261static int ufshcd_devfreq_get_dev_status(struct device *dev,
1262 struct devfreq_dev_status *stat)
1263{
1264 struct ufs_hba *hba = dev_get_drvdata(dev);
1265 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1266 unsigned long flags;
1267
1268 if (!ufshcd_is_clkscaling_supported(hba))
1269 return -EINVAL;
1270
1271 memset(stat, 0, sizeof(*stat));
1272
1273 spin_lock_irqsave(hba->host->host_lock, flags);
1274 if (!scaling->window_start_t)
1275 goto start_window;
1276
1277 if (scaling->is_busy_started)
1278 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1279 scaling->busy_start_t));
1280
1281 stat->total_time = jiffies_to_usecs((long)jiffies -
1282 (long)scaling->window_start_t);
1283 stat->busy_time = scaling->tot_busy_t;
1284start_window:
1285 scaling->window_start_t = jiffies;
1286 scaling->tot_busy_t = 0;
1287
1288 if (hba->outstanding_reqs) {
1289 scaling->busy_start_t = ktime_get();
1290 scaling->is_busy_started = true;
1291 } else {
1292 scaling->busy_start_t = 0;
1293 scaling->is_busy_started = false;
1294 }
1295 spin_unlock_irqrestore(hba->host->host_lock, flags);
1296 return 0;
1297}
1298
1299static struct devfreq_dev_profile ufs_devfreq_profile = {
1300 .polling_ms = 100,
1301 .target = ufshcd_devfreq_target,
1302 .get_dev_status = ufshcd_devfreq_get_dev_status,
1303};
1304
deac444f
BA		 1305static int ufshcd_devfreq_init(struct ufs_hba *hba)
1306{
092b4558
BA		 1307 struct list_head *clk_list = &hba->clk_list_head;
1308 struct ufs_clk_info *clki;
deac444f
BA		 1309 struct devfreq *devfreq;
1310 int ret;
1311
092b4558
BA		 1312 /* Skip devfreq if we don't have any clocks in the list */
1313 if (list_empty(clk_list))
1314 return 0;
1315
1316 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1317 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1318 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1319
1320 devfreq = devfreq_add_device(hba->dev,
deac444f
BA		 1321 &ufs_devfreq_profile,
1322 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1323 NULL);
1324 if (IS_ERR(devfreq)) {
1325 ret = PTR_ERR(devfreq);
1326 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1327
1328 dev_pm_opp_remove(hba->dev, clki->min_freq);
1329 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1330 return ret;
1331 }
1332
1333 hba->devfreq = devfreq;
1334
1335 return 0;
1336}
1337
092b4558
BA		 1338static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1339{
1340 struct list_head *clk_list = &hba->clk_list_head;
1341 struct ufs_clk_info *clki;
1342
1343 if (!hba->devfreq)
1344 return;
1345
1346 devfreq_remove_device(hba->devfreq);
1347 hba->devfreq = NULL;
1348
1349 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1350 dev_pm_opp_remove(hba->dev, clki->min_freq);
1351 dev_pm_opp_remove(hba->dev, clki->max_freq);
1352}
1353
401f1e44 1354static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1355{
1356 unsigned long flags;
1357
1358 devfreq_suspend_device(hba->devfreq);
1359 spin_lock_irqsave(hba->host->host_lock, flags);
1360 hba->clk_scaling.window_start_t = 0;
1361 spin_unlock_irqrestore(hba->host->host_lock, flags);
1362}
a3cd5ec5 1363
a508253d
GB		 1364static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1365{
401f1e44 1366 unsigned long flags;
1367 bool suspend = false;
1368
fcb0c4b0
ST		 1369 if (!ufshcd_is_clkscaling_supported(hba))
1370 return;
1371
401f1e44 1372 spin_lock_irqsave(hba->host->host_lock, flags);
1373 if (!hba->clk_scaling.is_suspended) {
1374 suspend = true;
1375 hba->clk_scaling.is_suspended = true;
1376 }
1377 spin_unlock_irqrestore(hba->host->host_lock, flags);
1378
1379 if (suspend)
1380 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1381}
1382
1383static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1384{
401f1e44 1385 unsigned long flags;
1386 bool resume = false;
1387
1388 if (!ufshcd_is_clkscaling_supported(hba))
1389 return;
1390
1391 spin_lock_irqsave(hba->host->host_lock, flags);
1392 if (hba->clk_scaling.is_suspended) {
1393 resume = true;
1394 hba->clk_scaling.is_suspended = false;
1395 }
1396 spin_unlock_irqrestore(hba->host->host_lock, flags);
1397
1398 if (resume)
1399 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1400}
1401
1402static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1403 struct device_attribute *attr, char *buf)
1404{
1405 struct ufs_hba *hba = dev_get_drvdata(dev);
1406
1407 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1408}
1409
1410static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1411 struct device_attribute *attr, const char *buf, size_t count)
1412{
1413 struct ufs_hba *hba = dev_get_drvdata(dev);
1414 u32 value;
1415 int err;
1416
1417 if (kstrtou32(buf, 0, &value))
1418 return -EINVAL;
1419
1420 value = !!value;
1421 if (value == hba->clk_scaling.is_allowed)
1422 goto out;
1423
1424 pm_runtime_get_sync(hba->dev);
1425 ufshcd_hold(hba, false);
1426
401f1e44 1427 cancel_work_sync(&hba->clk_scaling.suspend_work);
1428 cancel_work_sync(&hba->clk_scaling.resume_work);
1429
1430 hba->clk_scaling.is_allowed = value;
1431
fcb0c4b0
ST		 1432 if (value) {
1433 ufshcd_resume_clkscaling(hba);
1434 } else {
1435 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1436 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1437 if (err)
1438 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1439 __func__, err);
1440 }
fcb0c4b0
ST		 1441
1442 ufshcd_release(hba);
1443 pm_runtime_put_sync(hba->dev);
1444out:
1445 return count;
a508253d
GB		 1446}
1447
a3cd5ec5 1448static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1449{
1450 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1451 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1452 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1453 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1454 hba->clk_scaling.enable_attr.attr.mode = 0644;
1455 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1456 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1457}
1458
1ab27c9c
ST		 1459static void ufshcd_ungate_work(struct work_struct *work)
1460{
1461 int ret;
1462 unsigned long flags;
1463 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1464 clk_gating.ungate_work);
1465
1466 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1467
1468 spin_lock_irqsave(hba->host->host_lock, flags);
1469 if (hba->clk_gating.state == CLKS_ON) {
1470 spin_unlock_irqrestore(hba->host->host_lock, flags);
1471 goto unblock_reqs;
1472 }
1473
1474 spin_unlock_irqrestore(hba->host->host_lock, flags);
1475 ufshcd_setup_clocks(hba, true);
1476
1477 /* Exit from hibern8 */
1478 if (ufshcd_can_hibern8_during_gating(hba)) {
1479 /* Prevent gating in this path */
1480 hba->clk_gating.is_suspended = true;
1481 if (ufshcd_is_link_hibern8(hba)) {
1482 ret = ufshcd_uic_hibern8_exit(hba);
1483 if (ret)
1484 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1485 __func__, ret);
1486 else
1487 ufshcd_set_link_active(hba);
1488 }
1489 hba->clk_gating.is_suspended = false;
1490 }
1491unblock_reqs:
38135535 1492 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1493}
1494
1495/**
1496 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1497 * Also, exit from hibern8 mode and set the link as active.
1498 * @hba: per adapter instance
1499 * @async: This indicates whether caller should ungate clocks asynchronously.
1500 */
1501int ufshcd_hold(struct ufs_hba *hba, bool async)
1502{
1503 int rc = 0;
1504 unsigned long flags;
1505
1506 if (!ufshcd_is_clkgating_allowed(hba))
1507 goto out;
1ab27c9c
ST		 1508 spin_lock_irqsave(hba->host->host_lock, flags);
1509 hba->clk_gating.active_reqs++;
1510
53c12d0e
YG		 1511 if (ufshcd_eh_in_progress(hba)) {
1512 spin_unlock_irqrestore(hba->host->host_lock, flags);
1513 return 0;
1514 }
1515
856b3483 1516start:
1ab27c9c
ST		 1517 switch (hba->clk_gating.state) {
1518 case CLKS_ON:
f2a785ac
VG		 1519 /*
1520 * Wait for the ungate work to complete if in progress.
1521 * Though the clocks may be in ON state, the link could
1522 * still be in hibner8 state if hibern8 is allowed
1523 * during clock gating.
1524 * Make sure we exit hibern8 state also in addition to
1525 * clocks being ON.
1526 */
1527 if (ufshcd_can_hibern8_during_gating(hba) &&
1528 ufshcd_is_link_hibern8(hba)) {
1529 spin_unlock_irqrestore(hba->host->host_lock, flags);
1530 flush_work(&hba->clk_gating.ungate_work);
1531 spin_lock_irqsave(hba->host->host_lock, flags);
1532 goto start;
1533 }
1ab27c9c
ST		 1534 break;
1535 case REQ_CLKS_OFF:
1536 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1537 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1538 trace_ufshcd_clk_gating(dev_name(hba->dev),
1539 hba->clk_gating.state);
1ab27c9c
ST		 1540 break;
1541 }
1542 /*
9c490d2d 1543 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1544 * currently running. Hence, fall through to cancel gating
1545 * work and to enable clocks.
1546 */
30eb2e4c 1547 /* fallthrough */
1ab27c9c 1548 case CLKS_OFF:
38135535 1549 ufshcd_scsi_block_requests(hba);
1ab27c9c 1550 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47 1551 trace_ufshcd_clk_gating(dev_name(hba->dev),
1552 hba->clk_gating.state);
10e5e375
VV		 1553 queue_work(hba->clk_gating.clk_gating_workq,
1554 &hba->clk_gating.ungate_work);
1ab27c9c
ST		 1555 /*
1556 * fall through to check if we should wait for this
1557 * work to be done or not.
1558 */
30eb2e4c 1559 /* fallthrough */
1ab27c9c
ST		 1560 case REQ_CLKS_ON:
1561 if (async) {
1562 rc = -EAGAIN;
1563 hba->clk_gating.active_reqs--;
1564 break;
1565 }
1566
1567 spin_unlock_irqrestore(hba->host->host_lock, flags);
1568 flush_work(&hba->clk_gating.ungate_work);
1569 /* Make sure state is CLKS_ON before returning */
856b3483 1570 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1571 goto start;
1572 default:
1573 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1574 __func__, hba->clk_gating.state);
1575 break;
1576 }
1577 spin_unlock_irqrestore(hba->host->host_lock, flags);
1578out:
1579 return rc;
1580}
6e3fd44d 1581EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1582
1583static void ufshcd_gate_work(struct work_struct *work)
1584{
1585 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1586 clk_gating.gate_work.work);
1587 unsigned long flags;
1588
1589 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1590 /*
1591 * In case you are here to cancel this work the gating state
1592 * would be marked as REQ_CLKS_ON. In this case save time by
1593 * skipping the gating work and exit after changing the clock
1594 * state to CLKS_ON.
1595 */
1596 if (hba->clk_gating.is_suspended ||
18f01374 1597 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1598 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1599 trace_ufshcd_clk_gating(dev_name(hba->dev),
1600 hba->clk_gating.state);
1ab27c9c
ST		 1601 goto rel_lock;
1602 }
1603
1604 if (hba->clk_gating.active_reqs
1605 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1606 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1ab27c9c
ST		 1607 || hba->active_uic_cmd || hba->uic_async_done)
1608 goto rel_lock;
1609
1610 spin_unlock_irqrestore(hba->host->host_lock, flags);
1611
1612 /* put the link into hibern8 mode before turning off clocks */
1613 if (ufshcd_can_hibern8_during_gating(hba)) {
1614 if (ufshcd_uic_hibern8_enter(hba)) {
1615 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1616 trace_ufshcd_clk_gating(dev_name(hba->dev),
1617 hba->clk_gating.state);
1ab27c9c
ST		 1618 goto out;
1619 }
1620 ufshcd_set_link_hibern8(hba);
1621 }
1622
1623 if (!ufshcd_is_link_active(hba))
1624 ufshcd_setup_clocks(hba, false);
1625 else
1626 /* If link is active, device ref_clk can't be switched off */
1627 __ufshcd_setup_clocks(hba, false, true);
1628
1629 /*
1630 * In case you are here to cancel this work the gating state
1631 * would be marked as REQ_CLKS_ON. In this case keep the state
1632 * as REQ_CLKS_ON which would anyway imply that clocks are off
1633 * and a request to turn them on is pending. By doing this way,
1634 * we keep the state machine in tact and this would ultimately
1635 * prevent from doing cancel work multiple times when there are
1636 * new requests arriving before the current cancel work is done.
1637 */
1638 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1639 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1640 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 1641 trace_ufshcd_clk_gating(dev_name(hba->dev),
1642 hba->clk_gating.state);
1643 }
1ab27c9c
ST		 1644rel_lock:
1645 spin_unlock_irqrestore(hba->host->host_lock, flags);
1646out:
1647 return;
1648}
1649
1650/* host lock must be held before calling this variant */
1651static void __ufshcd_release(struct ufs_hba *hba)
1652{
1653 if (!ufshcd_is_clkgating_allowed(hba))
1654 return;
1655
1656 hba->clk_gating.active_reqs--;
1657
1658 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1659 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1660 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
53c12d0e
YG		 1661 || hba->active_uic_cmd || hba->uic_async_done
1662 || ufshcd_eh_in_progress(hba))
1ab27c9c
ST		 1663 return;
1664
1665 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1666 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1667 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1668 &hba->clk_gating.gate_work,
1669 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1670}
1671
1672void ufshcd_release(struct ufs_hba *hba)
1673{
1674 unsigned long flags;
1675
1676 spin_lock_irqsave(hba->host->host_lock, flags);
1677 __ufshcd_release(hba);
1678 spin_unlock_irqrestore(hba->host->host_lock, flags);
1679}
6e3fd44d 1680EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1681
1682static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1683 struct device_attribute *attr, char *buf)
1684{
1685 struct ufs_hba *hba = dev_get_drvdata(dev);
1686
1687 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1688}
1689
1690static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1691 struct device_attribute *attr, const char *buf, size_t count)
1692{
1693 struct ufs_hba *hba = dev_get_drvdata(dev);
1694 unsigned long flags, value;
1695
1696 if (kstrtoul(buf, 0, &value))
1697 return -EINVAL;
1698
1699 spin_lock_irqsave(hba->host->host_lock, flags);
1700 hba->clk_gating.delay_ms = value;
1701 spin_unlock_irqrestore(hba->host->host_lock, flags);
1702 return count;
1703}
1704
b427411a
ST		 1705static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1706 struct device_attribute *attr, char *buf)
1707{
1708 struct ufs_hba *hba = dev_get_drvdata(dev);
1709
1710 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1711}
1712
1713static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1714 struct device_attribute *attr, const char *buf, size_t count)
1715{
1716 struct ufs_hba *hba = dev_get_drvdata(dev);
1717 unsigned long flags;
1718 u32 value;
1719
1720 if (kstrtou32(buf, 0, &value))
1721 return -EINVAL;
1722
1723 value = !!value;
1724 if (value == hba->clk_gating.is_enabled)
1725 goto out;
1726
1727 if (value) {
1728 ufshcd_release(hba);
1729 } else {
1730 spin_lock_irqsave(hba->host->host_lock, flags);
1731 hba->clk_gating.active_reqs++;
1732 spin_unlock_irqrestore(hba->host->host_lock, flags);
1733 }
1734
1735 hba->clk_gating.is_enabled = value;
1736out:
1737 return count;
1738}
1739
eebcc196
VG		 1740static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1741{
1742 char wq_name[sizeof("ufs_clkscaling_00")];
1743
1744 if (!ufshcd_is_clkscaling_supported(hba))
1745 return;
1746
1747 INIT_WORK(&hba->clk_scaling.suspend_work,
1748 ufshcd_clk_scaling_suspend_work);
1749 INIT_WORK(&hba->clk_scaling.resume_work,
1750 ufshcd_clk_scaling_resume_work);
1751
1752 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1753 hba->host->host_no);
1754 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1755
1756 ufshcd_clkscaling_init_sysfs(hba);
1757}
1758
1759static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1760{
1761 if (!ufshcd_is_clkscaling_supported(hba))
1762 return;
1763
1764 destroy_workqueue(hba->clk_scaling.workq);
1765 ufshcd_devfreq_remove(hba);
1766}
1767
1ab27c9c
ST		 1768static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1769{
10e5e375
VV		 1770 char wq_name[sizeof("ufs_clk_gating_00")];
1771
1ab27c9c
ST		 1772 if (!ufshcd_is_clkgating_allowed(hba))
1773 return;
1774
1775 hba->clk_gating.delay_ms = 150;
1776 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1777 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1778
10e5e375
VV		 1779 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1780 hba->host->host_no);
1781 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1782 WQ_MEM_RECLAIM);
1783
b427411a
ST		 1784 hba->clk_gating.is_enabled = true;
1785
1ab27c9c
ST		 1786 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1787 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1788 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1789 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
b427411a 1790 hba->clk_gating.delay_attr.attr.mode = 0644;
1ab27c9c
ST		 1791 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1792 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
b427411a
ST		 1793
1794 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1795 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1796 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1797 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1798 hba->clk_gating.enable_attr.attr.mode = 0644;
1799 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1800 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1ab27c9c
ST		 1801}
1802
1803static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1804{
1805 if (!ufshcd_is_clkgating_allowed(hba))
1806 return;
1807 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
b427411a 1808 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
97cd6805
AM		 1809 cancel_work_sync(&hba->clk_gating.ungate_work);
1810 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1811 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1812}
1813
856b3483
ST		 1814/* Must be called with host lock acquired */
1815static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1816{
401f1e44 1817 bool queue_resume_work = false;
1818
fcb0c4b0 1819 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1820 return;
1821
401f1e44 1822 if (!hba->clk_scaling.active_reqs++)
1823 queue_resume_work = true;
1824
1825 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1826 return;
1827
1828 if (queue_resume_work)
1829 queue_work(hba->clk_scaling.workq,
1830 &hba->clk_scaling.resume_work);
1831
1832 if (!hba->clk_scaling.window_start_t) {
1833 hba->clk_scaling.window_start_t = jiffies;
1834 hba->clk_scaling.tot_busy_t = 0;
1835 hba->clk_scaling.is_busy_started = false;
1836 }
1837
856b3483
ST		 1838 if (!hba->clk_scaling.is_busy_started) {
1839 hba->clk_scaling.busy_start_t = ktime_get();
1840 hba->clk_scaling.is_busy_started = true;
1841 }
1842}
1843
1844static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1845{
1846 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1847
fcb0c4b0 1848 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1849 return;
1850
1851 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1852 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1853 scaling->busy_start_t));
8b0e1953 1854 scaling->busy_start_t = 0;
856b3483
ST		 1855 scaling->is_busy_started = false;
1856 }
1857}
7a3e97b0
SY		 1858/**
1859 * ufshcd_send_command - Send SCSI or device management commands
1860 * @hba: per adapter instance
1861 * @task_tag: Task tag of the command
1862 */
1863static inline
1864void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1865{
ff8e20c6 1866 hba->lrb[task_tag].issue_time_stamp = ktime_get();
09017188 1867 hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
856b3483 1868 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 1869 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 1870 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 1871 /* Make sure that doorbell is committed immediately */
1872 wmb();
1a07f2d9 1873 ufshcd_add_command_trace(hba, task_tag, "send");
7a3e97b0
SY		 1874}
1875
1876/**
1877 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 1878 * @lrbp: pointer to local reference block
7a3e97b0
SY		 1879 */
1880static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1881{
1882 int len;
1c2623c5
SJ		 1883 if (lrbp->sense_buffer &&
1884 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 1885 int len_to_copy;
1886
5a0b0cb9 1887 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 1888 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 1889
09a5a24f
AA		 1890 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1891 len_to_copy);
7a3e97b0
SY		 1892 }
1893}
1894
68078d5c
DR		 1895/**
1896 * ufshcd_copy_query_response() - Copy the Query Response and the data
1897 * descriptor
1898 * @hba: per adapter instance
8aa29f19 1899 * @lrbp: pointer to local reference block
68078d5c
DR		 1900 */
1901static
c6d4a831 1902int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 1903{
1904 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1905
68078d5c 1906 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 1907
68078d5c 1908 /* Get the descriptor */
1c90836f
AA		 1909 if (hba->dev_cmd.query.descriptor &&
1910 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 1911 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 1912 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 1913 u16 resp_len;
1914 u16 buf_len;
68078d5c
DR		 1915
1916 /* data segment length */
c6d4a831 1917 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 1918 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 1919 buf_len = be16_to_cpu(
1920 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 1921 if (likely(buf_len >= resp_len)) {
1922 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1923 } else {
1924 dev_warn(hba->dev,
3d4881d1
BH		 1925 "%s: rsp size %d is bigger than buffer size %d",
1926 __func__, resp_len, buf_len);
c6d4a831
DR		 1927 return -EINVAL;
1928 }
68078d5c 1929 }
c6d4a831
DR		 1930
1931 return 0;
68078d5c
DR		 1932}
1933
7a3e97b0
SY		 1934/**
1935 * ufshcd_hba_capabilities - Read controller capabilities
1936 * @hba: per adapter instance
1937 */
1938static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1939{
b873a275 1940 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 1941
1942 /* nutrs and nutmrs are 0 based values */
1943 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1944 hba->nutmrs =
1945 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1946}
1947
1948/**
6ccf44fe
SJ		 1949 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1950 * to accept UIC commands
7a3e97b0 1951 * @hba: per adapter instance
6ccf44fe
SJ		 1952 * Return true on success, else false
1953 */
1954static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1955{
1956 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1957 return true;
1958 else
1959 return false;
1960}
1961
53b3d9c3
SJ		 1962/**
1963 * ufshcd_get_upmcrs - Get the power mode change request status
1964 * @hba: Pointer to adapter instance
1965 *
1966 * This function gets the UPMCRS field of HCS register
1967 * Returns value of UPMCRS field
1968 */
1969static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1970{
1971 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1972}
1973
6ccf44fe
SJ		 1974/**
1975 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1976 * @hba: per adapter instance
1977 * @uic_cmd: UIC command
1978 *
1979 * Mutex must be held.
7a3e97b0
SY		 1980 */
1981static inline void
6ccf44fe 1982ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 1983{
6ccf44fe
SJ		 1984 WARN_ON(hba->active_uic_cmd);
1985
1986 hba->active_uic_cmd = uic_cmd;
1987
7a3e97b0 1988 /* Write Args */
6ccf44fe
SJ		 1989 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1990 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1991 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0
SY		 1992
1993 /* Write UIC Cmd */
6ccf44fe 1994 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 1995 REG_UIC_COMMAND);
7a3e97b0
SY		 1996}
1997
6ccf44fe
SJ		 1998/**
1999 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2000 * @hba: per adapter instance
8aa29f19 2001 * @uic_cmd: UIC command
6ccf44fe
SJ		 2002 *
2003 * Must be called with mutex held.
2004 * Returns 0 only if success.
2005 */
2006static int
2007ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2008{
2009 int ret;
2010 unsigned long flags;
2011
2012 if (wait_for_completion_timeout(&uic_cmd->done,
2013 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2014 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2015 else
2016 ret = -ETIMEDOUT;
2017
2018 spin_lock_irqsave(hba->host->host_lock, flags);
2019 hba->active_uic_cmd = NULL;
2020 spin_unlock_irqrestore(hba->host->host_lock, flags);
2021
2022 return ret;
2023}
2024
2025/**
2026 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2027 * @hba: per adapter instance
2028 * @uic_cmd: UIC command
d75f7fe4 2029 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2030 *
2031 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2032 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2033 * Returns 0 only if success.
2034 */
2035static int
d75f7fe4
YG		 2036__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2037 bool completion)
6ccf44fe 2038{
6ccf44fe
SJ		 2039 if (!ufshcd_ready_for_uic_cmd(hba)) {
2040 dev_err(hba->dev,
2041 "Controller not ready to accept UIC commands\n");
2042 return -EIO;
2043 }
2044
d75f7fe4
YG		 2045 if (completion)
2046 init_completion(&uic_cmd->done);
6ccf44fe 2047
6ccf44fe 2048 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2049
57d104c1 2050 return 0;
6ccf44fe
SJ		 2051}
2052
2053/**
2054 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2055 * @hba: per adapter instance
2056 * @uic_cmd: UIC command
2057 *
2058 * Returns 0 only if success.
2059 */
e77044c5 2060int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2061{
2062 int ret;
57d104c1 2063 unsigned long flags;
6ccf44fe 2064
1ab27c9c 2065 ufshcd_hold(hba, false);
6ccf44fe 2066 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2067 ufshcd_add_delay_before_dme_cmd(hba);
2068
57d104c1 2069 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2070 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2071 spin_unlock_irqrestore(hba->host->host_lock, flags);
2072 if (!ret)
2073 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2074
6ccf44fe
SJ		 2075 mutex_unlock(&hba->uic_cmd_mutex);
2076
1ab27c9c 2077 ufshcd_release(hba);
6ccf44fe
SJ		 2078 return ret;
2079}
2080
7a3e97b0
SY		 2081/**
2082 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2083 * @hba: per adapter instance
2084 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2085 *
2086 * Returns 0 in case of success, non-zero value in case of failure
2087 */
75b1cc4a 2088static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2089{
2090 struct ufshcd_sg_entry *prd_table;
2091 struct scatterlist *sg;
2092 struct scsi_cmnd *cmd;
2093 int sg_segments;
2094 int i;
2095
2096 cmd = lrbp->cmd;
2097 sg_segments = scsi_dma_map(cmd);
2098 if (sg_segments < 0)
2099 return sg_segments;
2100
2101 if (sg_segments) {
75b1cc4a
KK		 2102 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2103 lrbp->utr_descriptor_ptr->prd_table_length =
2104 cpu_to_le16((u16)(sg_segments *
2105 sizeof(struct ufshcd_sg_entry)));
2106 else
2107 lrbp->utr_descriptor_ptr->prd_table_length =
2108 cpu_to_le16((u16) (sg_segments));
7a3e97b0
SY		 2109
2110 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2111
2112 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2113 prd_table[i].size =
2114 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2115 prd_table[i].base_addr =
2116 cpu_to_le32(lower_32_bits(sg->dma_address));
2117 prd_table[i].upper_addr =
2118 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2119 prd_table[i].reserved = 0;
7a3e97b0
SY		 2120 }
2121 } else {
2122 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2123 }
2124
2125 return 0;
2126}
2127
2128/**
2fbd009b 2129 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2130 * @hba: per adapter instance
2fbd009b 2131 * @intrs: interrupt bits
7a3e97b0 2132 */
2fbd009b 2133static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2134{
2fbd009b
SJ		 2135 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2136
2137 if (hba->ufs_version == UFSHCI_VERSION_10) {
2138 u32 rw;
2139 rw = set & INTERRUPT_MASK_RW_VER_10;
2140 set = rw | ((set ^ intrs) & intrs);
2141 } else {
2142 set |= intrs;
2143 }
2144
2145 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2146}
2147
2148/**
2149 * ufshcd_disable_intr - disable interrupts
2150 * @hba: per adapter instance
2151 * @intrs: interrupt bits
2152 */
2153static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2154{
2155 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2156
2157 if (hba->ufs_version == UFSHCI_VERSION_10) {
2158 u32 rw;
2159 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2160 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2161 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2162
2163 } else {
2164 set &= ~intrs;
7a3e97b0 2165 }
2fbd009b
SJ		 2166
2167 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2168}
2169
5a0b0cb9
SRT		 2170/**
2171 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2172 * descriptor according to request
2173 * @lrbp: pointer to local reference block
2174 * @upiu_flags: flags required in the header
2175 * @cmd_dir: requests data direction
2176 */
2177static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
300bb13f 2178 u32 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2179{
2180 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2181 u32 data_direction;
2182 u32 dword_0;
2183
2184 if (cmd_dir == DMA_FROM_DEVICE) {
2185 data_direction = UTP_DEVICE_TO_HOST;
2186 *upiu_flags = UPIU_CMD_FLAGS_READ;
2187 } else if (cmd_dir == DMA_TO_DEVICE) {
2188 data_direction = UTP_HOST_TO_DEVICE;
2189 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2190 } else {
2191 data_direction = UTP_NO_DATA_TRANSFER;
2192 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2193 }
2194
2195 dword_0 = data_direction | (lrbp->command_type
2196 << UPIU_COMMAND_TYPE_OFFSET);
2197 if (lrbp->intr_cmd)
2198 dword_0 |= UTP_REQ_DESC_INT_CMD;
2199
2200 /* Transfer request descriptor header fields */
2201 req_desc->header.dword_0 = cpu_to_le32(dword_0);
52ac95fe
YG		 2202 /* dword_1 is reserved, hence it is set to 0 */
2203 req_desc->header.dword_1 = 0;
5a0b0cb9
SRT		 2204 /*
2205 * assigning invalid value for command status. Controller
2206 * updates OCS on command completion, with the command
2207 * status
2208 */
2209 req_desc->header.dword_2 =
2210 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
52ac95fe
YG		 2211 /* dword_3 is reserved, hence it is set to 0 */
2212 req_desc->header.dword_3 = 0;
51047266
YG		 2213
2214 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2215}
2216
2217/**
2218 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2219 * for scsi commands
8aa29f19
BVA		 2220 * @lrbp: local reference block pointer
2221 * @upiu_flags: flags
5a0b0cb9
SRT		 2222 */
2223static
2224void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2225{
2226 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2227 unsigned short cdb_len;
5a0b0cb9
SRT		 2228
2229 /* command descriptor fields */
2230 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2231 UPIU_TRANSACTION_COMMAND, upiu_flags,
2232 lrbp->lun, lrbp->task_tag);
2233 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2234 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2235
2236 /* Total EHS length and Data segment length will be zero */
2237 ucd_req_ptr->header.dword_2 = 0;
2238
2239 ucd_req_ptr->sc.exp_data_transfer_len =
2240 cpu_to_be32(lrbp->cmd->sdb.length);
2241
a851b2bd
AA		 2242 cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, UFS_CDB_SIZE);
2243 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
52ac95fe
YG		 2244 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2245
2246 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2247}
2248
68078d5c
DR		 2249/**
2250 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2251 * for query requsts
2252 * @hba: UFS hba
2253 * @lrbp: local reference block pointer
2254 * @upiu_flags: flags
2255 */
2256static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2257 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2258{
2259 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2260 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2261 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2262
2263 /* Query request header */
2264 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2265 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2266 lrbp->lun, lrbp->task_tag);
2267 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2268 0, query->request.query_func, 0, 0);
2269
6861285c
ZL		 2270 /* Data segment length only need for WRITE_DESC */
2271 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2272 ucd_req_ptr->header.dword_2 =
2273 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2274 else
2275 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2276
2277 /* Copy the Query Request buffer as is */
2278 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2279 QUERY_OSF_SIZE);
68078d5c
DR		 2280
2281 /* Copy the Descriptor */
c6d4a831 2282 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2283 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2284
51047266 2285 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2286}
2287
5a0b0cb9
SRT		 2288static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2289{
2290 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2291
2292 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2293
2294 /* command descriptor fields */
2295 ucd_req_ptr->header.dword_0 =
2296 UPIU_HEADER_DWORD(
2297 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2298 /* clear rest of the fields of basic header */
2299 ucd_req_ptr->header.dword_1 = 0;
2300 ucd_req_ptr->header.dword_2 = 0;
2301
2302 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2303}
2304
7a3e97b0 2305/**
300bb13f
JP		 2306 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2307 * for Device Management Purposes
8aa29f19
BVA		 2308 * @hba: per adapter instance
2309 * @lrbp: pointer to local reference block
7a3e97b0 2310 */
300bb13f 2311static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0 2312{
7a3e97b0 2313 u32 upiu_flags;
5a0b0cb9 2314 int ret = 0;
7a3e97b0 2315
83dc7e3d 2316 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2317 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2318 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2319 else
2320 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2321
2322 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2323 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2324 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2325 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2326 ufshcd_prepare_utp_nop_upiu(lrbp);
2327 else
2328 ret = -EINVAL;
2329
2330 return ret;
2331}
2332
2333/**
2334 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2335 * for SCSI Purposes
8aa29f19
BVA		 2336 * @hba: per adapter instance
2337 * @lrbp: pointer to local reference block
300bb13f
JP		 2338 */
2339static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2340{
2341 u32 upiu_flags;
2342 int ret = 0;
2343
83dc7e3d 2344 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2345 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2346 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2347 else
2348 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2349
2350 if (likely(lrbp->cmd)) {
2351 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2352 lrbp->cmd->sc_data_direction);
2353 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2354 } else {
2355 ret = -EINVAL;
2356 }
5a0b0cb9
SRT		 2357
2358 return ret;
7a3e97b0
SY		 2359}
2360
2a8fa600
SJ		 2361/**
2362 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2363 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2364 *
2365 * Returns SCSI W-LUN id
2366 */
2367static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2368{
2369 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2370}
2371
7a3e97b0
SY		 2372/**
2373 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2374 * @host: SCSI host pointer
7a3e97b0 2375 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2376 *
2377 * Returns 0 for success, non-zero in case of failure
2378 */
2379static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2380{
2381 struct ufshcd_lrb *lrbp;
2382 struct ufs_hba *hba;
2383 unsigned long flags;
2384 int tag;
2385 int err = 0;
2386
2387 hba = shost_priv(host);
2388
2389 tag = cmd->request->tag;
14497328
YG		 2390 if (!ufshcd_valid_tag(hba, tag)) {
2391 dev_err(hba->dev,
2392 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2393 __func__, tag, cmd, cmd->request);
2394 BUG();
2395 }
7a3e97b0 2396
a3cd5ec5 2397 if (!down_read_trylock(&hba->clk_scaling_lock))
2398 return SCSI_MLQUEUE_HOST_BUSY;
2399
3441da7d
SRT		 2400 spin_lock_irqsave(hba->host->host_lock, flags);
2401 switch (hba->ufshcd_state) {
2402 case UFSHCD_STATE_OPERATIONAL:
2403 break;
141f8165 2404 case UFSHCD_STATE_EH_SCHEDULED:
3441da7d 2405 case UFSHCD_STATE_RESET:
7a3e97b0 2406 err = SCSI_MLQUEUE_HOST_BUSY;
3441da7d
SRT		 2407 goto out_unlock;
2408 case UFSHCD_STATE_ERROR:
2409 set_host_byte(cmd, DID_ERROR);
2410 cmd->scsi_done(cmd);
2411 goto out_unlock;
2412 default:
2413 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2414 __func__, hba->ufshcd_state);
2415 set_host_byte(cmd, DID_BAD_TARGET);
2416 cmd->scsi_done(cmd);
2417 goto out_unlock;
7a3e97b0 2418 }
53c12d0e
YG		 2419
2420 /* if error handling is in progress, don't issue commands */
2421 if (ufshcd_eh_in_progress(hba)) {
2422 set_host_byte(cmd, DID_ERROR);
2423 cmd->scsi_done(cmd);
2424 goto out_unlock;
2425 }
3441da7d 2426 spin_unlock_irqrestore(hba->host->host_lock, flags);
7a3e97b0 2427
7fabb77b
GB		 2428 hba->req_abort_count = 0;
2429
1ab27c9c
ST		 2430 err = ufshcd_hold(hba, true);
2431 if (err) {
2432 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2433 goto out;
2434 }
2435 WARN_ON(hba->clk_gating.state != CLKS_ON);
2436
7a3e97b0
SY		 2437 lrbp = &hba->lrb[tag];
2438
5a0b0cb9 2439 WARN_ON(lrbp->cmd);
7a3e97b0 2440 lrbp->cmd = cmd;
09a5a24f 2441 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2442 lrbp->sense_buffer = cmd->sense_buffer;
2443 lrbp->task_tag = tag;
0ce147d4 2444 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2445 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
e0b299e3 2446 lrbp->req_abort_skip = false;
7a3e97b0 2447
300bb13f
JP		 2448 ufshcd_comp_scsi_upiu(hba, lrbp);
2449
75b1cc4a 2450 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2451 if (err) {
2452 lrbp->cmd = NULL;
7a3e97b0 2453 goto out;
5a0b0cb9 2454 }
ad1a1b9c
GB		 2455 /* Make sure descriptors are ready before ringing the doorbell */
2456 wmb();
7a3e97b0
SY		 2457
2458 /* issue command to the controller */
2459 spin_lock_irqsave(hba->host->host_lock, flags);
0e675efa 2460 ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
7a3e97b0 2461 ufshcd_send_command(hba, tag);
3441da7d 2462out_unlock:
7a3e97b0
SY		 2463 spin_unlock_irqrestore(hba->host->host_lock, flags);
2464out:
a3cd5ec5 2465 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2466 return err;
2467}
2468
5a0b0cb9
SRT		 2469static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2470 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2471{
2472 lrbp->cmd = NULL;
2473 lrbp->sense_bufflen = 0;
2474 lrbp->sense_buffer = NULL;
2475 lrbp->task_tag = tag;
2476 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9
SRT		 2477 lrbp->intr_cmd = true; /* No interrupt aggregation */
2478 hba->dev_cmd.type = cmd_type;
2479
300bb13f 2480 return ufshcd_comp_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2481}
2482
2483static int
2484ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2485{
2486 int err = 0;
2487 unsigned long flags;
2488 u32 mask = 1 << tag;
2489
2490 /* clear outstanding transaction before retry */
2491 spin_lock_irqsave(hba->host->host_lock, flags);
2492 ufshcd_utrl_clear(hba, tag);
2493 spin_unlock_irqrestore(hba->host->host_lock, flags);
2494
2495 /*
2496 * wait for for h/w to clear corresponding bit in door-bell.
2497 * max. wait is 1 sec.
2498 */
2499 err = ufshcd_wait_for_register(hba,
2500 REG_UTP_TRANSFER_REQ_DOOR_BELL,
596585a2 2501 mask, ~mask, 1000, 1000, true);
5a0b0cb9
SRT		 2502
2503 return err;
2504}
2505
c6d4a831
DR		 2506static int
2507ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2508{
2509 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2510
2511 /* Get the UPIU response */
2512 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2513 UPIU_RSP_CODE_OFFSET;
2514 return query_res->response;
2515}
2516
5a0b0cb9
SRT		 2517/**
2518 * ufshcd_dev_cmd_completion() - handles device management command responses
2519 * @hba: per adapter instance
2520 * @lrbp: pointer to local reference block
2521 */
2522static int
2523ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2524{
2525 int resp;
2526 int err = 0;
2527
ff8e20c6 2528 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2529 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2530
2531 switch (resp) {
2532 case UPIU_TRANSACTION_NOP_IN:
2533 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2534 err = -EINVAL;
2535 dev_err(hba->dev, "%s: unexpected response %x\n",
2536 __func__, resp);
2537 }
2538 break;
68078d5c 2539 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2540 err = ufshcd_check_query_response(hba, lrbp);
2541 if (!err)
2542 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2543 break;
5a0b0cb9
SRT		 2544 case UPIU_TRANSACTION_REJECT_UPIU:
2545 /* TODO: handle Reject UPIU Response */
2546 err = -EPERM;
2547 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2548 __func__);
2549 break;
2550 default:
2551 err = -EINVAL;
2552 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2553 __func__, resp);
2554 break;
2555 }
2556
2557 return err;
2558}
2559
2560static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2561 struct ufshcd_lrb *lrbp, int max_timeout)
2562{
2563 int err = 0;
2564 unsigned long time_left;
2565 unsigned long flags;
2566
2567 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2568 msecs_to_jiffies(max_timeout));
2569
ad1a1b9c
GB		 2570 /* Make sure descriptors are ready before ringing the doorbell */
2571 wmb();
5a0b0cb9
SRT		 2572 spin_lock_irqsave(hba->host->host_lock, flags);
2573 hba->dev_cmd.complete = NULL;
2574 if (likely(time_left)) {
2575 err = ufshcd_get_tr_ocs(lrbp);
2576 if (!err)
2577 err = ufshcd_dev_cmd_completion(hba, lrbp);
2578 }
2579 spin_unlock_irqrestore(hba->host->host_lock, flags);
2580
2581 if (!time_left) {
2582 err = -ETIMEDOUT;
a48353f6
YG		 2583 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2584 __func__, lrbp->task_tag);
5a0b0cb9 2585 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2586 /* successfully cleared the command, retry if needed */
5a0b0cb9 2587 err = -EAGAIN;
a48353f6
YG		 2588 /*
2589 * in case of an error, after clearing the doorbell,
2590 * we also need to clear the outstanding_request
2591 * field in hba
2592 */
2593 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2594 }
2595
2596 return err;
2597}
2598
5a0b0cb9
SRT		 2599/**
2600 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2601 * @hba: UFS hba
2602 * @cmd_type: specifies the type (NOP, Query...)
2603 * @timeout: time in seconds
5a0b0cb9 2604 *
68078d5c
DR		 2605 * NOTE: Since there is only one available tag for device management commands,
2606 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2607 */
2608static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2609 enum dev_cmd_type cmd_type, int timeout)
2610{
7252a360
BVA		 2611 struct request_queue *q = hba->cmd_queue;
2612 struct request *req;
5a0b0cb9
SRT		 2613 struct ufshcd_lrb *lrbp;
2614 int err;
2615 int tag;
2616 struct completion wait;
2617 unsigned long flags;
2618
a3cd5ec5 2619 down_read(&hba->clk_scaling_lock);
2620
5a0b0cb9
SRT		 2621 /*
2622 * Get free slot, sleep if slots are unavailable.
2623 * Even though we use wait_event() which sleeps indefinitely,
2624 * the maximum wait time is bounded by SCSI request timeout.
2625 */
7252a360
BVA		 2626 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
2627 if (IS_ERR(req))
2628 return PTR_ERR(req);
2629 tag = req->tag;
2630 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5a0b0cb9
SRT		 2631
2632 init_completion(&wait);
2633 lrbp = &hba->lrb[tag];
2634 WARN_ON(lrbp->cmd);
2635 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2636 if (unlikely(err))
2637 goto out_put_tag;
2638
2639 hba->dev_cmd.complete = &wait;
2640
6667e6d9 2641 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
e3dfdc53
YG		 2642 /* Make sure descriptors are ready before ringing the doorbell */
2643 wmb();
5a0b0cb9 2644 spin_lock_irqsave(hba->host->host_lock, flags);
0e675efa 2645 ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
5a0b0cb9
SRT		 2646 ufshcd_send_command(hba, tag);
2647 spin_unlock_irqrestore(hba->host->host_lock, flags);
2648
2649 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2650
6667e6d9
OS		 2651 ufshcd_add_query_upiu_trace(hba, tag,
2652 err ? "query_complete_err" : "query_complete");
2653
5a0b0cb9 2654out_put_tag:
7252a360 2655 blk_put_request(req);
a3cd5ec5 2656 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2657 return err;
2658}
2659
d44a5f98
DR		 2660/**
2661 * ufshcd_init_query() - init the query response and request parameters
2662 * @hba: per-adapter instance
2663 * @request: address of the request pointer to be initialized
2664 * @response: address of the response pointer to be initialized
2665 * @opcode: operation to perform
2666 * @idn: flag idn to access
2667 * @index: LU number to access
2668 * @selector: query/flag/descriptor further identification
2669 */
2670static inline void ufshcd_init_query(struct ufs_hba *hba,
2671 struct ufs_query_req **request, struct ufs_query_res **response,
2672 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2673{
2674 *request = &hba->dev_cmd.query.request;
2675 *response = &hba->dev_cmd.query.response;
2676 memset(*request, 0, sizeof(struct ufs_query_req));
2677 memset(*response, 0, sizeof(struct ufs_query_res));
2678 (*request)->upiu_req.opcode = opcode;
2679 (*request)->upiu_req.idn = idn;
2680 (*request)->upiu_req.index = index;
2681 (*request)->upiu_req.selector = selector;
2682}
2683
dc3c8d3a
YG		 2684static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2685 enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2686{
2687 int ret;
2688 int retries;
2689
2690 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2691 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2692 if (ret)
2693 dev_dbg(hba->dev,
2694 "%s: failed with error %d, retries %d\n",
2695 __func__, ret, retries);
2696 else
2697 break;
2698 }
2699
2700 if (ret)
2701 dev_err(hba->dev,
2702 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2703 __func__, opcode, idn, ret, retries);
2704 return ret;
2705}
2706
68078d5c
DR		 2707/**
2708 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2709 * @hba: per-adapter instance
2710 * @opcode: flag query to perform
2711 * @idn: flag idn to access
2712 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2713 *
2714 * Returns 0 for success, non-zero in case of failure
2715 */
dc3c8d3a 2716int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
68078d5c
DR		 2717 enum flag_idn idn, bool *flag_res)
2718{
d44a5f98
DR		 2719 struct ufs_query_req *request = NULL;
2720 struct ufs_query_res *response = NULL;
2721 int err, index = 0, selector = 0;
e5ad406c 2722 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2723
2724 BUG_ON(!hba);
2725
1ab27c9c 2726 ufshcd_hold(hba, false);
68078d5c 2727 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2728 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2729 selector);
68078d5c
DR		 2730
2731 switch (opcode) {
2732 case UPIU_QUERY_OPCODE_SET_FLAG:
2733 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2734 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2735 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2736 break;
2737 case UPIU_QUERY_OPCODE_READ_FLAG:
2738 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2739 if (!flag_res) {
2740 /* No dummy reads */
2741 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2742 __func__);
2743 err = -EINVAL;
2744 goto out_unlock;
2745 }
2746 break;
2747 default:
2748 dev_err(hba->dev,
2749 "%s: Expected query flag opcode but got = %d\n",
2750 __func__, opcode);
2751 err = -EINVAL;
2752 goto out_unlock;
2753 }
68078d5c 2754
e5ad406c 2755 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 2756
2757 if (err) {
2758 dev_err(hba->dev,
2759 "%s: Sending flag query for idn %d failed, err = %d\n",
2760 __func__, idn, err);
2761 goto out_unlock;
2762 }
2763
2764 if (flag_res)
e8c8e82a 2765 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 2766 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2767
2768out_unlock:
2769 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 2770 ufshcd_release(hba);
68078d5c
DR		 2771 return err;
2772}
2773
66ec6d59
SRT		 2774/**
2775 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 2776 * @hba: per-adapter instance
2777 * @opcode: attribute opcode
2778 * @idn: attribute idn to access
2779 * @index: index field
2780 * @selector: selector field
2781 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 2782 *
2783 * Returns 0 for success, non-zero in case of failure
2784*/
ec92b59c
SN		 2785int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2786 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 2787{
d44a5f98
DR		 2788 struct ufs_query_req *request = NULL;
2789 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 2790 int err;
2791
2792 BUG_ON(!hba);
2793
1ab27c9c 2794 ufshcd_hold(hba, false);
66ec6d59
SRT		 2795 if (!attr_val) {
2796 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2797 __func__, opcode);
2798 err = -EINVAL;
2799 goto out;
2800 }
2801
2802 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2803 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2804 selector);
66ec6d59
SRT		 2805
2806 switch (opcode) {
2807 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2808 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 2809 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 2810 break;
2811 case UPIU_QUERY_OPCODE_READ_ATTR:
2812 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2813 break;
2814 default:
2815 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2816 __func__, opcode);
2817 err = -EINVAL;
2818 goto out_unlock;
2819 }
2820
d44a5f98 2821 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 2822
2823 if (err) {
4b761b58
YG		 2824 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2825 __func__, opcode, idn, index, err);
66ec6d59
SRT		 2826 goto out_unlock;
2827 }
2828
e8c8e82a 2829 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 2830
2831out_unlock:
2832 mutex_unlock(&hba->dev_cmd.lock);
2833out:
1ab27c9c 2834 ufshcd_release(hba);
66ec6d59
SRT		 2835 return err;
2836}
2837
5e86ae44
YG		 2838/**
2839 * ufshcd_query_attr_retry() - API function for sending query
2840 * attribute with retries
2841 * @hba: per-adapter instance
2842 * @opcode: attribute opcode
2843 * @idn: attribute idn to access
2844 * @index: index field
2845 * @selector: selector field
2846 * @attr_val: the attribute value after the query request
2847 * completes
2848 *
2849 * Returns 0 for success, non-zero in case of failure
2850*/
2851static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2852 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2853 u32 *attr_val)
2854{
2855 int ret = 0;
2856 u32 retries;
2857
2858 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2859 ret = ufshcd_query_attr(hba, opcode, idn, index,
2860 selector, attr_val);
2861 if (ret)
2862 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2863 __func__, ret, retries);
2864 else
2865 break;
2866 }
2867
2868 if (ret)
2869 dev_err(hba->dev,
2870 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2871 __func__, idn, ret, QUERY_REQ_RETRIES);
2872 return ret;
2873}
2874
a70e91b8 2875static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 2876 enum query_opcode opcode, enum desc_idn idn, u8 index,
2877 u8 selector, u8 *desc_buf, int *buf_len)
2878{
2879 struct ufs_query_req *request = NULL;
2880 struct ufs_query_res *response = NULL;
2881 int err;
2882
2883 BUG_ON(!hba);
2884
1ab27c9c 2885 ufshcd_hold(hba, false);
d44a5f98
DR		 2886 if (!desc_buf) {
2887 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2888 __func__, opcode);
2889 err = -EINVAL;
2890 goto out;
2891 }
2892
a4b0e8a4 2893 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 2894 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2895 __func__, *buf_len);
2896 err = -EINVAL;
2897 goto out;
2898 }
2899
2900 mutex_lock(&hba->dev_cmd.lock);
2901 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2902 selector);
2903 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 2904 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 2905
2906 switch (opcode) {
2907 case UPIU_QUERY_OPCODE_WRITE_DESC:
2908 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2909 break;
2910 case UPIU_QUERY_OPCODE_READ_DESC:
2911 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2912 break;
2913 default:
2914 dev_err(hba->dev,
2915 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2916 __func__, opcode);
2917 err = -EINVAL;
2918 goto out_unlock;
2919 }
2920
2921 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2922
2923 if (err) {
4b761b58
YG		 2924 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2925 __func__, opcode, idn, index, err);
d44a5f98
DR		 2926 goto out_unlock;
2927 }
2928
ea2aab24 2929 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 2930
2931out_unlock:
cfcbae38 2932 hba->dev_cmd.query.descriptor = NULL;
d44a5f98
DR		 2933 mutex_unlock(&hba->dev_cmd.lock);
2934out:
1ab27c9c 2935 ufshcd_release(hba);
d44a5f98
DR		 2936 return err;
2937}
2938
a70e91b8 2939/**
8aa29f19
BVA		 2940 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2941 * @hba: per-adapter instance
2942 * @opcode: attribute opcode
2943 * @idn: attribute idn to access
2944 * @index: index field
2945 * @selector: selector field
2946 * @desc_buf: the buffer that contains the descriptor
2947 * @buf_len: length parameter passed to the device
a70e91b8
YG		 2948 *
2949 * Returns 0 for success, non-zero in case of failure.
2950 * The buf_len parameter will contain, on return, the length parameter
2951 * received on the response.
2952 */
2238d31c
SN		 2953int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
2954 enum query_opcode opcode,
2955 enum desc_idn idn, u8 index,
2956 u8 selector,
2957 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 2958{
2959 int err;
2960 int retries;
2961
2962 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2963 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
2964 selector, desc_buf, buf_len);
2965 if (!err || err == -EINVAL)
2966 break;
2967 }
2968
2969 return err;
2970}
a70e91b8 2971
a4b0e8a4
PM		 2972/**
2973 * ufshcd_read_desc_length - read the specified descriptor length from header
2974 * @hba: Pointer to adapter instance
2975 * @desc_id: descriptor idn value
2976 * @desc_index: descriptor index
2977 * @desc_length: pointer to variable to read the length of descriptor
2978 *
2979 * Return 0 in case of success, non-zero otherwise
2980 */
2981static int ufshcd_read_desc_length(struct ufs_hba *hba,
2982 enum desc_idn desc_id,
2983 int desc_index,
2984 int *desc_length)
2985{
2986 int ret;
2987 u8 header[QUERY_DESC_HDR_SIZE];
2988 int header_len = QUERY_DESC_HDR_SIZE;
2989
2990 if (desc_id >= QUERY_DESC_IDN_MAX)
2991 return -EINVAL;
2992
2993 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
2994 desc_id, desc_index, 0, header,
2995 &header_len);
2996
2997 if (ret) {
2998 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
2999 __func__, desc_id);
3000 return ret;
3001 } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3002 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3003 __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3004 desc_id);
3005 ret = -EINVAL;
3006 }
3007
3008 *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3009 return ret;
3010
3011}
3012
3013/**
3014 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3015 * @hba: Pointer to adapter instance
3016 * @desc_id: descriptor idn value
3017 * @desc_len: mapped desc length (out)
3018 *
3019 * Return 0 in case of success, non-zero otherwise
3020 */
3021int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3022 enum desc_idn desc_id, int *desc_len)
3023{
3024 switch (desc_id) {
3025 case QUERY_DESC_IDN_DEVICE:
3026 *desc_len = hba->desc_size.dev_desc;
3027 break;
3028 case QUERY_DESC_IDN_POWER:
3029 *desc_len = hba->desc_size.pwr_desc;
3030 break;
3031 case QUERY_DESC_IDN_GEOMETRY:
3032 *desc_len = hba->desc_size.geom_desc;
3033 break;
3034 case QUERY_DESC_IDN_CONFIGURATION:
3035 *desc_len = hba->desc_size.conf_desc;
3036 break;
3037 case QUERY_DESC_IDN_UNIT:
3038 *desc_len = hba->desc_size.unit_desc;
3039 break;
3040 case QUERY_DESC_IDN_INTERCONNECT:
3041 *desc_len = hba->desc_size.interc_desc;
3042 break;
3043 case QUERY_DESC_IDN_STRING:
3044 *desc_len = QUERY_DESC_MAX_SIZE;
3045 break;
c648c2d2
SN		 3046 case QUERY_DESC_IDN_HEALTH:
3047 *desc_len = hba->desc_size.hlth_desc;
3048 break;
a4b0e8a4
PM		 3049 case QUERY_DESC_IDN_RFU_0:
3050 case QUERY_DESC_IDN_RFU_1:
3051 *desc_len = 0;
3052 break;
3053 default:
3054 *desc_len = 0;
3055 return -EINVAL;
3056 }
3057 return 0;
3058}
3059EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3060
da461cec
SJ		 3061/**
3062 * ufshcd_read_desc_param - read the specified descriptor parameter
3063 * @hba: Pointer to adapter instance
3064 * @desc_id: descriptor idn value
3065 * @desc_index: descriptor index
3066 * @param_offset: offset of the parameter to read
3067 * @param_read_buf: pointer to buffer where parameter would be read
3068 * @param_size: sizeof(param_read_buf)
3069 *
3070 * Return 0 in case of success, non-zero otherwise
3071 */
45bced87
SN		 3072int ufshcd_read_desc_param(struct ufs_hba *hba,
3073 enum desc_idn desc_id,
3074 int desc_index,
3075 u8 param_offset,
3076 u8 *param_read_buf,
3077 u8 param_size)
da461cec
SJ		 3078{
3079 int ret;
3080 u8 *desc_buf;
a4b0e8a4 3081 int buff_len;
da461cec
SJ		 3082 bool is_kmalloc = true;
3083
a4b0e8a4
PM		 3084 /* Safety check */
3085 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3086 return -EINVAL;
3087
a4b0e8a4
PM		 3088 /* Get the max length of descriptor from structure filled up at probe
3089 * time.
3090 */
3091 ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
da461cec 3092
a4b0e8a4
PM		 3093 /* Sanity checks */
3094 if (ret || !buff_len) {
3095 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3096 __func__);
3097 return ret;
3098 }
3099
3100 /* Check whether we need temp memory */
3101 if (param_offset != 0 || param_size < buff_len) {
da461cec
SJ		 3102 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3103 if (!desc_buf)
3104 return -ENOMEM;
a4b0e8a4
PM		 3105 } else {
3106 desc_buf = param_read_buf;
3107 is_kmalloc = false;
da461cec
SJ		 3108 }
3109
a4b0e8a4 3110 /* Request for full descriptor */
a70e91b8 3111 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3112 desc_id, desc_index, 0,
3113 desc_buf, &buff_len);
da461cec 3114
bde44bb6 3115 if (ret) {
3116 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3117 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3118 goto out;
3119 }
3120
bde44bb6 3121 /* Sanity check */
3122 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3123 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3124 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3125 ret = -EINVAL;
3126 goto out;
3127 }
3128
a4b0e8a4
PM		 3129 /* Check wherher we will not copy more data, than available */
3130 if (is_kmalloc && param_size > buff_len)
3131 param_size = buff_len;
bde44bb6 3132
da461cec
SJ		 3133 if (is_kmalloc)
3134 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3135out:
3136 if (is_kmalloc)
3137 kfree(desc_buf);
3138 return ret;
3139}
3140
3141static inline int ufshcd_read_desc(struct ufs_hba *hba,
3142 enum desc_idn desc_id,
3143 int desc_index,
4b828fe1 3144 void *buf,
da461cec
SJ		 3145 u32 size)
3146{
3147 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3148}
3149
3150static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3151 u8 *buf,
3152 u32 size)
3153{
dbd34a61 3154 return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
da461cec
SJ		 3155}
3156
8209b6d5 3157static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
b573d484
YG		 3158{
3159 return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3160}
b573d484 3161
4b828fe1
TW		 3162/**
3163 * struct uc_string_id - unicode string
3164 *
3165 * @len: size of this descriptor inclusive
3166 * @type: descriptor type
3167 * @uc: unicode string character
3168 */
3169struct uc_string_id {
3170 u8 len;
3171 u8 type;
3172 wchar_t uc[0];
3173} __packed;
3174
3175/* replace non-printable or non-ASCII characters with spaces */
3176static inline char ufshcd_remove_non_printable(u8 ch)
3177{
3178 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3179}
3180
b573d484
YG		 3181/**
3182 * ufshcd_read_string_desc - read string descriptor
3183 * @hba: pointer to adapter instance
3184 * @desc_index: descriptor index
4b828fe1
TW		 3185 * @buf: pointer to buffer where descriptor would be read,
3186 * the caller should free the memory.
b573d484 3187 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3188 * null terminated string.
b573d484 3189 *
4b828fe1
TW		 3190 * Return:
3191 * * string size on success.
3192 * * -ENOMEM: on allocation failure
3193 * * -EINVAL: on a wrong parameter
b573d484 3194 */
4b828fe1
TW		 3195int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3196 u8 **buf, bool ascii)
b573d484 3197{
4b828fe1
TW		 3198 struct uc_string_id *uc_str;
3199 u8 *str;
3200 int ret;
b573d484 3201
4b828fe1
TW		 3202 if (!buf)
3203 return -EINVAL;
b573d484 3204
4b828fe1
TW		 3205 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3206 if (!uc_str)
3207 return -ENOMEM;
b573d484 3208
4b828fe1
TW		 3209 ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING,
3210 desc_index, uc_str,
3211 QUERY_DESC_MAX_SIZE);
3212 if (ret < 0) {
3213 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3214 QUERY_REQ_RETRIES, ret);
3215 str = NULL;
3216 goto out;
3217 }
3218
3219 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3220 dev_dbg(hba->dev, "String Desc is of zero length\n");
3221 str = NULL;
3222 ret = 0;
b573d484
YG		 3223 goto out;
3224 }
3225
3226 if (ascii) {
4b828fe1 3227 ssize_t ascii_len;
b573d484 3228 int i;
b573d484 3229 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3230 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3231 str = kzalloc(ascii_len, GFP_KERNEL);
3232 if (!str) {
3233 ret = -ENOMEM;
fcbefc3b 3234 goto out;
b573d484
YG		 3235 }
3236
3237 /*
3238 * the descriptor contains string in UTF16 format
3239 * we need to convert to utf-8 so it can be displayed
3240 */
4b828fe1
TW		 3241 ret = utf16s_to_utf8s(uc_str->uc,
3242 uc_str->len - QUERY_DESC_HDR_SIZE,
3243 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3244
3245 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3246 for (i = 0; i < ret; i++)
3247 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3248
4b828fe1
TW		 3249 str[ret++] = '\0';
3250
3251 } else {
5f57704d 3252 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3253 if (!str) {
3254 ret = -ENOMEM;
3255 goto out;
3256 }
4b828fe1 3257 ret = uc_str->len;
b573d484
YG		 3258 }
3259out:
4b828fe1
TW		 3260 *buf = str;
3261 kfree(uc_str);
3262 return ret;
b573d484 3263}
b573d484 3264
da461cec
SJ		 3265/**
3266 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3267 * @hba: Pointer to adapter instance
3268 * @lun: lun id
3269 * @param_offset: offset of the parameter to read
3270 * @param_read_buf: pointer to buffer where parameter would be read
3271 * @param_size: sizeof(param_read_buf)
3272 *
3273 * Return 0 in case of success, non-zero otherwise
3274 */
3275static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3276 int lun,
3277 enum unit_desc_param param_offset,
3278 u8 *param_read_buf,
3279 u32 param_size)
3280{
3281 /*
3282 * Unit descriptors are only available for general purpose LUs (LUN id
3283 * from 0 to 7) and RPMB Well known LU.
3284 */
d829fc8a 3285 if (!ufs_is_valid_unit_desc_lun(lun))
da461cec
SJ		 3286 return -EOPNOTSUPP;
3287
3288 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3289 param_offset, param_read_buf, param_size);
3290}
3291
7a3e97b0
SY		 3292/**
3293 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3294 * @hba: per adapter instance
3295 *
3296 * 1. Allocate DMA memory for Command Descriptor array
3297 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3298 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3299 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3300 * (UTMRDL)
3301 * 4. Allocate memory for local reference block(lrb).
3302 *
3303 * Returns 0 for success, non-zero in case of failure
3304 */
3305static int ufshcd_memory_alloc(struct ufs_hba *hba)
3306{
3307 size_t utmrdl_size, utrdl_size, ucdl_size;
3308
3309 /* Allocate memory for UTP command descriptors */
3310 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3311 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3312 ucdl_size,
3313 &hba->ucdl_dma_addr,
3314 GFP_KERNEL);
7a3e97b0
SY		 3315
3316 /*
3317 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3318 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3319 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3320 * be aligned to 128 bytes as well
3321 */
3322 if (!hba->ucdl_base_addr ||
3323 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3324 dev_err(hba->dev,
7a3e97b0
SY		 3325 "Command Descriptor Memory allocation failed\n");
3326 goto out;
3327 }
3328
3329 /*
3330 * Allocate memory for UTP Transfer descriptors
3331 * UFSHCI requires 1024 byte alignment of UTRD
3332 */
3333 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3334 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3335 utrdl_size,
3336 &hba->utrdl_dma_addr,
3337 GFP_KERNEL);
7a3e97b0
SY		 3338 if (!hba->utrdl_base_addr ||
3339 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3340 dev_err(hba->dev,
7a3e97b0
SY		 3341 "Transfer Descriptor Memory allocation failed\n");
3342 goto out;
3343 }
3344
3345 /*
3346 * Allocate memory for UTP Task Management descriptors
3347 * UFSHCI requires 1024 byte alignment of UTMRD
3348 */
3349 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3350 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3351 utmrdl_size,
3352 &hba->utmrdl_dma_addr,
3353 GFP_KERNEL);
7a3e97b0
SY		 3354 if (!hba->utmrdl_base_addr ||
3355 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3356 dev_err(hba->dev,
7a3e97b0
SY		 3357 "Task Management Descriptor Memory allocation failed\n");
3358 goto out;
3359 }
3360
3361 /* Allocate memory for local reference block */
a86854d0
KC		 3362 hba->lrb = devm_kcalloc(hba->dev,
3363 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3364 GFP_KERNEL);
7a3e97b0 3365 if (!hba->lrb) {
3b1d0580 3366 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3367 goto out;
3368 }
3369 return 0;
3370out:
7a3e97b0
SY		 3371 return -ENOMEM;
3372}
3373
3374/**
3375 * ufshcd_host_memory_configure - configure local reference block with
3376 * memory offsets
3377 * @hba: per adapter instance
3378 *
3379 * Configure Host memory space
3380 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3381 * address.
3382 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3383 * and PRDT offset.
3384 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3385 * into local reference block.
3386 */
3387static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3388{
3389 struct utp_transfer_cmd_desc *cmd_descp;
3390 struct utp_transfer_req_desc *utrdlp;
3391 dma_addr_t cmd_desc_dma_addr;
3392 dma_addr_t cmd_desc_element_addr;
3393 u16 response_offset;
3394 u16 prdt_offset;
3395 int cmd_desc_size;
3396 int i;
3397
3398 utrdlp = hba->utrdl_base_addr;
3399 cmd_descp = hba->ucdl_base_addr;
3400
3401 response_offset =
3402 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3403 prdt_offset =
3404 offsetof(struct utp_transfer_cmd_desc, prd_table);
3405
3406 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3407 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3408
3409 for (i = 0; i < hba->nutrs; i++) {
3410 /* Configure UTRD with command descriptor base address */
3411 cmd_desc_element_addr =
3412 (cmd_desc_dma_addr + (cmd_desc_size * i));
3413 utrdlp[i].command_desc_base_addr_lo =
3414 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3415 utrdlp[i].command_desc_base_addr_hi =
3416 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3417
3418 /* Response upiu and prdt offset should be in double words */
75b1cc4a
KK		 3419 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3420 utrdlp[i].response_upiu_offset =
3421 cpu_to_le16(response_offset);
3422 utrdlp[i].prd_table_offset =
3423 cpu_to_le16(prdt_offset);
3424 utrdlp[i].response_upiu_length =
3425 cpu_to_le16(ALIGNED_UPIU_SIZE);
3426 } else {
3427 utrdlp[i].response_upiu_offset =
7a3e97b0 3428 cpu_to_le16((response_offset >> 2));
75b1cc4a 3429 utrdlp[i].prd_table_offset =
7a3e97b0 3430 cpu_to_le16((prdt_offset >> 2));
75b1cc4a 3431 utrdlp[i].response_upiu_length =
3ca316c5 3432 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
75b1cc4a 3433 }
7a3e97b0
SY		 3434
3435 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
ff8e20c6
DR		 3436 hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr +
3437 (i * sizeof(struct utp_transfer_req_desc));
5a0b0cb9
SRT		 3438 hba->lrb[i].ucd_req_ptr =
3439 (struct utp_upiu_req *)(cmd_descp + i);
ff8e20c6 3440 hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr;
7a3e97b0
SY		 3441 hba->lrb[i].ucd_rsp_ptr =
3442 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
ff8e20c6
DR		 3443 hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr +
3444 response_offset;
7a3e97b0
SY		 3445 hba->lrb[i].ucd_prdt_ptr =
3446 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
ff8e20c6
DR		 3447 hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr +
3448 prdt_offset;
7a3e97b0
SY		 3449 }
3450}
3451
3452/**
3453 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3454 * @hba: per adapter instance
3455 *
3456 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3457 * in order to initialize the Unipro link startup procedure.
3458 * Once the Unipro links are up, the device connected to the controller
3459 * is detected.
3460 *
3461 * Returns 0 on success, non-zero value on failure
3462 */
3463static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3464{
6ccf44fe
SJ		 3465 struct uic_command uic_cmd = {0};
3466 int ret;
7a3e97b0 3467
6ccf44fe 3468 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3469
6ccf44fe
SJ		 3470 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3471 if (ret)
ff8e20c6 3472 dev_dbg(hba->dev,
6ccf44fe
SJ		 3473 "dme-link-startup: error code %d\n", ret);
3474 return ret;
7a3e97b0 3475}
4404c5de
AA		 3476/**
3477 * ufshcd_dme_reset - UIC command for DME_RESET
3478 * @hba: per adapter instance
3479 *
3480 * DME_RESET command is issued in order to reset UniPro stack.
3481 * This function now deal with cold reset.
3482 *
3483 * Returns 0 on success, non-zero value on failure
3484 */
3485static int ufshcd_dme_reset(struct ufs_hba *hba)
3486{
3487 struct uic_command uic_cmd = {0};
3488 int ret;
3489
3490 uic_cmd.command = UIC_CMD_DME_RESET;
3491
3492 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3493 if (ret)
3494 dev_err(hba->dev,
3495 "dme-reset: error code %d\n", ret);
3496
3497 return ret;
3498}
3499
3500/**
3501 * ufshcd_dme_enable - UIC command for DME_ENABLE
3502 * @hba: per adapter instance
3503 *
3504 * DME_ENABLE command is issued in order to enable UniPro stack.
3505 *
3506 * Returns 0 on success, non-zero value on failure
3507 */
3508static int ufshcd_dme_enable(struct ufs_hba *hba)
3509{
3510 struct uic_command uic_cmd = {0};
3511 int ret;
3512
3513 uic_cmd.command = UIC_CMD_DME_ENABLE;
3514
3515 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3516 if (ret)
3517 dev_err(hba->dev,
3518 "dme-reset: error code %d\n", ret);
3519
3520 return ret;
3521}
7a3e97b0 3522
cad2e03d
YG		 3523static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3524{
3525 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3526 unsigned long min_sleep_time_us;
3527
3528 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3529 return;
3530
3531 /*
3532 * last_dme_cmd_tstamp will be 0 only for 1st call to
3533 * this function
3534 */
3535 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3536 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3537 } else {
3538 unsigned long delta =
3539 (unsigned long) ktime_to_us(
3540 ktime_sub(ktime_get(),
3541 hba->last_dme_cmd_tstamp));
3542
3543 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3544 min_sleep_time_us =
3545 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3546 else
3547 return; /* no more delay required */
3548 }
3549
3550 /* allow sleep for extra 50us if needed */
3551 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3552}
3553
12b4fdb4
SJ		 3554/**
3555 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3556 * @hba: per adapter instance
3557 * @attr_sel: uic command argument1
3558 * @attr_set: attribute set type as uic command argument2
3559 * @mib_val: setting value as uic command argument3
3560 * @peer: indicate whether peer or local
3561 *
3562 * Returns 0 on success, non-zero value on failure
3563 */
3564int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3565 u8 attr_set, u32 mib_val, u8 peer)
3566{
3567 struct uic_command uic_cmd = {0};
3568 static const char *const action[] = {
3569 "dme-set",
3570 "dme-peer-set"
3571 };
3572 const char *set = action[!!peer];
3573 int ret;
64238fbd 3574 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3575
3576 uic_cmd.command = peer ?
3577 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3578 uic_cmd.argument1 = attr_sel;
3579 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3580 uic_cmd.argument3 = mib_val;
3581
64238fbd
YG		 3582 do {
3583 /* for peer attributes we retry upon failure */
3584 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3585 if (ret)
3586 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3587 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3588 } while (ret && peer && --retries);
3589
f37e9f8c 3590 if (ret)
64238fbd 3591 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3592 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3593 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3594
3595 return ret;
3596}
3597EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3598
3599/**
3600 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3601 * @hba: per adapter instance
3602 * @attr_sel: uic command argument1
3603 * @mib_val: the value of the attribute as returned by the UIC command
3604 * @peer: indicate whether peer or local
3605 *
3606 * Returns 0 on success, non-zero value on failure
3607 */
3608int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3609 u32 *mib_val, u8 peer)
3610{
3611 struct uic_command uic_cmd = {0};
3612 static const char *const action[] = {
3613 "dme-get",
3614 "dme-peer-get"
3615 };
3616 const char *get = action[!!peer];
3617 int ret;
64238fbd 3618 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3619 struct ufs_pa_layer_attr orig_pwr_info;
3620 struct ufs_pa_layer_attr temp_pwr_info;
3621 bool pwr_mode_change = false;
3622
3623 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3624 orig_pwr_info = hba->pwr_info;
3625 temp_pwr_info = orig_pwr_info;
3626
3627 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3628 orig_pwr_info.pwr_rx == FAST_MODE) {
3629 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3630 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3631 pwr_mode_change = true;
3632 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3633 orig_pwr_info.pwr_rx == SLOW_MODE) {
3634 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3635 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3636 pwr_mode_change = true;
3637 }
3638 if (pwr_mode_change) {
3639 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3640 if (ret)
3641 goto out;
3642 }
3643 }
12b4fdb4
SJ		 3644
3645 uic_cmd.command = peer ?
3646 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3647 uic_cmd.argument1 = attr_sel;
3648
64238fbd
YG		 3649 do {
3650 /* for peer attributes we retry upon failure */
3651 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3652 if (ret)
3653 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3654 get, UIC_GET_ATTR_ID(attr_sel), ret);
3655 } while (ret && peer && --retries);
3656
f37e9f8c 3657 if (ret)
64238fbd 3658 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3659 get, UIC_GET_ATTR_ID(attr_sel),
3660 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3661
64238fbd 3662 if (mib_val && !ret)
12b4fdb4 3663 *mib_val = uic_cmd.argument3;
874237f7
YG		 3664
3665 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3666 && pwr_mode_change)
3667 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3668out:
3669 return ret;
3670}
3671EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3672
53b3d9c3 3673/**
57d104c1
SJ		 3674 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3675 * state) and waits for it to take effect.
3676 *
53b3d9c3 3677 * @hba: per adapter instance
57d104c1
SJ		 3678 * @cmd: UIC command to execute
3679 *
3680 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3681 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3682 * and device UniPro link and hence it's final completion would be indicated by
3683 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3684 * addition to normal UIC command completion Status (UCCS). This function only
3685 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3686 *
3687 * Returns 0 on success, non-zero value on failure
3688 */
57d104c1 3689static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3690{
57d104c1 3691 struct completion uic_async_done;
53b3d9c3
SJ		 3692 unsigned long flags;
3693 u8 status;
3694 int ret;
d75f7fe4 3695 bool reenable_intr = false;
53b3d9c3 3696
53b3d9c3 3697 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3698 init_completion(&uic_async_done);
cad2e03d 3699 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3700
3701 spin_lock_irqsave(hba->host->host_lock, flags);
57d104c1 3702 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3703 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3704 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3705 /*
3706 * Make sure UIC command completion interrupt is disabled before
3707 * issuing UIC command.
3708 */
3709 wmb();
3710 reenable_intr = true;
57d104c1 3711 }
d75f7fe4
YG		 3712 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3713 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3714 if (ret) {
3715 dev_err(hba->dev,
3716 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3717 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3718 goto out;
3719 }
3720
57d104c1 3721 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3722 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3723 dev_err(hba->dev,
57d104c1
SJ		 3724 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3725 cmd->command, cmd->argument3);
53b3d9c3
SJ		 3726 ret = -ETIMEDOUT;
3727 goto out;
3728 }
3729
3730 status = ufshcd_get_upmcrs(hba);
3731 if (status != PWR_LOCAL) {
3732 dev_err(hba->dev,
479da360 3733 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3734 cmd->command, status);
53b3d9c3
SJ		 3735 ret = (status != PWR_OK) ? status : -1;
3736 }
3737out:
7942f7b5
VG		 3738 if (ret) {
3739 ufshcd_print_host_state(hba);
3740 ufshcd_print_pwr_info(hba);
3741 ufshcd_print_host_regs(hba);
3742 }
3743
53b3d9c3 3744 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3745 hba->active_uic_cmd = NULL;
57d104c1 3746 hba->uic_async_done = NULL;
d75f7fe4
YG		 3747 if (reenable_intr)
3748 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
53b3d9c3
SJ		 3749 spin_unlock_irqrestore(hba->host->host_lock, flags);
3750 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3751
53b3d9c3
SJ		 3752 return ret;
3753}
3754
57d104c1
SJ		 3755/**
3756 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3757 * using DME_SET primitives.
3758 * @hba: per adapter instance
3759 * @mode: powr mode value
3760 *
3761 * Returns 0 on success, non-zero value on failure
3762 */
3763static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3764{
3765 struct uic_command uic_cmd = {0};
1ab27c9c 3766 int ret;
57d104c1 3767
c3a2f9ee
YG		 3768 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3769 ret = ufshcd_dme_set(hba,
3770 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3771 if (ret) {
3772 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3773 __func__, ret);
3774 goto out;
3775 }
3776 }
3777
57d104c1
SJ		 3778 uic_cmd.command = UIC_CMD_DME_SET;
3779 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3780 uic_cmd.argument3 = mode;
1ab27c9c
ST		 3781 ufshcd_hold(hba, false);
3782 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3783 ufshcd_release(hba);
57d104c1 3784
c3a2f9ee 3785out:
1ab27c9c 3786 return ret;
57d104c1
SJ		 3787}
3788
53c12d0e
YG		 3789static int ufshcd_link_recovery(struct ufs_hba *hba)
3790{
3791 int ret;
3792 unsigned long flags;
3793
3794 spin_lock_irqsave(hba->host->host_lock, flags);
3795 hba->ufshcd_state = UFSHCD_STATE_RESET;
3796 ufshcd_set_eh_in_progress(hba);
3797 spin_unlock_irqrestore(hba->host->host_lock, flags);
3798
ebdd1dfd
CG		 3799 /* Reset the attached device */
3800 ufshcd_vops_device_reset(hba);
3801
53c12d0e
YG		 3802 ret = ufshcd_host_reset_and_restore(hba);
3803
3804 spin_lock_irqsave(hba->host->host_lock, flags);
3805 if (ret)
3806 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3807 ufshcd_clear_eh_in_progress(hba);
3808 spin_unlock_irqrestore(hba->host->host_lock, flags);
3809
3810 if (ret)
3811 dev_err(hba->dev, "%s: link recovery failed, err %d",
3812 __func__, ret);
3813
3814 return ret;
3815}
3816
87d0b4a6 3817static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 3818{
87d0b4a6 3819 int ret;
57d104c1 3820 struct uic_command uic_cmd = {0};
911a0771 3821 ktime_t start = ktime_get();
57d104c1 3822
ee32c909
KK		 3823 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3824
57d104c1 3825 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 3826 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3827 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3828 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 3829
53c12d0e 3830 if (ret) {
6d303e4b
SJ		 3831 int err;
3832
87d0b4a6
YG		 3833 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3834 __func__, ret);
3835
53c12d0e 3836 /*
6d303e4b
SJ		 3837 * If link recovery fails then return error code returned from
3838 * ufshcd_link_recovery().
3839 * If link recovery succeeds then return -EAGAIN to attempt
3840 * hibern8 enter retry again.
53c12d0e 3841 */
6d303e4b
SJ		 3842 err = ufshcd_link_recovery(hba);
3843 if (err) {
3844 dev_err(hba->dev, "%s: link recovery failed", __func__);
3845 ret = err;
3846 } else {
3847 ret = -EAGAIN;
3848 }
ee32c909
KK		 3849 } else
3850 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3851 POST_CHANGE);
53c12d0e 3852
87d0b4a6
YG		 3853 return ret;
3854}
3855
3856static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3857{
3858 int ret = 0, retries;
57d104c1 3859
87d0b4a6
YG		 3860 for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
3861 ret = __ufshcd_uic_hibern8_enter(hba);
6d303e4b 3862 if (!ret)
87d0b4a6
YG		 3863 goto out;
3864 }
3865out:
3866 return ret;
57d104c1
SJ		 3867}
3868
3869static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
3870{
3871 struct uic_command uic_cmd = {0};
3872 int ret;
911a0771 3873 ktime_t start = ktime_get();
57d104c1 3874
ee32c909
KK		 3875 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3876
57d104c1
SJ		 3877 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3878 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3879 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3880 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3881
57d104c1 3882 if (ret) {
53c12d0e
YG		 3883 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3884 __func__, ret);
3885 ret = ufshcd_link_recovery(hba);
ff8e20c6 3886 } else {
ee32c909
KK		 3887 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3888 POST_CHANGE);
ff8e20c6
DR		 3889 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3890 hba->ufs_stats.hibern8_exit_cnt++;
3891 }
57d104c1
SJ		 3892
3893 return ret;
3894}
3895
71d848b8 3896void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378
AH		 3897{
3898 unsigned long flags;
3899
ee5f1042 3900 if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
ad448378
AH		 3901 return;
3902
3903 spin_lock_irqsave(hba->host->host_lock, flags);
3904 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3905 spin_unlock_irqrestore(hba->host->host_lock, flags);
3906}
3907
5064636c
YG		 3908 /**
3909 * ufshcd_init_pwr_info - setting the POR (power on reset)
3910 * values in hba power info
3911 * @hba: per-adapter instance
3912 */
3913static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3914{
3915 hba->pwr_info.gear_rx = UFS_PWM_G1;
3916 hba->pwr_info.gear_tx = UFS_PWM_G1;
3917 hba->pwr_info.lane_rx = 1;
3918 hba->pwr_info.lane_tx = 1;
3919 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
3920 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
3921 hba->pwr_info.hs_rate = 0;
3922}
3923
d3e89bac 3924/**
7eb584db
DR		 3925 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3926 * @hba: per-adapter instance
d3e89bac 3927 */
7eb584db 3928static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 3929{
7eb584db
DR		 3930 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
3931
3932 if (hba->max_pwr_info.is_valid)
3933 return 0;
3934
2349b533 3935 pwr_info->pwr_tx = FAST_MODE;
3936 pwr_info->pwr_rx = FAST_MODE;
7eb584db 3937 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 3938
3939 /* Get the connected lane count */
7eb584db
DR		 3940 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
3941 &pwr_info->lane_rx);
3942 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3943 &pwr_info->lane_tx);
3944
3945 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
3946 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
3947 __func__,
3948 pwr_info->lane_rx,
3949 pwr_info->lane_tx);
3950 return -EINVAL;
3951 }
d3e89bac
SJ		 3952
3953 /*
3954 * First, get the maximum gears of HS speed.
3955 * If a zero value, it means there is no HSGEAR capability.
3956 * Then, get the maximum gears of PWM speed.
3957 */
7eb584db
DR		 3958 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
3959 if (!pwr_info->gear_rx) {
3960 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
3961 &pwr_info->gear_rx);
3962 if (!pwr_info->gear_rx) {
3963 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
3964 __func__, pwr_info->gear_rx);
3965 return -EINVAL;
3966 }
2349b533 3967 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 3968 }
3969
7eb584db
DR		 3970 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
3971 &pwr_info->gear_tx);
3972 if (!pwr_info->gear_tx) {
d3e89bac 3973 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 3974 &pwr_info->gear_tx);
3975 if (!pwr_info->gear_tx) {
3976 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
3977 __func__, pwr_info->gear_tx);
3978 return -EINVAL;
3979 }
2349b533 3980 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 3981 }
3982
3983 hba->max_pwr_info.is_valid = true;
3984 return 0;
3985}
3986
3987static int ufshcd_change_power_mode(struct ufs_hba *hba,
3988 struct ufs_pa_layer_attr *pwr_mode)
3989{
3990 int ret;
3991
3992 /* if already configured to the requested pwr_mode */
3993 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
3994 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
3995 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
3996 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
3997 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
3998 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
3999 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4000 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4001 return 0;
d3e89bac
SJ		 4002 }
4003
4004 /*
4005 * Configure attributes for power mode change with below.
4006 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4007 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4008 * - PA_HSSERIES
4009 */
7eb584db
DR		 4010 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4011 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4012 pwr_mode->lane_rx);
4013 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4014 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4015 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4016 else
4017 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4018
7eb584db
DR		 4019 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4020 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4021 pwr_mode->lane_tx);
4022 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4023 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4024 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4025 else
4026 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4027
7eb584db
DR		 4028 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4029 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4030 pwr_mode->pwr_rx == FAST_MODE ||
4031 pwr_mode->pwr_tx == FAST_MODE)
4032 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4033 pwr_mode->hs_rate);
d3e89bac 4034
7eb584db
DR		 4035 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4036 | pwr_mode->pwr_tx);
4037
4038 if (ret) {
d3e89bac 4039 dev_err(hba->dev,
7eb584db
DR		 4040 "%s: power mode change failed %d\n", __func__, ret);
4041 } else {
0263bcd0
YG		 4042 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4043 pwr_mode);
7eb584db
DR		 4044
4045 memcpy(&hba->pwr_info, pwr_mode,
4046 sizeof(struct ufs_pa_layer_attr));
4047 }
4048
4049 return ret;
4050}
4051
4052/**
4053 * ufshcd_config_pwr_mode - configure a new power mode
4054 * @hba: per-adapter instance
4055 * @desired_pwr_mode: desired power configuration
4056 */
0d846e70 4057int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4058 struct ufs_pa_layer_attr *desired_pwr_mode)
4059{
4060 struct ufs_pa_layer_attr final_params = { 0 };
4061 int ret;
4062
0263bcd0
YG		 4063 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4064 desired_pwr_mode, &final_params);
4065
4066 if (ret)
7eb584db
DR		 4067 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4068
4069 ret = ufshcd_change_power_mode(hba, &final_params);
a3cd5ec5 4070 if (!ret)
4071 ufshcd_print_pwr_info(hba);
d3e89bac
SJ		 4072
4073 return ret;
4074}
0d846e70 4075EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4076
68078d5c
DR		 4077/**
4078 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4079 * @hba: per-adapter instance
68078d5c
DR		 4080 *
4081 * Set fDeviceInit flag and poll until device toggles it.
4082 */
4083static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4084{
dc3c8d3a
YG		 4085 int i;
4086 int err;
68078d5c
DR		 4087 bool flag_res = 1;
4088
dc3c8d3a
YG		 4089 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4090 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
68078d5c
DR		 4091 if (err) {
4092 dev_err(hba->dev,
4093 "%s setting fDeviceInit flag failed with error %d\n",
4094 __func__, err);
4095 goto out;
4096 }
4097
dc3c8d3a
YG		 4098 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4099 for (i = 0; i < 1000 && !err && flag_res; i++)
4100 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4101 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
4102
68078d5c
DR		 4103 if (err)
4104 dev_err(hba->dev,
4105 "%s reading fDeviceInit flag failed with error %d\n",
4106 __func__, err);
4107 else if (flag_res)
4108 dev_err(hba->dev,
4109 "%s fDeviceInit was not cleared by the device\n",
4110 __func__);
4111
4112out:
4113 return err;
4114}
4115
7a3e97b0
SY		 4116/**
4117 * ufshcd_make_hba_operational - Make UFS controller operational
4118 * @hba: per adapter instance
4119 *
4120 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4121 * 1. Enable required interrupts
4122 * 2. Configure interrupt aggregation
897efe62 4123 * 3. Program UTRL and UTMRL base address
5c0c28a8 4124 * 4. Configure run-stop-registers
7a3e97b0
SY		 4125 *
4126 * Returns 0 on success, non-zero value on failure
4127 */
4128static int ufshcd_make_hba_operational(struct ufs_hba *hba)
4129{
4130 int err = 0;
4131 u32 reg;
4132
6ccf44fe
SJ		 4133 /* Enable required interrupts */
4134 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4135
4136 /* Configure interrupt aggregation */
b852190e
YG		 4137 if (ufshcd_is_intr_aggr_allowed(hba))
4138 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4139 else
4140 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4141
4142 /* Configure UTRL and UTMRL base address registers */
4143 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4144 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4145 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4146 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4147 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4148 REG_UTP_TASK_REQ_LIST_BASE_L);
4149 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4150 REG_UTP_TASK_REQ_LIST_BASE_H);
4151
897efe62
YG		 4152 /*
4153 * Make sure base address and interrupt setup are updated before
4154 * enabling the run/stop registers below.
4155 */
4156 wmb();
4157
7a3e97b0
SY		 4158 /*
4159 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4160 */
5c0c28a8 4161 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4162 if (!(ufshcd_get_lists_status(reg))) {
4163 ufshcd_enable_run_stop_reg(hba);
4164 } else {
3b1d0580 4165 dev_err(hba->dev,
7a3e97b0
SY		 4166 "Host controller not ready to process requests");
4167 err = -EIO;
4168 goto out;
4169 }
4170
7a3e97b0
SY		 4171out:
4172 return err;
4173}
4174
596585a2
YG		 4175/**
4176 * ufshcd_hba_stop - Send controller to reset state
4177 * @hba: per adapter instance
4178 * @can_sleep: perform sleep or just spin
4179 */
4180static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
4181{
4182 int err;
4183
4184 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4185 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4186 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4187 10, 1, can_sleep);
4188 if (err)
4189 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4190}
4191
7a3e97b0 4192/**
4404c5de 4193 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4194 * @hba: per adapter instance
4195 *
4196 * The controller resets itself and controller firmware initialization
4197 * sequence kicks off. When controller is ready it will set
4198 * the Host Controller Enable bit to 1.
4199 *
4200 * Returns 0 on success, non-zero value on failure
4201 */
4404c5de 4202static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0
SY		 4203{
4204 int retry;
4205
596585a2 4206 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4207 /* change controller state to "reset state" */
596585a2 4208 ufshcd_hba_stop(hba, true);
7a3e97b0 4209
57d104c1
SJ		 4210 /* UniPro link is disabled at this point */
4211 ufshcd_set_link_off(hba);
4212
0263bcd0 4213 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4214
7a3e97b0
SY		 4215 /* start controller initialization sequence */
4216 ufshcd_hba_start(hba);
4217
4218 /*
4219 * To initialize a UFS host controller HCE bit must be set to 1.
4220 * During initialization the HCE bit value changes from 1->0->1.
4221 * When the host controller completes initialization sequence
4222 * it sets the value of HCE bit to 1. The same HCE bit is read back
4223 * to check if the controller has completed initialization sequence.
4224 * So without this delay the value HCE = 1, set in the previous
4225 * instruction might be read back.
4226 * This delay can be changed based on the controller.
4227 */
838c1efc 4228 usleep_range(1000, 1100);
7a3e97b0
SY		 4229
4230 /* wait for the host controller to complete initialization */
4231 retry = 10;
4232 while (ufshcd_is_hba_active(hba)) {
4233 if (retry) {
4234 retry--;
4235 } else {
3b1d0580 4236 dev_err(hba->dev,
7a3e97b0
SY		 4237 "Controller enable failed\n");
4238 return -EIO;
4239 }
838c1efc 4240 usleep_range(5000, 5100);
7a3e97b0 4241 }
5c0c28a8 4242
1d337ec2 4243 /* enable UIC related interrupts */
57d104c1 4244 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4245
0263bcd0 4246 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4247
7a3e97b0
SY		 4248 return 0;
4249}
4250
4404c5de
AA		 4251static int ufshcd_hba_enable(struct ufs_hba *hba)
4252{
4253 int ret;
4254
4255 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4256 ufshcd_set_link_off(hba);
4257 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4258
4259 /* enable UIC related interrupts */
4260 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4261 ret = ufshcd_dme_reset(hba);
4262 if (!ret) {
4263 ret = ufshcd_dme_enable(hba);
4264 if (!ret)
4265 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4266 if (ret)
4267 dev_err(hba->dev,
4268 "Host controller enable failed with non-hce\n");
4269 }
4270 } else {
4271 ret = ufshcd_hba_execute_hce(hba);
4272 }
4273
4274 return ret;
4275}
7ca38cf3
YG		 4276static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4277{
4278 int tx_lanes, i, err = 0;
4279
4280 if (!peer)
4281 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4282 &tx_lanes);
4283 else
4284 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4285 &tx_lanes);
4286 for (i = 0; i < tx_lanes; i++) {
4287 if (!peer)
4288 err = ufshcd_dme_set(hba,
4289 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4290 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4291 0);
4292 else
4293 err = ufshcd_dme_peer_set(hba,
4294 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4295 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4296 0);
4297 if (err) {
4298 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4299 __func__, peer, i, err);
4300 break;
4301 }
4302 }
4303
4304 return err;
4305}
4306
4307static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4308{
4309 return ufshcd_disable_tx_lcc(hba, true);
4310}
4311
8808b4e9
SC		 4312static void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4313 u32 reg)
4314{
4315 reg_hist->reg[reg_hist->pos] = reg;
4316 reg_hist->tstamp[reg_hist->pos] = ktime_get();
4317 reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4318}
4319
7a3e97b0 4320/**
6ccf44fe 4321 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4322 * @hba: per adapter instance
4323 *
6ccf44fe 4324 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4325 */
6ccf44fe 4326static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4327{
6ccf44fe 4328 int ret;
1d337ec2 4329 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4330 bool link_startup_again = false;
7a3e97b0 4331
7caf489b 4332 /*
4333 * If UFS device isn't active then we will have to issue link startup
4334 * 2 times to make sure the device state move to active.
4335 */
4336 if (!ufshcd_is_ufs_dev_active(hba))
4337 link_startup_again = true;
7a3e97b0 4338
7caf489b 4339link_startup:
1d337ec2 4340 do {
0263bcd0 4341 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4342
1d337ec2 4343 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4344
1d337ec2
SRT		 4345 /* check if device is detected by inter-connect layer */
4346 if (!ret && !ufshcd_is_device_present(hba)) {
8808b4e9
SC		 4347 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4348 0);
1d337ec2
SRT		 4349 dev_err(hba->dev, "%s: Device not present\n", __func__);
4350 ret = -ENXIO;
4351 goto out;
4352 }
6ccf44fe 4353
1d337ec2
SRT		 4354 /*
4355 * DME link lost indication is only received when link is up,
4356 * but we can't be sure if the link is up until link startup
4357 * succeeds. So reset the local Uni-Pro and try again.
4358 */
8808b4e9
SC		 4359 if (ret && ufshcd_hba_enable(hba)) {
4360 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4361 (u32)ret);
1d337ec2 4362 goto out;
8808b4e9 4363 }
1d337ec2
SRT		 4364 } while (ret && retries--);
4365
8808b4e9 4366 if (ret) {
1d337ec2 4367 /* failed to get the link up... retire */
8808b4e9
SC		 4368 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4369 (u32)ret);
5c0c28a8 4370 goto out;
8808b4e9 4371 }
5c0c28a8 4372
7caf489b 4373 if (link_startup_again) {
4374 link_startup_again = false;
4375 retries = DME_LINKSTARTUP_RETRIES;
4376 goto link_startup;
4377 }
4378
d2aebb9b 4379 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4380 ufshcd_init_pwr_info(hba);
4381 ufshcd_print_pwr_info(hba);
4382
7ca38cf3
YG		 4383 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4384 ret = ufshcd_disable_device_tx_lcc(hba);
4385 if (ret)
4386 goto out;
4387 }
4388
5c0c28a8 4389 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4390 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4391 if (ret)
4392 goto out;
7a3e97b0 4393
5c0c28a8 4394 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4395out:
7942f7b5 4396 if (ret) {
6ccf44fe 4397 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4398 ufshcd_print_host_state(hba);
4399 ufshcd_print_pwr_info(hba);
4400 ufshcd_print_host_regs(hba);
4401 }
6ccf44fe 4402 return ret;
7a3e97b0
SY		 4403}
4404
5a0b0cb9
SRT		 4405/**
4406 * ufshcd_verify_dev_init() - Verify device initialization
4407 * @hba: per-adapter instance
4408 *
4409 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4410 * device Transport Protocol (UTP) layer is ready after a reset.
4411 * If the UTP layer at the device side is not initialized, it may
4412 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4413 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4414 */
4415static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4416{
4417 int err = 0;
4418 int retries;
4419
1ab27c9c 4420 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4421 mutex_lock(&hba->dev_cmd.lock);
4422 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4423 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4424 NOP_OUT_TIMEOUT);
4425
4426 if (!err || err == -ETIMEDOUT)
4427 break;
4428
4429 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4430 }
4431 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4432 ufshcd_release(hba);
5a0b0cb9
SRT		 4433
4434 if (err)
4435 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4436 return err;
4437}
4438
0ce147d4
SJ		 4439/**
4440 * ufshcd_set_queue_depth - set lun queue depth
4441 * @sdev: pointer to SCSI device
4442 *
4443 * Read bLUQueueDepth value and activate scsi tagged command
4444 * queueing. For WLUN, queue depth is set to 1. For best-effort
4445 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4446 * value that host can queue.
4447 */
4448static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4449{
4450 int ret = 0;
4451 u8 lun_qdepth;
4452 struct ufs_hba *hba;
4453
4454 hba = shost_priv(sdev->host);
4455
4456 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4457 ret = ufshcd_read_unit_desc_param(hba,
4458 ufshcd_scsi_to_upiu_lun(sdev->lun),
4459 UNIT_DESC_PARAM_LU_Q_DEPTH,
4460 &lun_qdepth,
4461 sizeof(lun_qdepth));
0ce147d4
SJ		 4462
4463 /* Some WLUN doesn't support unit descriptor */
4464 if (ret == -EOPNOTSUPP)
4465 lun_qdepth = 1;
4466 else if (!lun_qdepth)
4467 /* eventually, we can figure out the real queue depth */
4468 lun_qdepth = hba->nutrs;
4469 else
4470 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4471
4472 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4473 __func__, lun_qdepth);
db5ed4df 4474 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4475}
4476
57d104c1
SJ		 4477/*
4478 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4479 * @hba: per-adapter instance
4480 * @lun: UFS device lun id
4481 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4482 *
4483 * Returns 0 in case of success and b_lu_write_protect status would be returned
4484 * @b_lu_write_protect parameter.
4485 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4486 * Returns -EINVAL in case of invalid parameters passed to this function.
4487 */
4488static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4489 u8 lun,
4490 u8 *b_lu_write_protect)
4491{
4492 int ret;
4493
4494 if (!b_lu_write_protect)
4495 ret = -EINVAL;
4496 /*
4497 * According to UFS device spec, RPMB LU can't be write
4498 * protected so skip reading bLUWriteProtect parameter for
4499 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4500 */
4501 else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
4502 ret = -ENOTSUPP;
4503 else
4504 ret = ufshcd_read_unit_desc_param(hba,
4505 lun,
4506 UNIT_DESC_PARAM_LU_WR_PROTECT,
4507 b_lu_write_protect,
4508 sizeof(*b_lu_write_protect));
4509 return ret;
4510}
4511
4512/**
4513 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4514 * status
4515 * @hba: per-adapter instance
4516 * @sdev: pointer to SCSI device
4517 *
4518 */
4519static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4520 struct scsi_device *sdev)
4521{
4522 if (hba->dev_info.f_power_on_wp_en &&
4523 !hba->dev_info.is_lu_power_on_wp) {
4524 u8 b_lu_write_protect;
4525
4526 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4527 &b_lu_write_protect) &&
4528 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4529 hba->dev_info.is_lu_power_on_wp = true;
4530 }
4531}
4532
7a3e97b0
SY		 4533/**
4534 * ufshcd_slave_alloc - handle initial SCSI device configurations
4535 * @sdev: pointer to SCSI device
4536 *
4537 * Returns success
4538 */
4539static int ufshcd_slave_alloc(struct scsi_device *sdev)
4540{
4541 struct ufs_hba *hba;
4542
4543 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4544
4545 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4546 sdev->use_10_for_ms = 1;
7a3e97b0 4547
e8e7f271
SRT		 4548 /* allow SCSI layer to restart the device in case of errors */
4549 sdev->allow_restart = 1;
4264fd61 4550
b2a6c522
SRT		 4551 /* REPORT SUPPORTED OPERATION CODES is not supported */
4552 sdev->no_report_opcodes = 1;
4553
84af7e8b
SRT		 4554 /* WRITE_SAME command is not supported */
4555 sdev->no_write_same = 1;
e8e7f271 4556
0ce147d4 4557 ufshcd_set_queue_depth(sdev);
4264fd61 4558
57d104c1
SJ		 4559 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4560
7a3e97b0
SY		 4561 return 0;
4562}
4563
4264fd61
SRT		 4564/**
4565 * ufshcd_change_queue_depth - change queue depth
4566 * @sdev: pointer to SCSI device
4567 * @depth: required depth to set
4264fd61 4568 *
db5ed4df 4569 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4570 */
db5ed4df 4571static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4572{
4573 struct ufs_hba *hba = shost_priv(sdev->host);
4574
4575 if (depth > hba->nutrs)
4576 depth = hba->nutrs;
db5ed4df 4577 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4578}
4579
eeda4749
AM		 4580/**
4581 * ufshcd_slave_configure - adjust SCSI device configurations
4582 * @sdev: pointer to SCSI device
4583 */
4584static int ufshcd_slave_configure(struct scsi_device *sdev)
4585{
49615ba1 4586 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 4587 struct request_queue *q = sdev->request_queue;
4588
4589 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
49615ba1
SC		 4590
4591 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4592 sdev->rpm_autosuspend = 1;
4593
eeda4749
AM		 4594 return 0;
4595}
4596
7a3e97b0
SY		 4597/**
4598 * ufshcd_slave_destroy - remove SCSI device configurations
4599 * @sdev: pointer to SCSI device
4600 */
4601static void ufshcd_slave_destroy(struct scsi_device *sdev)
4602{
4603 struct ufs_hba *hba;
4604
4605 hba = shost_priv(sdev->host);
0ce147d4 4606 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4607 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4608 unsigned long flags;
4609
4610 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4611 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4612 spin_unlock_irqrestore(hba->host->host_lock, flags);
4613 }
7a3e97b0
SY		 4614}
4615
7a3e97b0
SY		 4616/**
4617 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4618 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4619 * @scsi_status: SCSI command status
4620 *
4621 * Returns value base on SCSI command status
4622 */
4623static inline int
4624ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4625{
4626 int result = 0;
4627
4628 switch (scsi_status) {
7a3e97b0 4629 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4630 ufshcd_copy_sense_data(lrbp);
30eb2e4c 4631 /* fallthrough */
1c2623c5 4632 case SAM_STAT_GOOD:
7a3e97b0
SY		 4633 result |= DID_OK << 16 |
4634 COMMAND_COMPLETE << 8 |
1c2623c5 4635 scsi_status;
7a3e97b0
SY		 4636 break;
4637 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4638 case SAM_STAT_BUSY:
7a3e97b0 4639 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4640 ufshcd_copy_sense_data(lrbp);
4641 result |= scsi_status;
7a3e97b0
SY		 4642 break;
4643 default:
4644 result |= DID_ERROR << 16;
4645 break;
4646 } /* end of switch */
4647
4648 return result;
4649}
4650
4651/**
4652 * ufshcd_transfer_rsp_status - Get overall status of the response
4653 * @hba: per adapter instance
8aa29f19 4654 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4655 *
4656 * Returns result of the command to notify SCSI midlayer
4657 */
4658static inline int
4659ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4660{
4661 int result = 0;
4662 int scsi_status;
4663 int ocs;
4664
4665 /* overall command status of utrd */
4666 ocs = ufshcd_get_tr_ocs(lrbp);
4667
4668 switch (ocs) {
4669 case OCS_SUCCESS:
5a0b0cb9 4670 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4671 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4672 switch (result) {
4673 case UPIU_TRANSACTION_RESPONSE:
4674 /*
4675 * get the response UPIU result to extract
4676 * the SCSI command status
4677 */
4678 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4679
4680 /*
4681 * get the result based on SCSI status response
4682 * to notify the SCSI midlayer of the command status
4683 */
4684 scsi_status = result & MASK_SCSI_STATUS;
4685 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4686
f05ac2e5
YG		 4687 /*
4688 * Currently we are only supporting BKOPs exception
4689 * events hence we can ignore BKOPs exception event
4690 * during power management callbacks. BKOPs exception
4691 * event is not expected to be raised in runtime suspend
4692 * callback as it allows the urgent bkops.
4693 * During system suspend, we are anyway forcefully
4694 * disabling the bkops and if urgent bkops is needed
4695 * it will be enabled on system resume. Long term
4696 * solution could be to abort the system suspend if
4697 * UFS device needs urgent BKOPs.
4698 */
4699 if (!hba->pm_op_in_progress &&
4700 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
66ec6d59 4701 schedule_work(&hba->eeh_work);
5a0b0cb9
SRT		 4702 break;
4703 case UPIU_TRANSACTION_REJECT_UPIU:
4704 /* TODO: handle Reject UPIU Response */
4705 result = DID_ERROR << 16;
3b1d0580 4706 dev_err(hba->dev,
5a0b0cb9
SRT		 4707 "Reject UPIU not fully implemented\n");
4708 break;
4709 default:
5a0b0cb9
SRT		 4710 dev_err(hba->dev,
4711 "Unexpected request response code = %x\n",
4712 result);
e0347d89 4713 result = DID_ERROR << 16;
7a3e97b0
SY		 4714 break;
4715 }
7a3e97b0
SY		 4716 break;
4717 case OCS_ABORTED:
4718 result |= DID_ABORT << 16;
4719 break;
e8e7f271
SRT		 4720 case OCS_INVALID_COMMAND_STATUS:
4721 result |= DID_REQUEUE << 16;
4722 break;
7a3e97b0
SY		 4723 case OCS_INVALID_CMD_TABLE_ATTR:
4724 case OCS_INVALID_PRDT_ATTR:
4725 case OCS_MISMATCH_DATA_BUF_SIZE:
4726 case OCS_MISMATCH_RESP_UPIU_SIZE:
4727 case OCS_PEER_COMM_FAILURE:
4728 case OCS_FATAL_ERROR:
4729 default:
4730 result |= DID_ERROR << 16;
3b1d0580 4731 dev_err(hba->dev,
ff8e20c6
DR		 4732 "OCS error from controller = %x for tag %d\n",
4733 ocs, lrbp->task_tag);
4734 ufshcd_print_host_regs(hba);
6ba65588 4735 ufshcd_print_host_state(hba);
7a3e97b0
SY		 4736 break;
4737 } /* end of switch */
4738
2df74b69 4739 if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
66cc820f 4740 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 4741 return result;
4742}
4743
6ccf44fe
SJ		 4744/**
4745 * ufshcd_uic_cmd_compl - handle completion of uic command
4746 * @hba: per adapter instance
53b3d9c3 4747 * @intr_status: interrupt status generated by the controller
9333d775
VG		 4748 *
4749 * Returns
4750 * IRQ_HANDLED - If interrupt is valid
4751 * IRQ_NONE - If invalid interrupt
6ccf44fe 4752 */
9333d775 4753static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 4754{
9333d775
VG		 4755 irqreturn_t retval = IRQ_NONE;
4756
53b3d9c3 4757 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 4758 hba->active_uic_cmd->argument2 |=
4759 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 4760 hba->active_uic_cmd->argument3 =
4761 ufshcd_get_dme_attr_val(hba);
6ccf44fe 4762 complete(&hba->active_uic_cmd->done);
9333d775 4763 retval = IRQ_HANDLED;
6ccf44fe 4764 }
53b3d9c3 4765
9333d775 4766 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
57d104c1 4767 complete(hba->uic_async_done);
9333d775
VG		 4768 retval = IRQ_HANDLED;
4769 }
4770 return retval;
6ccf44fe
SJ		 4771}
4772
7a3e97b0 4773/**
9a47ec7c 4774 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 4775 * @hba: per adapter instance
9a47ec7c 4776 * @completed_reqs: requests to complete
7a3e97b0 4777 */
9a47ec7c
YG		 4778static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4779 unsigned long completed_reqs)
7a3e97b0 4780{
5a0b0cb9
SRT		 4781 struct ufshcd_lrb *lrbp;
4782 struct scsi_cmnd *cmd;
7a3e97b0
SY		 4783 int result;
4784 int index;
e9d501b1 4785
e9d501b1
DR		 4786 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4787 lrbp = &hba->lrb[index];
4788 cmd = lrbp->cmd;
4789 if (cmd) {
1a07f2d9 4790 ufshcd_add_command_trace(hba, index, "complete");
e9d501b1
DR		 4791 result = ufshcd_transfer_rsp_status(hba, lrbp);
4792 scsi_dma_unmap(cmd);
4793 cmd->result = result;
4794 /* Mark completed command as NULL in LRB */
4795 lrbp->cmd = NULL;
74a527a2 4796 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 4797 /* Do not touch lrbp after scsi done */
4798 cmd->scsi_done(cmd);
1ab27c9c 4799 __ufshcd_release(hba);
300bb13f
JP		 4800 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4801 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
74a527a2 4802 lrbp->compl_time_stamp = ktime_get();
1a07f2d9
LS		 4803 if (hba->dev_cmd.complete) {
4804 ufshcd_add_command_trace(hba, index,
4805 "dev_complete");
e9d501b1 4806 complete(hba->dev_cmd.complete);
1a07f2d9 4807 }
e9d501b1 4808 }
401f1e44 4809 if (ufshcd_is_clkscaling_supported(hba))
4810 hba->clk_scaling.active_reqs--;
e9d501b1 4811 }
7a3e97b0
SY		 4812
4813 /* clear corresponding bits of completed commands */
4814 hba->outstanding_reqs ^= completed_reqs;
4815
856b3483 4816 ufshcd_clk_scaling_update_busy(hba);
7a3e97b0
SY		 4817}
4818
9a47ec7c
YG		 4819/**
4820 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4821 * @hba: per adapter instance
9333d775
VG		 4822 *
4823 * Returns
4824 * IRQ_HANDLED - If interrupt is valid
4825 * IRQ_NONE - If invalid interrupt
9a47ec7c 4826 */
9333d775 4827static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c
YG		 4828{
4829 unsigned long completed_reqs;
4830 u32 tr_doorbell;
4831
4832 /* Resetting interrupt aggregation counters first and reading the
4833 * DOOR_BELL afterward allows us to handle all the completed requests.
4834 * In order to prevent other interrupts starvation the DB is read once
4835 * after reset. The down side of this solution is the possibility of
4836 * false interrupt if device completes another request after resetting
4837 * aggregation and before reading the DB.
4838 */
5ac6abc9
AA		 4839 if (ufshcd_is_intr_aggr_allowed(hba) &&
4840 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 4841 ufshcd_reset_intr_aggr(hba);
4842
4843 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4844 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4845
9333d775
VG		 4846 if (completed_reqs) {
4847 __ufshcd_transfer_req_compl(hba, completed_reqs);
4848 return IRQ_HANDLED;
4849 } else {
4850 return IRQ_NONE;
4851 }
9a47ec7c
YG		 4852}
4853
66ec6d59
SRT		 4854/**
4855 * ufshcd_disable_ee - disable exception event
4856 * @hba: per-adapter instance
4857 * @mask: exception event to disable
4858 *
4859 * Disables exception event in the device so that the EVENT_ALERT
4860 * bit is not set.
4861 *
4862 * Returns zero on success, non-zero error value on failure.
4863 */
4864static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
4865{
4866 int err = 0;
4867 u32 val;
4868
4869 if (!(hba->ee_ctrl_mask & mask))
4870 goto out;
4871
4872 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 4873 val &= MASK_EE_STATUS;
5e86ae44 4874 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4875 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4876 if (!err)
4877 hba->ee_ctrl_mask &= ~mask;
4878out:
4879 return err;
4880}
4881
4882/**
4883 * ufshcd_enable_ee - enable exception event
4884 * @hba: per-adapter instance
4885 * @mask: exception event to enable
4886 *
4887 * Enable corresponding exception event in the device to allow
4888 * device to alert host in critical scenarios.
4889 *
4890 * Returns zero on success, non-zero error value on failure.
4891 */
4892static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
4893{
4894 int err = 0;
4895 u32 val;
4896
4897 if (hba->ee_ctrl_mask & mask)
4898 goto out;
4899
4900 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 4901 val &= MASK_EE_STATUS;
5e86ae44 4902 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4903 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4904 if (!err)
4905 hba->ee_ctrl_mask |= mask;
4906out:
4907 return err;
4908}
4909
4910/**
4911 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4912 * @hba: per-adapter instance
4913 *
4914 * Allow device to manage background operations on its own. Enabling
4915 * this might lead to inconsistent latencies during normal data transfers
4916 * as the device is allowed to manage its own way of handling background
4917 * operations.
4918 *
4919 * Returns zero on success, non-zero on failure.
4920 */
4921static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
4922{
4923 int err = 0;
4924
4925 if (hba->auto_bkops_enabled)
4926 goto out;
4927
dc3c8d3a 4928 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
66ec6d59
SRT		 4929 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4930 if (err) {
4931 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
4932 __func__, err);
4933 goto out;
4934 }
4935
4936 hba->auto_bkops_enabled = true;
7ff5ab47 4937 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 4938
4939 /* No need of URGENT_BKOPS exception from the device */
4940 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4941 if (err)
4942 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
4943 __func__, err);
4944out:
4945 return err;
4946}
4947
4948/**
4949 * ufshcd_disable_auto_bkops - block device in doing background operations
4950 * @hba: per-adapter instance
4951 *
4952 * Disabling background operations improves command response latency but
4953 * has drawback of device moving into critical state where the device is
4954 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4955 * host is idle so that BKOPS are managed effectively without any negative
4956 * impacts.
4957 *
4958 * Returns zero on success, non-zero on failure.
4959 */
4960static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
4961{
4962 int err = 0;
4963
4964 if (!hba->auto_bkops_enabled)
4965 goto out;
4966
4967 /*
4968 * If host assisted BKOPs is to be enabled, make sure
4969 * urgent bkops exception is allowed.
4970 */
4971 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
4972 if (err) {
4973 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
4974 __func__, err);
4975 goto out;
4976 }
4977
dc3c8d3a 4978 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
66ec6d59
SRT		 4979 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4980 if (err) {
4981 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
4982 __func__, err);
4983 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4984 goto out;
4985 }
4986
4987 hba->auto_bkops_enabled = false;
7ff5ab47 4988 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 4989 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 4990out:
4991 return err;
4992}
4993
4994/**
4e768e76 4995 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 4996 * @hba: per adapter instance
4997 *
4998 * After a device reset the device may toggle the BKOPS_EN flag
4999 * to default value. The s/w tracking variables should be updated
4e768e76 5000 * as well. This function would change the auto-bkops state based on
5001 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5002 */
4e768e76 5003static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5004{
4e768e76 5005 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5006 hba->auto_bkops_enabled = false;
5007 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5008 ufshcd_enable_auto_bkops(hba);
5009 } else {
5010 hba->auto_bkops_enabled = true;
5011 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5012 ufshcd_disable_auto_bkops(hba);
5013 }
24366c2a 5014 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5015}
5016
5017static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5018{
5e86ae44 5019 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5020 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5021}
5022
5023/**
57d104c1 5024 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5025 * @hba: per-adapter instance
57d104c1 5026 * @status: bkops_status value
66ec6d59 5027 *
57d104c1
SJ		 5028 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5029 * flag in the device to permit background operations if the device
5030 * bkops_status is greater than or equal to "status" argument passed to
5031 * this function, disable otherwise.
5032 *
5033 * Returns 0 for success, non-zero in case of failure.
5034 *
5035 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5036 * to know whether auto bkops is enabled or disabled after this function
5037 * returns control to it.
66ec6d59 5038 */
57d104c1
SJ		 5039static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5040 enum bkops_status status)
66ec6d59
SRT		 5041{
5042 int err;
57d104c1 5043 u32 curr_status = 0;
66ec6d59 5044
57d104c1 5045 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5046 if (err) {
5047 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5048 __func__, err);
5049 goto out;
57d104c1
SJ		 5050 } else if (curr_status > BKOPS_STATUS_MAX) {
5051 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5052 __func__, curr_status);
5053 err = -EINVAL;
5054 goto out;
66ec6d59
SRT		 5055 }
5056
57d104c1 5057 if (curr_status >= status)
66ec6d59 5058 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5059 else
5060 err = ufshcd_disable_auto_bkops(hba);
24366c2a 5061 hba->urgent_bkops_lvl = curr_status;
66ec6d59
SRT		 5062out:
5063 return err;
5064}
5065
57d104c1
SJ		 5066/**
5067 * ufshcd_urgent_bkops - handle urgent bkops exception event
5068 * @hba: per-adapter instance
5069 *
5070 * Enable fBackgroundOpsEn flag in the device to permit background
5071 * operations.
5072 *
5073 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5074 * and negative error value for any other failure.
5075 */
5076static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5077{
afdfff59 5078 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5079}
5080
66ec6d59
SRT		 5081static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5082{
5e86ae44 5083 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5084 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5085}
5086
afdfff59
YG		 5087static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5088{
5089 int err;
5090 u32 curr_status = 0;
5091
5092 if (hba->is_urgent_bkops_lvl_checked)
5093 goto enable_auto_bkops;
5094
5095 err = ufshcd_get_bkops_status(hba, &curr_status);
5096 if (err) {
5097 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5098 __func__, err);
5099 goto out;
5100 }
5101
5102 /*
5103 * We are seeing that some devices are raising the urgent bkops
5104 * exception events even when BKOPS status doesn't indicate performace
5105 * impacted or critical. Handle these device by determining their urgent
5106 * bkops status at runtime.
5107 */
5108 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5109 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5110 __func__, curr_status);
5111 /* update the current status as the urgent bkops level */
5112 hba->urgent_bkops_lvl = curr_status;
5113 hba->is_urgent_bkops_lvl_checked = true;
5114 }
5115
5116enable_auto_bkops:
5117 err = ufshcd_enable_auto_bkops(hba);
5118out:
5119 if (err < 0)
5120 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5121 __func__, err);
5122}
5123
66ec6d59
SRT		 5124/**
5125 * ufshcd_exception_event_handler - handle exceptions raised by device
5126 * @work: pointer to work data
5127 *
5128 * Read bExceptionEventStatus attribute from the device and handle the
5129 * exception event accordingly.
5130 */
5131static void ufshcd_exception_event_handler(struct work_struct *work)
5132{
5133 struct ufs_hba *hba;
5134 int err;
5135 u32 status = 0;
5136 hba = container_of(work, struct ufs_hba, eeh_work);
5137
62694735 5138 pm_runtime_get_sync(hba->dev);
2e3611e9 5139 scsi_block_requests(hba->host);
66ec6d59
SRT		 5140 err = ufshcd_get_ee_status(hba, &status);
5141 if (err) {
5142 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5143 __func__, err);
5144 goto out;
5145 }
5146
5147 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5148
5149 if (status & MASK_EE_URGENT_BKOPS)
5150 ufshcd_bkops_exception_event_handler(hba);
5151
66ec6d59 5152out:
2e3611e9 5153 scsi_unblock_requests(hba->host);
62694735 5154 pm_runtime_put_sync(hba->dev);
66ec6d59
SRT		 5155 return;
5156}
5157
9a47ec7c
YG		 5158/* Complete requests that have door-bell cleared */
5159static void ufshcd_complete_requests(struct ufs_hba *hba)
5160{
5161 ufshcd_transfer_req_compl(hba);
5162 ufshcd_tmc_handler(hba);
5163}
5164
583fa62d
YG		 5165/**
5166 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5167 * to recover from the DL NAC errors or not.
5168 * @hba: per-adapter instance
5169 *
5170 * Returns true if error handling is required, false otherwise
5171 */
5172static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5173{
5174 unsigned long flags;
5175 bool err_handling = true;
5176
5177 spin_lock_irqsave(hba->host->host_lock, flags);
5178 /*
5179 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5180 * device fatal error and/or DL NAC & REPLAY timeout errors.
5181 */
5182 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5183 goto out;
5184
5185 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5186 ((hba->saved_err & UIC_ERROR) &&
5187 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5188 goto out;
5189
5190 if ((hba->saved_err & UIC_ERROR) &&
5191 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5192 int err;
5193 /*
5194 * wait for 50ms to see if we can get any other errors or not.
5195 */
5196 spin_unlock_irqrestore(hba->host->host_lock, flags);
5197 msleep(50);
5198 spin_lock_irqsave(hba->host->host_lock, flags);
5199
5200 /*
5201 * now check if we have got any other severe errors other than
5202 * DL NAC error?
5203 */
5204 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5205 ((hba->saved_err & UIC_ERROR) &&
5206 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5207 goto out;
5208
5209 /*
5210 * As DL NAC is the only error received so far, send out NOP
5211 * command to confirm if link is still active or not.
5212 * - If we don't get any response then do error recovery.
5213 * - If we get response then clear the DL NAC error bit.
5214 */
5215
5216 spin_unlock_irqrestore(hba->host->host_lock, flags);
5217 err = ufshcd_verify_dev_init(hba);
5218 spin_lock_irqsave(hba->host->host_lock, flags);
5219
5220 if (err)
5221 goto out;
5222
5223 /* Link seems to be alive hence ignore the DL NAC errors */
5224 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5225 hba->saved_err &= ~UIC_ERROR;
5226 /* clear NAC error */
5227 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5228 if (!hba->saved_uic_err) {
5229 err_handling = false;
5230 goto out;
5231 }
5232 }
5233out:
5234 spin_unlock_irqrestore(hba->host->host_lock, flags);
5235 return err_handling;
5236}
5237
7a3e97b0 5238/**
e8e7f271
SRT		 5239 * ufshcd_err_handler - handle UFS errors that require s/w attention
5240 * @work: pointer to work structure
7a3e97b0 5241 */
e8e7f271 5242static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5243{
5244 struct ufs_hba *hba;
e8e7f271
SRT		 5245 unsigned long flags;
5246 u32 err_xfer = 0;
5247 u32 err_tm = 0;
5248 int err = 0;
5249 int tag;
9a47ec7c 5250 bool needs_reset = false;
e8e7f271
SRT		 5251
5252 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5253
62694735 5254 pm_runtime_get_sync(hba->dev);
1ab27c9c 5255 ufshcd_hold(hba, false);
e8e7f271
SRT		 5256
5257 spin_lock_irqsave(hba->host->host_lock, flags);
9a47ec7c 5258 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
e8e7f271 5259 goto out;
e8e7f271
SRT		 5260
5261 hba->ufshcd_state = UFSHCD_STATE_RESET;
5262 ufshcd_set_eh_in_progress(hba);
5263
5264 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5265 ufshcd_complete_requests(hba);
583fa62d
YG		 5266
5267 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5268 bool ret;
5269
5270 spin_unlock_irqrestore(hba->host->host_lock, flags);
5271 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5272 ret = ufshcd_quirk_dl_nac_errors(hba);
5273 spin_lock_irqsave(hba->host->host_lock, flags);
5274 if (!ret)
5275 goto skip_err_handling;
5276 }
9a47ec7c 5277 if ((hba->saved_err & INT_FATAL_ERRORS) ||
82174440 5278 (hba->saved_err & UFSHCD_UIC_HIBERN8_MASK) ||
9a47ec7c
YG		 5279 ((hba->saved_err & UIC_ERROR) &&
5280 (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
5281 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5282 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5283 needs_reset = true;
e8e7f271 5284
9a47ec7c
YG		 5285 /*
5286 * if host reset is required then skip clearing the pending
2df74b69
CG		 5287 * transfers forcefully because they will get cleared during
5288 * host reset and restore
9a47ec7c
YG		 5289 */
5290 if (needs_reset)
5291 goto skip_pending_xfer_clear;
5292
5293 /* release lock as clear command might sleep */
5294 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5295 /* Clear pending transfer requests */
9a47ec7c
YG		 5296 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5297 if (ufshcd_clear_cmd(hba, tag)) {
5298 err_xfer = true;
5299 goto lock_skip_pending_xfer_clear;
5300 }
5301 }
e8e7f271
SRT		 5302
5303 /* Clear pending task management requests */
9a47ec7c
YG		 5304 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5305 if (ufshcd_clear_tm_cmd(hba, tag)) {
5306 err_tm = true;
5307 goto lock_skip_pending_xfer_clear;
5308 }
5309 }
e8e7f271 5310
9a47ec7c 5311lock_skip_pending_xfer_clear:
e8e7f271 5312 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5313
9a47ec7c
YG		 5314 /* Complete the requests that are cleared by s/w */
5315 ufshcd_complete_requests(hba);
5316
5317 if (err_xfer || err_tm)
5318 needs_reset = true;
5319
5320skip_pending_xfer_clear:
e8e7f271 5321 /* Fatal errors need reset */
9a47ec7c
YG		 5322 if (needs_reset) {
5323 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5324
5325 /*
5326 * ufshcd_reset_and_restore() does the link reinitialization
5327 * which will need atleast one empty doorbell slot to send the
5328 * device management commands (NOP and query commands).
5329 * If there is no slot empty at this moment then free up last
5330 * slot forcefully.
5331 */
5332 if (hba->outstanding_reqs == max_doorbells)
5333 __ufshcd_transfer_req_compl(hba,
5334 (1UL << (hba->nutrs - 1)));
5335
5336 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5337 err = ufshcd_reset_and_restore(hba);
9a47ec7c 5338 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271
SRT		 5339 if (err) {
5340 dev_err(hba->dev, "%s: reset and restore failed\n",
5341 __func__);
5342 hba->ufshcd_state = UFSHCD_STATE_ERROR;
5343 }
5344 /*
5345 * Inform scsi mid-layer that we did reset and allow to handle
5346 * Unit Attention properly.
5347 */
5348 scsi_report_bus_reset(hba->host, 0);
5349 hba->saved_err = 0;
5350 hba->saved_uic_err = 0;
5351 }
9a47ec7c 5352
583fa62d 5353skip_err_handling:
9a47ec7c
YG		 5354 if (!needs_reset) {
5355 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5356 if (hba->saved_err || hba->saved_uic_err)
5357 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5358 __func__, hba->saved_err, hba->saved_uic_err);
5359 }
5360
e8e7f271
SRT		 5361 ufshcd_clear_eh_in_progress(hba);
5362
5363out:
9a47ec7c 5364 spin_unlock_irqrestore(hba->host->host_lock, flags);
38135535 5365 ufshcd_scsi_unblock_requests(hba);
1ab27c9c 5366 ufshcd_release(hba);
62694735 5367 pm_runtime_put_sync(hba->dev);
7a3e97b0
SY		 5368}
5369
5370/**
e8e7f271
SRT		 5371 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5372 * @hba: per-adapter instance
9333d775
VG		 5373 *
5374 * Returns
5375 * IRQ_HANDLED - If interrupt is valid
5376 * IRQ_NONE - If invalid interrupt
7a3e97b0 5377 */
9333d775 5378static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 5379{
5380 u32 reg;
9333d775 5381 irqreturn_t retval = IRQ_NONE;
7a3e97b0 5382
fb7b45f0
DR		 5383 /* PHY layer lane error */
5384 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5385 /* Ignore LINERESET indication, as this is not an error */
5386 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
9333d775 5387 (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
fb7b45f0
DR		 5388 /*
5389 * To know whether this error is fatal or not, DB timeout
5390 * must be checked but this error is handled separately.
5391 */
5392 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
48d5b973 5393 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
9333d775 5394 retval |= IRQ_HANDLED;
ff8e20c6 5395 }
fb7b45f0 5396
e8e7f271
SRT		 5397 /* PA_INIT_ERROR is fatal and needs UIC reset */
5398 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 5399 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5400 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5401 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
ff8e20c6 5402
9333d775
VG		 5403 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5404 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5405 else if (hba->dev_quirks &
5406 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5407 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5408 hba->uic_error |=
5409 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5410 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5411 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5412 }
5413 retval |= IRQ_HANDLED;
583fa62d 5414 }
e8e7f271
SRT		 5415
5416 /* UIC NL/TL/DME errors needs software retry */
5417 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 5418 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5419 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5420 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
e8e7f271 5421 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 5422 retval |= IRQ_HANDLED;
ff8e20c6 5423 }
e8e7f271
SRT		 5424
5425 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 5426 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
5427 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
48d5b973 5428 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
e8e7f271 5429 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 5430 retval |= IRQ_HANDLED;
ff8e20c6 5431 }
e8e7f271
SRT		 5432
5433 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 5434 if ((reg & UIC_DME_ERROR) &&
5435 (reg & UIC_DME_ERROR_CODE_MASK)) {
48d5b973 5436 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
e8e7f271 5437 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 5438 retval |= IRQ_HANDLED;
ff8e20c6 5439 }
e8e7f271
SRT		 5440
5441 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5442 __func__, hba->uic_error);
9333d775 5443 return retval;
e8e7f271
SRT		 5444}
5445
82174440
SC		 5446static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5447 u32 intr_mask)
5448{
5449 if (!ufshcd_is_auto_hibern8_supported(hba))
5450 return false;
5451
5452 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5453 return false;
5454
5455 if (hba->active_uic_cmd &&
5456 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5457 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5458 return false;
5459
5460 return true;
5461}
5462
e8e7f271
SRT		 5463/**
5464 * ufshcd_check_errors - Check for errors that need s/w attention
5465 * @hba: per-adapter instance
9333d775
VG		 5466 *
5467 * Returns
5468 * IRQ_HANDLED - If interrupt is valid
5469 * IRQ_NONE - If invalid interrupt
e8e7f271 5470 */
9333d775 5471static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
e8e7f271
SRT		 5472{
5473 bool queue_eh_work = false;
9333d775 5474 irqreturn_t retval = IRQ_NONE;
e8e7f271 5475
d3c615bf
SC		 5476 if (hba->errors & INT_FATAL_ERRORS) {
5477 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
e8e7f271 5478 queue_eh_work = true;
d3c615bf 5479 }
7a3e97b0
SY		 5480
5481 if (hba->errors & UIC_ERROR) {
e8e7f271 5482 hba->uic_error = 0;
9333d775 5483 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 5484 if (hba->uic_error)
5485 queue_eh_work = true;
7a3e97b0 5486 }
e8e7f271 5487
82174440
SC		 5488 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
5489 dev_err(hba->dev,
5490 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
5491 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
5492 "Enter" : "Exit",
5493 hba->errors, ufshcd_get_upmcrs(hba));
d3c615bf
SC		 5494 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
5495 hba->errors);
82174440
SC		 5496 queue_eh_work = true;
5497 }
5498
e8e7f271 5499 if (queue_eh_work) {
9a47ec7c
YG		 5500 /*
5501 * update the transfer error masks to sticky bits, let's do this
5502 * irrespective of current ufshcd_state.
5503 */
5504 hba->saved_err |= hba->errors;
5505 hba->saved_uic_err |= hba->uic_error;
5506
e8e7f271
SRT		 5507 /* handle fatal errors only when link is functional */
5508 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
5509 /* block commands from scsi mid-layer */
38135535 5510 ufshcd_scsi_block_requests(hba);
e8e7f271 5511
141f8165 5512 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
66cc820f
DR		 5513
5514 /* dump controller state before resetting */
5515 if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
5516 bool pr_prdt = !!(hba->saved_err &
5517 SYSTEM_BUS_FATAL_ERROR);
5518
5519 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
5520 __func__, hba->saved_err,
5521 hba->saved_uic_err);
5522
5523 ufshcd_print_host_regs(hba);
5524 ufshcd_print_pwr_info(hba);
5525 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5526 ufshcd_print_trs(hba, hba->outstanding_reqs,
5527 pr_prdt);
5528 }
e8e7f271
SRT		 5529 schedule_work(&hba->eh_work);
5530 }
9333d775 5531 retval |= IRQ_HANDLED;
3441da7d 5532 }
e8e7f271
SRT		 5533 /*
5534 * if (!queue_eh_work) -
5535 * Other errors are either non-fatal where host recovers
5536 * itself without s/w intervention or errors that will be
5537 * handled by the SCSI core layer.
5538 */
9333d775 5539 return retval;
7a3e97b0
SY		 5540}
5541
69a6c269
BVA		 5542struct ctm_info {
5543 struct ufs_hba *hba;
5544 unsigned long pending;
5545 unsigned int ncpl;
5546};
5547
5548static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
5549{
5550 struct ctm_info *const ci = priv;
5551 struct completion *c;
5552
5553 WARN_ON_ONCE(reserved);
5554 if (test_bit(req->tag, &ci->pending))
5555 return true;
5556 ci->ncpl++;
5557 c = req->end_io_data;
5558 if (c)
5559 complete(c);
5560 return true;
5561}
5562
7a3e97b0
SY		 5563/**
5564 * ufshcd_tmc_handler - handle task management function completion
5565 * @hba: per adapter instance
9333d775
VG		 5566 *
5567 * Returns
5568 * IRQ_HANDLED - If interrupt is valid
5569 * IRQ_NONE - If invalid interrupt
7a3e97b0 5570 */
9333d775 5571static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 5572{
69a6c269
BVA		 5573 struct request_queue *q = hba->tmf_queue;
5574 struct ctm_info ci = {
5575 .hba = hba,
5576 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
5577 };
7a3e97b0 5578
69a6c269
BVA		 5579 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
5580 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
7a3e97b0
SY		 5581}
5582
5583/**
5584 * ufshcd_sl_intr - Interrupt service routine
5585 * @hba: per adapter instance
5586 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 5587 *
5588 * Returns
5589 * IRQ_HANDLED - If interrupt is valid
5590 * IRQ_NONE - If invalid interrupt
7a3e97b0 5591 */
9333d775 5592static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 5593{
9333d775
VG		 5594 irqreturn_t retval = IRQ_NONE;
5595
7a3e97b0 5596 hba->errors = UFSHCD_ERROR_MASK & intr_status;
82174440
SC		 5597
5598 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5599 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5600
7a3e97b0 5601 if (hba->errors)
9333d775 5602 retval |= ufshcd_check_errors(hba);
7a3e97b0 5603
53b3d9c3 5604 if (intr_status & UFSHCD_UIC_MASK)
9333d775 5605 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 5606
5607 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 5608 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 5609
5610 if (intr_status & UTP_TRANSFER_REQ_COMPL)
9333d775
VG		 5611 retval |= ufshcd_transfer_req_compl(hba);
5612
5613 return retval;
7a3e97b0
SY		 5614}
5615
5616/**
5617 * ufshcd_intr - Main interrupt service routine
5618 * @irq: irq number
5619 * @__hba: pointer to adapter instance
5620 *
9333d775
VG		 5621 * Returns
5622 * IRQ_HANDLED - If interrupt is valid
5623 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 5624 */
5625static irqreturn_t ufshcd_intr(int irq, void *__hba)
5626{
d75f7fe4 5627 u32 intr_status, enabled_intr_status;
7a3e97b0
SY		 5628 irqreturn_t retval = IRQ_NONE;
5629 struct ufs_hba *hba = __hba;
7f6ba4f1 5630 int retries = hba->nutrs;
7a3e97b0
SY		 5631
5632 spin_lock(hba->host->host_lock);
b873a275 5633 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
7a3e97b0 5634
7f6ba4f1
VG		 5635 /*
5636 * There could be max of hba->nutrs reqs in flight and in worst case
5637 * if the reqs get finished 1 by 1 after the interrupt status is
5638 * read, make sure we handle them by checking the interrupt status
5639 * again in a loop until we process all of the reqs before returning.
5640 */
5641 do {
5642 enabled_intr_status =
5643 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
5644 if (intr_status)
5645 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 5646 if (enabled_intr_status)
5647 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 5648
5649 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5650 } while (intr_status && --retries);
d75f7fe4 5651
9333d775
VG		 5652 if (retval == IRQ_NONE) {
5653 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
5654 __func__, intr_status);
5655 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
5656 }
5657
7a3e97b0
SY		 5658 spin_unlock(hba->host->host_lock);
5659 return retval;
5660}
5661
e2933132
SRT		 5662static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
5663{
5664 int err = 0;
5665 u32 mask = 1 << tag;
5666 unsigned long flags;
5667
5668 if (!test_bit(tag, &hba->outstanding_tasks))
5669 goto out;
5670
5671 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 5672 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 5673 spin_unlock_irqrestore(hba->host->host_lock, flags);
5674
5675 /* poll for max. 1 sec to clear door bell register by h/w */
5676 err = ufshcd_wait_for_register(hba,
5677 REG_UTP_TASK_REQ_DOOR_BELL,
596585a2 5678 mask, 0, 1000, 1000, true);
e2933132
SRT		 5679out:
5680 return err;
5681}
5682
c6049cd9
CH		 5683static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
5684 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 5685{
69a6c269 5686 struct request_queue *q = hba->tmf_queue;
c6049cd9 5687 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 5688 DECLARE_COMPLETION_ONSTACK(wait);
5689 struct request *req;
7a3e97b0 5690 unsigned long flags;
c6049cd9 5691 int free_slot, task_tag, err;
7a3e97b0 5692
e2933132
SRT		 5693 /*
5694 * Get free slot, sleep if slots are unavailable.
5695 * Even though we use wait_event() which sleeps indefinitely,
5696 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5697 */
69a6c269
BVA		 5698 req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
5699 req->end_io_data = &wait;
5700 free_slot = req->tag;
5701 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
1ab27c9c 5702 ufshcd_hold(hba, false);
7a3e97b0 5703
e2933132 5704 spin_lock_irqsave(host->host_lock, flags);
e2933132 5705 task_tag = hba->nutrs + free_slot;
7a3e97b0 5706
c6049cd9
CH		 5707 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
5708
5709 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 5710 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
5711
7a3e97b0
SY		 5712 /* send command to the controller */
5713 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 5714
5715 /* Make sure descriptors are ready before ringing the task doorbell */
5716 wmb();
5717
b873a275 5718 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 5719 /* Make sure that doorbell is committed immediately */
5720 wmb();
7a3e97b0
SY		 5721
5722 spin_unlock_irqrestore(host->host_lock, flags);
5723
6667e6d9
OS		 5724 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
5725
7a3e97b0 5726 /* wait until the task management command is completed */
69a6c269 5727 err = wait_for_completion_io_timeout(&wait,
e2933132 5728 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 5729 if (!err) {
69a6c269
BVA		 5730 /*
5731 * Make sure that ufshcd_compl_tm() does not trigger a
5732 * use-after-free.
5733 */
5734 req->end_io_data = NULL;
6667e6d9 5735 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
e2933132
SRT		 5736 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
5737 __func__, tm_function);
5738 if (ufshcd_clear_tm_cmd(hba, free_slot))
5739 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
5740 __func__, free_slot);
5741 err = -ETIMEDOUT;
5742 } else {
c6049cd9
CH		 5743 err = 0;
5744 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
5745
6667e6d9 5746 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
7a3e97b0 5747 }
e2933132 5748
b557217c
SC		 5749 spin_lock_irqsave(hba->host->host_lock, flags);
5750 __clear_bit(free_slot, &hba->outstanding_tasks);
5751 spin_unlock_irqrestore(hba->host->host_lock, flags);
5752
69a6c269 5753 blk_put_request(req);
e2933132 5754
1ab27c9c 5755 ufshcd_release(hba);
7a3e97b0
SY		 5756 return err;
5757}
5758
c6049cd9
CH		 5759/**
5760 * ufshcd_issue_tm_cmd - issues task management commands to controller
5761 * @hba: per adapter instance
5762 * @lun_id: LUN ID to which TM command is sent
5763 * @task_id: task ID to which the TM command is applicable
5764 * @tm_function: task management function opcode
5765 * @tm_response: task management service response return value
5766 *
5767 * Returns non-zero value on error, zero on success.
5768 */
5769static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
5770 u8 tm_function, u8 *tm_response)
5771{
5772 struct utp_task_req_desc treq = { { 0 }, };
5773 int ocs_value, err;
5774
5775 /* Configure task request descriptor */
5776 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5777 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5778
5779 /* Configure task request UPIU */
5780 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
5781 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
5782 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
5783
5784 /*
5785 * The host shall provide the same value for LUN field in the basic
5786 * header and for Input Parameter.
5787 */
5788 treq.input_param1 = cpu_to_be32(lun_id);
5789 treq.input_param2 = cpu_to_be32(task_id);
5790
5791 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
5792 if (err == -ETIMEDOUT)
5793 return err;
5794
5795 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5796 if (ocs_value != OCS_SUCCESS)
5797 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
5798 __func__, ocs_value);
5799 else if (tm_response)
5800 *tm_response = be32_to_cpu(treq.output_param1) &
5801 MASK_TM_SERVICE_RESP;
5802 return err;
5803}
5804
5e0a86ee
AA		 5805/**
5806 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5807 * @hba: per-adapter instance
5808 * @req_upiu: upiu request
5809 * @rsp_upiu: upiu reply
5e0a86ee
AA		 5810 * @desc_buff: pointer to descriptor buffer, NULL if NA
5811 * @buff_len: descriptor size, 0 if NA
d0e9760d 5812 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 5813 * @desc_op: descriptor operation
5814 *
5815 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5816 * Therefore, it "rides" the device management infrastructure: uses its tag and
5817 * tasks work queues.
5818 *
5819 * Since there is only one available tag for device management commands,
5820 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5821 */
5822static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
5823 struct utp_upiu_req *req_upiu,
5824 struct utp_upiu_req *rsp_upiu,
5825 u8 *desc_buff, int *buff_len,
7f674c38 5826 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 5827 enum query_opcode desc_op)
5828{
7252a360
BVA		 5829 struct request_queue *q = hba->cmd_queue;
5830 struct request *req;
5e0a86ee
AA		 5831 struct ufshcd_lrb *lrbp;
5832 int err = 0;
5833 int tag;
5834 struct completion wait;
5835 unsigned long flags;
5836 u32 upiu_flags;
5837
5838 down_read(&hba->clk_scaling_lock);
5839
7252a360
BVA		 5840 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
5841 if (IS_ERR(req))
5842 return PTR_ERR(req);
5843 tag = req->tag;
5844 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5e0a86ee
AA		 5845
5846 init_completion(&wait);
5847 lrbp = &hba->lrb[tag];
5848 WARN_ON(lrbp->cmd);
5849
5850 lrbp->cmd = NULL;
5851 lrbp->sense_bufflen = 0;
5852 lrbp->sense_buffer = NULL;
5853 lrbp->task_tag = tag;
5854 lrbp->lun = 0;
5855 lrbp->intr_cmd = true;
5856 hba->dev_cmd.type = cmd_type;
5857
5858 switch (hba->ufs_version) {
5859 case UFSHCI_VERSION_10:
5860 case UFSHCI_VERSION_11:
5861 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
5862 break;
5863 default:
5864 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
5865 break;
5866 }
5867
5868 /* update the task tag in the request upiu */
5869 req_upiu->header.dword_0 |= cpu_to_be32(tag);
5870
5871 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
5872
5873 /* just copy the upiu request as it is */
5874 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
5875 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
5876 /* The Data Segment Area is optional depending upon the query
5877 * function value. for WRITE DESCRIPTOR, the data segment
5878 * follows right after the tsf.
5879 */
5880 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
5881 *buff_len = 0;
5882 }
5883
5884 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5885
5886 hba->dev_cmd.complete = &wait;
5887
5888 /* Make sure descriptors are ready before ringing the doorbell */
5889 wmb();
5890 spin_lock_irqsave(hba->host->host_lock, flags);
5891 ufshcd_send_command(hba, tag);
5892 spin_unlock_irqrestore(hba->host->host_lock, flags);
5893
5894 /*
5895 * ignore the returning value here - ufshcd_check_query_response is
5896 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5897 * read the response directly ignoring all errors.
5898 */
5899 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
5900
5901 /* just copy the upiu response as it is */
5902 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 5903 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
5904 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
5905 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
5906 MASK_QUERY_DATA_SEG_LEN;
5907
5908 if (*buff_len >= resp_len) {
5909 memcpy(desc_buff, descp, resp_len);
5910 *buff_len = resp_len;
5911 } else {
3d4881d1
BH		 5912 dev_warn(hba->dev,
5913 "%s: rsp size %d is bigger than buffer size %d",
5914 __func__, resp_len, *buff_len);
4bbbe242
AA		 5915 *buff_len = 0;
5916 err = -EINVAL;
5917 }
5918 }
5e0a86ee 5919
7252a360 5920 blk_put_request(req);
5e0a86ee
AA		 5921 up_read(&hba->clk_scaling_lock);
5922 return err;
5923}
5924
5925/**
5926 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5927 * @hba: per-adapter instance
5928 * @req_upiu: upiu request
5929 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5930 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5931 * @desc_buff: pointer to descriptor buffer, NULL if NA
5932 * @buff_len: descriptor size, 0 if NA
5933 * @desc_op: descriptor operation
5934 *
5935 * Supports UTP Transfer requests (nop and query), and UTP Task
5936 * Management requests.
5937 * It is up to the caller to fill the upiu conent properly, as it will
5938 * be copied without any further input validations.
5939 */
5940int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
5941 struct utp_upiu_req *req_upiu,
5942 struct utp_upiu_req *rsp_upiu,
5943 int msgcode,
5944 u8 *desc_buff, int *buff_len,
5945 enum query_opcode desc_op)
5946{
5947 int err;
7f674c38 5948 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee
AA		 5949 struct utp_task_req_desc treq = { { 0 }, };
5950 int ocs_value;
5951 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
5952
5e0a86ee
AA		 5953 switch (msgcode) {
5954 case UPIU_TRANSACTION_NOP_OUT:
5955 cmd_type = DEV_CMD_TYPE_NOP;
5956 /* fall through */
5957 case UPIU_TRANSACTION_QUERY_REQ:
5958 ufshcd_hold(hba, false);
5959 mutex_lock(&hba->dev_cmd.lock);
5960 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
5961 desc_buff, buff_len,
5962 cmd_type, desc_op);
5963 mutex_unlock(&hba->dev_cmd.lock);
5964 ufshcd_release(hba);
5965
5966 break;
5967 case UPIU_TRANSACTION_TASK_REQ:
5968 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5969 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5970
5971 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
5972
5973 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
5974 if (err == -ETIMEDOUT)
5975 break;
5976
5977 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5978 if (ocs_value != OCS_SUCCESS) {
5979 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
5980 ocs_value);
5981 break;
5982 }
5983
5984 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
5985
5986 break;
5987 default:
5988 err = -EINVAL;
5989
5990 break;
5991 }
5992
5e0a86ee
AA		 5993 return err;
5994}
5995
7a3e97b0 5996/**
3441da7d
SRT		 5997 * ufshcd_eh_device_reset_handler - device reset handler registered to
5998 * scsi layer.
7a3e97b0
SY		 5999 * @cmd: SCSI command pointer
6000 *
6001 * Returns SUCCESS/FAILED
6002 */
3441da7d 6003static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6004{
6005 struct Scsi_Host *host;
6006 struct ufs_hba *hba;
6007 unsigned int tag;
6008 u32 pos;
6009 int err;
e2933132
SRT		 6010 u8 resp = 0xF;
6011 struct ufshcd_lrb *lrbp;
3441da7d 6012 unsigned long flags;
7a3e97b0
SY		 6013
6014 host = cmd->device->host;
6015 hba = shost_priv(host);
6016 tag = cmd->request->tag;
6017
e2933132
SRT		 6018 lrbp = &hba->lrb[tag];
6019 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6020 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6021 if (!err)
6022 err = resp;
7a3e97b0 6023 goto out;
e2933132 6024 }
7a3e97b0 6025
3441da7d
SRT		 6026 /* clear the commands that were pending for corresponding LUN */
6027 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6028 if (hba->lrb[pos].lun == lrbp->lun) {
6029 err = ufshcd_clear_cmd(hba, pos);
6030 if (err)
6031 break;
7a3e97b0 6032 }
3441da7d
SRT		 6033 }
6034 spin_lock_irqsave(host->host_lock, flags);
6035 ufshcd_transfer_req_compl(hba);
6036 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 6037
7a3e97b0 6038out:
7fabb77b 6039 hba->req_abort_count = 0;
8808b4e9 6040 ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
3441da7d
SRT		 6041 if (!err) {
6042 err = SUCCESS;
6043 } else {
6044 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6045 err = FAILED;
6046 }
7a3e97b0
SY		 6047 return err;
6048}
6049
e0b299e3
GB		 6050static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6051{
6052 struct ufshcd_lrb *lrbp;
6053 int tag;
6054
6055 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6056 lrbp = &hba->lrb[tag];
6057 lrbp->req_abort_skip = true;
6058 }
6059}
6060
7a3e97b0
SY		 6061/**
6062 * ufshcd_abort - abort a specific command
6063 * @cmd: SCSI command pointer
6064 *
f20810d8
SRT		 6065 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6066 * command, and in host controller by clearing the door-bell register. There can
6067 * be race between controller sending the command to the device while abort is
6068 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6069 * really issued and then try to abort it.
6070 *
7a3e97b0
SY		 6071 * Returns SUCCESS/FAILED
6072 */
6073static int ufshcd_abort(struct scsi_cmnd *cmd)
6074{
6075 struct Scsi_Host *host;
6076 struct ufs_hba *hba;
6077 unsigned long flags;
6078 unsigned int tag;
f20810d8
SRT		 6079 int err = 0;
6080 int poll_cnt;
e2933132
SRT		 6081 u8 resp = 0xF;
6082 struct ufshcd_lrb *lrbp;
e9d501b1 6083 u32 reg;
7a3e97b0
SY		 6084
6085 host = cmd->device->host;
6086 hba = shost_priv(host);
6087 tag = cmd->request->tag;
e7d38257 6088 lrbp = &hba->lrb[tag];
14497328
YG		 6089 if (!ufshcd_valid_tag(hba, tag)) {
6090 dev_err(hba->dev,
6091 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6092 __func__, tag, cmd, cmd->request);
6093 BUG();
6094 }
7a3e97b0 6095
e7d38257
DR		 6096 /*
6097 * Task abort to the device W-LUN is illegal. When this command
6098 * will fail, due to spec violation, scsi err handling next step
6099 * will be to send LU reset which, again, is a spec violation.
6100 * To avoid these unnecessary/illegal step we skip to the last error
6101 * handling stage: reset and restore.
6102 */
6103 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6104 return ufshcd_eh_host_reset_handler(cmd);
6105
1ab27c9c 6106 ufshcd_hold(hba, false);
14497328 6107 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 6108 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 6109 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6110 dev_err(hba->dev,
6111 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6112 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 6113 goto out;
14497328 6114 }
7a3e97b0 6115
e9d501b1
DR		 6116 if (!(reg & (1 << tag))) {
6117 dev_err(hba->dev,
6118 "%s: cmd was completed, but without a notifying intr, tag = %d",
6119 __func__, tag);
6120 }
6121
66cc820f
DR		 6122 /* Print Transfer Request of aborted task */
6123 dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 6124
7fabb77b
GB		 6125 /*
6126 * Print detailed info about aborted request.
6127 * As more than one request might get aborted at the same time,
6128 * print full information only for the first aborted request in order
6129 * to reduce repeated printouts. For other aborted requests only print
6130 * basic details.
6131 */
6132 scsi_print_command(hba->lrb[tag].cmd);
6133 if (!hba->req_abort_count) {
8808b4e9 6134 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
7fabb77b 6135 ufshcd_print_host_regs(hba);
6ba65588 6136 ufshcd_print_host_state(hba);
7fabb77b
GB		 6137 ufshcd_print_pwr_info(hba);
6138 ufshcd_print_trs(hba, 1 << tag, true);
6139 } else {
6140 ufshcd_print_trs(hba, 1 << tag, false);
6141 }
6142 hba->req_abort_count++;
e0b299e3
GB		 6143
6144 /* Skip task abort in case previous aborts failed and report failure */
6145 if (lrbp->req_abort_skip) {
6146 err = -EIO;
6147 goto out;
6148 }
6149
f20810d8
SRT		 6150 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6151 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6152 UFS_QUERY_TASK, &resp);
6153 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6154 /* cmd pending in the device */
ff8e20c6
DR		 6155 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6156 __func__, tag);
f20810d8
SRT		 6157 break;
6158 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
f20810d8
SRT		 6159 /*
6160 * cmd not pending in the device, check if it is
6161 * in transition.
6162 */
ff8e20c6
DR		 6163 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6164 __func__, tag);
f20810d8
SRT		 6165 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6166 if (reg & (1 << tag)) {
6167 /* sleep for max. 200us to stabilize */
6168 usleep_range(100, 200);
6169 continue;
6170 }
6171 /* command completed already */
ff8e20c6
DR		 6172 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6173 __func__, tag);
f20810d8
SRT		 6174 goto out;
6175 } else {
ff8e20c6
DR		 6176 dev_err(hba->dev,
6177 "%s: no response from device. tag = %d, err %d\n",
6178 __func__, tag, err);
f20810d8
SRT		 6179 if (!err)
6180 err = resp; /* service response error */
6181 goto out;
6182 }
6183 }
6184
6185 if (!poll_cnt) {
6186 err = -EBUSY;
7a3e97b0
SY		 6187 goto out;
6188 }
7a3e97b0 6189
e2933132
SRT		 6190 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6191 UFS_ABORT_TASK, &resp);
6192 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
ff8e20c6 6193 if (!err) {
f20810d8 6194 err = resp; /* service response error */
ff8e20c6
DR		 6195 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6196 __func__, tag, err);
6197 }
7a3e97b0 6198 goto out;
e2933132 6199 }
7a3e97b0 6200
f20810d8 6201 err = ufshcd_clear_cmd(hba, tag);
ff8e20c6
DR		 6202 if (err) {
6203 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6204 __func__, tag, err);
f20810d8 6205 goto out;
ff8e20c6 6206 }
f20810d8 6207
7a3e97b0
SY		 6208 scsi_dma_unmap(cmd);
6209
6210 spin_lock_irqsave(host->host_lock, flags);
a48353f6 6211 ufshcd_outstanding_req_clear(hba, tag);
7a3e97b0
SY		 6212 hba->lrb[tag].cmd = NULL;
6213 spin_unlock_irqrestore(host->host_lock, flags);
5a0b0cb9 6214
7a3e97b0 6215out:
f20810d8
SRT		 6216 if (!err) {
6217 err = SUCCESS;
6218 } else {
6219 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6220 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6221 err = FAILED;
6222 }
6223
1ab27c9c
ST		 6224 /*
6225 * This ufshcd_release() corresponds to the original scsi cmd that got
6226 * aborted here (as we won't get any IRQ for it).
6227 */
6228 ufshcd_release(hba);
7a3e97b0
SY		 6229 return err;
6230}
6231
3441da7d
SRT		 6232/**
6233 * ufshcd_host_reset_and_restore - reset and restore host controller
6234 * @hba: per-adapter instance
6235 *
6236 * Note that host controller reset may issue DME_RESET to
6237 * local and remote (device) Uni-Pro stack and the attributes
6238 * are reset to default state.
6239 *
6240 * Returns zero on success, non-zero on failure
6241 */
6242static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6243{
6244 int err;
3441da7d
SRT		 6245 unsigned long flags;
6246
2df74b69
CG		 6247 /*
6248 * Stop the host controller and complete the requests
6249 * cleared by h/w
6250 */
3441da7d 6251 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 6252 ufshcd_hba_stop(hba, false);
2df74b69
CG		 6253 hba->silence_err_logs = true;
6254 ufshcd_complete_requests(hba);
6255 hba->silence_err_logs = false;
3441da7d
SRT		 6256 spin_unlock_irqrestore(hba->host->host_lock, flags);
6257
a3cd5ec5 6258 /* scale up clocks to max frequency before full reinitialization */
6259 ufshcd_scale_clks(hba, true);
6260
3441da7d
SRT		 6261 err = ufshcd_hba_enable(hba);
6262 if (err)
6263 goto out;
6264
6265 /* Establish the link again and restore the device */
1d337ec2
SRT		 6266 err = ufshcd_probe_hba(hba);
6267
6268 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
3441da7d
SRT		 6269 err = -EIO;
6270out:
6271 if (err)
6272 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
8808b4e9 6273 ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
3441da7d
SRT		 6274 return err;
6275}
6276
6277/**
6278 * ufshcd_reset_and_restore - reset and re-initialize host/device
6279 * @hba: per-adapter instance
6280 *
6281 * Reset and recover device, host and re-establish link. This
6282 * is helpful to recover the communication in fatal error conditions.
6283 *
6284 * Returns zero on success, non-zero on failure
6285 */
6286static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6287{
6288 int err = 0;
1d337ec2 6289 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 6290
1d337ec2 6291 do {
d8d9f793
BA		 6292 /* Reset the attached device */
6293 ufshcd_vops_device_reset(hba);
6294
1d337ec2
SRT		 6295 err = ufshcd_host_reset_and_restore(hba);
6296 } while (err && --retries);
3441da7d 6297
3441da7d
SRT		 6298 return err;
6299}
6300
6301/**
6302 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 6303 * @cmd: SCSI command pointer
3441da7d
SRT		 6304 *
6305 * Returns SUCCESS/FAILED
6306 */
6307static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6308{
6309 int err;
6310 unsigned long flags;
6311 struct ufs_hba *hba;
6312
6313 hba = shost_priv(cmd->device->host);
6314
1ab27c9c 6315 ufshcd_hold(hba, false);
3441da7d
SRT		 6316 /*
6317 * Check if there is any race with fatal error handling.
6318 * If so, wait for it to complete. Even though fatal error
6319 * handling does reset and restore in some cases, don't assume
6320 * anything out of it. We are just avoiding race here.
6321 */
6322 do {
6323 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 6324 if (!(work_pending(&hba->eh_work) ||
8dc0da79
ZL		 6325 hba->ufshcd_state == UFSHCD_STATE_RESET ||
6326 hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
3441da7d
SRT		 6327 break;
6328 spin_unlock_irqrestore(hba->host->host_lock, flags);
6329 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
e8e7f271 6330 flush_work(&hba->eh_work);
3441da7d
SRT		 6331 } while (1);
6332
6333 hba->ufshcd_state = UFSHCD_STATE_RESET;
6334 ufshcd_set_eh_in_progress(hba);
6335 spin_unlock_irqrestore(hba->host->host_lock, flags);
6336
6337 err = ufshcd_reset_and_restore(hba);
6338
6339 spin_lock_irqsave(hba->host->host_lock, flags);
6340 if (!err) {
6341 err = SUCCESS;
6342 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6343 } else {
6344 err = FAILED;
6345 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6346 }
6347 ufshcd_clear_eh_in_progress(hba);
6348 spin_unlock_irqrestore(hba->host->host_lock, flags);
6349
1ab27c9c 6350 ufshcd_release(hba);
3441da7d
SRT		 6351 return err;
6352}
6353
3a4bf06d
YG		 6354/**
6355 * ufshcd_get_max_icc_level - calculate the ICC level
6356 * @sup_curr_uA: max. current supported by the regulator
6357 * @start_scan: row at the desc table to start scan from
6358 * @buff: power descriptor buffer
6359 *
6360 * Returns calculated max ICC level for specific regulator
6361 */
6362static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6363{
6364 int i;
6365 int curr_uA;
6366 u16 data;
6367 u16 unit;
6368
6369 for (i = start_scan; i >= 0; i--) {
d79713f9 6370 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 6371 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6372 ATTR_ICC_LVL_UNIT_OFFSET;
6373 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6374 switch (unit) {
6375 case UFSHCD_NANO_AMP:
6376 curr_uA = curr_uA / 1000;
6377 break;
6378 case UFSHCD_MILI_AMP:
6379 curr_uA = curr_uA * 1000;
6380 break;
6381 case UFSHCD_AMP:
6382 curr_uA = curr_uA * 1000 * 1000;
6383 break;
6384 case UFSHCD_MICRO_AMP:
6385 default:
6386 break;
6387 }
6388 if (sup_curr_uA >= curr_uA)
6389 break;
6390 }
6391 if (i < 0) {
6392 i = 0;
6393 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6394 }
6395
6396 return (u32)i;
6397}
6398
6399/**
6400 * ufshcd_calc_icc_level - calculate the max ICC level
6401 * In case regulators are not initialized we'll return 0
6402 * @hba: per-adapter instance
6403 * @desc_buf: power descriptor buffer to extract ICC levels from.
6404 * @len: length of desc_buff
6405 *
6406 * Returns calculated ICC level
6407 */
6408static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6409 u8 *desc_buf, int len)
6410{
6411 u32 icc_level = 0;
6412
6413 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6414 !hba->vreg_info.vccq2) {
6415 dev_err(hba->dev,
6416 "%s: Regulator capability was not set, actvIccLevel=%d",
6417 __func__, icc_level);
6418 goto out;
6419 }
6420
0487fff7 6421 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 6422 icc_level = ufshcd_get_max_icc_level(
6423 hba->vreg_info.vcc->max_uA,
6424 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6425 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6426
0487fff7 6427 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 6428 icc_level = ufshcd_get_max_icc_level(
6429 hba->vreg_info.vccq->max_uA,
6430 icc_level,
6431 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6432
0487fff7 6433 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 6434 icc_level = ufshcd_get_max_icc_level(
6435 hba->vreg_info.vccq2->max_uA,
6436 icc_level,
6437 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6438out:
6439 return icc_level;
6440}
6441
6442static void ufshcd_init_icc_levels(struct ufs_hba *hba)
6443{
6444 int ret;
a4b0e8a4 6445 int buff_len = hba->desc_size.pwr_desc;
bbe21d7a
KC		 6446 u8 *desc_buf;
6447
6448 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6449 if (!desc_buf)
6450 return;
3a4bf06d
YG		 6451
6452 ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
6453 if (ret) {
6454 dev_err(hba->dev,
6455 "%s: Failed reading power descriptor.len = %d ret = %d",
6456 __func__, buff_len, ret);
bbe21d7a 6457 goto out;
3a4bf06d
YG		 6458 }
6459
6460 hba->init_prefetch_data.icc_level =
6461 ufshcd_find_max_sup_active_icc_level(hba,
6462 desc_buf, buff_len);
6463 dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
6464 __func__, hba->init_prefetch_data.icc_level);
6465
dbd34a61
SM		 6466 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6467 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
6468 &hba->init_prefetch_data.icc_level);
3a4bf06d
YG		 6469
6470 if (ret)
6471 dev_err(hba->dev,
6472 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
6473 __func__, hba->init_prefetch_data.icc_level , ret);
6474
bbe21d7a
KC		 6475out:
6476 kfree(desc_buf);
3a4bf06d
YG		 6477}
6478
2a8fa600
SJ		 6479/**
6480 * ufshcd_scsi_add_wlus - Adds required W-LUs
6481 * @hba: per-adapter instance
6482 *
6483 * UFS device specification requires the UFS devices to support 4 well known
6484 * logical units:
6485 * "REPORT_LUNS" (address: 01h)
6486 * "UFS Device" (address: 50h)
6487 * "RPMB" (address: 44h)
6488 * "BOOT" (address: 30h)
6489 * UFS device's power management needs to be controlled by "POWER CONDITION"
6490 * field of SSU (START STOP UNIT) command. But this "power condition" field
6491 * will take effect only when its sent to "UFS device" well known logical unit
6492 * hence we require the scsi_device instance to represent this logical unit in
6493 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 6494 *
2a8fa600
SJ		 6495 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6496 * Block) LU so user space process can control this LU. User space may also
6497 * want to have access to BOOT LU.
8aa29f19 6498 *
2a8fa600
SJ		 6499 * This function adds scsi device instances for each of all well known LUs
6500 * (except "REPORT LUNS" LU).
6501 *
6502 * Returns zero on success (all required W-LUs are added successfully),
6503 * non-zero error value on failure (if failed to add any of the required W-LU).
6504 */
6505static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
6506{
6507 int ret = 0;
7c48bfd0
AM		 6508 struct scsi_device *sdev_rpmb;
6509 struct scsi_device *sdev_boot;
2a8fa600
SJ		 6510
6511 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
6512 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
6513 if (IS_ERR(hba->sdev_ufs_device)) {
6514 ret = PTR_ERR(hba->sdev_ufs_device);
6515 hba->sdev_ufs_device = NULL;
6516 goto out;
6517 }
7c48bfd0 6518 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 6519
7c48bfd0 6520 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 6521 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7c48bfd0
AM		 6522 if (IS_ERR(sdev_rpmb)) {
6523 ret = PTR_ERR(sdev_rpmb);
3d21fbde 6524 goto remove_sdev_ufs_device;
2a8fa600 6525 }
7c48bfd0 6526 scsi_device_put(sdev_rpmb);
3d21fbde
HK		 6527
6528 sdev_boot = __scsi_add_device(hba->host, 0, 0,
6529 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
6530 if (IS_ERR(sdev_boot))
6531 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
6532 else
6533 scsi_device_put(sdev_boot);
2a8fa600
SJ		 6534 goto out;
6535
2a8fa600
SJ		 6536remove_sdev_ufs_device:
6537 scsi_remove_device(hba->sdev_ufs_device);
6538out:
6539 return ret;
6540}
6541
93fdd5ac
TW		 6542static int ufs_get_device_desc(struct ufs_hba *hba,
6543 struct ufs_dev_desc *dev_desc)
c58ab7aa
YG		 6544{
6545 int err;
bbe21d7a 6546 size_t buff_len;
c58ab7aa 6547 u8 model_index;
bbe21d7a
KC		 6548 u8 *desc_buf;
6549
4b828fe1
TW		 6550 if (!dev_desc)
6551 return -EINVAL;
6552
bbe21d7a
KC		 6553 buff_len = max_t(size_t, hba->desc_size.dev_desc,
6554 QUERY_DESC_MAX_SIZE + 1);
6555 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6556 if (!desc_buf) {
6557 err = -ENOMEM;
6558 goto out;
6559 }
c58ab7aa 6560
a4b0e8a4 6561 err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
c58ab7aa
YG		 6562 if (err) {
6563 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
6564 __func__, err);
6565 goto out;
6566 }
6567
6568 /*
6569 * getting vendor (manufacturerID) and Bank Index in big endian
6570 * format
6571 */
93fdd5ac 6572 dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 6573 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
6574
6575 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
4b828fe1
TW		 6576 err = ufshcd_read_string_desc(hba, model_index,
6577 &dev_desc->model, SD_ASCII_STD);
6578 if (err < 0) {
c58ab7aa
YG		 6579 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
6580 __func__, err);
6581 goto out;
6582 }
6583
4b828fe1
TW		 6584 /*
6585 * ufshcd_read_string_desc returns size of the string
6586 * reset the error value
6587 */
6588 err = 0;
c58ab7aa
YG		 6589
6590out:
bbe21d7a 6591 kfree(desc_buf);
c58ab7aa
YG		 6592 return err;
6593}
6594
4b828fe1
TW		 6595static void ufs_put_device_desc(struct ufs_dev_desc *dev_desc)
6596{
6597 kfree(dev_desc->model);
6598 dev_desc->model = NULL;
6599}
6600
93fdd5ac
TW		 6601static void ufs_fixup_device_setup(struct ufs_hba *hba,
6602 struct ufs_dev_desc *dev_desc)
c58ab7aa 6603{
c58ab7aa 6604 struct ufs_dev_fix *f;
c58ab7aa
YG		 6605
6606 for (f = ufs_fixups; f->quirk; f++) {
93fdd5ac
TW		 6607 if ((f->card.wmanufacturerid == dev_desc->wmanufacturerid ||
6608 f->card.wmanufacturerid == UFS_ANY_VENDOR) &&
4b828fe1
TW		 6609 ((dev_desc->model &&
6610 STR_PRFX_EQUAL(f->card.model, dev_desc->model)) ||
6611 !strcmp(f->card.model, UFS_ANY_MODEL)))
c58ab7aa
YG		 6612 hba->dev_quirks |= f->quirk;
6613 }
6614}
6615
37113106
YG		 6616/**
6617 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6618 * @hba: per-adapter instance
6619 *
6620 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6621 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6622 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6623 * the hibern8 exit latency.
6624 *
6625 * Returns zero on success, non-zero error value on failure.
6626 */
6627static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
6628{
6629 int ret = 0;
6630 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
6631
6632 ret = ufshcd_dme_peer_get(hba,
6633 UIC_ARG_MIB_SEL(
6634 RX_MIN_ACTIVATETIME_CAPABILITY,
6635 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6636 &peer_rx_min_activatetime);
6637 if (ret)
6638 goto out;
6639
6640 /* make sure proper unit conversion is applied */
6641 tuned_pa_tactivate =
6642 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
6643 / PA_TACTIVATE_TIME_UNIT_US);
6644 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6645 tuned_pa_tactivate);
6646
6647out:
6648 return ret;
6649}
6650
6651/**
6652 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6653 * @hba: per-adapter instance
6654 *
6655 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6656 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6657 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6658 * This optimal value can help reduce the hibern8 exit latency.
6659 *
6660 * Returns zero on success, non-zero error value on failure.
6661 */
6662static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
6663{
6664 int ret = 0;
6665 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
6666 u32 max_hibern8_time, tuned_pa_hibern8time;
6667
6668 ret = ufshcd_dme_get(hba,
6669 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
6670 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
6671 &local_tx_hibern8_time_cap);
6672 if (ret)
6673 goto out;
6674
6675 ret = ufshcd_dme_peer_get(hba,
6676 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
6677 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6678 &peer_rx_hibern8_time_cap);
6679 if (ret)
6680 goto out;
6681
6682 max_hibern8_time = max(local_tx_hibern8_time_cap,
6683 peer_rx_hibern8_time_cap);
6684 /* make sure proper unit conversion is applied */
6685 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
6686 / PA_HIBERN8_TIME_UNIT_US);
6687 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
6688 tuned_pa_hibern8time);
6689out:
6690 return ret;
6691}
6692
c6a6db43 6693/**
6694 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6695 * less than device PA_TACTIVATE time.
6696 * @hba: per-adapter instance
6697 *
6698 * Some UFS devices require host PA_TACTIVATE to be lower than device
6699 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6700 * for such devices.
6701 *
6702 * Returns zero on success, non-zero error value on failure.
6703 */
6704static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
6705{
6706 int ret = 0;
6707 u32 granularity, peer_granularity;
6708 u32 pa_tactivate, peer_pa_tactivate;
6709 u32 pa_tactivate_us, peer_pa_tactivate_us;
6710 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
6711
6712 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6713 &granularity);
6714 if (ret)
6715 goto out;
6716
6717 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6718 &peer_granularity);
6719 if (ret)
6720 goto out;
6721
6722 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
6723 (granularity > PA_GRANULARITY_MAX_VAL)) {
6724 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
6725 __func__, granularity);
6726 return -EINVAL;
6727 }
6728
6729 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
6730 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
6731 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
6732 __func__, peer_granularity);
6733 return -EINVAL;
6734 }
6735
6736 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
6737 if (ret)
6738 goto out;
6739
6740 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
6741 &peer_pa_tactivate);
6742 if (ret)
6743 goto out;
6744
6745 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
6746 peer_pa_tactivate_us = peer_pa_tactivate *
6747 gran_to_us_table[peer_granularity - 1];
6748
6749 if (pa_tactivate_us > peer_pa_tactivate_us) {
6750 u32 new_peer_pa_tactivate;
6751
6752 new_peer_pa_tactivate = pa_tactivate_us /
6753 gran_to_us_table[peer_granularity - 1];
6754 new_peer_pa_tactivate++;
6755 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6756 new_peer_pa_tactivate);
6757 }
6758
6759out:
6760 return ret;
6761}
6762
37113106
YG		 6763static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
6764{
6765 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
6766 ufshcd_tune_pa_tactivate(hba);
6767 ufshcd_tune_pa_hibern8time(hba);
6768 }
6769
6770 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
6771 /* set 1ms timeout for PA_TACTIVATE */
6772 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43 6773
6774 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
6775 ufshcd_quirk_tune_host_pa_tactivate(hba);
56d4a186
SJ		 6776
6777 ufshcd_vops_apply_dev_quirks(hba);
37113106
YG		 6778}
6779
ff8e20c6
DR		 6780static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
6781{
ff8e20c6
DR		 6782 hba->ufs_stats.hibern8_exit_cnt = 0;
6783 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 6784 hba->req_abort_count = 0;
ff8e20c6
DR		 6785}
6786
a4b0e8a4
PM		 6787static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
6788{
6789 int err;
6790
6791 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
6792 &hba->desc_size.dev_desc);
6793 if (err)
6794 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6795
6796 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
6797 &hba->desc_size.pwr_desc);
6798 if (err)
6799 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6800
6801 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
6802 &hba->desc_size.interc_desc);
6803 if (err)
6804 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6805
6806 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
6807 &hba->desc_size.conf_desc);
6808 if (err)
6809 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6810
6811 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
6812 &hba->desc_size.unit_desc);
6813 if (err)
6814 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6815
6816 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6817 &hba->desc_size.geom_desc);
6818 if (err)
6819 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
059efd84 6820
c648c2d2
SN		 6821 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
6822 &hba->desc_size.hlth_desc);
6823 if (err)
6824 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
a4b0e8a4
PM		 6825}
6826
9e1e8a75
SJ		 6827static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
6828 {19200000, REF_CLK_FREQ_19_2_MHZ},
6829 {26000000, REF_CLK_FREQ_26_MHZ},
6830 {38400000, REF_CLK_FREQ_38_4_MHZ},
6831 {52000000, REF_CLK_FREQ_52_MHZ},
6832 {0, REF_CLK_FREQ_INVAL},
6833};
6834
6835static enum ufs_ref_clk_freq
6836ufs_get_bref_clk_from_hz(unsigned long freq)
6837{
6838 int i;
6839
6840 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
6841 if (ufs_ref_clk_freqs[i].freq_hz == freq)
6842 return ufs_ref_clk_freqs[i].val;
6843
6844 return REF_CLK_FREQ_INVAL;
6845}
6846
6847void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
6848{
6849 unsigned long freq;
6850
6851 freq = clk_get_rate(refclk);
6852
6853 hba->dev_ref_clk_freq =
6854 ufs_get_bref_clk_from_hz(freq);
6855
6856 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
6857 dev_err(hba->dev,
6858 "invalid ref_clk setting = %ld\n", freq);
6859}
6860
6861static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
6862{
6863 int err;
6864 u32 ref_clk;
6865 u32 freq = hba->dev_ref_clk_freq;
6866
6867 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6868 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
6869
6870 if (err) {
6871 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
6872 err);
6873 goto out;
6874 }
6875
6876 if (ref_clk == freq)
6877 goto out; /* nothing to update */
6878
6879 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6880 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
6881
6882 if (err) {
6883 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
6884 ufs_ref_clk_freqs[freq].freq_hz);
6885 goto out;
6886 }
6887
6888 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
6889 ufs_ref_clk_freqs[freq].freq_hz);
6890
6891out:
6892 return err;
6893}
6894
6ccf44fe 6895/**
1d337ec2
SRT		 6896 * ufshcd_probe_hba - probe hba to detect device and initialize
6897 * @hba: per-adapter instance
6898 *
6899 * Execute link-startup and verify device initialization
6ccf44fe 6900 */
1d337ec2 6901static int ufshcd_probe_hba(struct ufs_hba *hba)
6ccf44fe 6902{
93fdd5ac 6903 struct ufs_dev_desc card = {0};
6ccf44fe 6904 int ret;
7ff5ab47 6905 ktime_t start = ktime_get();
6ccf44fe
SJ		 6906
6907 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 6908 if (ret)
6909 goto out;
6910
afdfff59
YG		 6911 /* set the default level for urgent bkops */
6912 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
6913 hba->is_urgent_bkops_lvl_checked = false;
6914
ff8e20c6
DR		 6915 /* Debug counters initialization */
6916 ufshcd_clear_dbg_ufs_stats(hba);
6917
57d104c1
SJ		 6918 /* UniPro link is active now */
6919 ufshcd_set_link_active(hba);
d3e89bac 6920
5a0b0cb9
SRT		 6921 ret = ufshcd_verify_dev_init(hba);
6922 if (ret)
6923 goto out;
68078d5c
DR		 6924
6925 ret = ufshcd_complete_dev_init(hba);
6926 if (ret)
6927 goto out;
5a0b0cb9 6928
a4b0e8a4
PM		 6929 /* Init check for device descriptor sizes */
6930 ufshcd_init_desc_sizes(hba);
6931
93fdd5ac
TW		 6932 ret = ufs_get_device_desc(hba, &card);
6933 if (ret) {
6934 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
6935 __func__, ret);
6936 goto out;
6937 }
6938
6939 ufs_fixup_device_setup(hba, &card);
4b828fe1
TW		 6940 ufs_put_device_desc(&card);
6941
37113106 6942 ufshcd_tune_unipro_params(hba);
60f01870 6943
57d104c1
SJ		 6944 /* UFS device is also active now */
6945 ufshcd_set_ufs_dev_active(hba);
66ec6d59 6946 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 6947 hba->wlun_dev_clr_ua = true;
6948
7eb584db
DR		 6949 if (ufshcd_get_max_pwr_mode(hba)) {
6950 dev_err(hba->dev,
6951 "%s: Failed getting max supported power mode\n",
6952 __func__);
6953 } else {
9e1e8a75
SJ		 6954 /*
6955 * Set the right value to bRefClkFreq before attempting to
6956 * switch to HS gears.
6957 */
6958 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
6959 ufshcd_set_dev_ref_clk(hba);
7eb584db 6960 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 6961 if (ret) {
7eb584db
DR		 6962 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
6963 __func__, ret);
8643ae66
DL		 6964 goto out;
6965 }
7eb584db 6966 }
57d104c1 6967
53c12d0e
YG		 6968 /* set the state as operational after switching to desired gear */
6969 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
a4b0e8a4 6970
71d848b8
CG		 6971 /* Enable Auto-Hibernate if configured */
6972 ufshcd_auto_hibern8_enable(hba);
6973
57d104c1
SJ		 6974 /*
6975 * If we are in error handling context or in power management callbacks
6976 * context, no need to scan the host
6977 */
6978 if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
6979 bool flag;
6980
6981 /* clear any previous UFS device information */
6982 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
dc3c8d3a
YG		 6983 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
6984 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
57d104c1 6985 hba->dev_info.f_power_on_wp_en = flag;
3441da7d 6986
3a4bf06d
YG		 6987 if (!hba->is_init_prefetch)
6988 ufshcd_init_icc_levels(hba);
6989
2a8fa600
SJ		 6990 /* Add required well known logical units to scsi mid layer */
6991 if (ufshcd_scsi_add_wlus(hba))
6992 goto out;
6993
0701e49d 6994 /* Initialize devfreq after UFS device is detected */
6995 if (ufshcd_is_clkscaling_supported(hba)) {
6996 memcpy(&hba->clk_scaling.saved_pwr_info.info,
6997 &hba->pwr_info,
6998 sizeof(struct ufs_pa_layer_attr));
6999 hba->clk_scaling.saved_pwr_info.is_valid = true;
7000 if (!hba->devfreq) {
deac444f
BA		 7001 ret = ufshcd_devfreq_init(hba);
7002 if (ret)
0701e49d 7003 goto out;
0701e49d 7004 }
7005 hba->clk_scaling.is_allowed = true;
7006 }
7007
df032bf2
AA		 7008 ufs_bsg_probe(hba);
7009
3441da7d
SRT		 7010 scsi_scan_host(hba->host);
7011 pm_runtime_put_sync(hba->dev);
7012 }
3a4bf06d
YG		 7013
7014 if (!hba->is_init_prefetch)
7015 hba->is_init_prefetch = true;
7016
5a0b0cb9 7017out:
1d337ec2
SRT		 7018 /*
7019 * If we failed to initialize the device or the device is not
7020 * present, turn off the power/clocks etc.
7021 */
57d104c1
SJ		 7022 if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
7023 pm_runtime_put_sync(hba->dev);
eebcc196 7024 ufshcd_exit_clk_scaling(hba);
1d337ec2 7025 ufshcd_hba_exit(hba);
57d104c1 7026 }
1d337ec2 7027
7ff5ab47 7028 trace_ufshcd_init(dev_name(hba->dev), ret,
7029 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7030 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 7031 return ret;
7032}
7033
7034/**
7035 * ufshcd_async_scan - asynchronous execution for probing hba
7036 * @data: data pointer to pass to this function
7037 * @cookie: cookie data
7038 */
7039static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7040{
7041 struct ufs_hba *hba = (struct ufs_hba *)data;
7042
7043 ufshcd_probe_hba(hba);
6ccf44fe
SJ		 7044}
7045
f550c65b
YG		 7046static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
7047{
7048 unsigned long flags;
7049 struct Scsi_Host *host;
7050 struct ufs_hba *hba;
7051 int index;
7052 bool found = false;
7053
7054 if (!scmd || !scmd->device || !scmd->device->host)
6600593c 7055 return BLK_EH_DONE;
f550c65b
YG		 7056
7057 host = scmd->device->host;
7058 hba = shost_priv(host);
7059 if (!hba)
6600593c 7060 return BLK_EH_DONE;
f550c65b
YG		 7061
7062 spin_lock_irqsave(host->host_lock, flags);
7063
7064 for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
7065 if (hba->lrb[index].cmd == scmd) {
7066 found = true;
7067 break;
7068 }
7069 }
7070
7071 spin_unlock_irqrestore(host->host_lock, flags);
7072
7073 /*
7074 * Bypass SCSI error handling and reset the block layer timer if this
7075 * SCSI command was not actually dispatched to UFS driver, otherwise
7076 * let SCSI layer handle the error as usual.
7077 */
6600593c 7078 return found ? BLK_EH_DONE : BLK_EH_RESET_TIMER;
f550c65b
YG		 7079}
7080
d829fc8a
SN		 7081static const struct attribute_group *ufshcd_driver_groups[] = {
7082 &ufs_sysfs_unit_descriptor_group,
ec92b59c 7083 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 7084 NULL,
7085};
7086
7a3e97b0
SY		 7087static struct scsi_host_template ufshcd_driver_template = {
7088 .module = THIS_MODULE,
7089 .name = UFSHCD,
7090 .proc_name = UFSHCD,
7091 .queuecommand = ufshcd_queuecommand,
7092 .slave_alloc = ufshcd_slave_alloc,
eeda4749 7093 .slave_configure = ufshcd_slave_configure,
7a3e97b0 7094 .slave_destroy = ufshcd_slave_destroy,
4264fd61 7095 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 7096 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 7097 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7098 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
f550c65b 7099 .eh_timed_out = ufshcd_eh_timed_out,
7a3e97b0
SY		 7100 .this_id = -1,
7101 .sg_tablesize = SG_ALL,
7102 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
7103 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 7104 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 7105 .max_host_blocked = 1,
c40ecc12 7106 .track_queue_depth = 1,
d829fc8a 7107 .sdev_groups = ufshcd_driver_groups,
4af14d11 7108 .dma_boundary = PAGE_SIZE - 1,
49615ba1 7109 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 7110};
7111
57d104c1
SJ		 7112static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7113 int ua)
7114{
7b16a07c 7115 int ret;
57d104c1 7116
7b16a07c
BA		 7117 if (!vreg)
7118 return 0;
57d104c1 7119
0487fff7
SC		 7120 /*
7121 * "set_load" operation shall be required on those regulators
7122 * which specifically configured current limitation. Otherwise
7123 * zero max_uA may cause unexpected behavior when regulator is
7124 * enabled or set as high power mode.
7125 */
7126 if (!vreg->max_uA)
7127 return 0;
7128
7b16a07c
BA		 7129 ret = regulator_set_load(vreg->reg, ua);
7130 if (ret < 0) {
7131 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7132 __func__, vreg->name, ua, ret);
57d104c1
SJ		 7133 }
7134
7135 return ret;
7136}
7137
7138static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7139 struct ufs_vreg *vreg)
7140{
73067981 7141 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 7142}
7143
7144static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7145 struct ufs_vreg *vreg)
7146{
7c7cfdcf
AH		 7147 if (!vreg)
7148 return 0;
7149
73067981 7150 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 7151}
7152
aa497613
SRT		 7153static int ufshcd_config_vreg(struct device *dev,
7154 struct ufs_vreg *vreg, bool on)
7155{
7156 int ret = 0;
72753590
GS		 7157 struct regulator *reg;
7158 const char *name;
aa497613
SRT		 7159 int min_uV, uA_load;
7160
7161 BUG_ON(!vreg);
7162
72753590
GS		 7163 reg = vreg->reg;
7164 name = vreg->name;
7165
aa497613 7166 if (regulator_count_voltages(reg) > 0) {
3b141e8c
SC		 7167 if (vreg->min_uV && vreg->max_uV) {
7168 min_uV = on ? vreg->min_uV : 0;
7169 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
7170 if (ret) {
7171 dev_err(dev,
7172 "%s: %s set voltage failed, err=%d\n",
aa497613 7173 __func__, name, ret);
3b141e8c
SC		 7174 goto out;
7175 }
aa497613
SRT		 7176 }
7177
7178 uA_load = on ? vreg->max_uA : 0;
57d104c1
SJ		 7179 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7180 if (ret)
aa497613 7181 goto out;
aa497613
SRT		 7182 }
7183out:
7184 return ret;
7185}
7186
7187static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7188{
7189 int ret = 0;
7190
73067981 7191 if (!vreg || vreg->enabled)
aa497613
SRT		 7192 goto out;
7193
7194 ret = ufshcd_config_vreg(dev, vreg, true);
7195 if (!ret)
7196 ret = regulator_enable(vreg->reg);
7197
7198 if (!ret)
7199 vreg->enabled = true;
7200 else
7201 dev_err(dev, "%s: %s enable failed, err=%d\n",
7202 __func__, vreg->name, ret);
7203out:
7204 return ret;
7205}
7206
7207static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7208{
7209 int ret = 0;
7210
73067981 7211 if (!vreg || !vreg->enabled)
aa497613
SRT		 7212 goto out;
7213
7214 ret = regulator_disable(vreg->reg);
7215
7216 if (!ret) {
7217 /* ignore errors on applying disable config */
7218 ufshcd_config_vreg(dev, vreg, false);
7219 vreg->enabled = false;
7220 } else {
7221 dev_err(dev, "%s: %s disable failed, err=%d\n",
7222 __func__, vreg->name, ret);
7223 }
7224out:
7225 return ret;
7226}
7227
7228static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7229{
7230 int ret = 0;
7231 struct device *dev = hba->dev;
7232 struct ufs_vreg_info *info = &hba->vreg_info;
7233
aa497613
SRT		 7234 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7235 if (ret)
7236 goto out;
7237
7238 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7239 if (ret)
7240 goto out;
7241
7242 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
7243 if (ret)
7244 goto out;
7245
7246out:
7247 if (ret) {
7248 ufshcd_toggle_vreg(dev, info->vccq2, false);
7249 ufshcd_toggle_vreg(dev, info->vccq, false);
7250 ufshcd_toggle_vreg(dev, info->vcc, false);
7251 }
7252 return ret;
7253}
7254
6a771a65
RS		 7255static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7256{
7257 struct ufs_vreg_info *info = &hba->vreg_info;
7258
60b7b823 7259 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 7260}
7261
aa497613
SRT		 7262static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7263{
7264 int ret = 0;
7265
7266 if (!vreg)
7267 goto out;
7268
7269 vreg->reg = devm_regulator_get(dev, vreg->name);
7270 if (IS_ERR(vreg->reg)) {
7271 ret = PTR_ERR(vreg->reg);
7272 dev_err(dev, "%s: %s get failed, err=%d\n",
7273 __func__, vreg->name, ret);
7274 }
7275out:
7276 return ret;
7277}
7278
7279static int ufshcd_init_vreg(struct ufs_hba *hba)
7280{
7281 int ret = 0;
7282 struct device *dev = hba->dev;
7283 struct ufs_vreg_info *info = &hba->vreg_info;
7284
aa497613
SRT		 7285 ret = ufshcd_get_vreg(dev, info->vcc);
7286 if (ret)
7287 goto out;
7288
7289 ret = ufshcd_get_vreg(dev, info->vccq);
7290 if (ret)
7291 goto out;
7292
7293 ret = ufshcd_get_vreg(dev, info->vccq2);
7294out:
7295 return ret;
7296}
7297
6a771a65
RS		 7298static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7299{
7300 struct ufs_vreg_info *info = &hba->vreg_info;
7301
7302 if (info)
7303 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7304
7305 return 0;
7306}
7307
57d104c1
SJ		 7308static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7309 bool skip_ref_clk)
c6e79dac
SRT		 7310{
7311 int ret = 0;
7312 struct ufs_clk_info *clki;
7313 struct list_head *head = &hba->clk_list_head;
1ab27c9c 7314 unsigned long flags;
911a0771 7315 ktime_t start = ktime_get();
7316 bool clk_state_changed = false;
c6e79dac 7317
566ec9ad 7318 if (list_empty(head))
c6e79dac
SRT		 7319 goto out;
7320
b334456e
SJ		 7321 /*
7322 * vendor specific setup_clocks ops may depend on clocks managed by
7323 * this standard driver hence call the vendor specific setup_clocks
7324 * before disabling the clocks managed here.
7325 */
7326 if (!on) {
7327 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7328 if (ret)
7329 return ret;
7330 }
1e879e8f 7331
c6e79dac
SRT		 7332 list_for_each_entry(clki, head, list) {
7333 if (!IS_ERR_OR_NULL(clki->clk)) {
57d104c1
SJ		 7334 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7335 continue;
7336
911a0771 7337 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 7338 if (on && !clki->enabled) {
7339 ret = clk_prepare_enable(clki->clk);
7340 if (ret) {
7341 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7342 __func__, clki->name, ret);
7343 goto out;
7344 }
7345 } else if (!on && clki->enabled) {
7346 clk_disable_unprepare(clki->clk);
7347 }
7348 clki->enabled = on;
7349 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7350 clki->name, on ? "en" : "dis");
7351 }
7352 }
1ab27c9c 7353
b334456e
SJ		 7354 /*
7355 * vendor specific setup_clocks ops may depend on clocks managed by
7356 * this standard driver hence call the vendor specific setup_clocks
7357 * after enabling the clocks managed here.
7358 */
7359 if (on) {
7360 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7361 if (ret)
7362 return ret;
7363 }
1e879e8f 7364
c6e79dac
SRT		 7365out:
7366 if (ret) {
7367 list_for_each_entry(clki, head, list) {
7368 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
7369 clk_disable_unprepare(clki->clk);
7370 }
7ff5ab47 7371 } else if (!ret && on) {
1ab27c9c
ST		 7372 spin_lock_irqsave(hba->host->host_lock, flags);
7373 hba->clk_gating.state = CLKS_ON;
7ff5ab47 7374 trace_ufshcd_clk_gating(dev_name(hba->dev),
7375 hba->clk_gating.state);
1ab27c9c 7376 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 7377 }
7ff5ab47 7378
911a0771 7379 if (clk_state_changed)
7380 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
7381 (on ? "on" : "off"),
7382 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 7383 return ret;
7384}
7385
57d104c1
SJ		 7386static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
7387{
7388 return __ufshcd_setup_clocks(hba, on, false);
7389}
7390
c6e79dac
SRT		 7391static int ufshcd_init_clocks(struct ufs_hba *hba)
7392{
7393 int ret = 0;
7394 struct ufs_clk_info *clki;
7395 struct device *dev = hba->dev;
7396 struct list_head *head = &hba->clk_list_head;
7397
566ec9ad 7398 if (list_empty(head))
c6e79dac
SRT		 7399 goto out;
7400
7401 list_for_each_entry(clki, head, list) {
7402 if (!clki->name)
7403 continue;
7404
7405 clki->clk = devm_clk_get(dev, clki->name);
7406 if (IS_ERR(clki->clk)) {
7407 ret = PTR_ERR(clki->clk);
7408 dev_err(dev, "%s: %s clk get failed, %d\n",
7409 __func__, clki->name, ret);
7410 goto out;
7411 }
7412
9e1e8a75
SJ		 7413 /*
7414 * Parse device ref clk freq as per device tree "ref_clk".
7415 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7416 * in ufshcd_alloc_host().
7417 */
7418 if (!strcmp(clki->name, "ref_clk"))
7419 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
7420
c6e79dac
SRT		 7421 if (clki->max_freq) {
7422 ret = clk_set_rate(clki->clk, clki->max_freq);
7423 if (ret) {
7424 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
7425 __func__, clki->name,
7426 clki->max_freq, ret);
7427 goto out;
7428 }
856b3483 7429 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 7430 }
7431 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
7432 clki->name, clk_get_rate(clki->clk));
7433 }
7434out:
7435 return ret;
7436}
7437
5c0c28a8
SRT		 7438static int ufshcd_variant_hba_init(struct ufs_hba *hba)
7439{
7440 int err = 0;
7441
7442 if (!hba->vops)
7443 goto out;
7444
0263bcd0
YG		 7445 err = ufshcd_vops_init(hba);
7446 if (err)
7447 goto out;
5c0c28a8 7448
0263bcd0
YG		 7449 err = ufshcd_vops_setup_regulators(hba, true);
7450 if (err)
7451 goto out_exit;
5c0c28a8
SRT		 7452
7453 goto out;
7454
5c0c28a8 7455out_exit:
0263bcd0 7456 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7457out:
7458 if (err)
7459 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 7460 __func__, ufshcd_get_var_name(hba), err);
5c0c28a8
SRT		 7461 return err;
7462}
7463
7464static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
7465{
7466 if (!hba->vops)
7467 return;
7468
0263bcd0 7469 ufshcd_vops_setup_regulators(hba, false);
5c0c28a8 7470
0263bcd0 7471 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7472}
7473
aa497613
SRT		 7474static int ufshcd_hba_init(struct ufs_hba *hba)
7475{
7476 int err;
7477
6a771a65
RS		 7478 /*
7479 * Handle host controller power separately from the UFS device power
7480 * rails as it will help controlling the UFS host controller power
7481 * collapse easily which is different than UFS device power collapse.
7482 * Also, enable the host controller power before we go ahead with rest
7483 * of the initialization here.
7484 */
7485 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 7486 if (err)
7487 goto out;
7488
6a771a65 7489 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 7490 if (err)
7491 goto out;
7492
6a771a65
RS		 7493 err = ufshcd_init_clocks(hba);
7494 if (err)
7495 goto out_disable_hba_vreg;
7496
7497 err = ufshcd_setup_clocks(hba, true);
7498 if (err)
7499 goto out_disable_hba_vreg;
7500
c6e79dac
SRT		 7501 err = ufshcd_init_vreg(hba);
7502 if (err)
7503 goto out_disable_clks;
7504
7505 err = ufshcd_setup_vreg(hba, true);
7506 if (err)
7507 goto out_disable_clks;
7508
aa497613
SRT		 7509 err = ufshcd_variant_hba_init(hba);
7510 if (err)
7511 goto out_disable_vreg;
7512
1d337ec2 7513 hba->is_powered = true;
aa497613
SRT		 7514 goto out;
7515
7516out_disable_vreg:
7517 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 7518out_disable_clks:
7519 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 7520out_disable_hba_vreg:
7521 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 7522out:
7523 return err;
7524}
7525
7526static void ufshcd_hba_exit(struct ufs_hba *hba)
7527{
1d337ec2
SRT		 7528 if (hba->is_powered) {
7529 ufshcd_variant_hba_exit(hba);
7530 ufshcd_setup_vreg(hba, false);
a508253d 7531 ufshcd_suspend_clkscaling(hba);
eebcc196 7532 if (ufshcd_is_clkscaling_supported(hba))
0701e49d 7533 if (hba->devfreq)
7534 ufshcd_suspend_clkscaling(hba);
1d337ec2
SRT		 7535 ufshcd_setup_clocks(hba, false);
7536 ufshcd_setup_hba_vreg(hba, false);
7537 hba->is_powered = false;
7538 }
aa497613
SRT		 7539}
7540
57d104c1
SJ		 7541static int
7542ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
7543{
7544 unsigned char cmd[6] = {REQUEST_SENSE,
7545 0,
7546 0,
7547 0,
09a5a24f 7548 UFS_SENSE_SIZE,
57d104c1
SJ		 7549 0};
7550 char *buffer;
7551 int ret;
7552
09a5a24f 7553 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 7554 if (!buffer) {
7555 ret = -ENOMEM;
7556 goto out;
7557 }
7558
fcbfffe2 7559 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 7560 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 7561 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 7562 if (ret)
7563 pr_err("%s: failed with err %d\n", __func__, ret);
7564
7565 kfree(buffer);
7566out:
7567 return ret;
7568}
7569
7570/**
7571 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7572 * power mode
7573 * @hba: per adapter instance
7574 * @pwr_mode: device power mode to set
7575 *
7576 * Returns 0 if requested power mode is set successfully
7577 * Returns non-zero if failed to set the requested power mode
7578 */
7579static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
7580 enum ufs_dev_pwr_mode pwr_mode)
7581{
7582 unsigned char cmd[6] = { START_STOP };
7583 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 7584 struct scsi_device *sdp;
7585 unsigned long flags;
57d104c1
SJ		 7586 int ret;
7587
7c48bfd0
AM		 7588 spin_lock_irqsave(hba->host->host_lock, flags);
7589 sdp = hba->sdev_ufs_device;
7590 if (sdp) {
7591 ret = scsi_device_get(sdp);
7592 if (!ret && !scsi_device_online(sdp)) {
7593 ret = -ENODEV;
7594 scsi_device_put(sdp);
7595 }
7596 } else {
7597 ret = -ENODEV;
7598 }
7599 spin_unlock_irqrestore(hba->host->host_lock, flags);
7600
7601 if (ret)
7602 return ret;
57d104c1
SJ		 7603
7604 /*
7605 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7606 * handling, which would wait for host to be resumed. Since we know
7607 * we are functional while we are here, skip host resume in error
7608 * handling context.
7609 */
7610 hba->host->eh_noresume = 1;
7611 if (hba->wlun_dev_clr_ua) {
7612 ret = ufshcd_send_request_sense(hba, sdp);
7613 if (ret)
7614 goto out;
7615 /* Unit attention condition is cleared now */
7616 hba->wlun_dev_clr_ua = false;
7617 }
7618
7619 cmd[4] = pwr_mode << 4;
7620
7621 /*
7622 * Current function would be generally called from the power management
e8064021 7623 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 7624 * already suspended childs.
7625 */
fcbfffe2
CH		 7626 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
7627 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 7628 if (ret) {
7629 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 7630 "START_STOP failed for power mode: %d, result %x\n",
7631 pwr_mode, ret);
c65be1a6 7632 if (driver_byte(ret) == DRIVER_SENSE)
21045519 7633 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 7634 }
7635
7636 if (!ret)
7637 hba->curr_dev_pwr_mode = pwr_mode;
7638out:
7c48bfd0 7639 scsi_device_put(sdp);
57d104c1
SJ		 7640 hba->host->eh_noresume = 0;
7641 return ret;
7642}
7643
7644static int ufshcd_link_state_transition(struct ufs_hba *hba,
7645 enum uic_link_state req_link_state,
7646 int check_for_bkops)
7647{
7648 int ret = 0;
7649
7650 if (req_link_state == hba->uic_link_state)
7651 return 0;
7652
7653 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
7654 ret = ufshcd_uic_hibern8_enter(hba);
7655 if (!ret)
7656 ufshcd_set_link_hibern8(hba);
7657 else
7658 goto out;
7659 }
7660 /*
7661 * If autobkops is enabled, link can't be turned off because
7662 * turning off the link would also turn off the device.
7663 */
7664 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
7665 (!check_for_bkops || (check_for_bkops &&
7666 !hba->auto_bkops_enabled))) {
f3099fbd
YG		 7667 /*
7668 * Let's make sure that link is in low power mode, we are doing
7669 * this currently by putting the link in Hibern8. Otherway to
7670 * put the link in low power mode is to send the DME end point
7671 * to device and then send the DME reset command to local
7672 * unipro. But putting the link in hibern8 is much faster.
7673 */
7674 ret = ufshcd_uic_hibern8_enter(hba);
7675 if (ret)
7676 goto out;
57d104c1
SJ		 7677 /*
7678 * Change controller state to "reset state" which
7679 * should also put the link in off/reset state
7680 */
596585a2 7681 ufshcd_hba_stop(hba, true);
57d104c1
SJ		 7682 /*
7683 * TODO: Check if we need any delay to make sure that
7684 * controller is reset
7685 */
7686 ufshcd_set_link_off(hba);
7687 }
7688
7689out:
7690 return ret;
7691}
7692
7693static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
7694{
b799fdf7
YG		 7695 /*
7696 * It seems some UFS devices may keep drawing more than sleep current
7697 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7698 * To avoid this situation, add 2ms delay before putting these UFS
7699 * rails in LPM mode.
7700 */
7701 if (!ufshcd_is_link_active(hba) &&
7702 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
7703 usleep_range(2000, 2100);
7704
57d104c1
SJ		 7705 /*
7706 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7707 * power.
7708 *
7709 * If UFS device and link is in OFF state, all power supplies (VCC,
7710 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7711 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7712 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7713 *
7714 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7715 * in low power state which would save some power.
7716 */
7717 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7718 !hba->dev_info.is_lu_power_on_wp) {
7719 ufshcd_setup_vreg(hba, false);
7720 } else if (!ufshcd_is_ufs_dev_active(hba)) {
7721 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7722 if (!ufshcd_is_link_active(hba)) {
7723 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7724 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
7725 }
7726 }
7727}
7728
7729static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
7730{
7731 int ret = 0;
7732
7733 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7734 !hba->dev_info.is_lu_power_on_wp) {
7735 ret = ufshcd_setup_vreg(hba, true);
7736 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 7737 if (!ret && !ufshcd_is_link_active(hba)) {
7738 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
7739 if (ret)
7740 goto vcc_disable;
7741 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
7742 if (ret)
7743 goto vccq_lpm;
7744 }
69d72ac8 7745 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 7746 }
7747 goto out;
7748
7749vccq_lpm:
7750 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7751vcc_disable:
7752 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7753out:
7754 return ret;
7755}
7756
7757static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
7758{
7759 if (ufshcd_is_link_off(hba))
7760 ufshcd_setup_hba_vreg(hba, false);
7761}
7762
7763static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
7764{
7765 if (ufshcd_is_link_off(hba))
7766 ufshcd_setup_hba_vreg(hba, true);
7767}
7768
7a3e97b0 7769/**
57d104c1 7770 * ufshcd_suspend - helper function for suspend operations
3b1d0580 7771 * @hba: per adapter instance
57d104c1
SJ		 7772 * @pm_op: desired low power operation type
7773 *
7774 * This function will try to put the UFS device and link into low power
7775 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7776 * (System PM level).
7777 *
7778 * If this function is called during shutdown, it will make sure that
7779 * both UFS device and UFS link is powered off.
7a3e97b0 7780 *
57d104c1
SJ		 7781 * NOTE: UFS device & link must be active before we enter in this function.
7782 *
7783 * Returns 0 for success and non-zero for failure
7a3e97b0 7784 */
57d104c1 7785static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7786{
57d104c1
SJ		 7787 int ret = 0;
7788 enum ufs_pm_level pm_lvl;
7789 enum ufs_dev_pwr_mode req_dev_pwr_mode;
7790 enum uic_link_state req_link_state;
7791
7792 hba->pm_op_in_progress = 1;
7793 if (!ufshcd_is_shutdown_pm(pm_op)) {
7794 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
7795 hba->rpm_lvl : hba->spm_lvl;
7796 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
7797 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
7798 } else {
7799 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
7800 req_link_state = UIC_LINK_OFF_STATE;
7801 }
7802
7a3e97b0 7803 /*
57d104c1
SJ		 7804 * If we can't transition into any of the low power modes
7805 * just gate the clocks.
7a3e97b0 7806 */
1ab27c9c
ST		 7807 ufshcd_hold(hba, false);
7808 hba->clk_gating.is_suspended = true;
7809
401f1e44 7810 if (hba->clk_scaling.is_allowed) {
7811 cancel_work_sync(&hba->clk_scaling.suspend_work);
7812 cancel_work_sync(&hba->clk_scaling.resume_work);
7813 ufshcd_suspend_clkscaling(hba);
7814 }
d6fcf81a 7815
57d104c1
SJ		 7816 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
7817 req_link_state == UIC_LINK_ACTIVE_STATE) {
7818 goto disable_clks;
7819 }
7a3e97b0 7820
57d104c1
SJ		 7821 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
7822 (req_link_state == hba->uic_link_state))
d6fcf81a 7823 goto enable_gating;
57d104c1
SJ		 7824
7825 /* UFS device & link must be active before we enter in this function */
7826 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
7827 ret = -EINVAL;
d6fcf81a 7828 goto enable_gating;
57d104c1
SJ		 7829 }
7830
7831 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 7832 if (ufshcd_can_autobkops_during_suspend(hba)) {
7833 /*
7834 * The device is idle with no requests in the queue,
7835 * allow background operations if bkops status shows
7836 * that performance might be impacted.
7837 */
7838 ret = ufshcd_urgent_bkops(hba);
7839 if (ret)
7840 goto enable_gating;
7841 } else {
7842 /* make sure that auto bkops is disabled */
7843 ufshcd_disable_auto_bkops(hba);
7844 }
57d104c1
SJ		 7845 }
7846
7847 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
7848 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
7849 !ufshcd_is_runtime_pm(pm_op))) {
7850 /* ensure that bkops is disabled */
7851 ufshcd_disable_auto_bkops(hba);
7852 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
7853 if (ret)
1ab27c9c 7854 goto enable_gating;
57d104c1
SJ		 7855 }
7856
7857 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
7858 if (ret)
7859 goto set_dev_active;
7860
7861 ufshcd_vreg_set_lpm(hba);
7862
7863disable_clks:
7864 /*
7865 * Call vendor specific suspend callback. As these callbacks may access
7866 * vendor specific host controller register space call them before the
7867 * host clocks are ON.
7868 */
0263bcd0
YG		 7869 ret = ufshcd_vops_suspend(hba, pm_op);
7870 if (ret)
7871 goto set_link_active;
57d104c1 7872
57d104c1
SJ		 7873 if (!ufshcd_is_link_active(hba))
7874 ufshcd_setup_clocks(hba, false);
7875 else
7876 /* If link is active, device ref_clk can't be switched off */
7877 __ufshcd_setup_clocks(hba, false, true);
7878
1ab27c9c 7879 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 7880 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
57d104c1
SJ		 7881 /*
7882 * Disable the host irq as host controller as there won't be any
0263bcd0 7883 * host controller transaction expected till resume.
57d104c1
SJ		 7884 */
7885 ufshcd_disable_irq(hba);
7886 /* Put the host controller in low power mode if possible */
7887 ufshcd_hba_vreg_set_lpm(hba);
7888 goto out;
7889
57d104c1 7890set_link_active:
401f1e44 7891 if (hba->clk_scaling.is_allowed)
7892 ufshcd_resume_clkscaling(hba);
57d104c1
SJ		 7893 ufshcd_vreg_set_hpm(hba);
7894 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
7895 ufshcd_set_link_active(hba);
7896 else if (ufshcd_is_link_off(hba))
7897 ufshcd_host_reset_and_restore(hba);
7898set_dev_active:
7899 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
7900 ufshcd_disable_auto_bkops(hba);
1ab27c9c 7901enable_gating:
401f1e44 7902 if (hba->clk_scaling.is_allowed)
7903 ufshcd_resume_clkscaling(hba);
1ab27c9c
ST		 7904 hba->clk_gating.is_suspended = false;
7905 ufshcd_release(hba);
57d104c1
SJ		 7906out:
7907 hba->pm_op_in_progress = 0;
8808b4e9
SC		 7908 if (ret)
7909 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
57d104c1 7910 return ret;
7a3e97b0
SY		 7911}
7912
7913/**
57d104c1 7914 * ufshcd_resume - helper function for resume operations
3b1d0580 7915 * @hba: per adapter instance
57d104c1 7916 * @pm_op: runtime PM or system PM
7a3e97b0 7917 *
57d104c1
SJ		 7918 * This function basically brings the UFS device, UniPro link and controller
7919 * to active state.
7920 *
7921 * Returns 0 for success and non-zero for failure
7a3e97b0 7922 */
57d104c1 7923static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7924{
57d104c1
SJ		 7925 int ret;
7926 enum uic_link_state old_link_state;
7927
7928 hba->pm_op_in_progress = 1;
7929 old_link_state = hba->uic_link_state;
7930
7931 ufshcd_hba_vreg_set_hpm(hba);
7932 /* Make sure clocks are enabled before accessing controller */
7933 ret = ufshcd_setup_clocks(hba, true);
7934 if (ret)
7935 goto out;
7936
57d104c1
SJ		 7937 /* enable the host irq as host controller would be active soon */
7938 ret = ufshcd_enable_irq(hba);
7939 if (ret)
7940 goto disable_irq_and_vops_clks;
7941
7942 ret = ufshcd_vreg_set_hpm(hba);
7943 if (ret)
7944 goto disable_irq_and_vops_clks;
7945
7a3e97b0 7946 /*
57d104c1
SJ		 7947 * Call vendor specific resume callback. As these callbacks may access
7948 * vendor specific host controller register space call them when the
7949 * host clocks are ON.
7a3e97b0 7950 */
0263bcd0
YG		 7951 ret = ufshcd_vops_resume(hba, pm_op);
7952 if (ret)
7953 goto disable_vreg;
57d104c1
SJ		 7954
7955 if (ufshcd_is_link_hibern8(hba)) {
7956 ret = ufshcd_uic_hibern8_exit(hba);
7957 if (!ret)
7958 ufshcd_set_link_active(hba);
7959 else
7960 goto vendor_suspend;
7961 } else if (ufshcd_is_link_off(hba)) {
7962 ret = ufshcd_host_reset_and_restore(hba);
7963 /*
7964 * ufshcd_host_reset_and_restore() should have already
7965 * set the link state as active
7966 */
7967 if (ret || !ufshcd_is_link_active(hba))
7968 goto vendor_suspend;
7969 }
7970
7971 if (!ufshcd_is_ufs_dev_active(hba)) {
7972 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
7973 if (ret)
7974 goto set_old_link_state;
7975 }
7976
4e768e76 7977 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
7978 ufshcd_enable_auto_bkops(hba);
7979 else
7980 /*
7981 * If BKOPs operations are urgently needed at this moment then
7982 * keep auto-bkops enabled or else disable it.
7983 */
7984 ufshcd_urgent_bkops(hba);
7985
1ab27c9c
ST		 7986 hba->clk_gating.is_suspended = false;
7987
fcb0c4b0
ST		 7988 if (hba->clk_scaling.is_allowed)
7989 ufshcd_resume_clkscaling(hba);
856b3483 7990
ad448378
AH		 7991 /* Enable Auto-Hibernate if configured */
7992 ufshcd_auto_hibern8_enable(hba);
7993
71d848b8
CG		 7994 /* Schedule clock gating in case of no access to UFS device yet */
7995 ufshcd_release(hba);
7996
57d104c1
SJ		 7997 goto out;
7998
7999set_old_link_state:
8000 ufshcd_link_state_transition(hba, old_link_state, 0);
8001vendor_suspend:
0263bcd0 8002 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 8003disable_vreg:
8004 ufshcd_vreg_set_lpm(hba);
8005disable_irq_and_vops_clks:
8006 ufshcd_disable_irq(hba);
401f1e44 8007 if (hba->clk_scaling.is_allowed)
8008 ufshcd_suspend_clkscaling(hba);
57d104c1
SJ		 8009 ufshcd_setup_clocks(hba, false);
8010out:
8011 hba->pm_op_in_progress = 0;
8808b4e9
SC		 8012 if (ret)
8013 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
57d104c1
SJ		 8014 return ret;
8015}
8016
8017/**
8018 * ufshcd_system_suspend - system suspend routine
8019 * @hba: per adapter instance
57d104c1
SJ		 8020 *
8021 * Check the description of ufshcd_suspend() function for more details.
8022 *
8023 * Returns 0 for success and non-zero for failure
8024 */
8025int ufshcd_system_suspend(struct ufs_hba *hba)
8026{
8027 int ret = 0;
7ff5ab47 8028 ktime_t start = ktime_get();
57d104c1
SJ		 8029
8030 if (!hba || !hba->is_powered)
233b594b 8031 return 0;
57d104c1 8032
0b257734 8033 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8034 hba->curr_dev_pwr_mode) &&
8035 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8036 hba->uic_link_state))
8037 goto out;
57d104c1 8038
0b257734 8039 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 8040 /*
8041 * UFS device and/or UFS link low power states during runtime
8042 * suspend seems to be different than what is expected during
8043 * system suspend. Hence runtime resume the devic & link and
8044 * let the system suspend low power states to take effect.
8045 * TODO: If resume takes longer time, we might have optimize
8046 * it in future by not resuming everything if possible.
8047 */
8048 ret = ufshcd_runtime_resume(hba);
8049 if (ret)
8050 goto out;
8051 }
8052
8053 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8054out:
7ff5ab47 8055 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8056 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8057 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 8058 if (!ret)
8059 hba->is_sys_suspended = true;
57d104c1
SJ		 8060 return ret;
8061}
8062EXPORT_SYMBOL(ufshcd_system_suspend);
8063
8064/**
8065 * ufshcd_system_resume - system resume routine
8066 * @hba: per adapter instance
8067 *
8068 * Returns 0 for success and non-zero for failure
8069 */
7a3e97b0 8070
57d104c1
SJ		 8071int ufshcd_system_resume(struct ufs_hba *hba)
8072{
7ff5ab47 8073 int ret = 0;
8074 ktime_t start = ktime_get();
8075
e3ce73d6
YG		 8076 if (!hba)
8077 return -EINVAL;
8078
8079 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 8080 /*
8081 * Let the runtime resume take care of resuming
8082 * if runtime suspended.
8083 */
7ff5ab47 8084 goto out;
8085 else
8086 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8087out:
8088 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8089 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8090 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 8091 if (!ret)
8092 hba->is_sys_suspended = false;
7ff5ab47 8093 return ret;
7a3e97b0 8094}
57d104c1 8095EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 8096
57d104c1
SJ		 8097/**
8098 * ufshcd_runtime_suspend - runtime suspend routine
8099 * @hba: per adapter instance
8100 *
8101 * Check the description of ufshcd_suspend() function for more details.
8102 *
8103 * Returns 0 for success and non-zero for failure
8104 */
66ec6d59
SRT		 8105int ufshcd_runtime_suspend(struct ufs_hba *hba)
8106{
7ff5ab47 8107 int ret = 0;
8108 ktime_t start = ktime_get();
8109
e3ce73d6
YG		 8110 if (!hba)
8111 return -EINVAL;
8112
8113 if (!hba->is_powered)
7ff5ab47 8114 goto out;
8115 else
8116 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8117out:
8118 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8119 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8120 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8121 return ret;
66ec6d59
SRT		 8122}
8123EXPORT_SYMBOL(ufshcd_runtime_suspend);
8124
57d104c1
SJ		 8125/**
8126 * ufshcd_runtime_resume - runtime resume routine
8127 * @hba: per adapter instance
8128 *
8129 * This function basically brings the UFS device, UniPro link and controller
8130 * to active state. Following operations are done in this function:
8131 *
8132 * 1. Turn on all the controller related clocks
8133 * 2. Bring the UniPro link out of Hibernate state
8134 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8135 * to active state.
8136 * 4. If auto-bkops is enabled on the device, disable it.
8137 *
8138 * So following would be the possible power state after this function return
8139 * successfully:
8140 * S1: UFS device in Active state with VCC rail ON
8141 * UniPro link in Active state
8142 * All the UFS/UniPro controller clocks are ON
8143 *
8144 * Returns 0 for success and non-zero for failure
8145 */
66ec6d59
SRT		 8146int ufshcd_runtime_resume(struct ufs_hba *hba)
8147{
7ff5ab47 8148 int ret = 0;
8149 ktime_t start = ktime_get();
8150
e3ce73d6
YG		 8151 if (!hba)
8152 return -EINVAL;
8153
8154 if (!hba->is_powered)
7ff5ab47 8155 goto out;
8156 else
8157 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8158out:
8159 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8160 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8161 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8162 return ret;
66ec6d59
SRT		 8163}
8164EXPORT_SYMBOL(ufshcd_runtime_resume);
8165
8166int ufshcd_runtime_idle(struct ufs_hba *hba)
8167{
8168 return 0;
8169}
8170EXPORT_SYMBOL(ufshcd_runtime_idle);
8171
57d104c1
SJ		 8172/**
8173 * ufshcd_shutdown - shutdown routine
8174 * @hba: per adapter instance
8175 *
8176 * This function would power off both UFS device and UFS link.
8177 *
8178 * Returns 0 always to allow force shutdown even in case of errors.
8179 */
8180int ufshcd_shutdown(struct ufs_hba *hba)
8181{
8182 int ret = 0;
8183
f51913ee
SC		 8184 if (!hba->is_powered)
8185 goto out;
8186
57d104c1
SJ		 8187 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8188 goto out;
8189
8190 if (pm_runtime_suspended(hba->dev)) {
8191 ret = ufshcd_runtime_resume(hba);
8192 if (ret)
8193 goto out;
8194 }
8195
8196 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8197out:
8198 if (ret)
8199 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8200 /* allow force shutdown even in case of errors */
8201 return 0;
8202}
8203EXPORT_SYMBOL(ufshcd_shutdown);
8204
7a3e97b0 8205/**
3b1d0580 8206 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 8207 * data structure memory
8aa29f19 8208 * @hba: per adapter instance
7a3e97b0 8209 */
3b1d0580 8210void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 8211{
df032bf2 8212 ufs_bsg_remove(hba);
cbb6813e 8213 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 8214 blk_cleanup_queue(hba->tmf_queue);
8215 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360 8216 blk_cleanup_queue(hba->cmd_queue);
cfdf9c91 8217 scsi_remove_host(hba->host);
7a3e97b0 8218 /* disable interrupts */
2fbd009b 8219 ufshcd_disable_intr(hba, hba->intr_mask);
596585a2 8220 ufshcd_hba_stop(hba, true);
7a3e97b0 8221
eebcc196 8222 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8223 ufshcd_exit_clk_gating(hba);
fcb0c4b0
ST		 8224 if (ufshcd_is_clkscaling_supported(hba))
8225 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
aa497613 8226 ufshcd_hba_exit(hba);
3b1d0580
VH		 8227}
8228EXPORT_SYMBOL_GPL(ufshcd_remove);
8229
47555a5c
YG		 8230/**
8231 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8232 * @hba: pointer to Host Bus Adapter (HBA)
8233 */
8234void ufshcd_dealloc_host(struct ufs_hba *hba)
8235{
8236 scsi_host_put(hba->host);
8237}
8238EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8239
ca3d7bf9
AM		 8240/**
8241 * ufshcd_set_dma_mask - Set dma mask based on the controller
8242 * addressing capability
8243 * @hba: per adapter instance
8244 *
8245 * Returns 0 for success, non-zero for failure
8246 */
8247static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8248{
8249 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8250 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8251 return 0;
8252 }
8253 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8254}
8255
7a3e97b0 8256/**
5c0c28a8 8257 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 8258 * @dev: pointer to device handle
8259 * @hba_handle: driver private handle
7a3e97b0
SY		 8260 * Returns 0 on success, non-zero value on failure
8261 */
5c0c28a8 8262int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 8263{
8264 struct Scsi_Host *host;
8265 struct ufs_hba *hba;
5c0c28a8 8266 int err = 0;
7a3e97b0 8267
3b1d0580
VH		 8268 if (!dev) {
8269 dev_err(dev,
8270 "Invalid memory reference for dev is NULL\n");
8271 err = -ENODEV;
7a3e97b0
SY		 8272 goto out_error;
8273 }
8274
7a3e97b0
SY		 8275 host = scsi_host_alloc(&ufshcd_driver_template,
8276 sizeof(struct ufs_hba));
8277 if (!host) {
3b1d0580 8278 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 8279 err = -ENOMEM;
3b1d0580 8280 goto out_error;
7a3e97b0
SY		 8281 }
8282 hba = shost_priv(host);
7a3e97b0 8283 hba->host = host;
3b1d0580 8284 hba->dev = dev;
5c0c28a8 8285 *hba_handle = hba;
9e1e8a75 8286 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 8287
566ec9ad
SM		 8288 INIT_LIST_HEAD(&hba->clk_list_head);
8289
5c0c28a8
SRT		 8290out_error:
8291 return err;
8292}
8293EXPORT_SYMBOL(ufshcd_alloc_host);
8294
69a6c269
BVA		 8295/* This function exists because blk_mq_alloc_tag_set() requires this. */
8296static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
8297 const struct blk_mq_queue_data *qd)
8298{
8299 WARN_ON_ONCE(true);
8300 return BLK_STS_NOTSUPP;
8301}
8302
8303static const struct blk_mq_ops ufshcd_tmf_ops = {
8304 .queue_rq = ufshcd_queue_tmf,
8305};
8306
5c0c28a8
SRT		 8307/**
8308 * ufshcd_init - Driver initialization routine
8309 * @hba: per-adapter instance
8310 * @mmio_base: base register address
8311 * @irq: Interrupt line of device
8312 * Returns 0 on success, non-zero value on failure
8313 */
8314int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
8315{
8316 int err;
8317 struct Scsi_Host *host = hba->host;
8318 struct device *dev = hba->dev;
8319
8320 if (!mmio_base) {
8321 dev_err(hba->dev,
8322 "Invalid memory reference for mmio_base is NULL\n");
8323 err = -ENODEV;
8324 goto out_error;
8325 }
8326
3b1d0580
VH		 8327 hba->mmio_base = mmio_base;
8328 hba->irq = irq;
7a3e97b0 8329
aa497613 8330 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 8331 if (err)
8332 goto out_error;
8333
7a3e97b0
SY		 8334 /* Read capabilities registers */
8335 ufshcd_hba_capabilities(hba);
8336
8337 /* Get UFS version supported by the controller */
8338 hba->ufs_version = ufshcd_get_ufs_version(hba);
8339
c01848c6
YG		 8340 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
8341 (hba->ufs_version != UFSHCI_VERSION_11) &&
8342 (hba->ufs_version != UFSHCI_VERSION_20) &&
8343 (hba->ufs_version != UFSHCI_VERSION_21))
8344 dev_err(hba->dev, "invalid UFS version 0x%x\n",
8345 hba->ufs_version);
8346
2fbd009b
SJ		 8347 /* Get Interrupt bit mask per version */
8348 hba->intr_mask = ufshcd_get_intr_mask(hba);
8349
ca3d7bf9
AM		 8350 err = ufshcd_set_dma_mask(hba);
8351 if (err) {
8352 dev_err(hba->dev, "set dma mask failed\n");
8353 goto out_disable;
8354 }
8355
7a3e97b0
SY		 8356 /* Allocate memory for host memory space */
8357 err = ufshcd_memory_alloc(hba);
8358 if (err) {
3b1d0580
VH		 8359 dev_err(hba->dev, "Memory allocation failed\n");
8360 goto out_disable;
7a3e97b0
SY		 8361 }
8362
8363 /* Configure LRB */
8364 ufshcd_host_memory_configure(hba);
8365
8366 host->can_queue = hba->nutrs;
8367 host->cmd_per_lun = hba->nutrs;
8368 host->max_id = UFSHCD_MAX_ID;
0ce147d4 8369 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 8370 host->max_channel = UFSHCD_MAX_CHANNEL;
8371 host->unique_id = host->host_no;
a851b2bd 8372 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 8373
7eb584db
DR		 8374 hba->max_pwr_info.is_valid = false;
8375
7a3e97b0 8376 /* Initialize work queues */
e8e7f271 8377 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 8378 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 8379
6ccf44fe
SJ		 8380 /* Initialize UIC command mutex */
8381 mutex_init(&hba->uic_cmd_mutex);
8382
5a0b0cb9
SRT		 8383 /* Initialize mutex for device management commands */
8384 mutex_init(&hba->dev_cmd.lock);
8385
a3cd5ec5 8386 init_rwsem(&hba->clk_scaling_lock);
8387
1ab27c9c 8388 ufshcd_init_clk_gating(hba);
199ef13c 8389
eebcc196
VG		 8390 ufshcd_init_clk_scaling(hba);
8391
199ef13c
YG		 8392 /*
8393 * In order to avoid any spurious interrupt immediately after
8394 * registering UFS controller interrupt handler, clear any pending UFS
8395 * interrupt status and disable all the UFS interrupts.
8396 */
8397 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
8398 REG_INTERRUPT_STATUS);
8399 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
8400 /*
8401 * Make sure that UFS interrupts are disabled and any pending interrupt
8402 * status is cleared before registering UFS interrupt handler.
8403 */
8404 mb();
8405
7a3e97b0 8406 /* IRQ registration */
2953f850 8407 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 8408 if (err) {
3b1d0580 8409 dev_err(hba->dev, "request irq failed\n");
1ab27c9c 8410 goto exit_gating;
57d104c1
SJ		 8411 } else {
8412 hba->is_irq_enabled = true;
7a3e97b0
SY		 8413 }
8414
3b1d0580 8415 err = scsi_add_host(host, hba->dev);
7a3e97b0 8416 if (err) {
3b1d0580 8417 dev_err(hba->dev, "scsi_add_host failed\n");
1ab27c9c 8418 goto exit_gating;
7a3e97b0
SY		 8419 }
8420
7252a360
BVA		 8421 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
8422 if (IS_ERR(hba->cmd_queue)) {
8423 err = PTR_ERR(hba->cmd_queue);
8424 goto out_remove_scsi_host;
8425 }
8426
69a6c269
BVA		 8427 hba->tmf_tag_set = (struct blk_mq_tag_set) {
8428 .nr_hw_queues = 1,
8429 .queue_depth = hba->nutmrs,
8430 .ops = &ufshcd_tmf_ops,
8431 .flags = BLK_MQ_F_NO_SCHED,
8432 };
8433 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
8434 if (err < 0)
8435 goto free_cmd_queue;
8436 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
8437 if (IS_ERR(hba->tmf_queue)) {
8438 err = PTR_ERR(hba->tmf_queue);
8439 goto free_tmf_tag_set;
8440 }
8441
d8d9f793
BA		 8442 /* Reset the attached device */
8443 ufshcd_vops_device_reset(hba);
8444
6ccf44fe
SJ		 8445 /* Host controller enable */
8446 err = ufshcd_hba_enable(hba);
7a3e97b0 8447 if (err) {
6ccf44fe 8448 dev_err(hba->dev, "Host controller enable failed\n");
66cc820f 8449 ufshcd_print_host_regs(hba);
6ba65588 8450 ufshcd_print_host_state(hba);
69a6c269 8451 goto free_tmf_queue;
7a3e97b0 8452 }
6ccf44fe 8453
0c8f7586 8454 /*
8455 * Set the default power management level for runtime and system PM.
8456 * Default power saving mode is to keep UFS link in Hibern8 state
8457 * and UFS device in sleep state.
8458 */
8459 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8460 UFS_SLEEP_PWR_MODE,
8461 UIC_LINK_HIBERN8_STATE);
8462 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8463 UFS_SLEEP_PWR_MODE,
8464 UIC_LINK_HIBERN8_STATE);
8465
ad448378 8466 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 8467 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 8468 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
8469 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
8470 }
8471
62694735
SRT		 8472 /* Hold auto suspend until async scan completes */
8473 pm_runtime_get_sync(dev);
38135535 8474 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 8475 /*
7caf489b 8476 * We are assuming that device wasn't put in sleep/power-down
8477 * state exclusively during the boot stage before kernel.
8478 * This assumption helps avoid doing link startup twice during
8479 * ufshcd_probe_hba().
57d104c1 8480 */
7caf489b 8481 ufshcd_set_ufs_dev_active(hba);
57d104c1 8482
6ccf44fe 8483 async_schedule(ufshcd_async_scan, hba);
cbb6813e 8484 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 8485
7a3e97b0
SY		 8486 return 0;
8487
69a6c269
BVA		 8488free_tmf_queue:
8489 blk_cleanup_queue(hba->tmf_queue);
8490free_tmf_tag_set:
8491 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360
BVA		 8492free_cmd_queue:
8493 blk_cleanup_queue(hba->cmd_queue);
3b1d0580
VH		 8494out_remove_scsi_host:
8495 scsi_remove_host(hba->host);
1ab27c9c 8496exit_gating:
eebcc196 8497 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8498 ufshcd_exit_clk_gating(hba);
3b1d0580 8499out_disable:
57d104c1 8500 hba->is_irq_enabled = false;
aa497613 8501 ufshcd_hba_exit(hba);
3b1d0580
VH		 8502out_error:
8503 return err;
8504}
8505EXPORT_SYMBOL_GPL(ufshcd_init);
8506
3b1d0580
VH		 8507MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
8508MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 8509MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 8510MODULE_LICENSE("GPL");
8511MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Linux 6.x block layer and io_uring tree(s)