projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: libsas: delete sas port if expander discover failed
[linux-2.6-block.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
7a3e97b0 1/*
e0eca63e 2 * Universal Flash Storage Host controller driver Core
7a3e97b0
SY		 3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
3b1d0580 5 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 7 *
3b1d0580
VH		 8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
3b1d0580
VH		 16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
7a3e97b0
SY		 18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
3b1d0580
VH		 24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
5c0c28a8
SRT		 35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
7a3e97b0
SY		 38 */
39
6ccf44fe 40#include <linux/async.h>
856b3483 41#include <linux/devfreq.h>
b573d484 42#include <linux/nls.h>
54b879b7 43#include <linux/of.h>
ad448378 44#include <linux/bitfield.h>
e0eca63e 45#include "ufshcd.h"
c58ab7aa 46#include "ufs_quirks.h"
53b3d9c3 47#include "unipro.h"
cbb6813e 48#include "ufs-sysfs.h"
df032bf2 49#include "ufs_bsg.h"
7a3e97b0 50
7ff5ab47 51#define CREATE_TRACE_POINTS
52#include <trace/events/ufs.h>
53
2fbd009b
SJ		 54#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
55 UTP_TASK_REQ_COMPL |\
56 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 57/* UIC command timeout, unit: ms */
58#define UIC_CMD_TIMEOUT 500
2fbd009b 59
5a0b0cb9
SRT		 60/* NOP OUT retries waiting for NOP IN response */
61#define NOP_OUT_RETRIES 10
62/* Timeout after 30 msecs if NOP OUT hangs without response */
63#define NOP_OUT_TIMEOUT 30 /* msecs */
64
68078d5c 65/* Query request retries */
10fe5888 66#define QUERY_REQ_RETRIES 3
68078d5c 67/* Query request timeout */
10fe5888 68#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 69
e2933132
SRT		 70/* Task management command timeout */
71#define TM_CMD_TIMEOUT 100 /* msecs */
72
64238fbd
YG		 73/* maximum number of retries for a general UIC command */
74#define UFS_UIC_COMMAND_RETRIES 3
75
1d337ec2
SRT		 76/* maximum number of link-startup retries */
77#define DME_LINKSTARTUP_RETRIES 3
78
87d0b4a6
YG		 79/* Maximum retries for Hibern8 enter */
80#define UIC_HIBERN8_ENTER_RETRIES 3
81
1d337ec2
SRT		 82/* maximum number of reset retries before giving up */
83#define MAX_HOST_RESET_RETRIES 5
84
68078d5c
DR		 85/* Expose the flag value from utp_upiu_query.value */
86#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87
7d568652
SJ		 88/* Interrupt aggregation default timeout, unit: 40us */
89#define INT_AGGR_DEF_TO 0x02
90
aa497613
SRT		 91#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
92 ({ \
93 int _ret; \
94 if (_on) \
95 _ret = ufshcd_enable_vreg(_dev, _vreg); \
96 else \
97 _ret = ufshcd_disable_vreg(_dev, _vreg); \
98 _ret; \
99 })
100
ba80917d
TW		 101#define ufshcd_hex_dump(prefix_str, buf, len) do { \
102 size_t __len = (len); \
103 print_hex_dump(KERN_ERR, prefix_str, \
104 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
105 16, 4, buf, __len, false); \
106} while (0)
107
108int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
109 const char *prefix)
110{
d6724756
MG		 111 u32 *regs;
112 size_t pos;
113
114 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
115 return -EINVAL;
ba80917d
TW		 116
117 regs = kzalloc(len, GFP_KERNEL);
118 if (!regs)
119 return -ENOMEM;
120
d6724756
MG		 121 for (pos = 0; pos < len; pos += 4)
122 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
123
ba80917d
TW		 124 ufshcd_hex_dump(prefix, regs, len);
125 kfree(regs);
126
127 return 0;
128}
129EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 130
7a3e97b0
SY		 131enum {
132 UFSHCD_MAX_CHANNEL = 0,
133 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 134 UFSHCD_CMD_PER_LUN = 32,
135 UFSHCD_CAN_QUEUE = 32,
136};
137
138/* UFSHCD states */
139enum {
7a3e97b0
SY		 140 UFSHCD_STATE_RESET,
141 UFSHCD_STATE_ERROR,
3441da7d 142 UFSHCD_STATE_OPERATIONAL,
141f8165 143 UFSHCD_STATE_EH_SCHEDULED,
3441da7d
SRT		 144};
145
146/* UFSHCD error handling flags */
147enum {
148 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 149};
150
e8e7f271
SRT		 151/* UFSHCD UIC layer error flags */
152enum {
153 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 154 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
155 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
156 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
157 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
158 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
e8e7f271
SRT		 159};
160
3441da7d 161#define ufshcd_set_eh_in_progress(h) \
9c490d2d 162 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 163#define ufshcd_eh_in_progress(h) \
9c490d2d 164 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 165#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 166 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 167
57d104c1
SJ		 168#define ufshcd_set_ufs_dev_active(h) \
169 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
170#define ufshcd_set_ufs_dev_sleep(h) \
171 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
172#define ufshcd_set_ufs_dev_poweroff(h) \
173 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
174#define ufshcd_is_ufs_dev_active(h) \
175 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
176#define ufshcd_is_ufs_dev_sleep(h) \
177 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
178#define ufshcd_is_ufs_dev_poweroff(h) \
179 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
180
cbb6813e 181struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 182 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
183 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
184 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
185 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
186 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
187 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
188};
189
190static inline enum ufs_dev_pwr_mode
191ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
192{
193 return ufs_pm_lvl_states[lvl].dev_state;
194}
195
196static inline enum uic_link_state
197ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
198{
199 return ufs_pm_lvl_states[lvl].link_state;
200}
201
0c8f7586 202static inline enum ufs_pm_level
203ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
204 enum uic_link_state link_state)
205{
206 enum ufs_pm_level lvl;
207
208 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
209 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
210 (ufs_pm_lvl_states[lvl].link_state == link_state))
211 return lvl;
212 }
213
214 /* if no match found, return the level 0 */
215 return UFS_PM_LVL_0;
216}
217
56d4a186
SJ		 218static struct ufs_dev_fix ufs_fixups[] = {
219 /* UFS cards deviations table */
220 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
221 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186
SJ		 222 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
223 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
56d4a186
SJ		 224 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
225 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
226 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
227 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
228 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
229 UFS_DEVICE_QUIRK_PA_TACTIVATE),
230 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
231 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 232 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
233 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
8e4829c6
WL		 234 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
235 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 236
237 END_FIX
238};
239
3441da7d
SRT		 240static void ufshcd_tmc_handler(struct ufs_hba *hba);
241static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 242static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 243static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 244static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2
SRT		 245static void ufshcd_hba_exit(struct ufs_hba *hba);
246static int ufshcd_probe_hba(struct ufs_hba *hba);
1ab27c9c
ST		 247static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
248 bool skip_ref_clk);
249static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
250static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
251static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 252static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 253static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 254static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
255static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 256static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 257static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 258static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 259static int ufshcd_change_power_mode(struct ufs_hba *hba,
260 struct ufs_pa_layer_attr *pwr_mode);
14497328
YG		 261static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
262{
263 return tag >= 0 && tag < hba->nutrs;
264}
57d104c1
SJ		 265
266static inline int ufshcd_enable_irq(struct ufs_hba *hba)
267{
268 int ret = 0;
269
270 if (!hba->is_irq_enabled) {
271 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
272 hba);
273 if (ret)
274 dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
275 __func__, ret);
276 hba->is_irq_enabled = true;
277 }
278
279 return ret;
280}
281
282static inline void ufshcd_disable_irq(struct ufs_hba *hba)
283{
284 if (hba->is_irq_enabled) {
285 free_irq(hba->irq, hba);
286 hba->is_irq_enabled = false;
287 }
288}
3441da7d 289
38135535
SJ		 290static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
291{
292 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
293 scsi_unblock_requests(hba->host);
294}
295
296static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
297{
298 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
299 scsi_block_requests(hba->host);
300}
301
b573d484
YG		 302/* replace non-printable or non-ASCII characters with spaces */
303static inline void ufshcd_remove_non_printable(char *val)
304{
305 if (!val)
306 return;
307
308 if (*val < 0x20 || *val > 0x7e)
309 *val = ' ';
310}
311
6667e6d9
OS		 312static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
313 const char *str)
314{
315 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
316
317 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
318}
319
320static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
321 const char *str)
322{
323 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
324
325 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
326}
327
328static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
329 const char *str)
330{
6667e6d9 331 int off = (int)tag - hba->nutrs;
391e388f 332 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 333
391e388f
CH		 334 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
335 &descp->input_param1);
6667e6d9
OS		 336}
337
1a07f2d9
LS		 338static void ufshcd_add_command_trace(struct ufs_hba *hba,
339 unsigned int tag, const char *str)
340{
341 sector_t lba = -1;
342 u8 opcode = 0;
343 u32 intr, doorbell;
e7c3b379 344 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
1a07f2d9
LS		 345 int transfer_len = -1;
346
e7c3b379
OS		 347 if (!trace_ufshcd_command_enabled()) {
348 /* trace UPIU W/O tracing command */
349 if (lrbp->cmd)
350 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9 351 return;
e7c3b379 352 }
1a07f2d9
LS		 353
354 if (lrbp->cmd) { /* data phase exists */
e7c3b379
OS		 355 /* trace UPIU also */
356 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9
LS		 357 opcode = (u8)(*lrbp->cmd->cmnd);
358 if ((opcode == READ_10) || (opcode == WRITE_10)) {
359 /*
360 * Currently we only fully trace read(10) and write(10)
361 * commands
362 */
363 if (lrbp->cmd->request && lrbp->cmd->request->bio)
364 lba =
365 lrbp->cmd->request->bio->bi_iter.bi_sector;
366 transfer_len = be32_to_cpu(
367 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
368 }
369 }
370
371 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
372 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
373 trace_ufshcd_command(dev_name(hba->dev), str, tag,
374 doorbell, transfer_len, intr, lba, opcode);
375}
376
ff8e20c6
DR		 377static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
378{
379 struct ufs_clk_info *clki;
380 struct list_head *head = &hba->clk_list_head;
381
566ec9ad 382 if (list_empty(head))
ff8e20c6
DR		 383 return;
384
385 list_for_each_entry(clki, head, list) {
386 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
387 clki->max_freq)
388 dev_err(hba->dev, "clk: %s, rate: %u\n",
389 clki->name, clki->curr_freq);
390 }
391}
392
393static void ufshcd_print_uic_err_hist(struct ufs_hba *hba,
394 struct ufs_uic_err_reg_hist *err_hist, char *err_name)
395{
396 int i;
27752647 397 bool found = false;
ff8e20c6
DR		 398
399 for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) {
27752647 400 int p = (i + err_hist->pos) % UIC_ERR_REG_HIST_LENGTH;
ff8e20c6
DR		 401
402 if (err_hist->reg[p] == 0)
403 continue;
404 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i,
405 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
27752647 406 found = true;
ff8e20c6 407 }
27752647
SC		 408
409 if (!found)
410 dev_err(hba->dev, "No record of %s uic errors\n", err_name);
ff8e20c6
DR		 411}
412
66cc820f
DR		 413static void ufshcd_print_host_regs(struct ufs_hba *hba)
414{
ba80917d 415 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
66cc820f
DR		 416 dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
417 hba->ufs_version, hba->capabilities);
418 dev_err(hba->dev,
419 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
420 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
ff8e20c6
DR		 421 dev_err(hba->dev,
422 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
423 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
424 hba->ufs_stats.hibern8_exit_cnt);
425
426 ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
427 ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
428 ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
429 ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
430 ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
431
432 ufshcd_print_clk_freqs(hba);
433
434 if (hba->vops && hba->vops->dbg_register_dump)
435 hba->vops->dbg_register_dump(hba);
66cc820f
DR		 436}
437
438static
439void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
440{
441 struct ufshcd_lrb *lrbp;
7fabb77b 442 int prdt_length;
66cc820f
DR		 443 int tag;
444
445 for_each_set_bit(tag, &bitmap, hba->nutrs) {
446 lrbp = &hba->lrb[tag];
447
ff8e20c6
DR		 448 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
449 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 450 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
451 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 452 dev_err(hba->dev,
453 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
454 tag, (u64)lrbp->utrd_dma_addr);
455
66cc820f
DR		 456 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
457 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 458 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
459 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 460 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
461 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 462 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
463 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 464 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
465 sizeof(struct utp_upiu_rsp));
66cc820f 466
7fabb77b
GB		 467 prdt_length = le16_to_cpu(
468 lrbp->utr_descriptor_ptr->prd_table_length);
469 dev_err(hba->dev,
470 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
471 tag, prdt_length,
472 (u64)lrbp->ucd_prdt_dma_addr);
473
474 if (pr_prdt)
66cc820f 475 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 476 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 477 }
478}
479
480static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
481{
66cc820f
DR		 482 int tag;
483
484 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 485 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
486
66cc820f 487 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 488 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 489 }
490}
491
6ba65588
GB		 492static void ufshcd_print_host_state(struct ufs_hba *hba)
493{
494 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
495 dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n",
e002e651 496 hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 497 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
498 hba->saved_err, hba->saved_uic_err);
499 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
500 hba->curr_dev_pwr_mode, hba->uic_link_state);
501 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
502 hba->pm_op_in_progress, hba->is_sys_suspended);
503 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
504 hba->auto_bkops_enabled, hba->host->host_self_blocked);
505 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
506 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
507 hba->eh_flags, hba->req_abort_count);
508 dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
509 hba->capabilities, hba->caps);
510 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
511 hba->dev_quirks);
512}
513
ff8e20c6
DR		 514/**
515 * ufshcd_print_pwr_info - print power params as saved in hba
516 * power info
517 * @hba: per-adapter instance
518 */
519static void ufshcd_print_pwr_info(struct ufs_hba *hba)
520{
521 static const char * const names[] = {
522 "INVALID MODE",
523 "FAST MODE",
524 "SLOW_MODE",
525 "INVALID MODE",
526 "FASTAUTO_MODE",
527 "SLOWAUTO_MODE",
528 "INVALID MODE",
529 };
530
531 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
532 __func__,
533 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
534 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
535 names[hba->pwr_info.pwr_rx],
536 names[hba->pwr_info.pwr_tx],
537 hba->pwr_info.hs_rate);
538}
539
5a0b0cb9
SRT		 540/*
541 * ufshcd_wait_for_register - wait for register value to change
542 * @hba - per-adapter interface
543 * @reg - mmio register offset
544 * @mask - mask to apply to read register value
545 * @val - wait condition
546 * @interval_us - polling interval in microsecs
547 * @timeout_ms - timeout in millisecs
596585a2 548 * @can_sleep - perform sleep or just spin
5a0b0cb9
SRT		 549 *
550 * Returns -ETIMEDOUT on error, zero on success
551 */
596585a2
YG		 552int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
553 u32 val, unsigned long interval_us,
554 unsigned long timeout_ms, bool can_sleep)
5a0b0cb9
SRT		 555{
556 int err = 0;
557 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
558
559 /* ignore bits that we don't intend to wait on */
560 val = val & mask;
561
562 while ((ufshcd_readl(hba, reg) & mask) != val) {
596585a2
YG		 563 if (can_sleep)
564 usleep_range(interval_us, interval_us + 50);
565 else
566 udelay(interval_us);
5a0b0cb9
SRT		 567 if (time_after(jiffies, timeout)) {
568 if ((ufshcd_readl(hba, reg) & mask) != val)
569 err = -ETIMEDOUT;
570 break;
571 }
572 }
573
574 return err;
575}
576
2fbd009b
SJ		 577/**
578 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 579 * @hba: Pointer to adapter instance
2fbd009b
SJ		 580 *
581 * Returns interrupt bit mask per version
582 */
583static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
584{
c01848c6
YG		 585 u32 intr_mask = 0;
586
587 switch (hba->ufs_version) {
588 case UFSHCI_VERSION_10:
589 intr_mask = INTERRUPT_MASK_ALL_VER_10;
590 break;
c01848c6
YG		 591 case UFSHCI_VERSION_11:
592 case UFSHCI_VERSION_20:
593 intr_mask = INTERRUPT_MASK_ALL_VER_11;
594 break;
c01848c6
YG		 595 case UFSHCI_VERSION_21:
596 default:
597 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 598 break;
c01848c6
YG		 599 }
600
601 return intr_mask;
2fbd009b
SJ		 602}
603
7a3e97b0
SY		 604/**
605 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 606 * @hba: Pointer to adapter instance
7a3e97b0
SY		 607 *
608 * Returns UFSHCI version supported by the controller
609 */
610static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
611{
0263bcd0
YG		 612 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
613 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 614
b873a275 615 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 616}
617
618/**
619 * ufshcd_is_device_present - Check if any device connected to
620 * the host controller
5c0c28a8 621 * @hba: pointer to adapter instance
7a3e97b0 622 *
c9e6010b 623 * Returns true if device present, false if no device detected
7a3e97b0 624 */
c9e6010b 625static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 626{
5c0c28a8 627 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 628 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 629}
630
631/**
632 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 633 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 634 *
635 * This function is used to get the OCS field from UTRD
636 * Returns the OCS field in the UTRD
637 */
638static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
639{
e8c8e82a 640 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 641}
642
7a3e97b0
SY		 643/**
644 * ufshcd_get_tm_free_slot - get a free slot for task management request
645 * @hba: per adapter instance
e2933132 646 * @free_slot: pointer to variable with available slot value
7a3e97b0 647 *
e2933132
SRT		 648 * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
649 * Returns 0 if free slot is not available, else return 1 with tag value
650 * in @free_slot.
7a3e97b0 651 */
e2933132 652static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
7a3e97b0 653{
e2933132
SRT		 654 int tag;
655 bool ret = false;
656
657 if (!free_slot)
658 goto out;
659
660 do {
661 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
662 if (tag >= hba->nutmrs)
663 goto out;
664 } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
665
666 *free_slot = tag;
667 ret = true;
668out:
669 return ret;
670}
671
672static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
673{
674 clear_bit_unlock(slot, &hba->tm_slots_in_use);
7a3e97b0
SY		 675}
676
677/**
678 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
679 * @hba: per adapter instance
680 * @pos: position of the bit to be cleared
681 */
682static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
683{
1399c5b0
AA		 684 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
685 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
686 else
687 ufshcd_writel(hba, ~(1 << pos),
688 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
689}
690
691/**
692 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
693 * @hba: per adapter instance
694 * @pos: position of the bit to be cleared
695 */
696static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
697{
698 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
699 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
700 else
701 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 702}
703
a48353f6
YG		 704/**
705 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
706 * @hba: per adapter instance
707 * @tag: position of the bit to be cleared
708 */
709static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
710{
711 __clear_bit(tag, &hba->outstanding_reqs);
712}
713
7a3e97b0
SY		 714/**
715 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
716 * @reg: Register value of host controller status
717 *
718 * Returns integer, 0 on Success and positive value if failed
719 */
720static inline int ufshcd_get_lists_status(u32 reg)
721{
6cf16115 722 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 723}
724
725/**
726 * ufshcd_get_uic_cmd_result - Get the UIC command result
727 * @hba: Pointer to adapter instance
728 *
729 * This function gets the result of UIC command completion
730 * Returns 0 on success, non zero value on error
731 */
732static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
733{
b873a275 734 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 735 MASK_UIC_COMMAND_RESULT;
736}
737
12b4fdb4
SJ		 738/**
739 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
740 * @hba: Pointer to adapter instance
741 *
742 * This function gets UIC command argument3
743 * Returns 0 on success, non zero value on error
744 */
745static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
746{
747 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
748}
749
7a3e97b0 750/**
5a0b0cb9 751 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 752 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 753 */
754static inline int
5a0b0cb9 755ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 756{
5a0b0cb9 757 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 758}
759
760/**
761 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
762 * @ucd_rsp_ptr: pointer to response UPIU
763 *
764 * This function gets the response status and scsi_status from response UPIU
765 * Returns the response result code.
766 */
767static inline int
768ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
769{
770 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
771}
772
1c2623c5
SJ		 773/*
774 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
775 * from response UPIU
776 * @ucd_rsp_ptr: pointer to response UPIU
777 *
778 * Return the data segment length.
779 */
780static inline unsigned int
781ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
782{
783 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
784 MASK_RSP_UPIU_DATA_SEG_LEN;
785}
786
66ec6d59
SRT		 787/**
788 * ufshcd_is_exception_event - Check if the device raised an exception event
789 * @ucd_rsp_ptr: pointer to response UPIU
790 *
791 * The function checks if the device raised an exception event indicated in
792 * the Device Information field of response UPIU.
793 *
794 * Returns true if exception is raised, false otherwise.
795 */
796static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
797{
798 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
799 MASK_RSP_EXCEPTION_EVENT ? true : false;
800}
801
7a3e97b0 802/**
7d568652 803 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 804 * @hba: per adapter instance
7a3e97b0
SY		 805 */
806static inline void
7d568652 807ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 808{
7d568652
SJ		 809 ufshcd_writel(hba, INT_AGGR_ENABLE |
810 INT_AGGR_COUNTER_AND_TIMER_RESET,
811 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
812}
813
814/**
815 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
816 * @hba: per adapter instance
817 * @cnt: Interrupt aggregation counter threshold
818 * @tmout: Interrupt aggregation timeout value
819 */
820static inline void
821ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
822{
823 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
824 INT_AGGR_COUNTER_THLD_VAL(cnt) |
825 INT_AGGR_TIMEOUT_VAL(tmout),
826 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 827}
828
b852190e
YG		 829/**
830 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
831 * @hba: per adapter instance
832 */
833static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
834{
835 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
836}
837
7a3e97b0
SY		 838/**
839 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
840 * When run-stop registers are set to 1, it indicates the
841 * host controller that it can process the requests
842 * @hba: per adapter instance
843 */
844static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
845{
b873a275
SJ		 846 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
847 REG_UTP_TASK_REQ_LIST_RUN_STOP);
848 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
849 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 850}
851
7a3e97b0
SY		 852/**
853 * ufshcd_hba_start - Start controller initialization sequence
854 * @hba: per adapter instance
855 */
856static inline void ufshcd_hba_start(struct ufs_hba *hba)
857{
b873a275 858 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 859}
860
861/**
862 * ufshcd_is_hba_active - Get controller state
863 * @hba: per adapter instance
864 *
c9e6010b 865 * Returns false if controller is active, true otherwise
7a3e97b0 866 */
c9e6010b 867static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 868{
4a8eec2b
TK		 869 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
870 ? false : true;
7a3e97b0
SY		 871}
872
37113106
YG		 873u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
874{
875 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
876 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
877 (hba->ufs_version == UFSHCI_VERSION_11))
878 return UFS_UNIPRO_VER_1_41;
879 else
880 return UFS_UNIPRO_VER_1_6;
881}
882EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
883
884static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
885{
886 /*
887 * If both host and device support UniPro ver1.6 or later, PA layer
888 * parameters tuning happens during link startup itself.
889 *
890 * We can manually tune PA layer parameters if either host or device
891 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
892 * logic simple, we will only do manual tuning if local unipro version
893 * doesn't support ver1.6 or later.
894 */
895 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
896 return true;
897 else
898 return false;
899}
900
a3cd5ec5 901static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
902{
903 int ret = 0;
904 struct ufs_clk_info *clki;
905 struct list_head *head = &hba->clk_list_head;
906 ktime_t start = ktime_get();
907 bool clk_state_changed = false;
908
566ec9ad 909 if (list_empty(head))
a3cd5ec5 910 goto out;
911
912 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
913 if (ret)
914 return ret;
915
916 list_for_each_entry(clki, head, list) {
917 if (!IS_ERR_OR_NULL(clki->clk)) {
918 if (scale_up && clki->max_freq) {
919 if (clki->curr_freq == clki->max_freq)
920 continue;
921
922 clk_state_changed = true;
923 ret = clk_set_rate(clki->clk, clki->max_freq);
924 if (ret) {
925 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
926 __func__, clki->name,
927 clki->max_freq, ret);
928 break;
929 }
930 trace_ufshcd_clk_scaling(dev_name(hba->dev),
931 "scaled up", clki->name,
932 clki->curr_freq,
933 clki->max_freq);
934
935 clki->curr_freq = clki->max_freq;
936
937 } else if (!scale_up && clki->min_freq) {
938 if (clki->curr_freq == clki->min_freq)
939 continue;
940
941 clk_state_changed = true;
942 ret = clk_set_rate(clki->clk, clki->min_freq);
943 if (ret) {
944 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
945 __func__, clki->name,
946 clki->min_freq, ret);
947 break;
948 }
949 trace_ufshcd_clk_scaling(dev_name(hba->dev),
950 "scaled down", clki->name,
951 clki->curr_freq,
952 clki->min_freq);
953 clki->curr_freq = clki->min_freq;
954 }
955 }
956 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
957 clki->name, clk_get_rate(clki->clk));
958 }
959
960 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
961
962out:
963 if (clk_state_changed)
964 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
965 (scale_up ? "up" : "down"),
966 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
967 return ret;
968}
969
970/**
971 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
972 * @hba: per adapter instance
973 * @scale_up: True if scaling up and false if scaling down
974 *
975 * Returns true if scaling is required, false otherwise.
976 */
977static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
978 bool scale_up)
979{
980 struct ufs_clk_info *clki;
981 struct list_head *head = &hba->clk_list_head;
982
566ec9ad 983 if (list_empty(head))
a3cd5ec5 984 return false;
985
986 list_for_each_entry(clki, head, list) {
987 if (!IS_ERR_OR_NULL(clki->clk)) {
988 if (scale_up && clki->max_freq) {
989 if (clki->curr_freq == clki->max_freq)
990 continue;
991 return true;
992 } else if (!scale_up && clki->min_freq) {
993 if (clki->curr_freq == clki->min_freq)
994 continue;
995 return true;
996 }
997 }
998 }
999
1000 return false;
1001}
1002
1003static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1004 u64 wait_timeout_us)
1005{
1006 unsigned long flags;
1007 int ret = 0;
1008 u32 tm_doorbell;
1009 u32 tr_doorbell;
1010 bool timeout = false, do_last_check = false;
1011 ktime_t start;
1012
1013 ufshcd_hold(hba, false);
1014 spin_lock_irqsave(hba->host->host_lock, flags);
1015 /*
1016 * Wait for all the outstanding tasks/transfer requests.
1017 * Verify by checking the doorbell registers are clear.
1018 */
1019 start = ktime_get();
1020 do {
1021 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1022 ret = -EBUSY;
1023 goto out;
1024 }
1025
1026 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1027 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1028 if (!tm_doorbell && !tr_doorbell) {
1029 timeout = false;
1030 break;
1031 } else if (do_last_check) {
1032 break;
1033 }
1034
1035 spin_unlock_irqrestore(hba->host->host_lock, flags);
1036 schedule();
1037 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1038 wait_timeout_us) {
1039 timeout = true;
1040 /*
1041 * We might have scheduled out for long time so make
1042 * sure to check if doorbells are cleared by this time
1043 * or not.
1044 */
1045 do_last_check = true;
1046 }
1047 spin_lock_irqsave(hba->host->host_lock, flags);
1048 } while (tm_doorbell || tr_doorbell);
1049
1050 if (timeout) {
1051 dev_err(hba->dev,
1052 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1053 __func__, tm_doorbell, tr_doorbell);
1054 ret = -EBUSY;
1055 }
1056out:
1057 spin_unlock_irqrestore(hba->host->host_lock, flags);
1058 ufshcd_release(hba);
1059 return ret;
1060}
1061
1062/**
1063 * ufshcd_scale_gear - scale up/down UFS gear
1064 * @hba: per adapter instance
1065 * @scale_up: True for scaling up gear and false for scaling down
1066 *
1067 * Returns 0 for success,
1068 * Returns -EBUSY if scaling can't happen at this time
1069 * Returns non-zero for any other errors
1070 */
1071static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1072{
1073 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1074 int ret = 0;
1075 struct ufs_pa_layer_attr new_pwr_info;
1076
1077 if (scale_up) {
1078 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1079 sizeof(struct ufs_pa_layer_attr));
1080 } else {
1081 memcpy(&new_pwr_info, &hba->pwr_info,
1082 sizeof(struct ufs_pa_layer_attr));
1083
1084 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1085 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1086 /* save the current power mode */
1087 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1088 &hba->pwr_info,
1089 sizeof(struct ufs_pa_layer_attr));
1090
1091 /* scale down gear */
1092 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1093 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1094 }
1095 }
1096
1097 /* check if the power mode needs to be changed or not? */
1098 ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1099
1100 if (ret)
1101 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1102 __func__, ret,
1103 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1104 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1105
1106 return ret;
1107}
1108
1109static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1110{
1111 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1112 int ret = 0;
1113 /*
1114 * make sure that there are no outstanding requests when
1115 * clock scaling is in progress
1116 */
38135535 1117 ufshcd_scsi_block_requests(hba);
a3cd5ec5 1118 down_write(&hba->clk_scaling_lock);
1119 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1120 ret = -EBUSY;
1121 up_write(&hba->clk_scaling_lock);
38135535 1122 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1123 }
1124
1125 return ret;
1126}
1127
1128static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1129{
1130 up_write(&hba->clk_scaling_lock);
38135535 1131 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5 1132}
1133
1134/**
1135 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1136 * @hba: per adapter instance
1137 * @scale_up: True for scaling up and false for scalin down
1138 *
1139 * Returns 0 for success,
1140 * Returns -EBUSY if scaling can't happen at this time
1141 * Returns non-zero for any other errors
1142 */
1143static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1144{
1145 int ret = 0;
1146
401f1e44 1147 /* let's not get into low power until clock scaling is completed */
1148 ufshcd_hold(hba, false);
1149
a3cd5ec5 1150 ret = ufshcd_clock_scaling_prepare(hba);
1151 if (ret)
1152 return ret;
1153
1154 /* scale down the gear before scaling down clocks */
1155 if (!scale_up) {
1156 ret = ufshcd_scale_gear(hba, false);
1157 if (ret)
1158 goto out;
1159 }
1160
1161 ret = ufshcd_scale_clks(hba, scale_up);
1162 if (ret) {
1163 if (!scale_up)
1164 ufshcd_scale_gear(hba, true);
1165 goto out;
1166 }
1167
1168 /* scale up the gear after scaling up clocks */
1169 if (scale_up) {
1170 ret = ufshcd_scale_gear(hba, true);
1171 if (ret) {
1172 ufshcd_scale_clks(hba, false);
1173 goto out;
1174 }
1175 }
1176
1177 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1178
1179out:
1180 ufshcd_clock_scaling_unprepare(hba);
401f1e44 1181 ufshcd_release(hba);
a3cd5ec5 1182 return ret;
1183}
1184
401f1e44 1185static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1186{
1187 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1188 clk_scaling.suspend_work);
1189 unsigned long irq_flags;
1190
1191 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1192 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1193 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1194 return;
1195 }
1196 hba->clk_scaling.is_suspended = true;
1197 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1198
1199 __ufshcd_suspend_clkscaling(hba);
1200}
1201
1202static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1203{
1204 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1205 clk_scaling.resume_work);
1206 unsigned long irq_flags;
1207
1208 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1209 if (!hba->clk_scaling.is_suspended) {
1210 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1211 return;
1212 }
1213 hba->clk_scaling.is_suspended = false;
1214 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1215
1216 devfreq_resume_device(hba->devfreq);
1217}
1218
a3cd5ec5 1219static int ufshcd_devfreq_target(struct device *dev,
1220 unsigned long *freq, u32 flags)
1221{
1222 int ret = 0;
1223 struct ufs_hba *hba = dev_get_drvdata(dev);
1224 ktime_t start;
401f1e44 1225 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1226 struct list_head *clk_list = &hba->clk_list_head;
1227 struct ufs_clk_info *clki;
a3cd5ec5 1228 unsigned long irq_flags;
1229
1230 if (!ufshcd_is_clkscaling_supported(hba))
1231 return -EINVAL;
1232
a3cd5ec5 1233 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1234 if (ufshcd_eh_in_progress(hba)) {
1235 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1236 return 0;
1237 }
1238
401f1e44 1239 if (!hba->clk_scaling.active_reqs)
1240 sched_clk_scaling_suspend_work = true;
1241
092b4558
BA		 1242 if (list_empty(clk_list)) {
1243 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1244 goto out;
1245 }
1246
1247 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1248 scale_up = (*freq == clki->max_freq) ? true : false;
401f1e44 1249 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1250 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1251 ret = 0;
1252 goto out; /* no state change required */
a3cd5ec5 1253 }
1254 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1255
1256 start = ktime_get();
a3cd5ec5 1257 ret = ufshcd_devfreq_scale(hba, scale_up);
1258
a3cd5ec5 1259 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1260 (scale_up ? "up" : "down"),
1261 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1262
401f1e44 1263out:
1264 if (sched_clk_scaling_suspend_work)
1265 queue_work(hba->clk_scaling.workq,
1266 &hba->clk_scaling.suspend_work);
1267
a3cd5ec5 1268 return ret;
1269}
1270
1271
1272static int ufshcd_devfreq_get_dev_status(struct device *dev,
1273 struct devfreq_dev_status *stat)
1274{
1275 struct ufs_hba *hba = dev_get_drvdata(dev);
1276 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1277 unsigned long flags;
1278
1279 if (!ufshcd_is_clkscaling_supported(hba))
1280 return -EINVAL;
1281
1282 memset(stat, 0, sizeof(*stat));
1283
1284 spin_lock_irqsave(hba->host->host_lock, flags);
1285 if (!scaling->window_start_t)
1286 goto start_window;
1287
1288 if (scaling->is_busy_started)
1289 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1290 scaling->busy_start_t));
1291
1292 stat->total_time = jiffies_to_usecs((long)jiffies -
1293 (long)scaling->window_start_t);
1294 stat->busy_time = scaling->tot_busy_t;
1295start_window:
1296 scaling->window_start_t = jiffies;
1297 scaling->tot_busy_t = 0;
1298
1299 if (hba->outstanding_reqs) {
1300 scaling->busy_start_t = ktime_get();
1301 scaling->is_busy_started = true;
1302 } else {
1303 scaling->busy_start_t = 0;
1304 scaling->is_busy_started = false;
1305 }
1306 spin_unlock_irqrestore(hba->host->host_lock, flags);
1307 return 0;
1308}
1309
1310static struct devfreq_dev_profile ufs_devfreq_profile = {
1311 .polling_ms = 100,
1312 .target = ufshcd_devfreq_target,
1313 .get_dev_status = ufshcd_devfreq_get_dev_status,
1314};
1315
deac444f
BA		 1316static int ufshcd_devfreq_init(struct ufs_hba *hba)
1317{
092b4558
BA		 1318 struct list_head *clk_list = &hba->clk_list_head;
1319 struct ufs_clk_info *clki;
deac444f
BA		 1320 struct devfreq *devfreq;
1321 int ret;
1322
092b4558
BA		 1323 /* Skip devfreq if we don't have any clocks in the list */
1324 if (list_empty(clk_list))
1325 return 0;
1326
1327 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1328 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1329 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1330
1331 devfreq = devfreq_add_device(hba->dev,
deac444f
BA		 1332 &ufs_devfreq_profile,
1333 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1334 NULL);
1335 if (IS_ERR(devfreq)) {
1336 ret = PTR_ERR(devfreq);
1337 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1338
1339 dev_pm_opp_remove(hba->dev, clki->min_freq);
1340 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1341 return ret;
1342 }
1343
1344 hba->devfreq = devfreq;
1345
1346 return 0;
1347}
1348
092b4558
BA		 1349static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1350{
1351 struct list_head *clk_list = &hba->clk_list_head;
1352 struct ufs_clk_info *clki;
1353
1354 if (!hba->devfreq)
1355 return;
1356
1357 devfreq_remove_device(hba->devfreq);
1358 hba->devfreq = NULL;
1359
1360 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1361 dev_pm_opp_remove(hba->dev, clki->min_freq);
1362 dev_pm_opp_remove(hba->dev, clki->max_freq);
1363}
1364
401f1e44 1365static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1366{
1367 unsigned long flags;
1368
1369 devfreq_suspend_device(hba->devfreq);
1370 spin_lock_irqsave(hba->host->host_lock, flags);
1371 hba->clk_scaling.window_start_t = 0;
1372 spin_unlock_irqrestore(hba->host->host_lock, flags);
1373}
a3cd5ec5 1374
a508253d
GB		 1375static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1376{
401f1e44 1377 unsigned long flags;
1378 bool suspend = false;
1379
fcb0c4b0
ST		 1380 if (!ufshcd_is_clkscaling_supported(hba))
1381 return;
1382
401f1e44 1383 spin_lock_irqsave(hba->host->host_lock, flags);
1384 if (!hba->clk_scaling.is_suspended) {
1385 suspend = true;
1386 hba->clk_scaling.is_suspended = true;
1387 }
1388 spin_unlock_irqrestore(hba->host->host_lock, flags);
1389
1390 if (suspend)
1391 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1392}
1393
1394static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1395{
401f1e44 1396 unsigned long flags;
1397 bool resume = false;
1398
1399 if (!ufshcd_is_clkscaling_supported(hba))
1400 return;
1401
1402 spin_lock_irqsave(hba->host->host_lock, flags);
1403 if (hba->clk_scaling.is_suspended) {
1404 resume = true;
1405 hba->clk_scaling.is_suspended = false;
1406 }
1407 spin_unlock_irqrestore(hba->host->host_lock, flags);
1408
1409 if (resume)
1410 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1411}
1412
1413static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1414 struct device_attribute *attr, char *buf)
1415{
1416 struct ufs_hba *hba = dev_get_drvdata(dev);
1417
1418 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1419}
1420
1421static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1422 struct device_attribute *attr, const char *buf, size_t count)
1423{
1424 struct ufs_hba *hba = dev_get_drvdata(dev);
1425 u32 value;
1426 int err;
1427
1428 if (kstrtou32(buf, 0, &value))
1429 return -EINVAL;
1430
1431 value = !!value;
1432 if (value == hba->clk_scaling.is_allowed)
1433 goto out;
1434
1435 pm_runtime_get_sync(hba->dev);
1436 ufshcd_hold(hba, false);
1437
401f1e44 1438 cancel_work_sync(&hba->clk_scaling.suspend_work);
1439 cancel_work_sync(&hba->clk_scaling.resume_work);
1440
1441 hba->clk_scaling.is_allowed = value;
1442
fcb0c4b0
ST		 1443 if (value) {
1444 ufshcd_resume_clkscaling(hba);
1445 } else {
1446 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1447 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1448 if (err)
1449 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1450 __func__, err);
1451 }
fcb0c4b0
ST		 1452
1453 ufshcd_release(hba);
1454 pm_runtime_put_sync(hba->dev);
1455out:
1456 return count;
a508253d
GB		 1457}
1458
a3cd5ec5 1459static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1460{
1461 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1462 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1463 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1464 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1465 hba->clk_scaling.enable_attr.attr.mode = 0644;
1466 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1467 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1468}
1469
1ab27c9c
ST		 1470static void ufshcd_ungate_work(struct work_struct *work)
1471{
1472 int ret;
1473 unsigned long flags;
1474 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1475 clk_gating.ungate_work);
1476
1477 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1478
1479 spin_lock_irqsave(hba->host->host_lock, flags);
1480 if (hba->clk_gating.state == CLKS_ON) {
1481 spin_unlock_irqrestore(hba->host->host_lock, flags);
1482 goto unblock_reqs;
1483 }
1484
1485 spin_unlock_irqrestore(hba->host->host_lock, flags);
1486 ufshcd_setup_clocks(hba, true);
1487
1488 /* Exit from hibern8 */
1489 if (ufshcd_can_hibern8_during_gating(hba)) {
1490 /* Prevent gating in this path */
1491 hba->clk_gating.is_suspended = true;
1492 if (ufshcd_is_link_hibern8(hba)) {
1493 ret = ufshcd_uic_hibern8_exit(hba);
1494 if (ret)
1495 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1496 __func__, ret);
1497 else
1498 ufshcd_set_link_active(hba);
1499 }
1500 hba->clk_gating.is_suspended = false;
1501 }
1502unblock_reqs:
38135535 1503 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1504}
1505
1506/**
1507 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1508 * Also, exit from hibern8 mode and set the link as active.
1509 * @hba: per adapter instance
1510 * @async: This indicates whether caller should ungate clocks asynchronously.
1511 */
1512int ufshcd_hold(struct ufs_hba *hba, bool async)
1513{
1514 int rc = 0;
1515 unsigned long flags;
1516
1517 if (!ufshcd_is_clkgating_allowed(hba))
1518 goto out;
1ab27c9c
ST		 1519 spin_lock_irqsave(hba->host->host_lock, flags);
1520 hba->clk_gating.active_reqs++;
1521
53c12d0e
YG		 1522 if (ufshcd_eh_in_progress(hba)) {
1523 spin_unlock_irqrestore(hba->host->host_lock, flags);
1524 return 0;
1525 }
1526
856b3483 1527start:
1ab27c9c
ST		 1528 switch (hba->clk_gating.state) {
1529 case CLKS_ON:
f2a785ac
VG		 1530 /*
1531 * Wait for the ungate work to complete if in progress.
1532 * Though the clocks may be in ON state, the link could
1533 * still be in hibner8 state if hibern8 is allowed
1534 * during clock gating.
1535 * Make sure we exit hibern8 state also in addition to
1536 * clocks being ON.
1537 */
1538 if (ufshcd_can_hibern8_during_gating(hba) &&
1539 ufshcd_is_link_hibern8(hba)) {
1540 spin_unlock_irqrestore(hba->host->host_lock, flags);
1541 flush_work(&hba->clk_gating.ungate_work);
1542 spin_lock_irqsave(hba->host->host_lock, flags);
1543 goto start;
1544 }
1ab27c9c
ST		 1545 break;
1546 case REQ_CLKS_OFF:
1547 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1548 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1549 trace_ufshcd_clk_gating(dev_name(hba->dev),
1550 hba->clk_gating.state);
1ab27c9c
ST		 1551 break;
1552 }
1553 /*
9c490d2d 1554 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1555 * currently running. Hence, fall through to cancel gating
1556 * work and to enable clocks.
1557 */
30eb2e4c 1558 /* fallthrough */
1ab27c9c 1559 case CLKS_OFF:
38135535 1560 ufshcd_scsi_block_requests(hba);
1ab27c9c 1561 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47 1562 trace_ufshcd_clk_gating(dev_name(hba->dev),
1563 hba->clk_gating.state);
10e5e375
VV		 1564 queue_work(hba->clk_gating.clk_gating_workq,
1565 &hba->clk_gating.ungate_work);
1ab27c9c
ST		 1566 /*
1567 * fall through to check if we should wait for this
1568 * work to be done or not.
1569 */
30eb2e4c 1570 /* fallthrough */
1ab27c9c
ST		 1571 case REQ_CLKS_ON:
1572 if (async) {
1573 rc = -EAGAIN;
1574 hba->clk_gating.active_reqs--;
1575 break;
1576 }
1577
1578 spin_unlock_irqrestore(hba->host->host_lock, flags);
1579 flush_work(&hba->clk_gating.ungate_work);
1580 /* Make sure state is CLKS_ON before returning */
856b3483 1581 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1582 goto start;
1583 default:
1584 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1585 __func__, hba->clk_gating.state);
1586 break;
1587 }
1588 spin_unlock_irqrestore(hba->host->host_lock, flags);
1589out:
1590 return rc;
1591}
6e3fd44d 1592EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1593
1594static void ufshcd_gate_work(struct work_struct *work)
1595{
1596 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1597 clk_gating.gate_work.work);
1598 unsigned long flags;
1599
1600 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1601 /*
1602 * In case you are here to cancel this work the gating state
1603 * would be marked as REQ_CLKS_ON. In this case save time by
1604 * skipping the gating work and exit after changing the clock
1605 * state to CLKS_ON.
1606 */
1607 if (hba->clk_gating.is_suspended ||
1608 (hba->clk_gating.state == REQ_CLKS_ON)) {
1ab27c9c 1609 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1610 trace_ufshcd_clk_gating(dev_name(hba->dev),
1611 hba->clk_gating.state);
1ab27c9c
ST		 1612 goto rel_lock;
1613 }
1614
1615 if (hba->clk_gating.active_reqs
1616 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1617 || hba->lrb_in_use || hba->outstanding_tasks
1618 || hba->active_uic_cmd || hba->uic_async_done)
1619 goto rel_lock;
1620
1621 spin_unlock_irqrestore(hba->host->host_lock, flags);
1622
1623 /* put the link into hibern8 mode before turning off clocks */
1624 if (ufshcd_can_hibern8_during_gating(hba)) {
1625 if (ufshcd_uic_hibern8_enter(hba)) {
1626 hba->clk_gating.state = CLKS_ON;
7ff5ab47 1627 trace_ufshcd_clk_gating(dev_name(hba->dev),
1628 hba->clk_gating.state);
1ab27c9c
ST		 1629 goto out;
1630 }
1631 ufshcd_set_link_hibern8(hba);
1632 }
1633
1634 if (!ufshcd_is_link_active(hba))
1635 ufshcd_setup_clocks(hba, false);
1636 else
1637 /* If link is active, device ref_clk can't be switched off */
1638 __ufshcd_setup_clocks(hba, false, true);
1639
1640 /*
1641 * In case you are here to cancel this work the gating state
1642 * would be marked as REQ_CLKS_ON. In this case keep the state
1643 * as REQ_CLKS_ON which would anyway imply that clocks are off
1644 * and a request to turn them on is pending. By doing this way,
1645 * we keep the state machine in tact and this would ultimately
1646 * prevent from doing cancel work multiple times when there are
1647 * new requests arriving before the current cancel work is done.
1648 */
1649 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1650 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1651 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 1652 trace_ufshcd_clk_gating(dev_name(hba->dev),
1653 hba->clk_gating.state);
1654 }
1ab27c9c
ST		 1655rel_lock:
1656 spin_unlock_irqrestore(hba->host->host_lock, flags);
1657out:
1658 return;
1659}
1660
1661/* host lock must be held before calling this variant */
1662static void __ufshcd_release(struct ufs_hba *hba)
1663{
1664 if (!ufshcd_is_clkgating_allowed(hba))
1665 return;
1666
1667 hba->clk_gating.active_reqs--;
1668
1669 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1670 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1671 || hba->lrb_in_use || hba->outstanding_tasks
53c12d0e
YG		 1672 || hba->active_uic_cmd || hba->uic_async_done
1673 || ufshcd_eh_in_progress(hba))
1ab27c9c
ST		 1674 return;
1675
1676 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1677 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1678 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1679 &hba->clk_gating.gate_work,
1680 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1681}
1682
1683void ufshcd_release(struct ufs_hba *hba)
1684{
1685 unsigned long flags;
1686
1687 spin_lock_irqsave(hba->host->host_lock, flags);
1688 __ufshcd_release(hba);
1689 spin_unlock_irqrestore(hba->host->host_lock, flags);
1690}
6e3fd44d 1691EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1692
1693static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1694 struct device_attribute *attr, char *buf)
1695{
1696 struct ufs_hba *hba = dev_get_drvdata(dev);
1697
1698 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1699}
1700
1701static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1702 struct device_attribute *attr, const char *buf, size_t count)
1703{
1704 struct ufs_hba *hba = dev_get_drvdata(dev);
1705 unsigned long flags, value;
1706
1707 if (kstrtoul(buf, 0, &value))
1708 return -EINVAL;
1709
1710 spin_lock_irqsave(hba->host->host_lock, flags);
1711 hba->clk_gating.delay_ms = value;
1712 spin_unlock_irqrestore(hba->host->host_lock, flags);
1713 return count;
1714}
1715
b427411a
ST		 1716static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1717 struct device_attribute *attr, char *buf)
1718{
1719 struct ufs_hba *hba = dev_get_drvdata(dev);
1720
1721 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1722}
1723
1724static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1725 struct device_attribute *attr, const char *buf, size_t count)
1726{
1727 struct ufs_hba *hba = dev_get_drvdata(dev);
1728 unsigned long flags;
1729 u32 value;
1730
1731 if (kstrtou32(buf, 0, &value))
1732 return -EINVAL;
1733
1734 value = !!value;
1735 if (value == hba->clk_gating.is_enabled)
1736 goto out;
1737
1738 if (value) {
1739 ufshcd_release(hba);
1740 } else {
1741 spin_lock_irqsave(hba->host->host_lock, flags);
1742 hba->clk_gating.active_reqs++;
1743 spin_unlock_irqrestore(hba->host->host_lock, flags);
1744 }
1745
1746 hba->clk_gating.is_enabled = value;
1747out:
1748 return count;
1749}
1750
eebcc196
VG		 1751static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1752{
1753 char wq_name[sizeof("ufs_clkscaling_00")];
1754
1755 if (!ufshcd_is_clkscaling_supported(hba))
1756 return;
1757
1758 INIT_WORK(&hba->clk_scaling.suspend_work,
1759 ufshcd_clk_scaling_suspend_work);
1760 INIT_WORK(&hba->clk_scaling.resume_work,
1761 ufshcd_clk_scaling_resume_work);
1762
1763 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1764 hba->host->host_no);
1765 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1766
1767 ufshcd_clkscaling_init_sysfs(hba);
1768}
1769
1770static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1771{
1772 if (!ufshcd_is_clkscaling_supported(hba))
1773 return;
1774
1775 destroy_workqueue(hba->clk_scaling.workq);
1776 ufshcd_devfreq_remove(hba);
1777}
1778
1ab27c9c
ST		 1779static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1780{
10e5e375
VV		 1781 char wq_name[sizeof("ufs_clk_gating_00")];
1782
1ab27c9c
ST		 1783 if (!ufshcd_is_clkgating_allowed(hba))
1784 return;
1785
1786 hba->clk_gating.delay_ms = 150;
1787 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1788 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1789
10e5e375
VV		 1790 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1791 hba->host->host_no);
1792 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1793 WQ_MEM_RECLAIM);
1794
b427411a
ST		 1795 hba->clk_gating.is_enabled = true;
1796
1ab27c9c
ST		 1797 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1798 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1799 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1800 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
b427411a 1801 hba->clk_gating.delay_attr.attr.mode = 0644;
1ab27c9c
ST		 1802 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1803 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
b427411a
ST		 1804
1805 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1806 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1807 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1808 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1809 hba->clk_gating.enable_attr.attr.mode = 0644;
1810 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1811 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1ab27c9c
ST		 1812}
1813
1814static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1815{
1816 if (!ufshcd_is_clkgating_allowed(hba))
1817 return;
1818 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
b427411a 1819 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
97cd6805
AM		 1820 cancel_work_sync(&hba->clk_gating.ungate_work);
1821 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1822 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1823}
1824
856b3483
ST		 1825/* Must be called with host lock acquired */
1826static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1827{
401f1e44 1828 bool queue_resume_work = false;
1829
fcb0c4b0 1830 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1831 return;
1832
401f1e44 1833 if (!hba->clk_scaling.active_reqs++)
1834 queue_resume_work = true;
1835
1836 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1837 return;
1838
1839 if (queue_resume_work)
1840 queue_work(hba->clk_scaling.workq,
1841 &hba->clk_scaling.resume_work);
1842
1843 if (!hba->clk_scaling.window_start_t) {
1844 hba->clk_scaling.window_start_t = jiffies;
1845 hba->clk_scaling.tot_busy_t = 0;
1846 hba->clk_scaling.is_busy_started = false;
1847 }
1848
856b3483
ST		 1849 if (!hba->clk_scaling.is_busy_started) {
1850 hba->clk_scaling.busy_start_t = ktime_get();
1851 hba->clk_scaling.is_busy_started = true;
1852 }
1853}
1854
1855static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1856{
1857 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1858
fcb0c4b0 1859 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1860 return;
1861
1862 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1863 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1864 scaling->busy_start_t));
8b0e1953 1865 scaling->busy_start_t = 0;
856b3483
ST		 1866 scaling->is_busy_started = false;
1867 }
1868}
7a3e97b0
SY		 1869/**
1870 * ufshcd_send_command - Send SCSI or device management commands
1871 * @hba: per adapter instance
1872 * @task_tag: Task tag of the command
1873 */
1874static inline
1875void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1876{
ff8e20c6 1877 hba->lrb[task_tag].issue_time_stamp = ktime_get();
09017188 1878 hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
856b3483 1879 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 1880 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 1881 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 1882 /* Make sure that doorbell is committed immediately */
1883 wmb();
1a07f2d9 1884 ufshcd_add_command_trace(hba, task_tag, "send");
7a3e97b0
SY		 1885}
1886
1887/**
1888 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 1889 * @lrbp: pointer to local reference block
7a3e97b0
SY		 1890 */
1891static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1892{
1893 int len;
1c2623c5
SJ		 1894 if (lrbp->sense_buffer &&
1895 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 1896 int len_to_copy;
1897
5a0b0cb9 1898 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 1899 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 1900
09a5a24f
AA		 1901 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1902 len_to_copy);
7a3e97b0
SY		 1903 }
1904}
1905
68078d5c
DR		 1906/**
1907 * ufshcd_copy_query_response() - Copy the Query Response and the data
1908 * descriptor
1909 * @hba: per adapter instance
8aa29f19 1910 * @lrbp: pointer to local reference block
68078d5c
DR		 1911 */
1912static
c6d4a831 1913int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 1914{
1915 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1916
68078d5c 1917 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 1918
68078d5c
DR		 1919 /* Get the descriptor */
1920 if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 1921 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 1922 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 1923 u16 resp_len;
1924 u16 buf_len;
68078d5c
DR		 1925
1926 /* data segment length */
c6d4a831 1927 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 1928 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 1929 buf_len = be16_to_cpu(
1930 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 1931 if (likely(buf_len >= resp_len)) {
1932 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1933 } else {
1934 dev_warn(hba->dev,
1935 "%s: Response size is bigger than buffer",
1936 __func__);
1937 return -EINVAL;
1938 }
68078d5c 1939 }
c6d4a831
DR		 1940
1941 return 0;
68078d5c
DR		 1942}
1943
7a3e97b0
SY		 1944/**
1945 * ufshcd_hba_capabilities - Read controller capabilities
1946 * @hba: per adapter instance
1947 */
1948static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1949{
b873a275 1950 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 1951
1952 /* nutrs and nutmrs are 0 based values */
1953 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1954 hba->nutmrs =
1955 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1956}
1957
1958/**
6ccf44fe
SJ		 1959 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1960 * to accept UIC commands
7a3e97b0 1961 * @hba: per adapter instance
6ccf44fe
SJ		 1962 * Return true on success, else false
1963 */
1964static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1965{
1966 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1967 return true;
1968 else
1969 return false;
1970}
1971
53b3d9c3
SJ		 1972/**
1973 * ufshcd_get_upmcrs - Get the power mode change request status
1974 * @hba: Pointer to adapter instance
1975 *
1976 * This function gets the UPMCRS field of HCS register
1977 * Returns value of UPMCRS field
1978 */
1979static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1980{
1981 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1982}
1983
6ccf44fe
SJ		 1984/**
1985 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1986 * @hba: per adapter instance
1987 * @uic_cmd: UIC command
1988 *
1989 * Mutex must be held.
7a3e97b0
SY		 1990 */
1991static inline void
6ccf44fe 1992ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 1993{
6ccf44fe
SJ		 1994 WARN_ON(hba->active_uic_cmd);
1995
1996 hba->active_uic_cmd = uic_cmd;
1997
7a3e97b0 1998 /* Write Args */
6ccf44fe
SJ		 1999 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2000 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2001 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0
SY		 2002
2003 /* Write UIC Cmd */
6ccf44fe 2004 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 2005 REG_UIC_COMMAND);
7a3e97b0
SY		 2006}
2007
6ccf44fe
SJ		 2008/**
2009 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2010 * @hba: per adapter instance
8aa29f19 2011 * @uic_cmd: UIC command
6ccf44fe
SJ		 2012 *
2013 * Must be called with mutex held.
2014 * Returns 0 only if success.
2015 */
2016static int
2017ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2018{
2019 int ret;
2020 unsigned long flags;
2021
2022 if (wait_for_completion_timeout(&uic_cmd->done,
2023 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2024 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2025 else
2026 ret = -ETIMEDOUT;
2027
2028 spin_lock_irqsave(hba->host->host_lock, flags);
2029 hba->active_uic_cmd = NULL;
2030 spin_unlock_irqrestore(hba->host->host_lock, flags);
2031
2032 return ret;
2033}
2034
2035/**
2036 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2037 * @hba: per adapter instance
2038 * @uic_cmd: UIC command
d75f7fe4 2039 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2040 *
2041 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2042 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2043 * Returns 0 only if success.
2044 */
2045static int
d75f7fe4
YG		 2046__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2047 bool completion)
6ccf44fe 2048{
6ccf44fe
SJ		 2049 if (!ufshcd_ready_for_uic_cmd(hba)) {
2050 dev_err(hba->dev,
2051 "Controller not ready to accept UIC commands\n");
2052 return -EIO;
2053 }
2054
d75f7fe4
YG		 2055 if (completion)
2056 init_completion(&uic_cmd->done);
6ccf44fe 2057
6ccf44fe 2058 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2059
57d104c1 2060 return 0;
6ccf44fe
SJ		 2061}
2062
2063/**
2064 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2065 * @hba: per adapter instance
2066 * @uic_cmd: UIC command
2067 *
2068 * Returns 0 only if success.
2069 */
e77044c5 2070int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2071{
2072 int ret;
57d104c1 2073 unsigned long flags;
6ccf44fe 2074
1ab27c9c 2075 ufshcd_hold(hba, false);
6ccf44fe 2076 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2077 ufshcd_add_delay_before_dme_cmd(hba);
2078
57d104c1 2079 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2080 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2081 spin_unlock_irqrestore(hba->host->host_lock, flags);
2082 if (!ret)
2083 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2084
6ccf44fe
SJ		 2085 mutex_unlock(&hba->uic_cmd_mutex);
2086
1ab27c9c 2087 ufshcd_release(hba);
6ccf44fe
SJ		 2088 return ret;
2089}
2090
7a3e97b0
SY		 2091/**
2092 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2093 * @hba: per adapter instance
2094 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2095 *
2096 * Returns 0 in case of success, non-zero value in case of failure
2097 */
75b1cc4a 2098static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2099{
2100 struct ufshcd_sg_entry *prd_table;
2101 struct scatterlist *sg;
2102 struct scsi_cmnd *cmd;
2103 int sg_segments;
2104 int i;
2105
2106 cmd = lrbp->cmd;
2107 sg_segments = scsi_dma_map(cmd);
2108 if (sg_segments < 0)
2109 return sg_segments;
2110
2111 if (sg_segments) {
75b1cc4a
KK		 2112 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2113 lrbp->utr_descriptor_ptr->prd_table_length =
2114 cpu_to_le16((u16)(sg_segments *
2115 sizeof(struct ufshcd_sg_entry)));
2116 else
2117 lrbp->utr_descriptor_ptr->prd_table_length =
2118 cpu_to_le16((u16) (sg_segments));
7a3e97b0
SY		 2119
2120 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2121
2122 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2123 prd_table[i].size =
2124 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2125 prd_table[i].base_addr =
2126 cpu_to_le32(lower_32_bits(sg->dma_address));
2127 prd_table[i].upper_addr =
2128 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2129 prd_table[i].reserved = 0;
7a3e97b0
SY		 2130 }
2131 } else {
2132 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2133 }
2134
2135 return 0;
2136}
2137
2138/**
2fbd009b 2139 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2140 * @hba: per adapter instance
2fbd009b 2141 * @intrs: interrupt bits
7a3e97b0 2142 */
2fbd009b 2143static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2144{
2fbd009b
SJ		 2145 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2146
2147 if (hba->ufs_version == UFSHCI_VERSION_10) {
2148 u32 rw;
2149 rw = set & INTERRUPT_MASK_RW_VER_10;
2150 set = rw | ((set ^ intrs) & intrs);
2151 } else {
2152 set |= intrs;
2153 }
2154
2155 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2156}
2157
2158/**
2159 * ufshcd_disable_intr - disable interrupts
2160 * @hba: per adapter instance
2161 * @intrs: interrupt bits
2162 */
2163static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2164{
2165 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2166
2167 if (hba->ufs_version == UFSHCI_VERSION_10) {
2168 u32 rw;
2169 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2170 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2171 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2172
2173 } else {
2174 set &= ~intrs;
7a3e97b0 2175 }
2fbd009b
SJ		 2176
2177 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2178}
2179
5a0b0cb9
SRT		 2180/**
2181 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2182 * descriptor according to request
2183 * @lrbp: pointer to local reference block
2184 * @upiu_flags: flags required in the header
2185 * @cmd_dir: requests data direction
2186 */
2187static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
300bb13f 2188 u32 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2189{
2190 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2191 u32 data_direction;
2192 u32 dword_0;
2193
2194 if (cmd_dir == DMA_FROM_DEVICE) {
2195 data_direction = UTP_DEVICE_TO_HOST;
2196 *upiu_flags = UPIU_CMD_FLAGS_READ;
2197 } else if (cmd_dir == DMA_TO_DEVICE) {
2198 data_direction = UTP_HOST_TO_DEVICE;
2199 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2200 } else {
2201 data_direction = UTP_NO_DATA_TRANSFER;
2202 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2203 }
2204
2205 dword_0 = data_direction | (lrbp->command_type
2206 << UPIU_COMMAND_TYPE_OFFSET);
2207 if (lrbp->intr_cmd)
2208 dword_0 |= UTP_REQ_DESC_INT_CMD;
2209
2210 /* Transfer request descriptor header fields */
2211 req_desc->header.dword_0 = cpu_to_le32(dword_0);
52ac95fe
YG		 2212 /* dword_1 is reserved, hence it is set to 0 */
2213 req_desc->header.dword_1 = 0;
5a0b0cb9
SRT		 2214 /*
2215 * assigning invalid value for command status. Controller
2216 * updates OCS on command completion, with the command
2217 * status
2218 */
2219 req_desc->header.dword_2 =
2220 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
52ac95fe
YG		 2221 /* dword_3 is reserved, hence it is set to 0 */
2222 req_desc->header.dword_3 = 0;
51047266
YG		 2223
2224 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2225}
2226
2227/**
2228 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2229 * for scsi commands
8aa29f19
BVA		 2230 * @lrbp: local reference block pointer
2231 * @upiu_flags: flags
5a0b0cb9
SRT		 2232 */
2233static
2234void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2235{
2236 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2237 unsigned short cdb_len;
5a0b0cb9
SRT		 2238
2239 /* command descriptor fields */
2240 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2241 UPIU_TRANSACTION_COMMAND, upiu_flags,
2242 lrbp->lun, lrbp->task_tag);
2243 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2244 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2245
2246 /* Total EHS length and Data segment length will be zero */
2247 ucd_req_ptr->header.dword_2 = 0;
2248
2249 ucd_req_ptr->sc.exp_data_transfer_len =
2250 cpu_to_be32(lrbp->cmd->sdb.length);
2251
a851b2bd
AA		 2252 cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, UFS_CDB_SIZE);
2253 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
52ac95fe
YG		 2254 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2255
2256 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2257}
2258
68078d5c
DR		 2259/**
2260 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2261 * for query requsts
2262 * @hba: UFS hba
2263 * @lrbp: local reference block pointer
2264 * @upiu_flags: flags
2265 */
2266static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2267 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2268{
2269 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2270 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2271 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2272
2273 /* Query request header */
2274 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2275 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2276 lrbp->lun, lrbp->task_tag);
2277 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2278 0, query->request.query_func, 0, 0);
2279
6861285c
ZL		 2280 /* Data segment length only need for WRITE_DESC */
2281 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2282 ucd_req_ptr->header.dword_2 =
2283 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2284 else
2285 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2286
2287 /* Copy the Query Request buffer as is */
2288 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2289 QUERY_OSF_SIZE);
68078d5c
DR		 2290
2291 /* Copy the Descriptor */
c6d4a831 2292 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2293 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2294
51047266 2295 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2296}
2297
5a0b0cb9
SRT		 2298static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2299{
2300 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2301
2302 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2303
2304 /* command descriptor fields */
2305 ucd_req_ptr->header.dword_0 =
2306 UPIU_HEADER_DWORD(
2307 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2308 /* clear rest of the fields of basic header */
2309 ucd_req_ptr->header.dword_1 = 0;
2310 ucd_req_ptr->header.dword_2 = 0;
2311
2312 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2313}
2314
7a3e97b0 2315/**
300bb13f
JP		 2316 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2317 * for Device Management Purposes
8aa29f19
BVA		 2318 * @hba: per adapter instance
2319 * @lrbp: pointer to local reference block
7a3e97b0 2320 */
300bb13f 2321static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0 2322{
7a3e97b0 2323 u32 upiu_flags;
5a0b0cb9 2324 int ret = 0;
7a3e97b0 2325
83dc7e3d 2326 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2327 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2328 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2329 else
2330 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2331
2332 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2333 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2334 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2335 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2336 ufshcd_prepare_utp_nop_upiu(lrbp);
2337 else
2338 ret = -EINVAL;
2339
2340 return ret;
2341}
2342
2343/**
2344 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2345 * for SCSI Purposes
8aa29f19
BVA		 2346 * @hba: per adapter instance
2347 * @lrbp: pointer to local reference block
300bb13f
JP		 2348 */
2349static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2350{
2351 u32 upiu_flags;
2352 int ret = 0;
2353
83dc7e3d 2354 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2355 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2356 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2357 else
2358 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2359
2360 if (likely(lrbp->cmd)) {
2361 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2362 lrbp->cmd->sc_data_direction);
2363 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2364 } else {
2365 ret = -EINVAL;
2366 }
5a0b0cb9
SRT		 2367
2368 return ret;
7a3e97b0
SY		 2369}
2370
2a8fa600
SJ		 2371/**
2372 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2373 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2374 *
2375 * Returns SCSI W-LUN id
2376 */
2377static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2378{
2379 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2380}
2381
7a3e97b0
SY		 2382/**
2383 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2384 * @host: SCSI host pointer
7a3e97b0 2385 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2386 *
2387 * Returns 0 for success, non-zero in case of failure
2388 */
2389static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2390{
2391 struct ufshcd_lrb *lrbp;
2392 struct ufs_hba *hba;
2393 unsigned long flags;
2394 int tag;
2395 int err = 0;
2396
2397 hba = shost_priv(host);
2398
2399 tag = cmd->request->tag;
14497328
YG		 2400 if (!ufshcd_valid_tag(hba, tag)) {
2401 dev_err(hba->dev,
2402 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2403 __func__, tag, cmd, cmd->request);
2404 BUG();
2405 }
7a3e97b0 2406
a3cd5ec5 2407 if (!down_read_trylock(&hba->clk_scaling_lock))
2408 return SCSI_MLQUEUE_HOST_BUSY;
2409
3441da7d
SRT		 2410 spin_lock_irqsave(hba->host->host_lock, flags);
2411 switch (hba->ufshcd_state) {
2412 case UFSHCD_STATE_OPERATIONAL:
2413 break;
141f8165 2414 case UFSHCD_STATE_EH_SCHEDULED:
3441da7d 2415 case UFSHCD_STATE_RESET:
7a3e97b0 2416 err = SCSI_MLQUEUE_HOST_BUSY;
3441da7d
SRT		 2417 goto out_unlock;
2418 case UFSHCD_STATE_ERROR:
2419 set_host_byte(cmd, DID_ERROR);
2420 cmd->scsi_done(cmd);
2421 goto out_unlock;
2422 default:
2423 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2424 __func__, hba->ufshcd_state);
2425 set_host_byte(cmd, DID_BAD_TARGET);
2426 cmd->scsi_done(cmd);
2427 goto out_unlock;
7a3e97b0 2428 }
53c12d0e
YG		 2429
2430 /* if error handling is in progress, don't issue commands */
2431 if (ufshcd_eh_in_progress(hba)) {
2432 set_host_byte(cmd, DID_ERROR);
2433 cmd->scsi_done(cmd);
2434 goto out_unlock;
2435 }
3441da7d 2436 spin_unlock_irqrestore(hba->host->host_lock, flags);
7a3e97b0 2437
7fabb77b
GB		 2438 hba->req_abort_count = 0;
2439
5a0b0cb9
SRT		 2440 /* acquire the tag to make sure device cmds don't use it */
2441 if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
2442 /*
2443 * Dev manage command in progress, requeue the command.
2444 * Requeuing the command helps in cases where the request *may*
2445 * find different tag instead of waiting for dev manage command
2446 * completion.
2447 */
2448 err = SCSI_MLQUEUE_HOST_BUSY;
2449 goto out;
2450 }
2451
1ab27c9c
ST		 2452 err = ufshcd_hold(hba, true);
2453 if (err) {
2454 err = SCSI_MLQUEUE_HOST_BUSY;
2455 clear_bit_unlock(tag, &hba->lrb_in_use);
2456 goto out;
2457 }
2458 WARN_ON(hba->clk_gating.state != CLKS_ON);
2459
7a3e97b0
SY		 2460 lrbp = &hba->lrb[tag];
2461
5a0b0cb9 2462 WARN_ON(lrbp->cmd);
7a3e97b0 2463 lrbp->cmd = cmd;
09a5a24f 2464 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2465 lrbp->sense_buffer = cmd->sense_buffer;
2466 lrbp->task_tag = tag;
0ce147d4 2467 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2468 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
e0b299e3 2469 lrbp->req_abort_skip = false;
7a3e97b0 2470
300bb13f
JP		 2471 ufshcd_comp_scsi_upiu(hba, lrbp);
2472
75b1cc4a 2473 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2474 if (err) {
2475 lrbp->cmd = NULL;
2476 clear_bit_unlock(tag, &hba->lrb_in_use);
7a3e97b0 2477 goto out;
5a0b0cb9 2478 }
ad1a1b9c
GB		 2479 /* Make sure descriptors are ready before ringing the doorbell */
2480 wmb();
7a3e97b0
SY		 2481
2482 /* issue command to the controller */
2483 spin_lock_irqsave(hba->host->host_lock, flags);
0e675efa 2484 ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
7a3e97b0 2485 ufshcd_send_command(hba, tag);
3441da7d 2486out_unlock:
7a3e97b0
SY		 2487 spin_unlock_irqrestore(hba->host->host_lock, flags);
2488out:
a3cd5ec5 2489 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2490 return err;
2491}
2492
5a0b0cb9
SRT		 2493static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2494 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2495{
2496 lrbp->cmd = NULL;
2497 lrbp->sense_bufflen = 0;
2498 lrbp->sense_buffer = NULL;
2499 lrbp->task_tag = tag;
2500 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9
SRT		 2501 lrbp->intr_cmd = true; /* No interrupt aggregation */
2502 hba->dev_cmd.type = cmd_type;
2503
300bb13f 2504 return ufshcd_comp_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2505}
2506
2507static int
2508ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2509{
2510 int err = 0;
2511 unsigned long flags;
2512 u32 mask = 1 << tag;
2513
2514 /* clear outstanding transaction before retry */
2515 spin_lock_irqsave(hba->host->host_lock, flags);
2516 ufshcd_utrl_clear(hba, tag);
2517 spin_unlock_irqrestore(hba->host->host_lock, flags);
2518
2519 /*
2520 * wait for for h/w to clear corresponding bit in door-bell.
2521 * max. wait is 1 sec.
2522 */
2523 err = ufshcd_wait_for_register(hba,
2524 REG_UTP_TRANSFER_REQ_DOOR_BELL,
596585a2 2525 mask, ~mask, 1000, 1000, true);
5a0b0cb9
SRT		 2526
2527 return err;
2528}
2529
c6d4a831
DR		 2530static int
2531ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2532{
2533 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2534
2535 /* Get the UPIU response */
2536 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2537 UPIU_RSP_CODE_OFFSET;
2538 return query_res->response;
2539}
2540
5a0b0cb9
SRT		 2541/**
2542 * ufshcd_dev_cmd_completion() - handles device management command responses
2543 * @hba: per adapter instance
2544 * @lrbp: pointer to local reference block
2545 */
2546static int
2547ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2548{
2549 int resp;
2550 int err = 0;
2551
ff8e20c6 2552 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2553 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2554
2555 switch (resp) {
2556 case UPIU_TRANSACTION_NOP_IN:
2557 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2558 err = -EINVAL;
2559 dev_err(hba->dev, "%s: unexpected response %x\n",
2560 __func__, resp);
2561 }
2562 break;
68078d5c 2563 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2564 err = ufshcd_check_query_response(hba, lrbp);
2565 if (!err)
2566 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2567 break;
5a0b0cb9
SRT		 2568 case UPIU_TRANSACTION_REJECT_UPIU:
2569 /* TODO: handle Reject UPIU Response */
2570 err = -EPERM;
2571 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2572 __func__);
2573 break;
2574 default:
2575 err = -EINVAL;
2576 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2577 __func__, resp);
2578 break;
2579 }
2580
2581 return err;
2582}
2583
2584static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2585 struct ufshcd_lrb *lrbp, int max_timeout)
2586{
2587 int err = 0;
2588 unsigned long time_left;
2589 unsigned long flags;
2590
2591 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2592 msecs_to_jiffies(max_timeout));
2593
ad1a1b9c
GB		 2594 /* Make sure descriptors are ready before ringing the doorbell */
2595 wmb();
5a0b0cb9
SRT		 2596 spin_lock_irqsave(hba->host->host_lock, flags);
2597 hba->dev_cmd.complete = NULL;
2598 if (likely(time_left)) {
2599 err = ufshcd_get_tr_ocs(lrbp);
2600 if (!err)
2601 err = ufshcd_dev_cmd_completion(hba, lrbp);
2602 }
2603 spin_unlock_irqrestore(hba->host->host_lock, flags);
2604
2605 if (!time_left) {
2606 err = -ETIMEDOUT;
a48353f6
YG		 2607 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2608 __func__, lrbp->task_tag);
5a0b0cb9 2609 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2610 /* successfully cleared the command, retry if needed */
5a0b0cb9 2611 err = -EAGAIN;
a48353f6
YG		 2612 /*
2613 * in case of an error, after clearing the doorbell,
2614 * we also need to clear the outstanding_request
2615 * field in hba
2616 */
2617 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2618 }
2619
2620 return err;
2621}
2622
2623/**
2624 * ufshcd_get_dev_cmd_tag - Get device management command tag
2625 * @hba: per-adapter instance
8aa29f19 2626 * @tag_out: pointer to variable with available slot value
5a0b0cb9
SRT		 2627 *
2628 * Get a free slot and lock it until device management command
2629 * completes.
2630 *
2631 * Returns false if free slot is unavailable for locking, else
2632 * return true with tag value in @tag.
2633 */
2634static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
2635{
2636 int tag;
2637 bool ret = false;
2638 unsigned long tmp;
2639
2640 if (!tag_out)
2641 goto out;
2642
2643 do {
2644 tmp = ~hba->lrb_in_use;
2645 tag = find_last_bit(&tmp, hba->nutrs);
2646 if (tag >= hba->nutrs)
2647 goto out;
2648 } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
2649
2650 *tag_out = tag;
2651 ret = true;
2652out:
2653 return ret;
2654}
2655
2656static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
2657{
2658 clear_bit_unlock(tag, &hba->lrb_in_use);
2659}
2660
2661/**
2662 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2663 * @hba: UFS hba
2664 * @cmd_type: specifies the type (NOP, Query...)
2665 * @timeout: time in seconds
5a0b0cb9 2666 *
68078d5c
DR		 2667 * NOTE: Since there is only one available tag for device management commands,
2668 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2669 */
2670static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2671 enum dev_cmd_type cmd_type, int timeout)
2672{
2673 struct ufshcd_lrb *lrbp;
2674 int err;
2675 int tag;
2676 struct completion wait;
2677 unsigned long flags;
2678
a3cd5ec5 2679 down_read(&hba->clk_scaling_lock);
2680
5a0b0cb9
SRT		 2681 /*
2682 * Get free slot, sleep if slots are unavailable.
2683 * Even though we use wait_event() which sleeps indefinitely,
2684 * the maximum wait time is bounded by SCSI request timeout.
2685 */
2686 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
2687
2688 init_completion(&wait);
2689 lrbp = &hba->lrb[tag];
2690 WARN_ON(lrbp->cmd);
2691 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2692 if (unlikely(err))
2693 goto out_put_tag;
2694
2695 hba->dev_cmd.complete = &wait;
2696
6667e6d9 2697 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
e3dfdc53
YG		 2698 /* Make sure descriptors are ready before ringing the doorbell */
2699 wmb();
5a0b0cb9 2700 spin_lock_irqsave(hba->host->host_lock, flags);
0e675efa 2701 ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
5a0b0cb9
SRT		 2702 ufshcd_send_command(hba, tag);
2703 spin_unlock_irqrestore(hba->host->host_lock, flags);
2704
2705 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2706
6667e6d9
OS		 2707 ufshcd_add_query_upiu_trace(hba, tag,
2708 err ? "query_complete_err" : "query_complete");
2709
5a0b0cb9
SRT		 2710out_put_tag:
2711 ufshcd_put_dev_cmd_tag(hba, tag);
2712 wake_up(&hba->dev_cmd.tag_wq);
a3cd5ec5 2713 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2714 return err;
2715}
2716
d44a5f98
DR		 2717/**
2718 * ufshcd_init_query() - init the query response and request parameters
2719 * @hba: per-adapter instance
2720 * @request: address of the request pointer to be initialized
2721 * @response: address of the response pointer to be initialized
2722 * @opcode: operation to perform
2723 * @idn: flag idn to access
2724 * @index: LU number to access
2725 * @selector: query/flag/descriptor further identification
2726 */
2727static inline void ufshcd_init_query(struct ufs_hba *hba,
2728 struct ufs_query_req **request, struct ufs_query_res **response,
2729 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2730{
2731 *request = &hba->dev_cmd.query.request;
2732 *response = &hba->dev_cmd.query.response;
2733 memset(*request, 0, sizeof(struct ufs_query_req));
2734 memset(*response, 0, sizeof(struct ufs_query_res));
2735 (*request)->upiu_req.opcode = opcode;
2736 (*request)->upiu_req.idn = idn;
2737 (*request)->upiu_req.index = index;
2738 (*request)->upiu_req.selector = selector;
2739}
2740
dc3c8d3a
YG		 2741static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2742 enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2743{
2744 int ret;
2745 int retries;
2746
2747 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2748 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2749 if (ret)
2750 dev_dbg(hba->dev,
2751 "%s: failed with error %d, retries %d\n",
2752 __func__, ret, retries);
2753 else
2754 break;
2755 }
2756
2757 if (ret)
2758 dev_err(hba->dev,
2759 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2760 __func__, opcode, idn, ret, retries);
2761 return ret;
2762}
2763
68078d5c
DR		 2764/**
2765 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2766 * @hba: per-adapter instance
2767 * @opcode: flag query to perform
2768 * @idn: flag idn to access
2769 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2770 *
2771 * Returns 0 for success, non-zero in case of failure
2772 */
dc3c8d3a 2773int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
68078d5c
DR		 2774 enum flag_idn idn, bool *flag_res)
2775{
d44a5f98
DR		 2776 struct ufs_query_req *request = NULL;
2777 struct ufs_query_res *response = NULL;
2778 int err, index = 0, selector = 0;
e5ad406c 2779 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2780
2781 BUG_ON(!hba);
2782
1ab27c9c 2783 ufshcd_hold(hba, false);
68078d5c 2784 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2785 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2786 selector);
68078d5c
DR		 2787
2788 switch (opcode) {
2789 case UPIU_QUERY_OPCODE_SET_FLAG:
2790 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2791 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2792 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2793 break;
2794 case UPIU_QUERY_OPCODE_READ_FLAG:
2795 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2796 if (!flag_res) {
2797 /* No dummy reads */
2798 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2799 __func__);
2800 err = -EINVAL;
2801 goto out_unlock;
2802 }
2803 break;
2804 default:
2805 dev_err(hba->dev,
2806 "%s: Expected query flag opcode but got = %d\n",
2807 __func__, opcode);
2808 err = -EINVAL;
2809 goto out_unlock;
2810 }
68078d5c 2811
e5ad406c 2812 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 2813
2814 if (err) {
2815 dev_err(hba->dev,
2816 "%s: Sending flag query for idn %d failed, err = %d\n",
2817 __func__, idn, err);
2818 goto out_unlock;
2819 }
2820
2821 if (flag_res)
e8c8e82a 2822 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 2823 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2824
2825out_unlock:
2826 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 2827 ufshcd_release(hba);
68078d5c
DR		 2828 return err;
2829}
2830
66ec6d59
SRT		 2831/**
2832 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 2833 * @hba: per-adapter instance
2834 * @opcode: attribute opcode
2835 * @idn: attribute idn to access
2836 * @index: index field
2837 * @selector: selector field
2838 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 2839 *
2840 * Returns 0 for success, non-zero in case of failure
2841*/
ec92b59c
SN		 2842int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2843 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 2844{
d44a5f98
DR		 2845 struct ufs_query_req *request = NULL;
2846 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 2847 int err;
2848
2849 BUG_ON(!hba);
2850
1ab27c9c 2851 ufshcd_hold(hba, false);
66ec6d59
SRT		 2852 if (!attr_val) {
2853 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2854 __func__, opcode);
2855 err = -EINVAL;
2856 goto out;
2857 }
2858
2859 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2860 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2861 selector);
66ec6d59
SRT		 2862
2863 switch (opcode) {
2864 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2865 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 2866 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 2867 break;
2868 case UPIU_QUERY_OPCODE_READ_ATTR:
2869 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2870 break;
2871 default:
2872 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2873 __func__, opcode);
2874 err = -EINVAL;
2875 goto out_unlock;
2876 }
2877
d44a5f98 2878 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 2879
2880 if (err) {
4b761b58
YG		 2881 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2882 __func__, opcode, idn, index, err);
66ec6d59
SRT		 2883 goto out_unlock;
2884 }
2885
e8c8e82a 2886 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 2887
2888out_unlock:
2889 mutex_unlock(&hba->dev_cmd.lock);
2890out:
1ab27c9c 2891 ufshcd_release(hba);
66ec6d59
SRT		 2892 return err;
2893}
2894
5e86ae44
YG		 2895/**
2896 * ufshcd_query_attr_retry() - API function for sending query
2897 * attribute with retries
2898 * @hba: per-adapter instance
2899 * @opcode: attribute opcode
2900 * @idn: attribute idn to access
2901 * @index: index field
2902 * @selector: selector field
2903 * @attr_val: the attribute value after the query request
2904 * completes
2905 *
2906 * Returns 0 for success, non-zero in case of failure
2907*/
2908static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2909 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2910 u32 *attr_val)
2911{
2912 int ret = 0;
2913 u32 retries;
2914
2915 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2916 ret = ufshcd_query_attr(hba, opcode, idn, index,
2917 selector, attr_val);
2918 if (ret)
2919 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2920 __func__, ret, retries);
2921 else
2922 break;
2923 }
2924
2925 if (ret)
2926 dev_err(hba->dev,
2927 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2928 __func__, idn, ret, QUERY_REQ_RETRIES);
2929 return ret;
2930}
2931
a70e91b8 2932static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 2933 enum query_opcode opcode, enum desc_idn idn, u8 index,
2934 u8 selector, u8 *desc_buf, int *buf_len)
2935{
2936 struct ufs_query_req *request = NULL;
2937 struct ufs_query_res *response = NULL;
2938 int err;
2939
2940 BUG_ON(!hba);
2941
1ab27c9c 2942 ufshcd_hold(hba, false);
d44a5f98
DR		 2943 if (!desc_buf) {
2944 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2945 __func__, opcode);
2946 err = -EINVAL;
2947 goto out;
2948 }
2949
a4b0e8a4 2950 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 2951 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2952 __func__, *buf_len);
2953 err = -EINVAL;
2954 goto out;
2955 }
2956
2957 mutex_lock(&hba->dev_cmd.lock);
2958 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2959 selector);
2960 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 2961 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 2962
2963 switch (opcode) {
2964 case UPIU_QUERY_OPCODE_WRITE_DESC:
2965 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2966 break;
2967 case UPIU_QUERY_OPCODE_READ_DESC:
2968 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2969 break;
2970 default:
2971 dev_err(hba->dev,
2972 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2973 __func__, opcode);
2974 err = -EINVAL;
2975 goto out_unlock;
2976 }
2977
2978 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2979
2980 if (err) {
4b761b58
YG		 2981 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2982 __func__, opcode, idn, index, err);
d44a5f98
DR		 2983 goto out_unlock;
2984 }
2985
2986 hba->dev_cmd.query.descriptor = NULL;
ea2aab24 2987 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 2988
2989out_unlock:
2990 mutex_unlock(&hba->dev_cmd.lock);
2991out:
1ab27c9c 2992 ufshcd_release(hba);
d44a5f98
DR		 2993 return err;
2994}
2995
a70e91b8 2996/**
8aa29f19
BVA		 2997 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2998 * @hba: per-adapter instance
2999 * @opcode: attribute opcode
3000 * @idn: attribute idn to access
3001 * @index: index field
3002 * @selector: selector field
3003 * @desc_buf: the buffer that contains the descriptor
3004 * @buf_len: length parameter passed to the device
a70e91b8
YG		 3005 *
3006 * Returns 0 for success, non-zero in case of failure.
3007 * The buf_len parameter will contain, on return, the length parameter
3008 * received on the response.
3009 */
2238d31c
SN		 3010int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3011 enum query_opcode opcode,
3012 enum desc_idn idn, u8 index,
3013 u8 selector,
3014 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 3015{
3016 int err;
3017 int retries;
3018
3019 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3020 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3021 selector, desc_buf, buf_len);
3022 if (!err || err == -EINVAL)
3023 break;
3024 }
3025
3026 return err;
3027}
a70e91b8 3028
a4b0e8a4
PM		 3029/**
3030 * ufshcd_read_desc_length - read the specified descriptor length from header
3031 * @hba: Pointer to adapter instance
3032 * @desc_id: descriptor idn value
3033 * @desc_index: descriptor index
3034 * @desc_length: pointer to variable to read the length of descriptor
3035 *
3036 * Return 0 in case of success, non-zero otherwise
3037 */
3038static int ufshcd_read_desc_length(struct ufs_hba *hba,
3039 enum desc_idn desc_id,
3040 int desc_index,
3041 int *desc_length)
3042{
3043 int ret;
3044 u8 header[QUERY_DESC_HDR_SIZE];
3045 int header_len = QUERY_DESC_HDR_SIZE;
3046
3047 if (desc_id >= QUERY_DESC_IDN_MAX)
3048 return -EINVAL;
3049
3050 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3051 desc_id, desc_index, 0, header,
3052 &header_len);
3053
3054 if (ret) {
3055 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
3056 __func__, desc_id);
3057 return ret;
3058 } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3059 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3060 __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3061 desc_id);
3062 ret = -EINVAL;
3063 }
3064
3065 *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3066 return ret;
3067
3068}
3069
3070/**
3071 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3072 * @hba: Pointer to adapter instance
3073 * @desc_id: descriptor idn value
3074 * @desc_len: mapped desc length (out)
3075 *
3076 * Return 0 in case of success, non-zero otherwise
3077 */
3078int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3079 enum desc_idn desc_id, int *desc_len)
3080{
3081 switch (desc_id) {
3082 case QUERY_DESC_IDN_DEVICE:
3083 *desc_len = hba->desc_size.dev_desc;
3084 break;
3085 case QUERY_DESC_IDN_POWER:
3086 *desc_len = hba->desc_size.pwr_desc;
3087 break;
3088 case QUERY_DESC_IDN_GEOMETRY:
3089 *desc_len = hba->desc_size.geom_desc;
3090 break;
3091 case QUERY_DESC_IDN_CONFIGURATION:
3092 *desc_len = hba->desc_size.conf_desc;
3093 break;
3094 case QUERY_DESC_IDN_UNIT:
3095 *desc_len = hba->desc_size.unit_desc;
3096 break;
3097 case QUERY_DESC_IDN_INTERCONNECT:
3098 *desc_len = hba->desc_size.interc_desc;
3099 break;
3100 case QUERY_DESC_IDN_STRING:
3101 *desc_len = QUERY_DESC_MAX_SIZE;
3102 break;
c648c2d2
SN		 3103 case QUERY_DESC_IDN_HEALTH:
3104 *desc_len = hba->desc_size.hlth_desc;
3105 break;
a4b0e8a4
PM		 3106 case QUERY_DESC_IDN_RFU_0:
3107 case QUERY_DESC_IDN_RFU_1:
3108 *desc_len = 0;
3109 break;
3110 default:
3111 *desc_len = 0;
3112 return -EINVAL;
3113 }
3114 return 0;
3115}
3116EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3117
da461cec
SJ		 3118/**
3119 * ufshcd_read_desc_param - read the specified descriptor parameter
3120 * @hba: Pointer to adapter instance
3121 * @desc_id: descriptor idn value
3122 * @desc_index: descriptor index
3123 * @param_offset: offset of the parameter to read
3124 * @param_read_buf: pointer to buffer where parameter would be read
3125 * @param_size: sizeof(param_read_buf)
3126 *
3127 * Return 0 in case of success, non-zero otherwise
3128 */
45bced87
SN		 3129int ufshcd_read_desc_param(struct ufs_hba *hba,
3130 enum desc_idn desc_id,
3131 int desc_index,
3132 u8 param_offset,
3133 u8 *param_read_buf,
3134 u8 param_size)
da461cec
SJ		 3135{
3136 int ret;
3137 u8 *desc_buf;
a4b0e8a4 3138 int buff_len;
da461cec
SJ		 3139 bool is_kmalloc = true;
3140
a4b0e8a4
PM		 3141 /* Safety check */
3142 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3143 return -EINVAL;
3144
a4b0e8a4
PM		 3145 /* Get the max length of descriptor from structure filled up at probe
3146 * time.
3147 */
3148 ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
da461cec 3149
a4b0e8a4
PM		 3150 /* Sanity checks */
3151 if (ret || !buff_len) {
3152 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3153 __func__);
3154 return ret;
3155 }
3156
3157 /* Check whether we need temp memory */
3158 if (param_offset != 0 || param_size < buff_len) {
da461cec
SJ		 3159 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3160 if (!desc_buf)
3161 return -ENOMEM;
a4b0e8a4
PM		 3162 } else {
3163 desc_buf = param_read_buf;
3164 is_kmalloc = false;
da461cec
SJ		 3165 }
3166
a4b0e8a4 3167 /* Request for full descriptor */
a70e91b8 3168 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3169 desc_id, desc_index, 0,
3170 desc_buf, &buff_len);
da461cec 3171
bde44bb6 3172 if (ret) {
3173 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3174 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3175 goto out;
3176 }
3177
bde44bb6 3178 /* Sanity check */
3179 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3180 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3181 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3182 ret = -EINVAL;
3183 goto out;
3184 }
3185
a4b0e8a4
PM		 3186 /* Check wherher we will not copy more data, than available */
3187 if (is_kmalloc && param_size > buff_len)
3188 param_size = buff_len;
bde44bb6 3189
da461cec
SJ		 3190 if (is_kmalloc)
3191 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3192out:
3193 if (is_kmalloc)
3194 kfree(desc_buf);
3195 return ret;
3196}
3197
3198static inline int ufshcd_read_desc(struct ufs_hba *hba,
3199 enum desc_idn desc_id,
3200 int desc_index,
3201 u8 *buf,
3202 u32 size)
3203{
3204 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3205}
3206
3207static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3208 u8 *buf,
3209 u32 size)
3210{
dbd34a61 3211 return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
da461cec
SJ		 3212}
3213
8209b6d5 3214static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
b573d484
YG		 3215{
3216 return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3217}
b573d484
YG		 3218
3219/**
3220 * ufshcd_read_string_desc - read string descriptor
3221 * @hba: pointer to adapter instance
3222 * @desc_index: descriptor index
3223 * @buf: pointer to buffer where descriptor would be read
3224 * @size: size of buf
3225 * @ascii: if true convert from unicode to ascii characters
3226 *
3227 * Return 0 in case of success, non-zero otherwise
3228 */
2238d31c
SN		 3229int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
3230 u8 *buf, u32 size, bool ascii)
b573d484
YG		 3231{
3232 int err = 0;
3233
3234 err = ufshcd_read_desc(hba,
3235 QUERY_DESC_IDN_STRING, desc_index, buf, size);
3236
3237 if (err) {
3238 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
3239 __func__, QUERY_REQ_RETRIES, err);
3240 goto out;
3241 }
3242
3243 if (ascii) {
3244 int desc_len;
3245 int ascii_len;
3246 int i;
3247 char *buff_ascii;
3248
3249 desc_len = buf[0];
3250 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3251 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3252 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
3253 dev_err(hba->dev, "%s: buffer allocated size is too small\n",
3254 __func__);
3255 err = -ENOMEM;
3256 goto out;
3257 }
3258
3259 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
3260 if (!buff_ascii) {
3261 err = -ENOMEM;
fcbefc3b 3262 goto out;
b573d484
YG		 3263 }
3264
3265 /*
3266 * the descriptor contains string in UTF16 format
3267 * we need to convert to utf-8 so it can be displayed
3268 */
3269 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
3270 desc_len - QUERY_DESC_HDR_SIZE,
3271 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
3272
3273 /* replace non-printable or non-ASCII characters with spaces */
3274 for (i = 0; i < ascii_len; i++)
3275 ufshcd_remove_non_printable(&buff_ascii[i]);
3276
3277 memset(buf + QUERY_DESC_HDR_SIZE, 0,
3278 size - QUERY_DESC_HDR_SIZE);
3279 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
3280 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
b573d484
YG		 3281 kfree(buff_ascii);
3282 }
3283out:
3284 return err;
3285}
b573d484 3286
da461cec
SJ		 3287/**
3288 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3289 * @hba: Pointer to adapter instance
3290 * @lun: lun id
3291 * @param_offset: offset of the parameter to read
3292 * @param_read_buf: pointer to buffer where parameter would be read
3293 * @param_size: sizeof(param_read_buf)
3294 *
3295 * Return 0 in case of success, non-zero otherwise
3296 */
3297static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3298 int lun,
3299 enum unit_desc_param param_offset,
3300 u8 *param_read_buf,
3301 u32 param_size)
3302{
3303 /*
3304 * Unit descriptors are only available for general purpose LUs (LUN id
3305 * from 0 to 7) and RPMB Well known LU.
3306 */
d829fc8a 3307 if (!ufs_is_valid_unit_desc_lun(lun))
da461cec
SJ		 3308 return -EOPNOTSUPP;
3309
3310 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3311 param_offset, param_read_buf, param_size);
3312}
3313
7a3e97b0
SY		 3314/**
3315 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3316 * @hba: per adapter instance
3317 *
3318 * 1. Allocate DMA memory for Command Descriptor array
3319 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3320 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3321 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3322 * (UTMRDL)
3323 * 4. Allocate memory for local reference block(lrb).
3324 *
3325 * Returns 0 for success, non-zero in case of failure
3326 */
3327static int ufshcd_memory_alloc(struct ufs_hba *hba)
3328{
3329 size_t utmrdl_size, utrdl_size, ucdl_size;
3330
3331 /* Allocate memory for UTP command descriptors */
3332 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3333 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3334 ucdl_size,
3335 &hba->ucdl_dma_addr,
3336 GFP_KERNEL);
7a3e97b0
SY		 3337
3338 /*
3339 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3340 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3341 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3342 * be aligned to 128 bytes as well
3343 */
3344 if (!hba->ucdl_base_addr ||
3345 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3346 dev_err(hba->dev,
7a3e97b0
SY		 3347 "Command Descriptor Memory allocation failed\n");
3348 goto out;
3349 }
3350
3351 /*
3352 * Allocate memory for UTP Transfer descriptors
3353 * UFSHCI requires 1024 byte alignment of UTRD
3354 */
3355 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3356 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3357 utrdl_size,
3358 &hba->utrdl_dma_addr,
3359 GFP_KERNEL);
7a3e97b0
SY		 3360 if (!hba->utrdl_base_addr ||
3361 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3362 dev_err(hba->dev,
7a3e97b0
SY		 3363 "Transfer Descriptor Memory allocation failed\n");
3364 goto out;
3365 }
3366
3367 /*
3368 * Allocate memory for UTP Task Management descriptors
3369 * UFSHCI requires 1024 byte alignment of UTMRD
3370 */
3371 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3372 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3373 utmrdl_size,
3374 &hba->utmrdl_dma_addr,
3375 GFP_KERNEL);
7a3e97b0
SY		 3376 if (!hba->utmrdl_base_addr ||
3377 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3378 dev_err(hba->dev,
7a3e97b0
SY		 3379 "Task Management Descriptor Memory allocation failed\n");
3380 goto out;
3381 }
3382
3383 /* Allocate memory for local reference block */
a86854d0
KC		 3384 hba->lrb = devm_kcalloc(hba->dev,
3385 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3386 GFP_KERNEL);
7a3e97b0 3387 if (!hba->lrb) {
3b1d0580 3388 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3389 goto out;
3390 }
3391 return 0;
3392out:
7a3e97b0
SY		 3393 return -ENOMEM;
3394}
3395
3396/**
3397 * ufshcd_host_memory_configure - configure local reference block with
3398 * memory offsets
3399 * @hba: per adapter instance
3400 *
3401 * Configure Host memory space
3402 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3403 * address.
3404 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3405 * and PRDT offset.
3406 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3407 * into local reference block.
3408 */
3409static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3410{
3411 struct utp_transfer_cmd_desc *cmd_descp;
3412 struct utp_transfer_req_desc *utrdlp;
3413 dma_addr_t cmd_desc_dma_addr;
3414 dma_addr_t cmd_desc_element_addr;
3415 u16 response_offset;
3416 u16 prdt_offset;
3417 int cmd_desc_size;
3418 int i;
3419
3420 utrdlp = hba->utrdl_base_addr;
3421 cmd_descp = hba->ucdl_base_addr;
3422
3423 response_offset =
3424 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3425 prdt_offset =
3426 offsetof(struct utp_transfer_cmd_desc, prd_table);
3427
3428 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3429 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3430
3431 for (i = 0; i < hba->nutrs; i++) {
3432 /* Configure UTRD with command descriptor base address */
3433 cmd_desc_element_addr =
3434 (cmd_desc_dma_addr + (cmd_desc_size * i));
3435 utrdlp[i].command_desc_base_addr_lo =
3436 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3437 utrdlp[i].command_desc_base_addr_hi =
3438 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3439
3440 /* Response upiu and prdt offset should be in double words */
75b1cc4a
KK		 3441 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3442 utrdlp[i].response_upiu_offset =
3443 cpu_to_le16(response_offset);
3444 utrdlp[i].prd_table_offset =
3445 cpu_to_le16(prdt_offset);
3446 utrdlp[i].response_upiu_length =
3447 cpu_to_le16(ALIGNED_UPIU_SIZE);
3448 } else {
3449 utrdlp[i].response_upiu_offset =
7a3e97b0 3450 cpu_to_le16((response_offset >> 2));
75b1cc4a 3451 utrdlp[i].prd_table_offset =
7a3e97b0 3452 cpu_to_le16((prdt_offset >> 2));
75b1cc4a 3453 utrdlp[i].response_upiu_length =
3ca316c5 3454 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
75b1cc4a 3455 }
7a3e97b0
SY		 3456
3457 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
ff8e20c6
DR		 3458 hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr +
3459 (i * sizeof(struct utp_transfer_req_desc));
5a0b0cb9
SRT		 3460 hba->lrb[i].ucd_req_ptr =
3461 (struct utp_upiu_req *)(cmd_descp + i);
ff8e20c6 3462 hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr;
7a3e97b0
SY		 3463 hba->lrb[i].ucd_rsp_ptr =
3464 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
ff8e20c6
DR		 3465 hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr +
3466 response_offset;
7a3e97b0
SY		 3467 hba->lrb[i].ucd_prdt_ptr =
3468 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
ff8e20c6
DR		 3469 hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr +
3470 prdt_offset;
7a3e97b0
SY		 3471 }
3472}
3473
3474/**
3475 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3476 * @hba: per adapter instance
3477 *
3478 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3479 * in order to initialize the Unipro link startup procedure.
3480 * Once the Unipro links are up, the device connected to the controller
3481 * is detected.
3482 *
3483 * Returns 0 on success, non-zero value on failure
3484 */
3485static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3486{
6ccf44fe
SJ		 3487 struct uic_command uic_cmd = {0};
3488 int ret;
7a3e97b0 3489
6ccf44fe 3490 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3491
6ccf44fe
SJ		 3492 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3493 if (ret)
ff8e20c6 3494 dev_dbg(hba->dev,
6ccf44fe
SJ		 3495 "dme-link-startup: error code %d\n", ret);
3496 return ret;
7a3e97b0 3497}
4404c5de
AA		 3498/**
3499 * ufshcd_dme_reset - UIC command for DME_RESET
3500 * @hba: per adapter instance
3501 *
3502 * DME_RESET command is issued in order to reset UniPro stack.
3503 * This function now deal with cold reset.
3504 *
3505 * Returns 0 on success, non-zero value on failure
3506 */
3507static int ufshcd_dme_reset(struct ufs_hba *hba)
3508{
3509 struct uic_command uic_cmd = {0};
3510 int ret;
3511
3512 uic_cmd.command = UIC_CMD_DME_RESET;
3513
3514 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3515 if (ret)
3516 dev_err(hba->dev,
3517 "dme-reset: error code %d\n", ret);
3518
3519 return ret;
3520}
3521
3522/**
3523 * ufshcd_dme_enable - UIC command for DME_ENABLE
3524 * @hba: per adapter instance
3525 *
3526 * DME_ENABLE command is issued in order to enable UniPro stack.
3527 *
3528 * Returns 0 on success, non-zero value on failure
3529 */
3530static int ufshcd_dme_enable(struct ufs_hba *hba)
3531{
3532 struct uic_command uic_cmd = {0};
3533 int ret;
3534
3535 uic_cmd.command = UIC_CMD_DME_ENABLE;
3536
3537 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3538 if (ret)
3539 dev_err(hba->dev,
3540 "dme-reset: error code %d\n", ret);
3541
3542 return ret;
3543}
7a3e97b0 3544
cad2e03d
YG		 3545static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3546{
3547 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3548 unsigned long min_sleep_time_us;
3549
3550 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3551 return;
3552
3553 /*
3554 * last_dme_cmd_tstamp will be 0 only for 1st call to
3555 * this function
3556 */
3557 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3558 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3559 } else {
3560 unsigned long delta =
3561 (unsigned long) ktime_to_us(
3562 ktime_sub(ktime_get(),
3563 hba->last_dme_cmd_tstamp));
3564
3565 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3566 min_sleep_time_us =
3567 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3568 else
3569 return; /* no more delay required */
3570 }
3571
3572 /* allow sleep for extra 50us if needed */
3573 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3574}
3575
12b4fdb4
SJ		 3576/**
3577 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3578 * @hba: per adapter instance
3579 * @attr_sel: uic command argument1
3580 * @attr_set: attribute set type as uic command argument2
3581 * @mib_val: setting value as uic command argument3
3582 * @peer: indicate whether peer or local
3583 *
3584 * Returns 0 on success, non-zero value on failure
3585 */
3586int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3587 u8 attr_set, u32 mib_val, u8 peer)
3588{
3589 struct uic_command uic_cmd = {0};
3590 static const char *const action[] = {
3591 "dme-set",
3592 "dme-peer-set"
3593 };
3594 const char *set = action[!!peer];
3595 int ret;
64238fbd 3596 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3597
3598 uic_cmd.command = peer ?
3599 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3600 uic_cmd.argument1 = attr_sel;
3601 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3602 uic_cmd.argument3 = mib_val;
3603
64238fbd
YG		 3604 do {
3605 /* for peer attributes we retry upon failure */
3606 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3607 if (ret)
3608 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3609 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3610 } while (ret && peer && --retries);
3611
f37e9f8c 3612 if (ret)
64238fbd 3613 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3614 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3615 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3616
3617 return ret;
3618}
3619EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3620
3621/**
3622 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3623 * @hba: per adapter instance
3624 * @attr_sel: uic command argument1
3625 * @mib_val: the value of the attribute as returned by the UIC command
3626 * @peer: indicate whether peer or local
3627 *
3628 * Returns 0 on success, non-zero value on failure
3629 */
3630int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3631 u32 *mib_val, u8 peer)
3632{
3633 struct uic_command uic_cmd = {0};
3634 static const char *const action[] = {
3635 "dme-get",
3636 "dme-peer-get"
3637 };
3638 const char *get = action[!!peer];
3639 int ret;
64238fbd 3640 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3641 struct ufs_pa_layer_attr orig_pwr_info;
3642 struct ufs_pa_layer_attr temp_pwr_info;
3643 bool pwr_mode_change = false;
3644
3645 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3646 orig_pwr_info = hba->pwr_info;
3647 temp_pwr_info = orig_pwr_info;
3648
3649 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3650 orig_pwr_info.pwr_rx == FAST_MODE) {
3651 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3652 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3653 pwr_mode_change = true;
3654 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3655 orig_pwr_info.pwr_rx == SLOW_MODE) {
3656 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3657 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3658 pwr_mode_change = true;
3659 }
3660 if (pwr_mode_change) {
3661 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3662 if (ret)
3663 goto out;
3664 }
3665 }
12b4fdb4
SJ		 3666
3667 uic_cmd.command = peer ?
3668 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3669 uic_cmd.argument1 = attr_sel;
3670
64238fbd
YG		 3671 do {
3672 /* for peer attributes we retry upon failure */
3673 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3674 if (ret)
3675 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3676 get, UIC_GET_ATTR_ID(attr_sel), ret);
3677 } while (ret && peer && --retries);
3678
f37e9f8c 3679 if (ret)
64238fbd 3680 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3681 get, UIC_GET_ATTR_ID(attr_sel),
3682 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3683
64238fbd 3684 if (mib_val && !ret)
12b4fdb4 3685 *mib_val = uic_cmd.argument3;
874237f7
YG		 3686
3687 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3688 && pwr_mode_change)
3689 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3690out:
3691 return ret;
3692}
3693EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3694
53b3d9c3 3695/**
57d104c1
SJ		 3696 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3697 * state) and waits for it to take effect.
3698 *
53b3d9c3 3699 * @hba: per adapter instance
57d104c1
SJ		 3700 * @cmd: UIC command to execute
3701 *
3702 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3703 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3704 * and device UniPro link and hence it's final completion would be indicated by
3705 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3706 * addition to normal UIC command completion Status (UCCS). This function only
3707 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3708 *
3709 * Returns 0 on success, non-zero value on failure
3710 */
57d104c1 3711static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3712{
57d104c1 3713 struct completion uic_async_done;
53b3d9c3
SJ		 3714 unsigned long flags;
3715 u8 status;
3716 int ret;
d75f7fe4 3717 bool reenable_intr = false;
53b3d9c3 3718
53b3d9c3 3719 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3720 init_completion(&uic_async_done);
cad2e03d 3721 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3722
3723 spin_lock_irqsave(hba->host->host_lock, flags);
57d104c1 3724 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3725 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3726 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3727 /*
3728 * Make sure UIC command completion interrupt is disabled before
3729 * issuing UIC command.
3730 */
3731 wmb();
3732 reenable_intr = true;
57d104c1 3733 }
d75f7fe4
YG		 3734 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3735 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3736 if (ret) {
3737 dev_err(hba->dev,
3738 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3739 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3740 goto out;
3741 }
3742
57d104c1 3743 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3744 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3745 dev_err(hba->dev,
57d104c1
SJ		 3746 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3747 cmd->command, cmd->argument3);
53b3d9c3
SJ		 3748 ret = -ETIMEDOUT;
3749 goto out;
3750 }
3751
3752 status = ufshcd_get_upmcrs(hba);
3753 if (status != PWR_LOCAL) {
3754 dev_err(hba->dev,
479da360 3755 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3756 cmd->command, status);
53b3d9c3
SJ		 3757 ret = (status != PWR_OK) ? status : -1;
3758 }
3759out:
7942f7b5
VG		 3760 if (ret) {
3761 ufshcd_print_host_state(hba);
3762 ufshcd_print_pwr_info(hba);
3763 ufshcd_print_host_regs(hba);
3764 }
3765
53b3d9c3 3766 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3767 hba->active_uic_cmd = NULL;
57d104c1 3768 hba->uic_async_done = NULL;
d75f7fe4
YG		 3769 if (reenable_intr)
3770 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
53b3d9c3
SJ		 3771 spin_unlock_irqrestore(hba->host->host_lock, flags);
3772 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3773
53b3d9c3
SJ		 3774 return ret;
3775}
3776
57d104c1
SJ		 3777/**
3778 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3779 * using DME_SET primitives.
3780 * @hba: per adapter instance
3781 * @mode: powr mode value
3782 *
3783 * Returns 0 on success, non-zero value on failure
3784 */
3785static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3786{
3787 struct uic_command uic_cmd = {0};
1ab27c9c 3788 int ret;
57d104c1 3789
c3a2f9ee
YG		 3790 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3791 ret = ufshcd_dme_set(hba,
3792 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3793 if (ret) {
3794 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3795 __func__, ret);
3796 goto out;
3797 }
3798 }
3799
57d104c1
SJ		 3800 uic_cmd.command = UIC_CMD_DME_SET;
3801 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3802 uic_cmd.argument3 = mode;
1ab27c9c
ST		 3803 ufshcd_hold(hba, false);
3804 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3805 ufshcd_release(hba);
57d104c1 3806
c3a2f9ee 3807out:
1ab27c9c 3808 return ret;
57d104c1
SJ		 3809}
3810
53c12d0e
YG		 3811static int ufshcd_link_recovery(struct ufs_hba *hba)
3812{
3813 int ret;
3814 unsigned long flags;
3815
3816 spin_lock_irqsave(hba->host->host_lock, flags);
3817 hba->ufshcd_state = UFSHCD_STATE_RESET;
3818 ufshcd_set_eh_in_progress(hba);
3819 spin_unlock_irqrestore(hba->host->host_lock, flags);
3820
3821 ret = ufshcd_host_reset_and_restore(hba);
3822
3823 spin_lock_irqsave(hba->host->host_lock, flags);
3824 if (ret)
3825 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3826 ufshcd_clear_eh_in_progress(hba);
3827 spin_unlock_irqrestore(hba->host->host_lock, flags);
3828
3829 if (ret)
3830 dev_err(hba->dev, "%s: link recovery failed, err %d",
3831 __func__, ret);
3832
3833 return ret;
3834}
3835
87d0b4a6 3836static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 3837{
87d0b4a6 3838 int ret;
57d104c1 3839 struct uic_command uic_cmd = {0};
911a0771 3840 ktime_t start = ktime_get();
57d104c1 3841
ee32c909
KK		 3842 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3843
57d104c1 3844 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 3845 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3846 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3847 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 3848
53c12d0e 3849 if (ret) {
87d0b4a6
YG		 3850 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3851 __func__, ret);
3852
53c12d0e
YG		 3853 /*
3854 * If link recovery fails then return error so that caller
3855 * don't retry the hibern8 enter again.
3856 */
3857 if (ufshcd_link_recovery(hba))
3858 ret = -ENOLINK;
ee32c909
KK		 3859 } else
3860 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3861 POST_CHANGE);
53c12d0e 3862
87d0b4a6
YG		 3863 return ret;
3864}
3865
3866static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3867{
3868 int ret = 0, retries;
57d104c1 3869
87d0b4a6
YG		 3870 for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
3871 ret = __ufshcd_uic_hibern8_enter(hba);
3872 if (!ret || ret == -ENOLINK)
3873 goto out;
3874 }
3875out:
3876 return ret;
57d104c1
SJ		 3877}
3878
3879static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
3880{
3881 struct uic_command uic_cmd = {0};
3882 int ret;
911a0771 3883 ktime_t start = ktime_get();
57d104c1 3884
ee32c909
KK		 3885 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3886
57d104c1
SJ		 3887 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3888 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771 3889 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3890 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3891
57d104c1 3892 if (ret) {
53c12d0e
YG		 3893 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3894 __func__, ret);
3895 ret = ufshcd_link_recovery(hba);
ff8e20c6 3896 } else {
ee32c909
KK		 3897 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3898 POST_CHANGE);
ff8e20c6
DR		 3899 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3900 hba->ufs_stats.hibern8_exit_cnt++;
3901 }
57d104c1
SJ		 3902
3903 return ret;
3904}
3905
ad448378
AH		 3906static void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
3907{
3908 unsigned long flags;
3909
3910 if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) || !hba->ahit)
3911 return;
3912
3913 spin_lock_irqsave(hba->host->host_lock, flags);
3914 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3915 spin_unlock_irqrestore(hba->host->host_lock, flags);
3916}
3917
5064636c
YG		 3918 /**
3919 * ufshcd_init_pwr_info - setting the POR (power on reset)
3920 * values in hba power info
3921 * @hba: per-adapter instance
3922 */
3923static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3924{
3925 hba->pwr_info.gear_rx = UFS_PWM_G1;
3926 hba->pwr_info.gear_tx = UFS_PWM_G1;
3927 hba->pwr_info.lane_rx = 1;
3928 hba->pwr_info.lane_tx = 1;
3929 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
3930 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
3931 hba->pwr_info.hs_rate = 0;
3932}
3933
d3e89bac 3934/**
7eb584db
DR		 3935 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3936 * @hba: per-adapter instance
d3e89bac 3937 */
7eb584db 3938static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 3939{
7eb584db
DR		 3940 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
3941
3942 if (hba->max_pwr_info.is_valid)
3943 return 0;
3944
2349b533 3945 pwr_info->pwr_tx = FAST_MODE;
3946 pwr_info->pwr_rx = FAST_MODE;
7eb584db 3947 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 3948
3949 /* Get the connected lane count */
7eb584db
DR		 3950 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
3951 &pwr_info->lane_rx);
3952 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3953 &pwr_info->lane_tx);
3954
3955 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
3956 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
3957 __func__,
3958 pwr_info->lane_rx,
3959 pwr_info->lane_tx);
3960 return -EINVAL;
3961 }
d3e89bac
SJ		 3962
3963 /*
3964 * First, get the maximum gears of HS speed.
3965 * If a zero value, it means there is no HSGEAR capability.
3966 * Then, get the maximum gears of PWM speed.
3967 */
7eb584db
DR		 3968 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
3969 if (!pwr_info->gear_rx) {
3970 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
3971 &pwr_info->gear_rx);
3972 if (!pwr_info->gear_rx) {
3973 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
3974 __func__, pwr_info->gear_rx);
3975 return -EINVAL;
3976 }
2349b533 3977 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 3978 }
3979
7eb584db
DR		 3980 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
3981 &pwr_info->gear_tx);
3982 if (!pwr_info->gear_tx) {
d3e89bac 3983 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 3984 &pwr_info->gear_tx);
3985 if (!pwr_info->gear_tx) {
3986 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
3987 __func__, pwr_info->gear_tx);
3988 return -EINVAL;
3989 }
2349b533 3990 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 3991 }
3992
3993 hba->max_pwr_info.is_valid = true;
3994 return 0;
3995}
3996
3997static int ufshcd_change_power_mode(struct ufs_hba *hba,
3998 struct ufs_pa_layer_attr *pwr_mode)
3999{
4000 int ret;
4001
4002 /* if already configured to the requested pwr_mode */
4003 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4004 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4005 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4006 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4007 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4008 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4009 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4010 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4011 return 0;
d3e89bac
SJ		 4012 }
4013
4014 /*
4015 * Configure attributes for power mode change with below.
4016 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4017 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4018 * - PA_HSSERIES
4019 */
7eb584db
DR		 4020 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4021 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4022 pwr_mode->lane_rx);
4023 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4024 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4025 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4026 else
4027 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4028
7eb584db
DR		 4029 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4030 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4031 pwr_mode->lane_tx);
4032 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4033 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4034 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4035 else
4036 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4037
7eb584db
DR		 4038 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4039 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4040 pwr_mode->pwr_rx == FAST_MODE ||
4041 pwr_mode->pwr_tx == FAST_MODE)
4042 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4043 pwr_mode->hs_rate);
d3e89bac 4044
7eb584db
DR		 4045 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4046 | pwr_mode->pwr_tx);
4047
4048 if (ret) {
d3e89bac 4049 dev_err(hba->dev,
7eb584db
DR		 4050 "%s: power mode change failed %d\n", __func__, ret);
4051 } else {
0263bcd0
YG		 4052 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4053 pwr_mode);
7eb584db
DR		 4054
4055 memcpy(&hba->pwr_info, pwr_mode,
4056 sizeof(struct ufs_pa_layer_attr));
4057 }
4058
4059 return ret;
4060}
4061
4062/**
4063 * ufshcd_config_pwr_mode - configure a new power mode
4064 * @hba: per-adapter instance
4065 * @desired_pwr_mode: desired power configuration
4066 */
0d846e70 4067int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4068 struct ufs_pa_layer_attr *desired_pwr_mode)
4069{
4070 struct ufs_pa_layer_attr final_params = { 0 };
4071 int ret;
4072
0263bcd0
YG		 4073 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4074 desired_pwr_mode, &final_params);
4075
4076 if (ret)
7eb584db
DR		 4077 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4078
4079 ret = ufshcd_change_power_mode(hba, &final_params);
a3cd5ec5 4080 if (!ret)
4081 ufshcd_print_pwr_info(hba);
d3e89bac
SJ		 4082
4083 return ret;
4084}
0d846e70 4085EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4086
68078d5c
DR		 4087/**
4088 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4089 * @hba: per-adapter instance
68078d5c
DR		 4090 *
4091 * Set fDeviceInit flag and poll until device toggles it.
4092 */
4093static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4094{
dc3c8d3a
YG		 4095 int i;
4096 int err;
68078d5c
DR		 4097 bool flag_res = 1;
4098
dc3c8d3a
YG		 4099 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4100 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
68078d5c
DR		 4101 if (err) {
4102 dev_err(hba->dev,
4103 "%s setting fDeviceInit flag failed with error %d\n",
4104 __func__, err);
4105 goto out;
4106 }
4107
dc3c8d3a
YG		 4108 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4109 for (i = 0; i < 1000 && !err && flag_res; i++)
4110 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4111 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
4112
68078d5c
DR		 4113 if (err)
4114 dev_err(hba->dev,
4115 "%s reading fDeviceInit flag failed with error %d\n",
4116 __func__, err);
4117 else if (flag_res)
4118 dev_err(hba->dev,
4119 "%s fDeviceInit was not cleared by the device\n",
4120 __func__);
4121
4122out:
4123 return err;
4124}
4125
7a3e97b0
SY		 4126/**
4127 * ufshcd_make_hba_operational - Make UFS controller operational
4128 * @hba: per adapter instance
4129 *
4130 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4131 * 1. Enable required interrupts
4132 * 2. Configure interrupt aggregation
897efe62 4133 * 3. Program UTRL and UTMRL base address
5c0c28a8 4134 * 4. Configure run-stop-registers
7a3e97b0
SY		 4135 *
4136 * Returns 0 on success, non-zero value on failure
4137 */
4138static int ufshcd_make_hba_operational(struct ufs_hba *hba)
4139{
4140 int err = 0;
4141 u32 reg;
4142
6ccf44fe
SJ		 4143 /* Enable required interrupts */
4144 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4145
4146 /* Configure interrupt aggregation */
b852190e
YG		 4147 if (ufshcd_is_intr_aggr_allowed(hba))
4148 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4149 else
4150 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4151
4152 /* Configure UTRL and UTMRL base address registers */
4153 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4154 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4155 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4156 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4157 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4158 REG_UTP_TASK_REQ_LIST_BASE_L);
4159 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4160 REG_UTP_TASK_REQ_LIST_BASE_H);
4161
897efe62
YG		 4162 /*
4163 * Make sure base address and interrupt setup are updated before
4164 * enabling the run/stop registers below.
4165 */
4166 wmb();
4167
7a3e97b0
SY		 4168 /*
4169 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4170 */
5c0c28a8 4171 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4172 if (!(ufshcd_get_lists_status(reg))) {
4173 ufshcd_enable_run_stop_reg(hba);
4174 } else {
3b1d0580 4175 dev_err(hba->dev,
7a3e97b0
SY		 4176 "Host controller not ready to process requests");
4177 err = -EIO;
4178 goto out;
4179 }
4180
7a3e97b0
SY		 4181out:
4182 return err;
4183}
4184
596585a2
YG		 4185/**
4186 * ufshcd_hba_stop - Send controller to reset state
4187 * @hba: per adapter instance
4188 * @can_sleep: perform sleep or just spin
4189 */
4190static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
4191{
4192 int err;
4193
4194 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4195 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4196 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4197 10, 1, can_sleep);
4198 if (err)
4199 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4200}
4201
7a3e97b0 4202/**
4404c5de 4203 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4204 * @hba: per adapter instance
4205 *
4206 * The controller resets itself and controller firmware initialization
4207 * sequence kicks off. When controller is ready it will set
4208 * the Host Controller Enable bit to 1.
4209 *
4210 * Returns 0 on success, non-zero value on failure
4211 */
4404c5de 4212static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0
SY		 4213{
4214 int retry;
4215
4216 /*
4217 * msleep of 1 and 5 used in this function might result in msleep(20),
4218 * but it was necessary to send the UFS FPGA to reset mode during
4219 * development and testing of this driver. msleep can be changed to
4220 * mdelay and retry count can be reduced based on the controller.
4221 */
596585a2 4222 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4223 /* change controller state to "reset state" */
596585a2 4224 ufshcd_hba_stop(hba, true);
7a3e97b0 4225
57d104c1
SJ		 4226 /* UniPro link is disabled at this point */
4227 ufshcd_set_link_off(hba);
4228
0263bcd0 4229 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4230
7a3e97b0
SY		 4231 /* start controller initialization sequence */
4232 ufshcd_hba_start(hba);
4233
4234 /*
4235 * To initialize a UFS host controller HCE bit must be set to 1.
4236 * During initialization the HCE bit value changes from 1->0->1.
4237 * When the host controller completes initialization sequence
4238 * it sets the value of HCE bit to 1. The same HCE bit is read back
4239 * to check if the controller has completed initialization sequence.
4240 * So without this delay the value HCE = 1, set in the previous
4241 * instruction might be read back.
4242 * This delay can be changed based on the controller.
4243 */
4244 msleep(1);
4245
4246 /* wait for the host controller to complete initialization */
4247 retry = 10;
4248 while (ufshcd_is_hba_active(hba)) {
4249 if (retry) {
4250 retry--;
4251 } else {
3b1d0580 4252 dev_err(hba->dev,
7a3e97b0
SY		 4253 "Controller enable failed\n");
4254 return -EIO;
4255 }
4256 msleep(5);
4257 }
5c0c28a8 4258
1d337ec2 4259 /* enable UIC related interrupts */
57d104c1 4260 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4261
0263bcd0 4262 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4263
7a3e97b0
SY		 4264 return 0;
4265}
4266
4404c5de
AA		 4267static int ufshcd_hba_enable(struct ufs_hba *hba)
4268{
4269 int ret;
4270
4271 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4272 ufshcd_set_link_off(hba);
4273 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4274
4275 /* enable UIC related interrupts */
4276 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4277 ret = ufshcd_dme_reset(hba);
4278 if (!ret) {
4279 ret = ufshcd_dme_enable(hba);
4280 if (!ret)
4281 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4282 if (ret)
4283 dev_err(hba->dev,
4284 "Host controller enable failed with non-hce\n");
4285 }
4286 } else {
4287 ret = ufshcd_hba_execute_hce(hba);
4288 }
4289
4290 return ret;
4291}
7ca38cf3
YG		 4292static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4293{
4294 int tx_lanes, i, err = 0;
4295
4296 if (!peer)
4297 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4298 &tx_lanes);
4299 else
4300 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4301 &tx_lanes);
4302 for (i = 0; i < tx_lanes; i++) {
4303 if (!peer)
4304 err = ufshcd_dme_set(hba,
4305 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4306 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4307 0);
4308 else
4309 err = ufshcd_dme_peer_set(hba,
4310 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4311 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4312 0);
4313 if (err) {
4314 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4315 __func__, peer, i, err);
4316 break;
4317 }
4318 }
4319
4320 return err;
4321}
4322
4323static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4324{
4325 return ufshcd_disable_tx_lcc(hba, true);
4326}
4327
7a3e97b0 4328/**
6ccf44fe 4329 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4330 * @hba: per adapter instance
4331 *
6ccf44fe 4332 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4333 */
6ccf44fe 4334static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4335{
6ccf44fe 4336 int ret;
1d337ec2 4337 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4338 bool link_startup_again = false;
7a3e97b0 4339
7caf489b 4340 /*
4341 * If UFS device isn't active then we will have to issue link startup
4342 * 2 times to make sure the device state move to active.
4343 */
4344 if (!ufshcd_is_ufs_dev_active(hba))
4345 link_startup_again = true;
7a3e97b0 4346
7caf489b 4347link_startup:
1d337ec2 4348 do {
0263bcd0 4349 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4350
1d337ec2 4351 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4352
1d337ec2
SRT		 4353 /* check if device is detected by inter-connect layer */
4354 if (!ret && !ufshcd_is_device_present(hba)) {
4355 dev_err(hba->dev, "%s: Device not present\n", __func__);
4356 ret = -ENXIO;
4357 goto out;
4358 }
6ccf44fe 4359
1d337ec2
SRT		 4360 /*
4361 * DME link lost indication is only received when link is up,
4362 * but we can't be sure if the link is up until link startup
4363 * succeeds. So reset the local Uni-Pro and try again.
4364 */
4365 if (ret && ufshcd_hba_enable(hba))
4366 goto out;
4367 } while (ret && retries--);
4368
4369 if (ret)
4370 /* failed to get the link up... retire */
5c0c28a8 4371 goto out;
5c0c28a8 4372
7caf489b 4373 if (link_startup_again) {
4374 link_startup_again = false;
4375 retries = DME_LINKSTARTUP_RETRIES;
4376 goto link_startup;
4377 }
4378
d2aebb9b 4379 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4380 ufshcd_init_pwr_info(hba);
4381 ufshcd_print_pwr_info(hba);
4382
7ca38cf3
YG		 4383 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4384 ret = ufshcd_disable_device_tx_lcc(hba);
4385 if (ret)
4386 goto out;
4387 }
4388
5c0c28a8 4389 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4390 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4391 if (ret)
4392 goto out;
7a3e97b0 4393
5c0c28a8 4394 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4395out:
7942f7b5 4396 if (ret) {
6ccf44fe 4397 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4398 ufshcd_print_host_state(hba);
4399 ufshcd_print_pwr_info(hba);
4400 ufshcd_print_host_regs(hba);
4401 }
6ccf44fe 4402 return ret;
7a3e97b0
SY		 4403}
4404
5a0b0cb9
SRT		 4405/**
4406 * ufshcd_verify_dev_init() - Verify device initialization
4407 * @hba: per-adapter instance
4408 *
4409 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4410 * device Transport Protocol (UTP) layer is ready after a reset.
4411 * If the UTP layer at the device side is not initialized, it may
4412 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4413 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4414 */
4415static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4416{
4417 int err = 0;
4418 int retries;
4419
1ab27c9c 4420 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4421 mutex_lock(&hba->dev_cmd.lock);
4422 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4423 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4424 NOP_OUT_TIMEOUT);
4425
4426 if (!err || err == -ETIMEDOUT)
4427 break;
4428
4429 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4430 }
4431 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4432 ufshcd_release(hba);
5a0b0cb9
SRT		 4433
4434 if (err)
4435 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4436 return err;
4437}
4438
0ce147d4
SJ		 4439/**
4440 * ufshcd_set_queue_depth - set lun queue depth
4441 * @sdev: pointer to SCSI device
4442 *
4443 * Read bLUQueueDepth value and activate scsi tagged command
4444 * queueing. For WLUN, queue depth is set to 1. For best-effort
4445 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4446 * value that host can queue.
4447 */
4448static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4449{
4450 int ret = 0;
4451 u8 lun_qdepth;
4452 struct ufs_hba *hba;
4453
4454 hba = shost_priv(sdev->host);
4455
4456 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4457 ret = ufshcd_read_unit_desc_param(hba,
4458 ufshcd_scsi_to_upiu_lun(sdev->lun),
4459 UNIT_DESC_PARAM_LU_Q_DEPTH,
4460 &lun_qdepth,
4461 sizeof(lun_qdepth));
0ce147d4
SJ		 4462
4463 /* Some WLUN doesn't support unit descriptor */
4464 if (ret == -EOPNOTSUPP)
4465 lun_qdepth = 1;
4466 else if (!lun_qdepth)
4467 /* eventually, we can figure out the real queue depth */
4468 lun_qdepth = hba->nutrs;
4469 else
4470 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4471
4472 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4473 __func__, lun_qdepth);
db5ed4df 4474 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4475}
4476
57d104c1
SJ		 4477/*
4478 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4479 * @hba: per-adapter instance
4480 * @lun: UFS device lun id
4481 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4482 *
4483 * Returns 0 in case of success and b_lu_write_protect status would be returned
4484 * @b_lu_write_protect parameter.
4485 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4486 * Returns -EINVAL in case of invalid parameters passed to this function.
4487 */
4488static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4489 u8 lun,
4490 u8 *b_lu_write_protect)
4491{
4492 int ret;
4493
4494 if (!b_lu_write_protect)
4495 ret = -EINVAL;
4496 /*
4497 * According to UFS device spec, RPMB LU can't be write
4498 * protected so skip reading bLUWriteProtect parameter for
4499 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4500 */
4501 else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
4502 ret = -ENOTSUPP;
4503 else
4504 ret = ufshcd_read_unit_desc_param(hba,
4505 lun,
4506 UNIT_DESC_PARAM_LU_WR_PROTECT,
4507 b_lu_write_protect,
4508 sizeof(*b_lu_write_protect));
4509 return ret;
4510}
4511
4512/**
4513 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4514 * status
4515 * @hba: per-adapter instance
4516 * @sdev: pointer to SCSI device
4517 *
4518 */
4519static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4520 struct scsi_device *sdev)
4521{
4522 if (hba->dev_info.f_power_on_wp_en &&
4523 !hba->dev_info.is_lu_power_on_wp) {
4524 u8 b_lu_write_protect;
4525
4526 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4527 &b_lu_write_protect) &&
4528 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4529 hba->dev_info.is_lu_power_on_wp = true;
4530 }
4531}
4532
7a3e97b0
SY		 4533/**
4534 * ufshcd_slave_alloc - handle initial SCSI device configurations
4535 * @sdev: pointer to SCSI device
4536 *
4537 * Returns success
4538 */
4539static int ufshcd_slave_alloc(struct scsi_device *sdev)
4540{
4541 struct ufs_hba *hba;
4542
4543 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4544
4545 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4546 sdev->use_10_for_ms = 1;
7a3e97b0 4547
e8e7f271
SRT		 4548 /* allow SCSI layer to restart the device in case of errors */
4549 sdev->allow_restart = 1;
4264fd61 4550
b2a6c522
SRT		 4551 /* REPORT SUPPORTED OPERATION CODES is not supported */
4552 sdev->no_report_opcodes = 1;
4553
84af7e8b
SRT		 4554 /* WRITE_SAME command is not supported */
4555 sdev->no_write_same = 1;
e8e7f271 4556
0ce147d4 4557 ufshcd_set_queue_depth(sdev);
4264fd61 4558
57d104c1
SJ		 4559 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4560
7a3e97b0
SY		 4561 return 0;
4562}
4563
4264fd61
SRT		 4564/**
4565 * ufshcd_change_queue_depth - change queue depth
4566 * @sdev: pointer to SCSI device
4567 * @depth: required depth to set
4264fd61 4568 *
db5ed4df 4569 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4570 */
db5ed4df 4571static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4572{
4573 struct ufs_hba *hba = shost_priv(sdev->host);
4574
4575 if (depth > hba->nutrs)
4576 depth = hba->nutrs;
db5ed4df 4577 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4578}
4579
eeda4749
AM		 4580/**
4581 * ufshcd_slave_configure - adjust SCSI device configurations
4582 * @sdev: pointer to SCSI device
4583 */
4584static int ufshcd_slave_configure(struct scsi_device *sdev)
4585{
4586 struct request_queue *q = sdev->request_queue;
4587
4588 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4589 blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX);
4590
4591 return 0;
4592}
4593
7a3e97b0
SY		 4594/**
4595 * ufshcd_slave_destroy - remove SCSI device configurations
4596 * @sdev: pointer to SCSI device
4597 */
4598static void ufshcd_slave_destroy(struct scsi_device *sdev)
4599{
4600 struct ufs_hba *hba;
4601
4602 hba = shost_priv(sdev->host);
0ce147d4 4603 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4604 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4605 unsigned long flags;
4606
4607 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4608 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4609 spin_unlock_irqrestore(hba->host->host_lock, flags);
4610 }
7a3e97b0
SY		 4611}
4612
7a3e97b0
SY		 4613/**
4614 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4615 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4616 * @scsi_status: SCSI command status
4617 *
4618 * Returns value base on SCSI command status
4619 */
4620static inline int
4621ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4622{
4623 int result = 0;
4624
4625 switch (scsi_status) {
7a3e97b0 4626 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4627 ufshcd_copy_sense_data(lrbp);
30eb2e4c 4628 /* fallthrough */
1c2623c5 4629 case SAM_STAT_GOOD:
7a3e97b0
SY		 4630 result |= DID_OK << 16 |
4631 COMMAND_COMPLETE << 8 |
1c2623c5 4632 scsi_status;
7a3e97b0
SY		 4633 break;
4634 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4635 case SAM_STAT_BUSY:
7a3e97b0 4636 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4637 ufshcd_copy_sense_data(lrbp);
4638 result |= scsi_status;
7a3e97b0
SY		 4639 break;
4640 default:
4641 result |= DID_ERROR << 16;
4642 break;
4643 } /* end of switch */
4644
4645 return result;
4646}
4647
4648/**
4649 * ufshcd_transfer_rsp_status - Get overall status of the response
4650 * @hba: per adapter instance
8aa29f19 4651 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4652 *
4653 * Returns result of the command to notify SCSI midlayer
4654 */
4655static inline int
4656ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4657{
4658 int result = 0;
4659 int scsi_status;
4660 int ocs;
4661
4662 /* overall command status of utrd */
4663 ocs = ufshcd_get_tr_ocs(lrbp);
4664
4665 switch (ocs) {
4666 case OCS_SUCCESS:
5a0b0cb9 4667 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4668 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4669 switch (result) {
4670 case UPIU_TRANSACTION_RESPONSE:
4671 /*
4672 * get the response UPIU result to extract
4673 * the SCSI command status
4674 */
4675 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4676
4677 /*
4678 * get the result based on SCSI status response
4679 * to notify the SCSI midlayer of the command status
4680 */
4681 scsi_status = result & MASK_SCSI_STATUS;
4682 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4683
f05ac2e5
YG		 4684 /*
4685 * Currently we are only supporting BKOPs exception
4686 * events hence we can ignore BKOPs exception event
4687 * during power management callbacks. BKOPs exception
4688 * event is not expected to be raised in runtime suspend
4689 * callback as it allows the urgent bkops.
4690 * During system suspend, we are anyway forcefully
4691 * disabling the bkops and if urgent bkops is needed
4692 * it will be enabled on system resume. Long term
4693 * solution could be to abort the system suspend if
4694 * UFS device needs urgent BKOPs.
4695 */
4696 if (!hba->pm_op_in_progress &&
4697 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
66ec6d59 4698 schedule_work(&hba->eeh_work);
5a0b0cb9
SRT		 4699 break;
4700 case UPIU_TRANSACTION_REJECT_UPIU:
4701 /* TODO: handle Reject UPIU Response */
4702 result = DID_ERROR << 16;
3b1d0580 4703 dev_err(hba->dev,
5a0b0cb9
SRT		 4704 "Reject UPIU not fully implemented\n");
4705 break;
4706 default:
5a0b0cb9
SRT		 4707 dev_err(hba->dev,
4708 "Unexpected request response code = %x\n",
4709 result);
e0347d89 4710 result = DID_ERROR << 16;
7a3e97b0
SY		 4711 break;
4712 }
7a3e97b0
SY		 4713 break;
4714 case OCS_ABORTED:
4715 result |= DID_ABORT << 16;
4716 break;
e8e7f271
SRT		 4717 case OCS_INVALID_COMMAND_STATUS:
4718 result |= DID_REQUEUE << 16;
4719 break;
7a3e97b0
SY		 4720 case OCS_INVALID_CMD_TABLE_ATTR:
4721 case OCS_INVALID_PRDT_ATTR:
4722 case OCS_MISMATCH_DATA_BUF_SIZE:
4723 case OCS_MISMATCH_RESP_UPIU_SIZE:
4724 case OCS_PEER_COMM_FAILURE:
4725 case OCS_FATAL_ERROR:
4726 default:
4727 result |= DID_ERROR << 16;
3b1d0580 4728 dev_err(hba->dev,
ff8e20c6
DR		 4729 "OCS error from controller = %x for tag %d\n",
4730 ocs, lrbp->task_tag);
4731 ufshcd_print_host_regs(hba);
6ba65588 4732 ufshcd_print_host_state(hba);
7a3e97b0
SY		 4733 break;
4734 } /* end of switch */
4735
66cc820f
DR		 4736 if (host_byte(result) != DID_OK)
4737 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 4738 return result;
4739}
4740
6ccf44fe
SJ		 4741/**
4742 * ufshcd_uic_cmd_compl - handle completion of uic command
4743 * @hba: per adapter instance
53b3d9c3 4744 * @intr_status: interrupt status generated by the controller
6ccf44fe 4745 */
53b3d9c3 4746static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 4747{
53b3d9c3 4748 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 4749 hba->active_uic_cmd->argument2 |=
4750 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 4751 hba->active_uic_cmd->argument3 =
4752 ufshcd_get_dme_attr_val(hba);
6ccf44fe
SJ		 4753 complete(&hba->active_uic_cmd->done);
4754 }
53b3d9c3 4755
57d104c1
SJ		 4756 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done)
4757 complete(hba->uic_async_done);
6ccf44fe
SJ		 4758}
4759
7a3e97b0 4760/**
9a47ec7c 4761 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 4762 * @hba: per adapter instance
9a47ec7c 4763 * @completed_reqs: requests to complete
7a3e97b0 4764 */
9a47ec7c
YG		 4765static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4766 unsigned long completed_reqs)
7a3e97b0 4767{
5a0b0cb9
SRT		 4768 struct ufshcd_lrb *lrbp;
4769 struct scsi_cmnd *cmd;
7a3e97b0
SY		 4770 int result;
4771 int index;
e9d501b1 4772
e9d501b1
DR		 4773 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4774 lrbp = &hba->lrb[index];
4775 cmd = lrbp->cmd;
4776 if (cmd) {
1a07f2d9 4777 ufshcd_add_command_trace(hba, index, "complete");
e9d501b1
DR		 4778 result = ufshcd_transfer_rsp_status(hba, lrbp);
4779 scsi_dma_unmap(cmd);
4780 cmd->result = result;
4781 /* Mark completed command as NULL in LRB */
4782 lrbp->cmd = NULL;
4783 clear_bit_unlock(index, &hba->lrb_in_use);
4784 /* Do not touch lrbp after scsi done */
4785 cmd->scsi_done(cmd);
1ab27c9c 4786 __ufshcd_release(hba);
300bb13f
JP		 4787 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4788 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
1a07f2d9
LS		 4789 if (hba->dev_cmd.complete) {
4790 ufshcd_add_command_trace(hba, index,
4791 "dev_complete");
e9d501b1 4792 complete(hba->dev_cmd.complete);
1a07f2d9 4793 }
e9d501b1 4794 }
401f1e44 4795 if (ufshcd_is_clkscaling_supported(hba))
4796 hba->clk_scaling.active_reqs--;
09017188
ZL		 4797
4798 lrbp->compl_time_stamp = ktime_get();
e9d501b1 4799 }
7a3e97b0
SY		 4800
4801 /* clear corresponding bits of completed commands */
4802 hba->outstanding_reqs ^= completed_reqs;
4803
856b3483
ST		 4804 ufshcd_clk_scaling_update_busy(hba);
4805
5a0b0cb9
SRT		 4806 /* we might have free'd some tags above */
4807 wake_up(&hba->dev_cmd.tag_wq);
7a3e97b0
SY		 4808}
4809
9a47ec7c
YG		 4810/**
4811 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4812 * @hba: per adapter instance
4813 */
4814static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
4815{
4816 unsigned long completed_reqs;
4817 u32 tr_doorbell;
4818
4819 /* Resetting interrupt aggregation counters first and reading the
4820 * DOOR_BELL afterward allows us to handle all the completed requests.
4821 * In order to prevent other interrupts starvation the DB is read once
4822 * after reset. The down side of this solution is the possibility of
4823 * false interrupt if device completes another request after resetting
4824 * aggregation and before reading the DB.
4825 */
5ac6abc9
AA		 4826 if (ufshcd_is_intr_aggr_allowed(hba) &&
4827 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 4828 ufshcd_reset_intr_aggr(hba);
4829
4830 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4831 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4832
4833 __ufshcd_transfer_req_compl(hba, completed_reqs);
4834}
4835
66ec6d59
SRT		 4836/**
4837 * ufshcd_disable_ee - disable exception event
4838 * @hba: per-adapter instance
4839 * @mask: exception event to disable
4840 *
4841 * Disables exception event in the device so that the EVENT_ALERT
4842 * bit is not set.
4843 *
4844 * Returns zero on success, non-zero error value on failure.
4845 */
4846static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
4847{
4848 int err = 0;
4849 u32 val;
4850
4851 if (!(hba->ee_ctrl_mask & mask))
4852 goto out;
4853
4854 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 4855 val &= MASK_EE_STATUS;
5e86ae44 4856 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4857 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4858 if (!err)
4859 hba->ee_ctrl_mask &= ~mask;
4860out:
4861 return err;
4862}
4863
4864/**
4865 * ufshcd_enable_ee - enable exception event
4866 * @hba: per-adapter instance
4867 * @mask: exception event to enable
4868 *
4869 * Enable corresponding exception event in the device to allow
4870 * device to alert host in critical scenarios.
4871 *
4872 * Returns zero on success, non-zero error value on failure.
4873 */
4874static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
4875{
4876 int err = 0;
4877 u32 val;
4878
4879 if (hba->ee_ctrl_mask & mask)
4880 goto out;
4881
4882 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 4883 val &= MASK_EE_STATUS;
5e86ae44 4884 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4885 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4886 if (!err)
4887 hba->ee_ctrl_mask |= mask;
4888out:
4889 return err;
4890}
4891
4892/**
4893 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4894 * @hba: per-adapter instance
4895 *
4896 * Allow device to manage background operations on its own. Enabling
4897 * this might lead to inconsistent latencies during normal data transfers
4898 * as the device is allowed to manage its own way of handling background
4899 * operations.
4900 *
4901 * Returns zero on success, non-zero on failure.
4902 */
4903static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
4904{
4905 int err = 0;
4906
4907 if (hba->auto_bkops_enabled)
4908 goto out;
4909
dc3c8d3a 4910 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
66ec6d59
SRT		 4911 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4912 if (err) {
4913 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
4914 __func__, err);
4915 goto out;
4916 }
4917
4918 hba->auto_bkops_enabled = true;
7ff5ab47 4919 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 4920
4921 /* No need of URGENT_BKOPS exception from the device */
4922 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4923 if (err)
4924 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
4925 __func__, err);
4926out:
4927 return err;
4928}
4929
4930/**
4931 * ufshcd_disable_auto_bkops - block device in doing background operations
4932 * @hba: per-adapter instance
4933 *
4934 * Disabling background operations improves command response latency but
4935 * has drawback of device moving into critical state where the device is
4936 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4937 * host is idle so that BKOPS are managed effectively without any negative
4938 * impacts.
4939 *
4940 * Returns zero on success, non-zero on failure.
4941 */
4942static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
4943{
4944 int err = 0;
4945
4946 if (!hba->auto_bkops_enabled)
4947 goto out;
4948
4949 /*
4950 * If host assisted BKOPs is to be enabled, make sure
4951 * urgent bkops exception is allowed.
4952 */
4953 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
4954 if (err) {
4955 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
4956 __func__, err);
4957 goto out;
4958 }
4959
dc3c8d3a 4960 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
66ec6d59
SRT		 4961 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4962 if (err) {
4963 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
4964 __func__, err);
4965 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4966 goto out;
4967 }
4968
4969 hba->auto_bkops_enabled = false;
7ff5ab47 4970 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
66ec6d59
SRT		 4971out:
4972 return err;
4973}
4974
4975/**
4e768e76 4976 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 4977 * @hba: per adapter instance
4978 *
4979 * After a device reset the device may toggle the BKOPS_EN flag
4980 * to default value. The s/w tracking variables should be updated
4e768e76 4981 * as well. This function would change the auto-bkops state based on
4982 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 4983 */
4e768e76 4984static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 4985{
4e768e76 4986 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
4987 hba->auto_bkops_enabled = false;
4988 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
4989 ufshcd_enable_auto_bkops(hba);
4990 } else {
4991 hba->auto_bkops_enabled = true;
4992 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
4993 ufshcd_disable_auto_bkops(hba);
4994 }
66ec6d59
SRT		 4995}
4996
4997static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
4998{
5e86ae44 4999 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5000 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5001}
5002
5003/**
57d104c1 5004 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5005 * @hba: per-adapter instance
57d104c1 5006 * @status: bkops_status value
66ec6d59 5007 *
57d104c1
SJ		 5008 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5009 * flag in the device to permit background operations if the device
5010 * bkops_status is greater than or equal to "status" argument passed to
5011 * this function, disable otherwise.
5012 *
5013 * Returns 0 for success, non-zero in case of failure.
5014 *
5015 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5016 * to know whether auto bkops is enabled or disabled after this function
5017 * returns control to it.
66ec6d59 5018 */
57d104c1
SJ		 5019static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5020 enum bkops_status status)
66ec6d59
SRT		 5021{
5022 int err;
57d104c1 5023 u32 curr_status = 0;
66ec6d59 5024
57d104c1 5025 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5026 if (err) {
5027 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5028 __func__, err);
5029 goto out;
57d104c1
SJ		 5030 } else if (curr_status > BKOPS_STATUS_MAX) {
5031 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5032 __func__, curr_status);
5033 err = -EINVAL;
5034 goto out;
66ec6d59
SRT		 5035 }
5036
57d104c1 5037 if (curr_status >= status)
66ec6d59 5038 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5039 else
5040 err = ufshcd_disable_auto_bkops(hba);
66ec6d59
SRT		 5041out:
5042 return err;
5043}
5044
57d104c1
SJ		 5045/**
5046 * ufshcd_urgent_bkops - handle urgent bkops exception event
5047 * @hba: per-adapter instance
5048 *
5049 * Enable fBackgroundOpsEn flag in the device to permit background
5050 * operations.
5051 *
5052 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5053 * and negative error value for any other failure.
5054 */
5055static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5056{
afdfff59 5057 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5058}
5059
66ec6d59
SRT		 5060static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5061{
5e86ae44 5062 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5063 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5064}
5065
afdfff59
YG		 5066static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5067{
5068 int err;
5069 u32 curr_status = 0;
5070
5071 if (hba->is_urgent_bkops_lvl_checked)
5072 goto enable_auto_bkops;
5073
5074 err = ufshcd_get_bkops_status(hba, &curr_status);
5075 if (err) {
5076 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5077 __func__, err);
5078 goto out;
5079 }
5080
5081 /*
5082 * We are seeing that some devices are raising the urgent bkops
5083 * exception events even when BKOPS status doesn't indicate performace
5084 * impacted or critical. Handle these device by determining their urgent
5085 * bkops status at runtime.
5086 */
5087 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5088 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5089 __func__, curr_status);
5090 /* update the current status as the urgent bkops level */
5091 hba->urgent_bkops_lvl = curr_status;
5092 hba->is_urgent_bkops_lvl_checked = true;
5093 }
5094
5095enable_auto_bkops:
5096 err = ufshcd_enable_auto_bkops(hba);
5097out:
5098 if (err < 0)
5099 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5100 __func__, err);
5101}
5102
66ec6d59
SRT		 5103/**
5104 * ufshcd_exception_event_handler - handle exceptions raised by device
5105 * @work: pointer to work data
5106 *
5107 * Read bExceptionEventStatus attribute from the device and handle the
5108 * exception event accordingly.
5109 */
5110static void ufshcd_exception_event_handler(struct work_struct *work)
5111{
5112 struct ufs_hba *hba;
5113 int err;
5114 u32 status = 0;
5115 hba = container_of(work, struct ufs_hba, eeh_work);
5116
62694735 5117 pm_runtime_get_sync(hba->dev);
2e3611e9 5118 scsi_block_requests(hba->host);
66ec6d59
SRT		 5119 err = ufshcd_get_ee_status(hba, &status);
5120 if (err) {
5121 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5122 __func__, err);
5123 goto out;
5124 }
5125
5126 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5127
5128 if (status & MASK_EE_URGENT_BKOPS)
5129 ufshcd_bkops_exception_event_handler(hba);
5130
66ec6d59 5131out:
2e3611e9 5132 scsi_unblock_requests(hba->host);
62694735 5133 pm_runtime_put_sync(hba->dev);
66ec6d59
SRT		 5134 return;
5135}
5136
9a47ec7c
YG		 5137/* Complete requests that have door-bell cleared */
5138static void ufshcd_complete_requests(struct ufs_hba *hba)
5139{
5140 ufshcd_transfer_req_compl(hba);
5141 ufshcd_tmc_handler(hba);
5142}
5143
583fa62d
YG		 5144/**
5145 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5146 * to recover from the DL NAC errors or not.
5147 * @hba: per-adapter instance
5148 *
5149 * Returns true if error handling is required, false otherwise
5150 */
5151static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5152{
5153 unsigned long flags;
5154 bool err_handling = true;
5155
5156 spin_lock_irqsave(hba->host->host_lock, flags);
5157 /*
5158 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5159 * device fatal error and/or DL NAC & REPLAY timeout errors.
5160 */
5161 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5162 goto out;
5163
5164 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5165 ((hba->saved_err & UIC_ERROR) &&
5166 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5167 goto out;
5168
5169 if ((hba->saved_err & UIC_ERROR) &&
5170 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5171 int err;
5172 /*
5173 * wait for 50ms to see if we can get any other errors or not.
5174 */
5175 spin_unlock_irqrestore(hba->host->host_lock, flags);
5176 msleep(50);
5177 spin_lock_irqsave(hba->host->host_lock, flags);
5178
5179 /*
5180 * now check if we have got any other severe errors other than
5181 * DL NAC error?
5182 */
5183 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5184 ((hba->saved_err & UIC_ERROR) &&
5185 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5186 goto out;
5187
5188 /*
5189 * As DL NAC is the only error received so far, send out NOP
5190 * command to confirm if link is still active or not.
5191 * - If we don't get any response then do error recovery.
5192 * - If we get response then clear the DL NAC error bit.
5193 */
5194
5195 spin_unlock_irqrestore(hba->host->host_lock, flags);
5196 err = ufshcd_verify_dev_init(hba);
5197 spin_lock_irqsave(hba->host->host_lock, flags);
5198
5199 if (err)
5200 goto out;
5201
5202 /* Link seems to be alive hence ignore the DL NAC errors */
5203 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5204 hba->saved_err &= ~UIC_ERROR;
5205 /* clear NAC error */
5206 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5207 if (!hba->saved_uic_err) {
5208 err_handling = false;
5209 goto out;
5210 }
5211 }
5212out:
5213 spin_unlock_irqrestore(hba->host->host_lock, flags);
5214 return err_handling;
5215}
5216
7a3e97b0 5217/**
e8e7f271
SRT		 5218 * ufshcd_err_handler - handle UFS errors that require s/w attention
5219 * @work: pointer to work structure
7a3e97b0 5220 */
e8e7f271 5221static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5222{
5223 struct ufs_hba *hba;
e8e7f271
SRT		 5224 unsigned long flags;
5225 u32 err_xfer = 0;
5226 u32 err_tm = 0;
5227 int err = 0;
5228 int tag;
9a47ec7c 5229 bool needs_reset = false;
e8e7f271
SRT		 5230
5231 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5232
62694735 5233 pm_runtime_get_sync(hba->dev);
1ab27c9c 5234 ufshcd_hold(hba, false);
e8e7f271
SRT		 5235
5236 spin_lock_irqsave(hba->host->host_lock, flags);
9a47ec7c 5237 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
e8e7f271 5238 goto out;
e8e7f271
SRT		 5239
5240 hba->ufshcd_state = UFSHCD_STATE_RESET;
5241 ufshcd_set_eh_in_progress(hba);
5242
5243 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5244 ufshcd_complete_requests(hba);
583fa62d
YG		 5245
5246 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5247 bool ret;
5248
5249 spin_unlock_irqrestore(hba->host->host_lock, flags);
5250 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5251 ret = ufshcd_quirk_dl_nac_errors(hba);
5252 spin_lock_irqsave(hba->host->host_lock, flags);
5253 if (!ret)
5254 goto skip_err_handling;
5255 }
9a47ec7c
YG		 5256 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5257 ((hba->saved_err & UIC_ERROR) &&
5258 (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
5259 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5260 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5261 needs_reset = true;
e8e7f271 5262
9a47ec7c
YG		 5263 /*
5264 * if host reset is required then skip clearing the pending
5265 * transfers forcefully because they will automatically get
5266 * cleared after link startup.
5267 */
5268 if (needs_reset)
5269 goto skip_pending_xfer_clear;
5270
5271 /* release lock as clear command might sleep */
5272 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5273 /* Clear pending transfer requests */
9a47ec7c
YG		 5274 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5275 if (ufshcd_clear_cmd(hba, tag)) {
5276 err_xfer = true;
5277 goto lock_skip_pending_xfer_clear;
5278 }
5279 }
e8e7f271
SRT		 5280
5281 /* Clear pending task management requests */
9a47ec7c
YG		 5282 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5283 if (ufshcd_clear_tm_cmd(hba, tag)) {
5284 err_tm = true;
5285 goto lock_skip_pending_xfer_clear;
5286 }
5287 }
e8e7f271 5288
9a47ec7c 5289lock_skip_pending_xfer_clear:
e8e7f271 5290 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5291
9a47ec7c
YG		 5292 /* Complete the requests that are cleared by s/w */
5293 ufshcd_complete_requests(hba);
5294
5295 if (err_xfer || err_tm)
5296 needs_reset = true;
5297
5298skip_pending_xfer_clear:
e8e7f271 5299 /* Fatal errors need reset */
9a47ec7c
YG		 5300 if (needs_reset) {
5301 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5302
5303 /*
5304 * ufshcd_reset_and_restore() does the link reinitialization
5305 * which will need atleast one empty doorbell slot to send the
5306 * device management commands (NOP and query commands).
5307 * If there is no slot empty at this moment then free up last
5308 * slot forcefully.
5309 */
5310 if (hba->outstanding_reqs == max_doorbells)
5311 __ufshcd_transfer_req_compl(hba,
5312 (1UL << (hba->nutrs - 1)));
5313
5314 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5315 err = ufshcd_reset_and_restore(hba);
9a47ec7c 5316 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271
SRT		 5317 if (err) {
5318 dev_err(hba->dev, "%s: reset and restore failed\n",
5319 __func__);
5320 hba->ufshcd_state = UFSHCD_STATE_ERROR;
5321 }
5322 /*
5323 * Inform scsi mid-layer that we did reset and allow to handle
5324 * Unit Attention properly.
5325 */
5326 scsi_report_bus_reset(hba->host, 0);
5327 hba->saved_err = 0;
5328 hba->saved_uic_err = 0;
5329 }
9a47ec7c 5330
583fa62d 5331skip_err_handling:
9a47ec7c
YG		 5332 if (!needs_reset) {
5333 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5334 if (hba->saved_err || hba->saved_uic_err)
5335 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5336 __func__, hba->saved_err, hba->saved_uic_err);
5337 }
5338
e8e7f271
SRT		 5339 ufshcd_clear_eh_in_progress(hba);
5340
5341out:
9a47ec7c 5342 spin_unlock_irqrestore(hba->host->host_lock, flags);
38135535 5343 ufshcd_scsi_unblock_requests(hba);
1ab27c9c 5344 ufshcd_release(hba);
62694735 5345 pm_runtime_put_sync(hba->dev);
7a3e97b0
SY		 5346}
5347
ff8e20c6
DR		 5348static void ufshcd_update_uic_reg_hist(struct ufs_uic_err_reg_hist *reg_hist,
5349 u32 reg)
5350{
5351 reg_hist->reg[reg_hist->pos] = reg;
5352 reg_hist->tstamp[reg_hist->pos] = ktime_get();
5353 reg_hist->pos = (reg_hist->pos + 1) % UIC_ERR_REG_HIST_LENGTH;
5354}
5355
7a3e97b0 5356/**
e8e7f271
SRT		 5357 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5358 * @hba: per-adapter instance
7a3e97b0 5359 */
e8e7f271 5360static void ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 5361{
5362 u32 reg;
5363
fb7b45f0
DR		 5364 /* PHY layer lane error */
5365 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5366 /* Ignore LINERESET indication, as this is not an error */
5367 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
ff8e20c6 5368 (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
fb7b45f0
DR		 5369 /*
5370 * To know whether this error is fatal or not, DB timeout
5371 * must be checked but this error is handled separately.
5372 */
5373 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
ff8e20c6
DR		 5374 ufshcd_update_uic_reg_hist(&hba->ufs_stats.pa_err, reg);
5375 }
fb7b45f0 5376
e8e7f271
SRT		 5377 /* PA_INIT_ERROR is fatal and needs UIC reset */
5378 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
ff8e20c6
DR		 5379 if (reg)
5380 ufshcd_update_uic_reg_hist(&hba->ufs_stats.dl_err, reg);
5381
e8e7f271
SRT		 5382 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5383 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
583fa62d
YG		 5384 else if (hba->dev_quirks &
5385 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5386 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5387 hba->uic_error |=
5388 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5389 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5390 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5391 }
e8e7f271
SRT		 5392
5393 /* UIC NL/TL/DME errors needs software retry */
5394 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
ff8e20c6
DR		 5395 if (reg) {
5396 ufshcd_update_uic_reg_hist(&hba->ufs_stats.nl_err, reg);
e8e7f271 5397 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
ff8e20c6 5398 }
e8e7f271
SRT		 5399
5400 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
ff8e20c6
DR		 5401 if (reg) {
5402 ufshcd_update_uic_reg_hist(&hba->ufs_stats.tl_err, reg);
e8e7f271 5403 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
ff8e20c6 5404 }
e8e7f271
SRT		 5405
5406 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
ff8e20c6
DR		 5407 if (reg) {
5408 ufshcd_update_uic_reg_hist(&hba->ufs_stats.dme_err, reg);
e8e7f271 5409 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
ff8e20c6 5410 }
e8e7f271
SRT		 5411
5412 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5413 __func__, hba->uic_error);
5414}
5415
5416/**
5417 * ufshcd_check_errors - Check for errors that need s/w attention
5418 * @hba: per-adapter instance
5419 */
5420static void ufshcd_check_errors(struct ufs_hba *hba)
5421{
5422 bool queue_eh_work = false;
5423
7a3e97b0 5424 if (hba->errors & INT_FATAL_ERRORS)
e8e7f271 5425 queue_eh_work = true;
7a3e97b0
SY		 5426
5427 if (hba->errors & UIC_ERROR) {
e8e7f271
SRT		 5428 hba->uic_error = 0;
5429 ufshcd_update_uic_error(hba);
5430 if (hba->uic_error)
5431 queue_eh_work = true;
7a3e97b0 5432 }
e8e7f271
SRT		 5433
5434 if (queue_eh_work) {
9a47ec7c
YG		 5435 /*
5436 * update the transfer error masks to sticky bits, let's do this
5437 * irrespective of current ufshcd_state.
5438 */
5439 hba->saved_err |= hba->errors;
5440 hba->saved_uic_err |= hba->uic_error;
5441
e8e7f271
SRT		 5442 /* handle fatal errors only when link is functional */
5443 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
5444 /* block commands from scsi mid-layer */
38135535 5445 ufshcd_scsi_block_requests(hba);
e8e7f271 5446
141f8165 5447 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
66cc820f
DR		 5448
5449 /* dump controller state before resetting */
5450 if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
5451 bool pr_prdt = !!(hba->saved_err &
5452 SYSTEM_BUS_FATAL_ERROR);
5453
5454 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
5455 __func__, hba->saved_err,
5456 hba->saved_uic_err);
5457
5458 ufshcd_print_host_regs(hba);
5459 ufshcd_print_pwr_info(hba);
5460 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5461 ufshcd_print_trs(hba, hba->outstanding_reqs,
5462 pr_prdt);
5463 }
e8e7f271
SRT		 5464 schedule_work(&hba->eh_work);
5465 }
3441da7d 5466 }
e8e7f271
SRT		 5467 /*
5468 * if (!queue_eh_work) -
5469 * Other errors are either non-fatal where host recovers
5470 * itself without s/w intervention or errors that will be
5471 * handled by the SCSI core layer.
5472 */
7a3e97b0
SY		 5473}
5474
5475/**
5476 * ufshcd_tmc_handler - handle task management function completion
5477 * @hba: per adapter instance
5478 */
5479static void ufshcd_tmc_handler(struct ufs_hba *hba)
5480{
5481 u32 tm_doorbell;
5482
b873a275 5483 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
7a3e97b0 5484 hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
e2933132 5485 wake_up(&hba->tm_wq);
7a3e97b0
SY		 5486}
5487
5488/**
5489 * ufshcd_sl_intr - Interrupt service routine
5490 * @hba: per adapter instance
5491 * @intr_status: contains interrupts generated by the controller
5492 */
5493static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
5494{
5495 hba->errors = UFSHCD_ERROR_MASK & intr_status;
5496 if (hba->errors)
e8e7f271 5497 ufshcd_check_errors(hba);
7a3e97b0 5498
53b3d9c3
SJ		 5499 if (intr_status & UFSHCD_UIC_MASK)
5500 ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 5501
5502 if (intr_status & UTP_TASK_REQ_COMPL)
5503 ufshcd_tmc_handler(hba);
5504
5505 if (intr_status & UTP_TRANSFER_REQ_COMPL)
5506 ufshcd_transfer_req_compl(hba);
5507}
5508
5509/**
5510 * ufshcd_intr - Main interrupt service routine
5511 * @irq: irq number
5512 * @__hba: pointer to adapter instance
5513 *
5514 * Returns IRQ_HANDLED - If interrupt is valid
5515 * IRQ_NONE - If invalid interrupt
5516 */
5517static irqreturn_t ufshcd_intr(int irq, void *__hba)
5518{
d75f7fe4 5519 u32 intr_status, enabled_intr_status;
7a3e97b0
SY		 5520 irqreturn_t retval = IRQ_NONE;
5521 struct ufs_hba *hba = __hba;
7f6ba4f1 5522 int retries = hba->nutrs;
7a3e97b0
SY		 5523
5524 spin_lock(hba->host->host_lock);
b873a275 5525 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
7a3e97b0 5526
7f6ba4f1
VG		 5527 /*
5528 * There could be max of hba->nutrs reqs in flight and in worst case
5529 * if the reqs get finished 1 by 1 after the interrupt status is
5530 * read, make sure we handle them by checking the interrupt status
5531 * again in a loop until we process all of the reqs before returning.
5532 */
5533 do {
5534 enabled_intr_status =
5535 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
5536 if (intr_status)
5537 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
5538 if (enabled_intr_status) {
5539 ufshcd_sl_intr(hba, enabled_intr_status);
5540 retval = IRQ_HANDLED;
5541 }
5542
5543 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5544 } while (intr_status && --retries);
d75f7fe4 5545
7a3e97b0
SY		 5546 spin_unlock(hba->host->host_lock);
5547 return retval;
5548}
5549
e2933132
SRT		 5550static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
5551{
5552 int err = 0;
5553 u32 mask = 1 << tag;
5554 unsigned long flags;
5555
5556 if (!test_bit(tag, &hba->outstanding_tasks))
5557 goto out;
5558
5559 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 5560 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 5561 spin_unlock_irqrestore(hba->host->host_lock, flags);
5562
5563 /* poll for max. 1 sec to clear door bell register by h/w */
5564 err = ufshcd_wait_for_register(hba,
5565 REG_UTP_TASK_REQ_DOOR_BELL,
596585a2 5566 mask, 0, 1000, 1000, true);
e2933132
SRT		 5567out:
5568 return err;
5569}
5570
c6049cd9
CH		 5571static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
5572 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 5573{
c6049cd9 5574 struct Scsi_Host *host = hba->host;
7a3e97b0 5575 unsigned long flags;
c6049cd9 5576 int free_slot, task_tag, err;
7a3e97b0 5577
e2933132
SRT		 5578 /*
5579 * Get free slot, sleep if slots are unavailable.
5580 * Even though we use wait_event() which sleeps indefinitely,
5581 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5582 */
5583 wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot));
1ab27c9c 5584 ufshcd_hold(hba, false);
7a3e97b0 5585
e2933132 5586 spin_lock_irqsave(host->host_lock, flags);
e2933132 5587 task_tag = hba->nutrs + free_slot;
7a3e97b0 5588
c6049cd9
CH		 5589 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
5590
5591 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 5592 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
5593
7a3e97b0
SY		 5594 /* send command to the controller */
5595 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 5596
5597 /* Make sure descriptors are ready before ringing the task doorbell */
5598 wmb();
5599
b873a275 5600 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 5601 /* Make sure that doorbell is committed immediately */
5602 wmb();
7a3e97b0
SY		 5603
5604 spin_unlock_irqrestore(host->host_lock, flags);
5605
6667e6d9
OS		 5606 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
5607
7a3e97b0 5608 /* wait until the task management command is completed */
e2933132
SRT		 5609 err = wait_event_timeout(hba->tm_wq,
5610 test_bit(free_slot, &hba->tm_condition),
5611 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 5612 if (!err) {
6667e6d9 5613 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
e2933132
SRT		 5614 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
5615 __func__, tm_function);
5616 if (ufshcd_clear_tm_cmd(hba, free_slot))
5617 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
5618 __func__, free_slot);
5619 err = -ETIMEDOUT;
5620 } else {
c6049cd9
CH		 5621 err = 0;
5622 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
5623
6667e6d9 5624 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
c6049cd9
CH		 5625
5626 spin_lock_irqsave(hba->host->host_lock, flags);
5627 __clear_bit(free_slot, &hba->outstanding_tasks);
5628 spin_unlock_irqrestore(hba->host->host_lock, flags);
5629
7a3e97b0 5630 }
e2933132 5631
7a3e97b0 5632 clear_bit(free_slot, &hba->tm_condition);
e2933132
SRT		 5633 ufshcd_put_tm_slot(hba, free_slot);
5634 wake_up(&hba->tm_tag_wq);
5635
1ab27c9c 5636 ufshcd_release(hba);
7a3e97b0
SY		 5637 return err;
5638}
5639
c6049cd9
CH		 5640/**
5641 * ufshcd_issue_tm_cmd - issues task management commands to controller
5642 * @hba: per adapter instance
5643 * @lun_id: LUN ID to which TM command is sent
5644 * @task_id: task ID to which the TM command is applicable
5645 * @tm_function: task management function opcode
5646 * @tm_response: task management service response return value
5647 *
5648 * Returns non-zero value on error, zero on success.
5649 */
5650static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
5651 u8 tm_function, u8 *tm_response)
5652{
5653 struct utp_task_req_desc treq = { { 0 }, };
5654 int ocs_value, err;
5655
5656 /* Configure task request descriptor */
5657 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5658 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5659
5660 /* Configure task request UPIU */
5661 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
5662 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
5663 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
5664
5665 /*
5666 * The host shall provide the same value for LUN field in the basic
5667 * header and for Input Parameter.
5668 */
5669 treq.input_param1 = cpu_to_be32(lun_id);
5670 treq.input_param2 = cpu_to_be32(task_id);
5671
5672 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
5673 if (err == -ETIMEDOUT)
5674 return err;
5675
5676 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5677 if (ocs_value != OCS_SUCCESS)
5678 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
5679 __func__, ocs_value);
5680 else if (tm_response)
5681 *tm_response = be32_to_cpu(treq.output_param1) &
5682 MASK_TM_SERVICE_RESP;
5683 return err;
5684}
5685
5e0a86ee
AA		 5686/**
5687 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5688 * @hba: per-adapter instance
5689 * @req_upiu: upiu request
5690 * @rsp_upiu: upiu reply
5691 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5692 * @desc_buff: pointer to descriptor buffer, NULL if NA
5693 * @buff_len: descriptor size, 0 if NA
5694 * @desc_op: descriptor operation
5695 *
5696 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5697 * Therefore, it "rides" the device management infrastructure: uses its tag and
5698 * tasks work queues.
5699 *
5700 * Since there is only one available tag for device management commands,
5701 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5702 */
5703static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
5704 struct utp_upiu_req *req_upiu,
5705 struct utp_upiu_req *rsp_upiu,
5706 u8 *desc_buff, int *buff_len,
5707 int cmd_type,
5708 enum query_opcode desc_op)
5709{
5710 struct ufshcd_lrb *lrbp;
5711 int err = 0;
5712 int tag;
5713 struct completion wait;
5714 unsigned long flags;
5715 u32 upiu_flags;
5716
5717 down_read(&hba->clk_scaling_lock);
5718
5719 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
5720
5721 init_completion(&wait);
5722 lrbp = &hba->lrb[tag];
5723 WARN_ON(lrbp->cmd);
5724
5725 lrbp->cmd = NULL;
5726 lrbp->sense_bufflen = 0;
5727 lrbp->sense_buffer = NULL;
5728 lrbp->task_tag = tag;
5729 lrbp->lun = 0;
5730 lrbp->intr_cmd = true;
5731 hba->dev_cmd.type = cmd_type;
5732
5733 switch (hba->ufs_version) {
5734 case UFSHCI_VERSION_10:
5735 case UFSHCI_VERSION_11:
5736 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
5737 break;
5738 default:
5739 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
5740 break;
5741 }
5742
5743 /* update the task tag in the request upiu */
5744 req_upiu->header.dword_0 |= cpu_to_be32(tag);
5745
5746 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
5747
5748 /* just copy the upiu request as it is */
5749 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
5750 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
5751 /* The Data Segment Area is optional depending upon the query
5752 * function value. for WRITE DESCRIPTOR, the data segment
5753 * follows right after the tsf.
5754 */
5755 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
5756 *buff_len = 0;
5757 }
5758
5759 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5760
5761 hba->dev_cmd.complete = &wait;
5762
5763 /* Make sure descriptors are ready before ringing the doorbell */
5764 wmb();
5765 spin_lock_irqsave(hba->host->host_lock, flags);
5766 ufshcd_send_command(hba, tag);
5767 spin_unlock_irqrestore(hba->host->host_lock, flags);
5768
5769 /*
5770 * ignore the returning value here - ufshcd_check_query_response is
5771 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5772 * read the response directly ignoring all errors.
5773 */
5774 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
5775
5776 /* just copy the upiu response as it is */
5777 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 5778 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
5779 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
5780 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
5781 MASK_QUERY_DATA_SEG_LEN;
5782
5783 if (*buff_len >= resp_len) {
5784 memcpy(desc_buff, descp, resp_len);
5785 *buff_len = resp_len;
5786 } else {
5787 dev_warn(hba->dev, "rsp size is bigger than buffer");
5788 *buff_len = 0;
5789 err = -EINVAL;
5790 }
5791 }
5e0a86ee
AA		 5792
5793 ufshcd_put_dev_cmd_tag(hba, tag);
5794 wake_up(&hba->dev_cmd.tag_wq);
5795 up_read(&hba->clk_scaling_lock);
5796 return err;
5797}
5798
5799/**
5800 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5801 * @hba: per-adapter instance
5802 * @req_upiu: upiu request
5803 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5804 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5805 * @desc_buff: pointer to descriptor buffer, NULL if NA
5806 * @buff_len: descriptor size, 0 if NA
5807 * @desc_op: descriptor operation
5808 *
5809 * Supports UTP Transfer requests (nop and query), and UTP Task
5810 * Management requests.
5811 * It is up to the caller to fill the upiu conent properly, as it will
5812 * be copied without any further input validations.
5813 */
5814int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
5815 struct utp_upiu_req *req_upiu,
5816 struct utp_upiu_req *rsp_upiu,
5817 int msgcode,
5818 u8 *desc_buff, int *buff_len,
5819 enum query_opcode desc_op)
5820{
5821 int err;
5822 int cmd_type = DEV_CMD_TYPE_QUERY;
5823 struct utp_task_req_desc treq = { { 0 }, };
5824 int ocs_value;
5825 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
5826
5e0a86ee
AA		 5827 switch (msgcode) {
5828 case UPIU_TRANSACTION_NOP_OUT:
5829 cmd_type = DEV_CMD_TYPE_NOP;
5830 /* fall through */
5831 case UPIU_TRANSACTION_QUERY_REQ:
5832 ufshcd_hold(hba, false);
5833 mutex_lock(&hba->dev_cmd.lock);
5834 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
5835 desc_buff, buff_len,
5836 cmd_type, desc_op);
5837 mutex_unlock(&hba->dev_cmd.lock);
5838 ufshcd_release(hba);
5839
5840 break;
5841 case UPIU_TRANSACTION_TASK_REQ:
5842 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5843 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5844
5845 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
5846
5847 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
5848 if (err == -ETIMEDOUT)
5849 break;
5850
5851 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5852 if (ocs_value != OCS_SUCCESS) {
5853 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
5854 ocs_value);
5855 break;
5856 }
5857
5858 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
5859
5860 break;
5861 default:
5862 err = -EINVAL;
5863
5864 break;
5865 }
5866
5e0a86ee
AA		 5867 return err;
5868}
5869
7a3e97b0 5870/**
3441da7d
SRT		 5871 * ufshcd_eh_device_reset_handler - device reset handler registered to
5872 * scsi layer.
7a3e97b0
SY		 5873 * @cmd: SCSI command pointer
5874 *
5875 * Returns SUCCESS/FAILED
5876 */
3441da7d 5877static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 5878{
5879 struct Scsi_Host *host;
5880 struct ufs_hba *hba;
5881 unsigned int tag;
5882 u32 pos;
5883 int err;
e2933132
SRT		 5884 u8 resp = 0xF;
5885 struct ufshcd_lrb *lrbp;
3441da7d 5886 unsigned long flags;
7a3e97b0
SY		 5887
5888 host = cmd->device->host;
5889 hba = shost_priv(host);
5890 tag = cmd->request->tag;
5891
e2933132
SRT		 5892 lrbp = &hba->lrb[tag];
5893 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
5894 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 5895 if (!err)
5896 err = resp;
7a3e97b0 5897 goto out;
e2933132 5898 }
7a3e97b0 5899
3441da7d
SRT		 5900 /* clear the commands that were pending for corresponding LUN */
5901 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
5902 if (hba->lrb[pos].lun == lrbp->lun) {
5903 err = ufshcd_clear_cmd(hba, pos);
5904 if (err)
5905 break;
7a3e97b0 5906 }
3441da7d
SRT		 5907 }
5908 spin_lock_irqsave(host->host_lock, flags);
5909 ufshcd_transfer_req_compl(hba);
5910 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 5911
7a3e97b0 5912out:
7fabb77b 5913 hba->req_abort_count = 0;
3441da7d
SRT		 5914 if (!err) {
5915 err = SUCCESS;
5916 } else {
5917 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
5918 err = FAILED;
5919 }
7a3e97b0
SY		 5920 return err;
5921}
5922
e0b299e3
GB		 5923static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
5924{
5925 struct ufshcd_lrb *lrbp;
5926 int tag;
5927
5928 for_each_set_bit(tag, &bitmap, hba->nutrs) {
5929 lrbp = &hba->lrb[tag];
5930 lrbp->req_abort_skip = true;
5931 }
5932}
5933
7a3e97b0
SY		 5934/**
5935 * ufshcd_abort - abort a specific command
5936 * @cmd: SCSI command pointer
5937 *
f20810d8
SRT		 5938 * Abort the pending command in device by sending UFS_ABORT_TASK task management
5939 * command, and in host controller by clearing the door-bell register. There can
5940 * be race between controller sending the command to the device while abort is
5941 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
5942 * really issued and then try to abort it.
5943 *
7a3e97b0
SY		 5944 * Returns SUCCESS/FAILED
5945 */
5946static int ufshcd_abort(struct scsi_cmnd *cmd)
5947{
5948 struct Scsi_Host *host;
5949 struct ufs_hba *hba;
5950 unsigned long flags;
5951 unsigned int tag;
f20810d8
SRT		 5952 int err = 0;
5953 int poll_cnt;
e2933132
SRT		 5954 u8 resp = 0xF;
5955 struct ufshcd_lrb *lrbp;
e9d501b1 5956 u32 reg;
7a3e97b0
SY		 5957
5958 host = cmd->device->host;
5959 hba = shost_priv(host);
5960 tag = cmd->request->tag;
e7d38257 5961 lrbp = &hba->lrb[tag];
14497328
YG		 5962 if (!ufshcd_valid_tag(hba, tag)) {
5963 dev_err(hba->dev,
5964 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
5965 __func__, tag, cmd, cmd->request);
5966 BUG();
5967 }
7a3e97b0 5968
e7d38257
DR		 5969 /*
5970 * Task abort to the device W-LUN is illegal. When this command
5971 * will fail, due to spec violation, scsi err handling next step
5972 * will be to send LU reset which, again, is a spec violation.
5973 * To avoid these unnecessary/illegal step we skip to the last error
5974 * handling stage: reset and restore.
5975 */
5976 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
5977 return ufshcd_eh_host_reset_handler(cmd);
5978
1ab27c9c 5979 ufshcd_hold(hba, false);
14497328 5980 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 5981 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 5982 if (!(test_bit(tag, &hba->outstanding_reqs))) {
5983 dev_err(hba->dev,
5984 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
5985 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 5986 goto out;
14497328 5987 }
7a3e97b0 5988
e9d501b1
DR		 5989 if (!(reg & (1 << tag))) {
5990 dev_err(hba->dev,
5991 "%s: cmd was completed, but without a notifying intr, tag = %d",
5992 __func__, tag);
5993 }
5994
66cc820f
DR		 5995 /* Print Transfer Request of aborted task */
5996 dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 5997
7fabb77b
GB		 5998 /*
5999 * Print detailed info about aborted request.
6000 * As more than one request might get aborted at the same time,
6001 * print full information only for the first aborted request in order
6002 * to reduce repeated printouts. For other aborted requests only print
6003 * basic details.
6004 */
6005 scsi_print_command(hba->lrb[tag].cmd);
6006 if (!hba->req_abort_count) {
6007 ufshcd_print_host_regs(hba);
6ba65588 6008 ufshcd_print_host_state(hba);
7fabb77b
GB		 6009 ufshcd_print_pwr_info(hba);
6010 ufshcd_print_trs(hba, 1 << tag, true);
6011 } else {
6012 ufshcd_print_trs(hba, 1 << tag, false);
6013 }
6014 hba->req_abort_count++;
e0b299e3
GB		 6015
6016 /* Skip task abort in case previous aborts failed and report failure */
6017 if (lrbp->req_abort_skip) {
6018 err = -EIO;
6019 goto out;
6020 }
6021
f20810d8
SRT		 6022 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6023 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6024 UFS_QUERY_TASK, &resp);
6025 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6026 /* cmd pending in the device */
ff8e20c6
DR		 6027 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6028 __func__, tag);
f20810d8
SRT		 6029 break;
6030 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
f20810d8
SRT		 6031 /*
6032 * cmd not pending in the device, check if it is
6033 * in transition.
6034 */
ff8e20c6
DR		 6035 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6036 __func__, tag);
f20810d8
SRT		 6037 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6038 if (reg & (1 << tag)) {
6039 /* sleep for max. 200us to stabilize */
6040 usleep_range(100, 200);
6041 continue;
6042 }
6043 /* command completed already */
ff8e20c6
DR		 6044 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6045 __func__, tag);
f20810d8
SRT		 6046 goto out;
6047 } else {
ff8e20c6
DR		 6048 dev_err(hba->dev,
6049 "%s: no response from device. tag = %d, err %d\n",
6050 __func__, tag, err);
f20810d8
SRT		 6051 if (!err)
6052 err = resp; /* service response error */
6053 goto out;
6054 }
6055 }
6056
6057 if (!poll_cnt) {
6058 err = -EBUSY;
7a3e97b0
SY		 6059 goto out;
6060 }
7a3e97b0 6061
e2933132
SRT		 6062 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6063 UFS_ABORT_TASK, &resp);
6064 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
ff8e20c6 6065 if (!err) {
f20810d8 6066 err = resp; /* service response error */
ff8e20c6
DR		 6067 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6068 __func__, tag, err);
6069 }
7a3e97b0 6070 goto out;
e2933132 6071 }
7a3e97b0 6072
f20810d8 6073 err = ufshcd_clear_cmd(hba, tag);
ff8e20c6
DR		 6074 if (err) {
6075 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6076 __func__, tag, err);
f20810d8 6077 goto out;
ff8e20c6 6078 }
f20810d8 6079
7a3e97b0
SY		 6080 scsi_dma_unmap(cmd);
6081
6082 spin_lock_irqsave(host->host_lock, flags);
a48353f6 6083 ufshcd_outstanding_req_clear(hba, tag);
7a3e97b0
SY		 6084 hba->lrb[tag].cmd = NULL;
6085 spin_unlock_irqrestore(host->host_lock, flags);
5a0b0cb9
SRT		 6086
6087 clear_bit_unlock(tag, &hba->lrb_in_use);
6088 wake_up(&hba->dev_cmd.tag_wq);
1ab27c9c 6089
7a3e97b0 6090out:
f20810d8
SRT		 6091 if (!err) {
6092 err = SUCCESS;
6093 } else {
6094 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6095 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6096 err = FAILED;
6097 }
6098
1ab27c9c
ST		 6099 /*
6100 * This ufshcd_release() corresponds to the original scsi cmd that got
6101 * aborted here (as we won't get any IRQ for it).
6102 */
6103 ufshcd_release(hba);
7a3e97b0
SY		 6104 return err;
6105}
6106
3441da7d
SRT		 6107/**
6108 * ufshcd_host_reset_and_restore - reset and restore host controller
6109 * @hba: per-adapter instance
6110 *
6111 * Note that host controller reset may issue DME_RESET to
6112 * local and remote (device) Uni-Pro stack and the attributes
6113 * are reset to default state.
6114 *
6115 * Returns zero on success, non-zero on failure
6116 */
6117static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6118{
6119 int err;
3441da7d
SRT		 6120 unsigned long flags;
6121
6122 /* Reset the host controller */
6123 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 6124 ufshcd_hba_stop(hba, false);
3441da7d
SRT		 6125 spin_unlock_irqrestore(hba->host->host_lock, flags);
6126
a3cd5ec5 6127 /* scale up clocks to max frequency before full reinitialization */
6128 ufshcd_scale_clks(hba, true);
6129
3441da7d
SRT		 6130 err = ufshcd_hba_enable(hba);
6131 if (err)
6132 goto out;
6133
6134 /* Establish the link again and restore the device */
1d337ec2
SRT		 6135 err = ufshcd_probe_hba(hba);
6136
6137 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
3441da7d
SRT		 6138 err = -EIO;
6139out:
6140 if (err)
6141 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
6142
6143 return err;
6144}
6145
6146/**
6147 * ufshcd_reset_and_restore - reset and re-initialize host/device
6148 * @hba: per-adapter instance
6149 *
6150 * Reset and recover device, host and re-establish link. This
6151 * is helpful to recover the communication in fatal error conditions.
6152 *
6153 * Returns zero on success, non-zero on failure
6154 */
6155static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6156{
6157 int err = 0;
6158 unsigned long flags;
1d337ec2 6159 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 6160
1d337ec2
SRT		 6161 do {
6162 err = ufshcd_host_reset_and_restore(hba);
6163 } while (err && --retries);
3441da7d
SRT		 6164
6165 /*
6166 * After reset the door-bell might be cleared, complete
6167 * outstanding requests in s/w here.
6168 */
6169 spin_lock_irqsave(hba->host->host_lock, flags);
6170 ufshcd_transfer_req_compl(hba);
6171 ufshcd_tmc_handler(hba);
6172 spin_unlock_irqrestore(hba->host->host_lock, flags);
6173
6174 return err;
6175}
6176
6177/**
6178 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 6179 * @cmd: SCSI command pointer
3441da7d
SRT		 6180 *
6181 * Returns SUCCESS/FAILED
6182 */
6183static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6184{
6185 int err;
6186 unsigned long flags;
6187 struct ufs_hba *hba;
6188
6189 hba = shost_priv(cmd->device->host);
6190
1ab27c9c 6191 ufshcd_hold(hba, false);
3441da7d
SRT		 6192 /*
6193 * Check if there is any race with fatal error handling.
6194 * If so, wait for it to complete. Even though fatal error
6195 * handling does reset and restore in some cases, don't assume
6196 * anything out of it. We are just avoiding race here.
6197 */
6198 do {
6199 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 6200 if (!(work_pending(&hba->eh_work) ||
8dc0da79
ZL		 6201 hba->ufshcd_state == UFSHCD_STATE_RESET ||
6202 hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
3441da7d
SRT		 6203 break;
6204 spin_unlock_irqrestore(hba->host->host_lock, flags);
6205 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
e8e7f271 6206 flush_work(&hba->eh_work);
3441da7d
SRT		 6207 } while (1);
6208
6209 hba->ufshcd_state = UFSHCD_STATE_RESET;
6210 ufshcd_set_eh_in_progress(hba);
6211 spin_unlock_irqrestore(hba->host->host_lock, flags);
6212
6213 err = ufshcd_reset_and_restore(hba);
6214
6215 spin_lock_irqsave(hba->host->host_lock, flags);
6216 if (!err) {
6217 err = SUCCESS;
6218 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6219 } else {
6220 err = FAILED;
6221 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6222 }
6223 ufshcd_clear_eh_in_progress(hba);
6224 spin_unlock_irqrestore(hba->host->host_lock, flags);
6225
1ab27c9c 6226 ufshcd_release(hba);
3441da7d
SRT		 6227 return err;
6228}
6229
3a4bf06d
YG		 6230/**
6231 * ufshcd_get_max_icc_level - calculate the ICC level
6232 * @sup_curr_uA: max. current supported by the regulator
6233 * @start_scan: row at the desc table to start scan from
6234 * @buff: power descriptor buffer
6235 *
6236 * Returns calculated max ICC level for specific regulator
6237 */
6238static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6239{
6240 int i;
6241 int curr_uA;
6242 u16 data;
6243 u16 unit;
6244
6245 for (i = start_scan; i >= 0; i--) {
d79713f9 6246 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 6247 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6248 ATTR_ICC_LVL_UNIT_OFFSET;
6249 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6250 switch (unit) {
6251 case UFSHCD_NANO_AMP:
6252 curr_uA = curr_uA / 1000;
6253 break;
6254 case UFSHCD_MILI_AMP:
6255 curr_uA = curr_uA * 1000;
6256 break;
6257 case UFSHCD_AMP:
6258 curr_uA = curr_uA * 1000 * 1000;
6259 break;
6260 case UFSHCD_MICRO_AMP:
6261 default:
6262 break;
6263 }
6264 if (sup_curr_uA >= curr_uA)
6265 break;
6266 }
6267 if (i < 0) {
6268 i = 0;
6269 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6270 }
6271
6272 return (u32)i;
6273}
6274
6275/**
6276 * ufshcd_calc_icc_level - calculate the max ICC level
6277 * In case regulators are not initialized we'll return 0
6278 * @hba: per-adapter instance
6279 * @desc_buf: power descriptor buffer to extract ICC levels from.
6280 * @len: length of desc_buff
6281 *
6282 * Returns calculated ICC level
6283 */
6284static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6285 u8 *desc_buf, int len)
6286{
6287 u32 icc_level = 0;
6288
6289 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6290 !hba->vreg_info.vccq2) {
6291 dev_err(hba->dev,
6292 "%s: Regulator capability was not set, actvIccLevel=%d",
6293 __func__, icc_level);
6294 goto out;
6295 }
6296
0487fff7 6297 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 6298 icc_level = ufshcd_get_max_icc_level(
6299 hba->vreg_info.vcc->max_uA,
6300 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6301 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6302
0487fff7 6303 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 6304 icc_level = ufshcd_get_max_icc_level(
6305 hba->vreg_info.vccq->max_uA,
6306 icc_level,
6307 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6308
0487fff7 6309 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 6310 icc_level = ufshcd_get_max_icc_level(
6311 hba->vreg_info.vccq2->max_uA,
6312 icc_level,
6313 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6314out:
6315 return icc_level;
6316}
6317
6318static void ufshcd_init_icc_levels(struct ufs_hba *hba)
6319{
6320 int ret;
a4b0e8a4 6321 int buff_len = hba->desc_size.pwr_desc;
bbe21d7a
KC		 6322 u8 *desc_buf;
6323
6324 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6325 if (!desc_buf)
6326 return;
3a4bf06d
YG		 6327
6328 ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
6329 if (ret) {
6330 dev_err(hba->dev,
6331 "%s: Failed reading power descriptor.len = %d ret = %d",
6332 __func__, buff_len, ret);
bbe21d7a 6333 goto out;
3a4bf06d
YG		 6334 }
6335
6336 hba->init_prefetch_data.icc_level =
6337 ufshcd_find_max_sup_active_icc_level(hba,
6338 desc_buf, buff_len);
6339 dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
6340 __func__, hba->init_prefetch_data.icc_level);
6341
dbd34a61
SM		 6342 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6343 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
6344 &hba->init_prefetch_data.icc_level);
3a4bf06d
YG		 6345
6346 if (ret)
6347 dev_err(hba->dev,
6348 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
6349 __func__, hba->init_prefetch_data.icc_level , ret);
6350
bbe21d7a
KC		 6351out:
6352 kfree(desc_buf);
3a4bf06d
YG		 6353}
6354
2a8fa600
SJ		 6355/**
6356 * ufshcd_scsi_add_wlus - Adds required W-LUs
6357 * @hba: per-adapter instance
6358 *
6359 * UFS device specification requires the UFS devices to support 4 well known
6360 * logical units:
6361 * "REPORT_LUNS" (address: 01h)
6362 * "UFS Device" (address: 50h)
6363 * "RPMB" (address: 44h)
6364 * "BOOT" (address: 30h)
6365 * UFS device's power management needs to be controlled by "POWER CONDITION"
6366 * field of SSU (START STOP UNIT) command. But this "power condition" field
6367 * will take effect only when its sent to "UFS device" well known logical unit
6368 * hence we require the scsi_device instance to represent this logical unit in
6369 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 6370 *
2a8fa600
SJ		 6371 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6372 * Block) LU so user space process can control this LU. User space may also
6373 * want to have access to BOOT LU.
8aa29f19 6374 *
2a8fa600
SJ		 6375 * This function adds scsi device instances for each of all well known LUs
6376 * (except "REPORT LUNS" LU).
6377 *
6378 * Returns zero on success (all required W-LUs are added successfully),
6379 * non-zero error value on failure (if failed to add any of the required W-LU).
6380 */
6381static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
6382{
6383 int ret = 0;
7c48bfd0
AM		 6384 struct scsi_device *sdev_rpmb;
6385 struct scsi_device *sdev_boot;
2a8fa600
SJ		 6386
6387 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
6388 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
6389 if (IS_ERR(hba->sdev_ufs_device)) {
6390 ret = PTR_ERR(hba->sdev_ufs_device);
6391 hba->sdev_ufs_device = NULL;
6392 goto out;
6393 }
7c48bfd0 6394 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 6395
7c48bfd0 6396 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 6397 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7c48bfd0
AM		 6398 if (IS_ERR(sdev_rpmb)) {
6399 ret = PTR_ERR(sdev_rpmb);
3d21fbde 6400 goto remove_sdev_ufs_device;
2a8fa600 6401 }
7c48bfd0 6402 scsi_device_put(sdev_rpmb);
3d21fbde
HK		 6403
6404 sdev_boot = __scsi_add_device(hba->host, 0, 0,
6405 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
6406 if (IS_ERR(sdev_boot))
6407 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
6408 else
6409 scsi_device_put(sdev_boot);
2a8fa600
SJ		 6410 goto out;
6411
2a8fa600
SJ		 6412remove_sdev_ufs_device:
6413 scsi_remove_device(hba->sdev_ufs_device);
6414out:
6415 return ret;
6416}
6417
93fdd5ac
TW		 6418static int ufs_get_device_desc(struct ufs_hba *hba,
6419 struct ufs_dev_desc *dev_desc)
c58ab7aa
YG		 6420{
6421 int err;
bbe21d7a 6422 size_t buff_len;
c58ab7aa 6423 u8 model_index;
bbe21d7a
KC		 6424 u8 *desc_buf;
6425
6426 buff_len = max_t(size_t, hba->desc_size.dev_desc,
6427 QUERY_DESC_MAX_SIZE + 1);
6428 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6429 if (!desc_buf) {
6430 err = -ENOMEM;
6431 goto out;
6432 }
c58ab7aa 6433
a4b0e8a4 6434 err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
c58ab7aa
YG		 6435 if (err) {
6436 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
6437 __func__, err);
6438 goto out;
6439 }
6440
6441 /*
6442 * getting vendor (manufacturerID) and Bank Index in big endian
6443 * format
6444 */
93fdd5ac 6445 dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 6446 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
6447
6448 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
6449
bbe21d7a
KC		 6450 /* Zero-pad entire buffer for string termination. */
6451 memset(desc_buf, 0, buff_len);
6452
6453 err = ufshcd_read_string_desc(hba, model_index, desc_buf,
8aa29f19 6454 QUERY_DESC_MAX_SIZE, true/*ASCII*/);
c58ab7aa
YG		 6455 if (err) {
6456 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
6457 __func__, err);
6458 goto out;
6459 }
6460
bbe21d7a
KC		 6461 desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
6462 strlcpy(dev_desc->model, (desc_buf + QUERY_DESC_HDR_SIZE),
6463 min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
c58ab7aa
YG		 6464 MAX_MODEL_LEN));
6465
6466 /* Null terminate the model string */
93fdd5ac 6467 dev_desc->model[MAX_MODEL_LEN] = '\0';
c58ab7aa
YG		 6468
6469out:
bbe21d7a 6470 kfree(desc_buf);
c58ab7aa
YG		 6471 return err;
6472}
6473
93fdd5ac
TW		 6474static void ufs_fixup_device_setup(struct ufs_hba *hba,
6475 struct ufs_dev_desc *dev_desc)
c58ab7aa 6476{
c58ab7aa 6477 struct ufs_dev_fix *f;
c58ab7aa
YG		 6478
6479 for (f = ufs_fixups; f->quirk; f++) {
93fdd5ac
TW		 6480 if ((f->card.wmanufacturerid == dev_desc->wmanufacturerid ||
6481 f->card.wmanufacturerid == UFS_ANY_VENDOR) &&
6482 (STR_PRFX_EQUAL(f->card.model, dev_desc->model) ||
c58ab7aa
YG		 6483 !strcmp(f->card.model, UFS_ANY_MODEL)))
6484 hba->dev_quirks |= f->quirk;
6485 }
6486}
6487
37113106
YG		 6488/**
6489 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6490 * @hba: per-adapter instance
6491 *
6492 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6493 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6494 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6495 * the hibern8 exit latency.
6496 *
6497 * Returns zero on success, non-zero error value on failure.
6498 */
6499static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
6500{
6501 int ret = 0;
6502 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
6503
6504 ret = ufshcd_dme_peer_get(hba,
6505 UIC_ARG_MIB_SEL(
6506 RX_MIN_ACTIVATETIME_CAPABILITY,
6507 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6508 &peer_rx_min_activatetime);
6509 if (ret)
6510 goto out;
6511
6512 /* make sure proper unit conversion is applied */
6513 tuned_pa_tactivate =
6514 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
6515 / PA_TACTIVATE_TIME_UNIT_US);
6516 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6517 tuned_pa_tactivate);
6518
6519out:
6520 return ret;
6521}
6522
6523/**
6524 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6525 * @hba: per-adapter instance
6526 *
6527 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6528 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6529 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6530 * This optimal value can help reduce the hibern8 exit latency.
6531 *
6532 * Returns zero on success, non-zero error value on failure.
6533 */
6534static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
6535{
6536 int ret = 0;
6537 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
6538 u32 max_hibern8_time, tuned_pa_hibern8time;
6539
6540 ret = ufshcd_dme_get(hba,
6541 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
6542 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
6543 &local_tx_hibern8_time_cap);
6544 if (ret)
6545 goto out;
6546
6547 ret = ufshcd_dme_peer_get(hba,
6548 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
6549 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6550 &peer_rx_hibern8_time_cap);
6551 if (ret)
6552 goto out;
6553
6554 max_hibern8_time = max(local_tx_hibern8_time_cap,
6555 peer_rx_hibern8_time_cap);
6556 /* make sure proper unit conversion is applied */
6557 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
6558 / PA_HIBERN8_TIME_UNIT_US);
6559 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
6560 tuned_pa_hibern8time);
6561out:
6562 return ret;
6563}
6564
c6a6db43 6565/**
6566 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6567 * less than device PA_TACTIVATE time.
6568 * @hba: per-adapter instance
6569 *
6570 * Some UFS devices require host PA_TACTIVATE to be lower than device
6571 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6572 * for such devices.
6573 *
6574 * Returns zero on success, non-zero error value on failure.
6575 */
6576static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
6577{
6578 int ret = 0;
6579 u32 granularity, peer_granularity;
6580 u32 pa_tactivate, peer_pa_tactivate;
6581 u32 pa_tactivate_us, peer_pa_tactivate_us;
6582 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
6583
6584 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6585 &granularity);
6586 if (ret)
6587 goto out;
6588
6589 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6590 &peer_granularity);
6591 if (ret)
6592 goto out;
6593
6594 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
6595 (granularity > PA_GRANULARITY_MAX_VAL)) {
6596 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
6597 __func__, granularity);
6598 return -EINVAL;
6599 }
6600
6601 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
6602 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
6603 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
6604 __func__, peer_granularity);
6605 return -EINVAL;
6606 }
6607
6608 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
6609 if (ret)
6610 goto out;
6611
6612 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
6613 &peer_pa_tactivate);
6614 if (ret)
6615 goto out;
6616
6617 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
6618 peer_pa_tactivate_us = peer_pa_tactivate *
6619 gran_to_us_table[peer_granularity - 1];
6620
6621 if (pa_tactivate_us > peer_pa_tactivate_us) {
6622 u32 new_peer_pa_tactivate;
6623
6624 new_peer_pa_tactivate = pa_tactivate_us /
6625 gran_to_us_table[peer_granularity - 1];
6626 new_peer_pa_tactivate++;
6627 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6628 new_peer_pa_tactivate);
6629 }
6630
6631out:
6632 return ret;
6633}
6634
37113106
YG		 6635static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
6636{
6637 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
6638 ufshcd_tune_pa_tactivate(hba);
6639 ufshcd_tune_pa_hibern8time(hba);
6640 }
6641
6642 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
6643 /* set 1ms timeout for PA_TACTIVATE */
6644 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43 6645
6646 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
6647 ufshcd_quirk_tune_host_pa_tactivate(hba);
56d4a186
SJ		 6648
6649 ufshcd_vops_apply_dev_quirks(hba);
37113106
YG		 6650}
6651
ff8e20c6
DR		 6652static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
6653{
6654 int err_reg_hist_size = sizeof(struct ufs_uic_err_reg_hist);
6655
6656 hba->ufs_stats.hibern8_exit_cnt = 0;
6657 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
6658
6659 memset(&hba->ufs_stats.pa_err, 0, err_reg_hist_size);
6660 memset(&hba->ufs_stats.dl_err, 0, err_reg_hist_size);
6661 memset(&hba->ufs_stats.nl_err, 0, err_reg_hist_size);
6662 memset(&hba->ufs_stats.tl_err, 0, err_reg_hist_size);
6663 memset(&hba->ufs_stats.dme_err, 0, err_reg_hist_size);
7fabb77b
GB		 6664
6665 hba->req_abort_count = 0;
ff8e20c6
DR		 6666}
6667
a4b0e8a4
PM		 6668static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
6669{
6670 int err;
6671
6672 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
6673 &hba->desc_size.dev_desc);
6674 if (err)
6675 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6676
6677 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
6678 &hba->desc_size.pwr_desc);
6679 if (err)
6680 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6681
6682 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
6683 &hba->desc_size.interc_desc);
6684 if (err)
6685 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6686
6687 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
6688 &hba->desc_size.conf_desc);
6689 if (err)
6690 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6691
6692 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
6693 &hba->desc_size.unit_desc);
6694 if (err)
6695 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6696
6697 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6698 &hba->desc_size.geom_desc);
6699 if (err)
6700 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
c648c2d2
SN		 6701 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
6702 &hba->desc_size.hlth_desc);
6703 if (err)
6704 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
a4b0e8a4
PM		 6705}
6706
6707static void ufshcd_def_desc_sizes(struct ufs_hba *hba)
6708{
6709 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6710 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6711 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6712 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6713 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6714 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
c648c2d2 6715 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
a4b0e8a4
PM		 6716}
6717
9e1e8a75
SJ		 6718static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
6719 {19200000, REF_CLK_FREQ_19_2_MHZ},
6720 {26000000, REF_CLK_FREQ_26_MHZ},
6721 {38400000, REF_CLK_FREQ_38_4_MHZ},
6722 {52000000, REF_CLK_FREQ_52_MHZ},
6723 {0, REF_CLK_FREQ_INVAL},
6724};
6725
6726static enum ufs_ref_clk_freq
6727ufs_get_bref_clk_from_hz(unsigned long freq)
6728{
6729 int i;
6730
6731 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
6732 if (ufs_ref_clk_freqs[i].freq_hz == freq)
6733 return ufs_ref_clk_freqs[i].val;
6734
6735 return REF_CLK_FREQ_INVAL;
6736}
6737
6738void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
6739{
6740 unsigned long freq;
6741
6742 freq = clk_get_rate(refclk);
6743
6744 hba->dev_ref_clk_freq =
6745 ufs_get_bref_clk_from_hz(freq);
6746
6747 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
6748 dev_err(hba->dev,
6749 "invalid ref_clk setting = %ld\n", freq);
6750}
6751
6752static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
6753{
6754 int err;
6755 u32 ref_clk;
6756 u32 freq = hba->dev_ref_clk_freq;
6757
6758 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6759 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
6760
6761 if (err) {
6762 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
6763 err);
6764 goto out;
6765 }
6766
6767 if (ref_clk == freq)
6768 goto out; /* nothing to update */
6769
6770 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6771 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
6772
6773 if (err) {
6774 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
6775 ufs_ref_clk_freqs[freq].freq_hz);
6776 goto out;
6777 }
6778
6779 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
6780 ufs_ref_clk_freqs[freq].freq_hz);
6781
6782out:
6783 return err;
6784}
6785
6ccf44fe 6786/**
1d337ec2
SRT		 6787 * ufshcd_probe_hba - probe hba to detect device and initialize
6788 * @hba: per-adapter instance
6789 *
6790 * Execute link-startup and verify device initialization
6ccf44fe 6791 */
1d337ec2 6792static int ufshcd_probe_hba(struct ufs_hba *hba)
6ccf44fe 6793{
93fdd5ac 6794 struct ufs_dev_desc card = {0};
6ccf44fe 6795 int ret;
7ff5ab47 6796 ktime_t start = ktime_get();
6ccf44fe
SJ		 6797
6798 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 6799 if (ret)
6800 goto out;
6801
afdfff59
YG		 6802 /* set the default level for urgent bkops */
6803 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
6804 hba->is_urgent_bkops_lvl_checked = false;
6805
ff8e20c6
DR		 6806 /* Debug counters initialization */
6807 ufshcd_clear_dbg_ufs_stats(hba);
6808
57d104c1
SJ		 6809 /* UniPro link is active now */
6810 ufshcd_set_link_active(hba);
d3e89bac 6811
ad448378
AH		 6812 /* Enable Auto-Hibernate if configured */
6813 ufshcd_auto_hibern8_enable(hba);
6814
5a0b0cb9
SRT		 6815 ret = ufshcd_verify_dev_init(hba);
6816 if (ret)
6817 goto out;
68078d5c
DR		 6818
6819 ret = ufshcd_complete_dev_init(hba);
6820 if (ret)
6821 goto out;
5a0b0cb9 6822
a4b0e8a4
PM		 6823 /* Init check for device descriptor sizes */
6824 ufshcd_init_desc_sizes(hba);
6825
93fdd5ac
TW		 6826 ret = ufs_get_device_desc(hba, &card);
6827 if (ret) {
6828 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
6829 __func__, ret);
6830 goto out;
6831 }
6832
6833 ufs_fixup_device_setup(hba, &card);
37113106 6834 ufshcd_tune_unipro_params(hba);
60f01870 6835
57d104c1
SJ		 6836 /* UFS device is also active now */
6837 ufshcd_set_ufs_dev_active(hba);
66ec6d59 6838 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 6839 hba->wlun_dev_clr_ua = true;
6840
7eb584db
DR		 6841 if (ufshcd_get_max_pwr_mode(hba)) {
6842 dev_err(hba->dev,
6843 "%s: Failed getting max supported power mode\n",
6844 __func__);
6845 } else {
9e1e8a75
SJ		 6846 /*
6847 * Set the right value to bRefClkFreq before attempting to
6848 * switch to HS gears.
6849 */
6850 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
6851 ufshcd_set_dev_ref_clk(hba);
7eb584db 6852 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 6853 if (ret) {
7eb584db
DR		 6854 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
6855 __func__, ret);
8643ae66
DL		 6856 goto out;
6857 }
7eb584db 6858 }
57d104c1 6859
53c12d0e
YG		 6860 /* set the state as operational after switching to desired gear */
6861 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
a4b0e8a4 6862
57d104c1
SJ		 6863 /*
6864 * If we are in error handling context or in power management callbacks
6865 * context, no need to scan the host
6866 */
6867 if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
6868 bool flag;
6869
6870 /* clear any previous UFS device information */
6871 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
dc3c8d3a
YG		 6872 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
6873 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
57d104c1 6874 hba->dev_info.f_power_on_wp_en = flag;
3441da7d 6875
3a4bf06d
YG		 6876 if (!hba->is_init_prefetch)
6877 ufshcd_init_icc_levels(hba);
6878
2a8fa600
SJ		 6879 /* Add required well known logical units to scsi mid layer */
6880 if (ufshcd_scsi_add_wlus(hba))
6881 goto out;
6882
0701e49d 6883 /* Initialize devfreq after UFS device is detected */
6884 if (ufshcd_is_clkscaling_supported(hba)) {
6885 memcpy(&hba->clk_scaling.saved_pwr_info.info,
6886 &hba->pwr_info,
6887 sizeof(struct ufs_pa_layer_attr));
6888 hba->clk_scaling.saved_pwr_info.is_valid = true;
6889 if (!hba->devfreq) {
deac444f
BA		 6890 ret = ufshcd_devfreq_init(hba);
6891 if (ret)
0701e49d 6892 goto out;
0701e49d 6893 }
6894 hba->clk_scaling.is_allowed = true;
6895 }
6896
df032bf2
AA		 6897 ufs_bsg_probe(hba);
6898
3441da7d
SRT		 6899 scsi_scan_host(hba->host);
6900 pm_runtime_put_sync(hba->dev);
6901 }
3a4bf06d
YG		 6902
6903 if (!hba->is_init_prefetch)
6904 hba->is_init_prefetch = true;
6905
5a0b0cb9 6906out:
1d337ec2
SRT		 6907 /*
6908 * If we failed to initialize the device or the device is not
6909 * present, turn off the power/clocks etc.
6910 */
57d104c1
SJ		 6911 if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
6912 pm_runtime_put_sync(hba->dev);
eebcc196 6913 ufshcd_exit_clk_scaling(hba);
1d337ec2 6914 ufshcd_hba_exit(hba);
57d104c1 6915 }
1d337ec2 6916
7ff5ab47 6917 trace_ufshcd_init(dev_name(hba->dev), ret,
6918 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 6919 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 6920 return ret;
6921}
6922
6923/**
6924 * ufshcd_async_scan - asynchronous execution for probing hba
6925 * @data: data pointer to pass to this function
6926 * @cookie: cookie data
6927 */
6928static void ufshcd_async_scan(void *data, async_cookie_t cookie)
6929{
6930 struct ufs_hba *hba = (struct ufs_hba *)data;
6931
6932 ufshcd_probe_hba(hba);
6ccf44fe
SJ		 6933}
6934
f550c65b
YG		 6935static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
6936{
6937 unsigned long flags;
6938 struct Scsi_Host *host;
6939 struct ufs_hba *hba;
6940 int index;
6941 bool found = false;
6942
6943 if (!scmd || !scmd->device || !scmd->device->host)
6600593c 6944 return BLK_EH_DONE;
f550c65b
YG		 6945
6946 host = scmd->device->host;
6947 hba = shost_priv(host);
6948 if (!hba)
6600593c 6949 return BLK_EH_DONE;
f550c65b
YG		 6950
6951 spin_lock_irqsave(host->host_lock, flags);
6952
6953 for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
6954 if (hba->lrb[index].cmd == scmd) {
6955 found = true;
6956 break;
6957 }
6958 }
6959
6960 spin_unlock_irqrestore(host->host_lock, flags);
6961
6962 /*
6963 * Bypass SCSI error handling and reset the block layer timer if this
6964 * SCSI command was not actually dispatched to UFS driver, otherwise
6965 * let SCSI layer handle the error as usual.
6966 */
6600593c 6967 return found ? BLK_EH_DONE : BLK_EH_RESET_TIMER;
f550c65b
YG		 6968}
6969
d829fc8a
SN		 6970static const struct attribute_group *ufshcd_driver_groups[] = {
6971 &ufs_sysfs_unit_descriptor_group,
ec92b59c 6972 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 6973 NULL,
6974};
6975
7a3e97b0
SY		 6976static struct scsi_host_template ufshcd_driver_template = {
6977 .module = THIS_MODULE,
6978 .name = UFSHCD,
6979 .proc_name = UFSHCD,
6980 .queuecommand = ufshcd_queuecommand,
6981 .slave_alloc = ufshcd_slave_alloc,
eeda4749 6982 .slave_configure = ufshcd_slave_configure,
7a3e97b0 6983 .slave_destroy = ufshcd_slave_destroy,
4264fd61 6984 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 6985 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 6986 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
6987 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
f550c65b 6988 .eh_timed_out = ufshcd_eh_timed_out,
7a3e97b0
SY		 6989 .this_id = -1,
6990 .sg_tablesize = SG_ALL,
6991 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
6992 .can_queue = UFSHCD_CAN_QUEUE,
1ab27c9c 6993 .max_host_blocked = 1,
c40ecc12 6994 .track_queue_depth = 1,
d829fc8a 6995 .sdev_groups = ufshcd_driver_groups,
4af14d11 6996 .dma_boundary = PAGE_SIZE - 1,
7a3e97b0
SY		 6997};
6998
57d104c1
SJ		 6999static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7000 int ua)
7001{
7b16a07c 7002 int ret;
57d104c1 7003
7b16a07c
BA		 7004 if (!vreg)
7005 return 0;
57d104c1 7006
0487fff7
SC		 7007 /*
7008 * "set_load" operation shall be required on those regulators
7009 * which specifically configured current limitation. Otherwise
7010 * zero max_uA may cause unexpected behavior when regulator is
7011 * enabled or set as high power mode.
7012 */
7013 if (!vreg->max_uA)
7014 return 0;
7015
7b16a07c
BA		 7016 ret = regulator_set_load(vreg->reg, ua);
7017 if (ret < 0) {
7018 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7019 __func__, vreg->name, ua, ret);
57d104c1
SJ		 7020 }
7021
7022 return ret;
7023}
7024
7025static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7026 struct ufs_vreg *vreg)
7027{
73067981 7028 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 7029}
7030
7031static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7032 struct ufs_vreg *vreg)
7033{
73067981 7034 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 7035}
7036
aa497613
SRT		 7037static int ufshcd_config_vreg(struct device *dev,
7038 struct ufs_vreg *vreg, bool on)
7039{
7040 int ret = 0;
72753590
GS		 7041 struct regulator *reg;
7042 const char *name;
aa497613
SRT		 7043 int min_uV, uA_load;
7044
7045 BUG_ON(!vreg);
7046
72753590
GS		 7047 reg = vreg->reg;
7048 name = vreg->name;
7049
aa497613 7050 if (regulator_count_voltages(reg) > 0) {
3b141e8c
SC		 7051 if (vreg->min_uV && vreg->max_uV) {
7052 min_uV = on ? vreg->min_uV : 0;
7053 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
7054 if (ret) {
7055 dev_err(dev,
7056 "%s: %s set voltage failed, err=%d\n",
aa497613 7057 __func__, name, ret);
3b141e8c
SC		 7058 goto out;
7059 }
aa497613
SRT		 7060 }
7061
7062 uA_load = on ? vreg->max_uA : 0;
57d104c1
SJ		 7063 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7064 if (ret)
aa497613 7065 goto out;
aa497613
SRT		 7066 }
7067out:
7068 return ret;
7069}
7070
7071static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7072{
7073 int ret = 0;
7074
73067981 7075 if (!vreg || vreg->enabled)
aa497613
SRT		 7076 goto out;
7077
7078 ret = ufshcd_config_vreg(dev, vreg, true);
7079 if (!ret)
7080 ret = regulator_enable(vreg->reg);
7081
7082 if (!ret)
7083 vreg->enabled = true;
7084 else
7085 dev_err(dev, "%s: %s enable failed, err=%d\n",
7086 __func__, vreg->name, ret);
7087out:
7088 return ret;
7089}
7090
7091static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7092{
7093 int ret = 0;
7094
73067981 7095 if (!vreg || !vreg->enabled)
aa497613
SRT		 7096 goto out;
7097
7098 ret = regulator_disable(vreg->reg);
7099
7100 if (!ret) {
7101 /* ignore errors on applying disable config */
7102 ufshcd_config_vreg(dev, vreg, false);
7103 vreg->enabled = false;
7104 } else {
7105 dev_err(dev, "%s: %s disable failed, err=%d\n",
7106 __func__, vreg->name, ret);
7107 }
7108out:
7109 return ret;
7110}
7111
7112static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7113{
7114 int ret = 0;
7115 struct device *dev = hba->dev;
7116 struct ufs_vreg_info *info = &hba->vreg_info;
7117
aa497613
SRT		 7118 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7119 if (ret)
7120 goto out;
7121
7122 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7123 if (ret)
7124 goto out;
7125
7126 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
7127 if (ret)
7128 goto out;
7129
7130out:
7131 if (ret) {
7132 ufshcd_toggle_vreg(dev, info->vccq2, false);
7133 ufshcd_toggle_vreg(dev, info->vccq, false);
7134 ufshcd_toggle_vreg(dev, info->vcc, false);
7135 }
7136 return ret;
7137}
7138
6a771a65
RS		 7139static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7140{
7141 struct ufs_vreg_info *info = &hba->vreg_info;
7142
60b7b823 7143 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 7144}
7145
aa497613
SRT		 7146static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7147{
7148 int ret = 0;
7149
7150 if (!vreg)
7151 goto out;
7152
7153 vreg->reg = devm_regulator_get(dev, vreg->name);
7154 if (IS_ERR(vreg->reg)) {
7155 ret = PTR_ERR(vreg->reg);
7156 dev_err(dev, "%s: %s get failed, err=%d\n",
7157 __func__, vreg->name, ret);
7158 }
7159out:
7160 return ret;
7161}
7162
7163static int ufshcd_init_vreg(struct ufs_hba *hba)
7164{
7165 int ret = 0;
7166 struct device *dev = hba->dev;
7167 struct ufs_vreg_info *info = &hba->vreg_info;
7168
aa497613
SRT		 7169 ret = ufshcd_get_vreg(dev, info->vcc);
7170 if (ret)
7171 goto out;
7172
7173 ret = ufshcd_get_vreg(dev, info->vccq);
7174 if (ret)
7175 goto out;
7176
7177 ret = ufshcd_get_vreg(dev, info->vccq2);
7178out:
7179 return ret;
7180}
7181
6a771a65
RS		 7182static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7183{
7184 struct ufs_vreg_info *info = &hba->vreg_info;
7185
7186 if (info)
7187 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7188
7189 return 0;
7190}
7191
57d104c1
SJ		 7192static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7193 bool skip_ref_clk)
c6e79dac
SRT		 7194{
7195 int ret = 0;
7196 struct ufs_clk_info *clki;
7197 struct list_head *head = &hba->clk_list_head;
1ab27c9c 7198 unsigned long flags;
911a0771 7199 ktime_t start = ktime_get();
7200 bool clk_state_changed = false;
c6e79dac 7201
566ec9ad 7202 if (list_empty(head))
c6e79dac
SRT		 7203 goto out;
7204
b334456e
SJ		 7205 /*
7206 * vendor specific setup_clocks ops may depend on clocks managed by
7207 * this standard driver hence call the vendor specific setup_clocks
7208 * before disabling the clocks managed here.
7209 */
7210 if (!on) {
7211 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7212 if (ret)
7213 return ret;
7214 }
1e879e8f 7215
c6e79dac
SRT		 7216 list_for_each_entry(clki, head, list) {
7217 if (!IS_ERR_OR_NULL(clki->clk)) {
57d104c1
SJ		 7218 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7219 continue;
7220
911a0771 7221 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 7222 if (on && !clki->enabled) {
7223 ret = clk_prepare_enable(clki->clk);
7224 if (ret) {
7225 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7226 __func__, clki->name, ret);
7227 goto out;
7228 }
7229 } else if (!on && clki->enabled) {
7230 clk_disable_unprepare(clki->clk);
7231 }
7232 clki->enabled = on;
7233 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7234 clki->name, on ? "en" : "dis");
7235 }
7236 }
1ab27c9c 7237
b334456e
SJ		 7238 /*
7239 * vendor specific setup_clocks ops may depend on clocks managed by
7240 * this standard driver hence call the vendor specific setup_clocks
7241 * after enabling the clocks managed here.
7242 */
7243 if (on) {
7244 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7245 if (ret)
7246 return ret;
7247 }
1e879e8f 7248
c6e79dac
SRT		 7249out:
7250 if (ret) {
7251 list_for_each_entry(clki, head, list) {
7252 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
7253 clk_disable_unprepare(clki->clk);
7254 }
7ff5ab47 7255 } else if (!ret && on) {
1ab27c9c
ST		 7256 spin_lock_irqsave(hba->host->host_lock, flags);
7257 hba->clk_gating.state = CLKS_ON;
7ff5ab47 7258 trace_ufshcd_clk_gating(dev_name(hba->dev),
7259 hba->clk_gating.state);
1ab27c9c 7260 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 7261 }
7ff5ab47 7262
911a0771 7263 if (clk_state_changed)
7264 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
7265 (on ? "on" : "off"),
7266 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 7267 return ret;
7268}
7269
57d104c1
SJ		 7270static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
7271{
7272 return __ufshcd_setup_clocks(hba, on, false);
7273}
7274
c6e79dac
SRT		 7275static int ufshcd_init_clocks(struct ufs_hba *hba)
7276{
7277 int ret = 0;
7278 struct ufs_clk_info *clki;
7279 struct device *dev = hba->dev;
7280 struct list_head *head = &hba->clk_list_head;
7281
566ec9ad 7282 if (list_empty(head))
c6e79dac
SRT		 7283 goto out;
7284
7285 list_for_each_entry(clki, head, list) {
7286 if (!clki->name)
7287 continue;
7288
7289 clki->clk = devm_clk_get(dev, clki->name);
7290 if (IS_ERR(clki->clk)) {
7291 ret = PTR_ERR(clki->clk);
7292 dev_err(dev, "%s: %s clk get failed, %d\n",
7293 __func__, clki->name, ret);
7294 goto out;
7295 }
7296
9e1e8a75
SJ		 7297 /*
7298 * Parse device ref clk freq as per device tree "ref_clk".
7299 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7300 * in ufshcd_alloc_host().
7301 */
7302 if (!strcmp(clki->name, "ref_clk"))
7303 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
7304
c6e79dac
SRT		 7305 if (clki->max_freq) {
7306 ret = clk_set_rate(clki->clk, clki->max_freq);
7307 if (ret) {
7308 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
7309 __func__, clki->name,
7310 clki->max_freq, ret);
7311 goto out;
7312 }
856b3483 7313 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 7314 }
7315 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
7316 clki->name, clk_get_rate(clki->clk));
7317 }
7318out:
7319 return ret;
7320}
7321
5c0c28a8
SRT		 7322static int ufshcd_variant_hba_init(struct ufs_hba *hba)
7323{
7324 int err = 0;
7325
7326 if (!hba->vops)
7327 goto out;
7328
0263bcd0
YG		 7329 err = ufshcd_vops_init(hba);
7330 if (err)
7331 goto out;
5c0c28a8 7332
0263bcd0
YG		 7333 err = ufshcd_vops_setup_regulators(hba, true);
7334 if (err)
7335 goto out_exit;
5c0c28a8
SRT		 7336
7337 goto out;
7338
5c0c28a8 7339out_exit:
0263bcd0 7340 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7341out:
7342 if (err)
7343 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 7344 __func__, ufshcd_get_var_name(hba), err);
5c0c28a8
SRT		 7345 return err;
7346}
7347
7348static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
7349{
7350 if (!hba->vops)
7351 return;
7352
0263bcd0 7353 ufshcd_vops_setup_regulators(hba, false);
5c0c28a8 7354
0263bcd0 7355 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7356}
7357
aa497613
SRT		 7358static int ufshcd_hba_init(struct ufs_hba *hba)
7359{
7360 int err;
7361
6a771a65
RS		 7362 /*
7363 * Handle host controller power separately from the UFS device power
7364 * rails as it will help controlling the UFS host controller power
7365 * collapse easily which is different than UFS device power collapse.
7366 * Also, enable the host controller power before we go ahead with rest
7367 * of the initialization here.
7368 */
7369 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 7370 if (err)
7371 goto out;
7372
6a771a65 7373 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 7374 if (err)
7375 goto out;
7376
6a771a65
RS		 7377 err = ufshcd_init_clocks(hba);
7378 if (err)
7379 goto out_disable_hba_vreg;
7380
7381 err = ufshcd_setup_clocks(hba, true);
7382 if (err)
7383 goto out_disable_hba_vreg;
7384
c6e79dac
SRT		 7385 err = ufshcd_init_vreg(hba);
7386 if (err)
7387 goto out_disable_clks;
7388
7389 err = ufshcd_setup_vreg(hba, true);
7390 if (err)
7391 goto out_disable_clks;
7392
aa497613
SRT		 7393 err = ufshcd_variant_hba_init(hba);
7394 if (err)
7395 goto out_disable_vreg;
7396
1d337ec2 7397 hba->is_powered = true;
aa497613
SRT		 7398 goto out;
7399
7400out_disable_vreg:
7401 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 7402out_disable_clks:
7403 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 7404out_disable_hba_vreg:
7405 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 7406out:
7407 return err;
7408}
7409
7410static void ufshcd_hba_exit(struct ufs_hba *hba)
7411{
1d337ec2
SRT		 7412 if (hba->is_powered) {
7413 ufshcd_variant_hba_exit(hba);
7414 ufshcd_setup_vreg(hba, false);
a508253d 7415 ufshcd_suspend_clkscaling(hba);
eebcc196 7416 if (ufshcd_is_clkscaling_supported(hba))
0701e49d 7417 if (hba->devfreq)
7418 ufshcd_suspend_clkscaling(hba);
1d337ec2
SRT		 7419 ufshcd_setup_clocks(hba, false);
7420 ufshcd_setup_hba_vreg(hba, false);
7421 hba->is_powered = false;
7422 }
aa497613
SRT		 7423}
7424
57d104c1
SJ		 7425static int
7426ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
7427{
7428 unsigned char cmd[6] = {REQUEST_SENSE,
7429 0,
7430 0,
7431 0,
09a5a24f 7432 UFS_SENSE_SIZE,
57d104c1
SJ		 7433 0};
7434 char *buffer;
7435 int ret;
7436
09a5a24f 7437 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 7438 if (!buffer) {
7439 ret = -ENOMEM;
7440 goto out;
7441 }
7442
fcbfffe2 7443 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 7444 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 7445 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 7446 if (ret)
7447 pr_err("%s: failed with err %d\n", __func__, ret);
7448
7449 kfree(buffer);
7450out:
7451 return ret;
7452}
7453
7454/**
7455 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7456 * power mode
7457 * @hba: per adapter instance
7458 * @pwr_mode: device power mode to set
7459 *
7460 * Returns 0 if requested power mode is set successfully
7461 * Returns non-zero if failed to set the requested power mode
7462 */
7463static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
7464 enum ufs_dev_pwr_mode pwr_mode)
7465{
7466 unsigned char cmd[6] = { START_STOP };
7467 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 7468 struct scsi_device *sdp;
7469 unsigned long flags;
57d104c1
SJ		 7470 int ret;
7471
7c48bfd0
AM		 7472 spin_lock_irqsave(hba->host->host_lock, flags);
7473 sdp = hba->sdev_ufs_device;
7474 if (sdp) {
7475 ret = scsi_device_get(sdp);
7476 if (!ret && !scsi_device_online(sdp)) {
7477 ret = -ENODEV;
7478 scsi_device_put(sdp);
7479 }
7480 } else {
7481 ret = -ENODEV;
7482 }
7483 spin_unlock_irqrestore(hba->host->host_lock, flags);
7484
7485 if (ret)
7486 return ret;
57d104c1
SJ		 7487
7488 /*
7489 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7490 * handling, which would wait for host to be resumed. Since we know
7491 * we are functional while we are here, skip host resume in error
7492 * handling context.
7493 */
7494 hba->host->eh_noresume = 1;
7495 if (hba->wlun_dev_clr_ua) {
7496 ret = ufshcd_send_request_sense(hba, sdp);
7497 if (ret)
7498 goto out;
7499 /* Unit attention condition is cleared now */
7500 hba->wlun_dev_clr_ua = false;
7501 }
7502
7503 cmd[4] = pwr_mode << 4;
7504
7505 /*
7506 * Current function would be generally called from the power management
e8064021 7507 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 7508 * already suspended childs.
7509 */
fcbfffe2
CH		 7510 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
7511 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 7512 if (ret) {
7513 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 7514 "START_STOP failed for power mode: %d, result %x\n",
7515 pwr_mode, ret);
c65be1a6 7516 if (driver_byte(ret) == DRIVER_SENSE)
21045519 7517 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 7518 }
7519
7520 if (!ret)
7521 hba->curr_dev_pwr_mode = pwr_mode;
7522out:
7c48bfd0 7523 scsi_device_put(sdp);
57d104c1
SJ		 7524 hba->host->eh_noresume = 0;
7525 return ret;
7526}
7527
7528static int ufshcd_link_state_transition(struct ufs_hba *hba,
7529 enum uic_link_state req_link_state,
7530 int check_for_bkops)
7531{
7532 int ret = 0;
7533
7534 if (req_link_state == hba->uic_link_state)
7535 return 0;
7536
7537 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
7538 ret = ufshcd_uic_hibern8_enter(hba);
7539 if (!ret)
7540 ufshcd_set_link_hibern8(hba);
7541 else
7542 goto out;
7543 }
7544 /*
7545 * If autobkops is enabled, link can't be turned off because
7546 * turning off the link would also turn off the device.
7547 */
7548 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
7549 (!check_for_bkops || (check_for_bkops &&
7550 !hba->auto_bkops_enabled))) {
f3099fbd
YG		 7551 /*
7552 * Let's make sure that link is in low power mode, we are doing
7553 * this currently by putting the link in Hibern8. Otherway to
7554 * put the link in low power mode is to send the DME end point
7555 * to device and then send the DME reset command to local
7556 * unipro. But putting the link in hibern8 is much faster.
7557 */
7558 ret = ufshcd_uic_hibern8_enter(hba);
7559 if (ret)
7560 goto out;
57d104c1
SJ		 7561 /*
7562 * Change controller state to "reset state" which
7563 * should also put the link in off/reset state
7564 */
596585a2 7565 ufshcd_hba_stop(hba, true);
57d104c1
SJ		 7566 /*
7567 * TODO: Check if we need any delay to make sure that
7568 * controller is reset
7569 */
7570 ufshcd_set_link_off(hba);
7571 }
7572
7573out:
7574 return ret;
7575}
7576
7577static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
7578{
b799fdf7
YG		 7579 /*
7580 * It seems some UFS devices may keep drawing more than sleep current
7581 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7582 * To avoid this situation, add 2ms delay before putting these UFS
7583 * rails in LPM mode.
7584 */
7585 if (!ufshcd_is_link_active(hba) &&
7586 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
7587 usleep_range(2000, 2100);
7588
57d104c1
SJ		 7589 /*
7590 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7591 * power.
7592 *
7593 * If UFS device and link is in OFF state, all power supplies (VCC,
7594 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7595 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7596 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7597 *
7598 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7599 * in low power state which would save some power.
7600 */
7601 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7602 !hba->dev_info.is_lu_power_on_wp) {
7603 ufshcd_setup_vreg(hba, false);
7604 } else if (!ufshcd_is_ufs_dev_active(hba)) {
7605 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7606 if (!ufshcd_is_link_active(hba)) {
7607 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7608 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
7609 }
7610 }
7611}
7612
7613static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
7614{
7615 int ret = 0;
7616
7617 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7618 !hba->dev_info.is_lu_power_on_wp) {
7619 ret = ufshcd_setup_vreg(hba, true);
7620 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 7621 if (!ret && !ufshcd_is_link_active(hba)) {
7622 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
7623 if (ret)
7624 goto vcc_disable;
7625 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
7626 if (ret)
7627 goto vccq_lpm;
7628 }
69d72ac8 7629 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 7630 }
7631 goto out;
7632
7633vccq_lpm:
7634 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7635vcc_disable:
7636 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7637out:
7638 return ret;
7639}
7640
7641static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
7642{
7643 if (ufshcd_is_link_off(hba))
7644 ufshcd_setup_hba_vreg(hba, false);
7645}
7646
7647static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
7648{
7649 if (ufshcd_is_link_off(hba))
7650 ufshcd_setup_hba_vreg(hba, true);
7651}
7652
7a3e97b0 7653/**
57d104c1 7654 * ufshcd_suspend - helper function for suspend operations
3b1d0580 7655 * @hba: per adapter instance
57d104c1
SJ		 7656 * @pm_op: desired low power operation type
7657 *
7658 * This function will try to put the UFS device and link into low power
7659 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7660 * (System PM level).
7661 *
7662 * If this function is called during shutdown, it will make sure that
7663 * both UFS device and UFS link is powered off.
7a3e97b0 7664 *
57d104c1
SJ		 7665 * NOTE: UFS device & link must be active before we enter in this function.
7666 *
7667 * Returns 0 for success and non-zero for failure
7a3e97b0 7668 */
57d104c1 7669static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7670{
57d104c1
SJ		 7671 int ret = 0;
7672 enum ufs_pm_level pm_lvl;
7673 enum ufs_dev_pwr_mode req_dev_pwr_mode;
7674 enum uic_link_state req_link_state;
7675
7676 hba->pm_op_in_progress = 1;
7677 if (!ufshcd_is_shutdown_pm(pm_op)) {
7678 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
7679 hba->rpm_lvl : hba->spm_lvl;
7680 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
7681 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
7682 } else {
7683 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
7684 req_link_state = UIC_LINK_OFF_STATE;
7685 }
7686
7a3e97b0 7687 /*
57d104c1
SJ		 7688 * If we can't transition into any of the low power modes
7689 * just gate the clocks.
7a3e97b0 7690 */
1ab27c9c
ST		 7691 ufshcd_hold(hba, false);
7692 hba->clk_gating.is_suspended = true;
7693
401f1e44 7694 if (hba->clk_scaling.is_allowed) {
7695 cancel_work_sync(&hba->clk_scaling.suspend_work);
7696 cancel_work_sync(&hba->clk_scaling.resume_work);
7697 ufshcd_suspend_clkscaling(hba);
7698 }
d6fcf81a 7699
57d104c1
SJ		 7700 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
7701 req_link_state == UIC_LINK_ACTIVE_STATE) {
7702 goto disable_clks;
7703 }
7a3e97b0 7704
57d104c1
SJ		 7705 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
7706 (req_link_state == hba->uic_link_state))
d6fcf81a 7707 goto enable_gating;
57d104c1
SJ		 7708
7709 /* UFS device & link must be active before we enter in this function */
7710 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
7711 ret = -EINVAL;
d6fcf81a 7712 goto enable_gating;
57d104c1
SJ		 7713 }
7714
7715 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 7716 if (ufshcd_can_autobkops_during_suspend(hba)) {
7717 /*
7718 * The device is idle with no requests in the queue,
7719 * allow background operations if bkops status shows
7720 * that performance might be impacted.
7721 */
7722 ret = ufshcd_urgent_bkops(hba);
7723 if (ret)
7724 goto enable_gating;
7725 } else {
7726 /* make sure that auto bkops is disabled */
7727 ufshcd_disable_auto_bkops(hba);
7728 }
57d104c1
SJ		 7729 }
7730
7731 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
7732 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
7733 !ufshcd_is_runtime_pm(pm_op))) {
7734 /* ensure that bkops is disabled */
7735 ufshcd_disable_auto_bkops(hba);
7736 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
7737 if (ret)
1ab27c9c 7738 goto enable_gating;
57d104c1
SJ		 7739 }
7740
7741 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
7742 if (ret)
7743 goto set_dev_active;
7744
7745 ufshcd_vreg_set_lpm(hba);
7746
7747disable_clks:
7748 /*
7749 * Call vendor specific suspend callback. As these callbacks may access
7750 * vendor specific host controller register space call them before the
7751 * host clocks are ON.
7752 */
0263bcd0
YG		 7753 ret = ufshcd_vops_suspend(hba, pm_op);
7754 if (ret)
7755 goto set_link_active;
57d104c1 7756
57d104c1
SJ		 7757 if (!ufshcd_is_link_active(hba))
7758 ufshcd_setup_clocks(hba, false);
7759 else
7760 /* If link is active, device ref_clk can't be switched off */
7761 __ufshcd_setup_clocks(hba, false, true);
7762
1ab27c9c 7763 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 7764 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
57d104c1
SJ		 7765 /*
7766 * Disable the host irq as host controller as there won't be any
0263bcd0 7767 * host controller transaction expected till resume.
57d104c1
SJ		 7768 */
7769 ufshcd_disable_irq(hba);
7770 /* Put the host controller in low power mode if possible */
7771 ufshcd_hba_vreg_set_lpm(hba);
7772 goto out;
7773
57d104c1 7774set_link_active:
401f1e44 7775 if (hba->clk_scaling.is_allowed)
7776 ufshcd_resume_clkscaling(hba);
57d104c1
SJ		 7777 ufshcd_vreg_set_hpm(hba);
7778 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
7779 ufshcd_set_link_active(hba);
7780 else if (ufshcd_is_link_off(hba))
7781 ufshcd_host_reset_and_restore(hba);
7782set_dev_active:
7783 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
7784 ufshcd_disable_auto_bkops(hba);
1ab27c9c 7785enable_gating:
401f1e44 7786 if (hba->clk_scaling.is_allowed)
7787 ufshcd_resume_clkscaling(hba);
1ab27c9c
ST		 7788 hba->clk_gating.is_suspended = false;
7789 ufshcd_release(hba);
57d104c1
SJ		 7790out:
7791 hba->pm_op_in_progress = 0;
7792 return ret;
7a3e97b0
SY		 7793}
7794
7795/**
57d104c1 7796 * ufshcd_resume - helper function for resume operations
3b1d0580 7797 * @hba: per adapter instance
57d104c1 7798 * @pm_op: runtime PM or system PM
7a3e97b0 7799 *
57d104c1
SJ		 7800 * This function basically brings the UFS device, UniPro link and controller
7801 * to active state.
7802 *
7803 * Returns 0 for success and non-zero for failure
7a3e97b0 7804 */
57d104c1 7805static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7806{
57d104c1
SJ		 7807 int ret;
7808 enum uic_link_state old_link_state;
7809
7810 hba->pm_op_in_progress = 1;
7811 old_link_state = hba->uic_link_state;
7812
7813 ufshcd_hba_vreg_set_hpm(hba);
7814 /* Make sure clocks are enabled before accessing controller */
7815 ret = ufshcd_setup_clocks(hba, true);
7816 if (ret)
7817 goto out;
7818
57d104c1
SJ		 7819 /* enable the host irq as host controller would be active soon */
7820 ret = ufshcd_enable_irq(hba);
7821 if (ret)
7822 goto disable_irq_and_vops_clks;
7823
7824 ret = ufshcd_vreg_set_hpm(hba);
7825 if (ret)
7826 goto disable_irq_and_vops_clks;
7827
7a3e97b0 7828 /*
57d104c1
SJ		 7829 * Call vendor specific resume callback. As these callbacks may access
7830 * vendor specific host controller register space call them when the
7831 * host clocks are ON.
7a3e97b0 7832 */
0263bcd0
YG		 7833 ret = ufshcd_vops_resume(hba, pm_op);
7834 if (ret)
7835 goto disable_vreg;
57d104c1
SJ		 7836
7837 if (ufshcd_is_link_hibern8(hba)) {
7838 ret = ufshcd_uic_hibern8_exit(hba);
7839 if (!ret)
7840 ufshcd_set_link_active(hba);
7841 else
7842 goto vendor_suspend;
7843 } else if (ufshcd_is_link_off(hba)) {
7844 ret = ufshcd_host_reset_and_restore(hba);
7845 /*
7846 * ufshcd_host_reset_and_restore() should have already
7847 * set the link state as active
7848 */
7849 if (ret || !ufshcd_is_link_active(hba))
7850 goto vendor_suspend;
7851 }
7852
7853 if (!ufshcd_is_ufs_dev_active(hba)) {
7854 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
7855 if (ret)
7856 goto set_old_link_state;
7857 }
7858
4e768e76 7859 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
7860 ufshcd_enable_auto_bkops(hba);
7861 else
7862 /*
7863 * If BKOPs operations are urgently needed at this moment then
7864 * keep auto-bkops enabled or else disable it.
7865 */
7866 ufshcd_urgent_bkops(hba);
7867
1ab27c9c
ST		 7868 hba->clk_gating.is_suspended = false;
7869
fcb0c4b0
ST		 7870 if (hba->clk_scaling.is_allowed)
7871 ufshcd_resume_clkscaling(hba);
856b3483 7872
1ab27c9c
ST		 7873 /* Schedule clock gating in case of no access to UFS device yet */
7874 ufshcd_release(hba);
ad448378
AH		 7875
7876 /* Enable Auto-Hibernate if configured */
7877 ufshcd_auto_hibern8_enable(hba);
7878
57d104c1
SJ		 7879 goto out;
7880
7881set_old_link_state:
7882 ufshcd_link_state_transition(hba, old_link_state, 0);
7883vendor_suspend:
0263bcd0 7884 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 7885disable_vreg:
7886 ufshcd_vreg_set_lpm(hba);
7887disable_irq_and_vops_clks:
7888 ufshcd_disable_irq(hba);
401f1e44 7889 if (hba->clk_scaling.is_allowed)
7890 ufshcd_suspend_clkscaling(hba);
57d104c1
SJ		 7891 ufshcd_setup_clocks(hba, false);
7892out:
7893 hba->pm_op_in_progress = 0;
7894 return ret;
7895}
7896
7897/**
7898 * ufshcd_system_suspend - system suspend routine
7899 * @hba: per adapter instance
57d104c1
SJ		 7900 *
7901 * Check the description of ufshcd_suspend() function for more details.
7902 *
7903 * Returns 0 for success and non-zero for failure
7904 */
7905int ufshcd_system_suspend(struct ufs_hba *hba)
7906{
7907 int ret = 0;
7ff5ab47 7908 ktime_t start = ktime_get();
57d104c1
SJ		 7909
7910 if (!hba || !hba->is_powered)
233b594b 7911 return 0;
57d104c1 7912
0b257734 7913 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
7914 hba->curr_dev_pwr_mode) &&
7915 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
7916 hba->uic_link_state))
7917 goto out;
57d104c1 7918
0b257734 7919 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 7920 /*
7921 * UFS device and/or UFS link low power states during runtime
7922 * suspend seems to be different than what is expected during
7923 * system suspend. Hence runtime resume the devic & link and
7924 * let the system suspend low power states to take effect.
7925 * TODO: If resume takes longer time, we might have optimize
7926 * it in future by not resuming everything if possible.
7927 */
7928 ret = ufshcd_runtime_resume(hba);
7929 if (ret)
7930 goto out;
7931 }
7932
7933 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
7934out:
7ff5ab47 7935 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
7936 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7937 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 7938 if (!ret)
7939 hba->is_sys_suspended = true;
57d104c1
SJ		 7940 return ret;
7941}
7942EXPORT_SYMBOL(ufshcd_system_suspend);
7943
7944/**
7945 * ufshcd_system_resume - system resume routine
7946 * @hba: per adapter instance
7947 *
7948 * Returns 0 for success and non-zero for failure
7949 */
7a3e97b0 7950
57d104c1
SJ		 7951int ufshcd_system_resume(struct ufs_hba *hba)
7952{
7ff5ab47 7953 int ret = 0;
7954 ktime_t start = ktime_get();
7955
e3ce73d6
YG		 7956 if (!hba)
7957 return -EINVAL;
7958
7959 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 7960 /*
7961 * Let the runtime resume take care of resuming
7962 * if runtime suspended.
7963 */
7ff5ab47 7964 goto out;
7965 else
7966 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
7967out:
7968 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
7969 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7970 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 7971 if (!ret)
7972 hba->is_sys_suspended = false;
7ff5ab47 7973 return ret;
7a3e97b0 7974}
57d104c1 7975EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 7976
57d104c1
SJ		 7977/**
7978 * ufshcd_runtime_suspend - runtime suspend routine
7979 * @hba: per adapter instance
7980 *
7981 * Check the description of ufshcd_suspend() function for more details.
7982 *
7983 * Returns 0 for success and non-zero for failure
7984 */
66ec6d59
SRT		 7985int ufshcd_runtime_suspend(struct ufs_hba *hba)
7986{
7ff5ab47 7987 int ret = 0;
7988 ktime_t start = ktime_get();
7989
e3ce73d6
YG		 7990 if (!hba)
7991 return -EINVAL;
7992
7993 if (!hba->is_powered)
7ff5ab47 7994 goto out;
7995 else
7996 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
7997out:
7998 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
7999 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8000 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8001 return ret;
66ec6d59
SRT		 8002}
8003EXPORT_SYMBOL(ufshcd_runtime_suspend);
8004
57d104c1
SJ		 8005/**
8006 * ufshcd_runtime_resume - runtime resume routine
8007 * @hba: per adapter instance
8008 *
8009 * This function basically brings the UFS device, UniPro link and controller
8010 * to active state. Following operations are done in this function:
8011 *
8012 * 1. Turn on all the controller related clocks
8013 * 2. Bring the UniPro link out of Hibernate state
8014 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8015 * to active state.
8016 * 4. If auto-bkops is enabled on the device, disable it.
8017 *
8018 * So following would be the possible power state after this function return
8019 * successfully:
8020 * S1: UFS device in Active state with VCC rail ON
8021 * UniPro link in Active state
8022 * All the UFS/UniPro controller clocks are ON
8023 *
8024 * Returns 0 for success and non-zero for failure
8025 */
66ec6d59
SRT		 8026int ufshcd_runtime_resume(struct ufs_hba *hba)
8027{
7ff5ab47 8028 int ret = 0;
8029 ktime_t start = ktime_get();
8030
e3ce73d6
YG		 8031 if (!hba)
8032 return -EINVAL;
8033
8034 if (!hba->is_powered)
7ff5ab47 8035 goto out;
8036 else
8037 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8038out:
8039 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8040 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8041 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8042 return ret;
66ec6d59
SRT		 8043}
8044EXPORT_SYMBOL(ufshcd_runtime_resume);
8045
8046int ufshcd_runtime_idle(struct ufs_hba *hba)
8047{
8048 return 0;
8049}
8050EXPORT_SYMBOL(ufshcd_runtime_idle);
8051
57d104c1
SJ		 8052/**
8053 * ufshcd_shutdown - shutdown routine
8054 * @hba: per adapter instance
8055 *
8056 * This function would power off both UFS device and UFS link.
8057 *
8058 * Returns 0 always to allow force shutdown even in case of errors.
8059 */
8060int ufshcd_shutdown(struct ufs_hba *hba)
8061{
8062 int ret = 0;
8063
8064 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8065 goto out;
8066
8067 if (pm_runtime_suspended(hba->dev)) {
8068 ret = ufshcd_runtime_resume(hba);
8069 if (ret)
8070 goto out;
8071 }
8072
8073 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8074out:
8075 if (ret)
8076 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8077 /* allow force shutdown even in case of errors */
8078 return 0;
8079}
8080EXPORT_SYMBOL(ufshcd_shutdown);
8081
7a3e97b0 8082/**
3b1d0580 8083 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 8084 * data structure memory
8aa29f19 8085 * @hba: per adapter instance
7a3e97b0 8086 */
3b1d0580 8087void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 8088{
df032bf2 8089 ufs_bsg_remove(hba);
cbb6813e 8090 ufs_sysfs_remove_nodes(hba->dev);
cfdf9c91 8091 scsi_remove_host(hba->host);
7a3e97b0 8092 /* disable interrupts */
2fbd009b 8093 ufshcd_disable_intr(hba, hba->intr_mask);
596585a2 8094 ufshcd_hba_stop(hba, true);
7a3e97b0 8095
eebcc196 8096 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8097 ufshcd_exit_clk_gating(hba);
fcb0c4b0
ST		 8098 if (ufshcd_is_clkscaling_supported(hba))
8099 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
aa497613 8100 ufshcd_hba_exit(hba);
3b1d0580
VH		 8101}
8102EXPORT_SYMBOL_GPL(ufshcd_remove);
8103
47555a5c
YG		 8104/**
8105 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8106 * @hba: pointer to Host Bus Adapter (HBA)
8107 */
8108void ufshcd_dealloc_host(struct ufs_hba *hba)
8109{
8110 scsi_host_put(hba->host);
8111}
8112EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8113
ca3d7bf9
AM		 8114/**
8115 * ufshcd_set_dma_mask - Set dma mask based on the controller
8116 * addressing capability
8117 * @hba: per adapter instance
8118 *
8119 * Returns 0 for success, non-zero for failure
8120 */
8121static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8122{
8123 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8124 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8125 return 0;
8126 }
8127 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8128}
8129
7a3e97b0 8130/**
5c0c28a8 8131 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 8132 * @dev: pointer to device handle
8133 * @hba_handle: driver private handle
7a3e97b0
SY		 8134 * Returns 0 on success, non-zero value on failure
8135 */
5c0c28a8 8136int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 8137{
8138 struct Scsi_Host *host;
8139 struct ufs_hba *hba;
5c0c28a8 8140 int err = 0;
7a3e97b0 8141
3b1d0580
VH		 8142 if (!dev) {
8143 dev_err(dev,
8144 "Invalid memory reference for dev is NULL\n");
8145 err = -ENODEV;
7a3e97b0
SY		 8146 goto out_error;
8147 }
8148
7a3e97b0
SY		 8149 host = scsi_host_alloc(&ufshcd_driver_template,
8150 sizeof(struct ufs_hba));
8151 if (!host) {
3b1d0580 8152 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 8153 err = -ENOMEM;
3b1d0580 8154 goto out_error;
7a3e97b0
SY		 8155 }
8156 hba = shost_priv(host);
7a3e97b0 8157 hba->host = host;
3b1d0580 8158 hba->dev = dev;
5c0c28a8 8159 *hba_handle = hba;
9e1e8a75 8160 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 8161
566ec9ad
SM		 8162 INIT_LIST_HEAD(&hba->clk_list_head);
8163
5c0c28a8
SRT		 8164out_error:
8165 return err;
8166}
8167EXPORT_SYMBOL(ufshcd_alloc_host);
8168
8169/**
8170 * ufshcd_init - Driver initialization routine
8171 * @hba: per-adapter instance
8172 * @mmio_base: base register address
8173 * @irq: Interrupt line of device
8174 * Returns 0 on success, non-zero value on failure
8175 */
8176int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
8177{
8178 int err;
8179 struct Scsi_Host *host = hba->host;
8180 struct device *dev = hba->dev;
8181
8182 if (!mmio_base) {
8183 dev_err(hba->dev,
8184 "Invalid memory reference for mmio_base is NULL\n");
8185 err = -ENODEV;
8186 goto out_error;
8187 }
8188
3b1d0580
VH		 8189 hba->mmio_base = mmio_base;
8190 hba->irq = irq;
7a3e97b0 8191
a4b0e8a4
PM		 8192 /* Set descriptor lengths to specification defaults */
8193 ufshcd_def_desc_sizes(hba);
8194
aa497613 8195 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 8196 if (err)
8197 goto out_error;
8198
7a3e97b0
SY		 8199 /* Read capabilities registers */
8200 ufshcd_hba_capabilities(hba);
8201
8202 /* Get UFS version supported by the controller */
8203 hba->ufs_version = ufshcd_get_ufs_version(hba);
8204
c01848c6
YG		 8205 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
8206 (hba->ufs_version != UFSHCI_VERSION_11) &&
8207 (hba->ufs_version != UFSHCI_VERSION_20) &&
8208 (hba->ufs_version != UFSHCI_VERSION_21))
8209 dev_err(hba->dev, "invalid UFS version 0x%x\n",
8210 hba->ufs_version);
8211
2fbd009b
SJ		 8212 /* Get Interrupt bit mask per version */
8213 hba->intr_mask = ufshcd_get_intr_mask(hba);
8214
ca3d7bf9
AM		 8215 err = ufshcd_set_dma_mask(hba);
8216 if (err) {
8217 dev_err(hba->dev, "set dma mask failed\n");
8218 goto out_disable;
8219 }
8220
7a3e97b0
SY		 8221 /* Allocate memory for host memory space */
8222 err = ufshcd_memory_alloc(hba);
8223 if (err) {
3b1d0580
VH		 8224 dev_err(hba->dev, "Memory allocation failed\n");
8225 goto out_disable;
7a3e97b0
SY		 8226 }
8227
8228 /* Configure LRB */
8229 ufshcd_host_memory_configure(hba);
8230
8231 host->can_queue = hba->nutrs;
8232 host->cmd_per_lun = hba->nutrs;
8233 host->max_id = UFSHCD_MAX_ID;
0ce147d4 8234 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 8235 host->max_channel = UFSHCD_MAX_CHANNEL;
8236 host->unique_id = host->host_no;
a851b2bd 8237 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 8238
7eb584db
DR		 8239 hba->max_pwr_info.is_valid = false;
8240
7a3e97b0 8241 /* Initailize wait queue for task management */
e2933132
SRT		 8242 init_waitqueue_head(&hba->tm_wq);
8243 init_waitqueue_head(&hba->tm_tag_wq);
7a3e97b0
SY		 8244
8245 /* Initialize work queues */
e8e7f271 8246 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 8247 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 8248
6ccf44fe
SJ		 8249 /* Initialize UIC command mutex */
8250 mutex_init(&hba->uic_cmd_mutex);
8251
5a0b0cb9
SRT		 8252 /* Initialize mutex for device management commands */
8253 mutex_init(&hba->dev_cmd.lock);
8254
a3cd5ec5 8255 init_rwsem(&hba->clk_scaling_lock);
8256
5a0b0cb9
SRT		 8257 /* Initialize device management tag acquire wait queue */
8258 init_waitqueue_head(&hba->dev_cmd.tag_wq);
8259
1ab27c9c 8260 ufshcd_init_clk_gating(hba);
199ef13c 8261
eebcc196
VG		 8262 ufshcd_init_clk_scaling(hba);
8263
199ef13c
YG		 8264 /*
8265 * In order to avoid any spurious interrupt immediately after
8266 * registering UFS controller interrupt handler, clear any pending UFS
8267 * interrupt status and disable all the UFS interrupts.
8268 */
8269 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
8270 REG_INTERRUPT_STATUS);
8271 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
8272 /*
8273 * Make sure that UFS interrupts are disabled and any pending interrupt
8274 * status is cleared before registering UFS interrupt handler.
8275 */
8276 mb();
8277
7a3e97b0 8278 /* IRQ registration */
2953f850 8279 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 8280 if (err) {
3b1d0580 8281 dev_err(hba->dev, "request irq failed\n");
1ab27c9c 8282 goto exit_gating;
57d104c1
SJ		 8283 } else {
8284 hba->is_irq_enabled = true;
7a3e97b0
SY		 8285 }
8286
3b1d0580 8287 err = scsi_add_host(host, hba->dev);
7a3e97b0 8288 if (err) {
3b1d0580 8289 dev_err(hba->dev, "scsi_add_host failed\n");
1ab27c9c 8290 goto exit_gating;
7a3e97b0
SY		 8291 }
8292
6ccf44fe
SJ		 8293 /* Host controller enable */
8294 err = ufshcd_hba_enable(hba);
7a3e97b0 8295 if (err) {
6ccf44fe 8296 dev_err(hba->dev, "Host controller enable failed\n");
66cc820f 8297 ufshcd_print_host_regs(hba);
6ba65588 8298 ufshcd_print_host_state(hba);
3b1d0580 8299 goto out_remove_scsi_host;
7a3e97b0 8300 }
6ccf44fe 8301
0c8f7586 8302 /*
8303 * Set the default power management level for runtime and system PM.
8304 * Default power saving mode is to keep UFS link in Hibern8 state
8305 * and UFS device in sleep state.
8306 */
8307 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8308 UFS_SLEEP_PWR_MODE,
8309 UIC_LINK_HIBERN8_STATE);
8310 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8311 UFS_SLEEP_PWR_MODE,
8312 UIC_LINK_HIBERN8_STATE);
8313
ad448378
AH		 8314 /* Set the default auto-hiberate idle timer value to 150 ms */
8315 if (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) {
8316 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
8317 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
8318 }
8319
62694735
SRT		 8320 /* Hold auto suspend until async scan completes */
8321 pm_runtime_get_sync(dev);
38135535 8322 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 8323 /*
7caf489b 8324 * We are assuming that device wasn't put in sleep/power-down
8325 * state exclusively during the boot stage before kernel.
8326 * This assumption helps avoid doing link startup twice during
8327 * ufshcd_probe_hba().
57d104c1 8328 */
7caf489b 8329 ufshcd_set_ufs_dev_active(hba);
57d104c1 8330
6ccf44fe 8331 async_schedule(ufshcd_async_scan, hba);
cbb6813e 8332 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 8333
7a3e97b0
SY		 8334 return 0;
8335
3b1d0580
VH		 8336out_remove_scsi_host:
8337 scsi_remove_host(hba->host);
1ab27c9c 8338exit_gating:
eebcc196 8339 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8340 ufshcd_exit_clk_gating(hba);
3b1d0580 8341out_disable:
57d104c1 8342 hba->is_irq_enabled = false;
aa497613 8343 ufshcd_hba_exit(hba);
3b1d0580
VH		 8344out_error:
8345 return err;
8346}
8347EXPORT_SYMBOL_GPL(ufshcd_init);
8348
3b1d0580
VH		 8349MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
8350MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 8351MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 8352MODULE_LICENSE("GPL");
8353MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Linux 6.x block layer and io_uring tree(s)