projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mmc: sdhci-pci-gli: Fix GL9763E runtime PM when the system resumes from suspend
[linux-block.git] / drivers / mmc / core / block.c
CommitLineData
7c45b226 1// SPDX-License-Identifier: GPL-2.0
1da177e4
LT		 2/*
3 * Block driver for media (i.e., flash cards)
4 *
5 * Copyright 2002 Hewlett-Packard Company
979ce720 6 * Copyright 2005-2008 Pierre Ossman
1da177e4
LT		 7 *
8 * Use consistent with the GNU GPL is permitted,
9 * provided that this copyright notice is
10 * preserved in its entirety in all copies and derived works.
11 *
12 * HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED,
13 * AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS
14 * FITNESS FOR ANY PARTICULAR PURPOSE.
15 *
16 * Many thanks to Alessandro Rubini and Jonathan Corbet!
17 *
18 * Author: Andrew Christian
19 * 28 May 2002
20 */
21#include <linux/moduleparam.h>
22#include <linux/module.h>
23#include <linux/init.h>
24
1da177e4
LT		 25#include <linux/kernel.h>
26#include <linux/fs.h>
5a0e3ad6 27#include <linux/slab.h>
1da177e4
LT		 28#include <linux/errno.h>
29#include <linux/hdreg.h>
30#include <linux/kdev_t.h>
edb25572 31#include <linux/kref.h>
1da177e4 32#include <linux/blkdev.h>
97548575 33#include <linux/cdev.h>
a621aaed 34#include <linux/mutex.h>
ec5a19dd 35#include <linux/scatterlist.h>
a7bbb573 36#include <linux/string_helpers.h>
cb87ea28
JC		 37#include <linux/delay.h>
38#include <linux/capability.h>
39#include <linux/compat.h>
e94cfef6 40#include <linux/pm_runtime.h>
b10fa99e 41#include <linux/idr.h>
627c3ccf 42#include <linux/debugfs.h>
1da177e4 43
cb87ea28 44#include <linux/mmc/ioctl.h>
1da177e4 45#include <linux/mmc/card.h>
385e3227 46#include <linux/mmc/host.h>
da7fbe58
PO		 47#include <linux/mmc/mmc.h>
48#include <linux/mmc/sd.h>
1da177e4 49
7c0f6ba6 50#include <linux/uaccess.h>
1da177e4 51
98ac2162 52#include "queue.h"
48ab086d 53#include "block.h"
55244c56 54#include "core.h"
4facdde1 55#include "card.h"
93f1c150 56#include "crypto.h"
5857b29b 57#include "host.h"
4facdde1 58#include "bus.h"
55244c56 59#include "mmc_ops.h"
28fc64af 60#include "quirks.h"
55244c56 61#include "sd_ops.h"
1da177e4 62
6b0b6285 63MODULE_ALIAS("mmc:block");
5e71b7a6
OJ		 64#ifdef MODULE_PARAM_PREFIX
65#undef MODULE_PARAM_PREFIX
66#endif
67#define MODULE_PARAM_PREFIX "mmcblk."
68
6b7a363d
AH		 69/*
70 * Set a 10 second timeout for polling write request busy state. Note, mmc core
71 * is setting a 3 second timeout for SD cards, and SDHCI has long had a 10
72 * second software timer to timeout the whole request, so 10 seconds should be
73 * ample.
74 */
75#define MMC_BLK_TIMEOUT_MS (10 * 1000)
775a9362 76#define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
a0e95766 77#define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8)
6a7a6b45 78
d3df0465 79#define mmc_req_rel_wr(req) ((req->cmd_flags & REQ_FUA) && \
ce39f9d1 80 (rq_data_dir(req) == WRITE))
5e71b7a6 81static DEFINE_MUTEX(block_mutex);
6b0b6285 82
1da177e4 83/*
5e71b7a6
OJ		 84 * The defaults come from config options but can be overriden by module
85 * or bootarg options.
1da177e4 86 */
5e71b7a6 87static int perdev_minors = CONFIG_MMC_BLOCK_MINORS;
1dff3144 88
5e71b7a6
OJ		 89/*
90 * We've only got one major, so number of mmcblk devices is
a26eba61 91 * limited to (1 << 20) / number of minors per device. It is also
b10fa99e 92 * limited by the MAX_DEVICES below.
5e71b7a6
OJ		 93 */
94static int max_devices;
95
a26eba61
BH		 96#define MAX_DEVICES 256
97
b10fa99e 98static DEFINE_IDA(mmc_blk_ida);
97548575 99static DEFINE_IDA(mmc_rpmb_ida);
1da177e4 100
6966e609
UH		 101struct mmc_blk_busy_data {
102 struct mmc_card *card;
103 u32 status;
104};
105
1da177e4
LT		 106/*
107 * There is one mmc_blk_data per slot.
108 */
109struct mmc_blk_data {
307d8e6f 110 struct device *parent;
1da177e4
LT		 111 struct gendisk *disk;
112 struct mmc_queue queue;
371a689f 113 struct list_head part;
97548575 114 struct list_head rpmbs;
1da177e4 115
d0c97cfb
AW		 116 unsigned int flags;
117#define MMC_BLK_CMD23 (1 << 0) /* Can do SET_BLOCK_COUNT for multiblock */
118#define MMC_BLK_REL_WR (1 << 1) /* MMC Reliable write support */
119
edb25572 120 struct kref kref;
a6f6c96b 121 unsigned int read_only;
371a689f 122 unsigned int part_type;
67716327
AH		 123 unsigned int reset_done;
124#define MMC_BLK_READ BIT(0)
125#define MMC_BLK_WRITE BIT(1)
126#define MMC_BLK_DISCARD BIT(2)
127#define MMC_BLK_SECDISCARD BIT(3)
1e8e55b6 128#define MMC_BLK_CQE_RECOVERY BIT(4)
f7b6fc32 129#define MMC_BLK_TRIM BIT(5)
371a689f
AW		 130
131 /*
132 * Only set in main mmc_blk_data associated
fc95e30b 133 * with mmc_card with dev_set_drvdata, and keeps
371a689f
AW		 134 * track of the current selected device partition.
135 */
136 unsigned int part_curr;
add710ea 137 int area_type;
f9f0da98
AH		 138
139 /* debugfs files (only in main mmc_blk_data) */
140 struct dentry *status_dentry;
141 struct dentry *ext_csd_dentry;
1da177e4
LT		 142};
143
97548575
LW		 144/* Device type for RPMB character devices */
145static dev_t mmc_rpmb_devt;
146
147/* Bus type for RPMB character devices */
148static struct bus_type mmc_rpmb_bus_type = {
149 .name = "mmc_rpmb",
150};
151
152/**
153 * struct mmc_rpmb_data - special RPMB device type for these areas
154 * @dev: the device for the RPMB area
155 * @chrdev: character device for the RPMB area
156 * @id: unique device ID number
157 * @part_index: partition index (0 on first)
158 * @md: parent MMC block device
159 * @node: list item, so we can put this device on a list
160 */
161struct mmc_rpmb_data {
162 struct device dev;
163 struct cdev chrdev;
164 int id;
165 unsigned int part_index;
166 struct mmc_blk_data *md;
167 struct list_head node;
168};
169
a621aaed 170static DEFINE_MUTEX(open_lock);
1da177e4 171
5e71b7a6
OJ		 172module_param(perdev_minors, int, 0444);
173MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");
174
8d1e977d 175static inline int mmc_blk_part_switch(struct mmc_card *card,
1f797edc 176 unsigned int part_type);
511ce378
BW		 177static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
178 struct mmc_card *card,
179 int disable_multi,
180 struct mmc_queue *mq);
181static void mmc_blk_hsq_req_done(struct mmc_request *mrq);
cdf8a6fb 182
1da177e4
LT		 183static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk)
184{
185 struct mmc_blk_data *md;
186
a621aaed 187 mutex_lock(&open_lock);
1da177e4 188 md = disk->private_data;
edb25572 189 if (md && !kref_get_unless_zero(&md->kref))
1da177e4 190 md = NULL;
a621aaed 191 mutex_unlock(&open_lock);
1da177e4
LT		 192
193 return md;
194}
195
371a689f
AW		 196static inline int mmc_get_devidx(struct gendisk *disk)
197{
382c55f8 198 int devidx = disk->first_minor / perdev_minors;
371a689f
AW		 199 return devidx;
200}
201
edb25572 202static void mmc_blk_kref_release(struct kref *ref)
1da177e4 203{
edb25572
SB		 204 struct mmc_blk_data *md = container_of(ref, struct mmc_blk_data, kref);
205 int devidx;
249cda33 206
edb25572
SB		 207 devidx = mmc_get_devidx(md->disk);
208 ida_simple_remove(&mmc_blk_ida, devidx);
209
210 mutex_lock(&open_lock);
211 md->disk->private_data = NULL;
a621aaed 212 mutex_unlock(&open_lock);
edb25572
SB		 213
214 put_disk(md->disk);
215 kfree(md);
216}
217
218static void mmc_blk_put(struct mmc_blk_data *md)
219{
220 kref_put(&md->kref, mmc_blk_kref_release);
1da177e4
LT		 221}
222
add710ea
JR		 223static ssize_t power_ro_lock_show(struct device *dev,
224 struct device_attribute *attr, char *buf)
225{
226 int ret;
227 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
228 struct mmc_card *card = md->queue.card;
229 int locked = 0;
230
231 if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN)
232 locked = 2;
233 else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN)
234 locked = 1;
235
236 ret = snprintf(buf, PAGE_SIZE, "%d\n", locked);
237
9098f84c
TW		 238 mmc_blk_put(md);
239
add710ea
JR		 240 return ret;
241}
242
243static ssize_t power_ro_lock_store(struct device *dev,
244 struct device_attribute *attr, const char *buf, size_t count)
245{
246 int ret;
247 struct mmc_blk_data *md, *part_md;
0493f6fe
LW		 248 struct mmc_queue *mq;
249 struct request *req;
add710ea
JR		 250 unsigned long set;
251
252 if (kstrtoul(buf, 0, &set))
253 return -EINVAL;
254
255 if (set != 1)
256 return count;
257
258 md = mmc_blk_get(dev_to_disk(dev));
0493f6fe 259 mq = &md->queue;
add710ea 260
0493f6fe 261 /* Dispatch locking to the block layer */
0bf6d96c 262 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0);
fb8e456e
AH		 263 if (IS_ERR(req)) {
264 count = PTR_ERR(req);
265 goto out_put;
266 }
0493f6fe 267 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;
b84ba30b 268 blk_execute_rq(req, false);
0493f6fe 269 ret = req_to_mmc_queue_req(req)->drv_op_result;
0bf6d96c 270 blk_mq_free_request(req);
add710ea
JR		 271
272 if (!ret) {
273 pr_info("%s: Locking boot partition ro until next power on\n",
274 md->disk->disk_name);
275 set_disk_ro(md->disk, 1);
276
277 list_for_each_entry(part_md, &md->part, part)
278 if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) {
279 pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name);
280 set_disk_ro(part_md->disk, 1);
281 }
282 }
fb8e456e 283out_put:
add710ea
JR		 284 mmc_blk_put(md);
285 return count;
286}
287
29e6a5e0
CH		 288static DEVICE_ATTR(ro_lock_until_next_power_on, 0,
289 power_ro_lock_show, power_ro_lock_store);
290
371a689f
AW		 291static ssize_t force_ro_show(struct device *dev, struct device_attribute *attr,
292 char *buf)
293{
294 int ret;
295 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
296
0031a98a 297 ret = snprintf(buf, PAGE_SIZE, "%d\n",
371a689f
AW		 298 get_disk_ro(dev_to_disk(dev)) ^
299 md->read_only);
300 mmc_blk_put(md);
301 return ret;
302}
303
304static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr,
305 const char *buf, size_t count)
306{
307 int ret;
308 char *end;
309 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
310 unsigned long set = simple_strtoul(buf, &end, 0);
311 if (end == buf) {
312 ret = -EINVAL;
313 goto out;
314 }
315
316 set_disk_ro(dev_to_disk(dev), set || md->read_only);
317 ret = count;
318out:
319 mmc_blk_put(md);
320 return ret;
321}
322
29e6a5e0
CH		 323static DEVICE_ATTR(force_ro, 0644, force_ro_show, force_ro_store);
324
325static struct attribute *mmc_disk_attrs[] = {
326 &dev_attr_force_ro.attr,
327 &dev_attr_ro_lock_until_next_power_on.attr,
328 NULL,
329};
330
331static umode_t mmc_disk_attrs_is_visible(struct kobject *kobj,
332 struct attribute *a, int n)
333{
0e8bb666 334 struct device *dev = kobj_to_dev(kobj);
29e6a5e0
CH		 335 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
336 umode_t mode = a->mode;
337
338 if (a == &dev_attr_ro_lock_until_next_power_on.attr &&
339 (md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
340 md->queue.card->ext_csd.boot_ro_lockable) {
341 mode = S_IRUGO;
342 if (!(md->queue.card->ext_csd.boot_ro_lock &
343 EXT_CSD_BOOT_WP_B_PWR_WP_DIS))
344 mode |= S_IWUSR;
345 }
346
347 mmc_blk_put(md);
348 return mode;
349}
350
351static const struct attribute_group mmc_disk_attr_group = {
352 .is_visible = mmc_disk_attrs_is_visible,
353 .attrs = mmc_disk_attrs,
354};
355
356static const struct attribute_group *mmc_disk_attr_groups[] = {
357 &mmc_disk_attr_group,
358 NULL,
359};
360
a5a1561f 361static int mmc_blk_open(struct block_device *bdev, fmode_t mode)
1da177e4 362{
a5a1561f 363 struct mmc_blk_data *md = mmc_blk_get(bdev->bd_disk);
1da177e4
LT		 364 int ret = -ENXIO;
365
2a48fc0a 366 mutex_lock(&block_mutex);
1da177e4 367 if (md) {
1da177e4 368 ret = 0;
a5a1561f 369 if ((mode & FMODE_WRITE) && md->read_only) {
70bb0896 370 mmc_blk_put(md);
a00fc090 371 ret = -EROFS;
70bb0896 372 }
1da177e4 373 }
2a48fc0a 374 mutex_unlock(&block_mutex);
1da177e4
LT		 375
376 return ret;
377}
378
db2a144b 379static void mmc_blk_release(struct gendisk *disk, fmode_t mode)
1da177e4 380{
a5a1561f 381 struct mmc_blk_data *md = disk->private_data;
1da177e4 382
2a48fc0a 383 mutex_lock(&block_mutex);
1da177e4 384 mmc_blk_put(md);
2a48fc0a 385 mutex_unlock(&block_mutex);
1da177e4
LT		 386}
387
388static int
a885c8c4 389mmc_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1da177e4 390{
a885c8c4
CH		 391 geo->cylinders = get_capacity(bdev->bd_disk) / (4 * 16);
392 geo->heads = 4;
393 geo->sectors = 16;
394 return 0;
1da177e4
LT		 395}
396
cb87ea28
JC		 397struct mmc_blk_ioc_data {
398 struct mmc_ioc_cmd ic;
399 unsigned char *buf;
400 u64 buf_bytes;
97548575 401 struct mmc_rpmb_data *rpmb;
cb87ea28
JC		 402};
403
404static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user(
405 struct mmc_ioc_cmd __user *user)
406{
407 struct mmc_blk_ioc_data *idata;
408 int err;
409
1ff8950c 410 idata = kmalloc(sizeof(*idata), GFP_KERNEL);
cb87ea28
JC		 411 if (!idata) {
412 err = -ENOMEM;
aea253ec 413 goto out;
cb87ea28
JC		 414 }
415
416 if (copy_from_user(&idata->ic, user, sizeof(idata->ic))) {
417 err = -EFAULT;
aea253ec 418 goto idata_err;
cb87ea28
JC		 419 }
420
421 idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks;
422 if (idata->buf_bytes > MMC_IOC_MAX_BYTES) {
423 err = -EOVERFLOW;
aea253ec 424 goto idata_err;
cb87ea28
JC		 425 }
426
bfe5b1b1
VV		 427 if (!idata->buf_bytes) {
428 idata->buf = NULL;
4d6144de 429 return idata;
bfe5b1b1 430 }
4d6144de 431
97a0c313
ME		 432 idata->buf = memdup_user((void __user *)(unsigned long)
433 idata->ic.data_ptr, idata->buf_bytes);
434 if (IS_ERR(idata->buf)) {
435 err = PTR_ERR(idata->buf);
aea253ec 436 goto idata_err;
cb87ea28
JC		 437 }
438
cb87ea28
JC		 439 return idata;
440
aea253ec 441idata_err:
cb87ea28 442 kfree(idata);
aea253ec 443out:
cb87ea28 444 return ERR_PTR(err);
cb87ea28
JC		 445}
446
a5f5774c
JH		 447static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr,
448 struct mmc_blk_ioc_data *idata)
449{
450 struct mmc_ioc_cmd *ic = &idata->ic;
451
452 if (copy_to_user(&(ic_ptr->response), ic->response,
453 sizeof(ic->response)))
454 return -EFAULT;
455
456 if (!idata->ic.write_flag) {
457 if (copy_to_user((void __user *)(unsigned long)ic->data_ptr,
458 idata->buf, idata->buf_bytes))
459 return -EFAULT;
460 }
461
462 return 0;
463}
464
a5f5774c
JH		 465static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
466 struct mmc_blk_ioc_data *idata)
cb87ea28 467{
a44f7cb9 468 struct mmc_command cmd = {}, sbc = {};
c7836d15
MY		 469 struct mmc_data data = {};
470 struct mmc_request mrq = {};
cb87ea28
JC		 471 struct scatterlist sg;
472 int err;
97548575 473 unsigned int target_part;
cb87ea28 474
a5f5774c
JH		 475 if (!card || !md || !idata)
476 return -EINVAL;
cb87ea28 477
97548575
LW		 478 /*
479 * The RPMB accesses comes in from the character device, so we
480 * need to target these explicitly. Else we just target the
481 * partition type for the block device the ioctl() was issued
482 * on.
483 */
484 if (idata->rpmb) {
485 /* Support multiple RPMB partitions */
486 target_part = idata->rpmb->part_index;
487 target_part |= EXT_CSD_PART_CONFIG_ACC_RPMB;
488 } else {
489 target_part = md->part_type;
490 }
8d1e977d 491
4d6144de
JR		 492 cmd.opcode = idata->ic.opcode;
493 cmd.arg = idata->ic.arg;
494 cmd.flags = idata->ic.flags;
495
496 if (idata->buf_bytes) {
497 data.sg = &sg;
498 data.sg_len = 1;
499 data.blksz = idata->ic.blksz;
500 data.blocks = idata->ic.blocks;
501
502 sg_init_one(data.sg, idata->buf, idata->buf_bytes);
503
504 if (idata->ic.write_flag)
505 data.flags = MMC_DATA_WRITE;
506 else
507 data.flags = MMC_DATA_READ;
508
509 /* data.flags must already be set before doing this. */
510 mmc_set_data_timeout(&data, card);
511
512 /* Allow overriding the timeout_ns for empirical tuning. */
513 if (idata->ic.data_timeout_ns)
514 data.timeout_ns = idata->ic.data_timeout_ns;
515
516 if ((cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B) {
517 /*
518 * Pretend this is a data transfer and rely on the
519 * host driver to compute timeout. When all host
520 * drivers support cmd.cmd_timeout for R1B, this
521 * can be changed to:
522 *
523 * mrq.data = NULL;
524 * cmd.cmd_timeout = idata->ic.cmd_timeout_ms;
525 */
526 data.timeout_ns = idata->ic.cmd_timeout_ms * 1000000;
527 }
528
529 mrq.data = &data;
530 }
531
532 mrq.cmd = &cmd;
533
97548575 534 err = mmc_blk_part_switch(card, target_part);
8d1e977d 535 if (err)
a5f5774c 536 return err;
8d1e977d 537
cb87ea28
JC		 538 if (idata->ic.is_acmd) {
539 err = mmc_app_cmd(card->host, card);
540 if (err)
a5f5774c 541 return err;
cb87ea28
JC		 542 }
543
97548575 544 if (idata->rpmb) {
a44f7cb9
WS		 545 sbc.opcode = MMC_SET_BLOCK_COUNT;
546 /*
547 * We don't do any blockcount validation because the max size
548 * may be increased by a future standard. We just copy the
549 * 'Reliable Write' bit here.
550 */
551 sbc.arg = data.blocks | (idata->ic.write_flag & BIT(31));
552 sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
553 mrq.sbc = &sbc;
8d1e977d
LP		 554 }
555
a82e484e 556 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) &&
55c2b8b9 557 (cmd.opcode == MMC_SWITCH))
4f111d04 558 return mmc_sanitize(card, idata->ic.cmd_timeout_ms);
775a9362 559
cb87ea28 560 mmc_wait_for_req(card->host, &mrq);
e72a55f2 561 memcpy(&idata->ic.response, cmd.resp, sizeof(cmd.resp));
cb87ea28
JC		 562
563 if (cmd.error) {
564 dev_err(mmc_dev(card->host), "%s: cmd error %d\n",
565 __func__, cmd.error);
a5f5774c 566 return cmd.error;
cb87ea28
JC		 567 }
568 if (data.error) {
569 dev_err(mmc_dev(card->host), "%s: data error %d\n",
570 __func__, data.error);
a5f5774c 571 return data.error;
cb87ea28
JC		 572 }
573
a0e95766
BS		 574 /*
575 * Make sure the cache of the PARTITION_CONFIG register and
576 * PARTITION_ACCESS bits is updated in case the ioctl ext_csd write
577 * changed it successfully.
578 */
579 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) &&
580 (cmd.opcode == MMC_SWITCH)) {
581 struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
582 u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg);
583
584 /*
585 * Update cache so the next mmc_blk_part_switch call operates
586 * on up-to-date data.
587 */
588 card->ext_csd.part_config = value;
589 main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK;
590 }
591
aea0440a
AA		 592 /*
593 * Make sure to update CACHE_CTRL in case it was changed. The cache
594 * will get turned back on if the card is re-initialized, e.g.
595 * suspend/resume or hw reset in recovery.
596 */
597 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) &&
598 (cmd.opcode == MMC_SWITCH)) {
599 u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1;
600
601 card->ext_csd.cache_ctrl = value;
602 }
603
cb87ea28
JC		 604 /*
605 * According to the SD specs, some commands require a delay after
606 * issuing the command.
607 */
608 if (idata->ic.postsleep_min_us)
609 usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us);
610
6246d7c9 611 if (idata->rpmb || (cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B) {
8d1e977d 612 /*
23e09be2
BH		 613 * Ensure RPMB/R1B command has completed by polling CMD13 "Send Status". Here we
614 * allow to override the default timeout value if a custom timeout is specified.
8d1e977d 615 */
23e09be2
BH		 616 err = mmc_poll_for_busy(card, idata->ic.cmd_timeout_ms ? : MMC_BLK_TIMEOUT_MS,
617 false, MMC_BUSY_IO);
8d1e977d
LP		 618 }
619
a5f5774c
JH		 620 return err;
621}
622
2fe20bae 623static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
97548575
LW		 624 struct mmc_ioc_cmd __user *ic_ptr,
625 struct mmc_rpmb_data *rpmb)
a5f5774c
JH		 626{
627 struct mmc_blk_ioc_data *idata;
3ecd8cf2 628 struct mmc_blk_ioc_data *idatas[1];
614f0388 629 struct mmc_queue *mq;
a5f5774c 630 struct mmc_card *card;
b093410c 631 int err = 0, ioc_err = 0;
614f0388 632 struct request *req;
a5f5774c
JH		 633
634 idata = mmc_blk_ioctl_copy_from_user(ic_ptr);
635 if (IS_ERR(idata))
636 return PTR_ERR(idata);
97548575
LW		 637 /* This will be NULL on non-RPMB ioctl():s */
638 idata->rpmb = rpmb;
a5f5774c 639
a5f5774c
JH		 640 card = md->queue.card;
641 if (IS_ERR(card)) {
642 err = PTR_ERR(card);
643 goto cmd_done;
644 }
645
614f0388
LW		 646 /*
647 * Dispatch the ioctl() into the block request queue.
648 */
649 mq = &md->queue;
0bf6d96c 650 req = blk_mq_alloc_request(mq->queue,
ff005a06 651 idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
fb8e456e
AH		 652 if (IS_ERR(req)) {
653 err = PTR_ERR(req);
654 goto cmd_done;
655 }
3ecd8cf2 656 idatas[0] = idata;
97548575
LW		 657 req_to_mmc_queue_req(req)->drv_op =
658 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
69f7599e 659 req_to_mmc_queue_req(req)->drv_op_data = idatas;
3ecd8cf2 660 req_to_mmc_queue_req(req)->ioc_count = 1;
b84ba30b 661 blk_execute_rq(req, false);
0493f6fe 662 ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
b093410c 663 err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
0bf6d96c 664 blk_mq_free_request(req);
a5f5774c 665
cb87ea28 666cmd_done:
cb87ea28
JC		 667 kfree(idata->buf);
668 kfree(idata);
b093410c 669 return ioc_err ? ioc_err : err;
cb87ea28
JC		 670}
671
2fe20bae 672static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
97548575
LW		 673 struct mmc_ioc_multi_cmd __user *user,
674 struct mmc_rpmb_data *rpmb)
a5f5774c
JH		 675{
676 struct mmc_blk_ioc_data **idata = NULL;
677 struct mmc_ioc_cmd __user *cmds = user->cmds;
678 struct mmc_card *card;
3ecd8cf2 679 struct mmc_queue *mq;
103da066 680 int err = 0, ioc_err = 0;
a5f5774c 681 __u64 num_of_cmds;
103da066 682 unsigned int i, n;
3ecd8cf2 683 struct request *req;
a5f5774c
JH		 684
685 if (copy_from_user(&num_of_cmds, &user->num_of_cmds,
686 sizeof(num_of_cmds)))
687 return -EFAULT;
688
aab2ee03
GU		 689 if (!num_of_cmds)
690 return 0;
691
a5f5774c
JH		 692 if (num_of_cmds > MMC_IOC_MAX_CMDS)
693 return -EINVAL;
694
103da066
SS		 695 n = num_of_cmds;
696 idata = kcalloc(n, sizeof(*idata), GFP_KERNEL);
a5f5774c
JH		 697 if (!idata)
698 return -ENOMEM;
699
103da066 700 for (i = 0; i < n; i++) {
a5f5774c
JH		 701 idata[i] = mmc_blk_ioctl_copy_from_user(&cmds[i]);
702 if (IS_ERR(idata[i])) {
703 err = PTR_ERR(idata[i]);
103da066 704 n = i;
a5f5774c
JH		 705 goto cmd_err;
706 }
97548575
LW		 707 /* This will be NULL on non-RPMB ioctl():s */
708 idata[i]->rpmb = rpmb;
a5f5774c
JH		 709 }
710
a5f5774c
JH		 711 card = md->queue.card;
712 if (IS_ERR(card)) {
713 err = PTR_ERR(card);
2fe20bae 714 goto cmd_err;
a5f5774c
JH		 715 }
716
a5f5774c 717
3ecd8cf2
LW		 718 /*
719 * Dispatch the ioctl()s into the block request queue.
720 */
721 mq = &md->queue;
0bf6d96c 722 req = blk_mq_alloc_request(mq->queue,
ff005a06 723 idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
fb8e456e
AH		 724 if (IS_ERR(req)) {
725 err = PTR_ERR(req);
726 goto cmd_err;
727 }
97548575
LW		 728 req_to_mmc_queue_req(req)->drv_op =
729 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
69f7599e 730 req_to_mmc_queue_req(req)->drv_op_data = idata;
103da066 731 req_to_mmc_queue_req(req)->ioc_count = n;
b84ba30b 732 blk_execute_rq(req, false);
0493f6fe 733 ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
a5f5774c
JH		 734
735 /* copy to user if data and response */
103da066 736 for (i = 0; i < n && !err; i++)
a5f5774c 737 err = mmc_blk_ioctl_copy_to_user(&cmds[i], idata[i]);
a5f5774c 738
0bf6d96c 739 blk_mq_free_request(req);
3ecd8cf2 740
a5f5774c 741cmd_err:
103da066 742 for (i = 0; i < n; i++) {
a5f5774c
JH		 743 kfree(idata[i]->buf);
744 kfree(idata[i]);
745 }
746 kfree(idata);
b093410c 747 return ioc_err ? ioc_err : err;
a5f5774c
JH		 748}
749
61fe0e2b
LW		 750static int mmc_blk_check_blkdev(struct block_device *bdev)
751{
752 /*
753 * The caller must have CAP_SYS_RAWIO, and must be calling this on the
754 * whole block device, not on a partition. This prevents overspray
755 * between sibling partitions.
756 */
fa01b1e9 757 if (!capable(CAP_SYS_RAWIO) || bdev_is_partition(bdev))
61fe0e2b
LW		 758 return -EPERM;
759 return 0;
760}
761
cb87ea28
JC		 762static int mmc_blk_ioctl(struct block_device *bdev, fmode_t mode,
763 unsigned int cmd, unsigned long arg)
764{
2fe20bae 765 struct mmc_blk_data *md;
61fe0e2b
LW		 766 int ret;
767
a5f5774c
JH		 768 switch (cmd) {
769 case MMC_IOC_CMD:
61fe0e2b
LW		 770 ret = mmc_blk_check_blkdev(bdev);
771 if (ret)
772 return ret;
2fe20bae
LW		 773 md = mmc_blk_get(bdev->bd_disk);
774 if (!md)
775 return -EINVAL;
776 ret = mmc_blk_ioctl_cmd(md,
97548575
LW		 777 (struct mmc_ioc_cmd __user *)arg,
778 NULL);
2fe20bae
LW		 779 mmc_blk_put(md);
780 return ret;
a5f5774c 781 case MMC_IOC_MULTI_CMD:
61fe0e2b
LW		 782 ret = mmc_blk_check_blkdev(bdev);
783 if (ret)
784 return ret;
2fe20bae
LW		 785 md = mmc_blk_get(bdev->bd_disk);
786 if (!md)
787 return -EINVAL;
788 ret = mmc_blk_ioctl_multi_cmd(md,
97548575
LW		 789 (struct mmc_ioc_multi_cmd __user *)arg,
790 NULL);
2fe20bae
LW		 791 mmc_blk_put(md);
792 return ret;
a5f5774c
JH		 793 default:
794 return -EINVAL;
795 }
cb87ea28
JC		 796}
797
798#ifdef CONFIG_COMPAT
799static int mmc_blk_compat_ioctl(struct block_device *bdev, fmode_t mode,
800 unsigned int cmd, unsigned long arg)
801{
802 return mmc_blk_ioctl(bdev, mode, cmd, (unsigned long) compat_ptr(arg));
803}
804#endif
805
dc913385
DO		 806static int mmc_blk_alternative_gpt_sector(struct gendisk *disk,
807 sector_t *sector)
808{
809 struct mmc_blk_data *md;
810 int ret;
811
812 md = mmc_blk_get(disk);
813 if (!md)
814 return -EINVAL;
815
816 if (md->queue.card)
817 ret = mmc_card_alternative_gpt_sector(md->queue.card, sector);
818 else
819 ret = -ENODEV;
820
821 mmc_blk_put(md);
822
823 return ret;
824}
825
83d5cde4 826static const struct block_device_operations mmc_bdops = {
a5a1561f
AV		 827 .open = mmc_blk_open,
828 .release = mmc_blk_release,
a885c8c4 829 .getgeo = mmc_blk_getgeo,
1da177e4 830 .owner = THIS_MODULE,
cb87ea28
JC		 831 .ioctl = mmc_blk_ioctl,
832#ifdef CONFIG_COMPAT
833 .compat_ioctl = mmc_blk_compat_ioctl,
834#endif
dc913385 835 .alternative_gpt_sector = mmc_blk_alternative_gpt_sector,
1da177e4
LT		 836};
837
025e3d5f
AH		 838static int mmc_blk_part_switch_pre(struct mmc_card *card,
839 unsigned int part_type)
840{
841 int ret = 0;
842
843 if (part_type == EXT_CSD_PART_CONFIG_ACC_RPMB) {
844 if (card->ext_csd.cmdq_en) {
845 ret = mmc_cmdq_disable(card);
846 if (ret)
847 return ret;
848 }
849 mmc_retune_pause(card->host);
850 }
851
852 return ret;
853}
854
855static int mmc_blk_part_switch_post(struct mmc_card *card,
856 unsigned int part_type)
857{
858 int ret = 0;
859
860 if (part_type == EXT_CSD_PART_CONFIG_ACC_RPMB) {
861 mmc_retune_unpause(card->host);
862 if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
863 ret = mmc_cmdq_enable(card);
864 }
865
866 return ret;
867}
868
371a689f 869static inline int mmc_blk_part_switch(struct mmc_card *card,
1f797edc 870 unsigned int part_type)
371a689f 871{
025e3d5f 872 int ret = 0;
fc95e30b 873 struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
0d7d85ca 874
1f797edc 875 if (main_md->part_curr == part_type)
371a689f
AW		 876 return 0;
877
878 if (mmc_card_mmc(card)) {
0d7d85ca
AH		 879 u8 part_config = card->ext_csd.part_config;
880
1f797edc 881 ret = mmc_blk_part_switch_pre(card, part_type);
025e3d5f
AH		 882 if (ret)
883 return ret;
57da0c04 884
0d7d85ca 885 part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK;
1f797edc 886 part_config |= part_type;
371a689f
AW		 887
888 ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
0d7d85ca 889 EXT_CSD_PART_CONFIG, part_config,
371a689f 890 card->ext_csd.part_time);
57da0c04 891 if (ret) {
1f797edc 892 mmc_blk_part_switch_post(card, part_type);
371a689f 893 return ret;
57da0c04 894 }
0d7d85ca
AH		 895
896 card->ext_csd.part_config = part_config;
57da0c04 897
025e3d5f 898 ret = mmc_blk_part_switch_post(card, main_md->part_curr);
67716327 899 }
371a689f 900
1f797edc 901 main_md->part_curr = part_type;
025e3d5f 902 return ret;
371a689f
AW		 903}
904
169f03a0 905static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks)
ec5a19dd
PO		 906{
907 int err;
051913da
BD		 908 u32 result;
909 __be32 *blocks;
ec5a19dd 910
c7836d15
MY		 911 struct mmc_request mrq = {};
912 struct mmc_command cmd = {};
913 struct mmc_data data = {};
ec5a19dd
PO		 914
915 struct scatterlist sg;
916
ec5a19dd
PO		 917 cmd.opcode = MMC_APP_CMD;
918 cmd.arg = card->rca << 16;
7213d175 919 cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
ec5a19dd
PO		 920
921 err = mmc_wait_for_cmd(card->host, &cmd, 0);
7213d175 922 if (err)
169f03a0 923 return err;
7213d175 924 if (!mmc_host_is_spi(card->host) && !(cmd.resp[0] & R1_APP_CMD))
169f03a0 925 return -EIO;
ec5a19dd
PO		 926
927 memset(&cmd, 0, sizeof(struct mmc_command));
928
929 cmd.opcode = SD_APP_SEND_NUM_WR_BLKS;
930 cmd.arg = 0;
7213d175 931 cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
ec5a19dd 932
ec5a19dd
PO		 933 data.blksz = 4;
934 data.blocks = 1;
935 data.flags = MMC_DATA_READ;
936 data.sg = &sg;
937 data.sg_len = 1;
d380443c 938 mmc_set_data_timeout(&data, card);
ec5a19dd 939
ec5a19dd
PO		 940 mrq.cmd = &cmd;
941 mrq.data = &data;
942
051913da
BD		 943 blocks = kmalloc(4, GFP_KERNEL);
944 if (!blocks)
169f03a0 945 return -ENOMEM;
051913da
BD		 946
947 sg_init_one(&sg, blocks, 4);
ec5a19dd
PO		 948
949 mmc_wait_for_req(card->host, &mrq);
950
051913da
BD		 951 result = ntohl(*blocks);
952 kfree(blocks);
953
17b0429d 954 if (cmd.error || data.error)
169f03a0
LW		 955 return -EIO;
956
957 *written_blocks = result;
ec5a19dd 958
169f03a0 959 return 0;
ec5a19dd
PO		 960}
961
92c0a0cc
AH		 962static unsigned int mmc_blk_clock_khz(struct mmc_host *host)
963{
964 if (host->actual_clock)
965 return host->actual_clock / 1000;
966
967 /* Clock may be subject to a divisor, fudge it by a factor of 2. */
968 if (host->ios.clock)
969 return host->ios.clock / 2000;
970
971 /* How can there be no clock */
972 WARN_ON_ONCE(1);
973 return 100; /* 100 kHz is minimum possible value */
974}
975
976static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host,
977 struct mmc_data *data)
978{
979 unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000);
980 unsigned int khz;
981
982 if (data->timeout_clks) {
983 khz = mmc_blk_clock_khz(host);
984 ms += DIV_ROUND_UP(data->timeout_clks, khz);
985 }
986
987 return ms;
988}
989
67716327
AH		 990static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host,
991 int type)
992{
993 int err;
994
995 if (md->reset_done & type)
996 return -EEXIST;
997
998 md->reset_done |= type;
b71597ed 999 err = mmc_hw_reset(host->card);
67716327 1000 /* Ensure we switch back to the correct partition */
fefdd3c9 1001 if (err) {
fc95e30b
UH		 1002 struct mmc_blk_data *main_md =
1003 dev_get_drvdata(&host->card->dev);
67716327
AH		 1004 int part_err;
1005
1006 main_md->part_curr = main_md->part_type;
1f797edc 1007 part_err = mmc_blk_part_switch(host->card, md->part_type);
67716327
AH		 1008 if (part_err) {
1009 /*
1010 * We have failed to get back into the correct
1011 * partition, so we need to abort the whole request.
1012 */
1013 return -ENODEV;
1014 }
1015 }
1016 return err;
1017}
1018
1019static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
1020{
1021 md->reset_done &= ~type;
1022}
1023
5ec12396
LW		 1024/*
1025 * The non-block commands come back from the block layer after it queued it and
1026 * processed it with all other requests and then they get issued in this
1027 * function.
1028 */
1029static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
1030{
1031 struct mmc_queue_req *mq_rq;
1032 struct mmc_card *card = mq->card;
1033 struct mmc_blk_data *md = mq->blkdata;
69f7599e 1034 struct mmc_blk_ioc_data **idata;
97548575 1035 bool rpmb_ioctl;
627c3ccf
LW		 1036 u8 **ext_csd;
1037 u32 status;
0493f6fe 1038 int ret;
5ec12396
LW		 1039 int i;
1040
1041 mq_rq = req_to_mmc_queue_req(req);
97548575 1042 rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB);
5ec12396
LW		 1043
1044 switch (mq_rq->drv_op) {
1045 case MMC_DRV_OP_IOCTL:
70b52f09
BH		 1046 if (card->ext_csd.cmdq_en) {
1047 ret = mmc_cmdq_disable(card);
1048 if (ret)
1049 break;
1050 }
1051 fallthrough;
97548575 1052 case MMC_DRV_OP_IOCTL_RPMB:
69f7599e 1053 idata = mq_rq->drv_op_data;
7432b49b 1054 for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) {
69f7599e 1055 ret = __mmc_blk_ioctl_cmd(card, md, idata[i]);
0493f6fe 1056 if (ret)
5ec12396
LW		 1057 break;
1058 }
5ec12396 1059 /* Always switch back to main area after RPMB access */
97548575
LW		 1060 if (rpmb_ioctl)
1061 mmc_blk_part_switch(card, 0);
70b52f09
BH		 1062 else if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
1063 mmc_cmdq_enable(card);
0493f6fe
LW		 1064 break;
1065 case MMC_DRV_OP_BOOT_WP:
1066 ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP,
1067 card->ext_csd.boot_ro_lock |
1068 EXT_CSD_BOOT_WP_B_PWR_WP_EN,
1069 card->ext_csd.part_time);
1070 if (ret)
1071 pr_err("%s: Locking boot partition ro until next power on failed: %d\n",
1072 md->disk->disk_name, ret);
1073 else
1074 card->ext_csd.boot_ro_lock |=
1075 EXT_CSD_BOOT_WP_B_PWR_WP_EN;
5ec12396 1076 break;
627c3ccf
LW		 1077 case MMC_DRV_OP_GET_CARD_STATUS:
1078 ret = mmc_send_status(card, &status);
1079 if (!ret)
1080 ret = status;
1081 break;
1082 case MMC_DRV_OP_GET_EXT_CSD:
1083 ext_csd = mq_rq->drv_op_data;
1084 ret = mmc_get_ext_csd(card, ext_csd);
1085 break;
5ec12396 1086 default:
0493f6fe
LW		 1087 pr_err("%s: unknown driver specific operation\n",
1088 md->disk->disk_name);
1089 ret = -EINVAL;
5ec12396
LW		 1090 break;
1091 }
0493f6fe 1092 mq_rq->drv_op_result = ret;
0fbfd125 1093 blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
5ec12396
LW		 1094}
1095
f7b6fc32
VW		 1096static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req,
1097 int type, unsigned int erase_arg)
bd788c96 1098{
7db3028e 1099 struct mmc_blk_data *md = mq->blkdata;
bd788c96 1100 struct mmc_card *card = md->queue.card;
01904ff7 1101 unsigned int from, nr;
f7b6fc32 1102 int err = 0;
2a842aca 1103 blk_status_t status = BLK_STS_OK;
bd788c96 1104
bd788c96 1105 if (!mmc_can_erase(card)) {
2a842aca 1106 status = BLK_STS_NOTSUPP;
8cb6ed17 1107 goto fail;
bd788c96
AH		 1108 }
1109
1110 from = blk_rq_pos(req);
1111 nr = blk_rq_sectors(req);
1112
164b50b3
GU		 1113 do {
1114 err = 0;
1115 if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1116 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1117 INAND_CMD38_ARG_EXT_CSD,
f7b6fc32 1118 erase_arg == MMC_TRIM_ARG ?
164b50b3
GU		 1119 INAND_CMD38_ARG_TRIM :
1120 INAND_CMD38_ARG_ERASE,
ad91619a 1121 card->ext_csd.generic_cmd6_time);
164b50b3
GU		 1122 }
1123 if (!err)
f7b6fc32 1124 err = mmc_erase(card, from, nr, erase_arg);
164b50b3 1125 } while (err == -EIO && !mmc_blk_reset(md, card->host, type));
2a842aca
CH		 1126 if (err)
1127 status = BLK_STS_IOERR;
1128 else
67716327 1129 mmc_blk_reset_success(md, type);
8cb6ed17 1130fail:
0fbfd125 1131 blk_mq_end_request(req, status);
bd788c96
AH		 1132}
1133
f7b6fc32
VW		 1134static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req)
1135{
1136 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG);
1137}
1138
1139static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
1140{
1141 struct mmc_blk_data *md = mq->blkdata;
1142 struct mmc_card *card = md->queue.card;
1143
1144 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, card->erase_arg);
1145}
1146
df061588 1147static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
49804548
AH		 1148 struct request *req)
1149{
7db3028e 1150 struct mmc_blk_data *md = mq->blkdata;
49804548 1151 struct mmc_card *card = md->queue.card;
775a9362 1152 unsigned int from, nr, arg;
67716327 1153 int err = 0, type = MMC_BLK_SECDISCARD;
2a842aca 1154 blk_status_t status = BLK_STS_OK;
49804548 1155
775a9362 1156 if (!(mmc_can_secure_erase_trim(card))) {
2a842aca 1157 status = BLK_STS_NOTSUPP;
49804548
AH		 1158 goto out;
1159 }
1160
28302812
AH		 1161 from = blk_rq_pos(req);
1162 nr = blk_rq_sectors(req);
1163
775a9362
ME		 1164 if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
1165 arg = MMC_SECURE_TRIM1_ARG;
1166 else
1167 arg = MMC_SECURE_ERASE_ARG;
d9ddd629 1168
67716327 1169retry:
6a7a6b45
AW		 1170 if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1171 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1172 INAND_CMD38_ARG_EXT_CSD,
1173 arg == MMC_SECURE_TRIM1_ARG ?
1174 INAND_CMD38_ARG_SECTRIM1 :
1175 INAND_CMD38_ARG_SECERASE,
ad91619a 1176 card->ext_csd.generic_cmd6_time);
6a7a6b45 1177 if (err)
28302812 1178 goto out_retry;
6a7a6b45 1179 }
28302812 1180
49804548 1181 err = mmc_erase(card, from, nr, arg);
28302812
AH		 1182 if (err == -EIO)
1183 goto out_retry;
2a842aca
CH		 1184 if (err) {
1185 status = BLK_STS_IOERR;
28302812 1186 goto out;
2a842aca 1187 }
28302812
AH		 1188
1189 if (arg == MMC_SECURE_TRIM1_ARG) {
6a7a6b45
AW		 1190 if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1191 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1192 INAND_CMD38_ARG_EXT_CSD,
1193 INAND_CMD38_ARG_SECTRIM2,
ad91619a 1194 card->ext_csd.generic_cmd6_time);
6a7a6b45 1195 if (err)
28302812 1196 goto out_retry;
6a7a6b45 1197 }
28302812 1198
49804548 1199 err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG);
28302812
AH		 1200 if (err == -EIO)
1201 goto out_retry;
2a842aca
CH		 1202 if (err) {
1203 status = BLK_STS_IOERR;
28302812 1204 goto out;
2a842aca 1205 }
6a7a6b45 1206 }
28302812 1207
28302812
AH		 1208out_retry:
1209 if (err && !mmc_blk_reset(md, card->host, type))
67716327
AH		 1210 goto retry;
1211 if (!err)
1212 mmc_blk_reset_success(md, type);
28302812 1213out:
0fbfd125 1214 blk_mq_end_request(req, status);
49804548
AH		 1215}
1216
df061588 1217static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
f4c5522b 1218{
7db3028e 1219 struct mmc_blk_data *md = mq->blkdata;
881d1c25
SJ		 1220 struct mmc_card *card = md->queue.card;
1221 int ret = 0;
1222
8ae11ede 1223 ret = mmc_flush_cache(card->host);
0fbfd125 1224 blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
f4c5522b
AW		 1225}
1226
1227/*
1228 * Reformat current write as a reliable write, supporting
1229 * both legacy and the enhanced reliable write MMC cards.
1230 * In each transfer we'll handle only as much as a single
1231 * reliable write can handle, thus finish the request in
1232 * partial completions.
1233 */
d0c97cfb
AW		 1234static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
1235 struct mmc_card *card,
1236 struct request *req)
f4c5522b 1237{
f4c5522b
AW		 1238 if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) {
1239 /* Legacy mode imposes restrictions on transfers. */
9cb38f7a 1240 if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors))
f4c5522b
AW		 1241 brq->data.blocks = 1;
1242
1243 if (brq->data.blocks > card->ext_csd.rel_sectors)
1244 brq->data.blocks = card->ext_csd.rel_sectors;
1245 else if (brq->data.blocks < card->ext_csd.rel_sectors)
1246 brq->data.blocks = 1;
1247 }
f4c5522b
AW		 1248}
1249
f47a1fe3
AH		 1250#define CMD_ERRORS_EXCL_OOR \
1251 (R1_ADDRESS_ERROR | /* Misaligned address */ \
4c2b8f26
RKAL		 1252 R1_BLOCK_LEN_ERROR | /* Transferred block length incorrect */\
1253 R1_WP_VIOLATION | /* Tried to write to protected block */ \
a04e6bae 1254 R1_CARD_ECC_FAILED | /* Card ECC failed */ \
4c2b8f26
RKAL		 1255 R1_CC_ERROR | /* Card controller error */ \
1256 R1_ERROR) /* General/unknown error */
1257
f47a1fe3
AH		 1258#define CMD_ERRORS \
1259 (CMD_ERRORS_EXCL_OOR | \
1260 R1_OUT_OF_RANGE) /* Command argument out of range */ \
1261
d83c2dba 1262static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
a04e6bae 1263{
d83c2dba 1264 u32 val;
a04e6bae 1265
d83c2dba
SL		 1266 /*
1267 * Per the SD specification(physical layer version 4.10)[1],
1268 * section 4.3.3, it explicitly states that "When the last
1269 * block of user area is read using CMD18, the host should
1270 * ignore OUT_OF_RANGE error that may occur even the sequence
1271 * is correct". And JESD84-B51 for eMMC also has a similar
1272 * statement on section 6.8.3.
1273 *
1274 * Multiple block read/write could be done by either predefined
1275 * method, namely CMD23, or open-ending mode. For open-ending mode,
1276 * we should ignore the OUT_OF_RANGE error as it's normal behaviour.
1277 *
1278 * However the spec[1] doesn't tell us whether we should also
1279 * ignore that for predefined method. But per the spec[1], section
1280 * 4.15 Set Block Count Command, it says"If illegal block count
1281 * is set, out of range error will be indicated during read/write
1282 * operation (For example, data transfer is stopped at user area
1283 * boundary)." In another word, we could expect a out of range error
1284 * in the response for the following CMD18/25. And if argument of
1285 * CMD23 + the argument of CMD18/25 exceed the max number of blocks,
1286 * we could also expect to get a -ETIMEDOUT or any error number from
1287 * the host drivers due to missing data response(for write)/data(for
1288 * read), as the cards will stop the data transfer by itself per the
1289 * spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode.
1290 */
1291
1292 if (!brq->stop.error) {
1293 bool oor_with_open_end;
1294 /* If there is no error yet, check R1 response */
1295
1296 val = brq->stop.resp[0] & CMD_ERRORS;
1297 oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc;
1298
1299 if (val && !oor_with_open_end)
1300 brq->stop.error = -EIO;
1301 }
a04e6bae
WS		 1302}
1303
ca5717f7 1304static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
d3377c01
AH		 1305 int disable_multi, bool *do_rel_wr_p,
1306 bool *do_data_tag_p)
1da177e4 1307{
ca5717f7
AH		 1308 struct mmc_blk_data *md = mq->blkdata;
1309 struct mmc_card *card = md->queue.card;
54d49d77 1310 struct mmc_blk_request *brq = &mqrq->brq;
67e69d52 1311 struct request *req = mmc_queue_req_to_req(mqrq);
d3377c01 1312 bool do_rel_wr, do_data_tag;
1da177e4 1313
f4c5522b
AW		 1314 /*
1315 * Reliable writes are used to implement Forced Unit Access and
d3df0465 1316 * are supported only on MMCs.
f4c5522b 1317 */
d3377c01
AH		 1318 do_rel_wr = (req->cmd_flags & REQ_FUA) &&
1319 rq_data_dir(req) == WRITE &&
1320 (md->flags & MMC_BLK_REL_WR);
f4c5522b 1321
54d49d77 1322 memset(brq, 0, sizeof(struct mmc_blk_request));
ca5717f7 1323
93f1c150
EB		 1324 mmc_crypto_prepare_req(mqrq);
1325
54d49d77 1326 brq->mrq.data = &brq->data;
93482b3d 1327 brq->mrq.tag = req->tag;
1da177e4 1328
54d49d77
PF		 1329 brq->stop.opcode = MMC_STOP_TRANSMISSION;
1330 brq->stop.arg = 0;
ca5717f7
AH		 1331
1332 if (rq_data_dir(req) == READ) {
1333 brq->data.flags = MMC_DATA_READ;
1334 brq->stop.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
1335 } else {
1336 brq->data.flags = MMC_DATA_WRITE;
1337 brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
1338 }
1339
1340 brq->data.blksz = 512;
54d49d77 1341 brq->data.blocks = blk_rq_sectors(req);
93482b3d
AH		 1342 brq->data.blk_addr = blk_rq_pos(req);
1343
1344 /*
1345 * The command queue supports 2 priorities: "high" (1) and "simple" (0).
1346 * The eMMC will give "high" priority tasks priority over "simple"
1347 * priority tasks. Here we always set "simple" priority by not setting
1348 * MMC_DATA_PRIO.
1349 */
6a79e391 1350
54d49d77
PF		 1351 /*
1352 * The block layer doesn't support all sector count
1353 * restrictions, so we need to be prepared for too big
1354 * requests.
1355 */
1356 if (brq->data.blocks > card->host->max_blk_count)
1357 brq->data.blocks = card->host->max_blk_count;
1da177e4 1358
2bf22b39 1359 if (brq->data.blocks > 1) {
41591b38
CB		 1360 /*
1361 * Some SD cards in SPI mode return a CRC error or even lock up
1362 * completely when trying to read the last block using a
1363 * multiblock read command.
1364 */
1365 if (mmc_host_is_spi(card->host) && (rq_data_dir(req) == READ) &&
1366 (blk_rq_pos(req) + blk_rq_sectors(req) ==
1367 get_capacity(md->disk)))
1368 brq->data.blocks--;
1369
2bf22b39
PW		 1370 /*
1371 * After a read error, we redo the request one sector
1372 * at a time in order to accurately determine which
1373 * sectors can be read successfully.
1374 */
1375 if (disable_multi)
1376 brq->data.blocks = 1;
1377
2e47e842
KM		 1378 /*
1379 * Some controllers have HW issues while operating
1380 * in multiple I/O mode
1381 */
1382 if (card->host->ops->multi_io_quirk)
1383 brq->data.blocks = card->host->ops->multi_io_quirk(card,
1384 (rq_data_dir(req) == READ) ?
1385 MMC_DATA_READ : MMC_DATA_WRITE,
1386 brq->data.blocks);
2bf22b39 1387 }
d0c97cfb 1388
93482b3d 1389 if (do_rel_wr) {
ca5717f7 1390 mmc_apply_rel_rw(brq, card, req);
93482b3d
AH		 1391 brq->data.flags |= MMC_DATA_REL_WR;
1392 }
ca5717f7
AH		 1393
1394 /*
1395 * Data tag is used only during writing meta data to speed
1396 * up write and any subsequent read of this meta data
1397 */
d3377c01
AH		 1398 do_data_tag = card->ext_csd.data_tag_unit_size &&
1399 (req->cmd_flags & REQ_META) &&
1400 (rq_data_dir(req) == WRITE) &&
1401 ((brq->data.blocks * brq->data.blksz) >=
1402 card->ext_csd.data_tag_unit_size);
ca5717f7 1403
93482b3d
AH		 1404 if (do_data_tag)
1405 brq->data.flags |= MMC_DATA_DAT_TAG;
1406
ca5717f7
AH		 1407 mmc_set_data_timeout(&brq->data, card);
1408
1409 brq->data.sg = mqrq->sg;
1410 brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
1411
1412 /*
1413 * Adjust the sg list so it is the same size as the
1414 * request.
1415 */
1416 if (brq->data.blocks != blk_rq_sectors(req)) {
1417 int i, data_size = brq->data.blocks << 9;
1418 struct scatterlist *sg;
1419
1420 for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) {
1421 data_size -= sg->length;
1422 if (data_size <= 0) {
1423 sg->length += data_size;
1424 i++;
1425 break;
1426 }
1427 }
1428 brq->data.sg_len = i;
1429 }
1430
d3377c01
AH		 1431 if (do_rel_wr_p)
1432 *do_rel_wr_p = do_rel_wr;
1433
1434 if (do_data_tag_p)
1435 *do_data_tag_p = do_data_tag;
ca5717f7
AH		 1436}
1437
1e8e55b6
AH		 1438#define MMC_CQE_RETRIES 2
1439
1440static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)
1441{
1442 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1443 struct mmc_request *mrq = &mqrq->brq.mrq;
1444 struct request_queue *q = req->q;
1445 struct mmc_host *host = mq->card->host;
e6bfb1bf 1446 enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
1e8e55b6
AH		 1447 unsigned long flags;
1448 bool put_card;
1449 int err;
1450
1451 mmc_cqe_post_req(host, mrq);
1452
1453 if (mrq->cmd && mrq->cmd->error)
1454 err = mrq->cmd->error;
1455 else if (mrq->data && mrq->data->error)
1456 err = mrq->data->error;
1457 else
1458 err = 0;
1459
1460 if (err) {
1461 if (mqrq->retries++ < MMC_CQE_RETRIES)
1462 blk_mq_requeue_request(req, true);
1463 else
1464 blk_mq_end_request(req, BLK_STS_IOERR);
1465 } else if (mrq->data) {
1466 if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered))
1467 blk_mq_requeue_request(req, true);
1468 else
1469 __blk_mq_end_request(req, BLK_STS_OK);
1470 } else {
1471 blk_mq_end_request(req, BLK_STS_OK);
1472 }
1473
f5d72c5c 1474 spin_lock_irqsave(&mq->lock, flags);
1e8e55b6 1475
e6bfb1bf 1476 mq->in_flight[issue_type] -= 1;
1e8e55b6
AH		 1477
1478 put_card = (mmc_tot_in_flight(mq) == 0);
1479
1480 mmc_cqe_check_busy(mq);
1481
f5d72c5c 1482 spin_unlock_irqrestore(&mq->lock, flags);
1e8e55b6
AH		 1483
1484 if (!mq->cqe_busy)
1485 blk_mq_run_hw_queues(q, true);
1486
1487 if (put_card)
1488 mmc_put_card(mq->card, &mq->ctx);
1489}
1490
1491void mmc_blk_cqe_recovery(struct mmc_queue *mq)
1492{
1493 struct mmc_card *card = mq->card;
1494 struct mmc_host *host = card->host;
1495 int err;
1496
1497 pr_debug("%s: CQE recovery start\n", mmc_hostname(host));
1498
1499 err = mmc_cqe_recovery(host);
1500 if (err)
1501 mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);
1502 else
1503 mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);
1504
1505 pr_debug("%s: CQE recovery done\n", mmc_hostname(host));
1506}
1507
1508static void mmc_blk_cqe_req_done(struct mmc_request *mrq)
1509{
1510 struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
1511 brq.mrq);
1512 struct request *req = mmc_queue_req_to_req(mqrq);
1513 struct request_queue *q = req->q;
1514 struct mmc_queue *mq = q->queuedata;
1515
1516 /*
1517 * Block layer timeouts race with completions which means the normal
1518 * completion path cannot be used during recovery.
1519 */
1520 if (mq->in_recovery)
1521 mmc_blk_cqe_complete_rq(mq, req);
15f73f5b 1522 else if (likely(!blk_should_fake_timeout(req->q)))
1e8e55b6
AH		 1523 blk_mq_complete_request(req);
1524}
1525
1526static int mmc_blk_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq)
1527{
1528 mrq->done = mmc_blk_cqe_req_done;
1529 mrq->recovery_notifier = mmc_cqe_recovery_notifier;
1530
1531 return mmc_cqe_start_req(host, mrq);
1532}
1533
1534static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq,
1535 struct request *req)
1536{
1537 struct mmc_blk_request *brq = &mqrq->brq;
1538
1539 memset(brq, 0, sizeof(*brq));
1540
1541 brq->mrq.cmd = &brq->cmd;
1542 brq->mrq.tag = req->tag;
1543
1544 return &brq->mrq;
1545}
1546
1547static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)
1548{
1549 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1550 struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req);
1551
1552 mrq->cmd->opcode = MMC_SWITCH;
1553 mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) |
1554 (EXT_CSD_FLUSH_CACHE << 16) |
1555 (1 << 8) |
1556 EXT_CSD_CMD_SET_NORMAL;
1557 mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B;
1558
1559 return mmc_blk_cqe_start_req(mq->card->host, mrq);
1560}
1561
511ce378
BW		 1562static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req)
1563{
1564 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1565 struct mmc_host *host = mq->card->host;
1566 int err;
1567
1568 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
1569 mqrq->brq.mrq.done = mmc_blk_hsq_req_done;
1570 mmc_pre_req(host, &mqrq->brq.mrq);
1571
1572 err = mmc_cqe_start_req(host, &mqrq->brq.mrq);
1573 if (err)
1574 mmc_post_req(host, &mqrq->brq.mrq, err);
1575
1576 return err;
1577}
1578
1e8e55b6
AH		 1579static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)
1580{
1581 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
511ce378
BW		 1582 struct mmc_host *host = mq->card->host;
1583
1584 if (host->hsq_enabled)
1585 return mmc_blk_hsq_issue_rw_rq(mq, req);
1e8e55b6
AH		 1586
1587 mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);
1588
1589 return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);
1590}
1591
ca5717f7
AH		 1592static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
1593 struct mmc_card *card,
1594 int disable_multi,
1595 struct mmc_queue *mq)
1596{
1597 u32 readcmd, writecmd;
1598 struct mmc_blk_request *brq = &mqrq->brq;
67e69d52 1599 struct request *req = mmc_queue_req_to_req(mqrq);
ca5717f7
AH		 1600 struct mmc_blk_data *md = mq->blkdata;
1601 bool do_rel_wr, do_data_tag;
1602
1603 mmc_blk_data_prep(mq, mqrq, disable_multi, &do_rel_wr, &do_data_tag);
1604
1605 brq->mrq.cmd = &brq->cmd;
1606
1607 brq->cmd.arg = blk_rq_pos(req);
1608 if (!mmc_card_blockaddr(card))
1609 brq->cmd.arg <<= 9;
1610 brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
1611
54d49d77
PF		 1612 if (brq->data.blocks > 1 || do_rel_wr) {
1613 /* SPI multiblock writes terminate using a special
1614 * token, not a STOP_TRANSMISSION request.
d0c97cfb 1615 */
54d49d77
PF		 1616 if (!mmc_host_is_spi(card->host) ||
1617 rq_data_dir(req) == READ)
1618 brq->mrq.stop = &brq->stop;
1619 readcmd = MMC_READ_MULTIPLE_BLOCK;
1620 writecmd = MMC_WRITE_MULTIPLE_BLOCK;
1621 } else {
1622 brq->mrq.stop = NULL;
1623 readcmd = MMC_READ_SINGLE_BLOCK;
1624 writecmd = MMC_WRITE_BLOCK;
1625 }
ca5717f7 1626 brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd;
4265900e 1627
54d49d77
PF		 1628 /*
1629 * Pre-defined multi-block transfers are preferable to
1630 * open ended-ones (and necessary for reliable writes).
1631 * However, it is not sufficient to just send CMD23,
1632 * and avoid the final CMD12, as on an error condition
1633 * CMD12 (stop) needs to be sent anyway. This, coupled
1634 * with Auto-CMD23 enhancements provided by some
1635 * hosts, means that the complexity of dealing
1636 * with this is best left to the host. If CMD23 is
1637 * supported by card and host, we'll fill sbc in and let
1638 * the host deal with handling it correctly. This means
1639 * that for hosts that don't expose MMC_CAP_CMD23, no
1640 * change of behavior will be observed.
1641 *
1642 * N.B: Some MMC cards experience perf degradation.
1643 * We'll avoid using CMD23-bounded multiblock writes for
1644 * these, while retaining features like reliable writes.
1645 */
4265900e
SD		 1646 if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(brq->cmd.opcode) &&
1647 (do_rel_wr || !(card->quirks & MMC_QUIRK_BLK_NO_CMD23) ||
1648 do_data_tag)) {
54d49d77
PF		 1649 brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
1650 brq->sbc.arg = brq->data.blocks |
4265900e
SD		 1651 (do_rel_wr ? (1 << 31) : 0) |
1652 (do_data_tag ? (1 << 29) : 0);
54d49d77
PF		 1653 brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
1654 brq->mrq.sbc = &brq->sbc;
1655 }
54d49d77 1656}
6a79e391 1657
81196976 1658#define MMC_MAX_RETRIES 5
7eb43d53 1659#define MMC_DATA_RETRIES 2
81196976
AH		 1660#define MMC_NO_RETRIES (MMC_MAX_RETRIES + 1)
1661
7eb43d53
AH		 1662static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout)
1663{
1664 struct mmc_command cmd = {
1665 .opcode = MMC_STOP_TRANSMISSION,
1666 .flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC,
1667 /* Some hosts wait for busy anyway, so provide a busy timeout */
1668 .busy_timeout = timeout,
1669 };
1670
1671 return mmc_wait_for_cmd(card->host, &cmd, 5);
1672}
1673
1674static int mmc_blk_fix_state(struct mmc_card *card, struct request *req)
1675{
1676 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1677 struct mmc_blk_request *brq = &mqrq->brq;
1678 unsigned int timeout = mmc_blk_data_timeout_ms(card->host, &brq->data);
1679 int err;
1680
1681 mmc_retune_hold_now(card->host);
1682
1683 mmc_blk_send_stop(card, timeout);
1684
972d5084 1685 err = mmc_poll_for_busy(card, timeout, false, MMC_BUSY_IO);
7eb43d53
AH		 1686
1687 mmc_retune_release(card->host);
1688
1689 return err;
1690}
1691
81196976
AH		 1692#define MMC_READ_SINGLE_RETRIES 2
1693
1694/* Single sector read during recovery */
1695static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)
1696{
1697 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1698 struct mmc_request *mrq = &mqrq->brq.mrq;
1699 struct mmc_card *card = mq->card;
1700 struct mmc_host *host = card->host;
1701 blk_status_t error = BLK_STS_OK;
81196976
AH		 1702
1703 do {
1704 u32 status;
1705 int err;
54309fde 1706 int retries = 0;
81196976 1707
54309fde
CL		 1708 while (retries++ <= MMC_READ_SINGLE_RETRIES) {
1709 mmc_blk_rw_rq_prep(mqrq, card, 1, mq);
81196976 1710
54309fde 1711 mmc_wait_for_req(host, mrq);
81196976 1712
54309fde 1713 err = mmc_send_status(card, &status);
81196976
AH		 1714 if (err)
1715 goto error_exit;
81196976 1716
54309fde
CL		 1717 if (!mmc_host_is_spi(host) &&
1718 !mmc_ready_for_data(status)) {
1719 err = mmc_blk_fix_state(card, req);
1720 if (err)
1721 goto error_exit;
1722 }
81196976 1723
54309fde
CL		 1724 if (!mrq->cmd->error)
1725 break;
1726 }
81196976
AH		 1727
1728 if (mrq->cmd->error ||
1729 mrq->data->error ||
1730 (!mmc_host_is_spi(host) &&
1731 (mrq->cmd->resp[0] & CMD_ERRORS || status & CMD_ERRORS)))
1732 error = BLK_STS_IOERR;
1733 else
1734 error = BLK_STS_OK;
1735
1736 } while (blk_update_request(req, error, 512));
1737
1738 return;
1739
1740error_exit:
1741 mrq->data->bytes_xfered = 0;
1742 blk_update_request(req, BLK_STS_IOERR, 512);
1743 /* Let it try the remaining request again */
1744 if (mqrq->retries > MMC_MAX_RETRIES - 1)
1745 mqrq->retries = MMC_MAX_RETRIES - 1;
1746}
1747
7eb43d53
AH		 1748static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq)
1749{
1750 return !!brq->mrq.sbc;
1751}
1752
1753static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq)
1754{
1755 return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR;
1756}
1757
1758/*
1759 * Check for errors the host controller driver might not have seen such as
1760 * response mode errors or invalid card state.
1761 */
1762static bool mmc_blk_status_error(struct request *req, u32 status)
1763{
1764 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1765 struct mmc_blk_request *brq = &mqrq->brq;
1766 struct mmc_queue *mq = req->q->queuedata;
1767 u32 stop_err_bits;
1768
1769 if (mmc_host_is_spi(mq->card->host))
aa950144 1770 return false;
7eb43d53
AH		 1771
1772 stop_err_bits = mmc_blk_stop_err_bits(brq);
1773
1774 return brq->cmd.resp[0] & CMD_ERRORS ||
1775 brq->stop.resp[0] & stop_err_bits ||
1776 status & stop_err_bits ||
40c96853 1777 (rq_data_dir(req) == WRITE && !mmc_ready_for_data(status));
7eb43d53
AH		 1778}
1779
1780static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq)
1781{
1782 return !brq->sbc.error && !brq->cmd.error &&
1783 !(brq->cmd.resp[0] & CMD_ERRORS);
1784}
1785
1786/*
1787 * Requests are completed by mmc_blk_mq_complete_rq() which sets simple
1788 * policy:
1789 * 1. A request that has transferred at least some data is considered
1790 * successful and will be requeued if there is remaining data to
1791 * transfer.
1792 * 2. Otherwise the number of retries is incremented and the request
1793 * will be requeued if there are remaining retries.
1794 * 3. Otherwise the request will be errored out.
1795 * That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and
1796 * mqrq->retries. So there are only 4 possible actions here:
1797 * 1. do not accept the bytes_xfered value i.e. set it to zero
1798 * 2. change mqrq->retries to determine the number of retries
1799 * 3. try to reset the card
1800 * 4. read one sector at a time
1801 */
81196976
AH		 1802static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)
1803{
1804 int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
1805 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1806 struct mmc_blk_request *brq = &mqrq->brq;
1807 struct mmc_blk_data *md = mq->blkdata;
1808 struct mmc_card *card = mq->card;
7eb43d53
AH		 1809 u32 status;
1810 u32 blocks;
1811 int err;
81196976 1812
7eb43d53
AH		 1813 /*
1814 * Some errors the host driver might not have seen. Set the number of
1815 * bytes transferred to zero in that case.
1816 */
1817 err = __mmc_send_status(card, &status, 0);
1818 if (err || mmc_blk_status_error(req, status))
1819 brq->data.bytes_xfered = 0;
81196976
AH		 1820
1821 mmc_retune_release(card->host);
1822
1823 /*
7eb43d53
AH		 1824 * Try again to get the status. This also provides an opportunity for
1825 * re-tuning.
81196976 1826 */
7eb43d53
AH		 1827 if (err)
1828 err = __mmc_send_status(card, &status, 0);
81196976 1829
7eb43d53
AH		 1830 /*
1831 * Nothing more to do after the number of bytes transferred has been
1832 * updated and there is no card.
1833 */
1834 if (err && mmc_detect_card_removed(card->host))
1835 return;
81196976 1836
7eb43d53
AH		 1837 /* Try to get back to "tran" state */
1838 if (!mmc_host_is_spi(mq->card->host) &&
40c96853 1839 (err || !mmc_ready_for_data(status)))
7eb43d53
AH		 1840 err = mmc_blk_fix_state(mq->card, req);
1841
1842 /*
1843 * Special case for SD cards where the card might record the number of
1844 * blocks written.
1845 */
1846 if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) &&
1847 rq_data_dir(req) == WRITE) {
1848 if (mmc_sd_num_wr_blocks(card, &blocks))
1849 brq->data.bytes_xfered = 0;
1850 else
1851 brq->data.bytes_xfered = blocks << 9;
81196976 1852 }
7eb43d53
AH		 1853
1854 /* Reset if the card is in a bad state */
1855 if (!mmc_host_is_spi(mq->card->host) &&
1856 err && mmc_blk_reset(md, card->host, type)) {
f3fa33ac 1857 pr_err("%s: recovery failed!\n", req->q->disk->disk_name);
81196976 1858 mqrq->retries = MMC_NO_RETRIES;
7eb43d53
AH		 1859 return;
1860 }
1861
1862 /*
1863 * If anything was done, just return and if there is anything remaining
1864 * on the request it will get requeued.
1865 */
1866 if (brq->data.bytes_xfered)
1867 return;
1868
1869 /* Reset before last retry */
1870 if (mqrq->retries + 1 == MMC_MAX_RETRIES)
1871 mmc_blk_reset(md, card->host, type);
1872
1873 /* Command errors fail fast, so use all MMC_MAX_RETRIES */
1874 if (brq->sbc.error || brq->cmd.error)
1875 return;
1876
1877 /* Reduce the remaining retries for data errors */
1878 if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) {
1879 mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES;
1880 return;
1881 }
1882
1883 /* FIXME: Missing single sector read for large sector size */
1884 if (!mmc_large_sector(card) && rq_data_dir(req) == READ &&
1885 brq->data.blocks > 1) {
1886 /* Read one sector at a time */
1887 mmc_blk_read_single(mq, req);
1888 return;
81196976
AH		 1889 }
1890}
1891
10f21df4
AH		 1892static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq)
1893{
1894 mmc_blk_eval_resp_error(brq);
1895
1896 return brq->sbc.error || brq->cmd.error || brq->stop.error ||
1897 brq->data.error || brq->cmd.resp[0] & CMD_ERRORS;
1898}
1899
5d435933
CL		 1900static int mmc_spi_err_check(struct mmc_card *card)
1901{
1902 u32 status = 0;
1903 int err;
1904
1905 /*
1906 * SPI does not have a TRAN state we have to wait on, instead the
1907 * card is ready again when it no longer holds the line LOW.
1908 * We still have to ensure two things here before we know the write
1909 * was successful:
1910 * 1. The card has not disconnected during busy and we actually read our
1911 * own pull-up, thinking it was still connected, so ensure it
1912 * still responds.
1913 * 2. Check for any error bits, in particular R1_SPI_IDLE to catch a
1914 * just reconnected card after being disconnected during busy.
1915 */
1916 err = __mmc_send_status(card, &status, 0);
1917 if (err)
1918 return err;
1919 /* All R1 and R2 bits of SPI are errors in our case */
1920 if (status)
1921 return -EIO;
1922 return 0;
1923}
1924
6966e609
UH		 1925static int mmc_blk_busy_cb(void *cb_data, bool *busy)
1926{
1927 struct mmc_blk_busy_data *data = cb_data;
1928 u32 status = 0;
1929 int err;
1930
1931 err = mmc_send_status(data->card, &status);
1932 if (err)
1933 return err;
1934
1935 /* Accumulate response error bits. */
1936 data->status |= status;
1937
1938 *busy = !mmc_ready_for_data(status);
1939 return 0;
1940}
1941
88a51646
AH		 1942static int mmc_blk_card_busy(struct mmc_card *card, struct request *req)
1943{
1944 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
6966e609 1945 struct mmc_blk_busy_data cb_data;
88a51646
AH		 1946 int err;
1947
5d435933 1948 if (rq_data_dir(req) == READ)
88a51646
AH		 1949 return 0;
1950
5d435933
CL		 1951 if (mmc_host_is_spi(card->host)) {
1952 err = mmc_spi_err_check(card);
1953 if (err)
1954 mqrq->brq.data.bytes_xfered = 0;
1955 return err;
1956 }
1957
6966e609
UH		 1958 cb_data.card = card;
1959 cb_data.status = 0;
1760fdb6 1960 err = __mmc_poll_for_busy(card->host, 0, MMC_BLK_TIMEOUT_MS,
2ebbdace 1961 &mmc_blk_busy_cb, &cb_data);
88a51646 1962
f47a1fe3
AH		 1963 /*
1964 * Do not assume data transferred correctly if there are any error bits
1965 * set.
1966 */
6966e609 1967 if (cb_data.status & mmc_blk_stop_err_bits(&mqrq->brq)) {
f47a1fe3 1968 mqrq->brq.data.bytes_xfered = 0;
88a51646
AH		 1969 err = err ? err : -EIO;
1970 }
1971
f47a1fe3 1972 /* Copy the exception bit so it will be seen later on */
6966e609 1973 if (mmc_card_mmc(card) && cb_data.status & R1_EXCEPTION_EVENT)
f47a1fe3
AH		 1974 mqrq->brq.cmd.resp[0] |= R1_EXCEPTION_EVENT;
1975
88a51646
AH		 1976 return err;
1977}
1978
10f21df4
AH		 1979static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,
1980 struct request *req)
1981{
1982 int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
1983
1984 mmc_blk_reset_success(mq->blkdata, type);
1985}
1986
81196976
AH		 1987static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)
1988{
1989 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1990 unsigned int nr_bytes = mqrq->brq.data.bytes_xfered;
1991
1992 if (nr_bytes) {
1993 if (blk_update_request(req, BLK_STS_OK, nr_bytes))
1994 blk_mq_requeue_request(req, true);
1995 else
1996 __blk_mq_end_request(req, BLK_STS_OK);
1997 } else if (!blk_rq_bytes(req)) {
1998 __blk_mq_end_request(req, BLK_STS_IOERR);
1999 } else if (mqrq->retries++ < MMC_MAX_RETRIES) {
2000 blk_mq_requeue_request(req, true);
2001 } else {
2002 if (mmc_card_removed(mq->card))
2003 req->rq_flags |= RQF_QUIET;
2004 blk_mq_end_request(req, BLK_STS_IOERR);
2005 }
2006}
2007
2008static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,
2009 struct mmc_queue_req *mqrq)
2010{
2011 return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&
2012 (mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT ||
2013 mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT);
2014}
2015
2016static void mmc_blk_urgent_bkops(struct mmc_queue *mq,
2017 struct mmc_queue_req *mqrq)
2018{
2019 if (mmc_blk_urgent_bkops_needed(mq, mqrq))
0c204979 2020 mmc_run_bkops(mq->card);
81196976
AH		 2021}
2022
511ce378
BW		 2023static void mmc_blk_hsq_req_done(struct mmc_request *mrq)
2024{
2025 struct mmc_queue_req *mqrq =
2026 container_of(mrq, struct mmc_queue_req, brq.mrq);
2027 struct request *req = mmc_queue_req_to_req(mqrq);
2028 struct request_queue *q = req->q;
2029 struct mmc_queue *mq = q->queuedata;
2030 struct mmc_host *host = mq->card->host;
2031 unsigned long flags;
2032
2033 if (mmc_blk_rq_error(&mqrq->brq) ||
2034 mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2035 spin_lock_irqsave(&mq->lock, flags);
2036 mq->recovery_needed = true;
2037 mq->recovery_req = req;
2038 spin_unlock_irqrestore(&mq->lock, flags);
2039
2040 host->cqe_ops->cqe_recovery_start(host);
2041
2042 schedule_work(&mq->recovery_work);
2043 return;
2044 }
2045
2046 mmc_blk_rw_reset_success(mq, req);
2047
2048 /*
2049 * Block layer timeouts race with completions which means the normal
2050 * completion path cannot be used during recovery.
2051 */
2052 if (mq->in_recovery)
2053 mmc_blk_cqe_complete_rq(mq, req);
15f73f5b 2054 else if (likely(!blk_should_fake_timeout(req->q)))
511ce378
BW		 2055 blk_mq_complete_request(req);
2056}
2057
81196976
AH		 2058void mmc_blk_mq_complete(struct request *req)
2059{
2060 struct mmc_queue *mq = req->q->queuedata;
407a1c57 2061 struct mmc_host *host = mq->card->host;
81196976 2062
407a1c57 2063 if (host->cqe_enabled)
1e8e55b6 2064 mmc_blk_cqe_complete_rq(mq, req);
15f73f5b 2065 else if (likely(!blk_should_fake_timeout(req->q)))
1e8e55b6 2066 mmc_blk_mq_complete_rq(mq, req);
81196976
AH		 2067}
2068
2069static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,
2070 struct request *req)
2071{
2072 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
88a51646 2073 struct mmc_host *host = mq->card->host;
81196976 2074
88a51646
AH		 2075 if (mmc_blk_rq_error(&mqrq->brq) ||
2076 mmc_blk_card_busy(mq->card, req)) {
2077 mmc_blk_mq_rw_recovery(mq, req);
2078 } else {
2079 mmc_blk_rw_reset_success(mq, req);
2080 mmc_retune_release(host);
2081 }
81196976
AH		 2082
2083 mmc_blk_urgent_bkops(mq, mqrq);
2084}
2085
2086static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, struct request *req)
2087{
81196976
AH		 2088 unsigned long flags;
2089 bool put_card;
2090
f5d72c5c 2091 spin_lock_irqsave(&mq->lock, flags);
81196976
AH		 2092
2093 mq->in_flight[mmc_issue_type(mq, req)] -= 1;
2094
2095 put_card = (mmc_tot_in_flight(mq) == 0);
2096
f5d72c5c 2097 spin_unlock_irqrestore(&mq->lock, flags);
81196976
AH		 2098
2099 if (put_card)
2100 mmc_put_card(mq->card, &mq->ctx);
2101}
2102
639d3531
SAS		 2103static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req,
2104 bool can_sleep)
81196976
AH		 2105{
2106 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2107 struct mmc_request *mrq = &mqrq->brq.mrq;
2108 struct mmc_host *host = mq->card->host;
2109
2110 mmc_post_req(host, mrq, 0);
2111
10f21df4
AH		 2112 /*
2113 * Block layer timeouts race with completions which means the normal
2114 * completion path cannot be used during recovery.
2115 */
639d3531 2116 if (mq->in_recovery) {
10f21df4 2117 mmc_blk_mq_complete_rq(mq, req);
639d3531
SAS		 2118 } else if (likely(!blk_should_fake_timeout(req->q))) {
2119 if (can_sleep)
2120 blk_mq_complete_request_direct(req, mmc_blk_mq_complete);
2121 else
2122 blk_mq_complete_request(req);
2123 }
81196976
AH		 2124
2125 mmc_blk_mq_dec_in_flight(mq, req);
2126}
2127
10f21df4
AH		 2128void mmc_blk_mq_recovery(struct mmc_queue *mq)
2129{
2130 struct request *req = mq->recovery_req;
2131 struct mmc_host *host = mq->card->host;
2132 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2133
2134 mq->recovery_req = NULL;
2135 mq->rw_wait = false;
2136
2137 if (mmc_blk_rq_error(&mqrq->brq)) {
2138 mmc_retune_hold_now(host);
2139 mmc_blk_mq_rw_recovery(mq, req);
2140 }
2141
2142 mmc_blk_urgent_bkops(mq, mqrq);
2143
639d3531 2144 mmc_blk_mq_post_req(mq, req, true);
10f21df4
AH		 2145}
2146
81196976
AH		 2147static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,
2148 struct request **prev_req)
2149{
10f21df4
AH		 2150 if (mmc_host_done_complete(mq->card->host))
2151 return;
2152
81196976
AH		 2153 mutex_lock(&mq->complete_lock);
2154
2155 if (!mq->complete_req)
2156 goto out_unlock;
2157
2158 mmc_blk_mq_poll_completion(mq, mq->complete_req);
2159
2160 if (prev_req)
2161 *prev_req = mq->complete_req;
2162 else
639d3531 2163 mmc_blk_mq_post_req(mq, mq->complete_req, true);
81196976
AH		 2164
2165 mq->complete_req = NULL;
2166
2167out_unlock:
2168 mutex_unlock(&mq->complete_lock);
2169}
2170
2171void mmc_blk_mq_complete_work(struct work_struct *work)
2172{
2173 struct mmc_queue *mq = container_of(work, struct mmc_queue,
2174 complete_work);
2175
2176 mmc_blk_mq_complete_prev_req(mq, NULL);
2177}
2178
2179static void mmc_blk_mq_req_done(struct mmc_request *mrq)
2180{
2181 struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
2182 brq.mrq);
2183 struct request *req = mmc_queue_req_to_req(mqrq);
2184 struct request_queue *q = req->q;
2185 struct mmc_queue *mq = q->queuedata;
10f21df4 2186 struct mmc_host *host = mq->card->host;
81196976 2187 unsigned long flags;
81196976 2188
10f21df4
AH		 2189 if (!mmc_host_done_complete(host)) {
2190 bool waiting;
81196976 2191
10f21df4
AH		 2192 /*
2193 * We cannot complete the request in this context, so record
2194 * that there is a request to complete, and that a following
2195 * request does not need to wait (although it does need to
2196 * complete complete_req first).
2197 */
f5d72c5c 2198 spin_lock_irqsave(&mq->lock, flags);
10f21df4
AH		 2199 mq->complete_req = req;
2200 mq->rw_wait = false;
2201 waiting = mq->waiting;
f5d72c5c 2202 spin_unlock_irqrestore(&mq->lock, flags);
10f21df4
AH		 2203
2204 /*
2205 * If 'waiting' then the waiting task will complete this
2206 * request, otherwise queue a work to do it. Note that
2207 * complete_work may still race with the dispatch of a following
2208 * request.
2209 */
2210 if (waiting)
2211 wake_up(&mq->wait);
2212 else
dcf6e2e3 2213 queue_work(mq->card->complete_wq, &mq->complete_work);
10f21df4
AH		 2214
2215 return;
2216 }
2217
2218 /* Take the recovery path for errors or urgent background operations */
2219 if (mmc_blk_rq_error(&mqrq->brq) ||
2220 mmc_blk_urgent_bkops_needed(mq, mqrq)) {
f5d72c5c 2221 spin_lock_irqsave(&mq->lock, flags);
10f21df4
AH		 2222 mq->recovery_needed = true;
2223 mq->recovery_req = req;
f5d72c5c 2224 spin_unlock_irqrestore(&mq->lock, flags);
81196976 2225 wake_up(&mq->wait);
10f21df4
AH		 2226 schedule_work(&mq->recovery_work);
2227 return;
2228 }
2229
2230 mmc_blk_rw_reset_success(mq, req);
2231
2232 mq->rw_wait = false;
2233 wake_up(&mq->wait);
2234
639d3531
SAS		 2235 /* context unknown */
2236 mmc_blk_mq_post_req(mq, req, false);
81196976
AH		 2237}
2238
2239static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)
2240{
81196976
AH		 2241 unsigned long flags;
2242 bool done;
2243
2244 /*
10f21df4
AH		 2245 * Wait while there is another request in progress, but not if recovery
2246 * is needed. Also indicate whether there is a request waiting to start.
81196976 2247 */
f5d72c5c 2248 spin_lock_irqsave(&mq->lock, flags);
10f21df4
AH		 2249 if (mq->recovery_needed) {
2250 *err = -EBUSY;
2251 done = true;
2252 } else {
2253 done = !mq->rw_wait;
2254 }
81196976 2255 mq->waiting = !done;
f5d72c5c 2256 spin_unlock_irqrestore(&mq->lock, flags);
81196976
AH		 2257
2258 return done;
2259}
2260
2261static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)
2262{
2263 int err = 0;
2264
2265 wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));
2266
2267 /* Always complete the previous request if there is one */
2268 mmc_blk_mq_complete_prev_req(mq, prev_req);
2269
2270 return err;
2271}
2272
2273static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,
2274 struct request *req)
2275{
2276 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2277 struct mmc_host *host = mq->card->host;
2278 struct request *prev_req = NULL;
2279 int err = 0;
2280
2281 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
2282
2283 mqrq->brq.mrq.done = mmc_blk_mq_req_done;
2284
2285 mmc_pre_req(host, &mqrq->brq.mrq);
2286
2287 err = mmc_blk_rw_wait(mq, &prev_req);
2288 if (err)
2289 goto out_post_req;
2290
2291 mq->rw_wait = true;
2292
2293 err = mmc_start_request(host, &mqrq->brq.mrq);
2294
2295 if (prev_req)
639d3531 2296 mmc_blk_mq_post_req(mq, prev_req, true);
81196976 2297
10f21df4 2298 if (err)
81196976 2299 mq->rw_wait = false;
10f21df4
AH		 2300
2301 /* Release re-tuning here where there is no synchronization required */
2302 if (err || mmc_host_done_complete(host))
81196976 2303 mmc_retune_release(host);
81196976
AH		 2304
2305out_post_req:
2306 if (err)
2307 mmc_post_req(host, &mqrq->brq.mrq, err);
2308
2309 return err;
2310}
2311
2312static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)
2313{
407a1c57 2314 if (host->cqe_enabled)
1e8e55b6
AH		 2315 return host->cqe_ops->cqe_wait_for_idle(host);
2316
81196976
AH		 2317 return mmc_blk_rw_wait(mq, NULL);
2318}
2319
2320enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)
2321{
2322 struct mmc_blk_data *md = mq->blkdata;
2323 struct mmc_card *card = md->queue.card;
2324 struct mmc_host *host = card->host;
2325 int ret;
2326
2327 ret = mmc_blk_part_switch(card, md->part_type);
2328 if (ret)
2329 return MMC_REQ_FAILED_TO_START;
2330
2331 switch (mmc_issue_type(mq, req)) {
2332 case MMC_ISSUE_SYNC:
2333 ret = mmc_blk_wait_for_idle(mq, host);
2334 if (ret)
2335 return MMC_REQ_BUSY;
2336 switch (req_op(req)) {
2337 case REQ_OP_DRV_IN:
2338 case REQ_OP_DRV_OUT:
2339 mmc_blk_issue_drv_op(mq, req);
2340 break;
2341 case REQ_OP_DISCARD:
2342 mmc_blk_issue_discard_rq(mq, req);
2343 break;
2344 case REQ_OP_SECURE_ERASE:
2345 mmc_blk_issue_secdiscard_rq(mq, req);
2346 break;
f7b6fc32
VW		 2347 case REQ_OP_WRITE_ZEROES:
2348 mmc_blk_issue_trim_rq(mq, req);
2349 break;
81196976
AH		 2350 case REQ_OP_FLUSH:
2351 mmc_blk_issue_flush(mq, req);
2352 break;
2353 default:
2354 WARN_ON_ONCE(1);
2355 return MMC_REQ_FAILED_TO_START;
2356 }
2357 return MMC_REQ_FINISHED;
1e8e55b6 2358 case MMC_ISSUE_DCMD:
81196976
AH		 2359 case MMC_ISSUE_ASYNC:
2360 switch (req_op(req)) {
1e8e55b6 2361 case REQ_OP_FLUSH:
97fce126
AA		 2362 if (!mmc_cache_enabled(host)) {
2363 blk_mq_end_request(req, BLK_STS_OK);
2364 return MMC_REQ_FINISHED;
2365 }
1e8e55b6
AH		 2366 ret = mmc_blk_cqe_issue_flush(mq, req);
2367 break;
81196976
AH		 2368 case REQ_OP_READ:
2369 case REQ_OP_WRITE:
407a1c57 2370 if (host->cqe_enabled)
1e8e55b6
AH		 2371 ret = mmc_blk_cqe_issue_rw_rq(mq, req);
2372 else
2373 ret = mmc_blk_mq_issue_rw_rq(mq, req);
81196976
AH		 2374 break;
2375 default:
2376 WARN_ON_ONCE(1);
2377 ret = -EINVAL;
2378 }
2379 if (!ret)
2380 return MMC_REQ_STARTED;
2381 return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START;
2382 default:
2383 WARN_ON_ONCE(1);
2384 return MMC_REQ_FAILED_TO_START;
2385 }
2386}
2387
a6f6c96b
RK		 2388static inline int mmc_blk_readonly(struct mmc_card *card)
2389{
2390 return mmc_card_readonly(card) ||
2391 !(card->csd.cmdclass & CCC_BLOCK_WRITE);
2392}
2393
371a689f
AW		 2394static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
2395 struct device *parent,
2396 sector_t size,
2397 bool default_ro,
add710ea 2398 const char *subname,
a94dcfce
CH		 2399 int area_type,
2400 unsigned int part_type)
1da177e4
LT		 2401{
2402 struct mmc_blk_data *md;
2403 int devidx, ret;
ce999ed1 2404 char cap_str[10];
08ebf903
MW		 2405 bool cache_enabled = false;
2406 bool fua_enabled = false;
1da177e4 2407
a04848c7 2408 devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL);
e7b42769
SL		 2409 if (devidx < 0) {
2410 /*
2411 * We get -ENOSPC because there are no more any available
2412 * devidx. The reason may be that, either userspace haven't yet
2413 * unmounted the partitions, which postpones mmc_blk_release()
2414 * from being called, or the device has more partitions than
2415 * what we support.
2416 */
2417 if (devidx == -ENOSPC)
2418 dev_err(mmc_dev(card->host),
2419 "no more device IDs available\n");
2420
a04848c7 2421 return ERR_PTR(devidx);
e7b42769 2422 }
1da177e4 2423
dd00cc48 2424 md = kzalloc(sizeof(struct mmc_blk_data), GFP_KERNEL);
a6f6c96b
RK		 2425 if (!md) {
2426 ret = -ENOMEM;
2427 goto out;
2428 }
1da177e4 2429
add710ea
JR		 2430 md->area_type = area_type;
2431
a6f6c96b
RK		 2432 /*
2433 * Set the read-only status based on the supported commands
2434 * and the write protect switch.
2435 */
2436 md->read_only = mmc_blk_readonly(card);
1da177e4 2437
607d968a
CH		 2438 md->disk = mmc_init_queue(&md->queue, card);
2439 if (IS_ERR(md->disk)) {
2440 ret = PTR_ERR(md->disk);
a6f6c96b
RK		 2441 goto err_kfree;
2442 }
1da177e4 2443
371a689f 2444 INIT_LIST_HEAD(&md->part);
97548575 2445 INIT_LIST_HEAD(&md->rpmbs);
edb25572
SB		 2446 kref_init(&md->kref);
2447
7db3028e 2448 md->queue.blkdata = md;
a94dcfce 2449 md->part_type = part_type;
d2b18394 2450
fe6b4c88 2451 md->disk->major = MMC_BLOCK_MAJOR;
1033d103 2452 md->disk->minors = perdev_minors;
5e71b7a6 2453 md->disk->first_minor = devidx * perdev_minors;
a6f6c96b
RK		 2454 md->disk->fops = &mmc_bdops;
2455 md->disk->private_data = md;
307d8e6f 2456 md->parent = parent;
371a689f 2457 set_disk_ro(md->disk, md->read_only || default_ro);
f5b4d71f 2458 if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT))
79b0f79a 2459 md->disk->flags |= GENHD_FL_NO_PART;
a6f6c96b
RK		 2460
2461 /*
2462 * As discussed on lkml, GENHD_FL_REMOVABLE should:
2463 *
2464 * - be set for removable media with permanent block devices
2465 * - be unset for removable block devices with permanent media
2466 *
2467 * Since MMC block devices clearly fall under the second
2468 * case, we do not set GENHD_FL_REMOVABLE. Userspace
2469 * should use the block device creation/destruction hotplug
2470 * messages to tell when the card is present.
2471 */
2472
f06c9153 2473 snprintf(md->disk->disk_name, sizeof(md->disk->disk_name),
9aaf3437 2474 "mmcblk%u%s", card->host->index, subname ? subname : "");
a6f6c96b 2475
371a689f 2476 set_capacity(md->disk, size);
d0c97cfb 2477
f0d89972 2478 if (mmc_host_cmd23(card->host)) {
0ed50abb
DG		 2479 if ((mmc_card_mmc(card) &&
2480 card->csd.mmca_vsn >= CSD_SPEC_VER_3) ||
f0d89972
AW		 2481 (mmc_card_sd(card) &&
2482 card->scr.cmds & SD_SCR_CMD23_SUPPORT))
2483 md->flags |= MMC_BLK_CMD23;
2484 }
d0c97cfb 2485
08ebf903 2486 if (md->flags & MMC_BLK_CMD23 &&
d0c97cfb
AW		 2487 ((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) ||
2488 card->ext_csd.rel_sectors)) {
2489 md->flags |= MMC_BLK_REL_WR;
08ebf903
MW		 2490 fua_enabled = true;
2491 cache_enabled = true;
d0c97cfb 2492 }
08ebf903
MW		 2493 if (mmc_cache_enabled(card->host))
2494 cache_enabled = true;
2495
2496 blk_queue_write_cache(md->queue.queue, cache_enabled, fua_enabled);
d0c97cfb 2497
ce999ed1
UH		 2498 string_get_size((u64)size, 512, STRING_UNITS_2,
2499 cap_str, sizeof(cap_str));
2500 pr_info("%s: %s %s %s %s\n",
2501 md->disk->disk_name, mmc_card_id(card), mmc_card_name(card),
2502 cap_str, md->read_only ? "(ro)" : "");
2503
a94dcfce
CH		 2504 /* used in ->open, must be set before add_disk: */
2505 if (area_type == MMC_BLK_DATA_AREA_MAIN)
2506 dev_set_drvdata(&card->dev, md);
9c1aaec4
LC		 2507 ret = device_add_disk(md->parent, md->disk, mmc_disk_attr_groups);
2508 if (ret)
2509 goto err_cleanup_queue;
371a689f
AW		 2510 return md;
2511
9c1aaec4
LC		 2512 err_cleanup_queue:
2513 blk_cleanup_queue(md->disk->queue);
2514 blk_mq_free_tag_set(&md->queue.tag_set);
371a689f
AW		 2515 err_kfree:
2516 kfree(md);
2517 out:
a04848c7 2518 ida_simple_remove(&mmc_blk_ida, devidx);
371a689f
AW		 2519 return ERR_PTR(ret);
2520}
2521
2522static struct mmc_blk_data *mmc_blk_alloc(struct mmc_card *card)
2523{
2524 sector_t size;
a6f6c96b 2525
85a18ad9
PO		 2526 if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
2527 /*
2528 * The EXT_CSD sector count is in number or 512 byte
2529 * sectors.
2530 */
371a689f 2531 size = card->ext_csd.sectors;
85a18ad9
PO		 2532 } else {
2533 /*
2534 * The CSD capacity field is in units of read_blkbits.
2535 * set_capacity takes units of 512 bytes.
2536 */
087de9ed
KM		 2537 size = (typeof(sector_t))card->csd.capacity
2538 << (card->csd.read_blkbits - 9);
85a18ad9 2539 }
371a689f 2540
7a30f2af 2541 return mmc_blk_alloc_req(card, &card->dev, size, false, NULL,
a94dcfce 2542 MMC_BLK_DATA_AREA_MAIN, 0);
371a689f 2543}
a6f6c96b 2544
371a689f
AW		 2545static int mmc_blk_alloc_part(struct mmc_card *card,
2546 struct mmc_blk_data *md,
2547 unsigned int part_type,
2548 sector_t size,
2549 bool default_ro,
add710ea
JR		 2550 const char *subname,
2551 int area_type)
371a689f 2552{
371a689f
AW		 2553 struct mmc_blk_data *part_md;
2554
2555 part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro,
a94dcfce 2556 subname, area_type, part_type);
371a689f
AW		 2557 if (IS_ERR(part_md))
2558 return PTR_ERR(part_md);
371a689f
AW		 2559 list_add(&part_md->part, &md->part);
2560
371a689f
AW		 2561 return 0;
2562}
2563
97548575
LW		 2564/**
2565 * mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev
2566 * @filp: the character device file
2567 * @cmd: the ioctl() command
2568 * @arg: the argument from userspace
2569 *
2570 * This will essentially just redirect the ioctl()s coming in over to
2571 * the main block device spawning the RPMB character device.
2572 */
2573static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd,
2574 unsigned long arg)
2575{
2576 struct mmc_rpmb_data *rpmb = filp->private_data;
2577 int ret;
2578
2579 switch (cmd) {
2580 case MMC_IOC_CMD:
2581 ret = mmc_blk_ioctl_cmd(rpmb->md,
2582 (struct mmc_ioc_cmd __user *)arg,
2583 rpmb);
2584 break;
2585 case MMC_IOC_MULTI_CMD:
2586 ret = mmc_blk_ioctl_multi_cmd(rpmb->md,
2587 (struct mmc_ioc_multi_cmd __user *)arg,
2588 rpmb);
2589 break;
2590 default:
2591 ret = -EINVAL;
2592 break;
2593 }
2594
b25b750d 2595 return ret;
97548575
LW		 2596}
2597
2598#ifdef CONFIG_COMPAT
2599static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd,
2600 unsigned long arg)
2601{
2602 return mmc_rpmb_ioctl(filp, cmd, (unsigned long)compat_ptr(arg));
2603}
2604#endif
2605
2606static int mmc_rpmb_chrdev_open(struct inode *inode, struct file *filp)
2607{
2608 struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2609 struct mmc_rpmb_data, chrdev);
2610
2611 get_device(&rpmb->dev);
2612 filp->private_data = rpmb;
1c87f735 2613 mmc_blk_get(rpmb->md->disk);
97548575
LW		 2614
2615 return nonseekable_open(inode, filp);
2616}
2617
2618static int mmc_rpmb_chrdev_release(struct inode *inode, struct file *filp)
2619{
2620 struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2621 struct mmc_rpmb_data, chrdev);
2622
1c87f735 2623 mmc_blk_put(rpmb->md);
202500d2 2624 put_device(&rpmb->dev);
97548575
LW		 2625
2626 return 0;
2627}
2628
2629static const struct file_operations mmc_rpmb_fileops = {
2630 .release = mmc_rpmb_chrdev_release,
2631 .open = mmc_rpmb_chrdev_open,
2632 .owner = THIS_MODULE,
2633 .llseek = no_llseek,
2634 .unlocked_ioctl = mmc_rpmb_ioctl,
2635#ifdef CONFIG_COMPAT
2636 .compat_ioctl = mmc_rpmb_ioctl_compat,
2637#endif
2638};
2639
1c87f735
LW		 2640static void mmc_blk_rpmb_device_release(struct device *dev)
2641{
2642 struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
2643
2644 ida_simple_remove(&mmc_rpmb_ida, rpmb->id);
2645 kfree(rpmb);
2646}
97548575
LW		 2647
2648static int mmc_blk_alloc_rpmb_part(struct mmc_card *card,
2649 struct mmc_blk_data *md,
2650 unsigned int part_index,
2651 sector_t size,
2652 const char *subname)
2653{
2654 int devidx, ret;
2655 char rpmb_name[DISK_NAME_LEN];
2656 char cap_str[10];
2657 struct mmc_rpmb_data *rpmb;
2658
2659 /* This creates the minor number for the RPMB char device */
2660 devidx = ida_simple_get(&mmc_rpmb_ida, 0, max_devices, GFP_KERNEL);
2661 if (devidx < 0)
2662 return devidx;
2663
2664 rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL);
1c87f735
LW		 2665 if (!rpmb) {
2666 ida_simple_remove(&mmc_rpmb_ida, devidx);
97548575 2667 return -ENOMEM;
1c87f735 2668 }
97548575
LW		 2669
2670 snprintf(rpmb_name, sizeof(rpmb_name),
2671 "mmcblk%u%s", card->host->index, subname ? subname : "");
2672
2673 rpmb->id = devidx;
2674 rpmb->part_index = part_index;
2675 rpmb->dev.init_name = rpmb_name;
2676 rpmb->dev.bus = &mmc_rpmb_bus_type;
2677 rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id);
2678 rpmb->dev.parent = &card->dev;
1c87f735 2679 rpmb->dev.release = mmc_blk_rpmb_device_release;
97548575
LW		 2680 device_initialize(&rpmb->dev);
2681 dev_set_drvdata(&rpmb->dev, rpmb);
2682 rpmb->md = md;
2683
2684 cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops);
2685 rpmb->chrdev.owner = THIS_MODULE;
2686 ret = cdev_device_add(&rpmb->chrdev, &rpmb->dev);
2687 if (ret) {
2688 pr_err("%s: could not add character device\n", rpmb_name);
1c87f735 2689 goto out_put_device;
97548575
LW		 2690 }
2691
2692 list_add(&rpmb->node, &md->rpmbs);
2693
2694 string_get_size((u64)size, 512, STRING_UNITS_2,
2695 cap_str, sizeof(cap_str));
2696
ce999ed1
UH		 2697 pr_info("%s: %s %s %s, chardev (%d:%d)\n",
2698 rpmb_name, mmc_card_id(card), mmc_card_name(card), cap_str,
97548575
LW		 2699 MAJOR(mmc_rpmb_devt), rpmb->id);
2700
2701 return 0;
2702
1c87f735
LW		 2703out_put_device:
2704 put_device(&rpmb->dev);
97548575
LW		 2705 return ret;
2706}
2707
2708static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb)
1c87f735 2709
97548575
LW		 2710{
2711 cdev_device_del(&rpmb->chrdev, &rpmb->dev);
1c87f735 2712 put_device(&rpmb->dev);
97548575
LW		 2713}
2714
e0c368d5
NJ		 2715/* MMC Physical partitions consist of two boot partitions and
2716 * up to four general purpose partitions.
2717 * For each partition enabled in EXT_CSD a block device will be allocatedi
2718 * to provide access to the partition.
2719 */
2720
371a689f
AW		 2721static int mmc_blk_alloc_parts(struct mmc_card *card, struct mmc_blk_data *md)
2722{
97548575 2723 int idx, ret;
371a689f
AW		 2724
2725 if (!mmc_card_mmc(card))
2726 return 0;
2727
e0c368d5 2728 for (idx = 0; idx < card->nr_parts; idx++) {
97548575
LW		 2729 if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) {
2730 /*
2731 * RPMB partitions does not provide block access, they
2732 * are only accessed using ioctl():s. Thus create
2733 * special RPMB block devices that do not have a
2734 * backing block queue for these.
2735 */
2736 ret = mmc_blk_alloc_rpmb_part(card, md,
2737 card->part[idx].part_cfg,
2738 card->part[idx].size >> 9,
2739 card->part[idx].name);
2740 if (ret)
2741 return ret;
2742 } else if (card->part[idx].size) {
e0c368d5
NJ		 2743 ret = mmc_blk_alloc_part(card, md,
2744 card->part[idx].part_cfg,
2745 card->part[idx].size >> 9,
2746 card->part[idx].force_ro,
add710ea
JR		 2747 card->part[idx].name,
2748 card->part[idx].area_type);
e0c368d5
NJ		 2749 if (ret)
2750 return ret;
2751 }
371a689f
AW		 2752 }
2753
97548575 2754 return 0;
1da177e4
LT		 2755}
2756
371a689f
AW		 2757static void mmc_blk_remove_req(struct mmc_blk_data *md)
2758{
a94dcfce
CH		 2759 /*
2760 * Flush remaining requests and free queues. It is freeing the queue
2761 * that stops new requests from being accepted.
2762 */
2763 del_gendisk(md->disk);
2764 mmc_cleanup_queue(&md->queue);
2765 mmc_blk_put(md);
371a689f
AW		 2766}
2767
2768static void mmc_blk_remove_parts(struct mmc_card *card,
2769 struct mmc_blk_data *md)
2770{
2771 struct list_head *pos, *q;
2772 struct mmc_blk_data *part_md;
97548575 2773 struct mmc_rpmb_data *rpmb;
371a689f 2774
97548575
LW		 2775 /* Remove RPMB partitions */
2776 list_for_each_safe(pos, q, &md->rpmbs) {
2777 rpmb = list_entry(pos, struct mmc_rpmb_data, node);
2778 list_del(pos);
2779 mmc_blk_remove_rpmb_part(rpmb);
2780 }
2781 /* Remove block partitions */
371a689f
AW		 2782 list_for_each_safe(pos, q, &md->part) {
2783 part_md = list_entry(pos, struct mmc_blk_data, part);
2784 list_del(pos);
2785 mmc_blk_remove_req(part_md);
2786 }
2787}
2788
627c3ccf
LW		 2789#ifdef CONFIG_DEBUG_FS
2790
2791static int mmc_dbg_card_status_get(void *data, u64 *val)
2792{
2793 struct mmc_card *card = data;
2794 struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
2795 struct mmc_queue *mq = &md->queue;
2796 struct request *req;
2797 int ret;
2798
2799 /* Ask the block layer about the card status */
0bf6d96c 2800 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);
fb8e456e
AH		 2801 if (IS_ERR(req))
2802 return PTR_ERR(req);
627c3ccf 2803 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
b84ba30b 2804 blk_execute_rq(req, false);
627c3ccf
LW		 2805 ret = req_to_mmc_queue_req(req)->drv_op_result;
2806 if (ret >= 0) {
2807 *val = ret;
2808 ret = 0;
2809 }
0bf6d96c 2810 blk_mq_free_request(req);
627c3ccf
LW		 2811
2812 return ret;
2813}
f6a3d9d9
Y		 2814DEFINE_DEBUGFS_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
2815 NULL, "%08llx\n");
627c3ccf
LW		 2816
2817/* That is two digits * 512 + 1 for newline */
2818#define EXT_CSD_STR_LEN 1025
2819
2820static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
2821{
2822 struct mmc_card *card = inode->i_private;
2823 struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
2824 struct mmc_queue *mq = &md->queue;
2825 struct request *req;
2826 char *buf;
2827 ssize_t n = 0;
2828 u8 *ext_csd;
2829 int err, i;
2830
2831 buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
2832 if (!buf)
2833 return -ENOMEM;
2834
2835 /* Ask the block layer for the EXT CSD */
0bf6d96c 2836 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);
fb8e456e
AH		 2837 if (IS_ERR(req)) {
2838 err = PTR_ERR(req);
2839 goto out_free;
2840 }
627c3ccf
LW		 2841 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
2842 req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
b84ba30b 2843 blk_execute_rq(req, false);
627c3ccf 2844 err = req_to_mmc_queue_req(req)->drv_op_result;
0bf6d96c 2845 blk_mq_free_request(req);
627c3ccf
LW		 2846 if (err) {
2847 pr_err("FAILED %d\n", err);
2848 goto out_free;
2849 }
2850
2851 for (i = 0; i < 512; i++)
2852 n += sprintf(buf + n, "%02x", ext_csd[i]);
2853 n += sprintf(buf + n, "\n");
2854
2855 if (n != EXT_CSD_STR_LEN) {
2856 err = -EINVAL;
0be55579 2857 kfree(ext_csd);
627c3ccf
LW		 2858 goto out_free;
2859 }
2860
2861 filp->private_data = buf;
2862 kfree(ext_csd);
2863 return 0;
2864
2865out_free:
2866 kfree(buf);
2867 return err;
2868}
2869
2870static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf,
2871 size_t cnt, loff_t *ppos)
2872{
2873 char *buf = filp->private_data;
2874
2875 return simple_read_from_buffer(ubuf, cnt, ppos,
2876 buf, EXT_CSD_STR_LEN);
2877}
2878
2879static int mmc_ext_csd_release(struct inode *inode, struct file *file)
2880{
2881 kfree(file->private_data);
2882 return 0;
2883}
2884
2885static const struct file_operations mmc_dbg_ext_csd_fops = {
2886 .open = mmc_ext_csd_open,
2887 .read = mmc_ext_csd_read,
2888 .release = mmc_ext_csd_release,
2889 .llseek = default_llseek,
2890};
2891
f9f0da98 2892static int mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
627c3ccf
LW		 2893{
2894 struct dentry *root;
2895
2896 if (!card->debugfs_root)
2897 return 0;
2898
2899 root = card->debugfs_root;
2900
2901 if (mmc_card_mmc(card) || mmc_card_sd(card)) {
f9f0da98 2902 md->status_dentry =
f6a3d9d9
Y		 2903 debugfs_create_file_unsafe("status", 0400, root,
2904 card,
2905 &mmc_dbg_card_status_fops);
f9f0da98 2906 if (!md->status_dentry)
627c3ccf
LW		 2907 return -EIO;
2908 }
2909
2910 if (mmc_card_mmc(card)) {
f9f0da98
AH		 2911 md->ext_csd_dentry =
2912 debugfs_create_file("ext_csd", S_IRUSR, root, card,
2913 &mmc_dbg_ext_csd_fops);
2914 if (!md->ext_csd_dentry)
627c3ccf
LW		 2915 return -EIO;
2916 }
2917
2918 return 0;
2919}
2920
f9f0da98
AH		 2921static void mmc_blk_remove_debugfs(struct mmc_card *card,
2922 struct mmc_blk_data *md)
2923{
2924 if (!card->debugfs_root)
2925 return;
2926
2927 if (!IS_ERR_OR_NULL(md->status_dentry)) {
2928 debugfs_remove(md->status_dentry);
2929 md->status_dentry = NULL;
2930 }
2931
2932 if (!IS_ERR_OR_NULL(md->ext_csd_dentry)) {
2933 debugfs_remove(md->ext_csd_dentry);
2934 md->ext_csd_dentry = NULL;
2935 }
2936}
627c3ccf
LW		 2937
2938#else
2939
f9f0da98 2940static int mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
627c3ccf
LW		 2941{
2942 return 0;
2943}
2944
f9f0da98
AH		 2945static void mmc_blk_remove_debugfs(struct mmc_card *card,
2946 struct mmc_blk_data *md)
2947{
2948}
2949
627c3ccf
LW		 2950#endif /* CONFIG_DEBUG_FS */
2951
96541bac 2952static int mmc_blk_probe(struct mmc_card *card)
1da177e4 2953{
a94dcfce 2954 struct mmc_blk_data *md;
6f1d3247 2955 int ret = 0;
a7bbb573 2956
912490db
PO		 2957 /*
2958 * Check that the card supports the command class(es) we need.
2959 */
2960 if (!(card->csd.cmdclass & CCC_BLOCK_READ))
1da177e4
LT		 2961 return -ENODEV;
2962
8c7cdbf9 2963 mmc_fixup_device(card, mmc_blk_fixups);
5204d00f 2964
dcf6e2e3
ZH		 2965 card->complete_wq = alloc_workqueue("mmc_complete",
2966 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1d848c28 2967 if (!card->complete_wq) {
dcf6e2e3
ZH		 2968 pr_err("Failed to create mmc completion workqueue");
2969 return -ENOMEM;
2970 }
2971
1da177e4 2972 md = mmc_blk_alloc(card);
6f1d3247
UH		 2973 if (IS_ERR(md)) {
2974 ret = PTR_ERR(md);
2975 goto out_free;
2976 }
1da177e4 2977
6f1d3247
UH		 2978 ret = mmc_blk_alloc_parts(card, md);
2979 if (ret)
371a689f
AW		 2980 goto out;
2981
627c3ccf 2982 /* Add two debugfs entries */
f9f0da98 2983 mmc_blk_add_debugfs(card, md);
627c3ccf 2984
e94cfef6
UH		 2985 pm_runtime_set_autosuspend_delay(&card->dev, 3000);
2986 pm_runtime_use_autosuspend(&card->dev);
2987
2988 /*
2989 * Don't enable runtime PM for SD-combo cards here. Leave that
2990 * decision to be taken during the SDIO init sequence instead.
2991 */
2992 if (card->type != MMC_TYPE_SD_COMBO) {
2993 pm_runtime_set_active(&card->dev);
2994 pm_runtime_enable(&card->dev);
2995 }
2996
1da177e4
LT		 2997 return 0;
2998
6f1d3247 2999out:
371a689f
AW		 3000 mmc_blk_remove_parts(card, md);
3001 mmc_blk_remove_req(md);
6f1d3247
UH		 3002out_free:
3003 destroy_workqueue(card->complete_wq);
3004 return ret;
1da177e4
LT		 3005}
3006
96541bac 3007static void mmc_blk_remove(struct mmc_card *card)
1da177e4 3008{
96541bac 3009 struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
1da177e4 3010
f9f0da98 3011 mmc_blk_remove_debugfs(card, md);
371a689f 3012 mmc_blk_remove_parts(card, md);
e94cfef6 3013 pm_runtime_get_sync(&card->dev);
65f9e20e
SL		 3014 if (md->part_curr != md->part_type) {
3015 mmc_claim_host(card->host);
3016 mmc_blk_part_switch(card, md->part_type);
3017 mmc_release_host(card->host);
3018 }
e94cfef6
UH		 3019 if (card->type != MMC_TYPE_SD_COMBO)
3020 pm_runtime_disable(&card->dev);
3021 pm_runtime_put_noidle(&card->dev);
371a689f 3022 mmc_blk_remove_req(md);
96541bac 3023 dev_set_drvdata(&card->dev, NULL);
dcf6e2e3 3024 destroy_workqueue(card->complete_wq);
1da177e4
LT		 3025}
3026
96541bac 3027static int _mmc_blk_suspend(struct mmc_card *card)
1da177e4 3028{
371a689f 3029 struct mmc_blk_data *part_md;
96541bac 3030 struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
1da177e4
LT		 3031
3032 if (md) {
3033 mmc_queue_suspend(&md->queue);
371a689f
AW		 3034 list_for_each_entry(part_md, &md->part, part) {
3035 mmc_queue_suspend(&part_md->queue);
3036 }
1da177e4
LT		 3037 }
3038 return 0;
3039}
3040
96541bac 3041static void mmc_blk_shutdown(struct mmc_card *card)
76287748 3042{
96541bac 3043 _mmc_blk_suspend(card);
76287748
UH		 3044}
3045
0967edc6
UH		 3046#ifdef CONFIG_PM_SLEEP
3047static int mmc_blk_suspend(struct device *dev)
76287748 3048{
96541bac
UH		 3049 struct mmc_card *card = mmc_dev_to_card(dev);
3050
3051 return _mmc_blk_suspend(card);
76287748
UH		 3052}
3053
0967edc6 3054static int mmc_blk_resume(struct device *dev)
1da177e4 3055{
371a689f 3056 struct mmc_blk_data *part_md;
fc95e30b 3057 struct mmc_blk_data *md = dev_get_drvdata(dev);
1da177e4
LT		 3058
3059 if (md) {
371a689f
AW		 3060 /*
3061 * Resume involves the card going into idle state,
3062 * so current partition is always the main one.
3063 */
3064 md->part_curr = md->part_type;
1da177e4 3065 mmc_queue_resume(&md->queue);
371a689f
AW		 3066 list_for_each_entry(part_md, &md->part, part) {
3067 mmc_queue_resume(&part_md->queue);
3068 }
1da177e4
LT		 3069 }
3070 return 0;
3071}
1da177e4
LT		 3072#endif
3073
0967edc6
UH		 3074static SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume);
3075
96541bac
UH		 3076static struct mmc_driver mmc_driver = {
3077 .drv = {
3078 .name = "mmcblk",
3079 .pm = &mmc_blk_pm_ops,
3080 },
1da177e4
LT		 3081 .probe = mmc_blk_probe,
3082 .remove = mmc_blk_remove,
76287748 3083 .shutdown = mmc_blk_shutdown,
1da177e4
LT		 3084};
3085
3086static int __init mmc_blk_init(void)
3087{
9d4e98e9 3088 int res;
1da177e4 3089
97548575
LW		 3090 res = bus_register(&mmc_rpmb_bus_type);
3091 if (res < 0) {
3092 pr_err("mmcblk: could not register RPMB bus type\n");
3093 return res;
3094 }
3095 res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb");
3096 if (res < 0) {
3097 pr_err("mmcblk: failed to allocate rpmb chrdev region\n");
3098 goto out_bus_unreg;
3099 }
3100
5e71b7a6
OJ		 3101 if (perdev_minors != CONFIG_MMC_BLOCK_MINORS)
3102 pr_info("mmcblk: using %d minors per device\n", perdev_minors);
3103
a26eba61 3104 max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors);
5e71b7a6 3105
fe6b4c88
PO		 3106 res = register_blkdev(MMC_BLOCK_MAJOR, "mmc");
3107 if (res)
97548575 3108 goto out_chrdev_unreg;
1da177e4 3109
9d4e98e9
AM		 3110 res = mmc_register_driver(&mmc_driver);
3111 if (res)
97548575 3112 goto out_blkdev_unreg;
1da177e4 3113
9d4e98e9 3114 return 0;
97548575
LW		 3115
3116out_blkdev_unreg:
9d4e98e9 3117 unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
97548575
LW		 3118out_chrdev_unreg:
3119 unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3120out_bus_unreg:
3121 bus_unregister(&mmc_rpmb_bus_type);
1da177e4
LT		 3122 return res;
3123}
3124
3125static void __exit mmc_blk_exit(void)
3126{
3127 mmc_unregister_driver(&mmc_driver);
fe6b4c88 3128 unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
97548575 3129 unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
d0a0852b 3130 bus_unregister(&mmc_rpmb_bus_type);
1da177e4
LT		 3131}
3132
3133module_init(mmc_blk_init);
3134module_exit(mmc_blk_exit);
3135
3136MODULE_LICENSE("GPL");
3137MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver");
3138
Linux 6.x block layer and io_uring tree(s)