projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dm raid: fix address sanitizer warning in raid_resume
[linux-block.git] / drivers / md / dm.c
CommitLineData
1da177e4
LT		 1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
784aae73 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
1da177e4
LT		 4 *
5 * This file is released under the GPL.
6 */
7
4cc96131
MS		 8#include "dm-core.h"
9#include "dm-rq.h"
51e5b2bd 10#include "dm-uevent.h"
91ccbbac 11#include "dm-ima.h"
1da177e4
LT		 12
13#include <linux/init.h>
14#include <linux/module.h>
48c9c27b 15#include <linux/mutex.h>
6958c1c6 16#include <linux/sched/mm.h>
174cd4b1 17#include <linux/sched/signal.h>
1da177e4
LT		 18#include <linux/blkpg.h>
19#include <linux/bio.h>
1da177e4 20#include <linux/mempool.h>
f26c5719 21#include <linux/dax.h>
1da177e4
LT		 22#include <linux/slab.h>
23#include <linux/idr.h>
7e026c8c 24#include <linux/uio.h>
3ac51e74 25#include <linux/hdreg.h>
3f77316d 26#include <linux/delay.h>
ffcc3936 27#include <linux/wait.h>
71cdb697 28#include <linux/pr.h>
b0b4d7c6 29#include <linux/refcount.h>
c6a564ff 30#include <linux/part_stat.h>
a892c8d5 31#include <linux/blk-crypto.h>
1e8d44bd 32#include <linux/blk-crypto-profile.h>
55782138 33
72d94861
AK		 34#define DM_MSG_PREFIX "core"
35
60935eb2
MB		 36/*
37 * Cookies are numeric values sent with CHANGE and REMOVE
38 * uevents while resuming, removing or renaming the device.
39 */
40#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
41#define DM_COOKIE_LENGTH 24
42
b99fdcdc
ML		 43/*
44 * For REQ_POLLED fs bio, this flag is set if we link mapped underlying
45 * dm_io into one list, and reuse bio->bi_private as the list head. Before
46 * ending this fs bio, we will recover its ->bi_private.
47 */
48#define REQ_DM_POLL_LIST REQ_DRV
49
1da177e4
LT		 50static const char *_name = DM_NAME;
51
52static unsigned int major = 0;
53static unsigned int _major = 0;
54
d15b774c
AK		 55static DEFINE_IDR(_minor_idr);
56
f32c10b0 57static DEFINE_SPINLOCK(_minor_lock);
2c140a24
MP		 58
59static void do_deferred_remove(struct work_struct *w);
60
61static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
62
acfe0ad7
MP		 63static struct workqueue_struct *deferred_remove_workqueue;
64
93e6442c
MP		 65atomic_t dm_global_event_nr = ATOMIC_INIT(0);
66DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
67
62e08243
MP		 68void dm_issue_global_event(void)
69{
70 atomic_inc(&dm_global_event_nr);
71 wake_up(&dm_global_eventq);
72}
73
442761fd
MS		 74DEFINE_STATIC_KEY_FALSE(stats_enabled);
75DEFINE_STATIC_KEY_FALSE(swap_bios_enabled);
76DEFINE_STATIC_KEY_FALSE(zoned_enabled);
77
1da177e4 78/*
64f52b0e 79 * One of these is allocated (on-stack) per original bio.
1da177e4 80 */
64f52b0e 81struct clone_info {
64f52b0e
MS		 82 struct dm_table *map;
83 struct bio *bio;
84 struct dm_io *io;
85 sector_t sector;
86 unsigned sector_count;
4edadf6d
MS		 87 bool is_abnormal_io:1;
88 bool submit_as_polled:1;
64f52b0e
MS		 89};
90
6c23f0bd
CH		 91static inline struct dm_target_io *clone_to_tio(struct bio *clone)
92{
93 return container_of(clone, struct dm_target_io, clone);
94}
95
64f52b0e
MS		 96void *dm_per_bio_data(struct bio *bio, size_t data_size)
97{
655f3aad 98 if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO))
62f26317
JX		 99 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
100 return (char *)bio - DM_IO_BIO_OFFSET - data_size;
64f52b0e
MS		 101}
102EXPORT_SYMBOL_GPL(dm_per_bio_data);
103
104struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
105{
106 struct dm_io *io = (struct dm_io *)((char *)data + data_size);
107 if (io->magic == DM_IO_MAGIC)
62f26317 108 return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
64f52b0e 109 BUG_ON(io->magic != DM_TIO_MAGIC);
62f26317 110 return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
64f52b0e
MS		 111}
112EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
113
114unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
115{
116 return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
117}
118EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
119
ba61fdd1
JM		 120#define MINOR_ALLOCED ((void *)-1)
121
115485e8 122#define DM_NUMA_NODE NUMA_NO_NODE
115485e8 123static int dm_numa_node = DM_NUMA_NODE;
faad87df 124
a666e5c0
MP		 125#define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE)
126static int swap_bios = DEFAULT_SWAP_BIOS;
127static int get_swap_bios(void)
128{
129 int latch = READ_ONCE(swap_bios);
130 if (unlikely(latch <= 0))
131 latch = DEFAULT_SWAP_BIOS;
132 return latch;
133}
134
86f1152b
BM		 135struct table_device {
136 struct list_head list;
b0b4d7c6 137 refcount_t count;
86f1152b
BM		 138 struct dm_dev dm_dev;
139};
140
e8603136
MS		 141/*
142 * Bio-based DM's mempools' reserved IOs set by the user.
143 */
4cc96131 144#define RESERVED_BIO_BASED_IOS 16
e8603136
MS		 145static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
146
115485e8
MS		 147static int __dm_get_module_param_int(int *module_param, int min, int max)
148{
6aa7de05 149 int param = READ_ONCE(*module_param);
115485e8
MS		 150 int modified_param = 0;
151 bool modified = true;
152
153 if (param < min)
154 modified_param = min;
155 else if (param > max)
156 modified_param = max;
157 else
158 modified = false;
159
160 if (modified) {
161 (void)cmpxchg(module_param, param, modified_param);
162 param = modified_param;
163 }
164
165 return param;
166}
167
4cc96131
MS		 168unsigned __dm_get_module_param(unsigned *module_param,
169 unsigned def, unsigned max)
f4790826 170{
6aa7de05 171 unsigned param = READ_ONCE(*module_param);
09c2d531 172 unsigned modified_param = 0;
f4790826 173
09c2d531
MS		 174 if (!param)
175 modified_param = def;
176 else if (param > max)
177 modified_param = max;
f4790826 178
09c2d531
MS		 179 if (modified_param) {
180 (void)cmpxchg(module_param, param, modified_param);
181 param = modified_param;
f4790826
MS		 182 }
183
09c2d531 184 return param;
f4790826
MS		 185}
186
e8603136
MS		 187unsigned dm_get_reserved_bio_based_ios(void)
188{
09c2d531 189 return __dm_get_module_param(&reserved_bio_based_ios,
4cc96131 190 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
e8603136
MS		 191}
192EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
193
115485e8
MS		 194static unsigned dm_get_numa_node(void)
195{
196 return __dm_get_module_param_int(&dm_numa_node,
197 DM_NUMA_NODE, num_online_nodes() - 1);
198}
199
1da177e4
LT		 200static int __init local_init(void)
201{
e689fbab 202 int r;
1ae49ea2 203
51e5b2bd 204 r = dm_uevent_init();
51157b4a 205 if (r)
e689fbab 206 return r;
51e5b2bd 207
acfe0ad7
MP		 208 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
209 if (!deferred_remove_workqueue) {
210 r = -ENOMEM;
211 goto out_uevent_exit;
212 }
213
1da177e4
LT		 214 _major = major;
215 r = register_blkdev(_major, _name);
51157b4a 216 if (r < 0)
acfe0ad7 217 goto out_free_workqueue;
1da177e4
LT		 218
219 if (!_major)
220 _major = r;
221
222 return 0;
51157b4a 223
acfe0ad7
MP		 224out_free_workqueue:
225 destroy_workqueue(deferred_remove_workqueue);
51157b4a
KU		 226out_uevent_exit:
227 dm_uevent_exit();
51157b4a
KU		 228
229 return r;
1da177e4
LT		 230}
231
232static void local_exit(void)
233{
2c140a24 234 flush_scheduled_work();
acfe0ad7 235 destroy_workqueue(deferred_remove_workqueue);
2c140a24 236
00d59405 237 unregister_blkdev(_major, _name);
51e5b2bd 238 dm_uevent_exit();
1da177e4
LT		 239
240 _major = 0;
241
242 DMINFO("cleaned up");
243}
244
b9249e55 245static int (*_inits[])(void) __initdata = {
1da177e4
LT		 246 local_init,
247 dm_target_init,
248 dm_linear_init,
249 dm_stripe_init,
952b3557 250 dm_io_init,
945fa4d2 251 dm_kcopyd_init,
1da177e4 252 dm_interface_init,
fd2ed4d2 253 dm_statistics_init,
1da177e4
LT		 254};
255
b9249e55 256static void (*_exits[])(void) = {
1da177e4
LT		 257 local_exit,
258 dm_target_exit,
259 dm_linear_exit,
260 dm_stripe_exit,
952b3557 261 dm_io_exit,
945fa4d2 262 dm_kcopyd_exit,
1da177e4 263 dm_interface_exit,
fd2ed4d2 264 dm_statistics_exit,
1da177e4
LT		 265};
266
267static int __init dm_init(void)
268{
269 const int count = ARRAY_SIZE(_inits);
1da177e4
LT		 270 int r, i;
271
f1cd6cb2
TS		 272#if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE))
273 DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled."
274 " Duplicate IMA measurements will not be recorded in the IMA log.");
275#endif
276
1da177e4
LT		 277 for (i = 0; i < count; i++) {
278 r = _inits[i]();
279 if (r)
280 goto bad;
281 }
282
283 return 0;
f1cd6cb2 284bad:
1da177e4
LT		 285 while (i--)
286 _exits[i]();
287
288 return r;
289}
290
291static void __exit dm_exit(void)
292{
293 int i = ARRAY_SIZE(_exits);
294
295 while (i--)
296 _exits[i]();
d15b774c
AK		 297
298 /*
299 * Should be empty by this point.
300 */
d15b774c 301 idr_destroy(&_minor_idr);
1da177e4
LT		 302}
303
304/*
305 * Block device functions
306 */
432a212c
MA		 307int dm_deleting_md(struct mapped_device *md)
308{
309 return test_bit(DMF_DELETING, &md->flags);
310}
311
fe5f9f2c 312static int dm_blk_open(struct block_device *bdev, fmode_t mode)
1da177e4
LT		 313{
314 struct mapped_device *md;
315
fba9f90e
JM		 316 spin_lock(&_minor_lock);
317
fe5f9f2c 318 md = bdev->bd_disk->private_data;
fba9f90e
JM		 319 if (!md)
320 goto out;
321
5c6bd75d 322 if (test_bit(DMF_FREEING, &md->flags) ||
432a212c 323 dm_deleting_md(md)) {
fba9f90e
JM		 324 md = NULL;
325 goto out;
326 }
327
1da177e4 328 dm_get(md);
5c6bd75d 329 atomic_inc(&md->open_count);
fba9f90e
JM		 330out:
331 spin_unlock(&_minor_lock);
332
333 return md ? 0 : -ENXIO;
1da177e4
LT		 334}
335
db2a144b 336static void dm_blk_close(struct gendisk *disk, fmode_t mode)
1da177e4 337{
63a4f065 338 struct mapped_device *md;
6e9624b8 339
4a1aeb98
MB		 340 spin_lock(&_minor_lock);
341
63a4f065
MS		 342 md = disk->private_data;
343 if (WARN_ON(!md))
344 goto out;
345
2c140a24
MP		 346 if (atomic_dec_and_test(&md->open_count) &&
347 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
acfe0ad7 348 queue_work(deferred_remove_workqueue, &deferred_remove_work);
2c140a24 349
1da177e4 350 dm_put(md);
63a4f065 351out:
4a1aeb98 352 spin_unlock(&_minor_lock);
1da177e4
LT		 353}
354
5c6bd75d
AK		 355int dm_open_count(struct mapped_device *md)
356{
357 return atomic_read(&md->open_count);
358}
359
360/*
361 * Guarantees nothing is using the device before it's deleted.
362 */
2c140a24 363int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
5c6bd75d
AK		 364{
365 int r = 0;
366
367 spin_lock(&_minor_lock);
368
2c140a24 369 if (dm_open_count(md)) {
5c6bd75d 370 r = -EBUSY;
2c140a24
MP		 371 if (mark_deferred)
372 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
373 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
374 r = -EEXIST;
5c6bd75d
AK		 375 else
376 set_bit(DMF_DELETING, &md->flags);
377
378 spin_unlock(&_minor_lock);
379
380 return r;
381}
382
2c140a24
MP		 383int dm_cancel_deferred_remove(struct mapped_device *md)
384{
385 int r = 0;
386
387 spin_lock(&_minor_lock);
388
389 if (test_bit(DMF_DELETING, &md->flags))
390 r = -EBUSY;
391 else
392 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
393
394 spin_unlock(&_minor_lock);
395
396 return r;
397}
398
399static void do_deferred_remove(struct work_struct *w)
400{
401 dm_deferred_remove();
402}
403
3ac51e74
DW		 404static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
405{
406 struct mapped_device *md = bdev->bd_disk->private_data;
407
408 return dm_get_geometry(md, geo);
409}
410
971888c4 411static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
5bd5e8d8 412 struct block_device **bdev)
aa129a22 413{
564b5c54 414 struct dm_target *ti;
6c182cd8 415 struct dm_table *map;
971888c4 416 int r;
aa129a22 417
6c182cd8 418retry:
e56f81e0 419 r = -ENOTTY;
971888c4 420 map = dm_get_live_table(md, srcu_idx);
aa129a22 421 if (!map || !dm_table_get_size(map))
971888c4 422 return r;
aa129a22
MB		 423
424 /* We only support devices that have a single target */
2aec377a 425 if (map->num_targets != 1)
971888c4 426 return r;
aa129a22 427
564b5c54
MS		 428 ti = dm_table_get_target(map, 0);
429 if (!ti->type->prepare_ioctl)
971888c4 430 return r;
519049af 431
971888c4
MS		 432 if (dm_suspended_md(md))
433 return -EAGAIN;
aa129a22 434
564b5c54 435 r = ti->type->prepare_ioctl(ti, bdev);
5bbbfdf6 436 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
971888c4 437 dm_put_live_table(md, *srcu_idx);
6c182cd8
HR		 438 msleep(10);
439 goto retry;
440 }
971888c4 441
e56f81e0
CH		 442 return r;
443}
444
971888c4 445static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
971888c4
MS		 446{
447 dm_put_live_table(md, srcu_idx);
448}
449
e56f81e0
CH		 450static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
451 unsigned int cmd, unsigned long arg)
452{
453 struct mapped_device *md = bdev->bd_disk->private_data;
971888c4 454 int r, srcu_idx;
e56f81e0 455
5bd5e8d8 456 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
e56f81e0 457 if (r < 0)
971888c4 458 goto out;
6c182cd8 459
e56f81e0
CH		 460 if (r > 0) {
461 /*
e980f623
CH		 462 * Target determined this ioctl is being issued against a
463 * subset of the parent bdev; require extra privileges.
e56f81e0 464 */
e980f623 465 if (!capable(CAP_SYS_RAWIO)) {
0378c625 466 DMDEBUG_LIMIT(
e980f623
CH		 467 "%s: sending ioctl %x to DM device without required privilege.",
468 current->comm, cmd);
469 r = -ENOIOCTLCMD;
e56f81e0 470 goto out;
e980f623 471 }
e56f81e0 472 }
6c182cd8 473
a7cb3d2f
CH		 474 if (!bdev->bd_disk->fops->ioctl)
475 r = -ENOTTY;
476 else
477 r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
e56f81e0 478out:
971888c4 479 dm_unprepare_ioctl(md, srcu_idx);
aa129a22
MB		 480 return r;
481}
482
7465d7ac
MS		 483u64 dm_start_time_ns_from_clone(struct bio *bio)
484{
6c23f0bd 485 return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
7465d7ac
MS		 486}
487EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
488
8d394bc4 489static bool bio_is_flush_with_data(struct bio *bio)
7465d7ac 490{
8d394bc4
MS		 491 return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
492}
493
e6926ad0 494static void dm_io_acct(struct dm_io *io, bool end)
8d394bc4 495{
e6926ad0
ML		 496 struct dm_stats_aux *stats_aux = &io->stats_aux;
497 unsigned long start_time = io->start_time;
498 struct mapped_device *md = io->md;
499 struct bio *bio = io->orig_bio;
d3de6d12
ML		 500 unsigned int sectors;
501
502 /*
503 * If REQ_PREFLUSH set, don't account payload, it will be
504 * submitted (and accounted) after this flush completes.
505 */
506 if (bio_is_flush_with_data(bio))
507 sectors = 0;
7dd76d1f 508 else if (likely(!(dm_io_flagged(io, DM_IO_WAS_SPLIT))))
d3de6d12 509 sectors = bio_sectors(bio);
7dd76d1f
ML		 510 else
511 sectors = io->sectors;
8d394bc4
MS		 512
513 if (!end)
d3de6d12
ML		 514 bdev_start_io_acct(bio->bi_bdev, sectors, bio_op(bio),
515 start_time);
8d394bc4 516 else
d3de6d12 517 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
7465d7ac 518
442761fd 519 if (static_branch_unlikely(&stats_enabled) &&
7dd76d1f
ML		 520 unlikely(dm_stats_used(&md->stats))) {
521 sector_t sector;
522
523 if (likely(!dm_io_flagged(io, DM_IO_WAS_SPLIT)))
524 sector = bio->bi_iter.bi_sector;
525 else
526 sector = bio_end_sector(bio) - io->sector_offset;
527
7465d7ac 528 dm_stats_account_io(&md->stats, bio_data_dir(bio),
7dd76d1f 529 sector, sectors,
8d394bc4 530 end, start_time, stats_aux);
7dd76d1f 531 }
8d394bc4
MS		 532}
533
b992b40d 534static void __dm_start_io_acct(struct dm_io *io)
8d394bc4 535{
e6926ad0 536 dm_io_acct(io, false);
7465d7ac
MS		 537}
538
0fbb4d93 539static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
7465d7ac 540{
0fbb4d93
MS		 541 /*
542 * Ensure IO accounting is only ever started once.
0fbb4d93 543 */
3b03f7c1
MS		 544 if (dm_io_flagged(io, DM_IO_ACCOUNTED))
545 return;
546
547 /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */
548 if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) {
82f6cdcc
MS		 549 dm_io_set_flag(io, DM_IO_ACCOUNTED);
550 } else {
551 unsigned long flags;
655f3aad 552 /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */
4d7bca13 553 spin_lock_irqsave(&io->lock, flags);
10eb3a0d
BM		 554 if (dm_io_flagged(io, DM_IO_ACCOUNTED)) {
555 spin_unlock_irqrestore(&io->lock, flags);
556 return;
557 }
82f6cdcc 558 dm_io_set_flag(io, DM_IO_ACCOUNTED);
4d7bca13 559 spin_unlock_irqrestore(&io->lock, flags);
82f6cdcc 560 }
7465d7ac 561
b992b40d 562 __dm_start_io_acct(io);
0fbb4d93 563}
7465d7ac 564
b992b40d 565static void dm_end_io_acct(struct dm_io *io)
0fbb4d93 566{
e6926ad0 567 dm_io_acct(io, true);
7465d7ac 568}
978e51ba
MS		 569
570static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
1da177e4 571{
64f52b0e
MS		 572 struct dm_io *io;
573 struct dm_target_io *tio;
574 struct bio *clone;
575
29dec90a 576 clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs);
ca522482
MS		 577 /* Set default bdev, but target must bio_set_dev() before issuing IO */
578 clone->bi_bdev = md->disk->part0;
64f52b0e 579
6c23f0bd 580 tio = clone_to_tio(clone);
655f3aad
MS		 581 tio->flags = 0;
582 dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);
64f52b0e
MS		 583 tio->io = NULL;
584
585 io = container_of(tio, struct dm_io, tio);
586 io->magic = DM_IO_MAGIC;
84b98f4c 587 io->status = BLK_STS_OK;
0f14d60a
ML		 588
589 /* one ref is for submission, the other is for completion */
590 atomic_set(&io->io_count, 2);
9f6dc633 591 this_cpu_inc(*md->pending_io);
7dd76d1f 592 io->orig_bio = bio;
978e51ba 593 io->md = md;
4d7bca13 594 spin_lock_init(&io->lock);
b879f915 595 io->start_time = jiffies;
82f6cdcc 596 io->flags = 0;
64f52b0e 597
442761fd
MS		 598 if (static_branch_unlikely(&stats_enabled))
599 dm_stats_record_start(&md->stats, &io->stats_aux);
64f52b0e
MS		 600
601 return io;
1da177e4
LT		 602}
603
0119ab14 604static void free_io(struct dm_io *io)
1da177e4 605{
64f52b0e
MS		 606 bio_put(&io->tio.clone);
607}
608
1d1068ce 609static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti,
ca522482 610 unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask)
64f52b0e
MS		 611{
612 struct dm_target_io *tio;
018b05eb 613 struct bio *clone;
64f52b0e
MS		 614
615 if (!ci->io->tio.io) {
616 /* the dm_target_io embedded in ci->io is available */
617 tio = &ci->io->tio;
018b05eb
MS		 618 /* alloc_io() already initialized embedded clone */
619 clone = &tio->clone;
64f52b0e 620 } else {
ca522482
MS		 621 struct mapped_device *md = ci->io->md;
622
29dec90a
CH		 623 clone = bio_alloc_clone(NULL, ci->bio, gfp_mask,
624 &md->mempools->bs);
64f52b0e
MS		 625 if (!clone)
626 return NULL;
ca522482
MS		 627 /* Set default bdev, but target must bio_set_dev() before issuing IO */
628 clone->bi_bdev = md->disk->part0;
64f52b0e 629
b99fdcdc
ML		 630 /* REQ_DM_POLL_LIST shouldn't be inherited */
631 clone->bi_opf &= ~REQ_DM_POLL_LIST;
632
6c23f0bd 633 tio = clone_to_tio(clone);
655f3aad 634 tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */
64f52b0e
MS		 635 }
636
637 tio->magic = DM_TIO_MAGIC;
638 tio->io = ci->io;
639 tio->ti = ti;
640 tio->target_bio_nr = target_bio_nr;
dc8e2021 641 tio->len_ptr = len;
743598f0 642 tio->old_sector = 0;
64f52b0e 643
018b05eb
MS		 644 if (len) {
645 clone->bi_iter.bi_size = to_bytes(*len);
646 if (bio_integrity(clone))
647 bio_integrity_trim(clone);
648 }
64f52b0e 649
018b05eb 650 return clone;
1da177e4
LT		 651}
652
1d1068ce 653static void free_tio(struct bio *clone)
1da177e4 654{
655f3aad 655 if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO))
64f52b0e 656 return;
1d1068ce 657 bio_put(clone);
1da177e4
LT		 658}
659
660/*
661 * Add the bio to the list of deferred io.
662 */
92c63902 663static void queue_io(struct mapped_device *md, struct bio *bio)
1da177e4 664{
05447420 665 unsigned long flags;
1da177e4 666
05447420 667 spin_lock_irqsave(&md->deferred_lock, flags);
1da177e4 668 bio_list_add(&md->deferred, bio);
05447420 669 spin_unlock_irqrestore(&md->deferred_lock, flags);
6a8736d1 670 queue_work(md->wq, &md->work);
1da177e4
LT		 671}
672
673/*
674 * Everyone (including functions in this file), should use this
675 * function to access the md->map field, and make sure they call
83d5e5b0 676 * dm_put_live_table() when finished.
1da177e4 677 */
563a225c
MS		 678struct dm_table *dm_get_live_table(struct mapped_device *md,
679 int *srcu_idx) __acquires(md->io_barrier)
1da177e4 680{
83d5e5b0
MP		 681 *srcu_idx = srcu_read_lock(&md->io_barrier);
682
683 return srcu_dereference(md->map, &md->io_barrier);
684}
1da177e4 685
563a225c
MS		 686void dm_put_live_table(struct mapped_device *md,
687 int srcu_idx) __releases(md->io_barrier)
83d5e5b0
MP		 688{
689 srcu_read_unlock(&md->io_barrier, srcu_idx);
690}
691
692void dm_sync_table(struct mapped_device *md)
693{
694 synchronize_srcu(&md->io_barrier);
695 synchronize_rcu_expedited();
696}
697
698/*
699 * A fast alternative to dm_get_live_table/dm_put_live_table.
700 * The caller must not block between these two functions.
701 */
702static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
703{
704 rcu_read_lock();
705 return rcu_dereference(md->map);
706}
1da177e4 707
83d5e5b0
MP		 708static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
709{
710 rcu_read_unlock();
1da177e4
LT		 711}
712
563a225c 713static inline struct dm_table *dm_get_live_table_bio(struct mapped_device *md,
5d7362d0 714 int *srcu_idx, unsigned bio_opf)
563a225c 715{
5d7362d0 716 if (bio_opf & REQ_NOWAIT)
563a225c
MS		 717 return dm_get_live_table_fast(md);
718 else
719 return dm_get_live_table(md, srcu_idx);
720}
721
722static inline void dm_put_live_table_bio(struct mapped_device *md, int srcu_idx,
5d7362d0 723 unsigned bio_opf)
563a225c 724{
5d7362d0 725 if (bio_opf & REQ_NOWAIT)
563a225c
MS		 726 dm_put_live_table_fast(md);
727 else
728 dm_put_live_table(md, srcu_idx);
729}
730
971888c4
MS		 731static char *_dm_claim_ptr = "I belong to device-mapper";
732
86f1152b
BM		 733/*
734 * Open a table device so we can use it as a map destination.
735 */
736static int open_table_device(struct table_device *td, dev_t dev,
737 struct mapped_device *md)
738{
86f1152b 739 struct block_device *bdev;
cd913c76 740 u64 part_off;
86f1152b
BM		 741 int r;
742
743 BUG_ON(td->dm_dev.bdev);
744
519049af 745 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
86f1152b
BM		 746 if (IS_ERR(bdev))
747 return PTR_ERR(bdev);
748
749 r = bd_link_disk_holder(bdev, dm_disk(md));
750 if (r) {
751 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
752 return r;
753 }
754
755 td->dm_dev.bdev = bdev;
cd913c76 756 td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off);
86f1152b
BM		 757 return 0;
758}
759
760/*
761 * Close a table device that we've been using.
762 */
763static void close_table_device(struct table_device *td, struct mapped_device *md)
764{
765 if (!td->dm_dev.bdev)
766 return;
767
768 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
769 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
817bf402 770 put_dax(td->dm_dev.dax_dev);
86f1152b 771 td->dm_dev.bdev = NULL;
817bf402 772 td->dm_dev.dax_dev = NULL;
86f1152b
BM		 773}
774
775static struct table_device *find_table_device(struct list_head *l, dev_t dev,
8454fca4
SS		 776 fmode_t mode)
777{
86f1152b
BM		 778 struct table_device *td;
779
780 list_for_each_entry(td, l, list)
781 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
782 return td;
783
784 return NULL;
785}
786
787int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
8454fca4
SS		 788 struct dm_dev **result)
789{
86f1152b
BM		 790 int r;
791 struct table_device *td;
792
793 mutex_lock(&md->table_devices_lock);
794 td = find_table_device(&md->table_devices, dev, mode);
795 if (!td) {
115485e8 796 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
86f1152b
BM		 797 if (!td) {
798 mutex_unlock(&md->table_devices_lock);
799 return -ENOMEM;
800 }
801
802 td->dm_dev.mode = mode;
803 td->dm_dev.bdev = NULL;
804
805 if ((r = open_table_device(td, dev, md))) {
806 mutex_unlock(&md->table_devices_lock);
807 kfree(td);
808 return r;
809 }
810
811 format_dev_t(td->dm_dev.name, dev);
812
b0b4d7c6 813 refcount_set(&td->count, 1);
86f1152b 814 list_add(&td->list, &md->table_devices);
b0b4d7c6
ER		 815 } else {
816 refcount_inc(&td->count);
86f1152b 817 }
86f1152b
BM		 818 mutex_unlock(&md->table_devices_lock);
819
820 *result = &td->dm_dev;
821 return 0;
822}
86f1152b
BM		 823
824void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
825{
826 struct table_device *td = container_of(d, struct table_device, dm_dev);
827
828 mutex_lock(&md->table_devices_lock);
b0b4d7c6 829 if (refcount_dec_and_test(&td->count)) {
86f1152b
BM		 830 close_table_device(td, md);
831 list_del(&td->list);
832 kfree(td);
833 }
834 mutex_unlock(&md->table_devices_lock);
835}
86f1152b
BM		 836
837static void free_table_devices(struct list_head *devices)
838{
839 struct list_head *tmp, *next;
840
841 list_for_each_safe(tmp, next, devices) {
842 struct table_device *td = list_entry(tmp, struct table_device, list);
843
844 DMWARN("dm_destroy: %s still exists with %d references",
b0b4d7c6 845 td->dm_dev.name, refcount_read(&td->count));
86f1152b
BM		 846 kfree(td);
847 }
848}
849
3ac51e74
DW		 850/*
851 * Get the geometry associated with a dm device
852 */
853int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
854{
855 *geo = md->geometry;
856
857 return 0;
858}
859
860/*
861 * Set the geometry of a device.
862 */
863int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
864{
865 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
866
867 if (geo->start > sz) {
868 DMWARN("Start sector is beyond the geometry limits.");
869 return -EINVAL;
870 }
871
872 md->geometry = *geo;
873
874 return 0;
875}
876
2e93ccc1
KU		 877static int __noflush_suspending(struct mapped_device *md)
878{
879 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
880}
881
8b211aac
ML		 882static void dm_requeue_add_io(struct dm_io *io, bool first_stage)
883{
884 struct mapped_device *md = io->md;
885
886 if (first_stage) {
887 struct dm_io *next = md->requeue_list;
888
889 md->requeue_list = io;
890 io->next = next;
891 } else {
892 bio_list_add_head(&md->deferred, io->orig_bio);
893 }
894}
895
896static void dm_kick_requeue(struct mapped_device *md, bool first_stage)
897{
898 if (first_stage)
899 queue_work(md->wq, &md->requeue_work);
900 else
901 queue_work(md->wq, &md->work);
902}
903
444fe04f
ML		 904/*
905 * Return true if the dm_io's original bio is requeued.
906 * io->status is updated with error if requeue disallowed.
907 */
8b211aac 908static bool dm_handle_requeue(struct dm_io *io, bool first_stage)
1da177e4 909{
8b211aac 910 struct bio *bio = io->orig_bio;
444fe04f
ML		 911 bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE);
912 bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) &&
913 (bio->bi_opf & REQ_POLLED));
914 struct mapped_device *md = io->md;
915 bool requeued = false;
1da177e4 916
444fe04f 917 if (handle_requeue || handle_polled_eagain) {
e2736347 918 unsigned long flags;
444fe04f
ML		 919
920 if (bio->bi_opf & REQ_POLLED) {
921 /*
922 * Upper layer won't help us poll split bio
923 * (io->orig_bio may only reflect a subset of the
924 * pre-split original) so clear REQ_POLLED.
925 */
926 bio_clear_polled(bio);
927 }
928
e2736347 929 /*
444fe04f
ML		 930 * Target requested pushing back the I/O or
931 * polled IO hit BLK_STS_AGAIN.
e2736347
MS		 932 */
933 spin_lock_irqsave(&md->deferred_lock, flags);
444fe04f
ML		 934 if ((__noflush_suspending(md) &&
935 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) ||
8b211aac
ML		 936 handle_polled_eagain || first_stage) {
937 dm_requeue_add_io(io, first_stage);
444fe04f 938 requeued = true;
e2736347 939 } else {
2e93ccc1 940 /*
e2736347
MS		 941 * noflush suspend was interrupted or this is
942 * a write to a zoned target.
2e93ccc1 943 */
e2736347 944 io->status = BLK_STS_IOERR;
2e93ccc1 945 }
e2736347
MS		 946 spin_unlock_irqrestore(&md->deferred_lock, flags);
947 }
2e93ccc1 948
444fe04f 949 if (requeued)
8b211aac 950 dm_kick_requeue(md, first_stage);
444fe04f
ML		 951
952 return requeued;
953}
954
8b211aac 955static void __dm_io_complete(struct dm_io *io, bool first_stage)
444fe04f 956{
8b211aac 957 struct bio *bio = io->orig_bio;
444fe04f
ML		 958 struct mapped_device *md = io->md;
959 blk_status_t io_error;
960 bool requeued;
961
8b211aac
ML		 962 requeued = dm_handle_requeue(io, first_stage);
963 if (requeued && first_stage)
964 return;
444fe04f 965
e2736347 966 io_error = io->status;
82f6cdcc 967 if (dm_io_flagged(io, DM_IO_ACCOUNTED))
b992b40d 968 dm_end_io_acct(io);
e2736347
MS		 969 else if (!io_error) {
970 /*
971 * Must handle target that DM_MAPIO_SUBMITTED only to
972 * then bio_endio() rather than dm_submit_bio_remap()
973 */
b992b40d
ML		 974 __dm_start_io_acct(io);
975 dm_end_io_acct(io);
e2736347
MS		 976 }
977 free_io(io);
978 smp_wmb();
979 this_cpu_dec(*md->pending_io);
6a8736d1 980
e2736347
MS		 981 /* nudge anyone waiting on suspend queue */
982 if (unlikely(wq_has_sleeper(&md->wait)))
983 wake_up(&md->wait);
2e93ccc1 984
444fe04f
ML		 985 /* Return early if the original bio was requeued */
986 if (requeued)
987 return;
e2736347
MS		 988
989 if (bio_is_flush_with_data(bio)) {
990 /*
991 * Preflush done for flush with data, reissue
992 * without REQ_PREFLUSH.
993 */
994 bio->bi_opf &= ~REQ_PREFLUSH;
995 queue_io(md, bio);
996 } else {
997 /* done with normal IO or empty flush */
998 if (io_error)
999 bio->bi_status = io_error;
1000 bio_endio(bio);
1001 }
1002}
1003
8b211aac
ML		 1004static void dm_wq_requeue_work(struct work_struct *work)
1005{
1006 struct mapped_device *md = container_of(work, struct mapped_device,
1007 requeue_work);
1008 unsigned long flags;
1009 struct dm_io *io;
1010
1011 /* reuse deferred lock to simplify dm_handle_requeue */
1012 spin_lock_irqsave(&md->deferred_lock, flags);
1013 io = md->requeue_list;
1014 md->requeue_list = NULL;
1015 spin_unlock_irqrestore(&md->deferred_lock, flags);
1016
1017 while (io) {
1018 struct dm_io *next = io->next;
1019
1020 dm_io_rewind(io, &md->queue->bio_split);
1021
1022 io->next = NULL;
1023 __dm_io_complete(io, false);
1024 io = next;
1025 }
1026}
1027
1028/*
1029 * Two staged requeue:
1030 *
1031 * 1) io->orig_bio points to the real original bio, and the part mapped to
1032 * this io must be requeued, instead of other parts of the original bio.
1033 *
1034 * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io.
1035 */
1036static void dm_io_complete(struct dm_io *io)
1037{
1038 bool first_requeue;
1039
1040 /*
1041 * Only dm_io that has been split needs two stage requeue, otherwise
1042 * we may run into long bio clone chain during suspend and OOM could
1043 * be triggered.
1044 *
1045 * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they
1046 * also aren't handled via the first stage requeue.
1047 */
1048 if (dm_io_flagged(io, DM_IO_WAS_SPLIT))
1049 first_requeue = true;
1050 else
1051 first_requeue = false;
1052
1053 __dm_io_complete(io, first_requeue);
1054}
1055
1da177e4
LT		 1056/*
1057 * Decrements the number of outstanding ios that a bio has been
1058 * cloned into, completing the original io if necc.
1059 */
84b98f4c
MS		 1060static inline void __dm_io_dec_pending(struct dm_io *io)
1061{
1062 if (atomic_dec_and_test(&io->io_count))
1063 dm_io_complete(io);
1064}
1065
1066static void dm_io_set_error(struct dm_io *io, blk_status_t error)
1da177e4 1067{
84b98f4c
MS		 1068 unsigned long flags;
1069
2e93ccc1 1070 /* Push-back supersedes any I/O errors */
84b98f4c
MS		 1071 spin_lock_irqsave(&io->lock, flags);
1072 if (!(io->status == BLK_STS_DM_REQUEUE &&
1073 __noflush_suspending(io->md))) {
1074 io->status = error;
1da177e4 1075 }
84b98f4c
MS		 1076 spin_unlock_irqrestore(&io->lock, flags);
1077}
1da177e4 1078
2e803cd9 1079static void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
84b98f4c
MS		 1080{
1081 if (unlikely(error))
1082 dm_io_set_error(io, error);
1083
1084 __dm_io_dec_pending(io);
1da177e4
LT		 1085}
1086
bcb44433
MS		 1087void disable_discard(struct mapped_device *md)
1088{
1089 struct queue_limits *limits = dm_get_queue_limits(md);
1090
1091 /* device doesn't really support DISCARD, disable it */
1092 limits->max_discard_sectors = 0;
bcb44433
MS		 1093}
1094
ac62d620
CH		 1095void disable_write_zeroes(struct mapped_device *md)
1096{
1097 struct queue_limits *limits = dm_get_queue_limits(md);
1098
1099 /* device doesn't really support WRITE ZEROES, disable it */
1100 limits->max_write_zeroes_sectors = 0;
1101}
1102
a666e5c0
MP		 1103static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
1104{
1105 return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
1106}
1107
4246a0b6 1108static void clone_endio(struct bio *bio)
1da177e4 1109{
4e4cbee9 1110 blk_status_t error = bio->bi_status;
6c23f0bd 1111 struct dm_target_io *tio = clone_to_tio(bio);
6cbce280
MS		 1112 struct dm_target *ti = tio->ti;
1113 dm_endio_fn endio = ti->type->end_io;
b35f8caa 1114 struct dm_io *io = tio->io;
6cbce280 1115 struct mapped_device *md = io->md;
1da177e4 1116
dddf3056
MS		 1117 if (unlikely(error == BLK_STS_TARGET)) {
1118 if (bio_op(bio) == REQ_OP_DISCARD &&
1119 !bdev_max_discard_sectors(bio->bi_bdev))
1120 disable_discard(md);
1121 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
1122 !bdev_write_zeroes_sectors(bio->bi_bdev))
1123 disable_write_zeroes(md);
ca522482 1124 }
415c79e1 1125
dddf3056
MS		 1126 if (static_branch_unlikely(&zoned_enabled) &&
1127 unlikely(blk_queue_is_zoned(bdev_get_queue(bio->bi_bdev))))
1128 dm_zone_endio(io, bio);
1129
1be56909 1130 if (endio) {
6cbce280 1131 int r = endio(ti, bio, &error);
1be56909
CH		 1132 switch (r) {
1133 case DM_ENDIO_REQUEUE:
442761fd
MS		 1134 if (static_branch_unlikely(&zoned_enabled)) {
1135 /*
1136 * Requeuing writes to a sequential zone of a zoned
1137 * target will break the sequential write pattern:
1138 * fail such IO.
1139 */
1140 if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
1141 error = BLK_STS_IOERR;
1142 else
1143 error = BLK_STS_DM_REQUEUE;
1144 } else
bf14e2b2 1145 error = BLK_STS_DM_REQUEUE;
df561f66 1146 fallthrough;
1be56909
CH		 1147 case DM_ENDIO_DONE:
1148 break;
1149 case DM_ENDIO_INCOMPLETE:
1150 /* The target will handle the io */
1151 return;
1152 default:
1153 DMWARN("unimplemented target endio return value: %d", r);
1154 BUG();
1155 }
1156 }
1157
442761fd
MS		 1158 if (static_branch_unlikely(&swap_bios_enabled) &&
1159 unlikely(swap_bios_limit(ti, bio)))
a666e5c0 1160 up(&md->swap_bios_semaphore);
a666e5c0 1161
1d1068ce 1162 free_tio(bio);
e2118b3c 1163 dm_io_dec_pending(io, error);
1da177e4
LT		 1164}
1165
56a67df7
MS		 1166/*
1167 * Return maximum size of I/O possible at the supplied sector up to the current
1168 * target boundary.
1169 */
3720281d
MS		 1170static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
1171 sector_t target_offset)
56a67df7 1172{
56a67df7
MS		 1173 return ti->len - target_offset;
1174}
1175
3720281d 1176static sector_t max_io_len(struct dm_target *ti, sector_t sector)
1da177e4 1177{
3720281d
MS		 1178 sector_t target_offset = dm_target_offset(ti, sector);
1179 sector_t len = max_io_len_target_boundary(ti, target_offset);
1da177e4
LT		 1180
1181 /*
3ee16db3
MS		 1182 * Does the target need to split IO even further?
1183 * - varied (per target) IO splitting is a tenet of DM; this
1184 * explains why stacked chunk_sectors based splitting via
c3949322 1185 * blk_queue_split() isn't possible here.
1da177e4 1186 */
c3949322
CH		 1187 if (!ti->max_io_len)
1188 return len;
1189 return min_t(sector_t, len,
1190 min(queue_max_sectors(ti->table->md->queue),
1191 blk_chunk_sectors_left(target_offset, ti->max_io_len)));
1da177e4
LT		 1192}
1193
542f9038
MS		 1194int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1195{
1196 if (len > UINT_MAX) {
1197 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1198 (unsigned long long)len, UINT_MAX);
1199 ti->error = "Maximum size of target IO is too large";
1200 return -EINVAL;
1201 }
1202
75ae1936 1203 ti->max_io_len = (uint32_t) len;
542f9038
MS		 1204
1205 return 0;
1206}
1207EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1208
f26c5719 1209static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
3d97c829
MS		 1210 sector_t sector, int *srcu_idx)
1211 __acquires(md->io_barrier)
545ed20e 1212{
545ed20e
TK		 1213 struct dm_table *map;
1214 struct dm_target *ti;
545ed20e 1215
f26c5719 1216 map = dm_get_live_table(md, srcu_idx);
545ed20e 1217 if (!map)
f26c5719 1218 return NULL;
545ed20e
TK		 1219
1220 ti = dm_table_find_target(map, sector);
123d87d5 1221 if (!ti)
f26c5719 1222 return NULL;
545ed20e 1223
f26c5719
DW		 1224 return ti;
1225}
545ed20e 1226
f26c5719 1227static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
e511c4a3
JC		 1228 long nr_pages, enum dax_access_mode mode, void **kaddr,
1229 pfn_t *pfn)
f26c5719
DW		 1230{
1231 struct mapped_device *md = dax_get_private(dax_dev);
1232 sector_t sector = pgoff * PAGE_SECTORS;
1233 struct dm_target *ti;
1234 long len, ret = -EIO;
1235 int srcu_idx;
545ed20e 1236
f26c5719 1237 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
545ed20e 1238
f26c5719
DW		 1239 if (!ti)
1240 goto out;
1241 if (!ti->type->direct_access)
1242 goto out;
3720281d 1243 len = max_io_len(ti, sector) / PAGE_SECTORS;
f26c5719
DW		 1244 if (len < 1)
1245 goto out;
1246 nr_pages = min(len, nr_pages);
e511c4a3 1247 ret = ti->type->direct_access(ti, pgoff, nr_pages, mode, kaddr, pfn);
817bf402 1248
f26c5719 1249 out:
545ed20e 1250 dm_put_live_table(md, srcu_idx);
f26c5719
DW		 1251
1252 return ret;
545ed20e
TK		 1253}
1254
cdf6cdcd
VG		 1255static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1256 size_t nr_pages)
1257{
1258 struct mapped_device *md = dax_get_private(dax_dev);
1259 sector_t sector = pgoff * PAGE_SECTORS;
1260 struct dm_target *ti;
1261 int ret = -EIO;
1262 int srcu_idx;
1263
1264 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1265
1266 if (!ti)
1267 goto out;
1268 if (WARN_ON(!ti->type->dax_zero_page_range)) {
1269 /*
1270 * ->zero_page_range() is mandatory dax operation. If we are
1271 * here, something is wrong.
1272 */
cdf6cdcd
VG		 1273 goto out;
1274 }
1275 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
cdf6cdcd
VG		 1276 out:
1277 dm_put_live_table(md, srcu_idx);
1278
1279 return ret;
1280}
1281
047218ec
JC		 1282static size_t dm_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
1283 void *addr, size_t bytes, struct iov_iter *i)
1284{
1285 struct mapped_device *md = dax_get_private(dax_dev);
1286 sector_t sector = pgoff * PAGE_SECTORS;
1287 struct dm_target *ti;
1288 int srcu_idx;
1289 long ret = 0;
1290
1291 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1292 if (!ti || !ti->type->dax_recovery_write)
1293 goto out;
1294
1295 ret = ti->type->dax_recovery_write(ti, pgoff, addr, bytes, i);
1296out:
1297 dm_put_live_table(md, srcu_idx);
1298 return ret;
1299}
1300
1dd40c3e
MP		 1301/*
1302 * A target may call dm_accept_partial_bio only from the map routine. It is
6842d264 1303 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
e6fc9f62
MS		 1304 * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by
1305 * __send_duplicate_bios().
1dd40c3e
MP		 1306 *
1307 * dm_accept_partial_bio informs the dm that the target only wants to process
1308 * additional n_sectors sectors of the bio and the rest of the data should be
1309 * sent in a next bio.
1310 *
1311 * A diagram that explains the arithmetics:
1312 * +--------------------+---------------+-------+
1313 * | 1 | 2 | 3 |
1314 * +--------------------+---------------+-------+
1315 *
1316 * <-------------- *tio->len_ptr --------------->
bdb34759 1317 * <----- bio_sectors ----->
1dd40c3e
MP		 1318 * <-- n_sectors -->
1319 *
1320 * Region 1 was already iterated over with bio_advance or similar function.
1321 * (it may be empty if the target doesn't use bio_advance)
1322 * Region 2 is the remaining bio size that the target wants to process.
1323 * (it may be empty if region 1 is non-empty, although there is no reason
1324 * to make it empty)
1325 * The target requires that region 3 is to be sent in the next bio.
1326 *
1327 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1328 * the partially processed part (the sum of regions 1+2) must be the same for all
1329 * copies of the bio.
1330 */
1331void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1332{
6c23f0bd 1333 struct dm_target_io *tio = clone_to_tio(bio);
8b211aac 1334 struct dm_io *io = tio->io;
bdb34759 1335 unsigned bio_sectors = bio_sectors(bio);
6842d264 1336
655f3aad 1337 BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
6842d264
DLM		 1338 BUG_ON(op_is_zone_mgmt(bio_op(bio)));
1339 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
bdb34759
MS		 1340 BUG_ON(bio_sectors > *tio->len_ptr);
1341 BUG_ON(n_sectors > bio_sectors);
6842d264 1342
bdb34759 1343 *tio->len_ptr -= bio_sectors - n_sectors;
1dd40c3e 1344 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
7dd76d1f
ML		 1345
1346 /*
1347 * __split_and_process_bio() may have already saved mapped part
1348 * for accounting but it is being reduced so update accordingly.
1349 */
8b211aac
ML		 1350 dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1351 io->sectors = n_sectors;
1352 io->sector_offset = bio_sectors(io->orig_bio);
1dd40c3e
MP		 1353}
1354EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1355
0fbb4d93
MS		 1356/*
1357 * @clone: clone bio that DM core passed to target's .map function
1358 * @tgt_clone: clone of @clone bio that target needs submitted
0fbb4d93
MS		 1359 *
1360 * Targets should use this interface to submit bios they take
1361 * ownership of when returning DM_MAPIO_SUBMITTED.
1362 *
1363 * Target should also enable ti->accounts_remapped_io
1364 */
b7f8dff0 1365void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone)
0fbb4d93
MS		 1366{
1367 struct dm_target_io *tio = clone_to_tio(clone);
1368 struct dm_io *io = tio->io;
1369
1370 /* establish bio that will get submitted */
1371 if (!tgt_clone)
1372 tgt_clone = clone;
1373
1374 /*
1375 * Account io->origin_bio to DM dev on behalf of target
1376 * that took ownership of IO with DM_MAPIO_SUBMITTED.
1377 */
9d20653f 1378 dm_start_io_acct(io, clone);
0fbb4d93 1379
9d20653f 1380 trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk),
0fbb4d93 1381 tio->old_sector);
9d20653f 1382 submit_bio_noacct(tgt_clone);
0fbb4d93
MS		 1383}
1384EXPORT_SYMBOL_GPL(dm_submit_bio_remap);
1385
a666e5c0
MP		 1386static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1387{
1388 mutex_lock(&md->swap_bios_lock);
1389 while (latch < md->swap_bios) {
1390 cond_resched();
1391 down(&md->swap_bios_semaphore);
1392 md->swap_bios--;
1393 }
1394 while (latch > md->swap_bios) {
1395 cond_resched();
1396 up(&md->swap_bios_semaphore);
1397 md->swap_bios++;
1398 }
1399 mutex_unlock(&md->swap_bios_lock);
1400}
1401
1561b396 1402static void __map_bio(struct bio *clone)
1da177e4 1403{
1561b396 1404 struct dm_target_io *tio = clone_to_tio(clone);
bd2a49b8 1405 struct dm_target *ti = tio->ti;
6cbce280
MS		 1406 struct dm_io *io = tio->io;
1407 struct mapped_device *md = io->md;
1408 int r;
1da177e4 1409
1da177e4 1410 clone->bi_end_io = clone_endio;
1da177e4
LT		 1411
1412 /*
0fbb4d93 1413 * Map the clone.
1da177e4 1414 */
743598f0 1415 tio->old_sector = clone->bi_iter.bi_sector;
d67a5f4b 1416
442761fd
MS		 1417 if (static_branch_unlikely(&swap_bios_enabled) &&
1418 unlikely(swap_bios_limit(ti, clone))) {
a666e5c0
MP		 1419 int latch = get_swap_bios();
1420 if (unlikely(latch != md->swap_bios))
1421 __set_swap_bios_limit(md, latch);
1422 down(&md->swap_bios_semaphore);
1423 }
1424
442761fd
MS		 1425 if (static_branch_unlikely(&zoned_enabled)) {
1426 /*
1427 * Check if the IO needs a special mapping due to zone append
1428 * emulation on zoned target. In this case, dm_zone_map_bio()
1429 * calls the target map operation.
1430 */
1431 if (unlikely(dm_emulate_zone_append(md)))
1432 r = dm_zone_map_bio(tio);
1433 else
1434 r = ti->type->map(ti, clone);
1435 } else
bb37d772
DLM		 1436 r = ti->type->map(ti, clone);
1437
846785e6
CH		 1438 switch (r) {
1439 case DM_MAPIO_SUBMITTED:
0fbb4d93
MS		 1440 /* target has assumed ownership of this io */
1441 if (!ti->accounts_remapped_io)
9d20653f 1442 dm_start_io_acct(io, clone);
846785e6
CH		 1443 break;
1444 case DM_MAPIO_REMAPPED:
9d20653f 1445 dm_submit_bio_remap(clone, NULL);
846785e6
CH		 1446 break;
1447 case DM_MAPIO_KILL:
846785e6 1448 case DM_MAPIO_REQUEUE:
442761fd
MS		 1449 if (static_branch_unlikely(&swap_bios_enabled) &&
1450 unlikely(swap_bios_limit(ti, clone)))
6cbce280 1451 up(&md->swap_bios_semaphore);
1d1068ce 1452 free_tio(clone);
90a2326e
MS		 1453 if (r == DM_MAPIO_KILL)
1454 dm_io_dec_pending(io, BLK_STS_IOERR);
1455 else
1456 dm_io_dec_pending(io, BLK_STS_DM_REQUEUE);
846785e6
CH		 1457 break;
1458 default:
45cbcd79
KU		 1459 DMWARN("unimplemented target map return value: %d", r);
1460 BUG();
1da177e4
LT		 1461 }
1462}
1da177e4 1463
7dd76d1f
ML		 1464static void setup_split_accounting(struct clone_info *ci, unsigned len)
1465{
1466 struct dm_io *io = ci->io;
1467
1468 if (ci->sector_count > len) {
1469 /*
1470 * Split needed, save the mapped part for accounting.
1471 * NOTE: dm_accept_partial_bio() will update accordingly.
1472 */
1473 dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1474 io->sectors = len;
8b211aac 1475 io->sector_offset = bio_sectors(ci->bio);
7dd76d1f
ML		 1476 }
1477}
1478
318716dd 1479static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
7dd06a25 1480 struct dm_target *ti, unsigned num_bios)
f9ab94ce 1481{
1d1068ce 1482 struct bio *bio;
318716dd 1483 int try;
dba14160 1484
318716dd
MS		 1485 for (try = 0; try < 2; try++) {
1486 int bio_nr;
318716dd
MS		 1487
1488 if (try)
bc02cdbe 1489 mutex_lock(&ci->io->md->table_devices_lock);
318716dd 1490 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
7dd06a25 1491 bio = alloc_tio(ci, ti, bio_nr, NULL,
dc8e2021 1492 try ? GFP_NOIO : GFP_NOWAIT);
1d1068ce 1493 if (!bio)
318716dd
MS		 1494 break;
1495
1d1068ce 1496 bio_list_add(blist, bio);
318716dd
MS		 1497 }
1498 if (try)
bc02cdbe 1499 mutex_unlock(&ci->io->md->table_devices_lock);
318716dd
MS		 1500 if (bio_nr == num_bios)
1501 return;
1502
6c23f0bd 1503 while ((bio = bio_list_pop(blist)))
1d1068ce 1504 free_tio(bio);
318716dd 1505 }
9015df24
AK		 1506}
1507
0f14d60a 1508static int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
564b5c54 1509 unsigned int num_bios, unsigned *len)
06a426ce 1510{
318716dd 1511 struct bio_list blist = BIO_EMPTY_LIST;
8eabf5d0 1512 struct bio *clone;
564b5c54 1513 unsigned int ret = 0;
06a426ce 1514
891fced6
CH		 1515 switch (num_bios) {
1516 case 0:
1517 break;
1518 case 1:
7dd76d1f
ML		 1519 if (len)
1520 setup_split_accounting(ci, *len);
891fced6 1521 clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1561b396 1522 __map_bio(clone);
0f14d60a 1523 ret = 1;
891fced6
CH		 1524 break;
1525 default:
7dd06a25
MS		 1526 /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */
1527 alloc_multiple_bios(&blist, ci, ti, num_bios);
891fced6 1528 while ((clone = bio_list_pop(&blist))) {
655f3aad 1529 dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
891fced6 1530 __map_bio(clone);
0f14d60a 1531 ret += 1;
891fced6
CH		 1532 }
1533 break;
318716dd 1534 }
0f14d60a
ML		 1535
1536 return ret;
06a426ce
MS		 1537}
1538
332f2b1e 1539static void __send_empty_flush(struct clone_info *ci)
f9ab94ce 1540{
564b5c54 1541 struct dm_table *t = ci->map;
828678b8
MS		 1542 struct bio flush_bio;
1543
1544 /*
1545 * Use an on-stack bio for this, it's safe since we don't
1546 * need to reference it after submit. It's just used as
1547 * the basis for the clone(s).
1548 */
49add496
CH		 1549 bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0,
1550 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC);
47d95102 1551
828678b8
MS		 1552 ci->bio = &flush_bio;
1553 ci->sector_count = 0;
92b914e2 1554 ci->io->tio.clone.bi_iter.bi_size = 0;
f9ab94ce 1555
564b5c54
MS		 1556 for (unsigned int i = 0; i < t->num_targets; i++) {
1557 unsigned int bios;
1558 struct dm_target *ti = dm_table_get_target(t, i);
0f14d60a
ML		 1559
1560 atomic_add(ti->num_flush_bios, &ci->io->io_count);
1561 bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1562 atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count);
1563 }
1564
1565 /*
1566 * alloc_io() takes one extra reference for submission, so the
1567 * reference won't reach 0 without the following subtraction
1568 */
1569 atomic_sub(1, &ci->io->io_count);
828678b8
MS		 1570
1571 bio_uninit(ci->bio);
f9ab94ce
MP		 1572}
1573
e6fc9f62
MS		 1574static void __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
1575 unsigned num_bios)
ba1cbad9 1576{
51b86f9a 1577 unsigned len;
564b5c54 1578 unsigned int bios;
ba1cbad9 1579
3720281d
MS		 1580 len = min_t(sector_t, ci->sector_count,
1581 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
51b86f9a 1582
0f14d60a
ML		 1583 atomic_add(num_bios, &ci->io->io_count);
1584 bios = __send_duplicate_bios(ci, ti, num_bios, &len);
1585 /*
1586 * alloc_io() takes one extra reference for submission, so the
1587 * reference won't reach 0 without the following (+1) subtraction
1588 */
1589 atomic_sub(num_bios - bios + 1, &ci->io->io_count);
7dd06a25 1590
3d7f4562
MS		 1591 ci->sector += len;
1592 ci->sector_count -= len;
ba1cbad9
MS		 1593}
1594
568c73a3
MS		 1595static bool is_abnormal_io(struct bio *bio)
1596{
4edadf6d 1597 unsigned int op = bio_op(bio);
568c73a3 1598
4edadf6d
MS		 1599 if (op != REQ_OP_READ && op != REQ_OP_WRITE && op != REQ_OP_FLUSH) {
1600 switch (op) {
1601 case REQ_OP_DISCARD:
1602 case REQ_OP_SECURE_ERASE:
1603 case REQ_OP_WRITE_ZEROES:
1604 return true;
1605 default:
1606 break;
1607 }
568c73a3
MS		 1608 }
1609
4edadf6d 1610 return false;
568c73a3
MS		 1611}
1612
4edadf6d
MS		 1613static blk_status_t __process_abnormal_io(struct clone_info *ci,
1614 struct dm_target *ti)
0519c71e 1615{
9679b5a7 1616 unsigned num_bios = 0;
0519c71e 1617
e6fc9f62 1618 switch (bio_op(ci->bio)) {
9679b5a7
MS		 1619 case REQ_OP_DISCARD:
1620 num_bios = ti->num_discard_bios;
1621 break;
1622 case REQ_OP_SECURE_ERASE:
1623 num_bios = ti->num_secure_erase_bios;
1624 break;
9679b5a7
MS		 1625 case REQ_OP_WRITE_ZEROES:
1626 num_bios = ti->num_write_zeroes_bios;
1627 break;
9679b5a7 1628 }
0519c71e 1629
e6fc9f62
MS		 1630 /*
1631 * Even though the device advertised support for this type of
1632 * request, that does not mean every target supports it, and
1633 * reconfiguration might also have changed that since the
1634 * check was performed.
1635 */
84b98f4c 1636 if (unlikely(!num_bios))
4edadf6d
MS		 1637 return BLK_STS_NOTSUPP;
1638
1639 __send_changing_extent_only(ci, ti, num_bios);
1640 return BLK_STS_OK;
0519c71e
MS		 1641}
1642
b99fdcdc 1643/*
ec211631 1644 * Reuse ->bi_private as dm_io list head for storing all dm_io instances
b99fdcdc
ML		 1645 * associated with this bio, and this bio's bi_private needs to be
1646 * stored in dm_io->data before the reuse.
1647 *
1648 * bio->bi_private is owned by fs or upper layer, so block layer won't
1649 * touch it after splitting. Meantime it won't be changed by anyone after
1650 * bio is submitted. So this reuse is safe.
1651 */
ec211631 1652static inline struct dm_io **dm_poll_list_head(struct bio *bio)
b99fdcdc 1653{
ec211631 1654 return (struct dm_io **)&bio->bi_private;
b99fdcdc
ML		 1655}
1656
1657static void dm_queue_poll_io(struct bio *bio, struct dm_io *io)
1658{
ec211631 1659 struct dm_io **head = dm_poll_list_head(bio);
b99fdcdc
ML		 1660
1661 if (!(bio->bi_opf & REQ_DM_POLL_LIST)) {
1662 bio->bi_opf |= REQ_DM_POLL_LIST;
1663 /*
1664 * Save .bi_private into dm_io, so that we can reuse
ec211631 1665 * .bi_private as dm_io list head for storing dm_io list
b99fdcdc
ML		 1666 */
1667 io->data = bio->bi_private;
1668
b99fdcdc
ML		 1669 /* tell block layer to poll for completion */
1670 bio->bi_cookie = ~BLK_QC_T_NONE;
ec211631
ML		 1671
1672 io->next = NULL;
b99fdcdc
ML		 1673 } else {
1674 /*
1675 * bio recursed due to split, reuse original poll list,
1676 * and save bio->bi_private too.
1677 */
ec211631
ML		 1678 io->data = (*head)->data;
1679 io->next = *head;
b99fdcdc
ML		 1680 }
1681
ec211631 1682 *head = io;
b99fdcdc
ML		 1683}
1684
e4c93811
AK		 1685/*
1686 * Select the correct strategy for processing a non-flush bio.
1687 */
84b98f4c 1688static blk_status_t __split_and_process_bio(struct clone_info *ci)
0ce65797 1689{
66bdaa43 1690 struct bio *clone;
512875bd 1691 struct dm_target *ti;
1c3b13e6 1692 unsigned len;
0ce65797 1693
512875bd 1694 ti = dm_table_find_target(ci->map, ci->sector);
4edadf6d
MS		 1695 if (unlikely(!ti))
1696 return BLK_STS_IOERR;
1ee88de3
MP		 1697
1698 if (unlikely((ci->bio->bi_opf & REQ_NOWAIT) != 0) &&
1699 unlikely(!dm_target_supports_nowait(ti->type)))
1700 return BLK_STS_NOTSUPP;
1701
1702 if (unlikely(ci->is_abnormal_io))
4edadf6d 1703 return __process_abnormal_io(ci, ti);
3d7f4562 1704
b99fdcdc
ML		 1705 /*
1706 * Only support bio polling for normal IO, and the target io is
1707 * exactly inside the dm_io instance (verified in dm_poll_dm_io)
1708 */
1709 ci->submit_as_polled = ci->bio->bi_opf & REQ_POLLED;
0ce65797 1710
3720281d 1711 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
7dd76d1f 1712 setup_split_accounting(ci, len);
66bdaa43 1713 clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO);
66bdaa43 1714 __map_bio(clone);
0ce65797 1715
1c3b13e6
KO		 1716 ci->sector += len;
1717 ci->sector_count -= len;
0ce65797 1718
84b98f4c 1719 return BLK_STS_OK;
0ce65797
MS		 1720}
1721
978e51ba 1722static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
4edadf6d 1723 struct dm_table *map, struct bio *bio, bool is_abnormal)
978e51ba
MS		 1724{
1725 ci->map = map;
1726 ci->io = alloc_io(md, bio);
d41e077a 1727 ci->bio = bio;
4edadf6d 1728 ci->is_abnormal_io = is_abnormal;
b99fdcdc 1729 ci->submit_as_polled = false;
978e51ba 1730 ci->sector = bio->bi_iter.bi_sector;
d41e077a
MS		 1731 ci->sector_count = bio_sectors(bio);
1732
1733 /* Shouldn't happen but sector_count was being set to 0 so... */
442761fd
MS		 1734 if (static_branch_unlikely(&zoned_enabled) &&
1735 WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count))
d41e077a 1736 ci->sector_count = 0;
978e51ba
MS		 1737}
1738
1da177e4 1739/*
14fe594d 1740 * Entry point to split a bio into clones and submit them to the targets.
1da177e4 1741 */
96c9865c
MS		 1742static void dm_split_and_process_bio(struct mapped_device *md,
1743 struct dm_table *map, struct bio *bio)
0ce65797 1744{
1da177e4 1745 struct clone_info ci;
4857abf6 1746 struct dm_io *io;
84b98f4c 1747 blk_status_t error = BLK_STS_OK;
4edadf6d
MS		 1748 bool is_abnormal;
1749
1750 is_abnormal = is_abnormal_io(bio);
1751 if (unlikely(is_abnormal)) {
1752 /*
1753 * Use blk_queue_split() for abnormal IO (e.g. discard, etc)
1754 * otherwise associated queue_limits won't be imposed.
1755 */
1756 blk_queue_split(&bio);
1757 }
1da177e4 1758
4edadf6d 1759 init_clone_info(&ci, md, map, bio, is_abnormal);
4857abf6 1760 io = ci.io;
0ce65797 1761
1eff9d32 1762 if (bio->bi_opf & REQ_PREFLUSH) {
332f2b1e 1763 __send_empty_flush(&ci);
e2736347 1764 /* dm_io_complete submits any data associated with flush */
d41e077a 1765 goto out;
d87f4c14 1766 }
0ce65797 1767
d41e077a
MS		 1768 error = __split_and_process_bio(&ci);
1769 if (error || !ci.sector_count)
1770 goto out;
d41e077a
MS		 1771 /*
1772 * Remainder must be passed to submit_bio_noacct() so it gets handled
1773 * *after* bios already submitted have been completely processed.
d41e077a 1774 */
8b211aac
ML		 1775 bio_trim(bio, io->sectors, ci.sector_count);
1776 trace_block_split(bio, bio->bi_iter.bi_sector);
1777 bio_inc_remaining(bio);
d41e077a
MS		 1778 submit_bio_noacct(bio);
1779out:
b99fdcdc
ML		 1780 /*
1781 * Drop the extra reference count for non-POLLED bio, and hold one
1782 * reference for POLLED bio, which will be released in dm_poll_bio
1783 *
ec211631
ML		 1784 * Add every dm_io instance into the dm_io list head which is stored
1785 * in bio->bi_private, so that dm_poll_bio can poll them all.
b99fdcdc 1786 */
0f14d60a
ML		 1787 if (error || !ci.submit_as_polled) {
1788 /*
1789 * In case of submission failure, the extra reference for
1790 * submitting io isn't consumed yet
1791 */
1792 if (error)
1793 atomic_dec(&io->io_count);
1794 dm_io_dec_pending(io, error);
1795 } else
4857abf6 1796 dm_queue_poll_io(bio, io);
0ce65797
MS		 1797}
1798
3e08773c 1799static void dm_submit_bio(struct bio *bio)
cec47e3d 1800{
309dca30 1801 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
83d5e5b0
MP		 1802 int srcu_idx;
1803 struct dm_table *map;
5d7362d0 1804 unsigned bio_opf = bio->bi_opf;
cec47e3d 1805
5d7362d0 1806 map = dm_get_live_table_bio(md, &srcu_idx, bio_opf);
29e4013d 1807
fa247089
MS		 1808 /* If suspended, or map not yet available, queue this IO for later */
1809 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) ||
1810 unlikely(!map)) {
6abc4946
KK		 1811 if (bio->bi_opf & REQ_NOWAIT)
1812 bio_wouldblock_error(bio);
b2abdb1b 1813 else if (bio->bi_opf & REQ_RAHEAD)
54d9a1b4 1814 bio_io_error(bio);
b2abdb1b
MS		 1815 else
1816 queue_io(md, bio);
1817 goto out;
cec47e3d 1818 }
1da177e4 1819
96c9865c 1820 dm_split_and_process_bio(md, map, bio);
b2abdb1b 1821out:
5d7362d0 1822 dm_put_live_table_bio(md, srcu_idx, bio_opf);
978e51ba
MS		 1823}
1824
b99fdcdc
ML		 1825static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
1826 unsigned int flags)
1827{
655f3aad 1828 WARN_ON_ONCE(!dm_tio_is_normal(&io->tio));
b99fdcdc
ML		 1829
1830 /* don't poll if the mapped io is done */
1831 if (atomic_read(&io->io_count) > 1)
1832 bio_poll(&io->tio.clone, iob, flags);
1833
1834 /* bio_poll holds the last reference */
1835 return atomic_read(&io->io_count) == 1;
1836}
1837
1838static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob,
1839 unsigned int flags)
1840{
ec211631
ML		 1841 struct dm_io **head = dm_poll_list_head(bio);
1842 struct dm_io *list = *head;
1843 struct dm_io *tmp = NULL;
1844 struct dm_io *curr, *next;
b99fdcdc
ML		 1845
1846 /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */
1847 if (!(bio->bi_opf & REQ_DM_POLL_LIST))
1848 return 0;
1849
ec211631 1850 WARN_ON_ONCE(!list);
b99fdcdc
ML		 1851
1852 /*
1853 * Restore .bi_private before possibly completing dm_io.
1854 *
1855 * bio_poll() is only possible once @bio has been completely
1856 * submitted via submit_bio_noacct()'s depth-first submission.
1857 * So there is no dm_queue_poll_io() race associated with
1858 * clearing REQ_DM_POLL_LIST here.
1859 */
1860 bio->bi_opf &= ~REQ_DM_POLL_LIST;
ec211631 1861 bio->bi_private = list->data;
b99fdcdc 1862
ec211631
ML		 1863 for (curr = list, next = curr->next; curr; curr = next, next =
1864 curr ? curr->next : NULL) {
1865 if (dm_poll_dm_io(curr, iob, flags)) {
b99fdcdc 1866 /*
84b98f4c
MS		 1867 * clone_endio() has already occurred, so no
1868 * error handling is needed here.
b99fdcdc 1869 */
ec211631
ML		 1870 __dm_io_dec_pending(curr);
1871 } else {
1872 curr->next = tmp;
1873 tmp = curr;
b99fdcdc
ML		 1874 }
1875 }
1876
1877 /* Not done? */
ec211631 1878 if (tmp) {
b99fdcdc
ML		 1879 bio->bi_opf |= REQ_DM_POLL_LIST;
1880 /* Reset bio->bi_private to dm_io list head */
ec211631 1881 *head = tmp;
b99fdcdc
ML		 1882 return 0;
1883 }
1884 return 1;
1885}
1886
1da177e4
LT		 1887/*-----------------------------------------------------------------
1888 * An IDR is used to keep track of allocated minor numbers.
1889 *---------------------------------------------------------------*/
2b06cfff 1890static void free_minor(int minor)
1da177e4 1891{
f32c10b0 1892 spin_lock(&_minor_lock);
1da177e4 1893 idr_remove(&_minor_idr, minor);
f32c10b0 1894 spin_unlock(&_minor_lock);
1da177e4
LT		 1895}
1896
1897/*
1898 * See if the device with a specific minor # is free.
1899 */
cf13ab8e 1900static int specific_minor(int minor)
1da177e4 1901{
c9d76be6 1902 int r;
1da177e4
LT		 1903
1904 if (minor >= (1 << MINORBITS))
1905 return -EINVAL;
1906
c9d76be6 1907 idr_preload(GFP_KERNEL);
f32c10b0 1908 spin_lock(&_minor_lock);
1da177e4 1909
c9d76be6 1910 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1da177e4 1911
f32c10b0 1912 spin_unlock(&_minor_lock);
c9d76be6
TH		 1913 idr_preload_end();
1914 if (r < 0)
1915 return r == -ENOSPC ? -EBUSY : r;
1916 return 0;
1da177e4
LT		 1917}
1918
cf13ab8e 1919static int next_free_minor(int *minor)
1da177e4 1920{
c9d76be6 1921 int r;
62f75c2f 1922
c9d76be6 1923 idr_preload(GFP_KERNEL);
f32c10b0 1924 spin_lock(&_minor_lock);
1da177e4 1925
c9d76be6 1926 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1da177e4 1927
f32c10b0 1928 spin_unlock(&_minor_lock);
c9d76be6
TH		 1929 idr_preload_end();
1930 if (r < 0)
1931 return r;
1932 *minor = r;
1933 return 0;
1da177e4
LT		 1934}
1935
83d5cde4 1936static const struct block_device_operations dm_blk_dops;
681cc5e8 1937static const struct block_device_operations dm_rq_blk_dops;
f26c5719 1938static const struct dax_operations dm_dax_ops;
1da177e4 1939
53d5914f
MP		 1940static void dm_wq_work(struct work_struct *work);
1941
aa6ce87a 1942#ifdef CONFIG_BLK_INLINE_ENCRYPTION
cb77cb5a 1943static void dm_queue_destroy_crypto_profile(struct request_queue *q)
aa6ce87a 1944{
cb77cb5a 1945 dm_destroy_crypto_profile(q->crypto_profile);
aa6ce87a
ST		 1946}
1947
1948#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1949
cb77cb5a 1950static inline void dm_queue_destroy_crypto_profile(struct request_queue *q)
aa6ce87a
ST		 1951{
1952}
1953#endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1954
0f20972f
MS		 1955static void cleanup_mapped_device(struct mapped_device *md)
1956{
0f20972f
MS		 1957 if (md->wq)
1958 destroy_workqueue(md->wq);
29dec90a 1959 dm_free_md_mempools(md->mempools);
0f20972f 1960
f26c5719 1961 if (md->dax_dev) {
fb08a190 1962 dax_remove_host(md->disk);
f26c5719
DW		 1963 kill_dax(md->dax_dev);
1964 put_dax(md->dax_dev);
1965 md->dax_dev = NULL;
1966 }
1967
588b7f5d 1968 dm_cleanup_zoned_dev(md);
0f20972f
MS		 1969 if (md->disk) {
1970 spin_lock(&_minor_lock);
1971 md->disk->private_data = NULL;
1972 spin_unlock(&_minor_lock);
89f871af
CH		 1973 if (dm_get_md_type(md) != DM_TYPE_NONE) {
1974 dm_sysfs_exit(md);
1975 del_gendisk(md->disk);
1976 }
cb77cb5a 1977 dm_queue_destroy_crypto_profile(md->queue);
8b9ab626 1978 put_disk(md->disk);
74a2b6ec 1979 }
0f20972f 1980
9f6dc633
MS		 1981 if (md->pending_io) {
1982 free_percpu(md->pending_io);
1983 md->pending_io = NULL;
1984 }
1985
d09960b0
TE		 1986 cleanup_srcu_struct(&md->io_barrier);
1987
d5ffebdd
MS		 1988 mutex_destroy(&md->suspend_lock);
1989 mutex_destroy(&md->type_lock);
1990 mutex_destroy(&md->table_devices_lock);
a666e5c0 1991 mutex_destroy(&md->swap_bios_lock);
d5ffebdd 1992
4cc96131 1993 dm_mq_cleanup_mapped_device(md);
0f20972f
MS		 1994}
1995
1da177e4
LT		 1996/*
1997 * Allocate and initialise a blank device with a given minor.
1998 */
2b06cfff 1999static struct mapped_device *alloc_dev(int minor)
1da177e4 2000{
115485e8
MS		 2001 int r, numa_node_id = dm_get_numa_node();
2002 struct mapped_device *md;
ba61fdd1 2003 void *old_md;
1da177e4 2004
856eb091 2005 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1da177e4
LT		 2006 if (!md) {
2007 DMWARN("unable to allocate device, out of memory.");
2008 return NULL;
2009 }
2010
10da4f79 2011 if (!try_module_get(THIS_MODULE))
6ed7ade8 2012 goto bad_module_get;
10da4f79 2013
1da177e4 2014 /* get a minor number for the dev */
2b06cfff 2015 if (minor == DM_ANY_MINOR)
cf13ab8e 2016 r = next_free_minor(&minor);
2b06cfff 2017 else
cf13ab8e 2018 r = specific_minor(minor);
1da177e4 2019 if (r < 0)
6ed7ade8 2020 goto bad_minor;
1da177e4 2021
83d5e5b0
MP		 2022 r = init_srcu_struct(&md->io_barrier);
2023 if (r < 0)
2024 goto bad_io_barrier;
2025
115485e8 2026 md->numa_node_id = numa_node_id;
591ddcfc 2027 md->init_tio_pdu = false;
a5664dad 2028 md->type = DM_TYPE_NONE;
e61290a4 2029 mutex_init(&md->suspend_lock);
a5664dad 2030 mutex_init(&md->type_lock);
86f1152b 2031 mutex_init(&md->table_devices_lock);
022c2611 2032 spin_lock_init(&md->deferred_lock);
1da177e4 2033 atomic_set(&md->holders, 1);
5c6bd75d 2034 atomic_set(&md->open_count, 0);
1da177e4 2035 atomic_set(&md->event_nr, 0);
7a8c3d3b
MA		 2036 atomic_set(&md->uevent_seq, 0);
2037 INIT_LIST_HEAD(&md->uevent_list);
86f1152b 2038 INIT_LIST_HEAD(&md->table_devices);
7a8c3d3b 2039 spin_lock_init(&md->uevent_lock);
1da177e4 2040
47ace7e0 2041 /*
c62b37d9
CH		 2042 * default to bio-based until DM table is loaded and md->type
2043 * established. If request-based table is loaded: blk-mq will
2044 * override accordingly.
47ace7e0 2045 */
74fe6ba9 2046 md->disk = blk_alloc_disk(md->numa_node_id);
1da177e4 2047 if (!md->disk)
0f20972f 2048 goto bad;
74fe6ba9 2049 md->queue = md->disk->queue;
1da177e4 2050
f0b04115 2051 init_waitqueue_head(&md->wait);
53d5914f 2052 INIT_WORK(&md->work, dm_wq_work);
8b211aac 2053 INIT_WORK(&md->requeue_work, dm_wq_requeue_work);
f0b04115 2054 init_waitqueue_head(&md->eventq);
2995fa78 2055 init_completion(&md->kobj_holder.completion);
f0b04115 2056
8b211aac 2057 md->requeue_list = NULL;
a666e5c0
MP		 2058 md->swap_bios = get_swap_bios();
2059 sema_init(&md->swap_bios_semaphore, md->swap_bios);
2060 mutex_init(&md->swap_bios_lock);
2061
1da177e4
LT		 2062 md->disk->major = _major;
2063 md->disk->first_minor = minor;
74fe6ba9 2064 md->disk->minors = 1;
1ebe2e5f 2065 md->disk->flags |= GENHD_FL_NO_PART;
1da177e4
LT		 2066 md->disk->fops = &dm_blk_dops;
2067 md->disk->queue = md->queue;
2068 md->disk->private_data = md;
2069 sprintf(md->disk->disk_name, "dm-%d", minor);
f26c5719 2070
5d2a228b 2071 if (IS_ENABLED(CONFIG_FS_DAX)) {
30c6828a 2072 md->dax_dev = alloc_dax(md, &dm_dax_ops);
d7519392
CH		 2073 if (IS_ERR(md->dax_dev)) {
2074 md->dax_dev = NULL;
976431b0 2075 goto bad;
d7519392 2076 }
7ac5360c
CH		 2077 set_dax_nocache(md->dax_dev);
2078 set_dax_nomc(md->dax_dev);
fb08a190 2079 if (dax_add_host(md->dax_dev, md->disk))
976431b0
DW		 2080 goto bad;
2081 }
f26c5719 2082
7e51f257 2083 format_dev_t(md->name, MKDEV(_major, minor));
1da177e4 2084
c7c879ee 2085 md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name);
304f3f6a 2086 if (!md->wq)
0f20972f 2087 goto bad;
304f3f6a 2088
9f6dc633
MS		 2089 md->pending_io = alloc_percpu(unsigned long);
2090 if (!md->pending_io)
2091 goto bad;
2092
fd2ed4d2
MP		 2093 dm_stats_init(&md->stats);
2094
ba61fdd1 2095 /* Populate the mapping, nobody knows we exist yet */
f32c10b0 2096 spin_lock(&_minor_lock);
ba61fdd1 2097 old_md = idr_replace(&_minor_idr, md, minor);
f32c10b0 2098 spin_unlock(&_minor_lock);
ba61fdd1
JM		 2099
2100 BUG_ON(old_md != MINOR_ALLOCED);
2101
1da177e4
LT		 2102 return md;
2103
0f20972f
MS		 2104bad:
2105 cleanup_mapped_device(md);
83d5e5b0 2106bad_io_barrier:
1da177e4 2107 free_minor(minor);
6ed7ade8 2108bad_minor:
10da4f79 2109 module_put(THIS_MODULE);
6ed7ade8 2110bad_module_get:
856eb091 2111 kvfree(md);
1da177e4
LT		 2112 return NULL;
2113}
2114
ae9da83f
JN		 2115static void unlock_fs(struct mapped_device *md);
2116
1da177e4
LT		 2117static void free_dev(struct mapped_device *md)
2118{
f331c029 2119 int minor = MINOR(disk_devt(md->disk));
63d94e48 2120
32a926da 2121 unlock_fs(md);
2eb6e1e3 2122
0f20972f 2123 cleanup_mapped_device(md);
63a4f065 2124
86f1152b 2125 free_table_devices(&md->table_devices);
63a4f065 2126 dm_stats_cleanup(&md->stats);
63a4f065
MS		 2127 free_minor(minor);
2128
10da4f79 2129 module_put(THIS_MODULE);
856eb091 2130 kvfree(md);
1da177e4
LT		 2131}
2132
2133/*
2134 * Bind a table to the device.
2135 */
2136static void event_callback(void *context)
2137{
7a8c3d3b
MA		 2138 unsigned long flags;
2139 LIST_HEAD(uevents);
1da177e4
LT		 2140 struct mapped_device *md = (struct mapped_device *) context;
2141
7a8c3d3b
MA		 2142 spin_lock_irqsave(&md->uevent_lock, flags);
2143 list_splice_init(&md->uevent_list, &uevents);
2144 spin_unlock_irqrestore(&md->uevent_lock, flags);
2145
ed9e1982 2146 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
7a8c3d3b 2147
1da177e4
LT		 2148 atomic_inc(&md->event_nr);
2149 wake_up(&md->eventq);
62e08243 2150 dm_issue_global_event();
1da177e4
LT		 2151}
2152
042d2a9b
AK		 2153/*
2154 * Returns old map, which caller must destroy.
2155 */
2156static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2157 struct queue_limits *limits)
1da177e4 2158{
042d2a9b 2159 struct dm_table *old_map;
1da177e4 2160 sector_t size;
2a2a4c51 2161 int ret;
1da177e4 2162
5a8f1f80
BVA		 2163 lockdep_assert_held(&md->suspend_lock);
2164
1da177e4 2165 size = dm_table_get_size(t);
3ac51e74
DW		 2166
2167 /*
2168 * Wipe any geometry if the size of the table changed.
2169 */
fd2ed4d2 2170 if (size != dm_get_size(md))
3ac51e74
DW		 2171 memset(&md->geometry, 0, sizeof(md->geometry));
2172
5424a0b8
MP		 2173 if (!get_capacity(md->disk))
2174 set_capacity(md->disk, size);
2175 else
2176 set_capacity_and_notify(md->disk, size);
d5816876 2177
2ca3310e
AK		 2178 dm_table_event_callback(t, event_callback, md);
2179
f5b4aee1 2180 if (dm_table_request_based(t)) {
16f12266 2181 /*
9c37de29
MS		 2182 * Leverage the fact that request-based DM targets are
2183 * immutable singletons - used to optimize dm_mq_queue_rq.
16f12266
MS		 2184 */
2185 md->immutable_target = dm_table_get_immutable_target(t);
e6ee8c0b 2186
29dec90a
CH		 2187 /*
2188 * There is no need to reload with request-based dm because the
2189 * size of front_pad doesn't change.
2190 *
2191 * Note for future: If you are to reload bioset, prep-ed
2192 * requests in the queue may refer to bio from the old bioset,
2193 * so you must walk through the queue to unprep.
2194 */
2195 if (!md->mempools) {
2196 md->mempools = t->mempools;
2197 t->mempools = NULL;
2198 }
2199 } else {
2200 /*
2201 * The md may already have mempools that need changing.
2202 * If so, reload bioset because front_pad may have changed
2203 * because a different table was loaded.
2204 */
2205 dm_free_md_mempools(md->mempools);
2206 md->mempools = t->mempools;
2207 t->mempools = NULL;
2a2a4c51 2208 }
e6ee8c0b 2209
f5b4aee1 2210 ret = dm_table_set_restrictions(t, md->queue, limits);
bb37d772
DLM		 2211 if (ret) {
2212 old_map = ERR_PTR(ret);
2213 goto out;
2214 }
2215
a12f5d48 2216 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1d3aa6f6 2217 rcu_assign_pointer(md->map, (void *)t);
36a0456f
AK		 2218 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2219
41abc4e1
HR		 2220 if (old_map)
2221 dm_sync_table(md);
2a2a4c51 2222out:
042d2a9b 2223 return old_map;
1da177e4
LT		 2224}
2225
a7940155
AK		 2226/*
2227 * Returns unbound table for the caller to free.
2228 */
2229static struct dm_table *__unbind(struct mapped_device *md)
1da177e4 2230{
a12f5d48 2231 struct dm_table *map = rcu_dereference_protected(md->map, 1);
1da177e4
LT		 2232
2233 if (!map)
a7940155 2234 return NULL;
1da177e4
LT		 2235
2236 dm_table_event_callback(map, NULL, NULL);
9cdb8520 2237 RCU_INIT_POINTER(md->map, NULL);
83d5e5b0 2238 dm_sync_table(md);
a7940155
AK		 2239
2240 return map;
1da177e4
LT		 2241}
2242
2243/*
2244 * Constructor for a new device.
2245 */
2b06cfff 2246int dm_create(int minor, struct mapped_device **result)
1da177e4
LT		 2247{
2248 struct mapped_device *md;
2249
2b06cfff 2250 md = alloc_dev(minor);
1da177e4
LT		 2251 if (!md)
2252 return -ENXIO;
2253
91ccbbac
TS		 2254 dm_ima_reset_data(md);
2255
1da177e4
LT		 2256 *result = md;
2257 return 0;
2258}
2259
a5664dad
MS		 2260/*
2261 * Functions to manage md->type.
2262 * All are required to hold md->type_lock.
2263 */
2264void dm_lock_md_type(struct mapped_device *md)
2265{
2266 mutex_lock(&md->type_lock);
2267}
2268
2269void dm_unlock_md_type(struct mapped_device *md)
2270{
2271 mutex_unlock(&md->type_lock);
2272}
2273
7e0d574f 2274void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
a5664dad 2275{
00c4fc3b 2276 BUG_ON(!mutex_is_locked(&md->type_lock));
a5664dad
MS		 2277 md->type = type;
2278}
2279
7e0d574f 2280enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
a5664dad
MS		 2281{
2282 return md->type;
2283}
2284
36a0456f
AK		 2285struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2286{
2287 return md->immutable_target_type;
2288}
2289
f84cb8a4
MS		 2290/*
2291 * The queue_limits are only valid as long as you have a reference
2292 * count on 'md'.
2293 */
2294struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2295{
2296 BUG_ON(!atomic_read(&md->holders));
2297 return &md->queue->limits;
2298}
2299EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2300
4a0b4ddf
MS		 2301/*
2302 * Setup the DM device's queue based on md's type
2303 */
591ddcfc 2304int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
4a0b4ddf 2305{
ba305859 2306 enum dm_queue_mode type = dm_table_get_type(t);
c100ec49 2307 struct queue_limits limits;
ba305859 2308 int r;
bfebd1cd 2309
545ed20e 2310 switch (type) {
bfebd1cd 2311 case DM_TYPE_REQUEST_BASED:
681cc5e8 2312 md->disk->fops = &dm_rq_blk_dops;
e83068a5 2313 r = dm_mq_init_request_queue(md, t);
bfebd1cd 2314 if (r) {
681cc5e8 2315 DMERR("Cannot initialize queue for request-based dm mapped device");
bfebd1cd
MS		 2316 return r;
2317 }
2318 break;
2319 case DM_TYPE_BIO_BASED:
545ed20e 2320 case DM_TYPE_DAX_BIO_BASED:
bfebd1cd 2321 break;
7e0d574f
BVA		 2322 case DM_TYPE_NONE:
2323 WARN_ON_ONCE(true);
2324 break;
4a0b4ddf
MS		 2325 }
2326
c100ec49
MS		 2327 r = dm_calculate_queue_limits(t, &limits);
2328 if (r) {
2329 DMERR("Cannot calculate initial queue limits");
2330 return r;
2331 }
bb37d772
DLM		 2332 r = dm_table_set_restrictions(t, md->queue, &limits);
2333 if (r)
2334 return r;
2335
e7089f65
LC		 2336 r = add_disk(md->disk);
2337 if (r)
2338 return r;
c100ec49 2339
89f871af
CH		 2340 r = dm_sysfs_init(md);
2341 if (r) {
2342 del_gendisk(md->disk);
2343 return r;
2344 }
2345 md->type = type;
4a0b4ddf
MS		 2346 return 0;
2347}
2348
2bec1f4a 2349struct mapped_device *dm_get_md(dev_t dev)
1da177e4
LT		 2350{
2351 struct mapped_device *md;
1da177e4
LT		 2352 unsigned minor = MINOR(dev);
2353
2354 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2355 return NULL;
2356
f32c10b0 2357 spin_lock(&_minor_lock);
1da177e4
LT		 2358
2359 md = idr_find(&_minor_idr, minor);
49de5769
MS		 2360 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2361 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2362 md = NULL;
2363 goto out;
fba9f90e 2364 }
49de5769 2365 dm_get(md);
fba9f90e 2366out:
f32c10b0 2367 spin_unlock(&_minor_lock);
1da177e4 2368
637842cf
DT		 2369 return md;
2370}
3cf2e4ba 2371EXPORT_SYMBOL_GPL(dm_get_md);
d229a958 2372
9ade92a9 2373void *dm_get_mdptr(struct mapped_device *md)
637842cf 2374{
9ade92a9 2375 return md->interface_ptr;
1da177e4
LT		 2376}
2377
2378void dm_set_mdptr(struct mapped_device *md, void *ptr)
2379{
2380 md->interface_ptr = ptr;
2381}
2382
2383void dm_get(struct mapped_device *md)
2384{
2385 atomic_inc(&md->holders);
3f77316d 2386 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1da177e4
LT		 2387}
2388
09ee96b2
MP		 2389int dm_hold(struct mapped_device *md)
2390{
2391 spin_lock(&_minor_lock);
2392 if (test_bit(DMF_FREEING, &md->flags)) {
2393 spin_unlock(&_minor_lock);
2394 return -EBUSY;
2395 }
2396 dm_get(md);
2397 spin_unlock(&_minor_lock);
2398 return 0;
2399}
2400EXPORT_SYMBOL_GPL(dm_hold);
2401
72d94861
AK		 2402const char *dm_device_name(struct mapped_device *md)
2403{
2404 return md->name;
2405}
2406EXPORT_SYMBOL_GPL(dm_device_name);
2407
3f77316d 2408static void __dm_destroy(struct mapped_device *md, bool wait)
1da177e4 2409{
1134e5ae 2410 struct dm_table *map;
83d5e5b0 2411 int srcu_idx;
1da177e4 2412
3f77316d 2413 might_sleep();
fba9f90e 2414
63a4f065 2415 spin_lock(&_minor_lock);
3f77316d
KU		 2416 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2417 set_bit(DMF_FREEING, &md->flags);
2418 spin_unlock(&_minor_lock);
3b785fbc 2419
7a5428dc 2420 blk_mark_disk_dead(md->disk);
3f77316d 2421
ab7c7bb6
MP		 2422 /*
2423 * Take suspend_lock so that presuspend and postsuspend methods
2424 * do not race with internal suspend.
2425 */
2426 mutex_lock(&md->suspend_lock);
2a708cff 2427 map = dm_get_live_table(md, &srcu_idx);
3f77316d
KU		 2428 if (!dm_suspended_md(md)) {
2429 dm_table_presuspend_targets(map);
adc0daad 2430 set_bit(DMF_SUSPENDED, &md->flags);
5df96f2b 2431 set_bit(DMF_POST_SUSPENDING, &md->flags);
3f77316d 2432 dm_table_postsuspend_targets(map);
1da177e4 2433 }
83d5e5b0
MP		 2434 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2435 dm_put_live_table(md, srcu_idx);
2a708cff 2436 mutex_unlock(&md->suspend_lock);
83d5e5b0 2437
3f77316d
KU		 2438 /*
2439 * Rare, but there may be I/O requests still going to complete,
2440 * for example. Wait for all references to disappear.
2441 * No one should increment the reference count of the mapped_device,
2442 * after the mapped_device state becomes DMF_FREEING.
2443 */
2444 if (wait)
2445 while (atomic_read(&md->holders))
2446 msleep(1);
2447 else if (atomic_read(&md->holders))
2448 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2449 dm_device_name(md), atomic_read(&md->holders));
2450
3f77316d
KU		 2451 dm_table_destroy(__unbind(md));
2452 free_dev(md);
2453}
2454
2455void dm_destroy(struct mapped_device *md)
2456{
2457 __dm_destroy(md, true);
2458}
2459
2460void dm_destroy_immediate(struct mapped_device *md)
2461{
2462 __dm_destroy(md, false);
2463}
2464
2465void dm_put(struct mapped_device *md)
2466{
2467 atomic_dec(&md->holders);
1da177e4 2468}
79eb885c 2469EXPORT_SYMBOL_GPL(dm_put);
1da177e4 2470
9f6dc633 2471static bool dm_in_flight_bios(struct mapped_device *md)
85067747
ML		 2472{
2473 int cpu;
9f6dc633 2474 unsigned long sum = 0;
85067747 2475
9f6dc633
MS		 2476 for_each_possible_cpu(cpu)
2477 sum += *per_cpu_ptr(md->pending_io, cpu);
85067747
ML		 2478
2479 return sum != 0;
2480}
2481
2f064a59 2482static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
46125c1c
MB		 2483{
2484 int r = 0;
9f4c3f87 2485 DEFINE_WAIT(wait);
46125c1c 2486
85067747 2487 while (true) {
9f4c3f87 2488 prepare_to_wait(&md->wait, &wait, task_state);
46125c1c 2489
9f6dc633 2490 if (!dm_in_flight_bios(md))
46125c1c
MB		 2491 break;
2492
e3fabdfd 2493 if (signal_pending_state(task_state, current)) {
46125c1c
MB		 2494 r = -EINTR;
2495 break;
2496 }
2497
2498 io_schedule();
2499 }
9f4c3f87 2500 finish_wait(&md->wait, &wait);
b44ebeb0 2501
9f6dc633
MS		 2502 smp_rmb();
2503
46125c1c
MB		 2504 return r;
2505}
2506
2f064a59 2507static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
85067747
ML		 2508{
2509 int r = 0;
2510
2511 if (!queue_is_mq(md->queue))
2512 return dm_wait_for_bios_completion(md, task_state);
2513
2514 while (true) {
2515 if (!blk_mq_queue_inflight(md->queue))
2516 break;
2517
2518 if (signal_pending_state(task_state, current)) {
2519 r = -EINTR;
2520 break;
2521 }
2522
2523 msleep(5);
2524 }
2525
2526 return r;
2527}
2528
1da177e4
LT		 2529/*
2530 * Process the deferred bios
2531 */
ef208587 2532static void dm_wq_work(struct work_struct *work)
1da177e4 2533{
0c2915b8
MS		 2534 struct mapped_device *md = container_of(work, struct mapped_device, work);
2535 struct bio *bio;
ef208587 2536
3b00b203 2537 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
df12ee99 2538 spin_lock_irq(&md->deferred_lock);
0c2915b8 2539 bio = bio_list_pop(&md->deferred);
df12ee99
AK		 2540 spin_unlock_irq(&md->deferred_lock);
2541
0c2915b8 2542 if (!bio)
df12ee99 2543 break;
022c2611 2544
0c2915b8 2545 submit_bio_noacct(bio);
022c2611 2546 }
1da177e4
LT		 2547}
2548
9a1fb464 2549static void dm_queue_flush(struct mapped_device *md)
304f3f6a 2550{
3b00b203 2551 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
4e857c58 2552 smp_mb__after_atomic();
53d5914f 2553 queue_work(md->wq, &md->work);
304f3f6a
MB		 2554}
2555
1da177e4 2556/*
042d2a9b 2557 * Swap in a new table, returning the old one for the caller to destroy.
1da177e4 2558 */
042d2a9b 2559struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
1da177e4 2560{
87eb5b21 2561 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
754c5fc7 2562 struct queue_limits limits;
042d2a9b 2563 int r;
1da177e4 2564
e61290a4 2565 mutex_lock(&md->suspend_lock);
1da177e4
LT		 2566
2567 /* device must be suspended */
4f186f8b 2568 if (!dm_suspended_md(md))
93c534ae 2569 goto out;
1da177e4 2570
3ae70656
MS		 2571 /*
2572 * If the new table has no data devices, retain the existing limits.
2573 * This helps multipath with queue_if_no_path if all paths disappear,
2574 * then new I/O is queued based on these limits, and then some paths
2575 * reappear.
2576 */
2577 if (dm_table_has_no_data_devices(table)) {
83d5e5b0 2578 live_map = dm_get_live_table_fast(md);
3ae70656
MS		 2579 if (live_map)
2580 limits = md->queue->limits;
83d5e5b0 2581 dm_put_live_table_fast(md);
3ae70656
MS		 2582 }
2583
87eb5b21
MC		 2584 if (!live_map) {
2585 r = dm_calculate_queue_limits(table, &limits);
2586 if (r) {
2587 map = ERR_PTR(r);
2588 goto out;
2589 }
042d2a9b 2590 }
754c5fc7 2591
042d2a9b 2592 map = __bind(md, table, &limits);
62e08243 2593 dm_issue_global_event();
1da177e4 2594
93c534ae 2595out:
e61290a4 2596 mutex_unlock(&md->suspend_lock);
042d2a9b 2597 return map;
1da177e4
LT		 2598}
2599
2600/*
2601 * Functions to lock and unlock any filesystem running on the
2602 * device.
2603 */
2ca3310e 2604static int lock_fs(struct mapped_device *md)
1da177e4 2605{
e39e2e95 2606 int r;
1da177e4 2607
040f04bd 2608 WARN_ON(test_bit(DMF_FROZEN, &md->flags));
aa8d7c2f 2609
977115c0 2610 r = freeze_bdev(md->disk->part0);
040f04bd
CH		 2611 if (!r)
2612 set_bit(DMF_FROZEN, &md->flags);
2613 return r;
1da177e4
LT		 2614}
2615
2ca3310e 2616static void unlock_fs(struct mapped_device *md)
1da177e4 2617{
aa8d7c2f
AK		 2618 if (!test_bit(DMF_FROZEN, &md->flags))
2619 return;
977115c0 2620 thaw_bdev(md->disk->part0);
aa8d7c2f 2621 clear_bit(DMF_FROZEN, &md->flags);
1da177e4
LT		 2622}
2623
2624/*
b48633f8
BVA		 2625 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2626 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2627 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2628 *
ffcc3936
MS		 2629 * If __dm_suspend returns 0, the device is completely quiescent
2630 * now. There is no request-processing activity. All new requests
2631 * are being added to md->deferred list.
cec47e3d 2632 */
ffcc3936 2633static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2f064a59 2634 unsigned suspend_flags, unsigned int task_state,
eaf9a736 2635 int dmf_suspended_flag)
1da177e4 2636{
ffcc3936
MS		 2637 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2638 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2639 int r;
1da177e4 2640
5a8f1f80
BVA		 2641 lockdep_assert_held(&md->suspend_lock);
2642
2e93ccc1
KU		 2643 /*
2644 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2645 * This flag is cleared before dm_suspend returns.
2646 */
2647 if (noflush)
2648 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
86331f39 2649 else
ac75b09f 2650 DMDEBUG("%s: suspending with flush", dm_device_name(md));
2e93ccc1 2651
d67ee213
MS		 2652 /*
2653 * This gets reverted if there's an error later and the targets
2654 * provide the .presuspend_undo hook.
2655 */
cf222b37
AK		 2656 dm_table_presuspend_targets(map);
2657
32a926da 2658 /*
9f518b27
KU		 2659 * Flush I/O to the device.
2660 * Any I/O submitted after lock_fs() may not be flushed.
2661 * noflush takes precedence over do_lockfs.
2662 * (lock_fs() flushes I/Os and waits for them to complete.)
32a926da
MP		 2663 */
2664 if (!noflush && do_lockfs) {
2665 r = lock_fs(md);
d67ee213
MS		 2666 if (r) {
2667 dm_table_presuspend_undo_targets(map);
ffcc3936 2668 return r;
d67ee213 2669 }
aa8d7c2f 2670 }
1da177e4
LT		 2671
2672 /*
3b00b203
MP		 2673 * Here we must make sure that no processes are submitting requests
2674 * to target drivers i.e. no one may be executing
96c9865c 2675 * dm_split_and_process_bio from dm_submit_bio.
3b00b203 2676 *
96c9865c 2677 * To get all processes out of dm_split_and_process_bio in dm_submit_bio,
3b00b203 2678 * we take the write lock. To prevent any process from reentering
96c9865c 2679 * dm_split_and_process_bio from dm_submit_bio and quiesce the thread
0cede372 2680 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
6a8736d1 2681 * flush_workqueue(md->wq).
1da177e4 2682 */
1eb787ec 2683 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
41abc4e1
HR		 2684 if (map)
2685 synchronize_srcu(&md->io_barrier);
1da177e4 2686
d0bcb878 2687 /*
29e4013d
TH		 2688 * Stop md->queue before flushing md->wq in case request-based
2689 * dm defers requests to md->wq from md->queue.
d0bcb878 2690 */
6a23e05c 2691 if (dm_request_based(md))
eca7ee6d 2692 dm_stop_queue(md->queue);
cec47e3d 2693
d0bcb878
KU		 2694 flush_workqueue(md->wq);
2695
1da177e4 2696 /*
3b00b203
MP		 2697 * At this point no more requests are entering target request routines.
2698 * We call dm_wait_for_completion to wait for all existing requests
2699 * to finish.
1da177e4 2700 */
b48633f8 2701 r = dm_wait_for_completion(md, task_state);
eaf9a736
MS		 2702 if (!r)
2703 set_bit(dmf_suspended_flag, &md->flags);
1da177e4 2704
6d6f10df 2705 if (noflush)
022c2611 2706 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
41abc4e1
HR		 2707 if (map)
2708 synchronize_srcu(&md->io_barrier);
2e93ccc1 2709
1da177e4 2710 /* were we interrupted ? */
46125c1c 2711 if (r < 0) {
9a1fb464 2712 dm_queue_flush(md);
73d410c0 2713
cec47e3d 2714 if (dm_request_based(md))
eca7ee6d 2715 dm_start_queue(md->queue);
cec47e3d 2716
2ca3310e 2717 unlock_fs(md);
d67ee213 2718 dm_table_presuspend_undo_targets(map);
ffcc3936 2719 /* pushback list is already flushed, so skip flush */
2ca3310e 2720 }
1da177e4 2721
ffcc3936
MS		 2722 return r;
2723}
2724
2725/*
2726 * We need to be able to change a mapping table under a mounted
2727 * filesystem. For example we might want to move some data in
2728 * the background. Before the table can be swapped with
2729 * dm_bind_table, dm_suspend must be called to flush any in
2730 * flight bios and ensure that any further io gets deferred.
2731 */
2732/*
2733 * Suspend mechanism in request-based dm.
2734 *
2735 * 1. Flush all I/Os by lock_fs() if needed.
2736 * 2. Stop dispatching any I/O by stopping the request_queue.
2737 * 3. Wait for all in-flight I/Os to be completed or requeued.
2738 *
2739 * To abort suspend, start the request_queue.
2740 */
2741int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2742{
2743 struct dm_table *map = NULL;
2744 int r = 0;
2745
2746retry:
2747 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2748
2749 if (dm_suspended_md(md)) {
2750 r = -EINVAL;
2751 goto out_unlock;
2752 }
2753
2754 if (dm_suspended_internally_md(md)) {
2755 /* already internally suspended, wait for internal resume */
2756 mutex_unlock(&md->suspend_lock);
2757 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2758 if (r)
2759 return r;
2760 goto retry;
2761 }
2762
a12f5d48 2763 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
ffcc3936 2764
eaf9a736 2765 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
ffcc3936
MS		 2766 if (r)
2767 goto out_unlock;
3b00b203 2768
5df96f2b 2769 set_bit(DMF_POST_SUSPENDING, &md->flags);
4d4471cb 2770 dm_table_postsuspend_targets(map);
5df96f2b 2771 clear_bit(DMF_POST_SUSPENDING, &md->flags);
4d4471cb 2772
d287483d 2773out_unlock:
e61290a4 2774 mutex_unlock(&md->suspend_lock);
cf222b37 2775 return r;
1da177e4
LT		 2776}
2777
ffcc3936
MS		 2778static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2779{
2780 if (map) {
2781 int r = dm_table_resume_targets(map);
2782 if (r)
2783 return r;
2784 }
2785
2786 dm_queue_flush(md);
2787
2788 /*
2789 * Flushing deferred I/Os must be done after targets are resumed
2790 * so that mapping of targets can work correctly.
2791 * Request-based dm is queueing the deferred I/Os in its request_queue.
2792 */
2793 if (dm_request_based(md))
eca7ee6d 2794 dm_start_queue(md->queue);
ffcc3936
MS		 2795
2796 unlock_fs(md);
2797
2798 return 0;
2799}
2800
1da177e4
LT		 2801int dm_resume(struct mapped_device *md)
2802{
8dc23658 2803 int r;
cf222b37 2804 struct dm_table *map = NULL;
1da177e4 2805
ffcc3936 2806retry:
8dc23658 2807 r = -EINVAL;
ffcc3936
MS		 2808 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2809
4f186f8b 2810 if (!dm_suspended_md(md))
cf222b37 2811 goto out;
cf222b37 2812
ffcc3936
MS		 2813 if (dm_suspended_internally_md(md)) {
2814 /* already internally suspended, wait for internal resume */
2815 mutex_unlock(&md->suspend_lock);
2816 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2817 if (r)
2818 return r;
2819 goto retry;
2820 }
2821
a12f5d48 2822 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2ca3310e 2823 if (!map || !dm_table_get_size(map))
cf222b37 2824 goto out;
1da177e4 2825
ffcc3936 2826 r = __dm_resume(md, map);
8757b776
MB		 2827 if (r)
2828 goto out;
2ca3310e 2829
2ca3310e 2830 clear_bit(DMF_SUSPENDED, &md->flags);
cf222b37 2831out:
e61290a4 2832 mutex_unlock(&md->suspend_lock);
2ca3310e 2833
cf222b37 2834 return r;
1da177e4
LT		 2835}
2836
fd2ed4d2
MP		 2837/*
2838 * Internal suspend/resume works like userspace-driven suspend. It waits
2839 * until all bios finish and prevents issuing new bios to the target drivers.
2840 * It may be used only from the kernel.
fd2ed4d2
MP		 2841 */
2842
ffcc3936 2843static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
fd2ed4d2 2844{
ffcc3936
MS		 2845 struct dm_table *map = NULL;
2846
1ea0654e
BVA		 2847 lockdep_assert_held(&md->suspend_lock);
2848
96b26c8c 2849 if (md->internal_suspend_count++)
ffcc3936
MS		 2850 return; /* nested internal suspend */
2851
2852 if (dm_suspended_md(md)) {
2853 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2854 return; /* nest suspend */
2855 }
2856
a12f5d48 2857 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
ffcc3936
MS		 2858
2859 /*
2860 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2861 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2862 * would require changing .presuspend to return an error -- avoid this
2863 * until there is a need for more elaborate variants of internal suspend.
2864 */
eaf9a736
MS		 2865 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2866 DMF_SUSPENDED_INTERNALLY);
ffcc3936 2867
5df96f2b 2868 set_bit(DMF_POST_SUSPENDING, &md->flags);
ffcc3936 2869 dm_table_postsuspend_targets(map);
5df96f2b 2870 clear_bit(DMF_POST_SUSPENDING, &md->flags);
ffcc3936
MS		 2871}
2872
2873static void __dm_internal_resume(struct mapped_device *md)
2874{
96b26c8c
MP		 2875 BUG_ON(!md->internal_suspend_count);
2876
2877 if (--md->internal_suspend_count)
ffcc3936
MS		 2878 return; /* resume from nested internal suspend */
2879
fd2ed4d2 2880 if (dm_suspended_md(md))
ffcc3936
MS		 2881 goto done; /* resume from nested suspend */
2882
2883 /*
2884 * NOTE: existing callers don't need to call dm_table_resume_targets
2885 * (which may fail -- so best to avoid it for now by passing NULL map)
2886 */
2887 (void) __dm_resume(md, NULL);
2888
2889done:
2890 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2891 smp_mb__after_atomic();
2892 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2893}
2894
2895void dm_internal_suspend_noflush(struct mapped_device *md)
2896{
2897 mutex_lock(&md->suspend_lock);
2898 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2899 mutex_unlock(&md->suspend_lock);
2900}
2901EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2902
2903void dm_internal_resume(struct mapped_device *md)
2904{
2905 mutex_lock(&md->suspend_lock);
2906 __dm_internal_resume(md);
2907 mutex_unlock(&md->suspend_lock);
2908}
2909EXPORT_SYMBOL_GPL(dm_internal_resume);
2910
2911/*
2912 * Fast variants of internal suspend/resume hold md->suspend_lock,
2913 * which prevents interaction with userspace-driven suspend.
2914 */
2915
2916void dm_internal_suspend_fast(struct mapped_device *md)
2917{
2918 mutex_lock(&md->suspend_lock);
2919 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
fd2ed4d2
MP		 2920 return;
2921
2922 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2923 synchronize_srcu(&md->io_barrier);
2924 flush_workqueue(md->wq);
2925 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2926}
b735fede 2927EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
fd2ed4d2 2928
ffcc3936 2929void dm_internal_resume_fast(struct mapped_device *md)
fd2ed4d2 2930{
ffcc3936 2931 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
fd2ed4d2
MP		 2932 goto done;
2933
2934 dm_queue_flush(md);
2935
2936done:
2937 mutex_unlock(&md->suspend_lock);
2938}
b735fede 2939EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
fd2ed4d2 2940
1da177e4
LT		 2941/*-----------------------------------------------------------------
2942 * Event notification.
2943 *---------------------------------------------------------------*/
3abf85b5 2944int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
60935eb2 2945 unsigned cookie)
69267a30 2946{
6958c1c6
MP		 2947 int r;
2948 unsigned noio_flag;
60935eb2
MB		 2949 char udev_cookie[DM_COOKIE_LENGTH];
2950 char *envp[] = { udev_cookie, NULL };
2951
6958c1c6
MP		 2952 noio_flag = memalloc_noio_save();
2953
60935eb2 2954 if (!cookie)
6958c1c6 2955 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
60935eb2
MB		 2956 else {
2957 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2958 DM_COOKIE_ENV_VAR_NAME, cookie);
6958c1c6
MP		 2959 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2960 action, envp);
60935eb2 2961 }
6958c1c6
MP		 2962
2963 memalloc_noio_restore(noio_flag);
2964
2965 return r;
69267a30
AK		 2966}
2967
7a8c3d3b
MA		 2968uint32_t dm_next_uevent_seq(struct mapped_device *md)
2969{
2970 return atomic_add_return(1, &md->uevent_seq);
2971}
2972
1da177e4
LT		 2973uint32_t dm_get_event_nr(struct mapped_device *md)
2974{
2975 return atomic_read(&md->event_nr);
2976}
2977
2978int dm_wait_event(struct mapped_device *md, int event_nr)
2979{
2980 return wait_event_interruptible(md->eventq,
2981 (event_nr != atomic_read(&md->event_nr)));
2982}
2983
7a8c3d3b
MA		 2984void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2985{
2986 unsigned long flags;
2987
2988 spin_lock_irqsave(&md->uevent_lock, flags);
2989 list_add(elist, &md->uevent_list);
2990 spin_unlock_irqrestore(&md->uevent_lock, flags);
2991}
2992
1da177e4
LT		 2993/*
2994 * The gendisk is only valid as long as you have a reference
2995 * count on 'md'.
2996 */
2997struct gendisk *dm_disk(struct mapped_device *md)
2998{
2999 return md->disk;
3000}
65ff5b7d 3001EXPORT_SYMBOL_GPL(dm_disk);
1da177e4 3002
784aae73
MB		 3003struct kobject *dm_kobject(struct mapped_device *md)
3004{
2995fa78 3005 return &md->kobj_holder.kobj;
784aae73
MB		 3006}
3007
784aae73
MB		 3008struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
3009{
3010 struct mapped_device *md;
3011
2995fa78 3012 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
784aae73 3013
b9a41d21
HT		 3014 spin_lock(&_minor_lock);
3015 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
3016 md = NULL;
3017 goto out;
3018 }
784aae73 3019 dm_get(md);
b9a41d21
HT		 3020out:
3021 spin_unlock(&_minor_lock);
3022
784aae73
MB		 3023 return md;
3024}
3025
4f186f8b 3026int dm_suspended_md(struct mapped_device *md)
1da177e4
LT		 3027{
3028 return test_bit(DMF_SUSPENDED, &md->flags);
3029}
3030
5df96f2b
MP		 3031static int dm_post_suspending_md(struct mapped_device *md)
3032{
3033 return test_bit(DMF_POST_SUSPENDING, &md->flags);
3034}
3035
ffcc3936
MS		 3036int dm_suspended_internally_md(struct mapped_device *md)
3037{
3038 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3039}
3040
2c140a24
MP		 3041int dm_test_deferred_remove_flag(struct mapped_device *md)
3042{
3043 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
3044}
3045
64dbce58
KU		 3046int dm_suspended(struct dm_target *ti)
3047{
33bd6f06 3048 return dm_suspended_md(ti->table->md);
64dbce58
KU		 3049}
3050EXPORT_SYMBOL_GPL(dm_suspended);
3051
5df96f2b
MP		 3052int dm_post_suspending(struct dm_target *ti)
3053{
33bd6f06 3054 return dm_post_suspending_md(ti->table->md);
5df96f2b
MP		 3055}
3056EXPORT_SYMBOL_GPL(dm_post_suspending);
3057
2e93ccc1
KU		 3058int dm_noflush_suspending(struct dm_target *ti)
3059{
33bd6f06 3060 return __noflush_suspending(ti->table->md);
2e93ccc1
KU		 3061}
3062EXPORT_SYMBOL_GPL(dm_noflush_suspending);
3063
e6ee8c0b
KU		 3064void dm_free_md_mempools(struct dm_md_mempools *pools)
3065{
3066 if (!pools)
3067 return;
3068
6f1c819c
KO		 3069 bioset_exit(&pools->bs);
3070 bioset_exit(&pools->io_bs);
e6ee8c0b
KU		 3071
3072 kfree(pools);
3073}
3074
9c72bad1
CH		 3075struct dm_pr {
3076 u64 old_key;
3077 u64 new_key;
3078 u32 flags;
c6adada5 3079 bool abort;
9c72bad1 3080 bool fail_early;
8dd87f3c 3081 int ret;
70151087 3082 enum pr_type type;
9c72bad1
CH		 3083};
3084
3085static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
8dd87f3c 3086 struct dm_pr *pr)
71cdb697
CH		 3087{
3088 struct mapped_device *md = bdev->bd_disk->private_data;
9c72bad1
CH		 3089 struct dm_table *table;
3090 struct dm_target *ti;
3091 int ret = -ENOTTY, srcu_idx;
71cdb697 3092
9c72bad1
CH		 3093 table = dm_get_live_table(md, &srcu_idx);
3094 if (!table || !dm_table_get_size(table))
3095 goto out;
71cdb697 3096
9c72bad1 3097 /* We only support devices that have a single target */
2aec377a 3098 if (table->num_targets != 1)
9c72bad1
CH		 3099 goto out;
3100 ti = dm_table_get_target(table, 0);
71cdb697 3101
e120a5f1
MS		 3102 if (dm_suspended_md(md)) {
3103 ret = -EAGAIN;
3104 goto out;
3105 }
3106
9c72bad1
CH		 3107 ret = -EINVAL;
3108 if (!ti->type->iterate_devices)
3109 goto out;
3110
8dd87f3c
MC		 3111 ti->type->iterate_devices(ti, fn, pr);
3112 ret = 0;
9c72bad1
CH		 3113out:
3114 dm_put_live_table(md, srcu_idx);
3115 return ret;
3116}
3117
3118/*
3119 * For register / unregister we need to manually call out to every path.
3120 */
3121static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
3122 sector_t start, sector_t len, void *data)
3123{
3124 struct dm_pr *pr = data;
3125 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
8dd87f3c
MC		 3126 int ret;
3127
3128 if (!ops || !ops->pr_register) {
3129 pr->ret = -EOPNOTSUPP;
3130 return -1;
3131 }
3132
3133 ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3134 if (!ret)
3135 return 0;
3136
3137 if (!pr->ret)
3138 pr->ret = ret;
9c72bad1 3139
8dd87f3c
MC		 3140 if (pr->fail_early)
3141 return -1;
3142
3143 return 0;
9c72bad1
CH		 3144}
3145
3146static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
3147 u32 flags)
3148{
3149 struct dm_pr pr = {
3150 .old_key = old_key,
3151 .new_key = new_key,
3152 .flags = flags,
3153 .fail_early = true,
8dd87f3c 3154 .ret = 0,
9c72bad1
CH		 3155 };
3156 int ret;
3157
3158 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
8dd87f3c
MC		 3159 if (ret) {
3160 /* Didn't even get to register a path */
3161 return ret;
9c72bad1
CH		 3162 }
3163
8dd87f3c
MC		 3164 if (!pr.ret)
3165 return 0;
3166 ret = pr.ret;
3167
3168 if (!new_key)
3169 return ret;
3170
3171 /* unregister all paths if we failed to register any path */
3172 pr.old_key = new_key;
3173 pr.new_key = 0;
3174 pr.flags = 0;
3175 pr.fail_early = false;
3176 (void) dm_call_pr(bdev, __dm_pr_register, &pr);
9c72bad1 3177 return ret;
71cdb697
CH		 3178}
3179
70151087
MC		 3180
3181static int __dm_pr_reserve(struct dm_target *ti, struct dm_dev *dev,
3182 sector_t start, sector_t len, void *data)
3183{
3184 struct dm_pr *pr = data;
3185 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3186
3187 if (!ops || !ops->pr_reserve) {
3188 pr->ret = -EOPNOTSUPP;
3189 return -1;
3190 }
3191
3192 pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags);
3193 if (!pr->ret)
3194 return -1;
3195
3196 return 0;
3197}
3198
71cdb697 3199static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
956a4025 3200 u32 flags)
71cdb697 3201{
70151087
MC		 3202 struct dm_pr pr = {
3203 .old_key = key,
3204 .flags = flags,
3205 .type = type,
3206 .fail_early = false,
3207 .ret = 0,
3208 };
3209 int ret;
71cdb697 3210
70151087
MC		 3211 ret = dm_call_pr(bdev, __dm_pr_reserve, &pr);
3212 if (ret)
3213 return ret;
71cdb697 3214
70151087 3215 return pr.ret;
71cdb697
CH		 3216}
3217
08a3c338
MC		 3218/*
3219 * If there is a non-All Registrants type of reservation, the release must be
3220 * sent down the holding path. For the cases where there is no reservation or
3221 * the path is not the holder the device will also return success, so we must
3222 * try each path to make sure we got the correct path.
3223 */
3224static int __dm_pr_release(struct dm_target *ti, struct dm_dev *dev,
3225 sector_t start, sector_t len, void *data)
3226{
3227 struct dm_pr *pr = data;
3228 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3229
3230 if (!ops || !ops->pr_release) {
3231 pr->ret = -EOPNOTSUPP;
3232 return -1;
3233 }
3234
3235 pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type);
3236 if (pr->ret)
3237 return -1;
3238
3239 return 0;
3240}
3241
71cdb697
CH		 3242static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3243{
08a3c338
MC		 3244 struct dm_pr pr = {
3245 .old_key = key,
3246 .type = type,
3247 .fail_early = false,
3248 };
3249 int ret;
71cdb697 3250
08a3c338
MC		 3251 ret = dm_call_pr(bdev, __dm_pr_release, &pr);
3252 if (ret)
3253 return ret;
71cdb697 3254
08a3c338 3255 return pr.ret;
71cdb697
CH		 3256}
3257
c6adada5
MC		 3258static int __dm_pr_preempt(struct dm_target *ti, struct dm_dev *dev,
3259 sector_t start, sector_t len, void *data)
3260{
3261 struct dm_pr *pr = data;
3262 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3263
3264 if (!ops || !ops->pr_preempt) {
3265 pr->ret = -EOPNOTSUPP;
3266 return -1;
3267 }
3268
3269 pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type,
3270 pr->abort);
3271 if (!pr->ret)
3272 return -1;
3273
3274 return 0;
3275}
3276
71cdb697 3277static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
956a4025 3278 enum pr_type type, bool abort)
71cdb697 3279{
c6adada5
MC		 3280 struct dm_pr pr = {
3281 .new_key = new_key,
3282 .old_key = old_key,
3283 .type = type,
3284 .fail_early = false,
3285 };
3286 int ret;
71cdb697 3287
c6adada5
MC		 3288 ret = dm_call_pr(bdev, __dm_pr_preempt, &pr);
3289 if (ret)
3290 return ret;
71cdb697 3291
c6adada5 3292 return pr.ret;
71cdb697
CH		 3293}
3294
3295static int dm_pr_clear(struct block_device *bdev, u64 key)
3296{
3297 struct mapped_device *md = bdev->bd_disk->private_data;
3298 const struct pr_ops *ops;
971888c4 3299 int r, srcu_idx;
71cdb697 3300
5bd5e8d8 3301 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
71cdb697 3302 if (r < 0)
971888c4 3303 goto out;
71cdb697
CH		 3304
3305 ops = bdev->bd_disk->fops->pr_ops;
3306 if (ops && ops->pr_clear)
3307 r = ops->pr_clear(bdev, key);
3308 else
3309 r = -EOPNOTSUPP;
971888c4
MS		 3310out:
3311 dm_unprepare_ioctl(md, srcu_idx);
71cdb697
CH		 3312 return r;
3313}
3314
3315static const struct pr_ops dm_pr_ops = {
3316 .pr_register = dm_pr_register,
3317 .pr_reserve = dm_pr_reserve,
3318 .pr_release = dm_pr_release,
3319 .pr_preempt = dm_pr_preempt,
3320 .pr_clear = dm_pr_clear,
3321};
3322
83d5cde4 3323static const struct block_device_operations dm_blk_dops = {
c62b37d9 3324 .submit_bio = dm_submit_bio,
b99fdcdc 3325 .poll_bio = dm_poll_bio,
1da177e4
LT		 3326 .open = dm_blk_open,
3327 .release = dm_blk_close,
aa129a22 3328 .ioctl = dm_blk_ioctl,
3ac51e74 3329 .getgeo = dm_blk_getgeo,
e76239a3 3330 .report_zones = dm_blk_report_zones,
71cdb697 3331 .pr_ops = &dm_pr_ops,
1da177e4
LT		 3332 .owner = THIS_MODULE
3333};
3334
681cc5e8
MS		 3335static const struct block_device_operations dm_rq_blk_dops = {
3336 .open = dm_blk_open,
3337 .release = dm_blk_close,
3338 .ioctl = dm_blk_ioctl,
3339 .getgeo = dm_blk_getgeo,
3340 .pr_ops = &dm_pr_ops,
3341 .owner = THIS_MODULE
3342};
3343
f26c5719
DW		 3344static const struct dax_operations dm_dax_ops = {
3345 .direct_access = dm_dax_direct_access,
cdf6cdcd 3346 .zero_page_range = dm_dax_zero_page_range,
047218ec 3347 .recovery_write = dm_dax_recovery_write,
f26c5719
DW		 3348};
3349
1da177e4
LT		 3350/*
3351 * module hooks
3352 */
3353module_init(dm_init);
3354module_exit(dm_exit);
3355
3356module_param(major, uint, 0);
3357MODULE_PARM_DESC(major, "The major number of the device mapper");
f4790826 3358
e8603136
MS		 3359module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3360MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3361
115485e8
MS		 3362module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3363MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3364
a666e5c0
MP		 3365module_param(swap_bios, int, S_IRUGO | S_IWUSR);
3366MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
3367
1da177e4
LT		 3368MODULE_DESCRIPTION(DM_NAME " driver");
3369MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3370MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)