projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
block: allocate struct hd_struct as part of struct bdev_inode
[linux-block.git] / drivers / md / dm.c
CommitLineData
1da177e4
LT		 1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
784aae73 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
1da177e4
LT		 4 *
5 * This file is released under the GPL.
6 */
7
4cc96131
MS		 8#include "dm-core.h"
9#include "dm-rq.h"
51e5b2bd 10#include "dm-uevent.h"
1da177e4
LT		 11
12#include <linux/init.h>
13#include <linux/module.h>
48c9c27b 14#include <linux/mutex.h>
6958c1c6 15#include <linux/sched/mm.h>
174cd4b1 16#include <linux/sched/signal.h>
1da177e4
LT		 17#include <linux/blkpg.h>
18#include <linux/bio.h>
1da177e4 19#include <linux/mempool.h>
f26c5719 20#include <linux/dax.h>
1da177e4
LT		 21#include <linux/slab.h>
22#include <linux/idr.h>
7e026c8c 23#include <linux/uio.h>
3ac51e74 24#include <linux/hdreg.h>
3f77316d 25#include <linux/delay.h>
ffcc3936 26#include <linux/wait.h>
71cdb697 27#include <linux/pr.h>
b0b4d7c6 28#include <linux/refcount.h>
c6a564ff 29#include <linux/part_stat.h>
a892c8d5 30#include <linux/blk-crypto.h>
55782138 31
72d94861
AK		 32#define DM_MSG_PREFIX "core"
33
60935eb2
MB		 34/*
35 * Cookies are numeric values sent with CHANGE and REMOVE
36 * uevents while resuming, removing or renaming the device.
37 */
38#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
39#define DM_COOKIE_LENGTH 24
40
1da177e4
LT		 41static const char *_name = DM_NAME;
42
43static unsigned int major = 0;
44static unsigned int _major = 0;
45
d15b774c
AK		 46static DEFINE_IDR(_minor_idr);
47
f32c10b0 48static DEFINE_SPINLOCK(_minor_lock);
2c140a24
MP		 49
50static void do_deferred_remove(struct work_struct *w);
51
52static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
53
acfe0ad7
MP		 54static struct workqueue_struct *deferred_remove_workqueue;
55
93e6442c
MP		 56atomic_t dm_global_event_nr = ATOMIC_INIT(0);
57DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
58
62e08243
MP		 59void dm_issue_global_event(void)
60{
61 atomic_inc(&dm_global_event_nr);
62 wake_up(&dm_global_eventq);
63}
64
1da177e4 65/*
64f52b0e 66 * One of these is allocated (on-stack) per original bio.
1da177e4 67 */
64f52b0e 68struct clone_info {
64f52b0e
MS		 69 struct dm_table *map;
70 struct bio *bio;
71 struct dm_io *io;
72 sector_t sector;
73 unsigned sector_count;
74};
75
76/*
77 * One of these is allocated per clone bio.
78 */
79#define DM_TIO_MAGIC 7282014
80struct dm_target_io {
81 unsigned magic;
82 struct dm_io *io;
83 struct dm_target *ti;
84 unsigned target_bio_nr;
85 unsigned *len_ptr;
86 bool inside_dm_io;
87 struct bio clone;
88};
89
1da177e4 90/*
745dc570 91 * One of these is allocated per original bio.
64f52b0e 92 * It contains the first clone used for that original.
1da177e4 93 */
64f52b0e 94#define DM_IO_MAGIC 5191977
1da177e4 95struct dm_io {
64f52b0e 96 unsigned magic;
1da177e4 97 struct mapped_device *md;
4e4cbee9 98 blk_status_t status;
1da177e4 99 atomic_t io_count;
745dc570 100 struct bio *orig_bio;
3eaf840e 101 unsigned long start_time;
f88fb981 102 spinlock_t endio_lock;
fd2ed4d2 103 struct dm_stats_aux stats_aux;
64f52b0e
MS		 104 /* last member of dm_target_io is 'struct bio' */
105 struct dm_target_io tio;
1da177e4
LT		 106};
107
64f52b0e
MS		 108void *dm_per_bio_data(struct bio *bio, size_t data_size)
109{
110 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
111 if (!tio->inside_dm_io)
112 return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
113 return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size;
114}
115EXPORT_SYMBOL_GPL(dm_per_bio_data);
116
117struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
118{
119 struct dm_io *io = (struct dm_io *)((char *)data + data_size);
120 if (io->magic == DM_IO_MAGIC)
121 return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone));
122 BUG_ON(io->magic != DM_TIO_MAGIC);
123 return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone));
124}
125EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
126
127unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
128{
129 return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
130}
131EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
132
ba61fdd1
JM		 133#define MINOR_ALLOCED ((void *)-1)
134
1da177e4
LT		 135/*
136 * Bits for the md->flags field.
137 */
1eb787ec 138#define DMF_BLOCK_IO_FOR_SUSPEND 0
1da177e4 139#define DMF_SUSPENDED 1
aa8d7c2f 140#define DMF_FROZEN 2
fba9f90e 141#define DMF_FREEING 3
5c6bd75d 142#define DMF_DELETING 4
2e93ccc1 143#define DMF_NOFLUSH_SUSPENDING 5
8ae12666
KO		 144#define DMF_DEFERRED_REMOVE 6
145#define DMF_SUSPENDED_INTERNALLY 7
5df96f2b 146#define DMF_POST_SUSPENDING 8
1da177e4 147
115485e8 148#define DM_NUMA_NODE NUMA_NO_NODE
115485e8 149static int dm_numa_node = DM_NUMA_NODE;
faad87df 150
e6ee8c0b
KU		 151/*
152 * For mempools pre-allocation at the table loading time.
153 */
154struct dm_md_mempools {
6f1c819c
KO		 155 struct bio_set bs;
156 struct bio_set io_bs;
e6ee8c0b
KU		 157};
158
86f1152b
BM		 159struct table_device {
160 struct list_head list;
b0b4d7c6 161 refcount_t count;
86f1152b
BM		 162 struct dm_dev dm_dev;
163};
164
e8603136
MS		 165/*
166 * Bio-based DM's mempools' reserved IOs set by the user.
167 */
4cc96131 168#define RESERVED_BIO_BASED_IOS 16
e8603136
MS		 169static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
170
115485e8
MS		 171static int __dm_get_module_param_int(int *module_param, int min, int max)
172{
6aa7de05 173 int param = READ_ONCE(*module_param);
115485e8
MS		 174 int modified_param = 0;
175 bool modified = true;
176
177 if (param < min)
178 modified_param = min;
179 else if (param > max)
180 modified_param = max;
181 else
182 modified = false;
183
184 if (modified) {
185 (void)cmpxchg(module_param, param, modified_param);
186 param = modified_param;
187 }
188
189 return param;
190}
191
4cc96131
MS		 192unsigned __dm_get_module_param(unsigned *module_param,
193 unsigned def, unsigned max)
f4790826 194{
6aa7de05 195 unsigned param = READ_ONCE(*module_param);
09c2d531 196 unsigned modified_param = 0;
f4790826 197
09c2d531
MS		 198 if (!param)
199 modified_param = def;
200 else if (param > max)
201 modified_param = max;
f4790826 202
09c2d531
MS		 203 if (modified_param) {
204 (void)cmpxchg(module_param, param, modified_param);
205 param = modified_param;
f4790826
MS		 206 }
207
09c2d531 208 return param;
f4790826
MS		 209}
210
e8603136
MS		 211unsigned dm_get_reserved_bio_based_ios(void)
212{
09c2d531 213 return __dm_get_module_param(&reserved_bio_based_ios,
4cc96131 214 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
e8603136
MS		 215}
216EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
217
115485e8
MS		 218static unsigned dm_get_numa_node(void)
219{
220 return __dm_get_module_param_int(&dm_numa_node,
221 DM_NUMA_NODE, num_online_nodes() - 1);
222}
223
1da177e4
LT		 224static int __init local_init(void)
225{
e689fbab 226 int r;
1ae49ea2 227
51e5b2bd 228 r = dm_uevent_init();
51157b4a 229 if (r)
e689fbab 230 return r;
51e5b2bd 231
acfe0ad7
MP		 232 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
233 if (!deferred_remove_workqueue) {
234 r = -ENOMEM;
235 goto out_uevent_exit;
236 }
237
1da177e4
LT		 238 _major = major;
239 r = register_blkdev(_major, _name);
51157b4a 240 if (r < 0)
acfe0ad7 241 goto out_free_workqueue;
1da177e4
LT		 242
243 if (!_major)
244 _major = r;
245
246 return 0;
51157b4a 247
acfe0ad7
MP		 248out_free_workqueue:
249 destroy_workqueue(deferred_remove_workqueue);
51157b4a
KU		 250out_uevent_exit:
251 dm_uevent_exit();
51157b4a
KU		 252
253 return r;
1da177e4
LT		 254}
255
256static void local_exit(void)
257{
2c140a24 258 flush_scheduled_work();
acfe0ad7 259 destroy_workqueue(deferred_remove_workqueue);
2c140a24 260
00d59405 261 unregister_blkdev(_major, _name);
51e5b2bd 262 dm_uevent_exit();
1da177e4
LT		 263
264 _major = 0;
265
266 DMINFO("cleaned up");
267}
268
b9249e55 269static int (*_inits[])(void) __initdata = {
1da177e4
LT		 270 local_init,
271 dm_target_init,
272 dm_linear_init,
273 dm_stripe_init,
952b3557 274 dm_io_init,
945fa4d2 275 dm_kcopyd_init,
1da177e4 276 dm_interface_init,
fd2ed4d2 277 dm_statistics_init,
1da177e4
LT		 278};
279
b9249e55 280static void (*_exits[])(void) = {
1da177e4
LT		 281 local_exit,
282 dm_target_exit,
283 dm_linear_exit,
284 dm_stripe_exit,
952b3557 285 dm_io_exit,
945fa4d2 286 dm_kcopyd_exit,
1da177e4 287 dm_interface_exit,
fd2ed4d2 288 dm_statistics_exit,
1da177e4
LT		 289};
290
291static int __init dm_init(void)
292{
293 const int count = ARRAY_SIZE(_inits);
294
295 int r, i;
296
297 for (i = 0; i < count; i++) {
298 r = _inits[i]();
299 if (r)
300 goto bad;
301 }
302
303 return 0;
304
305 bad:
306 while (i--)
307 _exits[i]();
308
309 return r;
310}
311
312static void __exit dm_exit(void)
313{
314 int i = ARRAY_SIZE(_exits);
315
316 while (i--)
317 _exits[i]();
d15b774c
AK		 318
319 /*
320 * Should be empty by this point.
321 */
d15b774c 322 idr_destroy(&_minor_idr);
1da177e4
LT		 323}
324
325/*
326 * Block device functions
327 */
432a212c
MA		 328int dm_deleting_md(struct mapped_device *md)
329{
330 return test_bit(DMF_DELETING, &md->flags);
331}
332
fe5f9f2c 333static int dm_blk_open(struct block_device *bdev, fmode_t mode)
1da177e4
LT		 334{
335 struct mapped_device *md;
336
fba9f90e
JM		 337 spin_lock(&_minor_lock);
338
fe5f9f2c 339 md = bdev->bd_disk->private_data;
fba9f90e
JM		 340 if (!md)
341 goto out;
342
5c6bd75d 343 if (test_bit(DMF_FREEING, &md->flags) ||
432a212c 344 dm_deleting_md(md)) {
fba9f90e
JM		 345 md = NULL;
346 goto out;
347 }
348
1da177e4 349 dm_get(md);
5c6bd75d 350 atomic_inc(&md->open_count);
fba9f90e
JM		 351out:
352 spin_unlock(&_minor_lock);
353
354 return md ? 0 : -ENXIO;
1da177e4
LT		 355}
356
db2a144b 357static void dm_blk_close(struct gendisk *disk, fmode_t mode)
1da177e4 358{
63a4f065 359 struct mapped_device *md;
6e9624b8 360
4a1aeb98
MB		 361 spin_lock(&_minor_lock);
362
63a4f065
MS		 363 md = disk->private_data;
364 if (WARN_ON(!md))
365 goto out;
366
2c140a24
MP		 367 if (atomic_dec_and_test(&md->open_count) &&
368 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
acfe0ad7 369 queue_work(deferred_remove_workqueue, &deferred_remove_work);
2c140a24 370
1da177e4 371 dm_put(md);
63a4f065 372out:
4a1aeb98 373 spin_unlock(&_minor_lock);
1da177e4
LT		 374}
375
5c6bd75d
AK		 376int dm_open_count(struct mapped_device *md)
377{
378 return atomic_read(&md->open_count);
379}
380
381/*
382 * Guarantees nothing is using the device before it's deleted.
383 */
2c140a24 384int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
5c6bd75d
AK		 385{
386 int r = 0;
387
388 spin_lock(&_minor_lock);
389
2c140a24 390 if (dm_open_count(md)) {
5c6bd75d 391 r = -EBUSY;
2c140a24
MP		 392 if (mark_deferred)
393 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
394 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
395 r = -EEXIST;
5c6bd75d
AK		 396 else
397 set_bit(DMF_DELETING, &md->flags);
398
399 spin_unlock(&_minor_lock);
400
401 return r;
402}
403
2c140a24
MP		 404int dm_cancel_deferred_remove(struct mapped_device *md)
405{
406 int r = 0;
407
408 spin_lock(&_minor_lock);
409
410 if (test_bit(DMF_DELETING, &md->flags))
411 r = -EBUSY;
412 else
413 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
414
415 spin_unlock(&_minor_lock);
416
417 return r;
418}
419
420static void do_deferred_remove(struct work_struct *w)
421{
422 dm_deferred_remove();
423}
424
3ac51e74
DW		 425static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
426{
427 struct mapped_device *md = bdev->bd_disk->private_data;
428
429 return dm_get_geometry(md, geo);
430}
431
d4100351
CH		 432#ifdef CONFIG_BLK_DEV_ZONED
433int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, void *data)
434{
435 struct dm_report_zones_args *args = data;
436 sector_t sector_diff = args->tgt->begin - args->start;
437
438 /*
439 * Ignore zones beyond the target range.
440 */
441 if (zone->start >= args->start + args->tgt->len)
442 return 0;
443
444 /*
445 * Remap the start sector and write pointer position of the zone
446 * to match its position in the target range.
447 */
448 zone->start += sector_diff;
449 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
450 if (zone->cond == BLK_ZONE_COND_FULL)
451 zone->wp = zone->start + zone->len;
452 else if (zone->cond == BLK_ZONE_COND_EMPTY)
453 zone->wp = zone->start;
454 else
455 zone->wp += sector_diff;
456 }
457
458 args->next_sector = zone->start + zone->len;
459 return args->orig_cb(zone, args->zone_idx++, args->orig_data);
460}
461EXPORT_SYMBOL_GPL(dm_report_zones_cb);
462
e76239a3 463static int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
d4100351 464 unsigned int nr_zones, report_zones_cb cb, void *data)
e76239a3 465{
e76239a3 466 struct mapped_device *md = disk->private_data;
e76239a3
CH		 467 struct dm_table *map;
468 int srcu_idx, ret;
d4100351
CH		 469 struct dm_report_zones_args args = {
470 .next_sector = sector,
471 .orig_data = data,
472 .orig_cb = cb,
473 };
e76239a3
CH		 474
475 if (dm_suspended_md(md))
476 return -EAGAIN;
477
478 map = dm_get_live_table(md, &srcu_idx);
479 if (!map)
480 return -EIO;
481
d4100351
CH		 482 do {
483 struct dm_target *tgt;
e76239a3 484
d4100351
CH		 485 tgt = dm_table_find_target(map, args.next_sector);
486 if (WARN_ON_ONCE(!tgt->type->report_zones)) {
487 ret = -EIO;
488 goto out;
489 }
e76239a3 490
d4100351 491 args.tgt = tgt;
a9cb9f41
JT		 492 ret = tgt->type->report_zones(tgt, &args,
493 nr_zones - args.zone_idx);
d4100351
CH		 494 if (ret < 0)
495 goto out;
496 } while (args.zone_idx < nr_zones &&
497 args.next_sector < get_capacity(disk));
e76239a3 498
d4100351 499 ret = args.zone_idx;
e76239a3
CH		 500out:
501 dm_put_live_table(md, srcu_idx);
502 return ret;
e76239a3 503}
d4100351
CH		 504#else
505#define dm_blk_report_zones NULL
506#endif /* CONFIG_BLK_DEV_ZONED */
e76239a3 507
971888c4 508static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
5bd5e8d8 509 struct block_device **bdev)
971888c4 510 __acquires(md->io_barrier)
aa129a22 511{
66482026 512 struct dm_target *tgt;
6c182cd8 513 struct dm_table *map;
971888c4 514 int r;
aa129a22 515
6c182cd8 516retry:
e56f81e0 517 r = -ENOTTY;
971888c4 518 map = dm_get_live_table(md, srcu_idx);
aa129a22 519 if (!map || !dm_table_get_size(map))
971888c4 520 return r;
aa129a22
MB		 521
522 /* We only support devices that have a single target */
523 if (dm_table_get_num_targets(map) != 1)
971888c4 524 return r;
aa129a22 525
66482026
MS		 526 tgt = dm_table_get_target(map, 0);
527 if (!tgt->type->prepare_ioctl)
971888c4 528 return r;
519049af 529
971888c4
MS		 530 if (dm_suspended_md(md))
531 return -EAGAIN;
aa129a22 532
5bd5e8d8 533 r = tgt->type->prepare_ioctl(tgt, bdev);
5bbbfdf6 534 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
971888c4 535 dm_put_live_table(md, *srcu_idx);
6c182cd8
HR		 536 msleep(10);
537 goto retry;
538 }
971888c4 539
e56f81e0
CH		 540 return r;
541}
542
971888c4
MS		 543static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
544 __releases(md->io_barrier)
545{
546 dm_put_live_table(md, srcu_idx);
547}
548
e56f81e0
CH		 549static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
550 unsigned int cmd, unsigned long arg)
551{
552 struct mapped_device *md = bdev->bd_disk->private_data;
971888c4 553 int r, srcu_idx;
e56f81e0 554
5bd5e8d8 555 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
e56f81e0 556 if (r < 0)
971888c4 557 goto out;
6c182cd8 558
e56f81e0
CH		 559 if (r > 0) {
560 /*
e980f623
CH		 561 * Target determined this ioctl is being issued against a
562 * subset of the parent bdev; require extra privileges.
e56f81e0 563 */
e980f623
CH		 564 if (!capable(CAP_SYS_RAWIO)) {
565 DMWARN_LIMIT(
566 "%s: sending ioctl %x to DM device without required privilege.",
567 current->comm, cmd);
568 r = -ENOIOCTLCMD;
e56f81e0 569 goto out;
e980f623 570 }
e56f81e0 571 }
6c182cd8 572
a7cb3d2f
CH		 573 if (!bdev->bd_disk->fops->ioctl)
574 r = -ENOTTY;
575 else
576 r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
e56f81e0 577out:
971888c4 578 dm_unprepare_ioctl(md, srcu_idx);
aa129a22
MB		 579 return r;
580}
581
7465d7ac
MS		 582u64 dm_start_time_ns_from_clone(struct bio *bio)
583{
584 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
585 struct dm_io *io = tio->io;
586
587 return jiffies_to_nsecs(io->start_time);
588}
589EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
590
591static void start_io_acct(struct dm_io *io)
592{
593 struct mapped_device *md = io->md;
594 struct bio *bio = io->orig_bio;
595
596 io->start_time = bio_start_io_acct(bio);
597 if (unlikely(dm_stats_used(&md->stats)))
598 dm_stats_account_io(&md->stats, bio_data_dir(bio),
599 bio->bi_iter.bi_sector, bio_sectors(bio),
600 false, 0, &io->stats_aux);
601}
602
603static void end_io_acct(struct dm_io *io)
604{
605 struct mapped_device *md = io->md;
606 struct bio *bio = io->orig_bio;
607 unsigned long duration = jiffies - io->start_time;
608
609 bio_end_io_acct(bio, io->start_time);
610
611 if (unlikely(dm_stats_used(&md->stats)))
612 dm_stats_account_io(&md->stats, bio_data_dir(bio),
613 bio->bi_iter.bi_sector, bio_sectors(bio),
614 true, duration, &io->stats_aux);
615
616 /* nudge anyone waiting on suspend queue */
617 if (unlikely(wq_has_sleeper(&md->wait)))
618 wake_up(&md->wait);
619}
978e51ba
MS		 620
621static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
1da177e4 622{
64f52b0e
MS		 623 struct dm_io *io;
624 struct dm_target_io *tio;
625 struct bio *clone;
626
6f1c819c 627 clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
64f52b0e
MS		 628 if (!clone)
629 return NULL;
630
631 tio = container_of(clone, struct dm_target_io, clone);
632 tio->inside_dm_io = true;
633 tio->io = NULL;
634
635 io = container_of(tio, struct dm_io, tio);
636 io->magic = DM_IO_MAGIC;
978e51ba
MS		 637 io->status = 0;
638 atomic_set(&io->io_count, 1);
639 io->orig_bio = bio;
640 io->md = md;
641 spin_lock_init(&io->endio_lock);
642
643 start_io_acct(io);
64f52b0e
MS		 644
645 return io;
1da177e4
LT		 646}
647
028867ac 648static void free_io(struct mapped_device *md, struct dm_io *io)
1da177e4 649{
64f52b0e
MS		 650 bio_put(&io->tio.clone);
651}
652
653static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
654 unsigned target_bio_nr, gfp_t gfp_mask)
655{
656 struct dm_target_io *tio;
657
658 if (!ci->io->tio.io) {
659 /* the dm_target_io embedded in ci->io is available */
660 tio = &ci->io->tio;
661 } else {
6f1c819c 662 struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
64f52b0e
MS		 663 if (!clone)
664 return NULL;
665
666 tio = container_of(clone, struct dm_target_io, clone);
667 tio->inside_dm_io = false;
668 }
669
670 tio->magic = DM_TIO_MAGIC;
671 tio->io = ci->io;
672 tio->ti = ti;
673 tio->target_bio_nr = target_bio_nr;
674
675 return tio;
1da177e4
LT		 676}
677
cfae7529 678static void free_tio(struct dm_target_io *tio)
1da177e4 679{
64f52b0e
MS		 680 if (tio->inside_dm_io)
681 return;
dba14160 682 bio_put(&tio->clone);
1da177e4
LT		 683}
684
685/*
686 * Add the bio to the list of deferred io.
687 */
92c63902 688static void queue_io(struct mapped_device *md, struct bio *bio)
1da177e4 689{
05447420 690 unsigned long flags;
1da177e4 691
05447420 692 spin_lock_irqsave(&md->deferred_lock, flags);
1da177e4 693 bio_list_add(&md->deferred, bio);
05447420 694 spin_unlock_irqrestore(&md->deferred_lock, flags);
6a8736d1 695 queue_work(md->wq, &md->work);
1da177e4
LT		 696}
697
698/*
699 * Everyone (including functions in this file), should use this
700 * function to access the md->map field, and make sure they call
83d5e5b0 701 * dm_put_live_table() when finished.
1da177e4 702 */
83d5e5b0 703struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
1da177e4 704{
83d5e5b0
MP		 705 *srcu_idx = srcu_read_lock(&md->io_barrier);
706
707 return srcu_dereference(md->map, &md->io_barrier);
708}
1da177e4 709
83d5e5b0
MP		 710void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
711{
712 srcu_read_unlock(&md->io_barrier, srcu_idx);
713}
714
715void dm_sync_table(struct mapped_device *md)
716{
717 synchronize_srcu(&md->io_barrier);
718 synchronize_rcu_expedited();
719}
720
721/*
722 * A fast alternative to dm_get_live_table/dm_put_live_table.
723 * The caller must not block between these two functions.
724 */
725static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
726{
727 rcu_read_lock();
728 return rcu_dereference(md->map);
729}
1da177e4 730
83d5e5b0
MP		 731static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
732{
733 rcu_read_unlock();
1da177e4
LT		 734}
735
971888c4
MS		 736static char *_dm_claim_ptr = "I belong to device-mapper";
737
86f1152b
BM		 738/*
739 * Open a table device so we can use it as a map destination.
740 */
741static int open_table_device(struct table_device *td, dev_t dev,
742 struct mapped_device *md)
743{
86f1152b
BM		 744 struct block_device *bdev;
745
746 int r;
747
748 BUG_ON(td->dm_dev.bdev);
749
519049af 750 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
86f1152b
BM		 751 if (IS_ERR(bdev))
752 return PTR_ERR(bdev);
753
754 r = bd_link_disk_holder(bdev, dm_disk(md));
755 if (r) {
756 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
757 return r;
758 }
759
760 td->dm_dev.bdev = bdev;
817bf402 761 td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
86f1152b
BM		 762 return 0;
763}
764
765/*
766 * Close a table device that we've been using.
767 */
768static void close_table_device(struct table_device *td, struct mapped_device *md)
769{
770 if (!td->dm_dev.bdev)
771 return;
772
773 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
774 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
817bf402 775 put_dax(td->dm_dev.dax_dev);
86f1152b 776 td->dm_dev.bdev = NULL;
817bf402 777 td->dm_dev.dax_dev = NULL;
86f1152b
BM		 778}
779
780static struct table_device *find_table_device(struct list_head *l, dev_t dev,
8454fca4
SS		 781 fmode_t mode)
782{
86f1152b
BM		 783 struct table_device *td;
784
785 list_for_each_entry(td, l, list)
786 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
787 return td;
788
789 return NULL;
790}
791
792int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
8454fca4
SS		 793 struct dm_dev **result)
794{
86f1152b
BM		 795 int r;
796 struct table_device *td;
797
798 mutex_lock(&md->table_devices_lock);
799 td = find_table_device(&md->table_devices, dev, mode);
800 if (!td) {
115485e8 801 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
86f1152b
BM		 802 if (!td) {
803 mutex_unlock(&md->table_devices_lock);
804 return -ENOMEM;
805 }
806
807 td->dm_dev.mode = mode;
808 td->dm_dev.bdev = NULL;
809
810 if ((r = open_table_device(td, dev, md))) {
811 mutex_unlock(&md->table_devices_lock);
812 kfree(td);
813 return r;
814 }
815
816 format_dev_t(td->dm_dev.name, dev);
817
b0b4d7c6 818 refcount_set(&td->count, 1);
86f1152b 819 list_add(&td->list, &md->table_devices);
b0b4d7c6
ER		 820 } else {
821 refcount_inc(&td->count);
86f1152b 822 }
86f1152b
BM		 823 mutex_unlock(&md->table_devices_lock);
824
825 *result = &td->dm_dev;
826 return 0;
827}
828EXPORT_SYMBOL_GPL(dm_get_table_device);
829
830void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
831{
832 struct table_device *td = container_of(d, struct table_device, dm_dev);
833
834 mutex_lock(&md->table_devices_lock);
b0b4d7c6 835 if (refcount_dec_and_test(&td->count)) {
86f1152b
BM		 836 close_table_device(td, md);
837 list_del(&td->list);
838 kfree(td);
839 }
840 mutex_unlock(&md->table_devices_lock);
841}
842EXPORT_SYMBOL(dm_put_table_device);
843
844static void free_table_devices(struct list_head *devices)
845{
846 struct list_head *tmp, *next;
847
848 list_for_each_safe(tmp, next, devices) {
849 struct table_device *td = list_entry(tmp, struct table_device, list);
850
851 DMWARN("dm_destroy: %s still exists with %d references",
b0b4d7c6 852 td->dm_dev.name, refcount_read(&td->count));
86f1152b
BM		 853 kfree(td);
854 }
855}
856
3ac51e74
DW		 857/*
858 * Get the geometry associated with a dm device
859 */
860int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
861{
862 *geo = md->geometry;
863
864 return 0;
865}
866
867/*
868 * Set the geometry of a device.
869 */
870int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
871{
872 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
873
874 if (geo->start > sz) {
875 DMWARN("Start sector is beyond the geometry limits.");
876 return -EINVAL;
877 }
878
879 md->geometry = *geo;
880
881 return 0;
882}
883
2e93ccc1
KU		 884static int __noflush_suspending(struct mapped_device *md)
885{
886 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
887}
888
1da177e4
LT		 889/*
890 * Decrements the number of outstanding ios that a bio has been
891 * cloned into, completing the original io if necc.
892 */
4e4cbee9 893static void dec_pending(struct dm_io *io, blk_status_t error)
1da177e4 894{
2e93ccc1 895 unsigned long flags;
4e4cbee9 896 blk_status_t io_error;
b35f8caa
MB		 897 struct bio *bio;
898 struct mapped_device *md = io->md;
2e93ccc1
KU		 899
900 /* Push-back supersedes any I/O errors */
f88fb981
KU		 901 if (unlikely(error)) {
902 spin_lock_irqsave(&io->endio_lock, flags);
745dc570 903 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
4e4cbee9 904 io->status = error;
f88fb981
KU		 905 spin_unlock_irqrestore(&io->endio_lock, flags);
906 }
1da177e4
LT		 907
908 if (atomic_dec_and_test(&io->io_count)) {
4e4cbee9 909 if (io->status == BLK_STS_DM_REQUEUE) {
2e93ccc1
KU		 910 /*
911 * Target requested pushing back the I/O.
2e93ccc1 912 */
022c2611 913 spin_lock_irqsave(&md->deferred_lock, flags);
6a8736d1 914 if (__noflush_suspending(md))
745dc570
MS		 915 /* NOTE early return due to BLK_STS_DM_REQUEUE below */
916 bio_list_add_head(&md->deferred, io->orig_bio);
6a8736d1 917 else
2e93ccc1 918 /* noflush suspend was interrupted. */
4e4cbee9 919 io->status = BLK_STS_IOERR;
022c2611 920 spin_unlock_irqrestore(&md->deferred_lock, flags);
2e93ccc1
KU		 921 }
922
4e4cbee9 923 io_error = io->status;
745dc570 924 bio = io->orig_bio;
6a8736d1
TH		 925 end_io_acct(io);
926 free_io(md, io);
927
4e4cbee9 928 if (io_error == BLK_STS_DM_REQUEUE)
6a8736d1 929 return;
2e93ccc1 930
1eff9d32 931 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
af7e466a 932 /*
6a8736d1 933 * Preflush done for flush with data, reissue
28a8f0d3 934 * without REQ_PREFLUSH.
af7e466a 935 */
1eff9d32 936 bio->bi_opf &= ~REQ_PREFLUSH;
6a8736d1 937 queue_io(md, bio);
af7e466a 938 } else {
b372d360 939 /* done with normal IO or empty flush */
8dd601fa
N		 940 if (io_error)
941 bio->bi_status = io_error;
4246a0b6 942 bio_endio(bio);
b35f8caa 943 }
1da177e4
LT		 944 }
945}
946
bcb44433
MS		 947void disable_discard(struct mapped_device *md)
948{
949 struct queue_limits *limits = dm_get_queue_limits(md);
950
951 /* device doesn't really support DISCARD, disable it */
952 limits->max_discard_sectors = 0;
953 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
954}
955
4cc96131 956void disable_write_same(struct mapped_device *md)
7eee4ae2
MS		 957{
958 struct queue_limits *limits = dm_get_queue_limits(md);
959
960 /* device doesn't really support WRITE SAME, disable it */
961 limits->max_write_same_sectors = 0;
962}
963
ac62d620
CH		 964void disable_write_zeroes(struct mapped_device *md)
965{
966 struct queue_limits *limits = dm_get_queue_limits(md);
967
968 /* device doesn't really support WRITE ZEROES, disable it */
969 limits->max_write_zeroes_sectors = 0;
970}
971
4246a0b6 972static void clone_endio(struct bio *bio)
1da177e4 973{
4e4cbee9 974 blk_status_t error = bio->bi_status;
bfc6d41c 975 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
b35f8caa 976 struct dm_io *io = tio->io;
9faf400f 977 struct mapped_device *md = tio->io->md;
1da177e4 978 dm_endio_fn endio = tio->ti->type->end_io;
415c79e1 979 struct bio *orig_bio = io->orig_bio;
1da177e4 980
9c37de29 981 if (unlikely(error == BLK_STS_TARGET)) {
bcb44433
MS		 982 if (bio_op(bio) == REQ_OP_DISCARD &&
983 !bio->bi_disk->queue->limits.max_discard_sectors)
984 disable_discard(md);
985 else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
986 !bio->bi_disk->queue->limits.max_write_same_sectors)
ac62d620 987 disable_write_same(md);
bcb44433
MS		 988 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
989 !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
ac62d620
CH		 990 disable_write_zeroes(md);
991 }
7eee4ae2 992
415c79e1
JT		 993 /*
994 * For zone-append bios get offset in zone of the written
995 * sector and add that to the original bio sector pos.
996 */
997 if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
998 sector_t written_sector = bio->bi_iter.bi_sector;
999 struct request_queue *q = orig_bio->bi_disk->queue;
1000 u64 mask = (u64)blk_queue_zone_sectors(q) - 1;
1001
1002 orig_bio->bi_iter.bi_sector += written_sector & mask;
1003 }
1004
1be56909 1005 if (endio) {
4e4cbee9 1006 int r = endio(tio->ti, bio, &error);
1be56909
CH		 1007 switch (r) {
1008 case DM_ENDIO_REQUEUE:
4e4cbee9 1009 error = BLK_STS_DM_REQUEUE;
df561f66 1010 fallthrough;
1be56909
CH		 1011 case DM_ENDIO_DONE:
1012 break;
1013 case DM_ENDIO_INCOMPLETE:
1014 /* The target will handle the io */
1015 return;
1016 default:
1017 DMWARN("unimplemented target endio return value: %d", r);
1018 BUG();
1019 }
1020 }
1021
cfae7529 1022 free_tio(tio);
b35f8caa 1023 dec_pending(io, error);
1da177e4
LT		 1024}
1025
56a67df7
MS		 1026/*
1027 * Return maximum size of I/O possible at the supplied sector up to the current
1028 * target boundary.
1029 */
3720281d
MS		 1030static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
1031 sector_t target_offset)
56a67df7 1032{
56a67df7
MS		 1033 return ti->len - target_offset;
1034}
1035
3720281d 1036static sector_t max_io_len(struct dm_target *ti, sector_t sector)
1da177e4 1037{
3720281d
MS		 1038 sector_t target_offset = dm_target_offset(ti, sector);
1039 sector_t len = max_io_len_target_boundary(ti, target_offset);
5091cdec 1040 sector_t max_len;
1da177e4
LT		 1041
1042 /*
542f9038 1043 * Does the target need to split even further?
5091cdec
MS		 1044 * - q->limits.chunk_sectors reflects ti->max_io_len so
1045 * blk_max_size_offset() provides required splitting.
1046 * - blk_max_size_offset() also respects q->limits.max_sectors
1da177e4 1047 */
33bd6f06 1048 max_len = blk_max_size_offset(ti->table->md->queue,
3720281d 1049 target_offset);
5091cdec
MS		 1050 if (len > max_len)
1051 len = max_len;
1da177e4
LT		 1052
1053 return len;
1054}
1055
542f9038
MS		 1056int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1057{
1058 if (len > UINT_MAX) {
1059 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1060 (unsigned long long)len, UINT_MAX);
1061 ti->error = "Maximum size of target IO is too large";
1062 return -EINVAL;
1063 }
1064
75ae1936 1065 ti->max_io_len = (uint32_t) len;
542f9038
MS		 1066
1067 return 0;
1068}
1069EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1070
f26c5719 1071static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
3d97c829
MS		 1072 sector_t sector, int *srcu_idx)
1073 __acquires(md->io_barrier)
545ed20e 1074{
545ed20e
TK		 1075 struct dm_table *map;
1076 struct dm_target *ti;
545ed20e 1077
f26c5719 1078 map = dm_get_live_table(md, srcu_idx);
545ed20e 1079 if (!map)
f26c5719 1080 return NULL;
545ed20e
TK		 1081
1082 ti = dm_table_find_target(map, sector);
123d87d5 1083 if (!ti)
f26c5719 1084 return NULL;
545ed20e 1085
f26c5719
DW		 1086 return ti;
1087}
545ed20e 1088
f26c5719 1089static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
3d97c829 1090 long nr_pages, void **kaddr, pfn_t *pfn)
f26c5719
DW		 1091{
1092 struct mapped_device *md = dax_get_private(dax_dev);
1093 sector_t sector = pgoff * PAGE_SECTORS;
1094 struct dm_target *ti;
1095 long len, ret = -EIO;
1096 int srcu_idx;
545ed20e 1097
f26c5719 1098 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
545ed20e 1099
f26c5719
DW		 1100 if (!ti)
1101 goto out;
1102 if (!ti->type->direct_access)
1103 goto out;
3720281d 1104 len = max_io_len(ti, sector) / PAGE_SECTORS;
f26c5719
DW		 1105 if (len < 1)
1106 goto out;
1107 nr_pages = min(len, nr_pages);
dbc62659 1108 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
817bf402 1109
f26c5719 1110 out:
545ed20e 1111 dm_put_live_table(md, srcu_idx);
f26c5719
DW		 1112
1113 return ret;
545ed20e
TK		 1114}
1115
7bf7eac8
DW		 1116static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
1117 int blocksize, sector_t start, sector_t len)
1118{
1119 struct mapped_device *md = dax_get_private(dax_dev);
1120 struct dm_table *map;
02186d88 1121 bool ret = false;
7bf7eac8 1122 int srcu_idx;
7bf7eac8
DW		 1123
1124 map = dm_get_live_table(md, &srcu_idx);
1125 if (!map)
02186d88 1126 goto out;
7bf7eac8 1127
2e9ee095 1128 ret = dm_table_supports_dax(map, device_supports_dax, &blocksize);
7bf7eac8 1129
02186d88 1130out:
7bf7eac8
DW		 1131 dm_put_live_table(md, srcu_idx);
1132
1133 return ret;
1134}
1135
7e026c8c 1136static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
3d97c829 1137 void *addr, size_t bytes, struct iov_iter *i)
7e026c8c
DW		 1138{
1139 struct mapped_device *md = dax_get_private(dax_dev);
1140 sector_t sector = pgoff * PAGE_SECTORS;
1141 struct dm_target *ti;
1142 long ret = 0;
1143 int srcu_idx;
1144
1145 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1146
1147 if (!ti)
1148 goto out;
1149 if (!ti->type->dax_copy_from_iter) {
1150 ret = copy_from_iter(addr, bytes, i);
1151 goto out;
1152 }
1153 ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
1154 out:
1155 dm_put_live_table(md, srcu_idx);
1156
1157 return ret;
1158}
1159
b3a9a0c3
DW		 1160static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1161 void *addr, size_t bytes, struct iov_iter *i)
1162{
1163 struct mapped_device *md = dax_get_private(dax_dev);
1164 sector_t sector = pgoff * PAGE_SECTORS;
1165 struct dm_target *ti;
1166 long ret = 0;
1167 int srcu_idx;
1168
1169 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1170
1171 if (!ti)
1172 goto out;
1173 if (!ti->type->dax_copy_to_iter) {
1174 ret = copy_to_iter(addr, bytes, i);
1175 goto out;
1176 }
1177 ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
1178 out:
1179 dm_put_live_table(md, srcu_idx);
1180
1181 return ret;
1182}
1183
cdf6cdcd
VG		 1184static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1185 size_t nr_pages)
1186{
1187 struct mapped_device *md = dax_get_private(dax_dev);
1188 sector_t sector = pgoff * PAGE_SECTORS;
1189 struct dm_target *ti;
1190 int ret = -EIO;
1191 int srcu_idx;
1192
1193 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1194
1195 if (!ti)
1196 goto out;
1197 if (WARN_ON(!ti->type->dax_zero_page_range)) {
1198 /*
1199 * ->zero_page_range() is mandatory dax operation. If we are
1200 * here, something is wrong.
1201 */
1202 dm_put_live_table(md, srcu_idx);
1203 goto out;
1204 }
1205 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1206
1207 out:
1208 dm_put_live_table(md, srcu_idx);
1209
1210 return ret;
1211}
1212
1dd40c3e
MP		 1213/*
1214 * A target may call dm_accept_partial_bio only from the map routine. It is
2e2d6f7e
AJ		 1215 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_RESET,
1216 * REQ_OP_ZONE_OPEN, REQ_OP_ZONE_CLOSE and REQ_OP_ZONE_FINISH.
1dd40c3e
MP		 1217 *
1218 * dm_accept_partial_bio informs the dm that the target only wants to process
1219 * additional n_sectors sectors of the bio and the rest of the data should be
1220 * sent in a next bio.
1221 *
1222 * A diagram that explains the arithmetics:
1223 * +--------------------+---------------+-------+
1224 * | 1 | 2 | 3 |
1225 * +--------------------+---------------+-------+
1226 *
1227 * <-------------- *tio->len_ptr --------------->
1228 * <------- bi_size ------->
1229 * <-- n_sectors -->
1230 *
1231 * Region 1 was already iterated over with bio_advance or similar function.
1232 * (it may be empty if the target doesn't use bio_advance)
1233 * Region 2 is the remaining bio size that the target wants to process.
1234 * (it may be empty if region 1 is non-empty, although there is no reason
1235 * to make it empty)
1236 * The target requires that region 3 is to be sent in the next bio.
1237 *
1238 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1239 * the partially processed part (the sum of regions 1+2) must be the same for all
1240 * copies of the bio.
1241 */
1242void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1243{
1244 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1245 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1eff9d32 1246 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1dd40c3e
MP		 1247 BUG_ON(bi_size > *tio->len_ptr);
1248 BUG_ON(n_sectors > bi_size);
1249 *tio->len_ptr -= bi_size - n_sectors;
1250 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1251}
1252EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1253
978e51ba 1254static blk_qc_t __map_bio(struct dm_target_io *tio)
1da177e4
LT		 1255{
1256 int r;
2056a782 1257 sector_t sector;
dba14160 1258 struct bio *clone = &tio->clone;
64f52b0e 1259 struct dm_io *io = tio->io;
bd2a49b8 1260 struct dm_target *ti = tio->ti;
978e51ba 1261 blk_qc_t ret = BLK_QC_T_NONE;
1da177e4 1262
1da177e4 1263 clone->bi_end_io = clone_endio;
1da177e4
LT		 1264
1265 /*
1266 * Map the clone. If r == 0 we don't need to do
1267 * anything, the target has assumed ownership of
1268 * this io.
1269 */
64f52b0e 1270 atomic_inc(&io->io_count);
4f024f37 1271 sector = clone->bi_iter.bi_sector;
d67a5f4b 1272
7de3ee57 1273 r = ti->type->map(ti, clone);
846785e6
CH		 1274 switch (r) {
1275 case DM_MAPIO_SUBMITTED:
1276 break;
1277 case DM_MAPIO_REMAPPED:
1da177e4 1278 /* the bio has been remapped so dispatch it */
74d46992 1279 trace_block_bio_remap(clone->bi_disk->queue, clone,
64f52b0e 1280 bio_dev(io->orig_bio), sector);
5a6c35f9 1281 ret = submit_bio_noacct(clone);
846785e6
CH		 1282 break;
1283 case DM_MAPIO_KILL:
4e4cbee9 1284 free_tio(tio);
64f52b0e 1285 dec_pending(io, BLK_STS_IOERR);
4e4cbee9 1286 break;
846785e6 1287 case DM_MAPIO_REQUEUE:
cfae7529 1288 free_tio(tio);
64f52b0e 1289 dec_pending(io, BLK_STS_DM_REQUEUE);
846785e6
CH		 1290 break;
1291 default:
45cbcd79
KU		 1292 DMWARN("unimplemented target map return value: %d", r);
1293 BUG();
1da177e4 1294 }
1da177e4 1295
978e51ba 1296 return ret;
1da177e4 1297}
1da177e4 1298
e0d6609a 1299static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
bd2a49b8 1300{
4f024f37
KO		 1301 bio->bi_iter.bi_sector = sector;
1302 bio->bi_iter.bi_size = to_bytes(len);
1da177e4
LT		 1303}
1304
1305/*
1306 * Creates a bio that consists of range of complete bvecs.
1307 */
c80914e8
MS		 1308static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1309 sector_t sector, unsigned len)
1da177e4 1310{
dba14160 1311 struct bio *clone = &tio->clone;
07560151 1312 int r;
1da177e4 1313
1c3b13e6
KO		 1314 __bio_clone_fast(clone, bio);
1315
07560151
EB		 1316 r = bio_crypt_clone(clone, bio, GFP_NOIO);
1317 if (r < 0)
1318 return r;
a892c8d5 1319
57c36519 1320 if (bio_integrity(bio)) {
e2460f2a
MP		 1321 if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1322 !dm_target_passes_integrity(tio->ti->type))) {
1323 DMWARN("%s: the target %s doesn't support integrity data.",
1324 dm_device_name(tio->io->md),
1325 tio->ti->type->name);
1326 return -EIO;
1327 }
1328
1329 r = bio_integrity_clone(clone, bio, GFP_NOIO);
c80914e8
MS		 1330 if (r < 0)
1331 return r;
1332 }
bd2a49b8 1333
fa8db494
MS		 1334 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1335 clone->bi_iter.bi_size = to_bytes(len);
1336
1337 if (bio_integrity(bio))
1338 bio_integrity_trim(clone);
c80914e8
MS		 1339
1340 return 0;
1da177e4
LT		 1341}
1342
318716dd
MS		 1343static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1344 struct dm_target *ti, unsigned num_bios)
f9ab94ce 1345{
dba14160 1346 struct dm_target_io *tio;
318716dd 1347 int try;
dba14160 1348
318716dd
MS		 1349 if (!num_bios)
1350 return;
f9ab94ce 1351
318716dd
MS		 1352 if (num_bios == 1) {
1353 tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1354 bio_list_add(blist, &tio->clone);
1355 return;
1356 }
9015df24 1357
318716dd
MS		 1358 for (try = 0; try < 2; try++) {
1359 int bio_nr;
1360 struct bio *bio;
1361
1362 if (try)
bc02cdbe 1363 mutex_lock(&ci->io->md->table_devices_lock);
318716dd
MS		 1364 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1365 tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
1366 if (!tio)
1367 break;
1368
1369 bio_list_add(blist, &tio->clone);
1370 }
1371 if (try)
bc02cdbe 1372 mutex_unlock(&ci->io->md->table_devices_lock);
318716dd
MS		 1373 if (bio_nr == num_bios)
1374 return;
1375
1376 while ((bio = bio_list_pop(blist))) {
1377 tio = container_of(bio, struct dm_target_io, clone);
1378 free_tio(tio);
1379 }
1380 }
9015df24
AK		 1381}
1382
978e51ba
MS		 1383static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
1384 struct dm_target_io *tio, unsigned *len)
9015df24 1385{
dba14160 1386 struct bio *clone = &tio->clone;
9015df24 1387
1dd40c3e
MP		 1388 tio->len_ptr = len;
1389
99778273 1390 __bio_clone_fast(clone, ci->bio);
bd2a49b8 1391 if (len)
1dd40c3e 1392 bio_setup_sector(clone, ci->sector, *len);
f9ab94ce 1393
978e51ba 1394 return __map_bio(tio);
f9ab94ce
MP		 1395}
1396
14fe594d 1397static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1dd40c3e 1398 unsigned num_bios, unsigned *len)
06a426ce 1399{
318716dd
MS		 1400 struct bio_list blist = BIO_EMPTY_LIST;
1401 struct bio *bio;
1402 struct dm_target_io *tio;
1403
1404 alloc_multiple_bios(&blist, ci, ti, num_bios);
06a426ce 1405
318716dd
MS		 1406 while ((bio = bio_list_pop(&blist))) {
1407 tio = container_of(bio, struct dm_target_io, clone);
978e51ba 1408 (void) __clone_and_map_simple_bio(ci, tio, len);
318716dd 1409 }
06a426ce
MS		 1410}
1411
14fe594d 1412static int __send_empty_flush(struct clone_info *ci)
f9ab94ce 1413{
06a426ce 1414 unsigned target_nr = 0;
f9ab94ce 1415 struct dm_target *ti;
828678b8
MS		 1416 struct bio flush_bio;
1417
1418 /*
1419 * Use an on-stack bio for this, it's safe since we don't
1420 * need to reference it after submit. It's just used as
1421 * the basis for the clone(s).
1422 */
1423 bio_init(&flush_bio, NULL, 0);
1424 flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
47d95102
CH		 1425 flush_bio.bi_disk = ci->io->md->disk;
1426 bio_associate_blkg(&flush_bio);
1427
828678b8
MS		 1428 ci->bio = &flush_bio;
1429 ci->sector_count = 0;
f9ab94ce 1430
b372d360 1431 BUG_ON(bio_has_data(ci->bio));
f9ab94ce 1432 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1dd40c3e 1433 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
828678b8
MS		 1434
1435 bio_uninit(ci->bio);
f9ab94ce
MP		 1436 return 0;
1437}
1438
c80914e8 1439static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
f31c21e4 1440 sector_t sector, unsigned *len)
5ae89a87 1441{
dba14160 1442 struct bio *bio = ci->bio;
5ae89a87 1443 struct dm_target_io *tio;
f31c21e4 1444 int r;
5ae89a87 1445
318716dd 1446 tio = alloc_tio(ci, ti, 0, GFP_NOIO);
f31c21e4
N		 1447 tio->len_ptr = len;
1448 r = clone_bio(tio, bio, sector, *len);
1449 if (r < 0) {
1450 free_tio(tio);
1451 return r;
b0d8ed4d 1452 }
978e51ba 1453 (void) __map_bio(tio);
c80914e8 1454
f31c21e4 1455 return 0;
5ae89a87
MS		 1456}
1457
3d7f4562 1458static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
61697a6a 1459 unsigned num_bios)
ba1cbad9 1460{
51b86f9a 1461 unsigned len;
ba1cbad9 1462
3d7f4562
MS		 1463 /*
1464 * Even though the device advertised support for this type of
1465 * request, that does not mean every target supports it, and
1466 * reconfiguration might also have changed that since the
1467 * check was performed.
1468 */
3d7f4562
MS		 1469 if (!num_bios)
1470 return -EOPNOTSUPP;
ba1cbad9 1471
3720281d
MS		 1472 len = min_t(sector_t, ci->sector_count,
1473 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
51b86f9a 1474
3d7f4562 1475 __send_duplicate_bios(ci, ti, num_bios, &len);
e262f347 1476
3d7f4562
MS		 1477 ci->sector += len;
1478 ci->sector_count -= len;
5ae89a87
MS		 1479
1480 return 0;
ba1cbad9
MS		 1481}
1482
568c73a3
MS		 1483static bool is_abnormal_io(struct bio *bio)
1484{
1485 bool r = false;
1486
1487 switch (bio_op(bio)) {
1488 case REQ_OP_DISCARD:
1489 case REQ_OP_SECURE_ERASE:
1490 case REQ_OP_WRITE_SAME:
1491 case REQ_OP_WRITE_ZEROES:
1492 r = true;
1493 break;
1494 }
1495
1496 return r;
1497}
1498
0519c71e
MS		 1499static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1500 int *result)
1501{
1502 struct bio *bio = ci->bio;
9679b5a7 1503 unsigned num_bios = 0;
0519c71e 1504
9679b5a7
MS		 1505 switch (bio_op(bio)) {
1506 case REQ_OP_DISCARD:
1507 num_bios = ti->num_discard_bios;
1508 break;
1509 case REQ_OP_SECURE_ERASE:
1510 num_bios = ti->num_secure_erase_bios;
1511 break;
1512 case REQ_OP_WRITE_SAME:
1513 num_bios = ti->num_write_same_bios;
1514 break;
1515 case REQ_OP_WRITE_ZEROES:
1516 num_bios = ti->num_write_zeroes_bios;
1517 break;
1518 default:
0519c71e 1519 return false;
9679b5a7 1520 }
0519c71e 1521
9679b5a7 1522 *result = __send_changing_extent_only(ci, ti, num_bios);
0519c71e
MS		 1523 return true;
1524}
1525
e4c93811
AK		 1526/*
1527 * Select the correct strategy for processing a non-flush bio.
1528 */
14fe594d 1529static int __split_and_process_non_flush(struct clone_info *ci)
0ce65797 1530{
512875bd 1531 struct dm_target *ti;
1c3b13e6 1532 unsigned len;
c80914e8 1533 int r;
0ce65797 1534
512875bd 1535 ti = dm_table_find_target(ci->map, ci->sector);
123d87d5 1536 if (!ti)
512875bd
JN		 1537 return -EIO;
1538
568c73a3 1539 if (__process_abnormal_io(ci, ti, &r))
0519c71e 1540 return r;
3d7f4562 1541
3720281d 1542 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
0ce65797 1543
c80914e8
MS		 1544 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1545 if (r < 0)
1546 return r;
0ce65797 1547
1c3b13e6
KO		 1548 ci->sector += len;
1549 ci->sector_count -= len;
0ce65797 1550
1c3b13e6 1551 return 0;
0ce65797
MS		 1552}
1553
978e51ba
MS		 1554static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1555 struct dm_table *map, struct bio *bio)
1556{
1557 ci->map = map;
1558 ci->io = alloc_io(md, bio);
1559 ci->sector = bio->bi_iter.bi_sector;
1560}
1561
a1e1cb72
MS		 1562#define __dm_part_stat_sub(part, field, subnd) \
1563 (part_stat_get(part, field) -= (subnd))
1564
1da177e4 1565/*
14fe594d 1566 * Entry point to split a bio into clones and submit them to the targets.
1da177e4 1567 */
978e51ba
MS		 1568static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1569 struct dm_table *map, struct bio *bio)
0ce65797 1570{
1da177e4 1571 struct clone_info ci;
978e51ba 1572 blk_qc_t ret = BLK_QC_T_NONE;
512875bd 1573 int error = 0;
1da177e4 1574
978e51ba 1575 init_clone_info(&ci, md, map, bio);
0ce65797 1576
1eff9d32 1577 if (bio->bi_opf & REQ_PREFLUSH) {
14fe594d 1578 error = __send_empty_flush(&ci);
b372d360 1579 /* dec_pending submits any data associated with flush */
2e2d6f7e 1580 } else if (op_is_zone_mgmt(bio_op(bio))) {
a4aa5e56
DLM		 1581 ci.bio = bio;
1582 ci.sector_count = 0;
1583 error = __split_and_process_non_flush(&ci);
b372d360 1584 } else {
6a8736d1 1585 ci.bio = bio;
d87f4c14 1586 ci.sector_count = bio_sectors(bio);
18a25da8 1587 while (ci.sector_count && !error) {
14fe594d 1588 error = __split_and_process_non_flush(&ci);
18a25da8
N		 1589 if (current->bio_list && ci.sector_count && !error) {
1590 /*
ed00aabd 1591 * Remainder must be passed to submit_bio_noacct()
18a25da8
N		 1592 * so that it gets handled *after* bios already submitted
1593 * have been completely processed.
1594 * We take a clone of the original to store in
745dc570 1595 * ci.io->orig_bio to be used by end_io_acct() and
18a25da8 1596 * for dec_pending to use for completion handling.
18a25da8 1597 */
f21c601a
MS		 1598 struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1599 GFP_NOIO, &md->queue->bio_split);
745dc570 1600 ci.io->orig_bio = b;
a1e1cb72
MS		 1601
1602 /*
1603 * Adjust IO stats for each split, otherwise upon queue
1604 * reentry there will be redundant IO accounting.
1605 * NOTE: this is a stop-gap fix, a proper fix involves
1606 * significant refactoring of DM core's bio splitting
1607 * (by eliminating DM's splitting and just using bio_split)
1608 */
1609 part_stat_lock();
cb8432d6 1610 __dm_part_stat_sub(dm_disk(md)->part0->bd_part,
a1e1cb72
MS		 1611 sectors[op_stat_group(bio_op(bio))], ci.sector_count);
1612 part_stat_unlock();
1613
18a25da8 1614 bio_chain(b, bio);
075c18c3 1615 trace_block_split(md->queue, b, bio->bi_iter.bi_sector);
ed00aabd 1616 ret = submit_bio_noacct(bio);
18a25da8
N		 1617 break;
1618 }
1619 }
d87f4c14 1620 }
0ce65797 1621
1da177e4 1622 /* drop the extra reference count */
54385bf7 1623 dec_pending(ci.io, errno_to_blk_status(error));
978e51ba 1624 return ret;
0ce65797
MS		 1625}
1626
c62b37d9 1627static blk_qc_t dm_submit_bio(struct bio *bio)
cec47e3d 1628{
c4a59c4e 1629 struct mapped_device *md = bio->bi_disk->private_data;
978e51ba 1630 blk_qc_t ret = BLK_QC_T_NONE;
83d5e5b0
MP		 1631 int srcu_idx;
1632 struct dm_table *map;
cec47e3d 1633
83d5e5b0 1634 map = dm_get_live_table(md, &srcu_idx);
b2abdb1b
MS		 1635 if (unlikely(!map)) {
1636 DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1637 dm_device_name(md));
1638 bio_io_error(bio);
1639 goto out;
1640 }
29e4013d 1641
b2abdb1b 1642 /* If suspended, queue this IO for later */
6a8736d1 1643 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
6abc4946
KK		 1644 if (bio->bi_opf & REQ_NOWAIT)
1645 bio_wouldblock_error(bio);
b2abdb1b 1646 else if (bio->bi_opf & REQ_RAHEAD)
54d9a1b4 1647 bio_io_error(bio);
b2abdb1b
MS		 1648 else
1649 queue_io(md, bio);
1650 goto out;
cec47e3d 1651 }
1da177e4 1652
b2abdb1b
MS		 1653 /*
1654 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc)
1655 * otherwise associated queue_limits won't be imposed.
1656 */
1657 if (is_abnormal_io(bio))
1658 blk_queue_split(&bio);
978e51ba 1659
9c37de29 1660 ret = __split_and_process_bio(md, map, bio);
b2abdb1b 1661out:
83d5e5b0 1662 dm_put_live_table(md, srcu_idx);
978e51ba
MS		 1663 return ret;
1664}
1665
1da177e4
LT		 1666/*-----------------------------------------------------------------
1667 * An IDR is used to keep track of allocated minor numbers.
1668 *---------------------------------------------------------------*/
2b06cfff 1669static void free_minor(int minor)
1da177e4 1670{
f32c10b0 1671 spin_lock(&_minor_lock);
1da177e4 1672 idr_remove(&_minor_idr, minor);
f32c10b0 1673 spin_unlock(&_minor_lock);
1da177e4
LT		 1674}
1675
1676/*
1677 * See if the device with a specific minor # is free.
1678 */
cf13ab8e 1679static int specific_minor(int minor)
1da177e4 1680{
c9d76be6 1681 int r;
1da177e4
LT		 1682
1683 if (minor >= (1 << MINORBITS))
1684 return -EINVAL;
1685
c9d76be6 1686 idr_preload(GFP_KERNEL);
f32c10b0 1687 spin_lock(&_minor_lock);
1da177e4 1688
c9d76be6 1689 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1da177e4 1690
f32c10b0 1691 spin_unlock(&_minor_lock);
c9d76be6
TH		 1692 idr_preload_end();
1693 if (r < 0)
1694 return r == -ENOSPC ? -EBUSY : r;
1695 return 0;
1da177e4
LT		 1696}
1697
cf13ab8e 1698static int next_free_minor(int *minor)
1da177e4 1699{
c9d76be6 1700 int r;
62f75c2f 1701
c9d76be6 1702 idr_preload(GFP_KERNEL);
f32c10b0 1703 spin_lock(&_minor_lock);
1da177e4 1704
c9d76be6 1705 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1da177e4 1706
f32c10b0 1707 spin_unlock(&_minor_lock);
c9d76be6
TH		 1708 idr_preload_end();
1709 if (r < 0)
1710 return r;
1711 *minor = r;
1712 return 0;
1da177e4
LT		 1713}
1714
83d5cde4 1715static const struct block_device_operations dm_blk_dops;
681cc5e8 1716static const struct block_device_operations dm_rq_blk_dops;
f26c5719 1717static const struct dax_operations dm_dax_ops;
1da177e4 1718
53d5914f
MP		 1719static void dm_wq_work(struct work_struct *work);
1720
0f20972f
MS		 1721static void cleanup_mapped_device(struct mapped_device *md)
1722{
0f20972f
MS		 1723 if (md->wq)
1724 destroy_workqueue(md->wq);
6f1c819c
KO		 1725 bioset_exit(&md->bs);
1726 bioset_exit(&md->io_bs);
0f20972f 1727
f26c5719
DW		 1728 if (md->dax_dev) {
1729 kill_dax(md->dax_dev);
1730 put_dax(md->dax_dev);
1731 md->dax_dev = NULL;
1732 }
1733
0f20972f
MS		 1734 if (md->disk) {
1735 spin_lock(&_minor_lock);
1736 md->disk->private_data = NULL;
1737 spin_unlock(&_minor_lock);
0f20972f
MS		 1738 del_gendisk(md->disk);
1739 put_disk(md->disk);
1740 }
1741
1742 if (md->queue)
1743 blk_cleanup_queue(md->queue);
1744
d09960b0
TE		 1745 cleanup_srcu_struct(&md->io_barrier);
1746
d5ffebdd
MS		 1747 mutex_destroy(&md->suspend_lock);
1748 mutex_destroy(&md->type_lock);
1749 mutex_destroy(&md->table_devices_lock);
1750
4cc96131 1751 dm_mq_cleanup_mapped_device(md);
0f20972f
MS		 1752}
1753
1da177e4
LT		 1754/*
1755 * Allocate and initialise a blank device with a given minor.
1756 */
2b06cfff 1757static struct mapped_device *alloc_dev(int minor)
1da177e4 1758{
115485e8
MS		 1759 int r, numa_node_id = dm_get_numa_node();
1760 struct mapped_device *md;
ba61fdd1 1761 void *old_md;
1da177e4 1762
856eb091 1763 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1da177e4
LT		 1764 if (!md) {
1765 DMWARN("unable to allocate device, out of memory.");
1766 return NULL;
1767 }
1768
10da4f79 1769 if (!try_module_get(THIS_MODULE))
6ed7ade8 1770 goto bad_module_get;
10da4f79 1771
1da177e4 1772 /* get a minor number for the dev */
2b06cfff 1773 if (minor == DM_ANY_MINOR)
cf13ab8e 1774 r = next_free_minor(&minor);
2b06cfff 1775 else
cf13ab8e 1776 r = specific_minor(minor);
1da177e4 1777 if (r < 0)
6ed7ade8 1778 goto bad_minor;
1da177e4 1779
83d5e5b0
MP		 1780 r = init_srcu_struct(&md->io_barrier);
1781 if (r < 0)
1782 goto bad_io_barrier;
1783
115485e8 1784 md->numa_node_id = numa_node_id;
591ddcfc 1785 md->init_tio_pdu = false;
a5664dad 1786 md->type = DM_TYPE_NONE;
e61290a4 1787 mutex_init(&md->suspend_lock);
a5664dad 1788 mutex_init(&md->type_lock);
86f1152b 1789 mutex_init(&md->table_devices_lock);
022c2611 1790 spin_lock_init(&md->deferred_lock);
1da177e4 1791 atomic_set(&md->holders, 1);
5c6bd75d 1792 atomic_set(&md->open_count, 0);
1da177e4 1793 atomic_set(&md->event_nr, 0);
7a8c3d3b
MA		 1794 atomic_set(&md->uevent_seq, 0);
1795 INIT_LIST_HEAD(&md->uevent_list);
86f1152b 1796 INIT_LIST_HEAD(&md->table_devices);
7a8c3d3b 1797 spin_lock_init(&md->uevent_lock);
1da177e4 1798
47ace7e0 1799 /*
c62b37d9
CH		 1800 * default to bio-based until DM table is loaded and md->type
1801 * established. If request-based table is loaded: blk-mq will
1802 * override accordingly.
47ace7e0 1803 */
c62b37d9 1804 md->queue = blk_alloc_queue(numa_node_id);
3d745ea5
CH		 1805 if (!md->queue)
1806 goto bad;
1da177e4 1807
c12c9a3c 1808 md->disk = alloc_disk_node(1, md->numa_node_id);
1da177e4 1809 if (!md->disk)
0f20972f 1810 goto bad;
1da177e4 1811
f0b04115 1812 init_waitqueue_head(&md->wait);
53d5914f 1813 INIT_WORK(&md->work, dm_wq_work);
f0b04115 1814 init_waitqueue_head(&md->eventq);
2995fa78 1815 init_completion(&md->kobj_holder.completion);
f0b04115 1816
1da177e4
LT		 1817 md->disk->major = _major;
1818 md->disk->first_minor = minor;
1819 md->disk->fops = &dm_blk_dops;
1820 md->disk->queue = md->queue;
1821 md->disk->private_data = md;
1822 sprintf(md->disk->disk_name, "dm-%d", minor);
f26c5719 1823
976431b0 1824 if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
fefc1d97
PG		 1825 md->dax_dev = alloc_dax(md, md->disk->disk_name,
1826 &dm_dax_ops, 0);
4e4ced93 1827 if (IS_ERR(md->dax_dev))
976431b0
DW		 1828 goto bad;
1829 }
f26c5719 1830
c100ec49 1831 add_disk_no_queue_reg(md->disk);
7e51f257 1832 format_dev_t(md->name, MKDEV(_major, minor));
1da177e4 1833
670368a8 1834 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
304f3f6a 1835 if (!md->wq)
0f20972f 1836 goto bad;
304f3f6a 1837
fd2ed4d2
MP		 1838 dm_stats_init(&md->stats);
1839
ba61fdd1 1840 /* Populate the mapping, nobody knows we exist yet */
f32c10b0 1841 spin_lock(&_minor_lock);
ba61fdd1 1842 old_md = idr_replace(&_minor_idr, md, minor);
f32c10b0 1843 spin_unlock(&_minor_lock);
ba61fdd1
JM		 1844
1845 BUG_ON(old_md != MINOR_ALLOCED);
1846
1da177e4
LT		 1847 return md;
1848
0f20972f
MS		 1849bad:
1850 cleanup_mapped_device(md);
83d5e5b0 1851bad_io_barrier:
1da177e4 1852 free_minor(minor);
6ed7ade8 1853bad_minor:
10da4f79 1854 module_put(THIS_MODULE);
6ed7ade8 1855bad_module_get:
856eb091 1856 kvfree(md);
1da177e4
LT		 1857 return NULL;
1858}
1859
ae9da83f
JN		 1860static void unlock_fs(struct mapped_device *md);
1861
1da177e4
LT		 1862static void free_dev(struct mapped_device *md)
1863{
f331c029 1864 int minor = MINOR(disk_devt(md->disk));
63d94e48 1865
32a926da 1866 unlock_fs(md);
2eb6e1e3 1867
0f20972f 1868 cleanup_mapped_device(md);
63a4f065 1869
86f1152b 1870 free_table_devices(&md->table_devices);
63a4f065 1871 dm_stats_cleanup(&md->stats);
63a4f065
MS		 1872 free_minor(minor);
1873
10da4f79 1874 module_put(THIS_MODULE);
856eb091 1875 kvfree(md);
1da177e4
LT		 1876}
1877
2a2a4c51 1878static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
e6ee8c0b 1879{
c0820cf5 1880 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
2a2a4c51 1881 int ret = 0;
e6ee8c0b 1882
0776aa0e 1883 if (dm_table_bio_based(t)) {
64f52b0e
MS		 1884 /*
1885 * The md may already have mempools that need changing.
1886 * If so, reload bioset because front_pad may have changed
1887 * because a different table was loaded.
1888 */
6f1c819c
KO		 1889 bioset_exit(&md->bs);
1890 bioset_exit(&md->io_bs);
0776aa0e 1891
6f1c819c 1892 } else if (bioset_initialized(&md->bs)) {
4e6e36c3
MS		 1893 /*
1894 * There's no need to reload with request-based dm
1895 * because the size of front_pad doesn't change.
1896 * Note for future: If you are to reload bioset,
1897 * prep-ed requests in the queue may refer
1898 * to bio from the old bioset, so you must walk
1899 * through the queue to unprep.
1900 */
1901 goto out;
c0820cf5 1902 }
e6ee8c0b 1903
6f1c819c
KO		 1904 BUG_ON(!p ||
1905 bioset_initialized(&md->bs) ||
1906 bioset_initialized(&md->io_bs));
cbc4e3c1 1907
2a2a4c51
JA		 1908 ret = bioset_init_from_src(&md->bs, &p->bs);
1909 if (ret)
1910 goto out;
1911 ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
1912 if (ret)
1913 bioset_exit(&md->bs);
e6ee8c0b 1914out:
02233342 1915 /* mempool bind completed, no longer need any mempools in the table */
e6ee8c0b 1916 dm_table_free_md_mempools(t);
2a2a4c51 1917 return ret;
e6ee8c0b
KU		 1918}
1919
1da177e4
LT		 1920/*
1921 * Bind a table to the device.
1922 */
1923static void event_callback(void *context)
1924{
7a8c3d3b
MA		 1925 unsigned long flags;
1926 LIST_HEAD(uevents);
1da177e4
LT		 1927 struct mapped_device *md = (struct mapped_device *) context;
1928
7a8c3d3b
MA		 1929 spin_lock_irqsave(&md->uevent_lock, flags);
1930 list_splice_init(&md->uevent_list, &uevents);
1931 spin_unlock_irqrestore(&md->uevent_lock, flags);
1932
ed9e1982 1933 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
7a8c3d3b 1934
1da177e4
LT		 1935 atomic_inc(&md->event_nr);
1936 wake_up(&md->eventq);
62e08243 1937 dm_issue_global_event();
1da177e4
LT		 1938}
1939
042d2a9b
AK		 1940/*
1941 * Returns old map, which caller must destroy.
1942 */
1943static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1944 struct queue_limits *limits)
1da177e4 1945{
042d2a9b 1946 struct dm_table *old_map;
165125e1 1947 struct request_queue *q = md->queue;
978e51ba 1948 bool request_based = dm_table_request_based(t);
1da177e4 1949 sector_t size;
2a2a4c51 1950 int ret;
1da177e4 1951
5a8f1f80
BVA		 1952 lockdep_assert_held(&md->suspend_lock);
1953
1da177e4 1954 size = dm_table_get_size(t);
3ac51e74
DW		 1955
1956 /*
1957 * Wipe any geometry if the size of the table changed.
1958 */
fd2ed4d2 1959 if (size != dm_get_size(md))
3ac51e74
DW		 1960 memset(&md->geometry, 0, sizeof(md->geometry));
1961
f64d9b2e 1962 set_capacity_and_notify(md->disk, size);
d5816876 1963
2ca3310e
AK		 1964 dm_table_event_callback(t, event_callback, md);
1965
e6ee8c0b
KU		 1966 /*
1967 * The queue hasn't been stopped yet, if the old table type wasn't
1968 * for request-based during suspension. So stop it to prevent
1969 * I/O mapping before resume.
1970 * This must be done before setting the queue restrictions,
1971 * because request-based dm may be run just after the setting.
1972 */
978e51ba 1973 if (request_based)
eca7ee6d 1974 dm_stop_queue(q);
978e51ba 1975
9c37de29 1976 if (request_based) {
16f12266 1977 /*
9c37de29
MS		 1978 * Leverage the fact that request-based DM targets are
1979 * immutable singletons - used to optimize dm_mq_queue_rq.
16f12266
MS		 1980 */
1981 md->immutable_target = dm_table_get_immutable_target(t);
1982 }
e6ee8c0b 1983
2a2a4c51
JA		 1984 ret = __bind_mempools(md, t);
1985 if (ret) {
1986 old_map = ERR_PTR(ret);
1987 goto out;
1988 }
e6ee8c0b 1989
a12f5d48 1990 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1d3aa6f6 1991 rcu_assign_pointer(md->map, (void *)t);
36a0456f
AK		 1992 md->immutable_target_type = dm_table_get_immutable_target_type(t);
1993
754c5fc7 1994 dm_table_set_restrictions(t, q, limits);
41abc4e1
HR		 1995 if (old_map)
1996 dm_sync_table(md);
1da177e4 1997
2a2a4c51 1998out:
042d2a9b 1999 return old_map;
1da177e4
LT		 2000}
2001
a7940155
AK		 2002/*
2003 * Returns unbound table for the caller to free.
2004 */
2005static struct dm_table *__unbind(struct mapped_device *md)
1da177e4 2006{
a12f5d48 2007 struct dm_table *map = rcu_dereference_protected(md->map, 1);
1da177e4
LT		 2008
2009 if (!map)
a7940155 2010 return NULL;
1da177e4
LT		 2011
2012 dm_table_event_callback(map, NULL, NULL);
9cdb8520 2013 RCU_INIT_POINTER(md->map, NULL);
83d5e5b0 2014 dm_sync_table(md);
a7940155
AK		 2015
2016 return map;
1da177e4
LT		 2017}
2018
2019/*
2020 * Constructor for a new device.
2021 */
2b06cfff 2022int dm_create(int minor, struct mapped_device **result)
1da177e4 2023{
c12c9a3c 2024 int r;
1da177e4
LT		 2025 struct mapped_device *md;
2026
2b06cfff 2027 md = alloc_dev(minor);
1da177e4
LT		 2028 if (!md)
2029 return -ENXIO;
2030
c12c9a3c
MS		 2031 r = dm_sysfs_init(md);
2032 if (r) {
2033 free_dev(md);
2034 return r;
2035 }
784aae73 2036
1da177e4
LT		 2037 *result = md;
2038 return 0;
2039}
2040
a5664dad
MS		 2041/*
2042 * Functions to manage md->type.
2043 * All are required to hold md->type_lock.
2044 */
2045void dm_lock_md_type(struct mapped_device *md)
2046{
2047 mutex_lock(&md->type_lock);
2048}
2049
2050void dm_unlock_md_type(struct mapped_device *md)
2051{
2052 mutex_unlock(&md->type_lock);
2053}
2054
7e0d574f 2055void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
a5664dad 2056{
00c4fc3b 2057 BUG_ON(!mutex_is_locked(&md->type_lock));
a5664dad
MS		 2058 md->type = type;
2059}
2060
7e0d574f 2061enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
a5664dad
MS		 2062{
2063 return md->type;
2064}
2065
36a0456f
AK		 2066struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2067{
2068 return md->immutable_target_type;
2069}
2070
f84cb8a4
MS		 2071/*
2072 * The queue_limits are only valid as long as you have a reference
2073 * count on 'md'.
2074 */
2075struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2076{
2077 BUG_ON(!atomic_read(&md->holders));
2078 return &md->queue->limits;
2079}
2080EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2081
4a0b4ddf
MS		 2082/*
2083 * Setup the DM device's queue based on md's type
2084 */
591ddcfc 2085int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
4a0b4ddf 2086{
bfebd1cd 2087 int r;
c100ec49 2088 struct queue_limits limits;
7e0d574f 2089 enum dm_queue_mode type = dm_get_md_type(md);
bfebd1cd 2090
545ed20e 2091 switch (type) {
bfebd1cd 2092 case DM_TYPE_REQUEST_BASED:
681cc5e8 2093 md->disk->fops = &dm_rq_blk_dops;
e83068a5 2094 r = dm_mq_init_request_queue(md, t);
bfebd1cd 2095 if (r) {
681cc5e8 2096 DMERR("Cannot initialize queue for request-based dm mapped device");
bfebd1cd
MS		 2097 return r;
2098 }
2099 break;
2100 case DM_TYPE_BIO_BASED:
545ed20e 2101 case DM_TYPE_DAX_BIO_BASED:
bfebd1cd 2102 break;
7e0d574f
BVA		 2103 case DM_TYPE_NONE:
2104 WARN_ON_ONCE(true);
2105 break;
4a0b4ddf
MS		 2106 }
2107
c100ec49
MS		 2108 r = dm_calculate_queue_limits(t, &limits);
2109 if (r) {
2110 DMERR("Cannot calculate initial queue limits");
2111 return r;
2112 }
2113 dm_table_set_restrictions(t, md->queue, &limits);
2114 blk_register_queue(md->disk);
2115
4a0b4ddf
MS		 2116 return 0;
2117}
2118
2bec1f4a 2119struct mapped_device *dm_get_md(dev_t dev)
1da177e4
LT		 2120{
2121 struct mapped_device *md;
1da177e4
LT		 2122 unsigned minor = MINOR(dev);
2123
2124 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2125 return NULL;
2126
f32c10b0 2127 spin_lock(&_minor_lock);
1da177e4
LT		 2128
2129 md = idr_find(&_minor_idr, minor);
49de5769
MS		 2130 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2131 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2132 md = NULL;
2133 goto out;
fba9f90e 2134 }
49de5769 2135 dm_get(md);
fba9f90e 2136out:
f32c10b0 2137 spin_unlock(&_minor_lock);
1da177e4 2138
637842cf
DT		 2139 return md;
2140}
3cf2e4ba 2141EXPORT_SYMBOL_GPL(dm_get_md);
d229a958 2142
9ade92a9 2143void *dm_get_mdptr(struct mapped_device *md)
637842cf 2144{
9ade92a9 2145 return md->interface_ptr;
1da177e4
LT		 2146}
2147
2148void dm_set_mdptr(struct mapped_device *md, void *ptr)
2149{
2150 md->interface_ptr = ptr;
2151}
2152
2153void dm_get(struct mapped_device *md)
2154{
2155 atomic_inc(&md->holders);
3f77316d 2156 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1da177e4
LT		 2157}
2158
09ee96b2
MP		 2159int dm_hold(struct mapped_device *md)
2160{
2161 spin_lock(&_minor_lock);
2162 if (test_bit(DMF_FREEING, &md->flags)) {
2163 spin_unlock(&_minor_lock);
2164 return -EBUSY;
2165 }
2166 dm_get(md);
2167 spin_unlock(&_minor_lock);
2168 return 0;
2169}
2170EXPORT_SYMBOL_GPL(dm_hold);
2171
72d94861
AK		 2172const char *dm_device_name(struct mapped_device *md)
2173{
2174 return md->name;
2175}
2176EXPORT_SYMBOL_GPL(dm_device_name);
2177
3f77316d 2178static void __dm_destroy(struct mapped_device *md, bool wait)
1da177e4 2179{
1134e5ae 2180 struct dm_table *map;
83d5e5b0 2181 int srcu_idx;
1da177e4 2182
3f77316d 2183 might_sleep();
fba9f90e 2184
63a4f065 2185 spin_lock(&_minor_lock);
3f77316d
KU		 2186 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2187 set_bit(DMF_FREEING, &md->flags);
2188 spin_unlock(&_minor_lock);
3b785fbc 2189
c12c9a3c 2190 blk_set_queue_dying(md->queue);
3f77316d 2191
ab7c7bb6
MP		 2192 /*
2193 * Take suspend_lock so that presuspend and postsuspend methods
2194 * do not race with internal suspend.
2195 */
2196 mutex_lock(&md->suspend_lock);
2a708cff 2197 map = dm_get_live_table(md, &srcu_idx);
3f77316d
KU		 2198 if (!dm_suspended_md(md)) {
2199 dm_table_presuspend_targets(map);
adc0daad 2200 set_bit(DMF_SUSPENDED, &md->flags);
5df96f2b 2201 set_bit(DMF_POST_SUSPENDING, &md->flags);
3f77316d 2202 dm_table_postsuspend_targets(map);
1da177e4 2203 }
83d5e5b0
MP		 2204 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2205 dm_put_live_table(md, srcu_idx);
2a708cff 2206 mutex_unlock(&md->suspend_lock);
83d5e5b0 2207
3f77316d
KU		 2208 /*
2209 * Rare, but there may be I/O requests still going to complete,
2210 * for example. Wait for all references to disappear.
2211 * No one should increment the reference count of the mapped_device,
2212 * after the mapped_device state becomes DMF_FREEING.
2213 */
2214 if (wait)
2215 while (atomic_read(&md->holders))
2216 msleep(1);
2217 else if (atomic_read(&md->holders))
2218 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2219 dm_device_name(md), atomic_read(&md->holders));
2220
2221 dm_sysfs_exit(md);
3f77316d
KU		 2222 dm_table_destroy(__unbind(md));
2223 free_dev(md);
2224}
2225
2226void dm_destroy(struct mapped_device *md)
2227{
2228 __dm_destroy(md, true);
2229}
2230
2231void dm_destroy_immediate(struct mapped_device *md)
2232{
2233 __dm_destroy(md, false);
2234}
2235
2236void dm_put(struct mapped_device *md)
2237{
2238 atomic_dec(&md->holders);
1da177e4 2239}
79eb885c 2240EXPORT_SYMBOL_GPL(dm_put);
1da177e4 2241
85067747
ML		 2242static bool md_in_flight_bios(struct mapped_device *md)
2243{
2244 int cpu;
cb8432d6 2245 struct hd_struct *part = dm_disk(md)->part0->bd_part;
85067747
ML		 2246 long sum = 0;
2247
2248 for_each_possible_cpu(cpu) {
2249 sum += part_stat_local_read_cpu(part, in_flight[0], cpu);
2250 sum += part_stat_local_read_cpu(part, in_flight[1], cpu);
2251 }
2252
2253 return sum != 0;
2254}
2255
2256static int dm_wait_for_bios_completion(struct mapped_device *md, long task_state)
46125c1c
MB		 2257{
2258 int r = 0;
9f4c3f87 2259 DEFINE_WAIT(wait);
46125c1c 2260
85067747 2261 while (true) {
9f4c3f87 2262 prepare_to_wait(&md->wait, &wait, task_state);
46125c1c 2263
85067747 2264 if (!md_in_flight_bios(md))
46125c1c
MB		 2265 break;
2266
e3fabdfd 2267 if (signal_pending_state(task_state, current)) {
46125c1c
MB		 2268 r = -EINTR;
2269 break;
2270 }
2271
2272 io_schedule();
2273 }
9f4c3f87 2274 finish_wait(&md->wait, &wait);
b44ebeb0 2275
46125c1c
MB		 2276 return r;
2277}
2278
85067747
ML		 2279static int dm_wait_for_completion(struct mapped_device *md, long task_state)
2280{
2281 int r = 0;
2282
2283 if (!queue_is_mq(md->queue))
2284 return dm_wait_for_bios_completion(md, task_state);
2285
2286 while (true) {
2287 if (!blk_mq_queue_inflight(md->queue))
2288 break;
2289
2290 if (signal_pending_state(task_state, current)) {
2291 r = -EINTR;
2292 break;
2293 }
2294
2295 msleep(5);
2296 }
2297
2298 return r;
2299}
2300
1da177e4
LT		 2301/*
2302 * Process the deferred bios
2303 */
ef208587 2304static void dm_wq_work(struct work_struct *work)
1da177e4 2305{
0c2915b8
MS		 2306 struct mapped_device *md = container_of(work, struct mapped_device, work);
2307 struct bio *bio;
ef208587 2308
3b00b203 2309 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
df12ee99 2310 spin_lock_irq(&md->deferred_lock);
0c2915b8 2311 bio = bio_list_pop(&md->deferred);
df12ee99
AK		 2312 spin_unlock_irq(&md->deferred_lock);
2313
0c2915b8 2314 if (!bio)
df12ee99 2315 break;
022c2611 2316
0c2915b8 2317 submit_bio_noacct(bio);
022c2611 2318 }
1da177e4
LT		 2319}
2320
9a1fb464 2321static void dm_queue_flush(struct mapped_device *md)
304f3f6a 2322{
3b00b203 2323 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
4e857c58 2324 smp_mb__after_atomic();
53d5914f 2325 queue_work(md->wq, &md->work);
304f3f6a
MB		 2326}
2327
1da177e4 2328/*
042d2a9b 2329 * Swap in a new table, returning the old one for the caller to destroy.
1da177e4 2330 */
042d2a9b 2331struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
1da177e4 2332{
87eb5b21 2333 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
754c5fc7 2334 struct queue_limits limits;
042d2a9b 2335 int r;
1da177e4 2336
e61290a4 2337 mutex_lock(&md->suspend_lock);
1da177e4
LT		 2338
2339 /* device must be suspended */
4f186f8b 2340 if (!dm_suspended_md(md))
93c534ae 2341 goto out;
1da177e4 2342
3ae70656
MS		 2343 /*
2344 * If the new table has no data devices, retain the existing limits.
2345 * This helps multipath with queue_if_no_path if all paths disappear,
2346 * then new I/O is queued based on these limits, and then some paths
2347 * reappear.
2348 */
2349 if (dm_table_has_no_data_devices(table)) {
83d5e5b0 2350 live_map = dm_get_live_table_fast(md);
3ae70656
MS		 2351 if (live_map)
2352 limits = md->queue->limits;
83d5e5b0 2353 dm_put_live_table_fast(md);
3ae70656
MS		 2354 }
2355
87eb5b21
MC		 2356 if (!live_map) {
2357 r = dm_calculate_queue_limits(table, &limits);
2358 if (r) {
2359 map = ERR_PTR(r);
2360 goto out;
2361 }
042d2a9b 2362 }
754c5fc7 2363
042d2a9b 2364 map = __bind(md, table, &limits);
62e08243 2365 dm_issue_global_event();
1da177e4 2366
93c534ae 2367out:
e61290a4 2368 mutex_unlock(&md->suspend_lock);
042d2a9b 2369 return map;
1da177e4
LT		 2370}
2371
2372/*
2373 * Functions to lock and unlock any filesystem running on the
2374 * device.
2375 */
2ca3310e 2376static int lock_fs(struct mapped_device *md)
1da177e4 2377{
b0519b54 2378 struct block_device *bdev;
e39e2e95 2379 int r;
1da177e4 2380
040f04bd 2381 WARN_ON(test_bit(DMF_FROZEN, &md->flags));
dfbe03f6 2382
b0519b54
CH		 2383 bdev = bdget_disk(md->disk, 0);
2384 if (!bdev)
2385 return -ENOMEM;
2386 r = freeze_bdev(bdev);
2387 bdput(bdev);
040f04bd
CH		 2388 if (!r)
2389 set_bit(DMF_FROZEN, &md->flags);
2390 return r;
1da177e4
LT		 2391}
2392
2ca3310e 2393static void unlock_fs(struct mapped_device *md)
1da177e4 2394{
b0519b54
CH		 2395 struct block_device *bdev;
2396
aa8d7c2f
AK		 2397 if (!test_bit(DMF_FROZEN, &md->flags))
2398 return;
b0519b54
CH		 2399
2400 bdev = bdget_disk(md->disk, 0);
2401 if (!bdev)
2402 return;
2403 thaw_bdev(bdev);
2404 bdput(bdev);
aa8d7c2f 2405 clear_bit(DMF_FROZEN, &md->flags);
1da177e4
LT		 2406}
2407
2408/*
b48633f8
BVA		 2409 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2410 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2411 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2412 *
ffcc3936
MS		 2413 * If __dm_suspend returns 0, the device is completely quiescent
2414 * now. There is no request-processing activity. All new requests
2415 * are being added to md->deferred list.
cec47e3d 2416 */
ffcc3936 2417static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
b48633f8 2418 unsigned suspend_flags, long task_state,
eaf9a736 2419 int dmf_suspended_flag)
1da177e4 2420{
ffcc3936
MS		 2421 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2422 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2423 int r;
1da177e4 2424
5a8f1f80
BVA		 2425 lockdep_assert_held(&md->suspend_lock);
2426
2e93ccc1
KU		 2427 /*
2428 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2429 * This flag is cleared before dm_suspend returns.
2430 */
2431 if (noflush)
2432 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
86331f39 2433 else
ac75b09f 2434 DMDEBUG("%s: suspending with flush", dm_device_name(md));
2e93ccc1 2435
d67ee213
MS		 2436 /*
2437 * This gets reverted if there's an error later and the targets
2438 * provide the .presuspend_undo hook.
2439 */
cf222b37
AK		 2440 dm_table_presuspend_targets(map);
2441
32a926da 2442 /*
9f518b27
KU		 2443 * Flush I/O to the device.
2444 * Any I/O submitted after lock_fs() may not be flushed.
2445 * noflush takes precedence over do_lockfs.
2446 * (lock_fs() flushes I/Os and waits for them to complete.)
32a926da
MP		 2447 */
2448 if (!noflush && do_lockfs) {
2449 r = lock_fs(md);
d67ee213
MS		 2450 if (r) {
2451 dm_table_presuspend_undo_targets(map);
ffcc3936 2452 return r;
d67ee213 2453 }
aa8d7c2f 2454 }
1da177e4
LT		 2455
2456 /*
3b00b203
MP		 2457 * Here we must make sure that no processes are submitting requests
2458 * to target drivers i.e. no one may be executing
0cede372 2459 * __split_and_process_bio from dm_submit_bio.
3b00b203 2460 *
0cede372 2461 * To get all processes out of __split_and_process_bio in dm_submit_bio,
3b00b203 2462 * we take the write lock. To prevent any process from reentering
0cede372
MS		 2463 * __split_and_process_bio from dm_submit_bio and quiesce the thread
2464 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
6a8736d1 2465 * flush_workqueue(md->wq).
1da177e4 2466 */
1eb787ec 2467 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
41abc4e1
HR		 2468 if (map)
2469 synchronize_srcu(&md->io_barrier);
1da177e4 2470
d0bcb878 2471 /*
29e4013d
TH		 2472 * Stop md->queue before flushing md->wq in case request-based
2473 * dm defers requests to md->wq from md->queue.
d0bcb878 2474 */
6a23e05c 2475 if (dm_request_based(md))
eca7ee6d 2476 dm_stop_queue(md->queue);
cec47e3d 2477
d0bcb878
KU		 2478 flush_workqueue(md->wq);
2479
1da177e4 2480 /*
3b00b203
MP		 2481 * At this point no more requests are entering target request routines.
2482 * We call dm_wait_for_completion to wait for all existing requests
2483 * to finish.
1da177e4 2484 */
b48633f8 2485 r = dm_wait_for_completion(md, task_state);
eaf9a736
MS		 2486 if (!r)
2487 set_bit(dmf_suspended_flag, &md->flags);
1da177e4 2488
6d6f10df 2489 if (noflush)
022c2611 2490 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
41abc4e1
HR		 2491 if (map)
2492 synchronize_srcu(&md->io_barrier);
2e93ccc1 2493
1da177e4 2494 /* were we interrupted ? */
46125c1c 2495 if (r < 0) {
9a1fb464 2496 dm_queue_flush(md);
73d410c0 2497
cec47e3d 2498 if (dm_request_based(md))
eca7ee6d 2499 dm_start_queue(md->queue);
cec47e3d 2500
2ca3310e 2501 unlock_fs(md);
d67ee213 2502 dm_table_presuspend_undo_targets(map);
ffcc3936 2503 /* pushback list is already flushed, so skip flush */
2ca3310e 2504 }
1da177e4 2505
ffcc3936
MS		 2506 return r;
2507}
2508
2509/*
2510 * We need to be able to change a mapping table under a mounted
2511 * filesystem. For example we might want to move some data in
2512 * the background. Before the table can be swapped with
2513 * dm_bind_table, dm_suspend must be called to flush any in
2514 * flight bios and ensure that any further io gets deferred.
2515 */
2516/*
2517 * Suspend mechanism in request-based dm.
2518 *
2519 * 1. Flush all I/Os by lock_fs() if needed.
2520 * 2. Stop dispatching any I/O by stopping the request_queue.
2521 * 3. Wait for all in-flight I/Os to be completed or requeued.
2522 *
2523 * To abort suspend, start the request_queue.
2524 */
2525int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2526{
2527 struct dm_table *map = NULL;
2528 int r = 0;
2529
2530retry:
2531 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2532
2533 if (dm_suspended_md(md)) {
2534 r = -EINVAL;
2535 goto out_unlock;
2536 }
2537
2538 if (dm_suspended_internally_md(md)) {
2539 /* already internally suspended, wait for internal resume */
2540 mutex_unlock(&md->suspend_lock);
2541 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2542 if (r)
2543 return r;
2544 goto retry;
2545 }
2546
a12f5d48 2547 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
ffcc3936 2548
eaf9a736 2549 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
ffcc3936
MS		 2550 if (r)
2551 goto out_unlock;
3b00b203 2552
5df96f2b 2553 set_bit(DMF_POST_SUSPENDING, &md->flags);
4d4471cb 2554 dm_table_postsuspend_targets(map);
5df96f2b 2555 clear_bit(DMF_POST_SUSPENDING, &md->flags);
4d4471cb 2556
d287483d 2557out_unlock:
e61290a4 2558 mutex_unlock(&md->suspend_lock);
cf222b37 2559 return r;
1da177e4
LT		 2560}
2561
ffcc3936
MS		 2562static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2563{
2564 if (map) {
2565 int r = dm_table_resume_targets(map);
2566 if (r)
2567 return r;
2568 }
2569
2570 dm_queue_flush(md);
2571
2572 /*
2573 * Flushing deferred I/Os must be done after targets are resumed
2574 * so that mapping of targets can work correctly.
2575 * Request-based dm is queueing the deferred I/Os in its request_queue.
2576 */
2577 if (dm_request_based(md))
eca7ee6d 2578 dm_start_queue(md->queue);
ffcc3936
MS		 2579
2580 unlock_fs(md);
2581
2582 return 0;
2583}
2584
1da177e4
LT		 2585int dm_resume(struct mapped_device *md)
2586{
8dc23658 2587 int r;
cf222b37 2588 struct dm_table *map = NULL;
1da177e4 2589
ffcc3936 2590retry:
8dc23658 2591 r = -EINVAL;
ffcc3936
MS		 2592 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2593
4f186f8b 2594 if (!dm_suspended_md(md))
cf222b37 2595 goto out;
cf222b37 2596
ffcc3936
MS		 2597 if (dm_suspended_internally_md(md)) {
2598 /* already internally suspended, wait for internal resume */
2599 mutex_unlock(&md->suspend_lock);
2600 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2601 if (r)
2602 return r;
2603 goto retry;
2604 }
2605
a12f5d48 2606 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2ca3310e 2607 if (!map || !dm_table_get_size(map))
cf222b37 2608 goto out;
1da177e4 2609
ffcc3936 2610 r = __dm_resume(md, map);
8757b776
MB		 2611 if (r)
2612 goto out;
2ca3310e 2613
2ca3310e 2614 clear_bit(DMF_SUSPENDED, &md->flags);
cf222b37 2615out:
e61290a4 2616 mutex_unlock(&md->suspend_lock);
2ca3310e 2617
cf222b37 2618 return r;
1da177e4
LT		 2619}
2620
fd2ed4d2
MP		 2621/*
2622 * Internal suspend/resume works like userspace-driven suspend. It waits
2623 * until all bios finish and prevents issuing new bios to the target drivers.
2624 * It may be used only from the kernel.
fd2ed4d2
MP		 2625 */
2626
ffcc3936 2627static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
fd2ed4d2 2628{
ffcc3936
MS		 2629 struct dm_table *map = NULL;
2630
1ea0654e
BVA		 2631 lockdep_assert_held(&md->suspend_lock);
2632
96b26c8c 2633 if (md->internal_suspend_count++)
ffcc3936
MS		 2634 return; /* nested internal suspend */
2635
2636 if (dm_suspended_md(md)) {
2637 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2638 return; /* nest suspend */
2639 }
2640
a12f5d48 2641 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
ffcc3936
MS		 2642
2643 /*
2644 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2645 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2646 * would require changing .presuspend to return an error -- avoid this
2647 * until there is a need for more elaborate variants of internal suspend.
2648 */
eaf9a736
MS		 2649 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2650 DMF_SUSPENDED_INTERNALLY);
ffcc3936 2651
5df96f2b 2652 set_bit(DMF_POST_SUSPENDING, &md->flags);
ffcc3936 2653 dm_table_postsuspend_targets(map);
5df96f2b 2654 clear_bit(DMF_POST_SUSPENDING, &md->flags);
ffcc3936
MS		 2655}
2656
2657static void __dm_internal_resume(struct mapped_device *md)
2658{
96b26c8c
MP		 2659 BUG_ON(!md->internal_suspend_count);
2660
2661 if (--md->internal_suspend_count)
ffcc3936
MS		 2662 return; /* resume from nested internal suspend */
2663
fd2ed4d2 2664 if (dm_suspended_md(md))
ffcc3936
MS		 2665 goto done; /* resume from nested suspend */
2666
2667 /*
2668 * NOTE: existing callers don't need to call dm_table_resume_targets
2669 * (which may fail -- so best to avoid it for now by passing NULL map)
2670 */
2671 (void) __dm_resume(md, NULL);
2672
2673done:
2674 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2675 smp_mb__after_atomic();
2676 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2677}
2678
2679void dm_internal_suspend_noflush(struct mapped_device *md)
2680{
2681 mutex_lock(&md->suspend_lock);
2682 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2683 mutex_unlock(&md->suspend_lock);
2684}
2685EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2686
2687void dm_internal_resume(struct mapped_device *md)
2688{
2689 mutex_lock(&md->suspend_lock);
2690 __dm_internal_resume(md);
2691 mutex_unlock(&md->suspend_lock);
2692}
2693EXPORT_SYMBOL_GPL(dm_internal_resume);
2694
2695/*
2696 * Fast variants of internal suspend/resume hold md->suspend_lock,
2697 * which prevents interaction with userspace-driven suspend.
2698 */
2699
2700void dm_internal_suspend_fast(struct mapped_device *md)
2701{
2702 mutex_lock(&md->suspend_lock);
2703 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
fd2ed4d2
MP		 2704 return;
2705
2706 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2707 synchronize_srcu(&md->io_barrier);
2708 flush_workqueue(md->wq);
2709 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2710}
b735fede 2711EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
fd2ed4d2 2712
ffcc3936 2713void dm_internal_resume_fast(struct mapped_device *md)
fd2ed4d2 2714{
ffcc3936 2715 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
fd2ed4d2
MP		 2716 goto done;
2717
2718 dm_queue_flush(md);
2719
2720done:
2721 mutex_unlock(&md->suspend_lock);
2722}
b735fede 2723EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
fd2ed4d2 2724
1da177e4
LT		 2725/*-----------------------------------------------------------------
2726 * Event notification.
2727 *---------------------------------------------------------------*/
3abf85b5 2728int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
60935eb2 2729 unsigned cookie)
69267a30 2730{
6958c1c6
MP		 2731 int r;
2732 unsigned noio_flag;
60935eb2
MB		 2733 char udev_cookie[DM_COOKIE_LENGTH];
2734 char *envp[] = { udev_cookie, NULL };
2735
6958c1c6
MP		 2736 noio_flag = memalloc_noio_save();
2737
60935eb2 2738 if (!cookie)
6958c1c6 2739 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
60935eb2
MB		 2740 else {
2741 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2742 DM_COOKIE_ENV_VAR_NAME, cookie);
6958c1c6
MP		 2743 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2744 action, envp);
60935eb2 2745 }
6958c1c6
MP		 2746
2747 memalloc_noio_restore(noio_flag);
2748
2749 return r;
69267a30
AK		 2750}
2751
7a8c3d3b
MA		 2752uint32_t dm_next_uevent_seq(struct mapped_device *md)
2753{
2754 return atomic_add_return(1, &md->uevent_seq);
2755}
2756
1da177e4
LT		 2757uint32_t dm_get_event_nr(struct mapped_device *md)
2758{
2759 return atomic_read(&md->event_nr);
2760}
2761
2762int dm_wait_event(struct mapped_device *md, int event_nr)
2763{
2764 return wait_event_interruptible(md->eventq,
2765 (event_nr != atomic_read(&md->event_nr)));
2766}
2767
7a8c3d3b
MA		 2768void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2769{
2770 unsigned long flags;
2771
2772 spin_lock_irqsave(&md->uevent_lock, flags);
2773 list_add(elist, &md->uevent_list);
2774 spin_unlock_irqrestore(&md->uevent_lock, flags);
2775}
2776
1da177e4
LT		 2777/*
2778 * The gendisk is only valid as long as you have a reference
2779 * count on 'md'.
2780 */
2781struct gendisk *dm_disk(struct mapped_device *md)
2782{
2783 return md->disk;
2784}
65ff5b7d 2785EXPORT_SYMBOL_GPL(dm_disk);
1da177e4 2786
784aae73
MB		 2787struct kobject *dm_kobject(struct mapped_device *md)
2788{
2995fa78 2789 return &md->kobj_holder.kobj;
784aae73
MB		 2790}
2791
784aae73
MB		 2792struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2793{
2794 struct mapped_device *md;
2795
2995fa78 2796 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
784aae73 2797
b9a41d21
HT		 2798 spin_lock(&_minor_lock);
2799 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2800 md = NULL;
2801 goto out;
2802 }
784aae73 2803 dm_get(md);
b9a41d21
HT		 2804out:
2805 spin_unlock(&_minor_lock);
2806
784aae73
MB		 2807 return md;
2808}
2809
4f186f8b 2810int dm_suspended_md(struct mapped_device *md)
1da177e4
LT		 2811{
2812 return test_bit(DMF_SUSPENDED, &md->flags);
2813}
2814
5df96f2b
MP		 2815static int dm_post_suspending_md(struct mapped_device *md)
2816{
2817 return test_bit(DMF_POST_SUSPENDING, &md->flags);
2818}
2819
ffcc3936
MS		 2820int dm_suspended_internally_md(struct mapped_device *md)
2821{
2822 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2823}
2824
2c140a24
MP		 2825int dm_test_deferred_remove_flag(struct mapped_device *md)
2826{
2827 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2828}
2829
64dbce58
KU		 2830int dm_suspended(struct dm_target *ti)
2831{
33bd6f06 2832 return dm_suspended_md(ti->table->md);
64dbce58
KU		 2833}
2834EXPORT_SYMBOL_GPL(dm_suspended);
2835
5df96f2b
MP		 2836int dm_post_suspending(struct dm_target *ti)
2837{
33bd6f06 2838 return dm_post_suspending_md(ti->table->md);
5df96f2b
MP		 2839}
2840EXPORT_SYMBOL_GPL(dm_post_suspending);
2841
2e93ccc1
KU		 2842int dm_noflush_suspending(struct dm_target *ti)
2843{
33bd6f06 2844 return __noflush_suspending(ti->table->md);
2e93ccc1
KU		 2845}
2846EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2847
7e0d574f 2848struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
0776aa0e
MS		 2849 unsigned integrity, unsigned per_io_data_size,
2850 unsigned min_pool_size)
e6ee8c0b 2851{
115485e8 2852 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
78d8e58a 2853 unsigned int pool_size = 0;
64f52b0e 2854 unsigned int front_pad, io_front_pad;
6f1c819c 2855 int ret;
e6ee8c0b
KU		 2856
2857 if (!pools)
4e6e36c3 2858 return NULL;
e6ee8c0b 2859
78d8e58a
MS		 2860 switch (type) {
2861 case DM_TYPE_BIO_BASED:
545ed20e 2862 case DM_TYPE_DAX_BIO_BASED:
0776aa0e 2863 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
30187e1d 2864 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
64f52b0e 2865 io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
6f1c819c
KO		 2866 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
2867 if (ret)
64f52b0e 2868 goto out;
6f1c819c 2869 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
eb8db831 2870 goto out;
78d8e58a
MS		 2871 break;
2872 case DM_TYPE_REQUEST_BASED:
0776aa0e 2873 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
78d8e58a 2874 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
591ddcfc 2875 /* per_io_data_size is used for blk-mq pdu at queue allocation */
78d8e58a
MS		 2876 break;
2877 default:
2878 BUG();
2879 }
2880
6f1c819c
KO		 2881 ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
2882 if (ret)
5f015204 2883 goto out;
e6ee8c0b 2884
6f1c819c 2885 if (integrity && bioset_integrity_create(&pools->bs, pool_size))
5f015204 2886 goto out;
a91a2785 2887
e6ee8c0b 2888 return pools;
5f1b670d 2889
5f1b670d
CH		 2890out:
2891 dm_free_md_mempools(pools);
78d8e58a 2892
4e6e36c3 2893 return NULL;
e6ee8c0b
KU		 2894}
2895
2896void dm_free_md_mempools(struct dm_md_mempools *pools)
2897{
2898 if (!pools)
2899 return;
2900
6f1c819c
KO		 2901 bioset_exit(&pools->bs);
2902 bioset_exit(&pools->io_bs);
e6ee8c0b
KU		 2903
2904 kfree(pools);
2905}
2906
9c72bad1
CH		 2907struct dm_pr {
2908 u64 old_key;
2909 u64 new_key;
2910 u32 flags;
2911 bool fail_early;
2912};
2913
2914static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2915 void *data)
71cdb697
CH		 2916{
2917 struct mapped_device *md = bdev->bd_disk->private_data;
9c72bad1
CH		 2918 struct dm_table *table;
2919 struct dm_target *ti;
2920 int ret = -ENOTTY, srcu_idx;
71cdb697 2921
9c72bad1
CH		 2922 table = dm_get_live_table(md, &srcu_idx);
2923 if (!table || !dm_table_get_size(table))
2924 goto out;
71cdb697 2925
9c72bad1
CH		 2926 /* We only support devices that have a single target */
2927 if (dm_table_get_num_targets(table) != 1)
2928 goto out;
2929 ti = dm_table_get_target(table, 0);
71cdb697 2930
9c72bad1
CH		 2931 ret = -EINVAL;
2932 if (!ti->type->iterate_devices)
2933 goto out;
2934
2935 ret = ti->type->iterate_devices(ti, fn, data);
2936out:
2937 dm_put_live_table(md, srcu_idx);
2938 return ret;
2939}
2940
2941/*
2942 * For register / unregister we need to manually call out to every path.
2943 */
2944static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2945 sector_t start, sector_t len, void *data)
2946{
2947 struct dm_pr *pr = data;
2948 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2949
2950 if (!ops || !ops->pr_register)
2951 return -EOPNOTSUPP;
2952 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2953}
2954
2955static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
2956 u32 flags)
2957{
2958 struct dm_pr pr = {
2959 .old_key = old_key,
2960 .new_key = new_key,
2961 .flags = flags,
2962 .fail_early = true,
2963 };
2964 int ret;
2965
2966 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2967 if (ret && new_key) {
2968 /* unregister all paths if we failed to register any path */
2969 pr.old_key = new_key;
2970 pr.new_key = 0;
2971 pr.flags = 0;
2972 pr.fail_early = false;
2973 dm_call_pr(bdev, __dm_pr_register, &pr);
2974 }
2975
2976 return ret;
71cdb697
CH		 2977}
2978
2979static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
956a4025 2980 u32 flags)
71cdb697
CH		 2981{
2982 struct mapped_device *md = bdev->bd_disk->private_data;
2983 const struct pr_ops *ops;
971888c4 2984 int r, srcu_idx;
71cdb697 2985
5bd5e8d8 2986 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
71cdb697 2987 if (r < 0)
971888c4 2988 goto out;
71cdb697
CH		 2989
2990 ops = bdev->bd_disk->fops->pr_ops;
2991 if (ops && ops->pr_reserve)
2992 r = ops->pr_reserve(bdev, key, type, flags);
2993 else
2994 r = -EOPNOTSUPP;
971888c4
MS		 2995out:
2996 dm_unprepare_ioctl(md, srcu_idx);
71cdb697
CH		 2997 return r;
2998}
2999
3000static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3001{
3002 struct mapped_device *md = bdev->bd_disk->private_data;
3003 const struct pr_ops *ops;
971888c4 3004 int r, srcu_idx;
71cdb697 3005
5bd5e8d8 3006 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
71cdb697 3007 if (r < 0)
971888c4 3008 goto out;
71cdb697
CH		 3009
3010 ops = bdev->bd_disk->fops->pr_ops;
3011 if (ops && ops->pr_release)
3012 r = ops->pr_release(bdev, key, type);
3013 else
3014 r = -EOPNOTSUPP;
971888c4
MS		 3015out:
3016 dm_unprepare_ioctl(md, srcu_idx);
71cdb697
CH		 3017 return r;
3018}
3019
3020static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
956a4025 3021 enum pr_type type, bool abort)
71cdb697
CH		 3022{
3023 struct mapped_device *md = bdev->bd_disk->private_data;
3024 const struct pr_ops *ops;
971888c4 3025 int r, srcu_idx;
71cdb697 3026
5bd5e8d8 3027 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
71cdb697 3028 if (r < 0)
971888c4 3029 goto out;
71cdb697
CH		 3030
3031 ops = bdev->bd_disk->fops->pr_ops;
3032 if (ops && ops->pr_preempt)
3033 r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3034 else
3035 r = -EOPNOTSUPP;
971888c4
MS		 3036out:
3037 dm_unprepare_ioctl(md, srcu_idx);
71cdb697
CH		 3038 return r;
3039}
3040
3041static int dm_pr_clear(struct block_device *bdev, u64 key)
3042{
3043 struct mapped_device *md = bdev->bd_disk->private_data;
3044 const struct pr_ops *ops;
971888c4 3045 int r, srcu_idx;
71cdb697 3046
5bd5e8d8 3047 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
71cdb697 3048 if (r < 0)
971888c4 3049 goto out;
71cdb697
CH		 3050
3051 ops = bdev->bd_disk->fops->pr_ops;
3052 if (ops && ops->pr_clear)
3053 r = ops->pr_clear(bdev, key);
3054 else
3055 r = -EOPNOTSUPP;
971888c4
MS		 3056out:
3057 dm_unprepare_ioctl(md, srcu_idx);
71cdb697
CH		 3058 return r;
3059}
3060
3061static const struct pr_ops dm_pr_ops = {
3062 .pr_register = dm_pr_register,
3063 .pr_reserve = dm_pr_reserve,
3064 .pr_release = dm_pr_release,
3065 .pr_preempt = dm_pr_preempt,
3066 .pr_clear = dm_pr_clear,
3067};
3068
83d5cde4 3069static const struct block_device_operations dm_blk_dops = {
c62b37d9 3070 .submit_bio = dm_submit_bio,
1da177e4
LT		 3071 .open = dm_blk_open,
3072 .release = dm_blk_close,
aa129a22 3073 .ioctl = dm_blk_ioctl,
3ac51e74 3074 .getgeo = dm_blk_getgeo,
e76239a3 3075 .report_zones = dm_blk_report_zones,
71cdb697 3076 .pr_ops = &dm_pr_ops,
1da177e4
LT		 3077 .owner = THIS_MODULE
3078};
3079
681cc5e8
MS		 3080static const struct block_device_operations dm_rq_blk_dops = {
3081 .open = dm_blk_open,
3082 .release = dm_blk_close,
3083 .ioctl = dm_blk_ioctl,
3084 .getgeo = dm_blk_getgeo,
3085 .pr_ops = &dm_pr_ops,
3086 .owner = THIS_MODULE
3087};
3088
f26c5719
DW		 3089static const struct dax_operations dm_dax_ops = {
3090 .direct_access = dm_dax_direct_access,
7bf7eac8 3091 .dax_supported = dm_dax_supported,
7e026c8c 3092 .copy_from_iter = dm_dax_copy_from_iter,
b3a9a0c3 3093 .copy_to_iter = dm_dax_copy_to_iter,
cdf6cdcd 3094 .zero_page_range = dm_dax_zero_page_range,
f26c5719
DW		 3095};
3096
1da177e4
LT		 3097/*
3098 * module hooks
3099 */
3100module_init(dm_init);
3101module_exit(dm_exit);
3102
3103module_param(major, uint, 0);
3104MODULE_PARM_DESC(major, "The major number of the device mapper");
f4790826 3105
e8603136
MS		 3106module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3107MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3108
115485e8
MS		 3109module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3110MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3111
1da177e4
LT		 3112MODULE_DESCRIPTION(DM_NAME " driver");
3113MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3114MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)