Merge tag 'for-linus-4.11' of git://git.code.sf.net/p/openipmi/linux-ipmi
[linux-2.6-block.git] / drivers / md / dm-thin.c
1 /*
2  * Copyright (C) 2011-2012 Red Hat UK.
3  *
4  * This file is released under the GPL.
5  */
6
7 #include "dm-thin-metadata.h"
8 #include "dm-bio-prison.h"
9 #include "dm.h"
10
11 #include <linux/device-mapper.h>
12 #include <linux/dm-io.h>
13 #include <linux/dm-kcopyd.h>
14 #include <linux/jiffies.h>
15 #include <linux/log2.h>
16 #include <linux/list.h>
17 #include <linux/rculist.h>
18 #include <linux/init.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 #include <linux/sort.h>
23 #include <linux/rbtree.h>
24
25 #define DM_MSG_PREFIX   "thin"
26
27 /*
28  * Tunable constants
29  */
30 #define ENDIO_HOOK_POOL_SIZE 1024
31 #define MAPPING_POOL_SIZE 1024
32 #define COMMIT_PERIOD HZ
33 #define NO_SPACE_TIMEOUT_SECS 60
34
35 static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
36
37 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
38                 "A percentage of time allocated for copy on write");
39
40 /*
41  * The block size of the device holding pool data must be
42  * between 64KB and 1GB.
43  */
44 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
45 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
46
47 /*
48  * Device id is restricted to 24 bits.
49  */
50 #define MAX_DEV_ID ((1 << 24) - 1)
51
52 /*
53  * How do we handle breaking sharing of data blocks?
54  * =================================================
55  *
56  * We use a standard copy-on-write btree to store the mappings for the
57  * devices (note I'm talking about copy-on-write of the metadata here, not
58  * the data).  When you take an internal snapshot you clone the root node
59  * of the origin btree.  After this there is no concept of an origin or a
60  * snapshot.  They are just two device trees that happen to point to the
61  * same data blocks.
62  *
63  * When we get a write in we decide if it's to a shared data block using
64  * some timestamp magic.  If it is, we have to break sharing.
65  *
66  * Let's say we write to a shared block in what was the origin.  The
67  * steps are:
68  *
69  * i) plug io further to this physical block. (see bio_prison code).
70  *
71  * ii) quiesce any read io to that shared data block.  Obviously
72  * including all devices that share this block.  (see dm_deferred_set code)
73  *
74  * iii) copy the data block to a newly allocate block.  This step can be
75  * missed out if the io covers the block. (schedule_copy).
76  *
77  * iv) insert the new mapping into the origin's btree
78  * (process_prepared_mapping).  This act of inserting breaks some
79  * sharing of btree nodes between the two devices.  Breaking sharing only
80  * effects the btree of that specific device.  Btrees for the other
81  * devices that share the block never change.  The btree for the origin
82  * device as it was after the last commit is untouched, ie. we're using
83  * persistent data structures in the functional programming sense.
84  *
85  * v) unplug io to this physical block, including the io that triggered
86  * the breaking of sharing.
87  *
88  * Steps (ii) and (iii) occur in parallel.
89  *
90  * The metadata _doesn't_ need to be committed before the io continues.  We
91  * get away with this because the io is always written to a _new_ block.
92  * If there's a crash, then:
93  *
94  * - The origin mapping will point to the old origin block (the shared
95  * one).  This will contain the data as it was before the io that triggered
96  * the breaking of sharing came in.
97  *
98  * - The snap mapping still points to the old block.  As it would after
99  * the commit.
100  *
101  * The downside of this scheme is the timestamp magic isn't perfect, and
102  * will continue to think that data block in the snapshot device is shared
103  * even after the write to the origin has broken sharing.  I suspect data
104  * blocks will typically be shared by many different devices, so we're
105  * breaking sharing n + 1 times, rather than n, where n is the number of
106  * devices that reference this data block.  At the moment I think the
107  * benefits far, far outweigh the disadvantages.
108  */
109
110 /*----------------------------------------------------------------*/
111
112 /*
113  * Key building.
114  */
115 enum lock_space {
116         VIRTUAL,
117         PHYSICAL
118 };
119
120 static void build_key(struct dm_thin_device *td, enum lock_space ls,
121                       dm_block_t b, dm_block_t e, struct dm_cell_key *key)
122 {
123         key->virtual = (ls == VIRTUAL);
124         key->dev = dm_thin_dev_id(td);
125         key->block_begin = b;
126         key->block_end = e;
127 }
128
129 static void build_data_key(struct dm_thin_device *td, dm_block_t b,
130                            struct dm_cell_key *key)
131 {
132         build_key(td, PHYSICAL, b, b + 1llu, key);
133 }
134
135 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
136                               struct dm_cell_key *key)
137 {
138         build_key(td, VIRTUAL, b, b + 1llu, key);
139 }
140
141 /*----------------------------------------------------------------*/
142
143 #define THROTTLE_THRESHOLD (1 * HZ)
144
145 struct throttle {
146         struct rw_semaphore lock;
147         unsigned long threshold;
148         bool throttle_applied;
149 };
150
151 static void throttle_init(struct throttle *t)
152 {
153         init_rwsem(&t->lock);
154         t->throttle_applied = false;
155 }
156
157 static void throttle_work_start(struct throttle *t)
158 {
159         t->threshold = jiffies + THROTTLE_THRESHOLD;
160 }
161
162 static void throttle_work_update(struct throttle *t)
163 {
164         if (!t->throttle_applied && jiffies > t->threshold) {
165                 down_write(&t->lock);
166                 t->throttle_applied = true;
167         }
168 }
169
170 static void throttle_work_complete(struct throttle *t)
171 {
172         if (t->throttle_applied) {
173                 t->throttle_applied = false;
174                 up_write(&t->lock);
175         }
176 }
177
178 static void throttle_lock(struct throttle *t)
179 {
180         down_read(&t->lock);
181 }
182
183 static void throttle_unlock(struct throttle *t)
184 {
185         up_read(&t->lock);
186 }
187
188 /*----------------------------------------------------------------*/
189
190 /*
191  * A pool device ties together a metadata device and a data device.  It
192  * also provides the interface for creating and destroying internal
193  * devices.
194  */
195 struct dm_thin_new_mapping;
196
197 /*
198  * The pool runs in 4 modes.  Ordered in degraded order for comparisons.
199  */
200 enum pool_mode {
201         PM_WRITE,               /* metadata may be changed */
202         PM_OUT_OF_DATA_SPACE,   /* metadata may be changed, though data may not be allocated */
203         PM_READ_ONLY,           /* metadata may not be changed */
204         PM_FAIL,                /* all I/O fails */
205 };
206
207 struct pool_features {
208         enum pool_mode mode;
209
210         bool zero_new_blocks:1;
211         bool discard_enabled:1;
212         bool discard_passdown:1;
213         bool error_if_no_space:1;
214 };
215
216 struct thin_c;
217 typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
218 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
219 typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
220
221 #define CELL_SORT_ARRAY_SIZE 8192
222
223 struct pool {
224         struct list_head list;
225         struct dm_target *ti;   /* Only set if a pool target is bound */
226
227         struct mapped_device *pool_md;
228         struct block_device *md_dev;
229         struct dm_pool_metadata *pmd;
230
231         dm_block_t low_water_blocks;
232         uint32_t sectors_per_block;
233         int sectors_per_block_shift;
234
235         struct pool_features pf;
236         bool low_water_triggered:1;     /* A dm event has been sent */
237         bool suspended:1;
238         bool out_of_data_space:1;
239
240         struct dm_bio_prison *prison;
241         struct dm_kcopyd_client *copier;
242
243         struct workqueue_struct *wq;
244         struct throttle throttle;
245         struct work_struct worker;
246         struct delayed_work waker;
247         struct delayed_work no_space_timeout;
248
249         unsigned long last_commit_jiffies;
250         unsigned ref_count;
251
252         spinlock_t lock;
253         struct bio_list deferred_flush_bios;
254         struct list_head prepared_mappings;
255         struct list_head prepared_discards;
256         struct list_head prepared_discards_pt2;
257         struct list_head active_thins;
258
259         struct dm_deferred_set *shared_read_ds;
260         struct dm_deferred_set *all_io_ds;
261
262         struct dm_thin_new_mapping *next_mapping;
263         mempool_t *mapping_pool;
264
265         process_bio_fn process_bio;
266         process_bio_fn process_discard;
267
268         process_cell_fn process_cell;
269         process_cell_fn process_discard_cell;
270
271         process_mapping_fn process_prepared_mapping;
272         process_mapping_fn process_prepared_discard;
273         process_mapping_fn process_prepared_discard_pt2;
274
275         struct dm_bio_prison_cell **cell_sort_array;
276 };
277
278 static enum pool_mode get_pool_mode(struct pool *pool);
279 static void metadata_operation_failed(struct pool *pool, const char *op, int r);
280
281 /*
282  * Target context for a pool.
283  */
284 struct pool_c {
285         struct dm_target *ti;
286         struct pool *pool;
287         struct dm_dev *data_dev;
288         struct dm_dev *metadata_dev;
289         struct dm_target_callbacks callbacks;
290
291         dm_block_t low_water_blocks;
292         struct pool_features requested_pf; /* Features requested during table load */
293         struct pool_features adjusted_pf;  /* Features used after adjusting for constituent devices */
294 };
295
296 /*
297  * Target context for a thin.
298  */
299 struct thin_c {
300         struct list_head list;
301         struct dm_dev *pool_dev;
302         struct dm_dev *origin_dev;
303         sector_t origin_size;
304         dm_thin_id dev_id;
305
306         struct pool *pool;
307         struct dm_thin_device *td;
308         struct mapped_device *thin_md;
309
310         bool requeue_mode:1;
311         spinlock_t lock;
312         struct list_head deferred_cells;
313         struct bio_list deferred_bio_list;
314         struct bio_list retry_on_resume_list;
315         struct rb_root sort_bio_list; /* sorted list of deferred bios */
316
317         /*
318          * Ensures the thin is not destroyed until the worker has finished
319          * iterating the active_thins list.
320          */
321         atomic_t refcount;
322         struct completion can_destroy;
323 };
324
325 /*----------------------------------------------------------------*/
326
327 static bool block_size_is_power_of_two(struct pool *pool)
328 {
329         return pool->sectors_per_block_shift >= 0;
330 }
331
332 static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
333 {
334         return block_size_is_power_of_two(pool) ?
335                 (b << pool->sectors_per_block_shift) :
336                 (b * pool->sectors_per_block);
337 }
338
339 /*----------------------------------------------------------------*/
340
341 struct discard_op {
342         struct thin_c *tc;
343         struct blk_plug plug;
344         struct bio *parent_bio;
345         struct bio *bio;
346 };
347
348 static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)
349 {
350         BUG_ON(!parent);
351
352         op->tc = tc;
353         blk_start_plug(&op->plug);
354         op->parent_bio = parent;
355         op->bio = NULL;
356 }
357
358 static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)
359 {
360         struct thin_c *tc = op->tc;
361         sector_t s = block_to_sectors(tc->pool, data_b);
362         sector_t len = block_to_sectors(tc->pool, data_e - data_b);
363
364         return __blkdev_issue_discard(tc->pool_dev->bdev, s, len,
365                                       GFP_NOWAIT, 0, &op->bio);
366 }
367
368 static void end_discard(struct discard_op *op, int r)
369 {
370         if (op->bio) {
371                 /*
372                  * Even if one of the calls to issue_discard failed, we
373                  * need to wait for the chain to complete.
374                  */
375                 bio_chain(op->bio, op->parent_bio);
376                 bio_set_op_attrs(op->bio, REQ_OP_DISCARD, 0);
377                 submit_bio(op->bio);
378         }
379
380         blk_finish_plug(&op->plug);
381
382         /*
383          * Even if r is set, there could be sub discards in flight that we
384          * need to wait for.
385          */
386         if (r && !op->parent_bio->bi_error)
387                 op->parent_bio->bi_error = r;
388         bio_endio(op->parent_bio);
389 }
390
391 /*----------------------------------------------------------------*/
392
393 /*
394  * wake_worker() is used when new work is queued and when pool_resume is
395  * ready to continue deferred IO processing.
396  */
397 static void wake_worker(struct pool *pool)
398 {
399         queue_work(pool->wq, &pool->worker);
400 }
401
402 /*----------------------------------------------------------------*/
403
404 static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
405                       struct dm_bio_prison_cell **cell_result)
406 {
407         int r;
408         struct dm_bio_prison_cell *cell_prealloc;
409
410         /*
411          * Allocate a cell from the prison's mempool.
412          * This might block but it can't fail.
413          */
414         cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
415
416         r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
417         if (r)
418                 /*
419                  * We reused an old cell; we can get rid of
420                  * the new one.
421                  */
422                 dm_bio_prison_free_cell(pool->prison, cell_prealloc);
423
424         return r;
425 }
426
427 static void cell_release(struct pool *pool,
428                          struct dm_bio_prison_cell *cell,
429                          struct bio_list *bios)
430 {
431         dm_cell_release(pool->prison, cell, bios);
432         dm_bio_prison_free_cell(pool->prison, cell);
433 }
434
435 static void cell_visit_release(struct pool *pool,
436                                void (*fn)(void *, struct dm_bio_prison_cell *),
437                                void *context,
438                                struct dm_bio_prison_cell *cell)
439 {
440         dm_cell_visit_release(pool->prison, fn, context, cell);
441         dm_bio_prison_free_cell(pool->prison, cell);
442 }
443
444 static void cell_release_no_holder(struct pool *pool,
445                                    struct dm_bio_prison_cell *cell,
446                                    struct bio_list *bios)
447 {
448         dm_cell_release_no_holder(pool->prison, cell, bios);
449         dm_bio_prison_free_cell(pool->prison, cell);
450 }
451
452 static void cell_error_with_code(struct pool *pool,
453                                  struct dm_bio_prison_cell *cell, int error_code)
454 {
455         dm_cell_error(pool->prison, cell, error_code);
456         dm_bio_prison_free_cell(pool->prison, cell);
457 }
458
459 static int get_pool_io_error_code(struct pool *pool)
460 {
461         return pool->out_of_data_space ? -ENOSPC : -EIO;
462 }
463
464 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
465 {
466         int error = get_pool_io_error_code(pool);
467
468         cell_error_with_code(pool, cell, error);
469 }
470
471 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
472 {
473         cell_error_with_code(pool, cell, 0);
474 }
475
476 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
477 {
478         cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE);
479 }
480
481 /*----------------------------------------------------------------*/
482
483 /*
484  * A global list of pools that uses a struct mapped_device as a key.
485  */
486 static struct dm_thin_pool_table {
487         struct mutex mutex;
488         struct list_head pools;
489 } dm_thin_pool_table;
490
491 static void pool_table_init(void)
492 {
493         mutex_init(&dm_thin_pool_table.mutex);
494         INIT_LIST_HEAD(&dm_thin_pool_table.pools);
495 }
496
497 static void __pool_table_insert(struct pool *pool)
498 {
499         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
500         list_add(&pool->list, &dm_thin_pool_table.pools);
501 }
502
503 static void __pool_table_remove(struct pool *pool)
504 {
505         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
506         list_del(&pool->list);
507 }
508
509 static struct pool *__pool_table_lookup(struct mapped_device *md)
510 {
511         struct pool *pool = NULL, *tmp;
512
513         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
514
515         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
516                 if (tmp->pool_md == md) {
517                         pool = tmp;
518                         break;
519                 }
520         }
521
522         return pool;
523 }
524
525 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
526 {
527         struct pool *pool = NULL, *tmp;
528
529         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
530
531         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
532                 if (tmp->md_dev == md_dev) {
533                         pool = tmp;
534                         break;
535                 }
536         }
537
538         return pool;
539 }
540
541 /*----------------------------------------------------------------*/
542
543 struct dm_thin_endio_hook {
544         struct thin_c *tc;
545         struct dm_deferred_entry *shared_read_entry;
546         struct dm_deferred_entry *all_io_entry;
547         struct dm_thin_new_mapping *overwrite_mapping;
548         struct rb_node rb_node;
549         struct dm_bio_prison_cell *cell;
550 };
551
552 static void __merge_bio_list(struct bio_list *bios, struct bio_list *master)
553 {
554         bio_list_merge(bios, master);
555         bio_list_init(master);
556 }
557
558 static void error_bio_list(struct bio_list *bios, int error)
559 {
560         struct bio *bio;
561
562         while ((bio = bio_list_pop(bios))) {
563                 bio->bi_error = error;
564                 bio_endio(bio);
565         }
566 }
567
568 static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error)
569 {
570         struct bio_list bios;
571         unsigned long flags;
572
573         bio_list_init(&bios);
574
575         spin_lock_irqsave(&tc->lock, flags);
576         __merge_bio_list(&bios, master);
577         spin_unlock_irqrestore(&tc->lock, flags);
578
579         error_bio_list(&bios, error);
580 }
581
582 static void requeue_deferred_cells(struct thin_c *tc)
583 {
584         struct pool *pool = tc->pool;
585         unsigned long flags;
586         struct list_head cells;
587         struct dm_bio_prison_cell *cell, *tmp;
588
589         INIT_LIST_HEAD(&cells);
590
591         spin_lock_irqsave(&tc->lock, flags);
592         list_splice_init(&tc->deferred_cells, &cells);
593         spin_unlock_irqrestore(&tc->lock, flags);
594
595         list_for_each_entry_safe(cell, tmp, &cells, user_list)
596                 cell_requeue(pool, cell);
597 }
598
599 static void requeue_io(struct thin_c *tc)
600 {
601         struct bio_list bios;
602         unsigned long flags;
603
604         bio_list_init(&bios);
605
606         spin_lock_irqsave(&tc->lock, flags);
607         __merge_bio_list(&bios, &tc->deferred_bio_list);
608         __merge_bio_list(&bios, &tc->retry_on_resume_list);
609         spin_unlock_irqrestore(&tc->lock, flags);
610
611         error_bio_list(&bios, DM_ENDIO_REQUEUE);
612         requeue_deferred_cells(tc);
613 }
614
615 static void error_retry_list_with_code(struct pool *pool, int error)
616 {
617         struct thin_c *tc;
618
619         rcu_read_lock();
620         list_for_each_entry_rcu(tc, &pool->active_thins, list)
621                 error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
622         rcu_read_unlock();
623 }
624
625 static void error_retry_list(struct pool *pool)
626 {
627         int error = get_pool_io_error_code(pool);
628
629         error_retry_list_with_code(pool, error);
630 }
631
632 /*
633  * This section of code contains the logic for processing a thin device's IO.
634  * Much of the code depends on pool object resources (lists, workqueues, etc)
635  * but most is exclusively called from the thin target rather than the thin-pool
636  * target.
637  */
638
639 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
640 {
641         struct pool *pool = tc->pool;
642         sector_t block_nr = bio->bi_iter.bi_sector;
643
644         if (block_size_is_power_of_two(pool))
645                 block_nr >>= pool->sectors_per_block_shift;
646         else
647                 (void) sector_div(block_nr, pool->sectors_per_block);
648
649         return block_nr;
650 }
651
652 /*
653  * Returns the _complete_ blocks that this bio covers.
654  */
655 static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
656                                 dm_block_t *begin, dm_block_t *end)
657 {
658         struct pool *pool = tc->pool;
659         sector_t b = bio->bi_iter.bi_sector;
660         sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
661
662         b += pool->sectors_per_block - 1ull; /* so we round up */
663
664         if (block_size_is_power_of_two(pool)) {
665                 b >>= pool->sectors_per_block_shift;
666                 e >>= pool->sectors_per_block_shift;
667         } else {
668                 (void) sector_div(b, pool->sectors_per_block);
669                 (void) sector_div(e, pool->sectors_per_block);
670         }
671
672         if (e < b)
673                 /* Can happen if the bio is within a single block. */
674                 e = b;
675
676         *begin = b;
677         *end = e;
678 }
679
680 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
681 {
682         struct pool *pool = tc->pool;
683         sector_t bi_sector = bio->bi_iter.bi_sector;
684
685         bio->bi_bdev = tc->pool_dev->bdev;
686         if (block_size_is_power_of_two(pool))
687                 bio->bi_iter.bi_sector =
688                         (block << pool->sectors_per_block_shift) |
689                         (bi_sector & (pool->sectors_per_block - 1));
690         else
691                 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
692                                  sector_div(bi_sector, pool->sectors_per_block);
693 }
694
695 static void remap_to_origin(struct thin_c *tc, struct bio *bio)
696 {
697         bio->bi_bdev = tc->origin_dev->bdev;
698 }
699
700 static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
701 {
702         return op_is_flush(bio->bi_opf) &&
703                 dm_thin_changed_this_transaction(tc->td);
704 }
705
706 static void inc_all_io_entry(struct pool *pool, struct bio *bio)
707 {
708         struct dm_thin_endio_hook *h;
709
710         if (bio_op(bio) == REQ_OP_DISCARD)
711                 return;
712
713         h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
714         h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
715 }
716
717 static void issue(struct thin_c *tc, struct bio *bio)
718 {
719         struct pool *pool = tc->pool;
720         unsigned long flags;
721
722         if (!bio_triggers_commit(tc, bio)) {
723                 generic_make_request(bio);
724                 return;
725         }
726
727         /*
728          * Complete bio with an error if earlier I/O caused changes to
729          * the metadata that can't be committed e.g, due to I/O errors
730          * on the metadata device.
731          */
732         if (dm_thin_aborted_changes(tc->td)) {
733                 bio_io_error(bio);
734                 return;
735         }
736
737         /*
738          * Batch together any bios that trigger commits and then issue a
739          * single commit for them in process_deferred_bios().
740          */
741         spin_lock_irqsave(&pool->lock, flags);
742         bio_list_add(&pool->deferred_flush_bios, bio);
743         spin_unlock_irqrestore(&pool->lock, flags);
744 }
745
746 static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
747 {
748         remap_to_origin(tc, bio);
749         issue(tc, bio);
750 }
751
752 static void remap_and_issue(struct thin_c *tc, struct bio *bio,
753                             dm_block_t block)
754 {
755         remap(tc, bio, block);
756         issue(tc, bio);
757 }
758
759 /*----------------------------------------------------------------*/
760
761 /*
762  * Bio endio functions.
763  */
764 struct dm_thin_new_mapping {
765         struct list_head list;
766
767         bool pass_discard:1;
768         bool maybe_shared:1;
769
770         /*
771          * Track quiescing, copying and zeroing preparation actions.  When this
772          * counter hits zero the block is prepared and can be inserted into the
773          * btree.
774          */
775         atomic_t prepare_actions;
776
777         int err;
778         struct thin_c *tc;
779         dm_block_t virt_begin, virt_end;
780         dm_block_t data_block;
781         struct dm_bio_prison_cell *cell;
782
783         /*
784          * If the bio covers the whole area of a block then we can avoid
785          * zeroing or copying.  Instead this bio is hooked.  The bio will
786          * still be in the cell, so care has to be taken to avoid issuing
787          * the bio twice.
788          */
789         struct bio *bio;
790         bio_end_io_t *saved_bi_end_io;
791 };
792
793 static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
794 {
795         struct pool *pool = m->tc->pool;
796
797         if (atomic_dec_and_test(&m->prepare_actions)) {
798                 list_add_tail(&m->list, &pool->prepared_mappings);
799                 wake_worker(pool);
800         }
801 }
802
803 static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
804 {
805         unsigned long flags;
806         struct pool *pool = m->tc->pool;
807
808         spin_lock_irqsave(&pool->lock, flags);
809         __complete_mapping_preparation(m);
810         spin_unlock_irqrestore(&pool->lock, flags);
811 }
812
813 static void copy_complete(int read_err, unsigned long write_err, void *context)
814 {
815         struct dm_thin_new_mapping *m = context;
816
817         m->err = read_err || write_err ? -EIO : 0;
818         complete_mapping_preparation(m);
819 }
820
821 static void overwrite_endio(struct bio *bio)
822 {
823         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
824         struct dm_thin_new_mapping *m = h->overwrite_mapping;
825
826         bio->bi_end_io = m->saved_bi_end_io;
827
828         m->err = bio->bi_error;
829         complete_mapping_preparation(m);
830 }
831
832 /*----------------------------------------------------------------*/
833
834 /*
835  * Workqueue.
836  */
837
838 /*
839  * Prepared mapping jobs.
840  */
841
842 /*
843  * This sends the bios in the cell, except the original holder, back
844  * to the deferred_bios list.
845  */
846 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
847 {
848         struct pool *pool = tc->pool;
849         unsigned long flags;
850
851         spin_lock_irqsave(&tc->lock, flags);
852         cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
853         spin_unlock_irqrestore(&tc->lock, flags);
854
855         wake_worker(pool);
856 }
857
858 static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
859
860 struct remap_info {
861         struct thin_c *tc;
862         struct bio_list defer_bios;
863         struct bio_list issue_bios;
864 };
865
866 static void __inc_remap_and_issue_cell(void *context,
867                                        struct dm_bio_prison_cell *cell)
868 {
869         struct remap_info *info = context;
870         struct bio *bio;
871
872         while ((bio = bio_list_pop(&cell->bios))) {
873                 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)
874                         bio_list_add(&info->defer_bios, bio);
875                 else {
876                         inc_all_io_entry(info->tc->pool, bio);
877
878                         /*
879                          * We can't issue the bios with the bio prison lock
880                          * held, so we add them to a list to issue on
881                          * return from this function.
882                          */
883                         bio_list_add(&info->issue_bios, bio);
884                 }
885         }
886 }
887
888 static void inc_remap_and_issue_cell(struct thin_c *tc,
889                                      struct dm_bio_prison_cell *cell,
890                                      dm_block_t block)
891 {
892         struct bio *bio;
893         struct remap_info info;
894
895         info.tc = tc;
896         bio_list_init(&info.defer_bios);
897         bio_list_init(&info.issue_bios);
898
899         /*
900          * We have to be careful to inc any bios we're about to issue
901          * before the cell is released, and avoid a race with new bios
902          * being added to the cell.
903          */
904         cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
905                            &info, cell);
906
907         while ((bio = bio_list_pop(&info.defer_bios)))
908                 thin_defer_bio(tc, bio);
909
910         while ((bio = bio_list_pop(&info.issue_bios)))
911                 remap_and_issue(info.tc, bio, block);
912 }
913
914 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
915 {
916         cell_error(m->tc->pool, m->cell);
917         list_del(&m->list);
918         mempool_free(m, m->tc->pool->mapping_pool);
919 }
920
921 static void process_prepared_mapping(struct dm_thin_new_mapping *m)
922 {
923         struct thin_c *tc = m->tc;
924         struct pool *pool = tc->pool;
925         struct bio *bio = m->bio;
926         int r;
927
928         if (m->err) {
929                 cell_error(pool, m->cell);
930                 goto out;
931         }
932
933         /*
934          * Commit the prepared block into the mapping btree.
935          * Any I/O for this block arriving after this point will get
936          * remapped to it directly.
937          */
938         r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
939         if (r) {
940                 metadata_operation_failed(pool, "dm_thin_insert_block", r);
941                 cell_error(pool, m->cell);
942                 goto out;
943         }
944
945         /*
946          * Release any bios held while the block was being provisioned.
947          * If we are processing a write bio that completely covers the block,
948          * we already processed it so can ignore it now when processing
949          * the bios in the cell.
950          */
951         if (bio) {
952                 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
953                 bio_endio(bio);
954         } else {
955                 inc_all_io_entry(tc->pool, m->cell->holder);
956                 remap_and_issue(tc, m->cell->holder, m->data_block);
957                 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
958         }
959
960 out:
961         list_del(&m->list);
962         mempool_free(m, pool->mapping_pool);
963 }
964
965 /*----------------------------------------------------------------*/
966
967 static void free_discard_mapping(struct dm_thin_new_mapping *m)
968 {
969         struct thin_c *tc = m->tc;
970         if (m->cell)
971                 cell_defer_no_holder(tc, m->cell);
972         mempool_free(m, tc->pool->mapping_pool);
973 }
974
975 static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
976 {
977         bio_io_error(m->bio);
978         free_discard_mapping(m);
979 }
980
981 static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
982 {
983         bio_endio(m->bio);
984         free_discard_mapping(m);
985 }
986
987 static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
988 {
989         int r;
990         struct thin_c *tc = m->tc;
991
992         r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
993         if (r) {
994                 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
995                 bio_io_error(m->bio);
996         } else
997                 bio_endio(m->bio);
998
999         cell_defer_no_holder(tc, m->cell);
1000         mempool_free(m, tc->pool->mapping_pool);
1001 }
1002
1003 /*----------------------------------------------------------------*/
1004
1005 static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,
1006                                                    struct bio *discard_parent)
1007 {
1008         /*
1009          * We've already unmapped this range of blocks, but before we
1010          * passdown we have to check that these blocks are now unused.
1011          */
1012         int r = 0;
1013         bool used = true;
1014         struct thin_c *tc = m->tc;
1015         struct pool *pool = tc->pool;
1016         dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
1017         struct discard_op op;
1018
1019         begin_discard(&op, tc, discard_parent);
1020         while (b != end) {
1021                 /* find start of unmapped run */
1022                 for (; b < end; b++) {
1023                         r = dm_pool_block_is_used(pool->pmd, b, &used);
1024                         if (r)
1025                                 goto out;
1026
1027                         if (!used)
1028                                 break;
1029                 }
1030
1031                 if (b == end)
1032                         break;
1033
1034                 /* find end of run */
1035                 for (e = b + 1; e != end; e++) {
1036                         r = dm_pool_block_is_used(pool->pmd, e, &used);
1037                         if (r)
1038                                 goto out;
1039
1040                         if (used)
1041                                 break;
1042                 }
1043
1044                 r = issue_discard(&op, b, e);
1045                 if (r)
1046                         goto out;
1047
1048                 b = e;
1049         }
1050 out:
1051         end_discard(&op, r);
1052 }
1053
1054 static void queue_passdown_pt2(struct dm_thin_new_mapping *m)
1055 {
1056         unsigned long flags;
1057         struct pool *pool = m->tc->pool;
1058
1059         spin_lock_irqsave(&pool->lock, flags);
1060         list_add_tail(&m->list, &pool->prepared_discards_pt2);
1061         spin_unlock_irqrestore(&pool->lock, flags);
1062         wake_worker(pool);
1063 }
1064
1065 static void passdown_endio(struct bio *bio)
1066 {
1067         /*
1068          * It doesn't matter if the passdown discard failed, we still want
1069          * to unmap (we ignore err).
1070          */
1071         queue_passdown_pt2(bio->bi_private);
1072 }
1073
1074 static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
1075 {
1076         int r;
1077         struct thin_c *tc = m->tc;
1078         struct pool *pool = tc->pool;
1079         struct bio *discard_parent;
1080         dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);
1081
1082         /*
1083          * Only this thread allocates blocks, so we can be sure that the
1084          * newly unmapped blocks will not be allocated before the end of
1085          * the function.
1086          */
1087         r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
1088         if (r) {
1089                 metadata_operation_failed(pool, "dm_thin_remove_range", r);
1090                 bio_io_error(m->bio);
1091                 cell_defer_no_holder(tc, m->cell);
1092                 mempool_free(m, pool->mapping_pool);
1093                 return;
1094         }
1095
1096         discard_parent = bio_alloc(GFP_NOIO, 1);
1097         if (!discard_parent) {
1098                 DMWARN("%s: unable to allocate top level discard bio for passdown. Skipping passdown.",
1099                        dm_device_name(tc->pool->pool_md));
1100                 queue_passdown_pt2(m);
1101
1102         } else {
1103                 discard_parent->bi_end_io = passdown_endio;
1104                 discard_parent->bi_private = m;
1105
1106                 if (m->maybe_shared)
1107                         passdown_double_checking_shared_status(m, discard_parent);
1108                 else {
1109                         struct discard_op op;
1110
1111                         begin_discard(&op, tc, discard_parent);
1112                         r = issue_discard(&op, m->data_block, data_end);
1113                         end_discard(&op, r);
1114                 }
1115         }
1116
1117         /*
1118          * Increment the unmapped blocks.  This prevents a race between the
1119          * passdown io and reallocation of freed blocks.
1120          */
1121         r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end);
1122         if (r) {
1123                 metadata_operation_failed(pool, "dm_pool_inc_data_range", r);
1124                 bio_io_error(m->bio);
1125                 cell_defer_no_holder(tc, m->cell);
1126                 mempool_free(m, pool->mapping_pool);
1127                 return;
1128         }
1129 }
1130
1131 static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
1132 {
1133         int r;
1134         struct thin_c *tc = m->tc;
1135         struct pool *pool = tc->pool;
1136
1137         /*
1138          * The passdown has completed, so now we can decrement all those
1139          * unmapped blocks.
1140          */
1141         r = dm_pool_dec_data_range(pool->pmd, m->data_block,
1142                                    m->data_block + (m->virt_end - m->virt_begin));
1143         if (r) {
1144                 metadata_operation_failed(pool, "dm_pool_dec_data_range", r);
1145                 bio_io_error(m->bio);
1146         } else
1147                 bio_endio(m->bio);
1148
1149         cell_defer_no_holder(tc, m->cell);
1150         mempool_free(m, pool->mapping_pool);
1151 }
1152
1153 static void process_prepared(struct pool *pool, struct list_head *head,
1154                              process_mapping_fn *fn)
1155 {
1156         unsigned long flags;
1157         struct list_head maps;
1158         struct dm_thin_new_mapping *m, *tmp;
1159
1160         INIT_LIST_HEAD(&maps);
1161         spin_lock_irqsave(&pool->lock, flags);
1162         list_splice_init(head, &maps);
1163         spin_unlock_irqrestore(&pool->lock, flags);
1164
1165         list_for_each_entry_safe(m, tmp, &maps, list)
1166                 (*fn)(m);
1167 }
1168
1169 /*
1170  * Deferred bio jobs.
1171  */
1172 static int io_overlaps_block(struct pool *pool, struct bio *bio)
1173 {
1174         return bio->bi_iter.bi_size ==
1175                 (pool->sectors_per_block << SECTOR_SHIFT);
1176 }
1177
1178 static int io_overwrites_block(struct pool *pool, struct bio *bio)
1179 {
1180         return (bio_data_dir(bio) == WRITE) &&
1181                 io_overlaps_block(pool, bio);
1182 }
1183
1184 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
1185                                bio_end_io_t *fn)
1186 {
1187         *save = bio->bi_end_io;
1188         bio->bi_end_io = fn;
1189 }
1190
1191 static int ensure_next_mapping(struct pool *pool)
1192 {
1193         if (pool->next_mapping)
1194                 return 0;
1195
1196         pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
1197
1198         return pool->next_mapping ? 0 : -ENOMEM;
1199 }
1200
1201 static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
1202 {
1203         struct dm_thin_new_mapping *m = pool->next_mapping;
1204
1205         BUG_ON(!pool->next_mapping);
1206
1207         memset(m, 0, sizeof(struct dm_thin_new_mapping));
1208         INIT_LIST_HEAD(&m->list);
1209         m->bio = NULL;
1210
1211         pool->next_mapping = NULL;
1212
1213         return m;
1214 }
1215
1216 static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
1217                     sector_t begin, sector_t end)
1218 {
1219         int r;
1220         struct dm_io_region to;
1221
1222         to.bdev = tc->pool_dev->bdev;
1223         to.sector = begin;
1224         to.count = end - begin;
1225
1226         r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
1227         if (r < 0) {
1228                 DMERR_LIMIT("dm_kcopyd_zero() failed");
1229                 copy_complete(1, 1, m);
1230         }
1231 }
1232
1233 static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
1234                                       dm_block_t data_begin,
1235                                       struct dm_thin_new_mapping *m)
1236 {
1237         struct pool *pool = tc->pool;
1238         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1239
1240         h->overwrite_mapping = m;
1241         m->bio = bio;
1242         save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1243         inc_all_io_entry(pool, bio);
1244         remap_and_issue(tc, bio, data_begin);
1245 }
1246
1247 /*
1248  * A partial copy also needs to zero the uncopied region.
1249  */
1250 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
1251                           struct dm_dev *origin, dm_block_t data_origin,
1252                           dm_block_t data_dest,
1253                           struct dm_bio_prison_cell *cell, struct bio *bio,
1254                           sector_t len)
1255 {
1256         int r;
1257         struct pool *pool = tc->pool;
1258         struct dm_thin_new_mapping *m = get_next_mapping(pool);
1259
1260         m->tc = tc;
1261         m->virt_begin = virt_block;
1262         m->virt_end = virt_block + 1u;
1263         m->data_block = data_dest;
1264         m->cell = cell;
1265
1266         /*
1267          * quiesce action + copy action + an extra reference held for the
1268          * duration of this function (we may need to inc later for a
1269          * partial zero).
1270          */
1271         atomic_set(&m->prepare_actions, 3);
1272
1273         if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
1274                 complete_mapping_preparation(m); /* already quiesced */
1275
1276         /*
1277          * IO to pool_dev remaps to the pool target's data_dev.
1278          *
1279          * If the whole block of data is being overwritten, we can issue the
1280          * bio immediately. Otherwise we use kcopyd to clone the data first.
1281          */
1282         if (io_overwrites_block(pool, bio))
1283                 remap_and_issue_overwrite(tc, bio, data_dest, m);
1284         else {
1285                 struct dm_io_region from, to;
1286
1287                 from.bdev = origin->bdev;
1288                 from.sector = data_origin * pool->sectors_per_block;
1289                 from.count = len;
1290
1291                 to.bdev = tc->pool_dev->bdev;
1292                 to.sector = data_dest * pool->sectors_per_block;
1293                 to.count = len;
1294
1295                 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
1296                                    0, copy_complete, m);
1297                 if (r < 0) {
1298                         DMERR_LIMIT("dm_kcopyd_copy() failed");
1299                         copy_complete(1, 1, m);
1300
1301                         /*
1302                          * We allow the zero to be issued, to simplify the
1303                          * error path.  Otherwise we'd need to start
1304                          * worrying about decrementing the prepare_actions
1305                          * counter.
1306                          */
1307                 }
1308
1309                 /*
1310                  * Do we need to zero a tail region?
1311                  */
1312                 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
1313                         atomic_inc(&m->prepare_actions);
1314                         ll_zero(tc, m,
1315                                 data_dest * pool->sectors_per_block + len,
1316                                 (data_dest + 1) * pool->sectors_per_block);
1317                 }
1318         }
1319
1320         complete_mapping_preparation(m); /* drop our ref */
1321 }
1322
1323 static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1324                                    dm_block_t data_origin, dm_block_t data_dest,
1325                                    struct dm_bio_prison_cell *cell, struct bio *bio)
1326 {
1327         schedule_copy(tc, virt_block, tc->pool_dev,
1328                       data_origin, data_dest, cell, bio,
1329                       tc->pool->sectors_per_block);
1330 }
1331
1332 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
1333                           dm_block_t data_block, struct dm_bio_prison_cell *cell,
1334                           struct bio *bio)
1335 {
1336         struct pool *pool = tc->pool;
1337         struct dm_thin_new_mapping *m = get_next_mapping(pool);
1338
1339         atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
1340         m->tc = tc;
1341         m->virt_begin = virt_block;
1342         m->virt_end = virt_block + 1u;
1343         m->data_block = data_block;
1344         m->cell = cell;
1345
1346         /*
1347          * If the whole block of data is being overwritten or we are not
1348          * zeroing pre-existing data, we can issue the bio immediately.
1349          * Otherwise we use kcopyd to zero the data first.
1350          */
1351         if (pool->pf.zero_new_blocks) {
1352                 if (io_overwrites_block(pool, bio))
1353                         remap_and_issue_overwrite(tc, bio, data_block, m);
1354                 else
1355                         ll_zero(tc, m, data_block * pool->sectors_per_block,
1356                                 (data_block + 1) * pool->sectors_per_block);
1357         } else
1358                 process_prepared_mapping(m);
1359 }
1360
1361 static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1362                                    dm_block_t data_dest,
1363                                    struct dm_bio_prison_cell *cell, struct bio *bio)
1364 {
1365         struct pool *pool = tc->pool;
1366         sector_t virt_block_begin = virt_block * pool->sectors_per_block;
1367         sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
1368
1369         if (virt_block_end <= tc->origin_size)
1370                 schedule_copy(tc, virt_block, tc->origin_dev,
1371                               virt_block, data_dest, cell, bio,
1372                               pool->sectors_per_block);
1373
1374         else if (virt_block_begin < tc->origin_size)
1375                 schedule_copy(tc, virt_block, tc->origin_dev,
1376                               virt_block, data_dest, cell, bio,
1377                               tc->origin_size - virt_block_begin);
1378
1379         else
1380                 schedule_zero(tc, virt_block, data_dest, cell, bio);
1381 }
1382
1383 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1384
1385 static void check_for_space(struct pool *pool)
1386 {
1387         int r;
1388         dm_block_t nr_free;
1389
1390         if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
1391                 return;
1392
1393         r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
1394         if (r)
1395                 return;
1396
1397         if (nr_free)
1398                 set_pool_mode(pool, PM_WRITE);
1399 }
1400
1401 /*
1402  * A non-zero return indicates read_only or fail_io mode.
1403  * Many callers don't care about the return value.
1404  */
1405 static int commit(struct pool *pool)
1406 {
1407         int r;
1408
1409         if (get_pool_mode(pool) >= PM_READ_ONLY)
1410                 return -EINVAL;
1411
1412         r = dm_pool_commit_metadata(pool->pmd);
1413         if (r)
1414                 metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
1415         else
1416                 check_for_space(pool);
1417
1418         return r;
1419 }
1420
1421 static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1422 {
1423         unsigned long flags;
1424
1425         if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1426                 DMWARN("%s: reached low water mark for data device: sending event.",
1427                        dm_device_name(pool->pool_md));
1428                 spin_lock_irqsave(&pool->lock, flags);
1429                 pool->low_water_triggered = true;
1430                 spin_unlock_irqrestore(&pool->lock, flags);
1431                 dm_table_event(pool->ti->table);
1432         }
1433 }
1434
1435 static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1436 {
1437         int r;
1438         dm_block_t free_blocks;
1439         struct pool *pool = tc->pool;
1440
1441         if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
1442                 return -EINVAL;
1443
1444         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1445         if (r) {
1446                 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
1447                 return r;
1448         }
1449
1450         check_low_water_mark(pool, free_blocks);
1451
1452         if (!free_blocks) {
1453                 /*
1454                  * Try to commit to see if that will free up some
1455                  * more space.
1456                  */
1457                 r = commit(pool);
1458                 if (r)
1459                         return r;
1460
1461                 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1462                 if (r) {
1463                         metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
1464                         return r;
1465                 }
1466
1467                 if (!free_blocks) {
1468                         set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
1469                         return -ENOSPC;
1470                 }
1471         }
1472
1473         r = dm_pool_alloc_data_block(pool->pmd, result);
1474         if (r) {
1475                 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
1476                 return r;
1477         }
1478
1479         return 0;
1480 }
1481
1482 /*
1483  * If we have run out of space, queue bios until the device is
1484  * resumed, presumably after having been reloaded with more space.
1485  */
1486 static void retry_on_resume(struct bio *bio)
1487 {
1488         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1489         struct thin_c *tc = h->tc;
1490         unsigned long flags;
1491
1492         spin_lock_irqsave(&tc->lock, flags);
1493         bio_list_add(&tc->retry_on_resume_list, bio);
1494         spin_unlock_irqrestore(&tc->lock, flags);
1495 }
1496
1497 static int should_error_unserviceable_bio(struct pool *pool)
1498 {
1499         enum pool_mode m = get_pool_mode(pool);
1500
1501         switch (m) {
1502         case PM_WRITE:
1503                 /* Shouldn't get here */
1504                 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
1505                 return -EIO;
1506
1507         case PM_OUT_OF_DATA_SPACE:
1508                 return pool->pf.error_if_no_space ? -ENOSPC : 0;
1509
1510         case PM_READ_ONLY:
1511         case PM_FAIL:
1512                 return -EIO;
1513         default:
1514                 /* Shouldn't get here */
1515                 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
1516                 return -EIO;
1517         }
1518 }
1519
1520 static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1521 {
1522         int error = should_error_unserviceable_bio(pool);
1523
1524         if (error) {
1525                 bio->bi_error = error;
1526                 bio_endio(bio);
1527         } else
1528                 retry_on_resume(bio);
1529 }
1530
1531 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
1532 {
1533         struct bio *bio;
1534         struct bio_list bios;
1535         int error;
1536
1537         error = should_error_unserviceable_bio(pool);
1538         if (error) {
1539                 cell_error_with_code(pool, cell, error);
1540                 return;
1541         }
1542
1543         bio_list_init(&bios);
1544         cell_release(pool, cell, &bios);
1545
1546         while ((bio = bio_list_pop(&bios)))
1547                 retry_on_resume(bio);
1548 }
1549
1550 static void process_discard_cell_no_passdown(struct thin_c *tc,
1551                                              struct dm_bio_prison_cell *virt_cell)
1552 {
1553         struct pool *pool = tc->pool;
1554         struct dm_thin_new_mapping *m = get_next_mapping(pool);
1555
1556         /*
1557          * We don't need to lock the data blocks, since there's no
1558          * passdown.  We only lock data blocks for allocation and breaking sharing.
1559          */
1560         m->tc = tc;
1561         m->virt_begin = virt_cell->key.block_begin;
1562         m->virt_end = virt_cell->key.block_end;
1563         m->cell = virt_cell;
1564         m->bio = virt_cell->holder;
1565
1566         if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1567                 pool->process_prepared_discard(m);
1568 }
1569
1570 static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
1571                                  struct bio *bio)
1572 {
1573         struct pool *pool = tc->pool;
1574
1575         int r;
1576         bool maybe_shared;
1577         struct dm_cell_key data_key;
1578         struct dm_bio_prison_cell *data_cell;
1579         struct dm_thin_new_mapping *m;
1580         dm_block_t virt_begin, virt_end, data_begin;
1581
1582         while (begin != end) {
1583                 r = ensure_next_mapping(pool);
1584                 if (r)
1585                         /* we did our best */
1586                         return;
1587
1588                 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
1589                                               &data_begin, &maybe_shared);
1590                 if (r)
1591                         /*
1592                          * Silently fail, letting any mappings we've
1593                          * created complete.
1594                          */
1595                         break;
1596
1597                 build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
1598                 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
1599                         /* contention, we'll give up with this range */
1600                         begin = virt_end;
1601                         continue;
1602                 }
1603
1604                 /*
1605                  * IO may still be going to the destination block.  We must
1606                  * quiesce before we can do the removal.
1607                  */
1608                 m = get_next_mapping(pool);
1609                 m->tc = tc;
1610                 m->maybe_shared = maybe_shared;
1611                 m->virt_begin = virt_begin;
1612                 m->virt_end = virt_end;
1613                 m->data_block = data_begin;
1614                 m->cell = data_cell;
1615                 m->bio = bio;
1616
1617                 /*
1618                  * The parent bio must not complete before sub discard bios are
1619                  * chained to it (see end_discard's bio_chain)!
1620                  *
1621                  * This per-mapping bi_remaining increment is paired with
1622                  * the implicit decrement that occurs via bio_endio() in
1623                  * end_discard().
1624                  */
1625                 bio_inc_remaining(bio);
1626                 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1627                         pool->process_prepared_discard(m);
1628
1629                 begin = virt_end;
1630         }
1631 }
1632
1633 static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
1634 {
1635         struct bio *bio = virt_cell->holder;
1636         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1637
1638         /*
1639          * The virt_cell will only get freed once the origin bio completes.
1640          * This means it will remain locked while all the individual
1641          * passdown bios are in flight.
1642          */
1643         h->cell = virt_cell;
1644         break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
1645
1646         /*
1647          * We complete the bio now, knowing that the bi_remaining field
1648          * will prevent completion until the sub range discards have
1649          * completed.
1650          */
1651         bio_endio(bio);
1652 }
1653
1654 static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1655 {
1656         dm_block_t begin, end;
1657         struct dm_cell_key virt_key;
1658         struct dm_bio_prison_cell *virt_cell;
1659
1660         get_bio_block_range(tc, bio, &begin, &end);
1661         if (begin == end) {
1662                 /*
1663                  * The discard covers less than a block.
1664                  */
1665                 bio_endio(bio);
1666                 return;
1667         }
1668
1669         build_key(tc->td, VIRTUAL, begin, end, &virt_key);
1670         if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
1671                 /*
1672                  * Potential starvation issue: We're relying on the
1673                  * fs/application being well behaved, and not trying to
1674                  * send IO to a region at the same time as discarding it.
1675                  * If they do this persistently then it's possible this
1676                  * cell will never be granted.
1677                  */
1678                 return;
1679
1680         tc->pool->process_discard_cell(tc, virt_cell);
1681 }
1682
1683 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1684                           struct dm_cell_key *key,
1685                           struct dm_thin_lookup_result *lookup_result,
1686                           struct dm_bio_prison_cell *cell)
1687 {
1688         int r;
1689         dm_block_t data_block;
1690         struct pool *pool = tc->pool;
1691
1692         r = alloc_data_block(tc, &data_block);
1693         switch (r) {
1694         case 0:
1695                 schedule_internal_copy(tc, block, lookup_result->block,
1696                                        data_block, cell, bio);
1697                 break;
1698
1699         case -ENOSPC:
1700                 retry_bios_on_resume(pool, cell);
1701                 break;
1702
1703         default:
1704                 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1705                             __func__, r);
1706                 cell_error(pool, cell);
1707                 break;
1708         }
1709 }
1710
1711 static void __remap_and_issue_shared_cell(void *context,
1712                                           struct dm_bio_prison_cell *cell)
1713 {
1714         struct remap_info *info = context;
1715         struct bio *bio;
1716
1717         while ((bio = bio_list_pop(&cell->bios))) {
1718                 if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) ||
1719                     bio_op(bio) == REQ_OP_DISCARD)
1720                         bio_list_add(&info->defer_bios, bio);
1721                 else {
1722                         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));;
1723
1724                         h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
1725                         inc_all_io_entry(info->tc->pool, bio);
1726                         bio_list_add(&info->issue_bios, bio);
1727                 }
1728         }
1729 }
1730
1731 static void remap_and_issue_shared_cell(struct thin_c *tc,
1732                                         struct dm_bio_prison_cell *cell,
1733                                         dm_block_t block)
1734 {
1735         struct bio *bio;
1736         struct remap_info info;
1737
1738         info.tc = tc;
1739         bio_list_init(&info.defer_bios);
1740         bio_list_init(&info.issue_bios);
1741
1742         cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
1743                            &info, cell);
1744
1745         while ((bio = bio_list_pop(&info.defer_bios)))
1746                 thin_defer_bio(tc, bio);
1747
1748         while ((bio = bio_list_pop(&info.issue_bios)))
1749                 remap_and_issue(tc, bio, block);
1750 }
1751
1752 static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1753                                dm_block_t block,
1754                                struct dm_thin_lookup_result *lookup_result,
1755                                struct dm_bio_prison_cell *virt_cell)
1756 {
1757         struct dm_bio_prison_cell *data_cell;
1758         struct pool *pool = tc->pool;
1759         struct dm_cell_key key;
1760
1761         /*
1762          * If cell is already occupied, then sharing is already in the process
1763          * of being broken so we have nothing further to do here.
1764          */
1765         build_data_key(tc->td, lookup_result->block, &key);
1766         if (bio_detain(pool, &key, bio, &data_cell)) {
1767                 cell_defer_no_holder(tc, virt_cell);
1768                 return;
1769         }
1770
1771         if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
1772                 break_sharing(tc, bio, block, &key, lookup_result, data_cell);
1773                 cell_defer_no_holder(tc, virt_cell);
1774         } else {
1775                 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1776
1777                 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
1778                 inc_all_io_entry(pool, bio);
1779                 remap_and_issue(tc, bio, lookup_result->block);
1780
1781                 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
1782                 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
1783         }
1784 }
1785
1786 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1787                             struct dm_bio_prison_cell *cell)
1788 {
1789         int r;
1790         dm_block_t data_block;
1791         struct pool *pool = tc->pool;
1792
1793         /*
1794          * Remap empty bios (flushes) immediately, without provisioning.
1795          */
1796         if (!bio->bi_iter.bi_size) {
1797                 inc_all_io_entry(pool, bio);
1798                 cell_defer_no_holder(tc, cell);
1799
1800                 remap_and_issue(tc, bio, 0);
1801                 return;
1802         }
1803
1804         /*
1805          * Fill read bios with zeroes and complete them immediately.
1806          */
1807         if (bio_data_dir(bio) == READ) {
1808                 zero_fill_bio(bio);
1809                 cell_defer_no_holder(tc, cell);
1810                 bio_endio(bio);
1811                 return;
1812         }
1813
1814         r = alloc_data_block(tc, &data_block);
1815         switch (r) {
1816         case 0:
1817                 if (tc->origin_dev)
1818                         schedule_external_copy(tc, block, data_block, cell, bio);
1819                 else
1820                         schedule_zero(tc, block, data_block, cell, bio);
1821                 break;
1822
1823         case -ENOSPC:
1824                 retry_bios_on_resume(pool, cell);
1825                 break;
1826
1827         default:
1828                 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1829                             __func__, r);
1830                 cell_error(pool, cell);
1831                 break;
1832         }
1833 }
1834
1835 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1836 {
1837         int r;
1838         struct pool *pool = tc->pool;
1839         struct bio *bio = cell->holder;
1840         dm_block_t block = get_bio_block(tc, bio);
1841         struct dm_thin_lookup_result lookup_result;
1842
1843         if (tc->requeue_mode) {
1844                 cell_requeue(pool, cell);
1845                 return;
1846         }
1847
1848         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1849         switch (r) {
1850         case 0:
1851                 if (lookup_result.shared)
1852                         process_shared_bio(tc, bio, block, &lookup_result, cell);
1853                 else {
1854                         inc_all_io_entry(pool, bio);
1855                         remap_and_issue(tc, bio, lookup_result.block);
1856                         inc_remap_and_issue_cell(tc, cell, lookup_result.block);
1857                 }
1858                 break;
1859
1860         case -ENODATA:
1861                 if (bio_data_dir(bio) == READ && tc->origin_dev) {
1862                         inc_all_io_entry(pool, bio);
1863                         cell_defer_no_holder(tc, cell);
1864
1865                         if (bio_end_sector(bio) <= tc->origin_size)
1866                                 remap_to_origin_and_issue(tc, bio);
1867
1868                         else if (bio->bi_iter.bi_sector < tc->origin_size) {
1869                                 zero_fill_bio(bio);
1870                                 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1871                                 remap_to_origin_and_issue(tc, bio);
1872
1873                         } else {
1874                                 zero_fill_bio(bio);
1875                                 bio_endio(bio);
1876                         }
1877                 } else
1878                         provision_block(tc, bio, block, cell);
1879                 break;
1880
1881         default:
1882                 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1883                             __func__, r);
1884                 cell_defer_no_holder(tc, cell);
1885                 bio_io_error(bio);
1886                 break;
1887         }
1888 }
1889
1890 static void process_bio(struct thin_c *tc, struct bio *bio)
1891 {
1892         struct pool *pool = tc->pool;
1893         dm_block_t block = get_bio_block(tc, bio);
1894         struct dm_bio_prison_cell *cell;
1895         struct dm_cell_key key;
1896
1897         /*
1898          * If cell is already occupied, then the block is already
1899          * being provisioned so we have nothing further to do here.
1900          */
1901         build_virtual_key(tc->td, block, &key);
1902         if (bio_detain(pool, &key, bio, &cell))
1903                 return;
1904
1905         process_cell(tc, cell);
1906 }
1907
1908 static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
1909                                     struct dm_bio_prison_cell *cell)
1910 {
1911         int r;
1912         int rw = bio_data_dir(bio);
1913         dm_block_t block = get_bio_block(tc, bio);
1914         struct dm_thin_lookup_result lookup_result;
1915
1916         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1917         switch (r) {
1918         case 0:
1919                 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
1920                         handle_unserviceable_bio(tc->pool, bio);
1921                         if (cell)
1922                                 cell_defer_no_holder(tc, cell);
1923                 } else {
1924                         inc_all_io_entry(tc->pool, bio);
1925                         remap_and_issue(tc, bio, lookup_result.block);
1926                         if (cell)
1927                                 inc_remap_and_issue_cell(tc, cell, lookup_result.block);
1928                 }
1929                 break;
1930
1931         case -ENODATA:
1932                 if (cell)
1933                         cell_defer_no_holder(tc, cell);
1934                 if (rw != READ) {
1935                         handle_unserviceable_bio(tc->pool, bio);
1936                         break;
1937                 }
1938
1939                 if (tc->origin_dev) {
1940                         inc_all_io_entry(tc->pool, bio);
1941                         remap_to_origin_and_issue(tc, bio);
1942                         break;
1943                 }
1944
1945                 zero_fill_bio(bio);
1946                 bio_endio(bio);
1947                 break;
1948
1949         default:
1950                 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1951                             __func__, r);
1952                 if (cell)
1953                         cell_defer_no_holder(tc, cell);
1954                 bio_io_error(bio);
1955                 break;
1956         }
1957 }
1958
1959 static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
1960 {
1961         __process_bio_read_only(tc, bio, NULL);
1962 }
1963
1964 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1965 {
1966         __process_bio_read_only(tc, cell->holder, cell);
1967 }
1968
1969 static void process_bio_success(struct thin_c *tc, struct bio *bio)
1970 {
1971         bio_endio(bio);
1972 }
1973
1974 static void process_bio_fail(struct thin_c *tc, struct bio *bio)
1975 {
1976         bio_io_error(bio);
1977 }
1978
1979 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1980 {
1981         cell_success(tc->pool, cell);
1982 }
1983
1984 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1985 {
1986         cell_error(tc->pool, cell);
1987 }
1988
1989 /*
1990  * FIXME: should we also commit due to size of transaction, measured in
1991  * metadata blocks?
1992  */
1993 static int need_commit_due_to_time(struct pool *pool)
1994 {
1995         return !time_in_range(jiffies, pool->last_commit_jiffies,
1996                               pool->last_commit_jiffies + COMMIT_PERIOD);
1997 }
1998
1999 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
2000 #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
2001
2002 static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
2003 {
2004         struct rb_node **rbp, *parent;
2005         struct dm_thin_endio_hook *pbd;
2006         sector_t bi_sector = bio->bi_iter.bi_sector;
2007
2008         rbp = &tc->sort_bio_list.rb_node;
2009         parent = NULL;
2010         while (*rbp) {
2011                 parent = *rbp;
2012                 pbd = thin_pbd(parent);
2013
2014                 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
2015                         rbp = &(*rbp)->rb_left;
2016                 else
2017                         rbp = &(*rbp)->rb_right;
2018         }
2019
2020         pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2021         rb_link_node(&pbd->rb_node, parent, rbp);
2022         rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
2023 }
2024
2025 static void __extract_sorted_bios(struct thin_c *tc)
2026 {
2027         struct rb_node *node;
2028         struct dm_thin_endio_hook *pbd;
2029         struct bio *bio;
2030
2031         for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
2032                 pbd = thin_pbd(node);
2033                 bio = thin_bio(pbd);
2034
2035                 bio_list_add(&tc->deferred_bio_list, bio);
2036                 rb_erase(&pbd->rb_node, &tc->sort_bio_list);
2037         }
2038
2039         WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
2040 }
2041
2042 static void __sort_thin_deferred_bios(struct thin_c *tc)
2043 {
2044         struct bio *bio;
2045         struct bio_list bios;
2046
2047         bio_list_init(&bios);
2048         bio_list_merge(&bios, &tc->deferred_bio_list);
2049         bio_list_init(&tc->deferred_bio_list);
2050
2051         /* Sort deferred_bio_list using rb-tree */
2052         while ((bio = bio_list_pop(&bios)))
2053                 __thin_bio_rb_add(tc, bio);
2054
2055         /*
2056          * Transfer the sorted bios in sort_bio_list back to
2057          * deferred_bio_list to allow lockless submission of
2058          * all bios.
2059          */
2060         __extract_sorted_bios(tc);
2061 }
2062
2063 static void process_thin_deferred_bios(struct thin_c *tc)
2064 {
2065         struct pool *pool = tc->pool;
2066         unsigned long flags;
2067         struct bio *bio;
2068         struct bio_list bios;
2069         struct blk_plug plug;
2070         unsigned count = 0;
2071
2072         if (tc->requeue_mode) {
2073                 error_thin_bio_list(tc, &tc->deferred_bio_list, DM_ENDIO_REQUEUE);
2074                 return;
2075         }
2076
2077         bio_list_init(&bios);
2078
2079         spin_lock_irqsave(&tc->lock, flags);
2080
2081         if (bio_list_empty(&tc->deferred_bio_list)) {
2082                 spin_unlock_irqrestore(&tc->lock, flags);
2083                 return;
2084         }
2085
2086         __sort_thin_deferred_bios(tc);
2087
2088         bio_list_merge(&bios, &tc->deferred_bio_list);
2089         bio_list_init(&tc->deferred_bio_list);
2090
2091         spin_unlock_irqrestore(&tc->lock, flags);
2092
2093         blk_start_plug(&plug);
2094         while ((bio = bio_list_pop(&bios))) {
2095                 /*
2096                  * If we've got no free new_mapping structs, and processing
2097                  * this bio might require one, we pause until there are some
2098                  * prepared mappings to process.
2099                  */
2100                 if (ensure_next_mapping(pool)) {
2101                         spin_lock_irqsave(&tc->lock, flags);
2102                         bio_list_add(&tc->deferred_bio_list, bio);
2103                         bio_list_merge(&tc->deferred_bio_list, &bios);
2104                         spin_unlock_irqrestore(&tc->lock, flags);
2105                         break;
2106                 }
2107
2108                 if (bio_op(bio) == REQ_OP_DISCARD)
2109                         pool->process_discard(tc, bio);
2110                 else
2111                         pool->process_bio(tc, bio);
2112
2113                 if ((count++ & 127) == 0) {
2114                         throttle_work_update(&pool->throttle);
2115                         dm_pool_issue_prefetches(pool->pmd);
2116                 }
2117         }
2118         blk_finish_plug(&plug);
2119 }
2120
2121 static int cmp_cells(const void *lhs, const void *rhs)
2122 {
2123         struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
2124         struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
2125
2126         BUG_ON(!lhs_cell->holder);
2127         BUG_ON(!rhs_cell->holder);
2128
2129         if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
2130                 return -1;
2131
2132         if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
2133                 return 1;
2134
2135         return 0;
2136 }
2137
2138 static unsigned sort_cells(struct pool *pool, struct list_head *cells)
2139 {
2140         unsigned count = 0;
2141         struct dm_bio_prison_cell *cell, *tmp;
2142
2143         list_for_each_entry_safe(cell, tmp, cells, user_list) {
2144                 if (count >= CELL_SORT_ARRAY_SIZE)
2145                         break;
2146
2147                 pool->cell_sort_array[count++] = cell;
2148                 list_del(&cell->user_list);
2149         }
2150
2151         sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
2152
2153         return count;
2154 }
2155
2156 static void process_thin_deferred_cells(struct thin_c *tc)
2157 {
2158         struct pool *pool = tc->pool;
2159         unsigned long flags;
2160         struct list_head cells;
2161         struct dm_bio_prison_cell *cell;
2162         unsigned i, j, count;
2163
2164         INIT_LIST_HEAD(&cells);
2165
2166         spin_lock_irqsave(&tc->lock, flags);
2167         list_splice_init(&tc->deferred_cells, &cells);
2168         spin_unlock_irqrestore(&tc->lock, flags);
2169
2170         if (list_empty(&cells))
2171                 return;
2172
2173         do {
2174                 count = sort_cells(tc->pool, &cells);
2175
2176                 for (i = 0; i < count; i++) {
2177                         cell = pool->cell_sort_array[i];
2178                         BUG_ON(!cell->holder);
2179
2180                         /*
2181                          * If we've got no free new_mapping structs, and processing
2182                          * this bio might require one, we pause until there are some
2183                          * prepared mappings to process.
2184                          */
2185                         if (ensure_next_mapping(pool)) {
2186                                 for (j = i; j < count; j++)
2187                                         list_add(&pool->cell_sort_array[j]->user_list, &cells);
2188
2189                                 spin_lock_irqsave(&tc->lock, flags);
2190                                 list_splice(&cells, &tc->deferred_cells);
2191                                 spin_unlock_irqrestore(&tc->lock, flags);
2192                                 return;
2193                         }
2194
2195                         if (bio_op(cell->holder) == REQ_OP_DISCARD)
2196                                 pool->process_discard_cell(tc, cell);
2197                         else
2198                                 pool->process_cell(tc, cell);
2199                 }
2200         } while (!list_empty(&cells));
2201 }
2202
2203 static void thin_get(struct thin_c *tc);
2204 static void thin_put(struct thin_c *tc);
2205
2206 /*
2207  * We can't hold rcu_read_lock() around code that can block.  So we
2208  * find a thin with the rcu lock held; bump a refcount; then drop
2209  * the lock.
2210  */
2211 static struct thin_c *get_first_thin(struct pool *pool)
2212 {
2213         struct thin_c *tc = NULL;
2214
2215         rcu_read_lock();
2216         if (!list_empty(&pool->active_thins)) {
2217                 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
2218                 thin_get(tc);
2219         }
2220         rcu_read_unlock();
2221
2222         return tc;
2223 }
2224
2225 static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
2226 {
2227         struct thin_c *old_tc = tc;
2228
2229         rcu_read_lock();
2230         list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
2231                 thin_get(tc);
2232                 thin_put(old_tc);
2233                 rcu_read_unlock();
2234                 return tc;
2235         }
2236         thin_put(old_tc);
2237         rcu_read_unlock();
2238
2239         return NULL;
2240 }
2241
2242 static void process_deferred_bios(struct pool *pool)
2243 {
2244         unsigned long flags;
2245         struct bio *bio;
2246         struct bio_list bios;
2247         struct thin_c *tc;
2248
2249         tc = get_first_thin(pool);
2250         while (tc) {
2251                 process_thin_deferred_cells(tc);
2252                 process_thin_deferred_bios(tc);
2253                 tc = get_next_thin(pool, tc);
2254         }
2255
2256         /*
2257          * If there are any deferred flush bios, we must commit
2258          * the metadata before issuing them.
2259          */
2260         bio_list_init(&bios);
2261         spin_lock_irqsave(&pool->lock, flags);
2262         bio_list_merge(&bios, &pool->deferred_flush_bios);
2263         bio_list_init(&pool->deferred_flush_bios);
2264         spin_unlock_irqrestore(&pool->lock, flags);
2265
2266         if (bio_list_empty(&bios) &&
2267             !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
2268                 return;
2269
2270         if (commit(pool)) {
2271                 while ((bio = bio_list_pop(&bios)))
2272                         bio_io_error(bio);
2273                 return;
2274         }
2275         pool->last_commit_jiffies = jiffies;
2276
2277         while ((bio = bio_list_pop(&bios)))
2278                 generic_make_request(bio);
2279 }
2280
2281 static void do_worker(struct work_struct *ws)
2282 {
2283         struct pool *pool = container_of(ws, struct pool, worker);
2284
2285         throttle_work_start(&pool->throttle);
2286         dm_pool_issue_prefetches(pool->pmd);
2287         throttle_work_update(&pool->throttle);
2288         process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
2289         throttle_work_update(&pool->throttle);
2290         process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
2291         throttle_work_update(&pool->throttle);
2292         process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2);
2293         throttle_work_update(&pool->throttle);
2294         process_deferred_bios(pool);
2295         throttle_work_complete(&pool->throttle);
2296 }
2297
2298 /*
2299  * We want to commit periodically so that not too much
2300  * unwritten data builds up.
2301  */
2302 static void do_waker(struct work_struct *ws)
2303 {
2304         struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
2305         wake_worker(pool);
2306         queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
2307 }
2308
2309 static void notify_of_pool_mode_change_to_oods(struct pool *pool);
2310
2311 /*
2312  * We're holding onto IO to allow userland time to react.  After the
2313  * timeout either the pool will have been resized (and thus back in
2314  * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
2315  */
2316 static void do_no_space_timeout(struct work_struct *ws)
2317 {
2318         struct pool *pool = container_of(to_delayed_work(ws), struct pool,
2319                                          no_space_timeout);
2320
2321         if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
2322                 pool->pf.error_if_no_space = true;
2323                 notify_of_pool_mode_change_to_oods(pool);
2324                 error_retry_list_with_code(pool, -ENOSPC);
2325         }
2326 }
2327
2328 /*----------------------------------------------------------------*/
2329
2330 struct pool_work {
2331         struct work_struct worker;
2332         struct completion complete;
2333 };
2334
2335 static struct pool_work *to_pool_work(struct work_struct *ws)
2336 {
2337         return container_of(ws, struct pool_work, worker);
2338 }
2339
2340 static void pool_work_complete(struct pool_work *pw)
2341 {
2342         complete(&pw->complete);
2343 }
2344
2345 static void pool_work_wait(struct pool_work *pw, struct pool *pool,
2346                            void (*fn)(struct work_struct *))
2347 {
2348         INIT_WORK_ONSTACK(&pw->worker, fn);
2349         init_completion(&pw->complete);
2350         queue_work(pool->wq, &pw->worker);
2351         wait_for_completion(&pw->complete);
2352 }
2353
2354 /*----------------------------------------------------------------*/
2355
2356 struct noflush_work {
2357         struct pool_work pw;
2358         struct thin_c *tc;
2359 };
2360
2361 static struct noflush_work *to_noflush(struct work_struct *ws)
2362 {
2363         return container_of(to_pool_work(ws), struct noflush_work, pw);
2364 }
2365
2366 static void do_noflush_start(struct work_struct *ws)
2367 {
2368         struct noflush_work *w = to_noflush(ws);
2369         w->tc->requeue_mode = true;
2370         requeue_io(w->tc);
2371         pool_work_complete(&w->pw);
2372 }
2373
2374 static void do_noflush_stop(struct work_struct *ws)
2375 {
2376         struct noflush_work *w = to_noflush(ws);
2377         w->tc->requeue_mode = false;
2378         pool_work_complete(&w->pw);
2379 }
2380
2381 static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
2382 {
2383         struct noflush_work w;
2384
2385         w.tc = tc;
2386         pool_work_wait(&w.pw, tc->pool, fn);
2387 }
2388
2389 /*----------------------------------------------------------------*/
2390
2391 static enum pool_mode get_pool_mode(struct pool *pool)
2392 {
2393         return pool->pf.mode;
2394 }
2395
2396 static void notify_of_pool_mode_change(struct pool *pool, const char *new_mode)
2397 {
2398         dm_table_event(pool->ti->table);
2399         DMINFO("%s: switching pool to %s mode",
2400                dm_device_name(pool->pool_md), new_mode);
2401 }
2402
2403 static void notify_of_pool_mode_change_to_oods(struct pool *pool)
2404 {
2405         if (!pool->pf.error_if_no_space)
2406                 notify_of_pool_mode_change(pool, "out-of-data-space (queue IO)");
2407         else
2408                 notify_of_pool_mode_change(pool, "out-of-data-space (error IO)");
2409 }
2410
2411 static bool passdown_enabled(struct pool_c *pt)
2412 {
2413         return pt->adjusted_pf.discard_passdown;
2414 }
2415
2416 static void set_discard_callbacks(struct pool *pool)
2417 {
2418         struct pool_c *pt = pool->ti->private;
2419
2420         if (passdown_enabled(pt)) {
2421                 pool->process_discard_cell = process_discard_cell_passdown;
2422                 pool->process_prepared_discard = process_prepared_discard_passdown_pt1;
2423                 pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2;
2424         } else {
2425                 pool->process_discard_cell = process_discard_cell_no_passdown;
2426                 pool->process_prepared_discard = process_prepared_discard_no_passdown;
2427         }
2428 }
2429
2430 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
2431 {
2432         struct pool_c *pt = pool->ti->private;
2433         bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
2434         enum pool_mode old_mode = get_pool_mode(pool);
2435         unsigned long no_space_timeout = ACCESS_ONCE(no_space_timeout_secs) * HZ;
2436
2437         /*
2438          * Never allow the pool to transition to PM_WRITE mode if user
2439          * intervention is required to verify metadata and data consistency.
2440          */
2441         if (new_mode == PM_WRITE && needs_check) {
2442                 DMERR("%s: unable to switch pool to write mode until repaired.",
2443                       dm_device_name(pool->pool_md));
2444                 if (old_mode != new_mode)
2445                         new_mode = old_mode;
2446                 else
2447                         new_mode = PM_READ_ONLY;
2448         }
2449         /*
2450          * If we were in PM_FAIL mode, rollback of metadata failed.  We're
2451          * not going to recover without a thin_repair.  So we never let the
2452          * pool move out of the old mode.
2453          */
2454         if (old_mode == PM_FAIL)
2455                 new_mode = old_mode;
2456
2457         switch (new_mode) {
2458         case PM_FAIL:
2459                 if (old_mode != new_mode)
2460                         notify_of_pool_mode_change(pool, "failure");
2461                 dm_pool_metadata_read_only(pool->pmd);
2462                 pool->process_bio = process_bio_fail;
2463                 pool->process_discard = process_bio_fail;
2464                 pool->process_cell = process_cell_fail;
2465                 pool->process_discard_cell = process_cell_fail;
2466                 pool->process_prepared_mapping = process_prepared_mapping_fail;
2467                 pool->process_prepared_discard = process_prepared_discard_fail;
2468
2469                 error_retry_list(pool);
2470                 break;
2471
2472         case PM_READ_ONLY:
2473                 if (old_mode != new_mode)
2474                         notify_of_pool_mode_change(pool, "read-only");
2475                 dm_pool_metadata_read_only(pool->pmd);
2476                 pool->process_bio = process_bio_read_only;
2477                 pool->process_discard = process_bio_success;
2478                 pool->process_cell = process_cell_read_only;
2479                 pool->process_discard_cell = process_cell_success;
2480                 pool->process_prepared_mapping = process_prepared_mapping_fail;
2481                 pool->process_prepared_discard = process_prepared_discard_success;
2482
2483                 error_retry_list(pool);
2484                 break;
2485
2486         case PM_OUT_OF_DATA_SPACE:
2487                 /*
2488                  * Ideally we'd never hit this state; the low water mark
2489                  * would trigger userland to extend the pool before we
2490                  * completely run out of data space.  However, many small
2491                  * IOs to unprovisioned space can consume data space at an
2492                  * alarming rate.  Adjust your low water mark if you're
2493                  * frequently seeing this mode.
2494                  */
2495                 if (old_mode != new_mode)
2496                         notify_of_pool_mode_change_to_oods(pool);
2497                 pool->out_of_data_space = true;
2498                 pool->process_bio = process_bio_read_only;
2499                 pool->process_discard = process_discard_bio;
2500                 pool->process_cell = process_cell_read_only;
2501                 pool->process_prepared_mapping = process_prepared_mapping;
2502                 set_discard_callbacks(pool);
2503
2504                 if (!pool->pf.error_if_no_space && no_space_timeout)
2505                         queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
2506                 break;
2507
2508         case PM_WRITE:
2509                 if (old_mode != new_mode)
2510                         notify_of_pool_mode_change(pool, "write");
2511                 pool->out_of_data_space = false;
2512                 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;
2513                 dm_pool_metadata_read_write(pool->pmd);
2514                 pool->process_bio = process_bio;
2515                 pool->process_discard = process_discard_bio;
2516                 pool->process_cell = process_cell;
2517                 pool->process_prepared_mapping = process_prepared_mapping;
2518                 set_discard_callbacks(pool);
2519                 break;
2520         }
2521
2522         pool->pf.mode = new_mode;
2523         /*
2524          * The pool mode may have changed, sync it so bind_control_target()
2525          * doesn't cause an unexpected mode transition on resume.
2526          */
2527         pt->adjusted_pf.mode = new_mode;
2528 }
2529
2530 static void abort_transaction(struct pool *pool)
2531 {
2532         const char *dev_name = dm_device_name(pool->pool_md);
2533
2534         DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
2535         if (dm_pool_abort_metadata(pool->pmd)) {
2536                 DMERR("%s: failed to abort metadata transaction", dev_name);
2537                 set_pool_mode(pool, PM_FAIL);
2538         }
2539
2540         if (dm_pool_metadata_set_needs_check(pool->pmd)) {
2541                 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
2542                 set_pool_mode(pool, PM_FAIL);
2543         }
2544 }
2545
2546 static void metadata_operation_failed(struct pool *pool, const char *op, int r)
2547 {
2548         DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2549                     dm_device_name(pool->pool_md), op, r);
2550
2551         abort_transaction(pool);
2552         set_pool_mode(pool, PM_READ_ONLY);
2553 }
2554
2555 /*----------------------------------------------------------------*/
2556
2557 /*
2558  * Mapping functions.
2559  */
2560
2561 /*
2562  * Called only while mapping a thin bio to hand it over to the workqueue.
2563  */
2564 static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
2565 {
2566         unsigned long flags;
2567         struct pool *pool = tc->pool;
2568
2569         spin_lock_irqsave(&tc->lock, flags);
2570         bio_list_add(&tc->deferred_bio_list, bio);
2571         spin_unlock_irqrestore(&tc->lock, flags);
2572
2573         wake_worker(pool);
2574 }
2575
2576 static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
2577 {
2578         struct pool *pool = tc->pool;
2579
2580         throttle_lock(&pool->throttle);
2581         thin_defer_bio(tc, bio);
2582         throttle_unlock(&pool->throttle);
2583 }
2584
2585 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2586 {
2587         unsigned long flags;
2588         struct pool *pool = tc->pool;
2589
2590         throttle_lock(&pool->throttle);
2591         spin_lock_irqsave(&tc->lock, flags);
2592         list_add_tail(&cell->user_list, &tc->deferred_cells);
2593         spin_unlock_irqrestore(&tc->lock, flags);
2594         throttle_unlock(&pool->throttle);
2595
2596         wake_worker(pool);
2597 }
2598
2599 static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
2600 {
2601         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2602
2603         h->tc = tc;
2604         h->shared_read_entry = NULL;
2605         h->all_io_entry = NULL;
2606         h->overwrite_mapping = NULL;
2607         h->cell = NULL;
2608 }
2609
2610 /*
2611  * Non-blocking function called from the thin target's map function.
2612  */
2613 static int thin_bio_map(struct dm_target *ti, struct bio *bio)
2614 {
2615         int r;
2616         struct thin_c *tc = ti->private;
2617         dm_block_t block = get_bio_block(tc, bio);
2618         struct dm_thin_device *td = tc->td;
2619         struct dm_thin_lookup_result result;
2620         struct dm_bio_prison_cell *virt_cell, *data_cell;
2621         struct dm_cell_key key;
2622
2623         thin_hook_bio(tc, bio);
2624
2625         if (tc->requeue_mode) {
2626                 bio->bi_error = DM_ENDIO_REQUEUE;
2627                 bio_endio(bio);
2628                 return DM_MAPIO_SUBMITTED;
2629         }
2630
2631         if (get_pool_mode(tc->pool) == PM_FAIL) {
2632                 bio_io_error(bio);
2633                 return DM_MAPIO_SUBMITTED;
2634         }
2635
2636         if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) {
2637                 thin_defer_bio_with_throttle(tc, bio);
2638                 return DM_MAPIO_SUBMITTED;
2639         }
2640
2641         /*
2642          * We must hold the virtual cell before doing the lookup, otherwise
2643          * there's a race with discard.
2644          */
2645         build_virtual_key(tc->td, block, &key);
2646         if (bio_detain(tc->pool, &key, bio, &virt_cell))
2647                 return DM_MAPIO_SUBMITTED;
2648
2649         r = dm_thin_find_block(td, block, 0, &result);
2650
2651         /*
2652          * Note that we defer readahead too.
2653          */
2654         switch (r) {
2655         case 0:
2656                 if (unlikely(result.shared)) {
2657                         /*
2658                          * We have a race condition here between the
2659                          * result.shared value returned by the lookup and
2660                          * snapshot creation, which may cause new
2661                          * sharing.
2662                          *
2663                          * To avoid this always quiesce the origin before
2664                          * taking the snap.  You want to do this anyway to
2665                          * ensure a consistent application view
2666                          * (i.e. lockfs).
2667                          *
2668                          * More distant ancestors are irrelevant. The
2669                          * shared flag will be set in their case.
2670                          */
2671                         thin_defer_cell(tc, virt_cell);
2672                         return DM_MAPIO_SUBMITTED;
2673                 }
2674
2675                 build_data_key(tc->td, result.block, &key);
2676                 if (bio_detain(tc->pool, &key, bio, &data_cell)) {
2677                         cell_defer_no_holder(tc, virt_cell);
2678                         return DM_MAPIO_SUBMITTED;
2679                 }
2680
2681                 inc_all_io_entry(tc->pool, bio);
2682                 cell_defer_no_holder(tc, data_cell);
2683                 cell_defer_no_holder(tc, virt_cell);
2684
2685                 remap(tc, bio, result.block);
2686                 return DM_MAPIO_REMAPPED;
2687
2688         case -ENODATA:
2689         case -EWOULDBLOCK:
2690                 thin_defer_cell(tc, virt_cell);
2691                 return DM_MAPIO_SUBMITTED;
2692
2693         default:
2694                 /*
2695                  * Must always call bio_io_error on failure.
2696                  * dm_thin_find_block can fail with -EINVAL if the
2697                  * pool is switched to fail-io mode.
2698                  */
2699                 bio_io_error(bio);
2700                 cell_defer_no_holder(tc, virt_cell);
2701                 return DM_MAPIO_SUBMITTED;
2702         }
2703 }
2704
2705 static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2706 {
2707         struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
2708         struct request_queue *q;
2709
2710         if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE)
2711                 return 1;
2712
2713         q = bdev_get_queue(pt->data_dev->bdev);
2714         return bdi_congested(q->backing_dev_info, bdi_bits);
2715 }
2716
2717 static void requeue_bios(struct pool *pool)
2718 {
2719         unsigned long flags;
2720         struct thin_c *tc;
2721
2722         rcu_read_lock();
2723         list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2724                 spin_lock_irqsave(&tc->lock, flags);
2725                 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
2726                 bio_list_init(&tc->retry_on_resume_list);
2727                 spin_unlock_irqrestore(&tc->lock, flags);
2728         }
2729         rcu_read_unlock();
2730 }
2731
2732 /*----------------------------------------------------------------
2733  * Binding of control targets to a pool object
2734  *--------------------------------------------------------------*/
2735 static bool data_dev_supports_discard(struct pool_c *pt)
2736 {
2737         struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2738
2739         return q && blk_queue_discard(q);
2740 }
2741
2742 static bool is_factor(sector_t block_size, uint32_t n)
2743 {
2744         return !sector_div(block_size, n);
2745 }
2746
2747 /*
2748  * If discard_passdown was enabled verify that the data device
2749  * supports discards.  Disable discard_passdown if not.
2750  */
2751 static void disable_passdown_if_not_supported(struct pool_c *pt)
2752 {
2753         struct pool *pool = pt->pool;
2754         struct block_device *data_bdev = pt->data_dev->bdev;
2755         struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;
2756         const char *reason = NULL;
2757         char buf[BDEVNAME_SIZE];
2758
2759         if (!pt->adjusted_pf.discard_passdown)
2760                 return;
2761
2762         if (!data_dev_supports_discard(pt))
2763                 reason = "discard unsupported";
2764
2765         else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2766                 reason = "max discard sectors smaller than a block";
2767
2768         if (reason) {
2769                 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason);
2770                 pt->adjusted_pf.discard_passdown = false;
2771         }
2772 }
2773
2774 static int bind_control_target(struct pool *pool, struct dm_target *ti)
2775 {
2776         struct pool_c *pt = ti->private;
2777
2778         /*
2779          * We want to make sure that a pool in PM_FAIL mode is never upgraded.
2780          */
2781         enum pool_mode old_mode = get_pool_mode(pool);
2782         enum pool_mode new_mode = pt->adjusted_pf.mode;
2783
2784         /*
2785          * Don't change the pool's mode until set_pool_mode() below.
2786          * Otherwise the pool's process_* function pointers may
2787          * not match the desired pool mode.
2788          */
2789         pt->adjusted_pf.mode = old_mode;
2790
2791         pool->ti = ti;
2792         pool->pf = pt->adjusted_pf;
2793         pool->low_water_blocks = pt->low_water_blocks;
2794
2795         set_pool_mode(pool, new_mode);
2796
2797         return 0;
2798 }
2799
2800 static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2801 {
2802         if (pool->ti == ti)
2803                 pool->ti = NULL;
2804 }
2805
2806 /*----------------------------------------------------------------
2807  * Pool creation
2808  *--------------------------------------------------------------*/
2809 /* Initialize pool features. */
2810 static void pool_features_init(struct pool_features *pf)
2811 {
2812         pf->mode = PM_WRITE;
2813         pf->zero_new_blocks = true;
2814         pf->discard_enabled = true;
2815         pf->discard_passdown = true;
2816         pf->error_if_no_space = false;
2817 }
2818
2819 static void __pool_destroy(struct pool *pool)
2820 {
2821         __pool_table_remove(pool);
2822
2823         vfree(pool->cell_sort_array);
2824         if (dm_pool_metadata_close(pool->pmd) < 0)
2825                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2826
2827         dm_bio_prison_destroy(pool->prison);
2828         dm_kcopyd_client_destroy(pool->copier);
2829
2830         if (pool->wq)
2831                 destroy_workqueue(pool->wq);
2832
2833         if (pool->next_mapping)
2834                 mempool_free(pool->next_mapping, pool->mapping_pool);
2835         mempool_destroy(pool->mapping_pool);
2836         dm_deferred_set_destroy(pool->shared_read_ds);
2837         dm_deferred_set_destroy(pool->all_io_ds);
2838         kfree(pool);
2839 }
2840
2841 static struct kmem_cache *_new_mapping_cache;
2842
2843 static struct pool *pool_create(struct mapped_device *pool_md,
2844                                 struct block_device *metadata_dev,
2845                                 unsigned long block_size,
2846                                 int read_only, char **error)
2847 {
2848         int r;
2849         void *err_p;
2850         struct pool *pool;
2851         struct dm_pool_metadata *pmd;
2852         bool format_device = read_only ? false : true;
2853
2854         pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
2855         if (IS_ERR(pmd)) {
2856                 *error = "Error creating metadata object";
2857                 return (struct pool *)pmd;
2858         }
2859
2860         pool = kmalloc(sizeof(*pool), GFP_KERNEL);
2861         if (!pool) {
2862                 *error = "Error allocating memory for pool";
2863                 err_p = ERR_PTR(-ENOMEM);
2864                 goto bad_pool;
2865         }
2866
2867         pool->pmd = pmd;
2868         pool->sectors_per_block = block_size;
2869         if (block_size & (block_size - 1))
2870                 pool->sectors_per_block_shift = -1;
2871         else
2872                 pool->sectors_per_block_shift = __ffs(block_size);
2873         pool->low_water_blocks = 0;
2874         pool_features_init(&pool->pf);
2875         pool->prison = dm_bio_prison_create();
2876         if (!pool->prison) {
2877                 *error = "Error creating pool's bio prison";
2878                 err_p = ERR_PTR(-ENOMEM);
2879                 goto bad_prison;
2880         }
2881
2882         pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2883         if (IS_ERR(pool->copier)) {
2884                 r = PTR_ERR(pool->copier);
2885                 *error = "Error creating pool's kcopyd client";
2886                 err_p = ERR_PTR(r);
2887                 goto bad_kcopyd_client;
2888         }
2889
2890         /*
2891          * Create singlethreaded workqueue that will service all devices
2892          * that use this metadata.
2893          */
2894         pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2895         if (!pool->wq) {
2896                 *error = "Error creating pool's workqueue";
2897                 err_p = ERR_PTR(-ENOMEM);
2898                 goto bad_wq;
2899         }
2900
2901         throttle_init(&pool->throttle);
2902         INIT_WORK(&pool->worker, do_worker);
2903         INIT_DELAYED_WORK(&pool->waker, do_waker);
2904         INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
2905         spin_lock_init(&pool->lock);
2906         bio_list_init(&pool->deferred_flush_bios);
2907         INIT_LIST_HEAD(&pool->prepared_mappings);
2908         INIT_LIST_HEAD(&pool->prepared_discards);
2909         INIT_LIST_HEAD(&pool->prepared_discards_pt2);
2910         INIT_LIST_HEAD(&pool->active_thins);
2911         pool->low_water_triggered = false;
2912         pool->suspended = true;
2913         pool->out_of_data_space = false;
2914
2915         pool->shared_read_ds = dm_deferred_set_create();
2916         if (!pool->shared_read_ds) {
2917                 *error = "Error creating pool's shared read deferred set";
2918                 err_p = ERR_PTR(-ENOMEM);
2919                 goto bad_shared_read_ds;
2920         }
2921
2922         pool->all_io_ds = dm_deferred_set_create();
2923         if (!pool->all_io_ds) {
2924                 *error = "Error creating pool's all io deferred set";
2925                 err_p = ERR_PTR(-ENOMEM);
2926                 goto bad_all_io_ds;
2927         }
2928
2929         pool->next_mapping = NULL;
2930         pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE,
2931                                                       _new_mapping_cache);
2932         if (!pool->mapping_pool) {
2933                 *error = "Error creating pool's mapping mempool";
2934                 err_p = ERR_PTR(-ENOMEM);
2935                 goto bad_mapping_pool;
2936         }
2937
2938         pool->cell_sort_array = vmalloc(sizeof(*pool->cell_sort_array) * CELL_SORT_ARRAY_SIZE);
2939         if (!pool->cell_sort_array) {
2940                 *error = "Error allocating cell sort array";
2941                 err_p = ERR_PTR(-ENOMEM);
2942                 goto bad_sort_array;
2943         }
2944
2945         pool->ref_count = 1;
2946         pool->last_commit_jiffies = jiffies;
2947         pool->pool_md = pool_md;
2948         pool->md_dev = metadata_dev;
2949         __pool_table_insert(pool);
2950
2951         return pool;
2952
2953 bad_sort_array:
2954         mempool_destroy(pool->mapping_pool);
2955 bad_mapping_pool:
2956         dm_deferred_set_destroy(pool->all_io_ds);
2957 bad_all_io_ds:
2958         dm_deferred_set_destroy(pool->shared_read_ds);
2959 bad_shared_read_ds:
2960         destroy_workqueue(pool->wq);
2961 bad_wq:
2962         dm_kcopyd_client_destroy(pool->copier);
2963 bad_kcopyd_client:
2964         dm_bio_prison_destroy(pool->prison);
2965 bad_prison:
2966         kfree(pool);
2967 bad_pool:
2968         if (dm_pool_metadata_close(pmd))
2969                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2970
2971         return err_p;
2972 }
2973
2974 static void __pool_inc(struct pool *pool)
2975 {
2976         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2977         pool->ref_count++;
2978 }
2979
2980 static void __pool_dec(struct pool *pool)
2981 {
2982         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2983         BUG_ON(!pool->ref_count);
2984         if (!--pool->ref_count)
2985                 __pool_destroy(pool);
2986 }
2987
2988 static struct pool *__pool_find(struct mapped_device *pool_md,
2989                                 struct block_device *metadata_dev,
2990                                 unsigned long block_size, int read_only,
2991                                 char **error, int *created)
2992 {
2993         struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
2994
2995         if (pool) {
2996                 if (pool->pool_md != pool_md) {
2997                         *error = "metadata device already in use by a pool";
2998                         return ERR_PTR(-EBUSY);
2999                 }
3000                 __pool_inc(pool);
3001
3002         } else {
3003                 pool = __pool_table_lookup(pool_md);
3004                 if (pool) {
3005                         if (pool->md_dev != metadata_dev) {
3006                                 *error = "different pool cannot replace a pool";
3007                                 return ERR_PTR(-EINVAL);
3008                         }
3009                         __pool_inc(pool);
3010
3011                 } else {
3012                         pool = pool_create(pool_md, metadata_dev, block_size, read_only, error);
3013                         *created = 1;
3014                 }
3015         }
3016
3017         return pool;
3018 }
3019
3020 /*----------------------------------------------------------------
3021  * Pool target methods
3022  *--------------------------------------------------------------*/
3023 static void pool_dtr(struct dm_target *ti)
3024 {
3025         struct pool_c *pt = ti->private;
3026
3027         mutex_lock(&dm_thin_pool_table.mutex);
3028
3029         unbind_control_target(pt->pool, ti);
3030         __pool_dec(pt->pool);
3031         dm_put_device(ti, pt->metadata_dev);
3032         dm_put_device(ti, pt->data_dev);
3033         kfree(pt);
3034
3035         mutex_unlock(&dm_thin_pool_table.mutex);
3036 }
3037
3038 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
3039                                struct dm_target *ti)
3040 {
3041         int r;
3042         unsigned argc;
3043         const char *arg_name;
3044
3045         static struct dm_arg _args[] = {
3046                 {0, 4, "Invalid number of pool feature arguments"},
3047         };
3048
3049         /*
3050          * No feature arguments supplied.
3051          */
3052         if (!as->argc)
3053                 return 0;
3054
3055         r = dm_read_arg_group(_args, as, &argc, &ti->error);
3056         if (r)
3057                 return -EINVAL;
3058
3059         while (argc && !r) {
3060                 arg_name = dm_shift_arg(as);
3061                 argc--;
3062
3063                 if (!strcasecmp(arg_name, "skip_block_zeroing"))
3064                         pf->zero_new_blocks = false;
3065
3066                 else if (!strcasecmp(arg_name, "ignore_discard"))
3067                         pf->discard_enabled = false;
3068
3069                 else if (!strcasecmp(arg_name, "no_discard_passdown"))
3070                         pf->discard_passdown = false;
3071
3072                 else if (!strcasecmp(arg_name, "read_only"))
3073                         pf->mode = PM_READ_ONLY;
3074
3075                 else if (!strcasecmp(arg_name, "error_if_no_space"))
3076                         pf->error_if_no_space = true;
3077
3078                 else {
3079                         ti->error = "Unrecognised pool feature requested";
3080                         r = -EINVAL;
3081                         break;
3082                 }
3083         }
3084
3085         return r;
3086 }
3087
3088 static void metadata_low_callback(void *context)
3089 {
3090         struct pool *pool = context;
3091
3092         DMWARN("%s: reached low water mark for metadata device: sending event.",
3093                dm_device_name(pool->pool_md));
3094
3095         dm_table_event(pool->ti->table);
3096 }
3097
3098 static sector_t get_dev_size(struct block_device *bdev)
3099 {
3100         return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
3101 }
3102
3103 static void warn_if_metadata_device_too_big(struct block_device *bdev)
3104 {
3105         sector_t metadata_dev_size = get_dev_size(bdev);
3106         char buffer[BDEVNAME_SIZE];
3107
3108         if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
3109                 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
3110                        bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS);
3111 }
3112
3113 static sector_t get_metadata_dev_size(struct block_device *bdev)
3114 {
3115         sector_t metadata_dev_size = get_dev_size(bdev);
3116
3117         if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
3118                 metadata_dev_size = THIN_METADATA_MAX_SECTORS;
3119
3120         return metadata_dev_size;
3121 }
3122
3123 static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
3124 {
3125         sector_t metadata_dev_size = get_metadata_dev_size(bdev);
3126
3127         sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
3128
3129         return metadata_dev_size;
3130 }
3131
3132 /*
3133  * When a metadata threshold is crossed a dm event is triggered, and
3134  * userland should respond by growing the metadata device.  We could let
3135  * userland set the threshold, like we do with the data threshold, but I'm
3136  * not sure they know enough to do this well.
3137  */
3138 static dm_block_t calc_metadata_threshold(struct pool_c *pt)
3139 {
3140         /*
3141          * 4M is ample for all ops with the possible exception of thin
3142          * device deletion which is harmless if it fails (just retry the
3143          * delete after you've grown the device).
3144          */
3145         dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
3146         return min((dm_block_t)1024ULL /* 4M */, quarter);
3147 }
3148
3149 /*
3150  * thin-pool <metadata dev> <data dev>
3151  *           <data block size (sectors)>
3152  *           <low water mark (blocks)>
3153  *           [<#feature args> [<arg>]*]
3154  *
3155  * Optional feature arguments are:
3156  *           skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
3157  *           ignore_discard: disable discard
3158  *           no_discard_passdown: don't pass discards down to the data device
3159  *           read_only: Don't allow any changes to be made to the pool metadata.
3160  *           error_if_no_space: error IOs, instead of queueing, if no space.
3161  */
3162 static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
3163 {
3164         int r, pool_created = 0;
3165         struct pool_c *pt;
3166         struct pool *pool;
3167         struct pool_features pf;
3168         struct dm_arg_set as;
3169         struct dm_dev *data_dev;
3170         unsigned long block_size;
3171         dm_block_t low_water_blocks;
3172         struct dm_dev *metadata_dev;
3173         fmode_t metadata_mode;
3174
3175         /*
3176          * FIXME Remove validation from scope of lock.
3177          */
3178         mutex_lock(&dm_thin_pool_table.mutex);
3179
3180         if (argc < 4) {
3181                 ti->error = "Invalid argument count";
3182                 r = -EINVAL;
3183                 goto out_unlock;
3184         }
3185
3186         as.argc = argc;
3187         as.argv = argv;
3188
3189         /*
3190          * Set default pool features.
3191          */
3192         pool_features_init(&pf);
3193
3194         dm_consume_args(&as, 4);
3195         r = parse_pool_features(&as, &pf, ti);
3196         if (r)
3197                 goto out_unlock;
3198
3199         metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE);
3200         r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
3201         if (r) {
3202                 ti->error = "Error opening metadata block device";
3203                 goto out_unlock;
3204         }
3205         warn_if_metadata_device_too_big(metadata_dev->bdev);
3206
3207         r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
3208         if (r) {
3209                 ti->error = "Error getting data device";
3210                 goto out_metadata;
3211         }
3212
3213         if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
3214             block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
3215             block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
3216             block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
3217                 ti->error = "Invalid block size";
3218                 r = -EINVAL;
3219                 goto out;
3220         }
3221
3222         if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
3223                 ti->error = "Invalid low water mark";
3224                 r = -EINVAL;
3225                 goto out;
3226         }
3227
3228         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
3229         if (!pt) {
3230                 r = -ENOMEM;
3231                 goto out;
3232         }
3233
3234         pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
3235                            block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
3236         if (IS_ERR(pool)) {
3237                 r = PTR_ERR(pool);
3238                 goto out_free_pt;
3239         }
3240
3241         /*
3242          * 'pool_created' reflects whether this is the first table load.
3243          * Top level discard support is not allowed to be changed after
3244          * initial load.  This would require a pool reload to trigger thin
3245          * device changes.
3246          */
3247         if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
3248                 ti->error = "Discard support cannot be disabled once enabled";
3249                 r = -EINVAL;
3250                 goto out_flags_changed;
3251         }
3252
3253         pt->pool = pool;
3254         pt->ti = ti;
3255         pt->metadata_dev = metadata_dev;
3256         pt->data_dev = data_dev;
3257         pt->low_water_blocks = low_water_blocks;
3258         pt->adjusted_pf = pt->requested_pf = pf;
3259         ti->num_flush_bios = 1;
3260
3261         /*
3262          * Only need to enable discards if the pool should pass
3263          * them down to the data device.  The thin device's discard
3264          * processing will cause mappings to be removed from the btree.
3265          */
3266         ti->discard_zeroes_data_unsupported = true;
3267         if (pf.discard_enabled && pf.discard_passdown) {
3268                 ti->num_discard_bios = 1;
3269
3270                 /*
3271                  * Setting 'discards_supported' circumvents the normal
3272                  * stacking of discard limits (this keeps the pool and
3273                  * thin devices' discard limits consistent).
3274                  */
3275                 ti->discards_supported = true;
3276         }
3277         ti->private = pt;
3278
3279         r = dm_pool_register_metadata_threshold(pt->pool->pmd,
3280                                                 calc_metadata_threshold(pt),
3281                                                 metadata_low_callback,
3282                                                 pool);
3283         if (r)
3284                 goto out_flags_changed;
3285
3286         pt->callbacks.congested_fn = pool_is_congested;
3287         dm_table_add_target_callbacks(ti->table, &pt->callbacks);
3288
3289         mutex_unlock(&dm_thin_pool_table.mutex);
3290
3291         return 0;
3292
3293 out_flags_changed:
3294         __pool_dec(pool);
3295 out_free_pt:
3296         kfree(pt);
3297 out:
3298         dm_put_device(ti, data_dev);
3299 out_metadata:
3300         dm_put_device(ti, metadata_dev);
3301 out_unlock:
3302         mutex_unlock(&dm_thin_pool_table.mutex);
3303
3304         return r;
3305 }
3306
3307 static int pool_map(struct dm_target *ti, struct bio *bio)
3308 {
3309         int r;
3310         struct pool_c *pt = ti->private;
3311         struct pool *pool = pt->pool;
3312         unsigned long flags;
3313
3314         /*
3315          * As this is a singleton target, ti->begin is always zero.
3316          */
3317         spin_lock_irqsave(&pool->lock, flags);
3318         bio->bi_bdev = pt->data_dev->bdev;
3319         r = DM_MAPIO_REMAPPED;
3320         spin_unlock_irqrestore(&pool->lock, flags);
3321
3322         return r;
3323 }
3324
3325 static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
3326 {
3327         int r;
3328         struct pool_c *pt = ti->private;
3329         struct pool *pool = pt->pool;
3330         sector_t data_size = ti->len;
3331         dm_block_t sb_data_size;
3332
3333         *need_commit = false;
3334
3335         (void) sector_div(data_size, pool->sectors_per_block);
3336
3337         r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
3338         if (r) {
3339                 DMERR("%s: failed to retrieve data device size",
3340                       dm_device_name(pool->pool_md));
3341                 return r;
3342         }
3343
3344         if (data_size < sb_data_size) {
3345                 DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3346                       dm_device_name(pool->pool_md),
3347                       (unsigned long long)data_size, sb_data_size);
3348                 return -EINVAL;
3349
3350         } else if (data_size > sb_data_size) {
3351                 if (dm_pool_metadata_needs_check(pool->pmd)) {
3352                         DMERR("%s: unable to grow the data device until repaired.",
3353                               dm_device_name(pool->pool_md));
3354                         return 0;
3355                 }
3356
3357                 if (sb_data_size)
3358                         DMINFO("%s: growing the data device from %llu to %llu blocks",
3359                                dm_device_name(pool->pool_md),
3360                                sb_data_size, (unsigned long long)data_size);
3361                 r = dm_pool_resize_data_dev(pool->pmd, data_size);
3362                 if (r) {
3363                         metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
3364                         return r;
3365                 }
3366
3367                 *need_commit = true;
3368         }
3369
3370         return 0;
3371 }
3372
3373 static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
3374 {
3375         int r;
3376         struct pool_c *pt = ti->private;
3377         struct pool *pool = pt->pool;
3378         dm_block_t metadata_dev_size, sb_metadata_dev_size;
3379
3380         *need_commit = false;
3381
3382         metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
3383
3384         r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
3385         if (r) {
3386                 DMERR("%s: failed to retrieve metadata device size",
3387                       dm_device_name(pool->pool_md));
3388                 return r;
3389         }
3390
3391         if (metadata_dev_size < sb_metadata_dev_size) {
3392                 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3393                       dm_device_name(pool->pool_md),
3394                       metadata_dev_size, sb_metadata_dev_size);
3395                 return -EINVAL;
3396
3397         } else if (metadata_dev_size > sb_metadata_dev_size) {
3398                 if (dm_pool_metadata_needs_check(pool->pmd)) {
3399                         DMERR("%s: unable to grow the metadata device until repaired.",
3400                               dm_device_name(pool->pool_md));
3401                         return 0;
3402                 }
3403
3404                 warn_if_metadata_device_too_big(pool->md_dev);
3405                 DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3406                        dm_device_name(pool->pool_md),
3407                        sb_metadata_dev_size, metadata_dev_size);
3408                 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
3409                 if (r) {
3410                         metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
3411                         return r;
3412                 }
3413
3414                 *need_commit = true;
3415         }
3416
3417         return 0;
3418 }
3419
3420 /*
3421  * Retrieves the number of blocks of the data device from
3422  * the superblock and compares it to the actual device size,
3423  * thus resizing the data device in case it has grown.
3424  *
3425  * This both copes with opening preallocated data devices in the ctr
3426  * being followed by a resume
3427  * -and-
3428  * calling the resume method individually after userspace has
3429  * grown the data device in reaction to a table event.
3430  */
3431 static int pool_preresume(struct dm_target *ti)
3432 {
3433         int r;
3434         bool need_commit1, need_commit2;
3435         struct pool_c *pt = ti->private;
3436         struct pool *pool = pt->pool;
3437
3438         /*
3439          * Take control of the pool object.
3440          */
3441         r = bind_control_target(pool, ti);
3442         if (r)
3443                 return r;
3444
3445         r = maybe_resize_data_dev(ti, &need_commit1);
3446         if (r)
3447                 return r;
3448
3449         r = maybe_resize_metadata_dev(ti, &need_commit2);
3450         if (r)
3451                 return r;
3452
3453         if (need_commit1 || need_commit2)
3454                 (void) commit(pool);
3455
3456         return 0;
3457 }
3458
3459 static void pool_suspend_active_thins(struct pool *pool)
3460 {
3461         struct thin_c *tc;
3462
3463         /* Suspend all active thin devices */
3464         tc = get_first_thin(pool);
3465         while (tc) {
3466                 dm_internal_suspend_noflush(tc->thin_md);
3467                 tc = get_next_thin(pool, tc);
3468         }
3469 }
3470
3471 static void pool_resume_active_thins(struct pool *pool)
3472 {
3473         struct thin_c *tc;
3474
3475         /* Resume all active thin devices */
3476         tc = get_first_thin(pool);
3477         while (tc) {
3478                 dm_internal_resume(tc->thin_md);
3479                 tc = get_next_thin(pool, tc);
3480         }
3481 }
3482
3483 static void pool_resume(struct dm_target *ti)
3484 {
3485         struct pool_c *pt = ti->private;
3486         struct pool *pool = pt->pool;
3487         unsigned long flags;
3488
3489         /*
3490          * Must requeue active_thins' bios and then resume
3491          * active_thins _before_ clearing 'suspend' flag.
3492          */
3493         requeue_bios(pool);
3494         pool_resume_active_thins(pool);
3495
3496         spin_lock_irqsave(&pool->lock, flags);
3497         pool->low_water_triggered = false;
3498         pool->suspended = false;
3499         spin_unlock_irqrestore(&pool->lock, flags);
3500
3501         do_waker(&pool->waker.work);
3502 }
3503
3504 static void pool_presuspend(struct dm_target *ti)
3505 {
3506         struct pool_c *pt = ti->private;
3507         struct pool *pool = pt->pool;
3508         unsigned long flags;
3509
3510         spin_lock_irqsave(&pool->lock, flags);
3511         pool->suspended = true;
3512         spin_unlock_irqrestore(&pool->lock, flags);
3513
3514         pool_suspend_active_thins(pool);
3515 }
3516
3517 static void pool_presuspend_undo(struct dm_target *ti)
3518 {
3519         struct pool_c *pt = ti->private;
3520         struct pool *pool = pt->pool;
3521         unsigned long flags;
3522
3523         pool_resume_active_thins(pool);
3524
3525         spin_lock_irqsave(&pool->lock, flags);
3526         pool->suspended = false;
3527         spin_unlock_irqrestore(&pool->lock, flags);
3528 }
3529
3530 static void pool_postsuspend(struct dm_target *ti)
3531 {
3532         struct pool_c *pt = ti->private;
3533         struct pool *pool = pt->pool;
3534
3535         cancel_delayed_work_sync(&pool->waker);
3536         cancel_delayed_work_sync(&pool->no_space_timeout);
3537         flush_workqueue(pool->wq);
3538         (void) commit(pool);
3539 }
3540
3541 static int check_arg_count(unsigned argc, unsigned args_required)
3542 {
3543         if (argc != args_required) {
3544                 DMWARN("Message received with %u arguments instead of %u.",
3545                        argc, args_required);
3546                 return -EINVAL;
3547         }
3548
3549         return 0;
3550 }
3551
3552 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
3553 {
3554         if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
3555             *dev_id <= MAX_DEV_ID)
3556                 return 0;
3557
3558         if (warning)
3559                 DMWARN("Message received with invalid device id: %s", arg);
3560
3561         return -EINVAL;
3562 }
3563
3564 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
3565 {
3566         dm_thin_id dev_id;
3567         int r;
3568
3569         r = check_arg_count(argc, 2);
3570         if (r)
3571                 return r;
3572
3573         r = read_dev_id(argv[1], &dev_id, 1);
3574         if (r)
3575                 return r;
3576
3577         r = dm_pool_create_thin(pool->pmd, dev_id);
3578         if (r) {
3579                 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3580                        argv[1]);
3581                 return r;
3582         }
3583
3584         return 0;
3585 }
3586
3587 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3588 {
3589         dm_thin_id dev_id;
3590         dm_thin_id origin_dev_id;
3591         int r;
3592
3593         r = check_arg_count(argc, 3);
3594         if (r)
3595                 return r;
3596
3597         r = read_dev_id(argv[1], &dev_id, 1);
3598         if (r)
3599                 return r;
3600
3601         r = read_dev_id(argv[2], &origin_dev_id, 1);
3602         if (r)
3603                 return r;
3604
3605         r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
3606         if (r) {
3607                 DMWARN("Creation of new snapshot %s of device %s failed.",
3608                        argv[1], argv[2]);
3609                 return r;
3610         }
3611
3612         return 0;
3613 }
3614
3615 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
3616 {
3617         dm_thin_id dev_id;
3618         int r;
3619
3620         r = check_arg_count(argc, 2);
3621         if (r)
3622                 return r;
3623
3624         r = read_dev_id(argv[1], &dev_id, 1);
3625         if (r)
3626                 return r;
3627
3628         r = dm_pool_delete_thin_device(pool->pmd, dev_id);
3629         if (r)
3630                 DMWARN("Deletion of thin device %s failed.", argv[1]);
3631
3632         return r;
3633 }
3634
3635 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
3636 {
3637         dm_thin_id old_id, new_id;
3638         int r;
3639
3640         r = check_arg_count(argc, 3);
3641         if (r)
3642                 return r;
3643
3644         if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
3645                 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
3646                 return -EINVAL;
3647         }
3648
3649         if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
3650                 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
3651                 return -EINVAL;
3652         }
3653
3654         r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
3655         if (r) {
3656                 DMWARN("Failed to change transaction id from %s to %s.",
3657                        argv[1], argv[2]);
3658                 return r;
3659         }
3660
3661         return 0;
3662 }
3663
3664 static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3665 {
3666         int r;
3667
3668         r = check_arg_count(argc, 1);
3669         if (r)
3670                 return r;
3671
3672         (void) commit(pool);
3673
3674         r = dm_pool_reserve_metadata_snap(pool->pmd);
3675         if (r)
3676                 DMWARN("reserve_metadata_snap message failed.");
3677
3678         return r;
3679 }
3680
3681 static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3682 {
3683         int r;
3684
3685         r = check_arg_count(argc, 1);
3686         if (r)
3687                 return r;
3688
3689         r = dm_pool_release_metadata_snap(pool->pmd);
3690         if (r)
3691                 DMWARN("release_metadata_snap message failed.");
3692
3693         return r;
3694 }
3695
3696 /*
3697  * Messages supported:
3698  *   create_thin        <dev_id>
3699  *   create_snap        <dev_id> <origin_id>
3700  *   delete             <dev_id>
3701  *   set_transaction_id <current_trans_id> <new_trans_id>
3702  *   reserve_metadata_snap
3703  *   release_metadata_snap
3704  */
3705 static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
3706 {
3707         int r = -EINVAL;
3708         struct pool_c *pt = ti->private;
3709         struct pool *pool = pt->pool;
3710
3711         if (get_pool_mode(pool) >= PM_READ_ONLY) {
3712                 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3713                       dm_device_name(pool->pool_md));
3714                 return -EOPNOTSUPP;
3715         }
3716
3717         if (!strcasecmp(argv[0], "create_thin"))
3718                 r = process_create_thin_mesg(argc, argv, pool);
3719
3720         else if (!strcasecmp(argv[0], "create_snap"))
3721                 r = process_create_snap_mesg(argc, argv, pool);
3722
3723         else if (!strcasecmp(argv[0], "delete"))
3724                 r = process_delete_mesg(argc, argv, pool);
3725
3726         else if (!strcasecmp(argv[0], "set_transaction_id"))
3727                 r = process_set_transaction_id_mesg(argc, argv, pool);
3728
3729         else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
3730                 r = process_reserve_metadata_snap_mesg(argc, argv, pool);
3731
3732         else if (!strcasecmp(argv[0], "release_metadata_snap"))
3733                 r = process_release_metadata_snap_mesg(argc, argv, pool);
3734
3735         else
3736                 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
3737
3738         if (!r)
3739                 (void) commit(pool);
3740
3741         return r;
3742 }
3743
3744 static void emit_flags(struct pool_features *pf, char *result,
3745                        unsigned sz, unsigned maxlen)
3746 {
3747         unsigned count = !pf->zero_new_blocks + !pf->discard_enabled +
3748                 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
3749                 pf->error_if_no_space;
3750         DMEMIT("%u ", count);
3751
3752         if (!pf->zero_new_blocks)
3753                 DMEMIT("skip_block_zeroing ");
3754
3755         if (!pf->discard_enabled)
3756                 DMEMIT("ignore_discard ");
3757
3758         if (!pf->discard_passdown)
3759                 DMEMIT("no_discard_passdown ");
3760
3761         if (pf->mode == PM_READ_ONLY)
3762                 DMEMIT("read_only ");
3763
3764         if (pf->error_if_no_space)
3765                 DMEMIT("error_if_no_space ");
3766 }
3767
3768 /*
3769  * Status line is:
3770  *    <transaction id> <used metadata sectors>/<total metadata sectors>
3771  *    <used data sectors>/<total data sectors> <held metadata root>
3772  *    <pool mode> <discard config> <no space config> <needs_check>
3773  */
3774 static void pool_status(struct dm_target *ti, status_type_t type,
3775                         unsigned status_flags, char *result, unsigned maxlen)
3776 {
3777         int r;
3778         unsigned sz = 0;
3779         uint64_t transaction_id;
3780         dm_block_t nr_free_blocks_data;
3781         dm_block_t nr_free_blocks_metadata;
3782         dm_block_t nr_blocks_data;
3783         dm_block_t nr_blocks_metadata;
3784         dm_block_t held_root;
3785         char buf[BDEVNAME_SIZE];
3786         char buf2[BDEVNAME_SIZE];
3787         struct pool_c *pt = ti->private;
3788         struct pool *pool = pt->pool;
3789
3790         switch (type) {
3791         case STATUSTYPE_INFO:
3792                 if (get_pool_mode(pool) == PM_FAIL) {
3793                         DMEMIT("Fail");
3794                         break;
3795                 }
3796
3797                 /* Commit to ensure statistics aren't out-of-date */
3798                 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3799                         (void) commit(pool);
3800
3801                 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
3802                 if (r) {
3803                         DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3804                               dm_device_name(pool->pool_md), r);
3805                         goto err;
3806                 }
3807
3808                 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
3809                 if (r) {
3810                         DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3811                               dm_device_name(pool->pool_md), r);
3812                         goto err;
3813                 }
3814
3815                 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
3816                 if (r) {
3817                         DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3818                               dm_device_name(pool->pool_md), r);
3819                         goto err;
3820                 }
3821
3822                 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
3823                 if (r) {
3824                         DMERR("%s: dm_pool_get_free_block_count returned %d",
3825                               dm_device_name(pool->pool_md), r);
3826                         goto err;
3827                 }
3828
3829                 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
3830                 if (r) {
3831                         DMERR("%s: dm_pool_get_data_dev_size returned %d",
3832                               dm_device_name(pool->pool_md), r);
3833                         goto err;
3834                 }
3835
3836                 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
3837                 if (r) {
3838                         DMERR("%s: dm_pool_get_metadata_snap returned %d",
3839                               dm_device_name(pool->pool_md), r);
3840                         goto err;
3841                 }
3842
3843                 DMEMIT("%llu %llu/%llu %llu/%llu ",
3844                        (unsigned long long)transaction_id,
3845                        (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3846                        (unsigned long long)nr_blocks_metadata,
3847                        (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
3848                        (unsigned long long)nr_blocks_data);
3849
3850                 if (held_root)
3851                         DMEMIT("%llu ", held_root);
3852                 else
3853                         DMEMIT("- ");
3854
3855                 if (pool->pf.mode == PM_OUT_OF_DATA_SPACE)
3856                         DMEMIT("out_of_data_space ");
3857                 else if (pool->pf.mode == PM_READ_ONLY)
3858                         DMEMIT("ro ");
3859                 else
3860                         DMEMIT("rw ");
3861
3862                 if (!pool->pf.discard_enabled)
3863                         DMEMIT("ignore_discard ");
3864                 else if (pool->pf.discard_passdown)
3865                         DMEMIT("discard_passdown ");
3866                 else
3867                         DMEMIT("no_discard_passdown ");
3868
3869                 if (pool->pf.error_if_no_space)
3870                         DMEMIT("error_if_no_space ");
3871                 else
3872                         DMEMIT("queue_if_no_space ");
3873
3874                 if (dm_pool_metadata_needs_check(pool->pmd))
3875                         DMEMIT("needs_check ");
3876                 else
3877                         DMEMIT("- ");
3878
3879                 break;
3880
3881         case STATUSTYPE_TABLE:
3882                 DMEMIT("%s %s %lu %llu ",
3883                        format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
3884                        format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
3885                        (unsigned long)pool->sectors_per_block,
3886                        (unsigned long long)pt->low_water_blocks);
3887                 emit_flags(&pt->requested_pf, result, sz, maxlen);
3888                 break;
3889         }
3890         return;
3891
3892 err:
3893         DMEMIT("Error");
3894 }
3895
3896 static int pool_iterate_devices(struct dm_target *ti,
3897                                 iterate_devices_callout_fn fn, void *data)
3898 {
3899         struct pool_c *pt = ti->private;
3900
3901         return fn(ti, pt->data_dev, 0, ti->len, data);
3902 }
3903
3904 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
3905 {
3906         struct pool_c *pt = ti->private;
3907         struct pool *pool = pt->pool;
3908         sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3909
3910         /*
3911          * If max_sectors is smaller than pool->sectors_per_block adjust it
3912          * to the highest possible power-of-2 factor of pool->sectors_per_block.
3913          * This is especially beneficial when the pool's data device is a RAID
3914          * device that has a full stripe width that matches pool->sectors_per_block
3915          * -- because even though partial RAID stripe-sized IOs will be issued to a
3916          *    single RAID stripe; when aggregated they will end on a full RAID stripe
3917          *    boundary.. which avoids additional partial RAID stripe writes cascading
3918          */
3919         if (limits->max_sectors < pool->sectors_per_block) {
3920                 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
3921                         if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
3922                                 limits->max_sectors--;
3923                         limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
3924                 }
3925         }
3926
3927         /*
3928          * If the system-determined stacked limits are compatible with the
3929          * pool's blocksize (io_opt is a factor) do not override them.
3930          */
3931         if (io_opt_sectors < pool->sectors_per_block ||
3932             !is_factor(io_opt_sectors, pool->sectors_per_block)) {
3933                 if (is_factor(pool->sectors_per_block, limits->max_sectors))
3934                         blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT);
3935                 else
3936                         blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
3937                 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
3938         }
3939
3940         /*
3941          * pt->adjusted_pf is a staging area for the actual features to use.
3942          * They get transferred to the live pool in bind_control_target()
3943          * called from pool_preresume().
3944          */
3945         if (!pt->adjusted_pf.discard_enabled) {
3946                 /*
3947                  * Must explicitly disallow stacking discard limits otherwise the
3948                  * block layer will stack them if pool's data device has support.
3949                  * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
3950                  * user to see that, so make sure to set all discard limits to 0.
3951                  */
3952                 limits->discard_granularity = 0;
3953                 return;
3954         }
3955
3956         disable_passdown_if_not_supported(pt);
3957
3958         /*
3959          * The pool uses the same discard limits as the underlying data
3960          * device.  DM core has already set this up.
3961          */
3962 }
3963
3964 static struct target_type pool_target = {
3965         .name = "thin-pool",
3966         .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
3967                     DM_TARGET_IMMUTABLE,
3968         .version = {1, 19, 0},
3969         .module = THIS_MODULE,
3970         .ctr = pool_ctr,
3971         .dtr = pool_dtr,
3972         .map = pool_map,
3973         .presuspend = pool_presuspend,
3974         .presuspend_undo = pool_presuspend_undo,
3975         .postsuspend = pool_postsuspend,
3976         .preresume = pool_preresume,
3977         .resume = pool_resume,
3978         .message = pool_message,
3979         .status = pool_status,
3980         .iterate_devices = pool_iterate_devices,
3981         .io_hints = pool_io_hints,
3982 };
3983
3984 /*----------------------------------------------------------------
3985  * Thin target methods
3986  *--------------------------------------------------------------*/
3987 static void thin_get(struct thin_c *tc)
3988 {
3989         atomic_inc(&tc->refcount);
3990 }
3991
3992 static void thin_put(struct thin_c *tc)
3993 {
3994         if (atomic_dec_and_test(&tc->refcount))
3995                 complete(&tc->can_destroy);
3996 }
3997
3998 static void thin_dtr(struct dm_target *ti)
3999 {
4000         struct thin_c *tc = ti->private;
4001         unsigned long flags;
4002
4003         spin_lock_irqsave(&tc->pool->lock, flags);
4004         list_del_rcu(&tc->list);
4005         spin_unlock_irqrestore(&tc->pool->lock, flags);
4006         synchronize_rcu();
4007
4008         thin_put(tc);
4009         wait_for_completion(&tc->can_destroy);
4010
4011         mutex_lock(&dm_thin_pool_table.mutex);
4012
4013         __pool_dec(tc->pool);
4014         dm_pool_close_thin_device(tc->td);
4015         dm_put_device(ti, tc->pool_dev);
4016         if (tc->origin_dev)
4017                 dm_put_device(ti, tc->origin_dev);
4018         kfree(tc);
4019
4020         mutex_unlock(&dm_thin_pool_table.mutex);
4021 }
4022
4023 /*
4024  * Thin target parameters:
4025  *
4026  * <pool_dev> <dev_id> [origin_dev]
4027  *
4028  * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
4029  * dev_id: the internal device identifier
4030  * origin_dev: a device external to the pool that should act as the origin
4031  *
4032  * If the pool device has discards disabled, they get disabled for the thin
4033  * device as well.
4034  */
4035 static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
4036 {
4037         int r;
4038         struct thin_c *tc;
4039         struct dm_dev *pool_dev, *origin_dev;
4040         struct mapped_device *pool_md;
4041         unsigned long flags;
4042
4043         mutex_lock(&dm_thin_pool_table.mutex);
4044
4045         if (argc != 2 && argc != 3) {
4046                 ti->error = "Invalid argument count";
4047                 r = -EINVAL;
4048                 goto out_unlock;
4049         }
4050
4051         tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
4052         if (!tc) {
4053                 ti->error = "Out of memory";
4054                 r = -ENOMEM;
4055                 goto out_unlock;
4056         }
4057         tc->thin_md = dm_table_get_md(ti->table);
4058         spin_lock_init(&tc->lock);
4059         INIT_LIST_HEAD(&tc->deferred_cells);
4060         bio_list_init(&tc->deferred_bio_list);
4061         bio_list_init(&tc->retry_on_resume_list);
4062         tc->sort_bio_list = RB_ROOT;
4063
4064         if (argc == 3) {
4065                 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
4066                 if (r) {
4067                         ti->error = "Error opening origin device";
4068                         goto bad_origin_dev;
4069                 }
4070                 tc->origin_dev = origin_dev;
4071         }
4072
4073         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
4074         if (r) {
4075                 ti->error = "Error opening pool device";
4076                 goto bad_pool_dev;
4077         }
4078         tc->pool_dev = pool_dev;
4079
4080         if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
4081                 ti->error = "Invalid device id";
4082                 r = -EINVAL;
4083                 goto bad_common;
4084         }
4085
4086         pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
4087         if (!pool_md) {
4088                 ti->error = "Couldn't get pool mapped device";
4089                 r = -EINVAL;
4090                 goto bad_common;
4091         }
4092
4093         tc->pool = __pool_table_lookup(pool_md);
4094         if (!tc->pool) {
4095                 ti->error = "Couldn't find pool object";
4096                 r = -EINVAL;
4097                 goto bad_pool_lookup;
4098         }
4099         __pool_inc(tc->pool);
4100
4101         if (get_pool_mode(tc->pool) == PM_FAIL) {
4102                 ti->error = "Couldn't open thin device, Pool is in fail mode";
4103                 r = -EINVAL;
4104                 goto bad_pool;
4105         }
4106
4107         r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
4108         if (r) {
4109                 ti->error = "Couldn't open thin internal device";
4110                 goto bad_pool;
4111         }
4112
4113         r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
4114         if (r)
4115                 goto bad;
4116
4117         ti->num_flush_bios = 1;
4118         ti->flush_supported = true;
4119         ti->per_io_data_size = sizeof(struct dm_thin_endio_hook);
4120
4121         /* In case the pool supports discards, pass them on. */
4122         ti->discard_zeroes_data_unsupported = true;
4123         if (tc->pool->pf.discard_enabled) {
4124                 ti->discards_supported = true;
4125                 ti->num_discard_bios = 1;
4126                 ti->split_discard_bios = false;
4127         }
4128
4129         mutex_unlock(&dm_thin_pool_table.mutex);
4130
4131         spin_lock_irqsave(&tc->pool->lock, flags);
4132         if (tc->pool->suspended) {
4133                 spin_unlock_irqrestore(&tc->pool->lock, flags);
4134                 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
4135                 ti->error = "Unable to activate thin device while pool is suspended";
4136                 r = -EINVAL;
4137                 goto bad;
4138         }
4139         atomic_set(&tc->refcount, 1);
4140         init_completion(&tc->can_destroy);
4141         list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
4142         spin_unlock_irqrestore(&tc->pool->lock, flags);
4143         /*
4144          * This synchronize_rcu() call is needed here otherwise we risk a
4145          * wake_worker() call finding no bios to process (because the newly
4146          * added tc isn't yet visible).  So this reduces latency since we
4147          * aren't then dependent on the periodic commit to wake_worker().
4148          */
4149         synchronize_rcu();
4150
4151         dm_put(pool_md);
4152
4153         return 0;
4154
4155 bad:
4156         dm_pool_close_thin_device(tc->td);
4157 bad_pool:
4158         __pool_dec(tc->pool);
4159 bad_pool_lookup:
4160         dm_put(pool_md);
4161 bad_common:
4162         dm_put_device(ti, tc->pool_dev);
4163 bad_pool_dev:
4164         if (tc->origin_dev)
4165                 dm_put_device(ti, tc->origin_dev);
4166 bad_origin_dev:
4167         kfree(tc);
4168 out_unlock:
4169         mutex_unlock(&dm_thin_pool_table.mutex);
4170
4171         return r;
4172 }
4173
4174 static int thin_map(struct dm_target *ti, struct bio *bio)
4175 {
4176         bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
4177
4178         return thin_bio_map(ti, bio);
4179 }
4180
4181 static int thin_endio(struct dm_target *ti, struct bio *bio, int err)
4182 {
4183         unsigned long flags;
4184         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
4185         struct list_head work;
4186         struct dm_thin_new_mapping *m, *tmp;
4187         struct pool *pool = h->tc->pool;
4188
4189         if (h->shared_read_entry) {
4190                 INIT_LIST_HEAD(&work);
4191                 dm_deferred_entry_dec(h->shared_read_entry, &work);
4192
4193                 spin_lock_irqsave(&pool->lock, flags);
4194                 list_for_each_entry_safe(m, tmp, &work, list) {
4195                         list_del(&m->list);
4196                         __complete_mapping_preparation(m);
4197                 }
4198                 spin_unlock_irqrestore(&pool->lock, flags);
4199         }
4200
4201         if (h->all_io_entry) {
4202                 INIT_LIST_HEAD(&work);
4203                 dm_deferred_entry_dec(h->all_io_entry, &work);
4204                 if (!list_empty(&work)) {
4205                         spin_lock_irqsave(&pool->lock, flags);
4206                         list_for_each_entry_safe(m, tmp, &work, list)
4207                                 list_add_tail(&m->list, &pool->prepared_discards);
4208                         spin_unlock_irqrestore(&pool->lock, flags);
4209                         wake_worker(pool);
4210                 }
4211         }
4212
4213         if (h->cell)
4214                 cell_defer_no_holder(h->tc, h->cell);
4215
4216         return 0;
4217 }
4218
4219 static void thin_presuspend(struct dm_target *ti)
4220 {
4221         struct thin_c *tc = ti->private;
4222
4223         if (dm_noflush_suspending(ti))
4224                 noflush_work(tc, do_noflush_start);
4225 }
4226
4227 static void thin_postsuspend(struct dm_target *ti)
4228 {
4229         struct thin_c *tc = ti->private;
4230
4231         /*
4232          * The dm_noflush_suspending flag has been cleared by now, so
4233          * unfortunately we must always run this.
4234          */
4235         noflush_work(tc, do_noflush_stop);
4236 }
4237
4238 static int thin_preresume(struct dm_target *ti)
4239 {
4240         struct thin_c *tc = ti->private;
4241
4242         if (tc->origin_dev)
4243                 tc->origin_size = get_dev_size(tc->origin_dev->bdev);
4244
4245         return 0;
4246 }
4247
4248 /*
4249  * <nr mapped sectors> <highest mapped sector>
4250  */
4251 static void thin_status(struct dm_target *ti, status_type_t type,
4252                         unsigned status_flags, char *result, unsigned maxlen)
4253 {
4254         int r;
4255         ssize_t sz = 0;
4256         dm_block_t mapped, highest;
4257         char buf[BDEVNAME_SIZE];
4258         struct thin_c *tc = ti->private;
4259
4260         if (get_pool_mode(tc->pool) == PM_FAIL) {
4261                 DMEMIT("Fail");
4262                 return;
4263         }
4264
4265         if (!tc->td)
4266                 DMEMIT("-");
4267         else {
4268                 switch (type) {
4269                 case STATUSTYPE_INFO:
4270                         r = dm_thin_get_mapped_count(tc->td, &mapped);
4271                         if (r) {
4272                                 DMERR("dm_thin_get_mapped_count returned %d", r);
4273                                 goto err;
4274                         }
4275
4276                         r = dm_thin_get_highest_mapped_block(tc->td, &highest);
4277                         if (r < 0) {
4278                                 DMERR("dm_thin_get_highest_mapped_block returned %d", r);
4279                                 goto err;
4280                         }
4281
4282                         DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
4283                         if (r)
4284                                 DMEMIT("%llu", ((highest + 1) *
4285                                                 tc->pool->sectors_per_block) - 1);
4286                         else
4287                                 DMEMIT("-");
4288                         break;
4289
4290                 case STATUSTYPE_TABLE:
4291                         DMEMIT("%s %lu",
4292                                format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
4293                                (unsigned long) tc->dev_id);
4294                         if (tc->origin_dev)
4295                                 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
4296                         break;
4297                 }
4298         }
4299
4300         return;
4301
4302 err:
4303         DMEMIT("Error");
4304 }
4305
4306 static int thin_iterate_devices(struct dm_target *ti,
4307                                 iterate_devices_callout_fn fn, void *data)
4308 {
4309         sector_t blocks;
4310         struct thin_c *tc = ti->private;
4311         struct pool *pool = tc->pool;
4312
4313         /*
4314          * We can't call dm_pool_get_data_dev_size() since that blocks.  So
4315          * we follow a more convoluted path through to the pool's target.
4316          */
4317         if (!pool->ti)
4318                 return 0;       /* nothing is bound */
4319
4320         blocks = pool->ti->len;
4321         (void) sector_div(blocks, pool->sectors_per_block);
4322         if (blocks)
4323                 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
4324
4325         return 0;
4326 }
4327
4328 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
4329 {
4330         struct thin_c *tc = ti->private;
4331         struct pool *pool = tc->pool;
4332
4333         if (!pool->pf.discard_enabled)
4334                 return;
4335
4336         limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
4337         limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
4338 }
4339
4340 static struct target_type thin_target = {
4341         .name = "thin",
4342         .version = {1, 19, 0},
4343         .module = THIS_MODULE,
4344         .ctr = thin_ctr,
4345         .dtr = thin_dtr,
4346         .map = thin_map,
4347         .end_io = thin_endio,
4348         .preresume = thin_preresume,
4349         .presuspend = thin_presuspend,
4350         .postsuspend = thin_postsuspend,
4351         .status = thin_status,
4352         .iterate_devices = thin_iterate_devices,
4353         .io_hints = thin_io_hints,
4354 };
4355
4356 /*----------------------------------------------------------------*/
4357
4358 static int __init dm_thin_init(void)
4359 {
4360         int r;
4361
4362         pool_table_init();
4363
4364         r = dm_register_target(&thin_target);
4365         if (r)
4366                 return r;
4367
4368         r = dm_register_target(&pool_target);
4369         if (r)
4370                 goto bad_pool_target;
4371
4372         r = -ENOMEM;
4373
4374         _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
4375         if (!_new_mapping_cache)
4376                 goto bad_new_mapping_cache;
4377
4378         return 0;
4379
4380 bad_new_mapping_cache:
4381         dm_unregister_target(&pool_target);
4382 bad_pool_target:
4383         dm_unregister_target(&thin_target);
4384
4385         return r;
4386 }
4387
4388 static void dm_thin_exit(void)
4389 {
4390         dm_unregister_target(&thin_target);
4391         dm_unregister_target(&pool_target);
4392
4393         kmem_cache_destroy(_new_mapping_cache);
4394 }
4395
4396 module_init(dm_thin_init);
4397 module_exit(dm_thin_exit);
4398
4399 module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
4400 MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
4401
4402 MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
4403 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
4404 MODULE_LICENSE("GPL");