188121ca00aab7a4bcd65343ab116d14b362a579
[~shefty/rdma-dev.git] / drivers / md / dm-thin.c
1 /*
2  * Copyright (C) 2011 Red Hat UK.
3  *
4  * This file is released under the GPL.
5  */
6
7 #include "dm-thin-metadata.h"
8
9 #include <linux/device-mapper.h>
10 #include <linux/dm-io.h>
11 #include <linux/dm-kcopyd.h>
12 #include <linux/list.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/slab.h>
16
17 #define DM_MSG_PREFIX   "thin"
18
19 /*
20  * Tunable constants
21  */
22 #define ENDIO_HOOK_POOL_SIZE 10240
23 #define DEFERRED_SET_SIZE 64
24 #define MAPPING_POOL_SIZE 1024
25 #define PRISON_CELLS 1024
26 #define COMMIT_PERIOD HZ
27
28 /*
29  * The block size of the device holding pool data must be
30  * between 64KB and 1GB.
31  */
32 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
33 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
34
35 /*
36  * Device id is restricted to 24 bits.
37  */
38 #define MAX_DEV_ID ((1 << 24) - 1)
39
40 /*
41  * How do we handle breaking sharing of data blocks?
42  * =================================================
43  *
44  * We use a standard copy-on-write btree to store the mappings for the
45  * devices (note I'm talking about copy-on-write of the metadata here, not
46  * the data).  When you take an internal snapshot you clone the root node
47  * of the origin btree.  After this there is no concept of an origin or a
48  * snapshot.  They are just two device trees that happen to point to the
49  * same data blocks.
50  *
51  * When we get a write in we decide if it's to a shared data block using
52  * some timestamp magic.  If it is, we have to break sharing.
53  *
54  * Let's say we write to a shared block in what was the origin.  The
55  * steps are:
56  *
57  * i) plug io further to this physical block. (see bio_prison code).
58  *
59  * ii) quiesce any read io to that shared data block.  Obviously
60  * including all devices that share this block.  (see deferred_set code)
61  *
62  * iii) copy the data block to a newly allocate block.  This step can be
63  * missed out if the io covers the block. (schedule_copy).
64  *
65  * iv) insert the new mapping into the origin's btree
66  * (process_prepared_mapping).  This act of inserting breaks some
67  * sharing of btree nodes between the two devices.  Breaking sharing only
68  * effects the btree of that specific device.  Btrees for the other
69  * devices that share the block never change.  The btree for the origin
70  * device as it was after the last commit is untouched, ie. we're using
71  * persistent data structures in the functional programming sense.
72  *
73  * v) unplug io to this physical block, including the io that triggered
74  * the breaking of sharing.
75  *
76  * Steps (ii) and (iii) occur in parallel.
77  *
78  * The metadata _doesn't_ need to be committed before the io continues.  We
79  * get away with this because the io is always written to a _new_ block.
80  * If there's a crash, then:
81  *
82  * - The origin mapping will point to the old origin block (the shared
83  * one).  This will contain the data as it was before the io that triggered
84  * the breaking of sharing came in.
85  *
86  * - The snap mapping still points to the old block.  As it would after
87  * the commit.
88  *
89  * The downside of this scheme is the timestamp magic isn't perfect, and
90  * will continue to think that data block in the snapshot device is shared
91  * even after the write to the origin has broken sharing.  I suspect data
92  * blocks will typically be shared by many different devices, so we're
93  * breaking sharing n + 1 times, rather than n, where n is the number of
94  * devices that reference this data block.  At the moment I think the
95  * benefits far, far outweigh the disadvantages.
96  */
97
98 /*----------------------------------------------------------------*/
99
100 /*
101  * Sometimes we can't deal with a bio straight away.  We put them in prison
102  * where they can't cause any mischief.  Bios are put in a cell identified
103  * by a key, multiple bios can be in the same cell.  When the cell is
104  * subsequently unlocked the bios become available.
105  */
106 struct bio_prison;
107
108 struct cell_key {
109         int virtual;
110         dm_thin_id dev;
111         dm_block_t block;
112 };
113
114 struct cell {
115         struct hlist_node list;
116         struct bio_prison *prison;
117         struct cell_key key;
118         struct bio *holder;
119         struct bio_list bios;
120 };
121
122 struct bio_prison {
123         spinlock_t lock;
124         mempool_t *cell_pool;
125
126         unsigned nr_buckets;
127         unsigned hash_mask;
128         struct hlist_head *cells;
129 };
130
131 static uint32_t calc_nr_buckets(unsigned nr_cells)
132 {
133         uint32_t n = 128;
134
135         nr_cells /= 4;
136         nr_cells = min(nr_cells, 8192u);
137
138         while (n < nr_cells)
139                 n <<= 1;
140
141         return n;
142 }
143
144 /*
145  * @nr_cells should be the number of cells you want in use _concurrently_.
146  * Don't confuse it with the number of distinct keys.
147  */
148 static struct bio_prison *prison_create(unsigned nr_cells)
149 {
150         unsigned i;
151         uint32_t nr_buckets = calc_nr_buckets(nr_cells);
152         size_t len = sizeof(struct bio_prison) +
153                 (sizeof(struct hlist_head) * nr_buckets);
154         struct bio_prison *prison = kmalloc(len, GFP_KERNEL);
155
156         if (!prison)
157                 return NULL;
158
159         spin_lock_init(&prison->lock);
160         prison->cell_pool = mempool_create_kmalloc_pool(nr_cells,
161                                                         sizeof(struct cell));
162         if (!prison->cell_pool) {
163                 kfree(prison);
164                 return NULL;
165         }
166
167         prison->nr_buckets = nr_buckets;
168         prison->hash_mask = nr_buckets - 1;
169         prison->cells = (struct hlist_head *) (prison + 1);
170         for (i = 0; i < nr_buckets; i++)
171                 INIT_HLIST_HEAD(prison->cells + i);
172
173         return prison;
174 }
175
176 static void prison_destroy(struct bio_prison *prison)
177 {
178         mempool_destroy(prison->cell_pool);
179         kfree(prison);
180 }
181
182 static uint32_t hash_key(struct bio_prison *prison, struct cell_key *key)
183 {
184         const unsigned long BIG_PRIME = 4294967291UL;
185         uint64_t hash = key->block * BIG_PRIME;
186
187         return (uint32_t) (hash & prison->hash_mask);
188 }
189
190 static int keys_equal(struct cell_key *lhs, struct cell_key *rhs)
191 {
192                return (lhs->virtual == rhs->virtual) &&
193                        (lhs->dev == rhs->dev) &&
194                        (lhs->block == rhs->block);
195 }
196
197 static struct cell *__search_bucket(struct hlist_head *bucket,
198                                     struct cell_key *key)
199 {
200         struct cell *cell;
201         struct hlist_node *tmp;
202
203         hlist_for_each_entry(cell, tmp, bucket, list)
204                 if (keys_equal(&cell->key, key))
205                         return cell;
206
207         return NULL;
208 }
209
210 /*
211  * This may block if a new cell needs allocating.  You must ensure that
212  * cells will be unlocked even if the calling thread is blocked.
213  *
214  * Returns 1 if the cell was already held, 0 if @inmate is the new holder.
215  */
216 static int bio_detain(struct bio_prison *prison, struct cell_key *key,
217                       struct bio *inmate, struct cell **ref)
218 {
219         int r = 1;
220         unsigned long flags;
221         uint32_t hash = hash_key(prison, key);
222         struct cell *cell, *cell2;
223
224         BUG_ON(hash > prison->nr_buckets);
225
226         spin_lock_irqsave(&prison->lock, flags);
227
228         cell = __search_bucket(prison->cells + hash, key);
229         if (cell) {
230                 bio_list_add(&cell->bios, inmate);
231                 goto out;
232         }
233
234         /*
235          * Allocate a new cell
236          */
237         spin_unlock_irqrestore(&prison->lock, flags);
238         cell2 = mempool_alloc(prison->cell_pool, GFP_NOIO);
239         spin_lock_irqsave(&prison->lock, flags);
240
241         /*
242          * We've been unlocked, so we have to double check that
243          * nobody else has inserted this cell in the meantime.
244          */
245         cell = __search_bucket(prison->cells + hash, key);
246         if (cell) {
247                 mempool_free(cell2, prison->cell_pool);
248                 bio_list_add(&cell->bios, inmate);
249                 goto out;
250         }
251
252         /*
253          * Use new cell.
254          */
255         cell = cell2;
256
257         cell->prison = prison;
258         memcpy(&cell->key, key, sizeof(cell->key));
259         cell->holder = inmate;
260         bio_list_init(&cell->bios);
261         hlist_add_head(&cell->list, prison->cells + hash);
262
263         r = 0;
264
265 out:
266         spin_unlock_irqrestore(&prison->lock, flags);
267
268         *ref = cell;
269
270         return r;
271 }
272
273 /*
274  * @inmates must have been initialised prior to this call
275  */
276 static void __cell_release(struct cell *cell, struct bio_list *inmates)
277 {
278         struct bio_prison *prison = cell->prison;
279
280         hlist_del(&cell->list);
281
282         bio_list_add(inmates, cell->holder);
283         bio_list_merge(inmates, &cell->bios);
284
285         mempool_free(cell, prison->cell_pool);
286 }
287
288 static void cell_release(struct cell *cell, struct bio_list *bios)
289 {
290         unsigned long flags;
291         struct bio_prison *prison = cell->prison;
292
293         spin_lock_irqsave(&prison->lock, flags);
294         __cell_release(cell, bios);
295         spin_unlock_irqrestore(&prison->lock, flags);
296 }
297
298 /*
299  * There are a couple of places where we put a bio into a cell briefly
300  * before taking it out again.  In these situations we know that no other
301  * bio may be in the cell.  This function releases the cell, and also does
302  * a sanity check.
303  */
304 static void __cell_release_singleton(struct cell *cell, struct bio *bio)
305 {
306         hlist_del(&cell->list);
307         BUG_ON(cell->holder != bio);
308         BUG_ON(!bio_list_empty(&cell->bios));
309 }
310
311 static void cell_release_singleton(struct cell *cell, struct bio *bio)
312 {
313         unsigned long flags;
314         struct bio_prison *prison = cell->prison;
315
316         spin_lock_irqsave(&prison->lock, flags);
317         __cell_release_singleton(cell, bio);
318         spin_unlock_irqrestore(&prison->lock, flags);
319 }
320
321 /*
322  * Sometimes we don't want the holder, just the additional bios.
323  */
324 static void __cell_release_no_holder(struct cell *cell, struct bio_list *inmates)
325 {
326         struct bio_prison *prison = cell->prison;
327
328         hlist_del(&cell->list);
329         bio_list_merge(inmates, &cell->bios);
330
331         mempool_free(cell, prison->cell_pool);
332 }
333
334 static void cell_release_no_holder(struct cell *cell, struct bio_list *inmates)
335 {
336         unsigned long flags;
337         struct bio_prison *prison = cell->prison;
338
339         spin_lock_irqsave(&prison->lock, flags);
340         __cell_release_no_holder(cell, inmates);
341         spin_unlock_irqrestore(&prison->lock, flags);
342 }
343
344 static void cell_error(struct cell *cell)
345 {
346         struct bio_prison *prison = cell->prison;
347         struct bio_list bios;
348         struct bio *bio;
349         unsigned long flags;
350
351         bio_list_init(&bios);
352
353         spin_lock_irqsave(&prison->lock, flags);
354         __cell_release(cell, &bios);
355         spin_unlock_irqrestore(&prison->lock, flags);
356
357         while ((bio = bio_list_pop(&bios)))
358                 bio_io_error(bio);
359 }
360
361 /*----------------------------------------------------------------*/
362
363 /*
364  * We use the deferred set to keep track of pending reads to shared blocks.
365  * We do this to ensure the new mapping caused by a write isn't performed
366  * until these prior reads have completed.  Otherwise the insertion of the
367  * new mapping could free the old block that the read bios are mapped to.
368  */
369
370 struct deferred_set;
371 struct deferred_entry {
372         struct deferred_set *ds;
373         unsigned count;
374         struct list_head work_items;
375 };
376
377 struct deferred_set {
378         spinlock_t lock;
379         unsigned current_entry;
380         unsigned sweeper;
381         struct deferred_entry entries[DEFERRED_SET_SIZE];
382 };
383
384 static void ds_init(struct deferred_set *ds)
385 {
386         int i;
387
388         spin_lock_init(&ds->lock);
389         ds->current_entry = 0;
390         ds->sweeper = 0;
391         for (i = 0; i < DEFERRED_SET_SIZE; i++) {
392                 ds->entries[i].ds = ds;
393                 ds->entries[i].count = 0;
394                 INIT_LIST_HEAD(&ds->entries[i].work_items);
395         }
396 }
397
398 static struct deferred_entry *ds_inc(struct deferred_set *ds)
399 {
400         unsigned long flags;
401         struct deferred_entry *entry;
402
403         spin_lock_irqsave(&ds->lock, flags);
404         entry = ds->entries + ds->current_entry;
405         entry->count++;
406         spin_unlock_irqrestore(&ds->lock, flags);
407
408         return entry;
409 }
410
411 static unsigned ds_next(unsigned index)
412 {
413         return (index + 1) % DEFERRED_SET_SIZE;
414 }
415
416 static void __sweep(struct deferred_set *ds, struct list_head *head)
417 {
418         while ((ds->sweeper != ds->current_entry) &&
419                !ds->entries[ds->sweeper].count) {
420                 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
421                 ds->sweeper = ds_next(ds->sweeper);
422         }
423
424         if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
425                 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
426 }
427
428 static void ds_dec(struct deferred_entry *entry, struct list_head *head)
429 {
430         unsigned long flags;
431
432         spin_lock_irqsave(&entry->ds->lock, flags);
433         BUG_ON(!entry->count);
434         --entry->count;
435         __sweep(entry->ds, head);
436         spin_unlock_irqrestore(&entry->ds->lock, flags);
437 }
438
439 /*
440  * Returns 1 if deferred or 0 if no pending items to delay job.
441  */
442 static int ds_add_work(struct deferred_set *ds, struct list_head *work)
443 {
444         int r = 1;
445         unsigned long flags;
446         unsigned next_entry;
447
448         spin_lock_irqsave(&ds->lock, flags);
449         if ((ds->sweeper == ds->current_entry) &&
450             !ds->entries[ds->current_entry].count)
451                 r = 0;
452         else {
453                 list_add(work, &ds->entries[ds->current_entry].work_items);
454                 next_entry = ds_next(ds->current_entry);
455                 if (!ds->entries[next_entry].count)
456                         ds->current_entry = next_entry;
457         }
458         spin_unlock_irqrestore(&ds->lock, flags);
459
460         return r;
461 }
462
463 /*----------------------------------------------------------------*/
464
465 /*
466  * Key building.
467  */
468 static void build_data_key(struct dm_thin_device *td,
469                            dm_block_t b, struct cell_key *key)
470 {
471         key->virtual = 0;
472         key->dev = dm_thin_dev_id(td);
473         key->block = b;
474 }
475
476 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
477                               struct cell_key *key)
478 {
479         key->virtual = 1;
480         key->dev = dm_thin_dev_id(td);
481         key->block = b;
482 }
483
484 /*----------------------------------------------------------------*/
485
486 /*
487  * A pool device ties together a metadata device and a data device.  It
488  * also provides the interface for creating and destroying internal
489  * devices.
490  */
491 struct new_mapping;
492 struct pool {
493         struct list_head list;
494         struct dm_target *ti;   /* Only set if a pool target is bound */
495
496         struct mapped_device *pool_md;
497         struct block_device *md_dev;
498         struct dm_pool_metadata *pmd;
499
500         uint32_t sectors_per_block;
501         unsigned block_shift;
502         dm_block_t offset_mask;
503         dm_block_t low_water_blocks;
504
505         unsigned zero_new_blocks:1;
506         unsigned low_water_triggered:1; /* A dm event has been sent */
507         unsigned no_free_space:1;       /* A -ENOSPC warning has been issued */
508
509         struct bio_prison *prison;
510         struct dm_kcopyd_client *copier;
511
512         struct workqueue_struct *wq;
513         struct work_struct worker;
514         struct delayed_work waker;
515
516         unsigned ref_count;
517         unsigned long last_commit_jiffies;
518
519         spinlock_t lock;
520         struct bio_list deferred_bios;
521         struct bio_list deferred_flush_bios;
522         struct list_head prepared_mappings;
523
524         struct bio_list retry_on_resume_list;
525
526         struct deferred_set shared_read_ds;
527
528         struct new_mapping *next_mapping;
529         mempool_t *mapping_pool;
530         mempool_t *endio_hook_pool;
531 };
532
533 /*
534  * Target context for a pool.
535  */
536 struct pool_c {
537         struct dm_target *ti;
538         struct pool *pool;
539         struct dm_dev *data_dev;
540         struct dm_dev *metadata_dev;
541         struct dm_target_callbacks callbacks;
542
543         dm_block_t low_water_blocks;
544         unsigned zero_new_blocks:1;
545 };
546
547 /*
548  * Target context for a thin.
549  */
550 struct thin_c {
551         struct dm_dev *pool_dev;
552         struct dm_dev *origin_dev;
553         dm_thin_id dev_id;
554
555         struct pool *pool;
556         struct dm_thin_device *td;
557 };
558
559 /*----------------------------------------------------------------*/
560
561 /*
562  * A global list of pools that uses a struct mapped_device as a key.
563  */
564 static struct dm_thin_pool_table {
565         struct mutex mutex;
566         struct list_head pools;
567 } dm_thin_pool_table;
568
569 static void pool_table_init(void)
570 {
571         mutex_init(&dm_thin_pool_table.mutex);
572         INIT_LIST_HEAD(&dm_thin_pool_table.pools);
573 }
574
575 static void __pool_table_insert(struct pool *pool)
576 {
577         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
578         list_add(&pool->list, &dm_thin_pool_table.pools);
579 }
580
581 static void __pool_table_remove(struct pool *pool)
582 {
583         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
584         list_del(&pool->list);
585 }
586
587 static struct pool *__pool_table_lookup(struct mapped_device *md)
588 {
589         struct pool *pool = NULL, *tmp;
590
591         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
592
593         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
594                 if (tmp->pool_md == md) {
595                         pool = tmp;
596                         break;
597                 }
598         }
599
600         return pool;
601 }
602
603 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
604 {
605         struct pool *pool = NULL, *tmp;
606
607         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
608
609         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
610                 if (tmp->md_dev == md_dev) {
611                         pool = tmp;
612                         break;
613                 }
614         }
615
616         return pool;
617 }
618
619 /*----------------------------------------------------------------*/
620
621 struct endio_hook {
622         struct thin_c *tc;
623         struct deferred_entry *shared_read_entry;
624         struct new_mapping *overwrite_mapping;
625 };
626
627 static void __requeue_bio_list(struct thin_c *tc, struct bio_list *master)
628 {
629         struct bio *bio;
630         struct bio_list bios;
631
632         bio_list_init(&bios);
633         bio_list_merge(&bios, master);
634         bio_list_init(master);
635
636         while ((bio = bio_list_pop(&bios))) {
637                 struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
638                 if (h->tc == tc)
639                         bio_endio(bio, DM_ENDIO_REQUEUE);
640                 else
641                         bio_list_add(master, bio);
642         }
643 }
644
645 static void requeue_io(struct thin_c *tc)
646 {
647         struct pool *pool = tc->pool;
648         unsigned long flags;
649
650         spin_lock_irqsave(&pool->lock, flags);
651         __requeue_bio_list(tc, &pool->deferred_bios);
652         __requeue_bio_list(tc, &pool->retry_on_resume_list);
653         spin_unlock_irqrestore(&pool->lock, flags);
654 }
655
656 /*
657  * This section of code contains the logic for processing a thin device's IO.
658  * Much of the code depends on pool object resources (lists, workqueues, etc)
659  * but most is exclusively called from the thin target rather than the thin-pool
660  * target.
661  */
662
663 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
664 {
665         return bio->bi_sector >> tc->pool->block_shift;
666 }
667
668 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
669 {
670         struct pool *pool = tc->pool;
671
672         bio->bi_bdev = tc->pool_dev->bdev;
673         bio->bi_sector = (block << pool->block_shift) +
674                 (bio->bi_sector & pool->offset_mask);
675 }
676
677 static void remap_to_origin(struct thin_c *tc, struct bio *bio)
678 {
679         bio->bi_bdev = tc->origin_dev->bdev;
680 }
681
682 static void issue(struct thin_c *tc, struct bio *bio)
683 {
684         struct pool *pool = tc->pool;
685         unsigned long flags;
686
687         /*
688          * Batch together any FUA/FLUSH bios we find and then issue
689          * a single commit for them in process_deferred_bios().
690          */
691         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
692                 spin_lock_irqsave(&pool->lock, flags);
693                 bio_list_add(&pool->deferred_flush_bios, bio);
694                 spin_unlock_irqrestore(&pool->lock, flags);
695         } else
696                 generic_make_request(bio);
697 }
698
699 static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
700 {
701         remap_to_origin(tc, bio);
702         issue(tc, bio);
703 }
704
705 static void remap_and_issue(struct thin_c *tc, struct bio *bio,
706                             dm_block_t block)
707 {
708         remap(tc, bio, block);
709         issue(tc, bio);
710 }
711
712 /*
713  * wake_worker() is used when new work is queued and when pool_resume is
714  * ready to continue deferred IO processing.
715  */
716 static void wake_worker(struct pool *pool)
717 {
718         queue_work(pool->wq, &pool->worker);
719 }
720
721 /*----------------------------------------------------------------*/
722
723 /*
724  * Bio endio functions.
725  */
726 struct new_mapping {
727         struct list_head list;
728
729         unsigned quiesced:1;
730         unsigned prepared:1;
731
732         struct thin_c *tc;
733         dm_block_t virt_block;
734         dm_block_t data_block;
735         struct cell *cell;
736         int err;
737
738         /*
739          * If the bio covers the whole area of a block then we can avoid
740          * zeroing or copying.  Instead this bio is hooked.  The bio will
741          * still be in the cell, so care has to be taken to avoid issuing
742          * the bio twice.
743          */
744         struct bio *bio;
745         bio_end_io_t *saved_bi_end_io;
746 };
747
748 static void __maybe_add_mapping(struct new_mapping *m)
749 {
750         struct pool *pool = m->tc->pool;
751
752         if (m->quiesced && m->prepared) {
753                 list_add(&m->list, &pool->prepared_mappings);
754                 wake_worker(pool);
755         }
756 }
757
758 static void copy_complete(int read_err, unsigned long write_err, void *context)
759 {
760         unsigned long flags;
761         struct new_mapping *m = context;
762         struct pool *pool = m->tc->pool;
763
764         m->err = read_err || write_err ? -EIO : 0;
765
766         spin_lock_irqsave(&pool->lock, flags);
767         m->prepared = 1;
768         __maybe_add_mapping(m);
769         spin_unlock_irqrestore(&pool->lock, flags);
770 }
771
772 static void overwrite_endio(struct bio *bio, int err)
773 {
774         unsigned long flags;
775         struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
776         struct new_mapping *m = h->overwrite_mapping;
777         struct pool *pool = m->tc->pool;
778
779         m->err = err;
780
781         spin_lock_irqsave(&pool->lock, flags);
782         m->prepared = 1;
783         __maybe_add_mapping(m);
784         spin_unlock_irqrestore(&pool->lock, flags);
785 }
786
787 /*----------------------------------------------------------------*/
788
789 /*
790  * Workqueue.
791  */
792
793 /*
794  * Prepared mapping jobs.
795  */
796
797 /*
798  * This sends the bios in the cell back to the deferred_bios list.
799  */
800 static void cell_defer(struct thin_c *tc, struct cell *cell,
801                        dm_block_t data_block)
802 {
803         struct pool *pool = tc->pool;
804         unsigned long flags;
805
806         spin_lock_irqsave(&pool->lock, flags);
807         cell_release(cell, &pool->deferred_bios);
808         spin_unlock_irqrestore(&tc->pool->lock, flags);
809
810         wake_worker(pool);
811 }
812
813 /*
814  * Same as cell_defer above, except it omits one particular detainee,
815  * a write bio that covers the block and has already been processed.
816  */
817 static void cell_defer_except(struct thin_c *tc, struct cell *cell)
818 {
819         struct bio_list bios;
820         struct pool *pool = tc->pool;
821         unsigned long flags;
822
823         bio_list_init(&bios);
824
825         spin_lock_irqsave(&pool->lock, flags);
826         cell_release_no_holder(cell, &pool->deferred_bios);
827         spin_unlock_irqrestore(&pool->lock, flags);
828
829         wake_worker(pool);
830 }
831
832 static void process_prepared_mapping(struct new_mapping *m)
833 {
834         struct thin_c *tc = m->tc;
835         struct bio *bio;
836         int r;
837
838         bio = m->bio;
839         if (bio)
840                 bio->bi_end_io = m->saved_bi_end_io;
841
842         if (m->err) {
843                 cell_error(m->cell);
844                 return;
845         }
846
847         /*
848          * Commit the prepared block into the mapping btree.
849          * Any I/O for this block arriving after this point will get
850          * remapped to it directly.
851          */
852         r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);
853         if (r) {
854                 DMERR("dm_thin_insert_block() failed");
855                 cell_error(m->cell);
856                 return;
857         }
858
859         /*
860          * Release any bios held while the block was being provisioned.
861          * If we are processing a write bio that completely covers the block,
862          * we already processed it so can ignore it now when processing
863          * the bios in the cell.
864          */
865         if (bio) {
866                 cell_defer_except(tc, m->cell);
867                 bio_endio(bio, 0);
868         } else
869                 cell_defer(tc, m->cell, m->data_block);
870
871         list_del(&m->list);
872         mempool_free(m, tc->pool->mapping_pool);
873 }
874
875 static void process_prepared_mappings(struct pool *pool)
876 {
877         unsigned long flags;
878         struct list_head maps;
879         struct new_mapping *m, *tmp;
880
881         INIT_LIST_HEAD(&maps);
882         spin_lock_irqsave(&pool->lock, flags);
883         list_splice_init(&pool->prepared_mappings, &maps);
884         spin_unlock_irqrestore(&pool->lock, flags);
885
886         list_for_each_entry_safe(m, tmp, &maps, list)
887                 process_prepared_mapping(m);
888 }
889
890 /*
891  * Deferred bio jobs.
892  */
893 static int io_overwrites_block(struct pool *pool, struct bio *bio)
894 {
895         return ((bio_data_dir(bio) == WRITE) &&
896                 !(bio->bi_sector & pool->offset_mask)) &&
897                 (bio->bi_size == (pool->sectors_per_block << SECTOR_SHIFT));
898 }
899
900 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
901                                bio_end_io_t *fn)
902 {
903         *save = bio->bi_end_io;
904         bio->bi_end_io = fn;
905 }
906
907 static int ensure_next_mapping(struct pool *pool)
908 {
909         if (pool->next_mapping)
910                 return 0;
911
912         pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
913
914         return pool->next_mapping ? 0 : -ENOMEM;
915 }
916
917 static struct new_mapping *get_next_mapping(struct pool *pool)
918 {
919         struct new_mapping *r = pool->next_mapping;
920
921         BUG_ON(!pool->next_mapping);
922
923         pool->next_mapping = NULL;
924
925         return r;
926 }
927
928 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
929                           struct dm_dev *origin, dm_block_t data_origin,
930                           dm_block_t data_dest,
931                           struct cell *cell, struct bio *bio)
932 {
933         int r;
934         struct pool *pool = tc->pool;
935         struct new_mapping *m = get_next_mapping(pool);
936
937         INIT_LIST_HEAD(&m->list);
938         m->quiesced = 0;
939         m->prepared = 0;
940         m->tc = tc;
941         m->virt_block = virt_block;
942         m->data_block = data_dest;
943         m->cell = cell;
944         m->err = 0;
945         m->bio = NULL;
946
947         if (!ds_add_work(&pool->shared_read_ds, &m->list))
948                 m->quiesced = 1;
949
950         /*
951          * IO to pool_dev remaps to the pool target's data_dev.
952          *
953          * If the whole block of data is being overwritten, we can issue the
954          * bio immediately. Otherwise we use kcopyd to clone the data first.
955          */
956         if (io_overwrites_block(pool, bio)) {
957                 struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
958                 h->overwrite_mapping = m;
959                 m->bio = bio;
960                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
961                 remap_and_issue(tc, bio, data_dest);
962         } else {
963                 struct dm_io_region from, to;
964
965                 from.bdev = origin->bdev;
966                 from.sector = data_origin * pool->sectors_per_block;
967                 from.count = pool->sectors_per_block;
968
969                 to.bdev = tc->pool_dev->bdev;
970                 to.sector = data_dest * pool->sectors_per_block;
971                 to.count = pool->sectors_per_block;
972
973                 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
974                                    0, copy_complete, m);
975                 if (r < 0) {
976                         mempool_free(m, pool->mapping_pool);
977                         DMERR("dm_kcopyd_copy() failed");
978                         cell_error(cell);
979                 }
980         }
981 }
982
983 static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
984                                    dm_block_t data_origin, dm_block_t data_dest,
985                                    struct cell *cell, struct bio *bio)
986 {
987         schedule_copy(tc, virt_block, tc->pool_dev,
988                       data_origin, data_dest, cell, bio);
989 }
990
991 static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
992                                    dm_block_t data_dest,
993                                    struct cell *cell, struct bio *bio)
994 {
995         schedule_copy(tc, virt_block, tc->origin_dev,
996                       virt_block, data_dest, cell, bio);
997 }
998
999 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
1000                           dm_block_t data_block, struct cell *cell,
1001                           struct bio *bio)
1002 {
1003         struct pool *pool = tc->pool;
1004         struct new_mapping *m = get_next_mapping(pool);
1005
1006         INIT_LIST_HEAD(&m->list);
1007         m->quiesced = 1;
1008         m->prepared = 0;
1009         m->tc = tc;
1010         m->virt_block = virt_block;
1011         m->data_block = data_block;
1012         m->cell = cell;
1013         m->err = 0;
1014         m->bio = NULL;
1015
1016         /*
1017          * If the whole block of data is being overwritten or we are not
1018          * zeroing pre-existing data, we can issue the bio immediately.
1019          * Otherwise we use kcopyd to zero the data first.
1020          */
1021         if (!pool->zero_new_blocks)
1022                 process_prepared_mapping(m);
1023
1024         else if (io_overwrites_block(pool, bio)) {
1025                 struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
1026                 h->overwrite_mapping = m;
1027                 m->bio = bio;
1028                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1029                 remap_and_issue(tc, bio, data_block);
1030
1031         } else {
1032                 int r;
1033                 struct dm_io_region to;
1034
1035                 to.bdev = tc->pool_dev->bdev;
1036                 to.sector = data_block * pool->sectors_per_block;
1037                 to.count = pool->sectors_per_block;
1038
1039                 r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m);
1040                 if (r < 0) {
1041                         mempool_free(m, pool->mapping_pool);
1042                         DMERR("dm_kcopyd_zero() failed");
1043                         cell_error(cell);
1044                 }
1045         }
1046 }
1047
1048 static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1049 {
1050         int r;
1051         dm_block_t free_blocks;
1052         unsigned long flags;
1053         struct pool *pool = tc->pool;
1054
1055         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1056         if (r)
1057                 return r;
1058
1059         if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1060                 DMWARN("%s: reached low water mark, sending event.",
1061                        dm_device_name(pool->pool_md));
1062                 spin_lock_irqsave(&pool->lock, flags);
1063                 pool->low_water_triggered = 1;
1064                 spin_unlock_irqrestore(&pool->lock, flags);
1065                 dm_table_event(pool->ti->table);
1066         }
1067
1068         if (!free_blocks) {
1069                 if (pool->no_free_space)
1070                         return -ENOSPC;
1071                 else {
1072                         /*
1073                          * Try to commit to see if that will free up some
1074                          * more space.
1075                          */
1076                         r = dm_pool_commit_metadata(pool->pmd);
1077                         if (r) {
1078                                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1079                                       __func__, r);
1080                                 return r;
1081                         }
1082
1083                         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1084                         if (r)
1085                                 return r;
1086
1087                         /*
1088                          * If we still have no space we set a flag to avoid
1089                          * doing all this checking and return -ENOSPC.
1090                          */
1091                         if (!free_blocks) {
1092                                 DMWARN("%s: no free space available.",
1093                                        dm_device_name(pool->pool_md));
1094                                 spin_lock_irqsave(&pool->lock, flags);
1095                                 pool->no_free_space = 1;
1096                                 spin_unlock_irqrestore(&pool->lock, flags);
1097                                 return -ENOSPC;
1098                         }
1099                 }
1100         }
1101
1102         r = dm_pool_alloc_data_block(pool->pmd, result);
1103         if (r)
1104                 return r;
1105
1106         return 0;
1107 }
1108
1109 /*
1110  * If we have run out of space, queue bios until the device is
1111  * resumed, presumably after having been reloaded with more space.
1112  */
1113 static void retry_on_resume(struct bio *bio)
1114 {
1115         struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
1116         struct thin_c *tc = h->tc;
1117         struct pool *pool = tc->pool;
1118         unsigned long flags;
1119
1120         spin_lock_irqsave(&pool->lock, flags);
1121         bio_list_add(&pool->retry_on_resume_list, bio);
1122         spin_unlock_irqrestore(&pool->lock, flags);
1123 }
1124
1125 static void no_space(struct cell *cell)
1126 {
1127         struct bio *bio;
1128         struct bio_list bios;
1129
1130         bio_list_init(&bios);
1131         cell_release(cell, &bios);
1132
1133         while ((bio = bio_list_pop(&bios)))
1134                 retry_on_resume(bio);
1135 }
1136
1137 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1138                           struct cell_key *key,
1139                           struct dm_thin_lookup_result *lookup_result,
1140                           struct cell *cell)
1141 {
1142         int r;
1143         dm_block_t data_block;
1144
1145         r = alloc_data_block(tc, &data_block);
1146         switch (r) {
1147         case 0:
1148                 schedule_internal_copy(tc, block, lookup_result->block,
1149                                        data_block, cell, bio);
1150                 break;
1151
1152         case -ENOSPC:
1153                 no_space(cell);
1154                 break;
1155
1156         default:
1157                 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1158                 cell_error(cell);
1159                 break;
1160         }
1161 }
1162
1163 static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1164                                dm_block_t block,
1165                                struct dm_thin_lookup_result *lookup_result)
1166 {
1167         struct cell *cell;
1168         struct pool *pool = tc->pool;
1169         struct cell_key key;
1170
1171         /*
1172          * If cell is already occupied, then sharing is already in the process
1173          * of being broken so we have nothing further to do here.
1174          */
1175         build_data_key(tc->td, lookup_result->block, &key);
1176         if (bio_detain(pool->prison, &key, bio, &cell))
1177                 return;
1178
1179         if (bio_data_dir(bio) == WRITE)
1180                 break_sharing(tc, bio, block, &key, lookup_result, cell);
1181         else {
1182                 struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
1183
1184                 h->shared_read_entry = ds_inc(&pool->shared_read_ds);
1185
1186                 cell_release_singleton(cell, bio);
1187                 remap_and_issue(tc, bio, lookup_result->block);
1188         }
1189 }
1190
1191 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1192                             struct cell *cell)
1193 {
1194         int r;
1195         dm_block_t data_block;
1196
1197         /*
1198          * Remap empty bios (flushes) immediately, without provisioning.
1199          */
1200         if (!bio->bi_size) {
1201                 cell_release_singleton(cell, bio);
1202                 remap_and_issue(tc, bio, 0);
1203                 return;
1204         }
1205
1206         /*
1207          * Fill read bios with zeroes and complete them immediately.
1208          */
1209         if (bio_data_dir(bio) == READ) {
1210                 zero_fill_bio(bio);
1211                 cell_release_singleton(cell, bio);
1212                 bio_endio(bio, 0);
1213                 return;
1214         }
1215
1216         r = alloc_data_block(tc, &data_block);
1217         switch (r) {
1218         case 0:
1219                 if (tc->origin_dev)
1220                         schedule_external_copy(tc, block, data_block, cell, bio);
1221                 else
1222                         schedule_zero(tc, block, data_block, cell, bio);
1223                 break;
1224
1225         case -ENOSPC:
1226                 no_space(cell);
1227                 break;
1228
1229         default:
1230                 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1231                 cell_error(cell);
1232                 break;
1233         }
1234 }
1235
1236 static void process_bio(struct thin_c *tc, struct bio *bio)
1237 {
1238         int r;
1239         dm_block_t block = get_bio_block(tc, bio);
1240         struct cell *cell;
1241         struct cell_key key;
1242         struct dm_thin_lookup_result lookup_result;
1243
1244         /*
1245          * If cell is already occupied, then the block is already
1246          * being provisioned so we have nothing further to do here.
1247          */
1248         build_virtual_key(tc->td, block, &key);
1249         if (bio_detain(tc->pool->prison, &key, bio, &cell))
1250                 return;
1251
1252         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1253         switch (r) {
1254         case 0:
1255                 /*
1256                  * We can release this cell now.  This thread is the only
1257                  * one that puts bios into a cell, and we know there were
1258                  * no preceding bios.
1259                  */
1260                 /*
1261                  * TODO: this will probably have to change when discard goes
1262                  * back in.
1263                  */
1264                 cell_release_singleton(cell, bio);
1265
1266                 if (lookup_result.shared)
1267                         process_shared_bio(tc, bio, block, &lookup_result);
1268                 else
1269                         remap_and_issue(tc, bio, lookup_result.block);
1270                 break;
1271
1272         case -ENODATA:
1273                 if (bio_data_dir(bio) == READ && tc->origin_dev) {
1274                         cell_release_singleton(cell, bio);
1275                         remap_to_origin_and_issue(tc, bio);
1276                 } else
1277                         provision_block(tc, bio, block, cell);
1278                 break;
1279
1280         default:
1281                 DMERR("dm_thin_find_block() failed, error = %d", r);
1282                 bio_io_error(bio);
1283                 break;
1284         }
1285 }
1286
1287 static int need_commit_due_to_time(struct pool *pool)
1288 {
1289         return jiffies < pool->last_commit_jiffies ||
1290                jiffies > pool->last_commit_jiffies + COMMIT_PERIOD;
1291 }
1292
1293 static void process_deferred_bios(struct pool *pool)
1294 {
1295         unsigned long flags;
1296         struct bio *bio;
1297         struct bio_list bios;
1298         int r;
1299
1300         bio_list_init(&bios);
1301
1302         spin_lock_irqsave(&pool->lock, flags);
1303         bio_list_merge(&bios, &pool->deferred_bios);
1304         bio_list_init(&pool->deferred_bios);
1305         spin_unlock_irqrestore(&pool->lock, flags);
1306
1307         while ((bio = bio_list_pop(&bios))) {
1308                 struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
1309                 struct thin_c *tc = h->tc;
1310
1311                 /*
1312                  * If we've got no free new_mapping structs, and processing
1313                  * this bio might require one, we pause until there are some
1314                  * prepared mappings to process.
1315                  */
1316                 if (ensure_next_mapping(pool)) {
1317                         spin_lock_irqsave(&pool->lock, flags);
1318                         bio_list_merge(&pool->deferred_bios, &bios);
1319                         spin_unlock_irqrestore(&pool->lock, flags);
1320
1321                         break;
1322                 }
1323                 process_bio(tc, bio);
1324         }
1325
1326         /*
1327          * If there are any deferred flush bios, we must commit
1328          * the metadata before issuing them.
1329          */
1330         bio_list_init(&bios);
1331         spin_lock_irqsave(&pool->lock, flags);
1332         bio_list_merge(&bios, &pool->deferred_flush_bios);
1333         bio_list_init(&pool->deferred_flush_bios);
1334         spin_unlock_irqrestore(&pool->lock, flags);
1335
1336         if (bio_list_empty(&bios) && !need_commit_due_to_time(pool))
1337                 return;
1338
1339         r = dm_pool_commit_metadata(pool->pmd);
1340         if (r) {
1341                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1342                       __func__, r);
1343                 while ((bio = bio_list_pop(&bios)))
1344                         bio_io_error(bio);
1345                 return;
1346         }
1347         pool->last_commit_jiffies = jiffies;
1348
1349         while ((bio = bio_list_pop(&bios)))
1350                 generic_make_request(bio);
1351 }
1352
1353 static void do_worker(struct work_struct *ws)
1354 {
1355         struct pool *pool = container_of(ws, struct pool, worker);
1356
1357         process_prepared_mappings(pool);
1358         process_deferred_bios(pool);
1359 }
1360
1361 /*
1362  * We want to commit periodically so that not too much
1363  * unwritten data builds up.
1364  */
1365 static void do_waker(struct work_struct *ws)
1366 {
1367         struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
1368         wake_worker(pool);
1369         queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
1370 }
1371
1372 /*----------------------------------------------------------------*/
1373
1374 /*
1375  * Mapping functions.
1376  */
1377
1378 /*
1379  * Called only while mapping a thin bio to hand it over to the workqueue.
1380  */
1381 static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
1382 {
1383         unsigned long flags;
1384         struct pool *pool = tc->pool;
1385
1386         spin_lock_irqsave(&pool->lock, flags);
1387         bio_list_add(&pool->deferred_bios, bio);
1388         spin_unlock_irqrestore(&pool->lock, flags);
1389
1390         wake_worker(pool);
1391 }
1392
1393 static struct endio_hook *thin_hook_bio(struct thin_c *tc, struct bio *bio)
1394 {
1395         struct pool *pool = tc->pool;
1396         struct endio_hook *h = mempool_alloc(pool->endio_hook_pool, GFP_NOIO);
1397
1398         h->tc = tc;
1399         h->shared_read_entry = NULL;
1400         h->overwrite_mapping = NULL;
1401
1402         return h;
1403 }
1404
1405 /*
1406  * Non-blocking function called from the thin target's map function.
1407  */
1408 static int thin_bio_map(struct dm_target *ti, struct bio *bio,
1409                         union map_info *map_context)
1410 {
1411         int r;
1412         struct thin_c *tc = ti->private;
1413         dm_block_t block = get_bio_block(tc, bio);
1414         struct dm_thin_device *td = tc->td;
1415         struct dm_thin_lookup_result result;
1416
1417         map_context->ptr = thin_hook_bio(tc, bio);
1418         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1419                 thin_defer_bio(tc, bio);
1420                 return DM_MAPIO_SUBMITTED;
1421         }
1422
1423         r = dm_thin_find_block(td, block, 0, &result);
1424
1425         /*
1426          * Note that we defer readahead too.
1427          */
1428         switch (r) {
1429         case 0:
1430                 if (unlikely(result.shared)) {
1431                         /*
1432                          * We have a race condition here between the
1433                          * result.shared value returned by the lookup and
1434                          * snapshot creation, which may cause new
1435                          * sharing.
1436                          *
1437                          * To avoid this always quiesce the origin before
1438                          * taking the snap.  You want to do this anyway to
1439                          * ensure a consistent application view
1440                          * (i.e. lockfs).
1441                          *
1442                          * More distant ancestors are irrelevant. The
1443                          * shared flag will be set in their case.
1444                          */
1445                         thin_defer_bio(tc, bio);
1446                         r = DM_MAPIO_SUBMITTED;
1447                 } else {
1448                         remap(tc, bio, result.block);
1449                         r = DM_MAPIO_REMAPPED;
1450                 }
1451                 break;
1452
1453         case -ENODATA:
1454                 /*
1455                  * In future, the failed dm_thin_find_block above could
1456                  * provide the hint to load the metadata into cache.
1457                  */
1458         case -EWOULDBLOCK:
1459                 thin_defer_bio(tc, bio);
1460                 r = DM_MAPIO_SUBMITTED;
1461                 break;
1462         }
1463
1464         return r;
1465 }
1466
1467 static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1468 {
1469         int r;
1470         unsigned long flags;
1471         struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
1472
1473         spin_lock_irqsave(&pt->pool->lock, flags);
1474         r = !bio_list_empty(&pt->pool->retry_on_resume_list);
1475         spin_unlock_irqrestore(&pt->pool->lock, flags);
1476
1477         if (!r) {
1478                 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
1479                 r = bdi_congested(&q->backing_dev_info, bdi_bits);
1480         }
1481
1482         return r;
1483 }
1484
1485 static void __requeue_bios(struct pool *pool)
1486 {
1487         bio_list_merge(&pool->deferred_bios, &pool->retry_on_resume_list);
1488         bio_list_init(&pool->retry_on_resume_list);
1489 }
1490
1491 /*----------------------------------------------------------------
1492  * Binding of control targets to a pool object
1493  *--------------------------------------------------------------*/
1494 static int bind_control_target(struct pool *pool, struct dm_target *ti)
1495 {
1496         struct pool_c *pt = ti->private;
1497
1498         pool->ti = ti;
1499         pool->low_water_blocks = pt->low_water_blocks;
1500         pool->zero_new_blocks = pt->zero_new_blocks;
1501
1502         return 0;
1503 }
1504
1505 static void unbind_control_target(struct pool *pool, struct dm_target *ti)
1506 {
1507         if (pool->ti == ti)
1508                 pool->ti = NULL;
1509 }
1510
1511 /*----------------------------------------------------------------
1512  * Pool creation
1513  *--------------------------------------------------------------*/
1514 static void __pool_destroy(struct pool *pool)
1515 {
1516         __pool_table_remove(pool);
1517
1518         if (dm_pool_metadata_close(pool->pmd) < 0)
1519                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1520
1521         prison_destroy(pool->prison);
1522         dm_kcopyd_client_destroy(pool->copier);
1523
1524         if (pool->wq)
1525                 destroy_workqueue(pool->wq);
1526
1527         if (pool->next_mapping)
1528                 mempool_free(pool->next_mapping, pool->mapping_pool);
1529         mempool_destroy(pool->mapping_pool);
1530         mempool_destroy(pool->endio_hook_pool);
1531         kfree(pool);
1532 }
1533
1534 static struct pool *pool_create(struct mapped_device *pool_md,
1535                                 struct block_device *metadata_dev,
1536                                 unsigned long block_size, char **error)
1537 {
1538         int r;
1539         void *err_p;
1540         struct pool *pool;
1541         struct dm_pool_metadata *pmd;
1542
1543         pmd = dm_pool_metadata_open(metadata_dev, block_size);
1544         if (IS_ERR(pmd)) {
1545                 *error = "Error creating metadata object";
1546                 return (struct pool *)pmd;
1547         }
1548
1549         pool = kmalloc(sizeof(*pool), GFP_KERNEL);
1550         if (!pool) {
1551                 *error = "Error allocating memory for pool";
1552                 err_p = ERR_PTR(-ENOMEM);
1553                 goto bad_pool;
1554         }
1555
1556         pool->pmd = pmd;
1557         pool->sectors_per_block = block_size;
1558         pool->block_shift = ffs(block_size) - 1;
1559         pool->offset_mask = block_size - 1;
1560         pool->low_water_blocks = 0;
1561         pool->zero_new_blocks = 1;
1562         pool->prison = prison_create(PRISON_CELLS);
1563         if (!pool->prison) {
1564                 *error = "Error creating pool's bio prison";
1565                 err_p = ERR_PTR(-ENOMEM);
1566                 goto bad_prison;
1567         }
1568
1569         pool->copier = dm_kcopyd_client_create();
1570         if (IS_ERR(pool->copier)) {
1571                 r = PTR_ERR(pool->copier);
1572                 *error = "Error creating pool's kcopyd client";
1573                 err_p = ERR_PTR(r);
1574                 goto bad_kcopyd_client;
1575         }
1576
1577         /*
1578          * Create singlethreaded workqueue that will service all devices
1579          * that use this metadata.
1580          */
1581         pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
1582         if (!pool->wq) {
1583                 *error = "Error creating pool's workqueue";
1584                 err_p = ERR_PTR(-ENOMEM);
1585                 goto bad_wq;
1586         }
1587
1588         INIT_WORK(&pool->worker, do_worker);
1589         INIT_DELAYED_WORK(&pool->waker, do_waker);
1590         spin_lock_init(&pool->lock);
1591         bio_list_init(&pool->deferred_bios);
1592         bio_list_init(&pool->deferred_flush_bios);
1593         INIT_LIST_HEAD(&pool->prepared_mappings);
1594         pool->low_water_triggered = 0;
1595         pool->no_free_space = 0;
1596         bio_list_init(&pool->retry_on_resume_list);
1597         ds_init(&pool->shared_read_ds);
1598
1599         pool->next_mapping = NULL;
1600         pool->mapping_pool =
1601                 mempool_create_kmalloc_pool(MAPPING_POOL_SIZE, sizeof(struct new_mapping));
1602         if (!pool->mapping_pool) {
1603                 *error = "Error creating pool's mapping mempool";
1604                 err_p = ERR_PTR(-ENOMEM);
1605                 goto bad_mapping_pool;
1606         }
1607
1608         pool->endio_hook_pool =
1609                 mempool_create_kmalloc_pool(ENDIO_HOOK_POOL_SIZE, sizeof(struct endio_hook));
1610         if (!pool->endio_hook_pool) {
1611                 *error = "Error creating pool's endio_hook mempool";
1612                 err_p = ERR_PTR(-ENOMEM);
1613                 goto bad_endio_hook_pool;
1614         }
1615         pool->ref_count = 1;
1616         pool->last_commit_jiffies = jiffies;
1617         pool->pool_md = pool_md;
1618         pool->md_dev = metadata_dev;
1619         __pool_table_insert(pool);
1620
1621         return pool;
1622
1623 bad_endio_hook_pool:
1624         mempool_destroy(pool->mapping_pool);
1625 bad_mapping_pool:
1626         destroy_workqueue(pool->wq);
1627 bad_wq:
1628         dm_kcopyd_client_destroy(pool->copier);
1629 bad_kcopyd_client:
1630         prison_destroy(pool->prison);
1631 bad_prison:
1632         kfree(pool);
1633 bad_pool:
1634         if (dm_pool_metadata_close(pmd))
1635                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1636
1637         return err_p;
1638 }
1639
1640 static void __pool_inc(struct pool *pool)
1641 {
1642         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1643         pool->ref_count++;
1644 }
1645
1646 static void __pool_dec(struct pool *pool)
1647 {
1648         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1649         BUG_ON(!pool->ref_count);
1650         if (!--pool->ref_count)
1651                 __pool_destroy(pool);
1652 }
1653
1654 static struct pool *__pool_find(struct mapped_device *pool_md,
1655                                 struct block_device *metadata_dev,
1656                                 unsigned long block_size, char **error)
1657 {
1658         struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
1659
1660         if (pool) {
1661                 if (pool->pool_md != pool_md)
1662                         return ERR_PTR(-EBUSY);
1663                 __pool_inc(pool);
1664
1665         } else {
1666                 pool = __pool_table_lookup(pool_md);
1667                 if (pool) {
1668                         if (pool->md_dev != metadata_dev)
1669                                 return ERR_PTR(-EINVAL);
1670                         __pool_inc(pool);
1671
1672                 } else
1673                         pool = pool_create(pool_md, metadata_dev, block_size, error);
1674         }
1675
1676         return pool;
1677 }
1678
1679 /*----------------------------------------------------------------
1680  * Pool target methods
1681  *--------------------------------------------------------------*/
1682 static void pool_dtr(struct dm_target *ti)
1683 {
1684         struct pool_c *pt = ti->private;
1685
1686         mutex_lock(&dm_thin_pool_table.mutex);
1687
1688         unbind_control_target(pt->pool, ti);
1689         __pool_dec(pt->pool);
1690         dm_put_device(ti, pt->metadata_dev);
1691         dm_put_device(ti, pt->data_dev);
1692         kfree(pt);
1693
1694         mutex_unlock(&dm_thin_pool_table.mutex);
1695 }
1696
1697 struct pool_features {
1698         unsigned zero_new_blocks:1;
1699 };
1700
1701 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
1702                                struct dm_target *ti)
1703 {
1704         int r;
1705         unsigned argc;
1706         const char *arg_name;
1707
1708         static struct dm_arg _args[] = {
1709                 {0, 1, "Invalid number of pool feature arguments"},
1710         };
1711
1712         /*
1713          * No feature arguments supplied.
1714          */
1715         if (!as->argc)
1716                 return 0;
1717
1718         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1719         if (r)
1720                 return -EINVAL;
1721
1722         while (argc && !r) {
1723                 arg_name = dm_shift_arg(as);
1724                 argc--;
1725
1726                 if (!strcasecmp(arg_name, "skip_block_zeroing")) {
1727                         pf->zero_new_blocks = 0;
1728                         continue;
1729                 }
1730
1731                 ti->error = "Unrecognised pool feature requested";
1732                 r = -EINVAL;
1733         }
1734
1735         return r;
1736 }
1737
1738 /*
1739  * thin-pool <metadata dev> <data dev>
1740  *           <data block size (sectors)>
1741  *           <low water mark (blocks)>
1742  *           [<#feature args> [<arg>]*]
1743  *
1744  * Optional feature arguments are:
1745  *           skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
1746  */
1747 static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
1748 {
1749         int r;
1750         struct pool_c *pt;
1751         struct pool *pool;
1752         struct pool_features pf;
1753         struct dm_arg_set as;
1754         struct dm_dev *data_dev;
1755         unsigned long block_size;
1756         dm_block_t low_water_blocks;
1757         struct dm_dev *metadata_dev;
1758         sector_t metadata_dev_size;
1759         char b[BDEVNAME_SIZE];
1760
1761         /*
1762          * FIXME Remove validation from scope of lock.
1763          */
1764         mutex_lock(&dm_thin_pool_table.mutex);
1765
1766         if (argc < 4) {
1767                 ti->error = "Invalid argument count";
1768                 r = -EINVAL;
1769                 goto out_unlock;
1770         }
1771         as.argc = argc;
1772         as.argv = argv;
1773
1774         r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &metadata_dev);
1775         if (r) {
1776                 ti->error = "Error opening metadata block device";
1777                 goto out_unlock;
1778         }
1779
1780         metadata_dev_size = i_size_read(metadata_dev->bdev->bd_inode) >> SECTOR_SHIFT;
1781         if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
1782                 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
1783                        bdevname(metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
1784
1785         r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
1786         if (r) {
1787                 ti->error = "Error getting data device";
1788                 goto out_metadata;
1789         }
1790
1791         if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
1792             block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
1793             block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
1794             !is_power_of_2(block_size)) {
1795                 ti->error = "Invalid block size";
1796                 r = -EINVAL;
1797                 goto out;
1798         }
1799
1800         if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
1801                 ti->error = "Invalid low water mark";
1802                 r = -EINVAL;
1803                 goto out;
1804         }
1805
1806         /*
1807          * Set default pool features.
1808          */
1809         memset(&pf, 0, sizeof(pf));
1810         pf.zero_new_blocks = 1;
1811
1812         dm_consume_args(&as, 4);
1813         r = parse_pool_features(&as, &pf, ti);
1814         if (r)
1815                 goto out;
1816
1817         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
1818         if (!pt) {
1819                 r = -ENOMEM;
1820                 goto out;
1821         }
1822
1823         pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
1824                            block_size, &ti->error);
1825         if (IS_ERR(pool)) {
1826                 r = PTR_ERR(pool);
1827                 goto out_free_pt;
1828         }
1829
1830         pt->pool = pool;
1831         pt->ti = ti;
1832         pt->metadata_dev = metadata_dev;
1833         pt->data_dev = data_dev;
1834         pt->low_water_blocks = low_water_blocks;
1835         pt->zero_new_blocks = pf.zero_new_blocks;
1836         ti->num_flush_requests = 1;
1837         ti->num_discard_requests = 0;
1838         ti->private = pt;
1839
1840         pt->callbacks.congested_fn = pool_is_congested;
1841         dm_table_add_target_callbacks(ti->table, &pt->callbacks);
1842
1843         mutex_unlock(&dm_thin_pool_table.mutex);
1844
1845         return 0;
1846
1847 out_free_pt:
1848         kfree(pt);
1849 out:
1850         dm_put_device(ti, data_dev);
1851 out_metadata:
1852         dm_put_device(ti, metadata_dev);
1853 out_unlock:
1854         mutex_unlock(&dm_thin_pool_table.mutex);
1855
1856         return r;
1857 }
1858
1859 static int pool_map(struct dm_target *ti, struct bio *bio,
1860                     union map_info *map_context)
1861 {
1862         int r;
1863         struct pool_c *pt = ti->private;
1864         struct pool *pool = pt->pool;
1865         unsigned long flags;
1866
1867         /*
1868          * As this is a singleton target, ti->begin is always zero.
1869          */
1870         spin_lock_irqsave(&pool->lock, flags);
1871         bio->bi_bdev = pt->data_dev->bdev;
1872         r = DM_MAPIO_REMAPPED;
1873         spin_unlock_irqrestore(&pool->lock, flags);
1874
1875         return r;
1876 }
1877
1878 /*
1879  * Retrieves the number of blocks of the data device from
1880  * the superblock and compares it to the actual device size,
1881  * thus resizing the data device in case it has grown.
1882  *
1883  * This both copes with opening preallocated data devices in the ctr
1884  * being followed by a resume
1885  * -and-
1886  * calling the resume method individually after userspace has
1887  * grown the data device in reaction to a table event.
1888  */
1889 static int pool_preresume(struct dm_target *ti)
1890 {
1891         int r;
1892         struct pool_c *pt = ti->private;
1893         struct pool *pool = pt->pool;
1894         dm_block_t data_size, sb_data_size;
1895
1896         /*
1897          * Take control of the pool object.
1898          */
1899         r = bind_control_target(pool, ti);
1900         if (r)
1901                 return r;
1902
1903         data_size = ti->len >> pool->block_shift;
1904         r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
1905         if (r) {
1906                 DMERR("failed to retrieve data device size");
1907                 return r;
1908         }
1909
1910         if (data_size < sb_data_size) {
1911                 DMERR("pool target too small, is %llu blocks (expected %llu)",
1912                       data_size, sb_data_size);
1913                 return -EINVAL;
1914
1915         } else if (data_size > sb_data_size) {
1916                 r = dm_pool_resize_data_dev(pool->pmd, data_size);
1917                 if (r) {
1918                         DMERR("failed to resize data device");
1919                         return r;
1920                 }
1921
1922                 r = dm_pool_commit_metadata(pool->pmd);
1923                 if (r) {
1924                         DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1925                               __func__, r);
1926                         return r;
1927                 }
1928         }
1929
1930         return 0;
1931 }
1932
1933 static void pool_resume(struct dm_target *ti)
1934 {
1935         struct pool_c *pt = ti->private;
1936         struct pool *pool = pt->pool;
1937         unsigned long flags;
1938
1939         spin_lock_irqsave(&pool->lock, flags);
1940         pool->low_water_triggered = 0;
1941         pool->no_free_space = 0;
1942         __requeue_bios(pool);
1943         spin_unlock_irqrestore(&pool->lock, flags);
1944
1945         do_waker(&pool->waker.work);
1946 }
1947
1948 static void pool_postsuspend(struct dm_target *ti)
1949 {
1950         int r;
1951         struct pool_c *pt = ti->private;
1952         struct pool *pool = pt->pool;
1953
1954         cancel_delayed_work(&pool->waker);
1955         flush_workqueue(pool->wq);
1956
1957         r = dm_pool_commit_metadata(pool->pmd);
1958         if (r < 0) {
1959                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1960                       __func__, r);
1961                 /* FIXME: invalidate device? error the next FUA or FLUSH bio ?*/
1962         }
1963 }
1964
1965 static int check_arg_count(unsigned argc, unsigned args_required)
1966 {
1967         if (argc != args_required) {
1968                 DMWARN("Message received with %u arguments instead of %u.",
1969                        argc, args_required);
1970                 return -EINVAL;
1971         }
1972
1973         return 0;
1974 }
1975
1976 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
1977 {
1978         if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
1979             *dev_id <= MAX_DEV_ID)
1980                 return 0;
1981
1982         if (warning)
1983                 DMWARN("Message received with invalid device id: %s", arg);
1984
1985         return -EINVAL;
1986 }
1987
1988 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
1989 {
1990         dm_thin_id dev_id;
1991         int r;
1992
1993         r = check_arg_count(argc, 2);
1994         if (r)
1995                 return r;
1996
1997         r = read_dev_id(argv[1], &dev_id, 1);
1998         if (r)
1999                 return r;
2000
2001         r = dm_pool_create_thin(pool->pmd, dev_id);
2002         if (r) {
2003                 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
2004                        argv[1]);
2005                 return r;
2006         }
2007
2008         return 0;
2009 }
2010
2011 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2012 {
2013         dm_thin_id dev_id;
2014         dm_thin_id origin_dev_id;
2015         int r;
2016
2017         r = check_arg_count(argc, 3);
2018         if (r)
2019                 return r;
2020
2021         r = read_dev_id(argv[1], &dev_id, 1);
2022         if (r)
2023                 return r;
2024
2025         r = read_dev_id(argv[2], &origin_dev_id, 1);
2026         if (r)
2027                 return r;
2028
2029         r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
2030         if (r) {
2031                 DMWARN("Creation of new snapshot %s of device %s failed.",
2032                        argv[1], argv[2]);
2033                 return r;
2034         }
2035
2036         return 0;
2037 }
2038
2039 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
2040 {
2041         dm_thin_id dev_id;
2042         int r;
2043
2044         r = check_arg_count(argc, 2);
2045         if (r)
2046                 return r;
2047
2048         r = read_dev_id(argv[1], &dev_id, 1);
2049         if (r)
2050                 return r;
2051
2052         r = dm_pool_delete_thin_device(pool->pmd, dev_id);
2053         if (r)
2054                 DMWARN("Deletion of thin device %s failed.", argv[1]);
2055
2056         return r;
2057 }
2058
2059 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
2060 {
2061         dm_thin_id old_id, new_id;
2062         int r;
2063
2064         r = check_arg_count(argc, 3);
2065         if (r)
2066                 return r;
2067
2068         if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
2069                 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
2070                 return -EINVAL;
2071         }
2072
2073         if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
2074                 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
2075                 return -EINVAL;
2076         }
2077
2078         r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
2079         if (r) {
2080                 DMWARN("Failed to change transaction id from %s to %s.",
2081                        argv[1], argv[2]);
2082                 return r;
2083         }
2084
2085         return 0;
2086 }
2087
2088 /*
2089  * Messages supported:
2090  *   create_thin        <dev_id>
2091  *   create_snap        <dev_id> <origin_id>
2092  *   delete             <dev_id>
2093  *   trim               <dev_id> <new_size_in_sectors>
2094  *   set_transaction_id <current_trans_id> <new_trans_id>
2095  */
2096 static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
2097 {
2098         int r = -EINVAL;
2099         struct pool_c *pt = ti->private;
2100         struct pool *pool = pt->pool;
2101
2102         if (!strcasecmp(argv[0], "create_thin"))
2103                 r = process_create_thin_mesg(argc, argv, pool);
2104
2105         else if (!strcasecmp(argv[0], "create_snap"))
2106                 r = process_create_snap_mesg(argc, argv, pool);
2107
2108         else if (!strcasecmp(argv[0], "delete"))
2109                 r = process_delete_mesg(argc, argv, pool);
2110
2111         else if (!strcasecmp(argv[0], "set_transaction_id"))
2112                 r = process_set_transaction_id_mesg(argc, argv, pool);
2113
2114         else
2115                 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
2116
2117         if (!r) {
2118                 r = dm_pool_commit_metadata(pool->pmd);
2119                 if (r)
2120                         DMERR("%s message: dm_pool_commit_metadata() failed, error = %d",
2121                               argv[0], r);
2122         }
2123
2124         return r;
2125 }
2126
2127 /*
2128  * Status line is:
2129  *    <transaction id> <used metadata sectors>/<total metadata sectors>
2130  *    <used data sectors>/<total data sectors> <held metadata root>
2131  */
2132 static int pool_status(struct dm_target *ti, status_type_t type,
2133                        char *result, unsigned maxlen)
2134 {
2135         int r;
2136         unsigned sz = 0;
2137         uint64_t transaction_id;
2138         dm_block_t nr_free_blocks_data;
2139         dm_block_t nr_free_blocks_metadata;
2140         dm_block_t nr_blocks_data;
2141         dm_block_t nr_blocks_metadata;
2142         dm_block_t held_root;
2143         char buf[BDEVNAME_SIZE];
2144         char buf2[BDEVNAME_SIZE];
2145         struct pool_c *pt = ti->private;
2146         struct pool *pool = pt->pool;
2147
2148         switch (type) {
2149         case STATUSTYPE_INFO:
2150                 r = dm_pool_get_metadata_transaction_id(pool->pmd,
2151                                                         &transaction_id);
2152                 if (r)
2153                         return r;
2154
2155                 r = dm_pool_get_free_metadata_block_count(pool->pmd,
2156                                                           &nr_free_blocks_metadata);
2157                 if (r)
2158                         return r;
2159
2160                 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
2161                 if (r)
2162                         return r;
2163
2164                 r = dm_pool_get_free_block_count(pool->pmd,
2165                                                  &nr_free_blocks_data);
2166                 if (r)
2167                         return r;
2168
2169                 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
2170                 if (r)
2171                         return r;
2172
2173                 r = dm_pool_get_held_metadata_root(pool->pmd, &held_root);
2174                 if (r)
2175                         return r;
2176
2177                 DMEMIT("%llu %llu/%llu %llu/%llu ",
2178                        (unsigned long long)transaction_id,
2179                        (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
2180                        (unsigned long long)nr_blocks_metadata,
2181                        (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
2182                        (unsigned long long)nr_blocks_data);
2183
2184                 if (held_root)
2185                         DMEMIT("%llu", held_root);
2186                 else
2187                         DMEMIT("-");
2188
2189                 break;
2190
2191         case STATUSTYPE_TABLE:
2192                 DMEMIT("%s %s %lu %llu ",
2193                        format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
2194                        format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
2195                        (unsigned long)pool->sectors_per_block,
2196                        (unsigned long long)pt->low_water_blocks);
2197
2198                 DMEMIT("%u ", !pool->zero_new_blocks);
2199
2200                 if (!pool->zero_new_blocks)
2201                         DMEMIT("skip_block_zeroing ");
2202                 break;
2203         }
2204
2205         return 0;
2206 }
2207
2208 static int pool_iterate_devices(struct dm_target *ti,
2209                                 iterate_devices_callout_fn fn, void *data)
2210 {
2211         struct pool_c *pt = ti->private;
2212
2213         return fn(ti, pt->data_dev, 0, ti->len, data);
2214 }
2215
2216 static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2217                       struct bio_vec *biovec, int max_size)
2218 {
2219         struct pool_c *pt = ti->private;
2220         struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2221
2222         if (!q->merge_bvec_fn)
2223                 return max_size;
2224
2225         bvm->bi_bdev = pt->data_dev->bdev;
2226
2227         return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2228 }
2229
2230 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
2231 {
2232         struct pool_c *pt = ti->private;
2233         struct pool *pool = pt->pool;
2234
2235         blk_limits_io_min(limits, 0);
2236         blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
2237 }
2238
2239 static struct target_type pool_target = {
2240         .name = "thin-pool",
2241         .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
2242                     DM_TARGET_IMMUTABLE,
2243         .version = {1, 0, 0},
2244         .module = THIS_MODULE,
2245         .ctr = pool_ctr,
2246         .dtr = pool_dtr,
2247         .map = pool_map,
2248         .postsuspend = pool_postsuspend,
2249         .preresume = pool_preresume,
2250         .resume = pool_resume,
2251         .message = pool_message,
2252         .status = pool_status,
2253         .merge = pool_merge,
2254         .iterate_devices = pool_iterate_devices,
2255         .io_hints = pool_io_hints,
2256 };
2257
2258 /*----------------------------------------------------------------
2259  * Thin target methods
2260  *--------------------------------------------------------------*/
2261 static void thin_dtr(struct dm_target *ti)
2262 {
2263         struct thin_c *tc = ti->private;
2264
2265         mutex_lock(&dm_thin_pool_table.mutex);
2266
2267         __pool_dec(tc->pool);
2268         dm_pool_close_thin_device(tc->td);
2269         dm_put_device(ti, tc->pool_dev);
2270         if (tc->origin_dev)
2271                 dm_put_device(ti, tc->origin_dev);
2272         kfree(tc);
2273
2274         mutex_unlock(&dm_thin_pool_table.mutex);
2275 }
2276
2277 /*
2278  * Thin target parameters:
2279  *
2280  * <pool_dev> <dev_id> [origin_dev]
2281  *
2282  * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
2283  * dev_id: the internal device identifier
2284  * origin_dev: a device external to the pool that should act as the origin
2285  */
2286 static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
2287 {
2288         int r;
2289         struct thin_c *tc;
2290         struct dm_dev *pool_dev, *origin_dev;
2291         struct mapped_device *pool_md;
2292
2293         mutex_lock(&dm_thin_pool_table.mutex);
2294
2295         if (argc != 2 && argc != 3) {
2296                 ti->error = "Invalid argument count";
2297                 r = -EINVAL;
2298                 goto out_unlock;
2299         }
2300
2301         tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
2302         if (!tc) {
2303                 ti->error = "Out of memory";
2304                 r = -ENOMEM;
2305                 goto out_unlock;
2306         }
2307
2308         if (argc == 3) {
2309                 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
2310                 if (r) {
2311                         ti->error = "Error opening origin device";
2312                         goto bad_origin_dev;
2313                 }
2314                 tc->origin_dev = origin_dev;
2315         }
2316
2317         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
2318         if (r) {
2319                 ti->error = "Error opening pool device";
2320                 goto bad_pool_dev;
2321         }
2322         tc->pool_dev = pool_dev;
2323
2324         if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
2325                 ti->error = "Invalid device id";
2326                 r = -EINVAL;
2327                 goto bad_common;
2328         }
2329
2330         pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
2331         if (!pool_md) {
2332                 ti->error = "Couldn't get pool mapped device";
2333                 r = -EINVAL;
2334                 goto bad_common;
2335         }
2336
2337         tc->pool = __pool_table_lookup(pool_md);
2338         if (!tc->pool) {
2339                 ti->error = "Couldn't find pool object";
2340                 r = -EINVAL;
2341                 goto bad_pool_lookup;
2342         }
2343         __pool_inc(tc->pool);
2344
2345         r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
2346         if (r) {
2347                 ti->error = "Couldn't open thin internal device";
2348                 goto bad_thin_open;
2349         }
2350
2351         ti->split_io = tc->pool->sectors_per_block;
2352         ti->num_flush_requests = 1;
2353         ti->num_discard_requests = 0;
2354         ti->discards_supported = 0;
2355
2356         dm_put(pool_md);
2357
2358         mutex_unlock(&dm_thin_pool_table.mutex);
2359
2360         return 0;
2361
2362 bad_thin_open:
2363         __pool_dec(tc->pool);
2364 bad_pool_lookup:
2365         dm_put(pool_md);
2366 bad_common:
2367         dm_put_device(ti, tc->pool_dev);
2368 bad_pool_dev:
2369         if (tc->origin_dev)
2370                 dm_put_device(ti, tc->origin_dev);
2371 bad_origin_dev:
2372         kfree(tc);
2373 out_unlock:
2374         mutex_unlock(&dm_thin_pool_table.mutex);
2375
2376         return r;
2377 }
2378
2379 static int thin_map(struct dm_target *ti, struct bio *bio,
2380                     union map_info *map_context)
2381 {
2382         bio->bi_sector = dm_target_offset(ti, bio->bi_sector);
2383
2384         return thin_bio_map(ti, bio, map_context);
2385 }
2386
2387 static int thin_endio(struct dm_target *ti,
2388                       struct bio *bio, int err,
2389                       union map_info *map_context)
2390 {
2391         unsigned long flags;
2392         struct endio_hook *h = map_context->ptr;
2393         struct list_head work;
2394         struct new_mapping *m, *tmp;
2395         struct pool *pool = h->tc->pool;
2396
2397         if (h->shared_read_entry) {
2398                 INIT_LIST_HEAD(&work);
2399                 ds_dec(h->shared_read_entry, &work);
2400
2401                 spin_lock_irqsave(&pool->lock, flags);
2402                 list_for_each_entry_safe(m, tmp, &work, list) {
2403                         list_del(&m->list);
2404                         m->quiesced = 1;
2405                         __maybe_add_mapping(m);
2406                 }
2407                 spin_unlock_irqrestore(&pool->lock, flags);
2408         }
2409
2410         mempool_free(h, pool->endio_hook_pool);
2411
2412         return 0;
2413 }
2414
2415 static void thin_postsuspend(struct dm_target *ti)
2416 {
2417         if (dm_noflush_suspending(ti))
2418                 requeue_io((struct thin_c *)ti->private);
2419 }
2420
2421 /*
2422  * <nr mapped sectors> <highest mapped sector>
2423  */
2424 static int thin_status(struct dm_target *ti, status_type_t type,
2425                        char *result, unsigned maxlen)
2426 {
2427         int r;
2428         ssize_t sz = 0;
2429         dm_block_t mapped, highest;
2430         char buf[BDEVNAME_SIZE];
2431         struct thin_c *tc = ti->private;
2432
2433         if (!tc->td)
2434                 DMEMIT("-");
2435         else {
2436                 switch (type) {
2437                 case STATUSTYPE_INFO:
2438                         r = dm_thin_get_mapped_count(tc->td, &mapped);
2439                         if (r)
2440                                 return r;
2441
2442                         r = dm_thin_get_highest_mapped_block(tc->td, &highest);
2443                         if (r < 0)
2444                                 return r;
2445
2446                         DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
2447                         if (r)
2448                                 DMEMIT("%llu", ((highest + 1) *
2449                                                 tc->pool->sectors_per_block) - 1);
2450                         else
2451                                 DMEMIT("-");
2452                         break;
2453
2454                 case STATUSTYPE_TABLE:
2455                         DMEMIT("%s %lu",
2456                                format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
2457                                (unsigned long) tc->dev_id);
2458                         if (tc->origin_dev)
2459                                 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
2460                         break;
2461                 }
2462         }
2463
2464         return 0;
2465 }
2466
2467 static int thin_iterate_devices(struct dm_target *ti,
2468                                 iterate_devices_callout_fn fn, void *data)
2469 {
2470         dm_block_t blocks;
2471         struct thin_c *tc = ti->private;
2472
2473         /*
2474          * We can't call dm_pool_get_data_dev_size() since that blocks.  So
2475          * we follow a more convoluted path through to the pool's target.
2476          */
2477         if (!tc->pool->ti)
2478                 return 0;       /* nothing is bound */
2479
2480         blocks = tc->pool->ti->len >> tc->pool->block_shift;
2481         if (blocks)
2482                 return fn(ti, tc->pool_dev, 0, tc->pool->sectors_per_block * blocks, data);
2483
2484         return 0;
2485 }
2486
2487 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
2488 {
2489         struct thin_c *tc = ti->private;
2490
2491         blk_limits_io_min(limits, 0);
2492         blk_limits_io_opt(limits, tc->pool->sectors_per_block << SECTOR_SHIFT);
2493 }
2494
2495 static struct target_type thin_target = {
2496         .name = "thin",
2497         .version = {1, 1, 0},
2498         .module = THIS_MODULE,
2499         .ctr = thin_ctr,
2500         .dtr = thin_dtr,
2501         .map = thin_map,
2502         .end_io = thin_endio,
2503         .postsuspend = thin_postsuspend,
2504         .status = thin_status,
2505         .iterate_devices = thin_iterate_devices,
2506         .io_hints = thin_io_hints,
2507 };
2508
2509 /*----------------------------------------------------------------*/
2510
2511 static int __init dm_thin_init(void)
2512 {
2513         int r;
2514
2515         pool_table_init();
2516
2517         r = dm_register_target(&thin_target);
2518         if (r)
2519                 return r;
2520
2521         r = dm_register_target(&pool_target);
2522         if (r)
2523                 dm_unregister_target(&thin_target);
2524
2525         return r;
2526 }
2527
2528 static void dm_thin_exit(void)
2529 {
2530         dm_unregister_target(&thin_target);
2531         dm_unregister_target(&pool_target);
2532 }
2533
2534 module_init(dm_thin_init);
2535 module_exit(dm_thin_exit);
2536
2537 MODULE_DESCRIPTION(DM_NAME "device-mapper thin provisioning target");
2538 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2539 MODULE_LICENSE("GPL");