Merge branch 'for-3.8/drivers' of git://git.kernel.dk/linux-block
[~shefty/rdma-dev.git] / drivers / md / raid10.c
1 /*
2  * raid10.c : Multiple Devices driver for Linux
3  *
4  * Copyright (C) 2000-2004 Neil Brown
5  *
6  * RAID-10 support for md.
7  *
8  * Base on code in raid1.c.  See raid1.c for further copyright information.
9  *
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/delay.h>
23 #include <linux/blkdev.h>
24 #include <linux/module.h>
25 #include <linux/seq_file.h>
26 #include <linux/ratelimit.h>
27 #include <linux/kthread.h>
28 #include "md.h"
29 #include "raid10.h"
30 #include "raid0.h"
31 #include "bitmap.h"
32
33 /*
34  * RAID10 provides a combination of RAID0 and RAID1 functionality.
35  * The layout of data is defined by
36  *    chunk_size
37  *    raid_disks
38  *    near_copies (stored in low byte of layout)
39  *    far_copies (stored in second byte of layout)
40  *    far_offset (stored in bit 16 of layout )
41  *
42  * The data to be stored is divided into chunks using chunksize.
43  * Each device is divided into far_copies sections.
44  * In each section, chunks are laid out in a style similar to raid0, but
45  * near_copies copies of each chunk is stored (each on a different drive).
46  * The starting device for each section is offset near_copies from the starting
47  * device of the previous section.
48  * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
49  * drive.
50  * near_copies and far_copies must be at least one, and their product is at most
51  * raid_disks.
52  *
53  * If far_offset is true, then the far_copies are handled a bit differently.
54  * The copies are still in different stripes, but instead of be very far apart
55  * on disk, there are adjacent stripes.
56  */
57
58 /*
59  * Number of guaranteed r10bios in case of extreme VM load:
60  */
61 #define NR_RAID10_BIOS 256
62
63 /* when we get a read error on a read-only array, we redirect to another
64  * device without failing the first device, or trying to over-write to
65  * correct the read error.  To keep track of bad blocks on a per-bio
66  * level, we store IO_BLOCKED in the appropriate 'bios' pointer
67  */
68 #define IO_BLOCKED ((struct bio *)1)
69 /* When we successfully write to a known bad-block, we need to remove the
70  * bad-block marking which must be done from process context.  So we record
71  * the success by setting devs[n].bio to IO_MADE_GOOD
72  */
73 #define IO_MADE_GOOD ((struct bio *)2)
74
75 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
76
77 /* When there are this many requests queued to be written by
78  * the raid10 thread, we become 'congested' to provide back-pressure
79  * for writeback.
80  */
81 static int max_queued_requests = 1024;
82
83 static void allow_barrier(struct r10conf *conf);
84 static void lower_barrier(struct r10conf *conf);
85 static int enough(struct r10conf *conf, int ignore);
86 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
87                                 int *skipped);
88 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
89 static void end_reshape_write(struct bio *bio, int error);
90 static void end_reshape(struct r10conf *conf);
91
92 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
93 {
94         struct r10conf *conf = data;
95         int size = offsetof(struct r10bio, devs[conf->copies]);
96
97         /* allocate a r10bio with room for raid_disks entries in the
98          * bios array */
99         return kzalloc(size, gfp_flags);
100 }
101
102 static void r10bio_pool_free(void *r10_bio, void *data)
103 {
104         kfree(r10_bio);
105 }
106
107 /* Maximum size of each resync request */
108 #define RESYNC_BLOCK_SIZE (64*1024)
109 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
110 /* amount of memory to reserve for resync requests */
111 #define RESYNC_WINDOW (1024*1024)
112 /* maximum number of concurrent requests, memory permitting */
113 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
114
115 /*
116  * When performing a resync, we need to read and compare, so
117  * we need as many pages are there are copies.
118  * When performing a recovery, we need 2 bios, one for read,
119  * one for write (we recover only one drive per r10buf)
120  *
121  */
122 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
123 {
124         struct r10conf *conf = data;
125         struct page *page;
126         struct r10bio *r10_bio;
127         struct bio *bio;
128         int i, j;
129         int nalloc;
130
131         r10_bio = r10bio_pool_alloc(gfp_flags, conf);
132         if (!r10_bio)
133                 return NULL;
134
135         if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
136             test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
137                 nalloc = conf->copies; /* resync */
138         else
139                 nalloc = 2; /* recovery */
140
141         /*
142          * Allocate bios.
143          */
144         for (j = nalloc ; j-- ; ) {
145                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
146                 if (!bio)
147                         goto out_free_bio;
148                 r10_bio->devs[j].bio = bio;
149                 if (!conf->have_replacement)
150                         continue;
151                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
152                 if (!bio)
153                         goto out_free_bio;
154                 r10_bio->devs[j].repl_bio = bio;
155         }
156         /*
157          * Allocate RESYNC_PAGES data pages and attach them
158          * where needed.
159          */
160         for (j = 0 ; j < nalloc; j++) {
161                 struct bio *rbio = r10_bio->devs[j].repl_bio;
162                 bio = r10_bio->devs[j].bio;
163                 for (i = 0; i < RESYNC_PAGES; i++) {
164                         if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
165                                                &conf->mddev->recovery)) {
166                                 /* we can share bv_page's during recovery
167                                  * and reshape */
168                                 struct bio *rbio = r10_bio->devs[0].bio;
169                                 page = rbio->bi_io_vec[i].bv_page;
170                                 get_page(page);
171                         } else
172                                 page = alloc_page(gfp_flags);
173                         if (unlikely(!page))
174                                 goto out_free_pages;
175
176                         bio->bi_io_vec[i].bv_page = page;
177                         if (rbio)
178                                 rbio->bi_io_vec[i].bv_page = page;
179                 }
180         }
181
182         return r10_bio;
183
184 out_free_pages:
185         for ( ; i > 0 ; i--)
186                 safe_put_page(bio->bi_io_vec[i-1].bv_page);
187         while (j--)
188                 for (i = 0; i < RESYNC_PAGES ; i++)
189                         safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
190         j = 0;
191 out_free_bio:
192         for ( ; j < nalloc; j++) {
193                 if (r10_bio->devs[j].bio)
194                         bio_put(r10_bio->devs[j].bio);
195                 if (r10_bio->devs[j].repl_bio)
196                         bio_put(r10_bio->devs[j].repl_bio);
197         }
198         r10bio_pool_free(r10_bio, conf);
199         return NULL;
200 }
201
202 static void r10buf_pool_free(void *__r10_bio, void *data)
203 {
204         int i;
205         struct r10conf *conf = data;
206         struct r10bio *r10bio = __r10_bio;
207         int j;
208
209         for (j=0; j < conf->copies; j++) {
210                 struct bio *bio = r10bio->devs[j].bio;
211                 if (bio) {
212                         for (i = 0; i < RESYNC_PAGES; i++) {
213                                 safe_put_page(bio->bi_io_vec[i].bv_page);
214                                 bio->bi_io_vec[i].bv_page = NULL;
215                         }
216                         bio_put(bio);
217                 }
218                 bio = r10bio->devs[j].repl_bio;
219                 if (bio)
220                         bio_put(bio);
221         }
222         r10bio_pool_free(r10bio, conf);
223 }
224
225 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
226 {
227         int i;
228
229         for (i = 0; i < conf->copies; i++) {
230                 struct bio **bio = & r10_bio->devs[i].bio;
231                 if (!BIO_SPECIAL(*bio))
232                         bio_put(*bio);
233                 *bio = NULL;
234                 bio = &r10_bio->devs[i].repl_bio;
235                 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
236                         bio_put(*bio);
237                 *bio = NULL;
238         }
239 }
240
241 static void free_r10bio(struct r10bio *r10_bio)
242 {
243         struct r10conf *conf = r10_bio->mddev->private;
244
245         put_all_bios(conf, r10_bio);
246         mempool_free(r10_bio, conf->r10bio_pool);
247 }
248
249 static void put_buf(struct r10bio *r10_bio)
250 {
251         struct r10conf *conf = r10_bio->mddev->private;
252
253         mempool_free(r10_bio, conf->r10buf_pool);
254
255         lower_barrier(conf);
256 }
257
258 static void reschedule_retry(struct r10bio *r10_bio)
259 {
260         unsigned long flags;
261         struct mddev *mddev = r10_bio->mddev;
262         struct r10conf *conf = mddev->private;
263
264         spin_lock_irqsave(&conf->device_lock, flags);
265         list_add(&r10_bio->retry_list, &conf->retry_list);
266         conf->nr_queued ++;
267         spin_unlock_irqrestore(&conf->device_lock, flags);
268
269         /* wake up frozen array... */
270         wake_up(&conf->wait_barrier);
271
272         md_wakeup_thread(mddev->thread);
273 }
274
275 /*
276  * raid_end_bio_io() is called when we have finished servicing a mirrored
277  * operation and are ready to return a success/failure code to the buffer
278  * cache layer.
279  */
280 static void raid_end_bio_io(struct r10bio *r10_bio)
281 {
282         struct bio *bio = r10_bio->master_bio;
283         int done;
284         struct r10conf *conf = r10_bio->mddev->private;
285
286         if (bio->bi_phys_segments) {
287                 unsigned long flags;
288                 spin_lock_irqsave(&conf->device_lock, flags);
289                 bio->bi_phys_segments--;
290                 done = (bio->bi_phys_segments == 0);
291                 spin_unlock_irqrestore(&conf->device_lock, flags);
292         } else
293                 done = 1;
294         if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
295                 clear_bit(BIO_UPTODATE, &bio->bi_flags);
296         if (done) {
297                 bio_endio(bio, 0);
298                 /*
299                  * Wake up any possible resync thread that waits for the device
300                  * to go idle.
301                  */
302                 allow_barrier(conf);
303         }
304         free_r10bio(r10_bio);
305 }
306
307 /*
308  * Update disk head position estimator based on IRQ completion info.
309  */
310 static inline void update_head_pos(int slot, struct r10bio *r10_bio)
311 {
312         struct r10conf *conf = r10_bio->mddev->private;
313
314         conf->mirrors[r10_bio->devs[slot].devnum].head_position =
315                 r10_bio->devs[slot].addr + (r10_bio->sectors);
316 }
317
318 /*
319  * Find the disk number which triggered given bio
320  */
321 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
322                          struct bio *bio, int *slotp, int *replp)
323 {
324         int slot;
325         int repl = 0;
326
327         for (slot = 0; slot < conf->copies; slot++) {
328                 if (r10_bio->devs[slot].bio == bio)
329                         break;
330                 if (r10_bio->devs[slot].repl_bio == bio) {
331                         repl = 1;
332                         break;
333                 }
334         }
335
336         BUG_ON(slot == conf->copies);
337         update_head_pos(slot, r10_bio);
338
339         if (slotp)
340                 *slotp = slot;
341         if (replp)
342                 *replp = repl;
343         return r10_bio->devs[slot].devnum;
344 }
345
346 static void raid10_end_read_request(struct bio *bio, int error)
347 {
348         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
349         struct r10bio *r10_bio = bio->bi_private;
350         int slot, dev;
351         struct md_rdev *rdev;
352         struct r10conf *conf = r10_bio->mddev->private;
353
354
355         slot = r10_bio->read_slot;
356         dev = r10_bio->devs[slot].devnum;
357         rdev = r10_bio->devs[slot].rdev;
358         /*
359          * this branch is our 'one mirror IO has finished' event handler:
360          */
361         update_head_pos(slot, r10_bio);
362
363         if (uptodate) {
364                 /*
365                  * Set R10BIO_Uptodate in our master bio, so that
366                  * we will return a good error code to the higher
367                  * levels even if IO on some other mirrored buffer fails.
368                  *
369                  * The 'master' represents the composite IO operation to
370                  * user-side. So if something waits for IO, then it will
371                  * wait for the 'master' bio.
372                  */
373                 set_bit(R10BIO_Uptodate, &r10_bio->state);
374         } else {
375                 /* If all other devices that store this block have
376                  * failed, we want to return the error upwards rather
377                  * than fail the last device.  Here we redefine
378                  * "uptodate" to mean "Don't want to retry"
379                  */
380                 unsigned long flags;
381                 spin_lock_irqsave(&conf->device_lock, flags);
382                 if (!enough(conf, rdev->raid_disk))
383                         uptodate = 1;
384                 spin_unlock_irqrestore(&conf->device_lock, flags);
385         }
386         if (uptodate) {
387                 raid_end_bio_io(r10_bio);
388                 rdev_dec_pending(rdev, conf->mddev);
389         } else {
390                 /*
391                  * oops, read error - keep the refcount on the rdev
392                  */
393                 char b[BDEVNAME_SIZE];
394                 printk_ratelimited(KERN_ERR
395                                    "md/raid10:%s: %s: rescheduling sector %llu\n",
396                                    mdname(conf->mddev),
397                                    bdevname(rdev->bdev, b),
398                                    (unsigned long long)r10_bio->sector);
399                 set_bit(R10BIO_ReadError, &r10_bio->state);
400                 reschedule_retry(r10_bio);
401         }
402 }
403
404 static void close_write(struct r10bio *r10_bio)
405 {
406         /* clear the bitmap if all writes complete successfully */
407         bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
408                         r10_bio->sectors,
409                         !test_bit(R10BIO_Degraded, &r10_bio->state),
410                         0);
411         md_write_end(r10_bio->mddev);
412 }
413
414 static void one_write_done(struct r10bio *r10_bio)
415 {
416         if (atomic_dec_and_test(&r10_bio->remaining)) {
417                 if (test_bit(R10BIO_WriteError, &r10_bio->state))
418                         reschedule_retry(r10_bio);
419                 else {
420                         close_write(r10_bio);
421                         if (test_bit(R10BIO_MadeGood, &r10_bio->state))
422                                 reschedule_retry(r10_bio);
423                         else
424                                 raid_end_bio_io(r10_bio);
425                 }
426         }
427 }
428
429 static void raid10_end_write_request(struct bio *bio, int error)
430 {
431         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
432         struct r10bio *r10_bio = bio->bi_private;
433         int dev;
434         int dec_rdev = 1;
435         struct r10conf *conf = r10_bio->mddev->private;
436         int slot, repl;
437         struct md_rdev *rdev = NULL;
438
439         dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
440
441         if (repl)
442                 rdev = conf->mirrors[dev].replacement;
443         if (!rdev) {
444                 smp_rmb();
445                 repl = 0;
446                 rdev = conf->mirrors[dev].rdev;
447         }
448         /*
449          * this branch is our 'one mirror IO has finished' event handler:
450          */
451         if (!uptodate) {
452                 if (repl)
453                         /* Never record new bad blocks to replacement,
454                          * just fail it.
455                          */
456                         md_error(rdev->mddev, rdev);
457                 else {
458                         set_bit(WriteErrorSeen, &rdev->flags);
459                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
460                                 set_bit(MD_RECOVERY_NEEDED,
461                                         &rdev->mddev->recovery);
462                         set_bit(R10BIO_WriteError, &r10_bio->state);
463                         dec_rdev = 0;
464                 }
465         } else {
466                 /*
467                  * Set R10BIO_Uptodate in our master bio, so that
468                  * we will return a good error code for to the higher
469                  * levels even if IO on some other mirrored buffer fails.
470                  *
471                  * The 'master' represents the composite IO operation to
472                  * user-side. So if something waits for IO, then it will
473                  * wait for the 'master' bio.
474                  */
475                 sector_t first_bad;
476                 int bad_sectors;
477
478                 set_bit(R10BIO_Uptodate, &r10_bio->state);
479
480                 /* Maybe we can clear some bad blocks. */
481                 if (is_badblock(rdev,
482                                 r10_bio->devs[slot].addr,
483                                 r10_bio->sectors,
484                                 &first_bad, &bad_sectors)) {
485                         bio_put(bio);
486                         if (repl)
487                                 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
488                         else
489                                 r10_bio->devs[slot].bio = IO_MADE_GOOD;
490                         dec_rdev = 0;
491                         set_bit(R10BIO_MadeGood, &r10_bio->state);
492                 }
493         }
494
495         /*
496          *
497          * Let's see if all mirrored write operations have finished
498          * already.
499          */
500         one_write_done(r10_bio);
501         if (dec_rdev)
502                 rdev_dec_pending(rdev, conf->mddev);
503 }
504
505 /*
506  * RAID10 layout manager
507  * As well as the chunksize and raid_disks count, there are two
508  * parameters: near_copies and far_copies.
509  * near_copies * far_copies must be <= raid_disks.
510  * Normally one of these will be 1.
511  * If both are 1, we get raid0.
512  * If near_copies == raid_disks, we get raid1.
513  *
514  * Chunks are laid out in raid0 style with near_copies copies of the
515  * first chunk, followed by near_copies copies of the next chunk and
516  * so on.
517  * If far_copies > 1, then after 1/far_copies of the array has been assigned
518  * as described above, we start again with a device offset of near_copies.
519  * So we effectively have another copy of the whole array further down all
520  * the drives, but with blocks on different drives.
521  * With this layout, and block is never stored twice on the one device.
522  *
523  * raid10_find_phys finds the sector offset of a given virtual sector
524  * on each device that it is on.
525  *
526  * raid10_find_virt does the reverse mapping, from a device and a
527  * sector offset to a virtual address
528  */
529
530 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
531 {
532         int n,f;
533         sector_t sector;
534         sector_t chunk;
535         sector_t stripe;
536         int dev;
537         int slot = 0;
538
539         /* now calculate first sector/dev */
540         chunk = r10bio->sector >> geo->chunk_shift;
541         sector = r10bio->sector & geo->chunk_mask;
542
543         chunk *= geo->near_copies;
544         stripe = chunk;
545         dev = sector_div(stripe, geo->raid_disks);
546         if (geo->far_offset)
547                 stripe *= geo->far_copies;
548
549         sector += stripe << geo->chunk_shift;
550
551         /* and calculate all the others */
552         for (n = 0; n < geo->near_copies; n++) {
553                 int d = dev;
554                 sector_t s = sector;
555                 r10bio->devs[slot].addr = sector;
556                 r10bio->devs[slot].devnum = d;
557                 slot++;
558
559                 for (f = 1; f < geo->far_copies; f++) {
560                         d += geo->near_copies;
561                         if (d >= geo->raid_disks)
562                                 d -= geo->raid_disks;
563                         s += geo->stride;
564                         r10bio->devs[slot].devnum = d;
565                         r10bio->devs[slot].addr = s;
566                         slot++;
567                 }
568                 dev++;
569                 if (dev >= geo->raid_disks) {
570                         dev = 0;
571                         sector += (geo->chunk_mask + 1);
572                 }
573         }
574 }
575
576 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
577 {
578         struct geom *geo = &conf->geo;
579
580         if (conf->reshape_progress != MaxSector &&
581             ((r10bio->sector >= conf->reshape_progress) !=
582              conf->mddev->reshape_backwards)) {
583                 set_bit(R10BIO_Previous, &r10bio->state);
584                 geo = &conf->prev;
585         } else
586                 clear_bit(R10BIO_Previous, &r10bio->state);
587
588         __raid10_find_phys(geo, r10bio);
589 }
590
591 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
592 {
593         sector_t offset, chunk, vchunk;
594         /* Never use conf->prev as this is only called during resync
595          * or recovery, so reshape isn't happening
596          */
597         struct geom *geo = &conf->geo;
598
599         offset = sector & geo->chunk_mask;
600         if (geo->far_offset) {
601                 int fc;
602                 chunk = sector >> geo->chunk_shift;
603                 fc = sector_div(chunk, geo->far_copies);
604                 dev -= fc * geo->near_copies;
605                 if (dev < 0)
606                         dev += geo->raid_disks;
607         } else {
608                 while (sector >= geo->stride) {
609                         sector -= geo->stride;
610                         if (dev < geo->near_copies)
611                                 dev += geo->raid_disks - geo->near_copies;
612                         else
613                                 dev -= geo->near_copies;
614                 }
615                 chunk = sector >> geo->chunk_shift;
616         }
617         vchunk = chunk * geo->raid_disks + dev;
618         sector_div(vchunk, geo->near_copies);
619         return (vchunk << geo->chunk_shift) + offset;
620 }
621
622 /**
623  *      raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
624  *      @q: request queue
625  *      @bvm: properties of new bio
626  *      @biovec: the request that could be merged to it.
627  *
628  *      Return amount of bytes we can accept at this offset
629  *      This requires checking for end-of-chunk if near_copies != raid_disks,
630  *      and for subordinate merge_bvec_fns if merge_check_needed.
631  */
632 static int raid10_mergeable_bvec(struct request_queue *q,
633                                  struct bvec_merge_data *bvm,
634                                  struct bio_vec *biovec)
635 {
636         struct mddev *mddev = q->queuedata;
637         struct r10conf *conf = mddev->private;
638         sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
639         int max;
640         unsigned int chunk_sectors;
641         unsigned int bio_sectors = bvm->bi_size >> 9;
642         struct geom *geo = &conf->geo;
643
644         chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
645         if (conf->reshape_progress != MaxSector &&
646             ((sector >= conf->reshape_progress) !=
647              conf->mddev->reshape_backwards))
648                 geo = &conf->prev;
649
650         if (geo->near_copies < geo->raid_disks) {
651                 max = (chunk_sectors - ((sector & (chunk_sectors - 1))
652                                         + bio_sectors)) << 9;
653                 if (max < 0)
654                         /* bio_add cannot handle a negative return */
655                         max = 0;
656                 if (max <= biovec->bv_len && bio_sectors == 0)
657                         return biovec->bv_len;
658         } else
659                 max = biovec->bv_len;
660
661         if (mddev->merge_check_needed) {
662                 struct {
663                         struct r10bio r10_bio;
664                         struct r10dev devs[conf->copies];
665                 } on_stack;
666                 struct r10bio *r10_bio = &on_stack.r10_bio;
667                 int s;
668                 if (conf->reshape_progress != MaxSector) {
669                         /* Cannot give any guidance during reshape */
670                         if (max <= biovec->bv_len && bio_sectors == 0)
671                                 return biovec->bv_len;
672                         return 0;
673                 }
674                 r10_bio->sector = sector;
675                 raid10_find_phys(conf, r10_bio);
676                 rcu_read_lock();
677                 for (s = 0; s < conf->copies; s++) {
678                         int disk = r10_bio->devs[s].devnum;
679                         struct md_rdev *rdev = rcu_dereference(
680                                 conf->mirrors[disk].rdev);
681                         if (rdev && !test_bit(Faulty, &rdev->flags)) {
682                                 struct request_queue *q =
683                                         bdev_get_queue(rdev->bdev);
684                                 if (q->merge_bvec_fn) {
685                                         bvm->bi_sector = r10_bio->devs[s].addr
686                                                 + rdev->data_offset;
687                                         bvm->bi_bdev = rdev->bdev;
688                                         max = min(max, q->merge_bvec_fn(
689                                                           q, bvm, biovec));
690                                 }
691                         }
692                         rdev = rcu_dereference(conf->mirrors[disk].replacement);
693                         if (rdev && !test_bit(Faulty, &rdev->flags)) {
694                                 struct request_queue *q =
695                                         bdev_get_queue(rdev->bdev);
696                                 if (q->merge_bvec_fn) {
697                                         bvm->bi_sector = r10_bio->devs[s].addr
698                                                 + rdev->data_offset;
699                                         bvm->bi_bdev = rdev->bdev;
700                                         max = min(max, q->merge_bvec_fn(
701                                                           q, bvm, biovec));
702                                 }
703                         }
704                 }
705                 rcu_read_unlock();
706         }
707         return max;
708 }
709
710 /*
711  * This routine returns the disk from which the requested read should
712  * be done. There is a per-array 'next expected sequential IO' sector
713  * number - if this matches on the next IO then we use the last disk.
714  * There is also a per-disk 'last know head position' sector that is
715  * maintained from IRQ contexts, both the normal and the resync IO
716  * completion handlers update this position correctly. If there is no
717  * perfect sequential match then we pick the disk whose head is closest.
718  *
719  * If there are 2 mirrors in the same 2 devices, performance degrades
720  * because position is mirror, not device based.
721  *
722  * The rdev for the device selected will have nr_pending incremented.
723  */
724
725 /*
726  * FIXME: possibly should rethink readbalancing and do it differently
727  * depending on near_copies / far_copies geometry.
728  */
729 static struct md_rdev *read_balance(struct r10conf *conf,
730                                     struct r10bio *r10_bio,
731                                     int *max_sectors)
732 {
733         const sector_t this_sector = r10_bio->sector;
734         int disk, slot;
735         int sectors = r10_bio->sectors;
736         int best_good_sectors;
737         sector_t new_distance, best_dist;
738         struct md_rdev *best_rdev, *rdev = NULL;
739         int do_balance;
740         int best_slot;
741         struct geom *geo = &conf->geo;
742
743         raid10_find_phys(conf, r10_bio);
744         rcu_read_lock();
745 retry:
746         sectors = r10_bio->sectors;
747         best_slot = -1;
748         best_rdev = NULL;
749         best_dist = MaxSector;
750         best_good_sectors = 0;
751         do_balance = 1;
752         /*
753          * Check if we can balance. We can balance on the whole
754          * device if no resync is going on (recovery is ok), or below
755          * the resync window. We take the first readable disk when
756          * above the resync window.
757          */
758         if (conf->mddev->recovery_cp < MaxSector
759             && (this_sector + sectors >= conf->next_resync))
760                 do_balance = 0;
761
762         for (slot = 0; slot < conf->copies ; slot++) {
763                 sector_t first_bad;
764                 int bad_sectors;
765                 sector_t dev_sector;
766
767                 if (r10_bio->devs[slot].bio == IO_BLOCKED)
768                         continue;
769                 disk = r10_bio->devs[slot].devnum;
770                 rdev = rcu_dereference(conf->mirrors[disk].replacement);
771                 if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
772                     test_bit(Unmerged, &rdev->flags) ||
773                     r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
774                         rdev = rcu_dereference(conf->mirrors[disk].rdev);
775                 if (rdev == NULL ||
776                     test_bit(Faulty, &rdev->flags) ||
777                     test_bit(Unmerged, &rdev->flags))
778                         continue;
779                 if (!test_bit(In_sync, &rdev->flags) &&
780                     r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
781                         continue;
782
783                 dev_sector = r10_bio->devs[slot].addr;
784                 if (is_badblock(rdev, dev_sector, sectors,
785                                 &first_bad, &bad_sectors)) {
786                         if (best_dist < MaxSector)
787                                 /* Already have a better slot */
788                                 continue;
789                         if (first_bad <= dev_sector) {
790                                 /* Cannot read here.  If this is the
791                                  * 'primary' device, then we must not read
792                                  * beyond 'bad_sectors' from another device.
793                                  */
794                                 bad_sectors -= (dev_sector - first_bad);
795                                 if (!do_balance && sectors > bad_sectors)
796                                         sectors = bad_sectors;
797                                 if (best_good_sectors > sectors)
798                                         best_good_sectors = sectors;
799                         } else {
800                                 sector_t good_sectors =
801                                         first_bad - dev_sector;
802                                 if (good_sectors > best_good_sectors) {
803                                         best_good_sectors = good_sectors;
804                                         best_slot = slot;
805                                         best_rdev = rdev;
806                                 }
807                                 if (!do_balance)
808                                         /* Must read from here */
809                                         break;
810                         }
811                         continue;
812                 } else
813                         best_good_sectors = sectors;
814
815                 if (!do_balance)
816                         break;
817
818                 /* This optimisation is debatable, and completely destroys
819                  * sequential read speed for 'far copies' arrays.  So only
820                  * keep it for 'near' arrays, and review those later.
821                  */
822                 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
823                         break;
824
825                 /* for far > 1 always use the lowest address */
826                 if (geo->far_copies > 1)
827                         new_distance = r10_bio->devs[slot].addr;
828                 else
829                         new_distance = abs(r10_bio->devs[slot].addr -
830                                            conf->mirrors[disk].head_position);
831                 if (new_distance < best_dist) {
832                         best_dist = new_distance;
833                         best_slot = slot;
834                         best_rdev = rdev;
835                 }
836         }
837         if (slot >= conf->copies) {
838                 slot = best_slot;
839                 rdev = best_rdev;
840         }
841
842         if (slot >= 0) {
843                 atomic_inc(&rdev->nr_pending);
844                 if (test_bit(Faulty, &rdev->flags)) {
845                         /* Cannot risk returning a device that failed
846                          * before we inc'ed nr_pending
847                          */
848                         rdev_dec_pending(rdev, conf->mddev);
849                         goto retry;
850                 }
851                 r10_bio->read_slot = slot;
852         } else
853                 rdev = NULL;
854         rcu_read_unlock();
855         *max_sectors = best_good_sectors;
856
857         return rdev;
858 }
859
860 int md_raid10_congested(struct mddev *mddev, int bits)
861 {
862         struct r10conf *conf = mddev->private;
863         int i, ret = 0;
864
865         if ((bits & (1 << BDI_async_congested)) &&
866             conf->pending_count >= max_queued_requests)
867                 return 1;
868
869         rcu_read_lock();
870         for (i = 0;
871              (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
872                      && ret == 0;
873              i++) {
874                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
875                 if (rdev && !test_bit(Faulty, &rdev->flags)) {
876                         struct request_queue *q = bdev_get_queue(rdev->bdev);
877
878                         ret |= bdi_congested(&q->backing_dev_info, bits);
879                 }
880         }
881         rcu_read_unlock();
882         return ret;
883 }
884 EXPORT_SYMBOL_GPL(md_raid10_congested);
885
886 static int raid10_congested(void *data, int bits)
887 {
888         struct mddev *mddev = data;
889
890         return mddev_congested(mddev, bits) ||
891                 md_raid10_congested(mddev, bits);
892 }
893
894 static void flush_pending_writes(struct r10conf *conf)
895 {
896         /* Any writes that have been queued but are awaiting
897          * bitmap updates get flushed here.
898          */
899         spin_lock_irq(&conf->device_lock);
900
901         if (conf->pending_bio_list.head) {
902                 struct bio *bio;
903                 bio = bio_list_get(&conf->pending_bio_list);
904                 conf->pending_count = 0;
905                 spin_unlock_irq(&conf->device_lock);
906                 /* flush any pending bitmap writes to disk
907                  * before proceeding w/ I/O */
908                 bitmap_unplug(conf->mddev->bitmap);
909                 wake_up(&conf->wait_barrier);
910
911                 while (bio) { /* submit pending writes */
912                         struct bio *next = bio->bi_next;
913                         bio->bi_next = NULL;
914                         if (unlikely((bio->bi_rw & REQ_DISCARD) &&
915                             !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
916                                 /* Just ignore it */
917                                 bio_endio(bio, 0);
918                         else
919                                 generic_make_request(bio);
920                         bio = next;
921                 }
922         } else
923                 spin_unlock_irq(&conf->device_lock);
924 }
925
926 /* Barriers....
927  * Sometimes we need to suspend IO while we do something else,
928  * either some resync/recovery, or reconfigure the array.
929  * To do this we raise a 'barrier'.
930  * The 'barrier' is a counter that can be raised multiple times
931  * to count how many activities are happening which preclude
932  * normal IO.
933  * We can only raise the barrier if there is no pending IO.
934  * i.e. if nr_pending == 0.
935  * We choose only to raise the barrier if no-one is waiting for the
936  * barrier to go down.  This means that as soon as an IO request
937  * is ready, no other operations which require a barrier will start
938  * until the IO request has had a chance.
939  *
940  * So: regular IO calls 'wait_barrier'.  When that returns there
941  *    is no backgroup IO happening,  It must arrange to call
942  *    allow_barrier when it has finished its IO.
943  * backgroup IO calls must call raise_barrier.  Once that returns
944  *    there is no normal IO happeing.  It must arrange to call
945  *    lower_barrier when the particular background IO completes.
946  */
947
948 static void raise_barrier(struct r10conf *conf, int force)
949 {
950         BUG_ON(force && !conf->barrier);
951         spin_lock_irq(&conf->resync_lock);
952
953         /* Wait until no block IO is waiting (unless 'force') */
954         wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
955                             conf->resync_lock);
956
957         /* block any new IO from starting */
958         conf->barrier++;
959
960         /* Now wait for all pending IO to complete */
961         wait_event_lock_irq(conf->wait_barrier,
962                             !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
963                             conf->resync_lock);
964
965         spin_unlock_irq(&conf->resync_lock);
966 }
967
968 static void lower_barrier(struct r10conf *conf)
969 {
970         unsigned long flags;
971         spin_lock_irqsave(&conf->resync_lock, flags);
972         conf->barrier--;
973         spin_unlock_irqrestore(&conf->resync_lock, flags);
974         wake_up(&conf->wait_barrier);
975 }
976
977 static void wait_barrier(struct r10conf *conf)
978 {
979         spin_lock_irq(&conf->resync_lock);
980         if (conf->barrier) {
981                 conf->nr_waiting++;
982                 /* Wait for the barrier to drop.
983                  * However if there are already pending
984                  * requests (preventing the barrier from
985                  * rising completely), and the
986                  * pre-process bio queue isn't empty,
987                  * then don't wait, as we need to empty
988                  * that queue to get the nr_pending
989                  * count down.
990                  */
991                 wait_event_lock_irq(conf->wait_barrier,
992                                     !conf->barrier ||
993                                     (conf->nr_pending &&
994                                      current->bio_list &&
995                                      !bio_list_empty(current->bio_list)),
996                                     conf->resync_lock);
997                 conf->nr_waiting--;
998         }
999         conf->nr_pending++;
1000         spin_unlock_irq(&conf->resync_lock);
1001 }
1002
1003 static void allow_barrier(struct r10conf *conf)
1004 {
1005         unsigned long flags;
1006         spin_lock_irqsave(&conf->resync_lock, flags);
1007         conf->nr_pending--;
1008         spin_unlock_irqrestore(&conf->resync_lock, flags);
1009         wake_up(&conf->wait_barrier);
1010 }
1011
1012 static void freeze_array(struct r10conf *conf)
1013 {
1014         /* stop syncio and normal IO and wait for everything to
1015          * go quiet.
1016          * We increment barrier and nr_waiting, and then
1017          * wait until nr_pending match nr_queued+1
1018          * This is called in the context of one normal IO request
1019          * that has failed. Thus any sync request that might be pending
1020          * will be blocked by nr_pending, and we need to wait for
1021          * pending IO requests to complete or be queued for re-try.
1022          * Thus the number queued (nr_queued) plus this request (1)
1023          * must match the number of pending IOs (nr_pending) before
1024          * we continue.
1025          */
1026         spin_lock_irq(&conf->resync_lock);
1027         conf->barrier++;
1028         conf->nr_waiting++;
1029         wait_event_lock_irq_cmd(conf->wait_barrier,
1030                                 conf->nr_pending == conf->nr_queued+1,
1031                                 conf->resync_lock,
1032                                 flush_pending_writes(conf));
1033
1034         spin_unlock_irq(&conf->resync_lock);
1035 }
1036
1037 static void unfreeze_array(struct r10conf *conf)
1038 {
1039         /* reverse the effect of the freeze */
1040         spin_lock_irq(&conf->resync_lock);
1041         conf->barrier--;
1042         conf->nr_waiting--;
1043         wake_up(&conf->wait_barrier);
1044         spin_unlock_irq(&conf->resync_lock);
1045 }
1046
1047 static sector_t choose_data_offset(struct r10bio *r10_bio,
1048                                    struct md_rdev *rdev)
1049 {
1050         if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1051             test_bit(R10BIO_Previous, &r10_bio->state))
1052                 return rdev->data_offset;
1053         else
1054                 return rdev->new_data_offset;
1055 }
1056
1057 struct raid10_plug_cb {
1058         struct blk_plug_cb      cb;
1059         struct bio_list         pending;
1060         int                     pending_cnt;
1061 };
1062
1063 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1064 {
1065         struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
1066                                                    cb);
1067         struct mddev *mddev = plug->cb.data;
1068         struct r10conf *conf = mddev->private;
1069         struct bio *bio;
1070
1071         if (from_schedule || current->bio_list) {
1072                 spin_lock_irq(&conf->device_lock);
1073                 bio_list_merge(&conf->pending_bio_list, &plug->pending);
1074                 conf->pending_count += plug->pending_cnt;
1075                 spin_unlock_irq(&conf->device_lock);
1076                 md_wakeup_thread(mddev->thread);
1077                 kfree(plug);
1078                 return;
1079         }
1080
1081         /* we aren't scheduling, so we can do the write-out directly. */
1082         bio = bio_list_get(&plug->pending);
1083         bitmap_unplug(mddev->bitmap);
1084         wake_up(&conf->wait_barrier);
1085
1086         while (bio) { /* submit pending writes */
1087                 struct bio *next = bio->bi_next;
1088                 bio->bi_next = NULL;
1089                 generic_make_request(bio);
1090                 bio = next;
1091         }
1092         kfree(plug);
1093 }
1094
1095 static void make_request(struct mddev *mddev, struct bio * bio)
1096 {
1097         struct r10conf *conf = mddev->private;
1098         struct r10bio *r10_bio;
1099         struct bio *read_bio;
1100         int i;
1101         sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1102         int chunk_sects = chunk_mask + 1;
1103         const int rw = bio_data_dir(bio);
1104         const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
1105         const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
1106         const unsigned long do_discard = (bio->bi_rw
1107                                           & (REQ_DISCARD | REQ_SECURE));
1108         unsigned long flags;
1109         struct md_rdev *blocked_rdev;
1110         struct blk_plug_cb *cb;
1111         struct raid10_plug_cb *plug = NULL;
1112         int sectors_handled;
1113         int max_sectors;
1114         int sectors;
1115
1116         if (unlikely(bio->bi_rw & REQ_FLUSH)) {
1117                 md_flush_request(mddev, bio);
1118                 return;
1119         }
1120
1121         /* If this request crosses a chunk boundary, we need to
1122          * split it.  This will only happen for 1 PAGE (or less) requests.
1123          */
1124         if (unlikely((bio->bi_sector & chunk_mask) + (bio->bi_size >> 9)
1125                      > chunk_sects
1126                      && (conf->geo.near_copies < conf->geo.raid_disks
1127                          || conf->prev.near_copies < conf->prev.raid_disks))) {
1128                 struct bio_pair *bp;
1129                 /* Sanity check -- queue functions should prevent this happening */
1130                 if ((bio->bi_vcnt != 1 && bio->bi_vcnt != 0) ||
1131                     bio->bi_idx != 0)
1132                         goto bad_map;
1133                 /* This is a one page bio that upper layers
1134                  * refuse to split for us, so we need to split it.
1135                  */
1136                 bp = bio_split(bio,
1137                                chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
1138
1139                 /* Each of these 'make_request' calls will call 'wait_barrier'.
1140                  * If the first succeeds but the second blocks due to the resync
1141                  * thread raising the barrier, we will deadlock because the
1142                  * IO to the underlying device will be queued in generic_make_request
1143                  * and will never complete, so will never reduce nr_pending.
1144                  * So increment nr_waiting here so no new raise_barriers will
1145                  * succeed, and so the second wait_barrier cannot block.
1146                  */
1147                 spin_lock_irq(&conf->resync_lock);
1148                 conf->nr_waiting++;
1149                 spin_unlock_irq(&conf->resync_lock);
1150
1151                 make_request(mddev, &bp->bio1);
1152                 make_request(mddev, &bp->bio2);
1153
1154                 spin_lock_irq(&conf->resync_lock);
1155                 conf->nr_waiting--;
1156                 wake_up(&conf->wait_barrier);
1157                 spin_unlock_irq(&conf->resync_lock);
1158
1159                 bio_pair_release(bp);
1160                 return;
1161         bad_map:
1162                 printk("md/raid10:%s: make_request bug: can't convert block across chunks"
1163                        " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
1164                        (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
1165
1166                 bio_io_error(bio);
1167                 return;
1168         }
1169
1170         md_write_start(mddev, bio);
1171
1172         /*
1173          * Register the new request and wait if the reconstruction
1174          * thread has put up a bar for new requests.
1175          * Continue immediately if no resync is active currently.
1176          */
1177         wait_barrier(conf);
1178
1179         sectors = bio->bi_size >> 9;
1180         while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1181             bio->bi_sector < conf->reshape_progress &&
1182             bio->bi_sector + sectors > conf->reshape_progress) {
1183                 /* IO spans the reshape position.  Need to wait for
1184                  * reshape to pass
1185                  */
1186                 allow_barrier(conf);
1187                 wait_event(conf->wait_barrier,
1188                            conf->reshape_progress <= bio->bi_sector ||
1189                            conf->reshape_progress >= bio->bi_sector + sectors);
1190                 wait_barrier(conf);
1191         }
1192         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1193             bio_data_dir(bio) == WRITE &&
1194             (mddev->reshape_backwards
1195              ? (bio->bi_sector < conf->reshape_safe &&
1196                 bio->bi_sector + sectors > conf->reshape_progress)
1197              : (bio->bi_sector + sectors > conf->reshape_safe &&
1198                 bio->bi_sector < conf->reshape_progress))) {
1199                 /* Need to update reshape_position in metadata */
1200                 mddev->reshape_position = conf->reshape_progress;
1201                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
1202                 set_bit(MD_CHANGE_PENDING, &mddev->flags);
1203                 md_wakeup_thread(mddev->thread);
1204                 wait_event(mddev->sb_wait,
1205                            !test_bit(MD_CHANGE_PENDING, &mddev->flags));
1206
1207                 conf->reshape_safe = mddev->reshape_position;
1208         }
1209
1210         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1211
1212         r10_bio->master_bio = bio;
1213         r10_bio->sectors = sectors;
1214
1215         r10_bio->mddev = mddev;
1216         r10_bio->sector = bio->bi_sector;
1217         r10_bio->state = 0;
1218
1219         /* We might need to issue multiple reads to different
1220          * devices if there are bad blocks around, so we keep
1221          * track of the number of reads in bio->bi_phys_segments.
1222          * If this is 0, there is only one r10_bio and no locking
1223          * will be needed when the request completes.  If it is
1224          * non-zero, then it is the number of not-completed requests.
1225          */
1226         bio->bi_phys_segments = 0;
1227         clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1228
1229         if (rw == READ) {
1230                 /*
1231                  * read balancing logic:
1232                  */
1233                 struct md_rdev *rdev;
1234                 int slot;
1235
1236 read_again:
1237                 rdev = read_balance(conf, r10_bio, &max_sectors);
1238                 if (!rdev) {
1239                         raid_end_bio_io(r10_bio);
1240                         return;
1241                 }
1242                 slot = r10_bio->read_slot;
1243
1244                 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1245                 md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector,
1246                             max_sectors);
1247
1248                 r10_bio->devs[slot].bio = read_bio;
1249                 r10_bio->devs[slot].rdev = rdev;
1250
1251                 read_bio->bi_sector = r10_bio->devs[slot].addr +
1252                         choose_data_offset(r10_bio, rdev);
1253                 read_bio->bi_bdev = rdev->bdev;
1254                 read_bio->bi_end_io = raid10_end_read_request;
1255                 read_bio->bi_rw = READ | do_sync;
1256                 read_bio->bi_private = r10_bio;
1257
1258                 if (max_sectors < r10_bio->sectors) {
1259                         /* Could not read all from this device, so we will
1260                          * need another r10_bio.
1261                          */
1262                         sectors_handled = (r10_bio->sectors + max_sectors
1263                                            - bio->bi_sector);
1264                         r10_bio->sectors = max_sectors;
1265                         spin_lock_irq(&conf->device_lock);
1266                         if (bio->bi_phys_segments == 0)
1267                                 bio->bi_phys_segments = 2;
1268                         else
1269                                 bio->bi_phys_segments++;
1270                         spin_unlock(&conf->device_lock);
1271                         /* Cannot call generic_make_request directly
1272                          * as that will be queued in __generic_make_request
1273                          * and subsequent mempool_alloc might block
1274                          * waiting for it.  so hand bio over to raid10d.
1275                          */
1276                         reschedule_retry(r10_bio);
1277
1278                         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1279
1280                         r10_bio->master_bio = bio;
1281                         r10_bio->sectors = ((bio->bi_size >> 9)
1282                                             - sectors_handled);
1283                         r10_bio->state = 0;
1284                         r10_bio->mddev = mddev;
1285                         r10_bio->sector = bio->bi_sector + sectors_handled;
1286                         goto read_again;
1287                 } else
1288                         generic_make_request(read_bio);
1289                 return;
1290         }
1291
1292         /*
1293          * WRITE:
1294          */
1295         if (conf->pending_count >= max_queued_requests) {
1296                 md_wakeup_thread(mddev->thread);
1297                 wait_event(conf->wait_barrier,
1298                            conf->pending_count < max_queued_requests);
1299         }
1300         /* first select target devices under rcu_lock and
1301          * inc refcount on their rdev.  Record them by setting
1302          * bios[x] to bio
1303          * If there are known/acknowledged bad blocks on any device
1304          * on which we have seen a write error, we want to avoid
1305          * writing to those blocks.  This potentially requires several
1306          * writes to write around the bad blocks.  Each set of writes
1307          * gets its own r10_bio with a set of bios attached.  The number
1308          * of r10_bios is recored in bio->bi_phys_segments just as with
1309          * the read case.
1310          */
1311
1312         r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1313         raid10_find_phys(conf, r10_bio);
1314 retry_write:
1315         blocked_rdev = NULL;
1316         rcu_read_lock();
1317         max_sectors = r10_bio->sectors;
1318
1319         for (i = 0;  i < conf->copies; i++) {
1320                 int d = r10_bio->devs[i].devnum;
1321                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1322                 struct md_rdev *rrdev = rcu_dereference(
1323                         conf->mirrors[d].replacement);
1324                 if (rdev == rrdev)
1325                         rrdev = NULL;
1326                 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1327                         atomic_inc(&rdev->nr_pending);
1328                         blocked_rdev = rdev;
1329                         break;
1330                 }
1331                 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1332                         atomic_inc(&rrdev->nr_pending);
1333                         blocked_rdev = rrdev;
1334                         break;
1335                 }
1336                 if (rdev && (test_bit(Faulty, &rdev->flags)
1337                              || test_bit(Unmerged, &rdev->flags)))
1338                         rdev = NULL;
1339                 if (rrdev && (test_bit(Faulty, &rrdev->flags)
1340                               || test_bit(Unmerged, &rrdev->flags)))
1341                         rrdev = NULL;
1342
1343                 r10_bio->devs[i].bio = NULL;
1344                 r10_bio->devs[i].repl_bio = NULL;
1345
1346                 if (!rdev && !rrdev) {
1347                         set_bit(R10BIO_Degraded, &r10_bio->state);
1348                         continue;
1349                 }
1350                 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1351                         sector_t first_bad;
1352                         sector_t dev_sector = r10_bio->devs[i].addr;
1353                         int bad_sectors;
1354                         int is_bad;
1355
1356                         is_bad = is_badblock(rdev, dev_sector,
1357                                              max_sectors,
1358                                              &first_bad, &bad_sectors);
1359                         if (is_bad < 0) {
1360                                 /* Mustn't write here until the bad block
1361                                  * is acknowledged
1362                                  */
1363                                 atomic_inc(&rdev->nr_pending);
1364                                 set_bit(BlockedBadBlocks, &rdev->flags);
1365                                 blocked_rdev = rdev;
1366                                 break;
1367                         }
1368                         if (is_bad && first_bad <= dev_sector) {
1369                                 /* Cannot write here at all */
1370                                 bad_sectors -= (dev_sector - first_bad);
1371                                 if (bad_sectors < max_sectors)
1372                                         /* Mustn't write more than bad_sectors
1373                                          * to other devices yet
1374                                          */
1375                                         max_sectors = bad_sectors;
1376                                 /* We don't set R10BIO_Degraded as that
1377                                  * only applies if the disk is missing,
1378                                  * so it might be re-added, and we want to
1379                                  * know to recover this chunk.
1380                                  * In this case the device is here, and the
1381                                  * fact that this chunk is not in-sync is
1382                                  * recorded in the bad block log.
1383                                  */
1384                                 continue;
1385                         }
1386                         if (is_bad) {
1387                                 int good_sectors = first_bad - dev_sector;
1388                                 if (good_sectors < max_sectors)
1389                                         max_sectors = good_sectors;
1390                         }
1391                 }
1392                 if (rdev) {
1393                         r10_bio->devs[i].bio = bio;
1394                         atomic_inc(&rdev->nr_pending);
1395                 }
1396                 if (rrdev) {
1397                         r10_bio->devs[i].repl_bio = bio;
1398                         atomic_inc(&rrdev->nr_pending);
1399                 }
1400         }
1401         rcu_read_unlock();
1402
1403         if (unlikely(blocked_rdev)) {
1404                 /* Have to wait for this device to get unblocked, then retry */
1405                 int j;
1406                 int d;
1407
1408                 for (j = 0; j < i; j++) {
1409                         if (r10_bio->devs[j].bio) {
1410                                 d = r10_bio->devs[j].devnum;
1411                                 rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1412                         }
1413                         if (r10_bio->devs[j].repl_bio) {
1414                                 struct md_rdev *rdev;
1415                                 d = r10_bio->devs[j].devnum;
1416                                 rdev = conf->mirrors[d].replacement;
1417                                 if (!rdev) {
1418                                         /* Race with remove_disk */
1419                                         smp_mb();
1420                                         rdev = conf->mirrors[d].rdev;
1421                                 }
1422                                 rdev_dec_pending(rdev, mddev);
1423                         }
1424                 }
1425                 allow_barrier(conf);
1426                 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1427                 wait_barrier(conf);
1428                 goto retry_write;
1429         }
1430
1431         if (max_sectors < r10_bio->sectors) {
1432                 /* We are splitting this into multiple parts, so
1433                  * we need to prepare for allocating another r10_bio.
1434                  */
1435                 r10_bio->sectors = max_sectors;
1436                 spin_lock_irq(&conf->device_lock);
1437                 if (bio->bi_phys_segments == 0)
1438                         bio->bi_phys_segments = 2;
1439                 else
1440                         bio->bi_phys_segments++;
1441                 spin_unlock_irq(&conf->device_lock);
1442         }
1443         sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector;
1444
1445         atomic_set(&r10_bio->remaining, 1);
1446         bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1447
1448         for (i = 0; i < conf->copies; i++) {
1449                 struct bio *mbio;
1450                 int d = r10_bio->devs[i].devnum;
1451                 if (r10_bio->devs[i].bio) {
1452                         struct md_rdev *rdev = conf->mirrors[d].rdev;
1453                         mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1454                         md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1455                                     max_sectors);
1456                         r10_bio->devs[i].bio = mbio;
1457
1458                         mbio->bi_sector = (r10_bio->devs[i].addr+
1459                                            choose_data_offset(r10_bio,
1460                                                               rdev));
1461                         mbio->bi_bdev = rdev->bdev;
1462                         mbio->bi_end_io = raid10_end_write_request;
1463                         mbio->bi_rw = WRITE | do_sync | do_fua | do_discard;
1464                         mbio->bi_private = r10_bio;
1465
1466                         atomic_inc(&r10_bio->remaining);
1467
1468                         cb = blk_check_plugged(raid10_unplug, mddev,
1469                                                sizeof(*plug));
1470                         if (cb)
1471                                 plug = container_of(cb, struct raid10_plug_cb,
1472                                                     cb);
1473                         else
1474                                 plug = NULL;
1475                         spin_lock_irqsave(&conf->device_lock, flags);
1476                         if (plug) {
1477                                 bio_list_add(&plug->pending, mbio);
1478                                 plug->pending_cnt++;
1479                         } else {
1480                                 bio_list_add(&conf->pending_bio_list, mbio);
1481                                 conf->pending_count++;
1482                         }
1483                         spin_unlock_irqrestore(&conf->device_lock, flags);
1484                         if (!plug)
1485                                 md_wakeup_thread(mddev->thread);
1486                 }
1487
1488                 if (r10_bio->devs[i].repl_bio) {
1489                         struct md_rdev *rdev = conf->mirrors[d].replacement;
1490                         if (rdev == NULL) {
1491                                 /* Replacement just got moved to main 'rdev' */
1492                                 smp_mb();
1493                                 rdev = conf->mirrors[d].rdev;
1494                         }
1495                         mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1496                         md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1497                                     max_sectors);
1498                         r10_bio->devs[i].repl_bio = mbio;
1499
1500                         mbio->bi_sector = (r10_bio->devs[i].addr +
1501                                            choose_data_offset(
1502                                                    r10_bio, rdev));
1503                         mbio->bi_bdev = rdev->bdev;
1504                         mbio->bi_end_io = raid10_end_write_request;
1505                         mbio->bi_rw = WRITE | do_sync | do_fua | do_discard;
1506                         mbio->bi_private = r10_bio;
1507
1508                         atomic_inc(&r10_bio->remaining);
1509                         spin_lock_irqsave(&conf->device_lock, flags);
1510                         bio_list_add(&conf->pending_bio_list, mbio);
1511                         conf->pending_count++;
1512                         spin_unlock_irqrestore(&conf->device_lock, flags);
1513                         if (!mddev_check_plugged(mddev))
1514                                 md_wakeup_thread(mddev->thread);
1515                 }
1516         }
1517
1518         /* Don't remove the bias on 'remaining' (one_write_done) until
1519          * after checking if we need to go around again.
1520          */
1521
1522         if (sectors_handled < (bio->bi_size >> 9)) {
1523                 one_write_done(r10_bio);
1524                 /* We need another r10_bio.  It has already been counted
1525                  * in bio->bi_phys_segments.
1526                  */
1527                 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1528
1529                 r10_bio->master_bio = bio;
1530                 r10_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
1531
1532                 r10_bio->mddev = mddev;
1533                 r10_bio->sector = bio->bi_sector + sectors_handled;
1534                 r10_bio->state = 0;
1535                 goto retry_write;
1536         }
1537         one_write_done(r10_bio);
1538
1539         /* In case raid10d snuck in to freeze_array */
1540         wake_up(&conf->wait_barrier);
1541 }
1542
1543 static void status(struct seq_file *seq, struct mddev *mddev)
1544 {
1545         struct r10conf *conf = mddev->private;
1546         int i;
1547
1548         if (conf->geo.near_copies < conf->geo.raid_disks)
1549                 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1550         if (conf->geo.near_copies > 1)
1551                 seq_printf(seq, " %d near-copies", conf->geo.near_copies);
1552         if (conf->geo.far_copies > 1) {
1553                 if (conf->geo.far_offset)
1554                         seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1555                 else
1556                         seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1557         }
1558         seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
1559                                         conf->geo.raid_disks - mddev->degraded);
1560         for (i = 0; i < conf->geo.raid_disks; i++)
1561                 seq_printf(seq, "%s",
1562                               conf->mirrors[i].rdev &&
1563                               test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
1564         seq_printf(seq, "]");
1565 }
1566
1567 /* check if there are enough drives for
1568  * every block to appear on atleast one.
1569  * Don't consider the device numbered 'ignore'
1570  * as we might be about to remove it.
1571  */
1572 static int _enough(struct r10conf *conf, struct geom *geo, int ignore)
1573 {
1574         int first = 0;
1575
1576         do {
1577                 int n = conf->copies;
1578                 int cnt = 0;
1579                 int this = first;
1580                 while (n--) {
1581                         if (conf->mirrors[this].rdev &&
1582                             this != ignore)
1583                                 cnt++;
1584                         this = (this+1) % geo->raid_disks;
1585                 }
1586                 if (cnt == 0)
1587                         return 0;
1588                 first = (first + geo->near_copies) % geo->raid_disks;
1589         } while (first != 0);
1590         return 1;
1591 }
1592
1593 static int enough(struct r10conf *conf, int ignore)
1594 {
1595         return _enough(conf, &conf->geo, ignore) &&
1596                 _enough(conf, &conf->prev, ignore);
1597 }
1598
1599 static void error(struct mddev *mddev, struct md_rdev *rdev)
1600 {
1601         char b[BDEVNAME_SIZE];
1602         struct r10conf *conf = mddev->private;
1603
1604         /*
1605          * If it is not operational, then we have already marked it as dead
1606          * else if it is the last working disks, ignore the error, let the
1607          * next level up know.
1608          * else mark the drive as failed
1609          */
1610         if (test_bit(In_sync, &rdev->flags)
1611             && !enough(conf, rdev->raid_disk))
1612                 /*
1613                  * Don't fail the drive, just return an IO error.
1614                  */
1615                 return;
1616         if (test_and_clear_bit(In_sync, &rdev->flags)) {
1617                 unsigned long flags;
1618                 spin_lock_irqsave(&conf->device_lock, flags);
1619                 mddev->degraded++;
1620                 spin_unlock_irqrestore(&conf->device_lock, flags);
1621                 /*
1622                  * if recovery is running, make sure it aborts.
1623                  */
1624                 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1625         }
1626         set_bit(Blocked, &rdev->flags);
1627         set_bit(Faulty, &rdev->flags);
1628         set_bit(MD_CHANGE_DEVS, &mddev->flags);
1629         printk(KERN_ALERT
1630                "md/raid10:%s: Disk failure on %s, disabling device.\n"
1631                "md/raid10:%s: Operation continuing on %d devices.\n",
1632                mdname(mddev), bdevname(rdev->bdev, b),
1633                mdname(mddev), conf->geo.raid_disks - mddev->degraded);
1634 }
1635
1636 static void print_conf(struct r10conf *conf)
1637 {
1638         int i;
1639         struct raid10_info *tmp;
1640
1641         printk(KERN_DEBUG "RAID10 conf printout:\n");
1642         if (!conf) {
1643                 printk(KERN_DEBUG "(!conf)\n");
1644                 return;
1645         }
1646         printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
1647                 conf->geo.raid_disks);
1648
1649         for (i = 0; i < conf->geo.raid_disks; i++) {
1650                 char b[BDEVNAME_SIZE];
1651                 tmp = conf->mirrors + i;
1652                 if (tmp->rdev)
1653                         printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1654                                 i, !test_bit(In_sync, &tmp->rdev->flags),
1655                                 !test_bit(Faulty, &tmp->rdev->flags),
1656                                 bdevname(tmp->rdev->bdev,b));
1657         }
1658 }
1659
1660 static void close_sync(struct r10conf *conf)
1661 {
1662         wait_barrier(conf);
1663         allow_barrier(conf);
1664
1665         mempool_destroy(conf->r10buf_pool);
1666         conf->r10buf_pool = NULL;
1667 }
1668
1669 static int raid10_spare_active(struct mddev *mddev)
1670 {
1671         int i;
1672         struct r10conf *conf = mddev->private;
1673         struct raid10_info *tmp;
1674         int count = 0;
1675         unsigned long flags;
1676
1677         /*
1678          * Find all non-in_sync disks within the RAID10 configuration
1679          * and mark them in_sync
1680          */
1681         for (i = 0; i < conf->geo.raid_disks; i++) {
1682                 tmp = conf->mirrors + i;
1683                 if (tmp->replacement
1684                     && tmp->replacement->recovery_offset == MaxSector
1685                     && !test_bit(Faulty, &tmp->replacement->flags)
1686                     && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
1687                         /* Replacement has just become active */
1688                         if (!tmp->rdev
1689                             || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
1690                                 count++;
1691                         if (tmp->rdev) {
1692                                 /* Replaced device not technically faulty,
1693                                  * but we need to be sure it gets removed
1694                                  * and never re-added.
1695                                  */
1696                                 set_bit(Faulty, &tmp->rdev->flags);
1697                                 sysfs_notify_dirent_safe(
1698                                         tmp->rdev->sysfs_state);
1699                         }
1700                         sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
1701                 } else if (tmp->rdev
1702                            && !test_bit(Faulty, &tmp->rdev->flags)
1703                            && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1704                         count++;
1705                         sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
1706                 }
1707         }
1708         spin_lock_irqsave(&conf->device_lock, flags);
1709         mddev->degraded -= count;
1710         spin_unlock_irqrestore(&conf->device_lock, flags);
1711
1712         print_conf(conf);
1713         return count;
1714 }
1715
1716
1717 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1718 {
1719         struct r10conf *conf = mddev->private;
1720         int err = -EEXIST;
1721         int mirror;
1722         int first = 0;
1723         int last = conf->geo.raid_disks - 1;
1724         struct request_queue *q = bdev_get_queue(rdev->bdev);
1725
1726         if (mddev->recovery_cp < MaxSector)
1727                 /* only hot-add to in-sync arrays, as recovery is
1728                  * very different from resync
1729                  */
1730                 return -EBUSY;
1731         if (rdev->saved_raid_disk < 0 && !_enough(conf, &conf->prev, -1))
1732                 return -EINVAL;
1733
1734         if (rdev->raid_disk >= 0)
1735                 first = last = rdev->raid_disk;
1736
1737         if (q->merge_bvec_fn) {
1738                 set_bit(Unmerged, &rdev->flags);
1739                 mddev->merge_check_needed = 1;
1740         }
1741
1742         if (rdev->saved_raid_disk >= first &&
1743             conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1744                 mirror = rdev->saved_raid_disk;
1745         else
1746                 mirror = first;
1747         for ( ; mirror <= last ; mirror++) {
1748                 struct raid10_info *p = &conf->mirrors[mirror];
1749                 if (p->recovery_disabled == mddev->recovery_disabled)
1750                         continue;
1751                 if (p->rdev) {
1752                         if (!test_bit(WantReplacement, &p->rdev->flags) ||
1753                             p->replacement != NULL)
1754                                 continue;
1755                         clear_bit(In_sync, &rdev->flags);
1756                         set_bit(Replacement, &rdev->flags);
1757                         rdev->raid_disk = mirror;
1758                         err = 0;
1759                         disk_stack_limits(mddev->gendisk, rdev->bdev,
1760                                           rdev->data_offset << 9);
1761                         conf->fullsync = 1;
1762                         rcu_assign_pointer(p->replacement, rdev);
1763                         break;
1764                 }
1765
1766                 disk_stack_limits(mddev->gendisk, rdev->bdev,
1767                                   rdev->data_offset << 9);
1768
1769                 p->head_position = 0;
1770                 p->recovery_disabled = mddev->recovery_disabled - 1;
1771                 rdev->raid_disk = mirror;
1772                 err = 0;
1773                 if (rdev->saved_raid_disk != mirror)
1774                         conf->fullsync = 1;
1775                 rcu_assign_pointer(p->rdev, rdev);
1776                 break;
1777         }
1778         if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
1779                 /* Some requests might not have seen this new
1780                  * merge_bvec_fn.  We must wait for them to complete
1781                  * before merging the device fully.
1782                  * First we make sure any code which has tested
1783                  * our function has submitted the request, then
1784                  * we wait for all outstanding requests to complete.
1785                  */
1786                 synchronize_sched();
1787                 raise_barrier(conf, 0);
1788                 lower_barrier(conf);
1789                 clear_bit(Unmerged, &rdev->flags);
1790         }
1791         md_integrity_add_rdev(rdev, mddev);
1792         if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1793                 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
1794
1795         print_conf(conf);
1796         return err;
1797 }
1798
1799 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1800 {
1801         struct r10conf *conf = mddev->private;
1802         int err = 0;
1803         int number = rdev->raid_disk;
1804         struct md_rdev **rdevp;
1805         struct raid10_info *p = conf->mirrors + number;
1806
1807         print_conf(conf);
1808         if (rdev == p->rdev)
1809                 rdevp = &p->rdev;
1810         else if (rdev == p->replacement)
1811                 rdevp = &p->replacement;
1812         else
1813                 return 0;
1814
1815         if (test_bit(In_sync, &rdev->flags) ||
1816             atomic_read(&rdev->nr_pending)) {
1817                 err = -EBUSY;
1818                 goto abort;
1819         }
1820         /* Only remove faulty devices if recovery
1821          * is not possible.
1822          */
1823         if (!test_bit(Faulty, &rdev->flags) &&
1824             mddev->recovery_disabled != p->recovery_disabled &&
1825             (!p->replacement || p->replacement == rdev) &&
1826             number < conf->geo.raid_disks &&
1827             enough(conf, -1)) {
1828                 err = -EBUSY;
1829                 goto abort;
1830         }
1831         *rdevp = NULL;
1832         synchronize_rcu();
1833         if (atomic_read(&rdev->nr_pending)) {
1834                 /* lost the race, try later */
1835                 err = -EBUSY;
1836                 *rdevp = rdev;
1837                 goto abort;
1838         } else if (p->replacement) {
1839                 /* We must have just cleared 'rdev' */
1840                 p->rdev = p->replacement;
1841                 clear_bit(Replacement, &p->replacement->flags);
1842                 smp_mb(); /* Make sure other CPUs may see both as identical
1843                            * but will never see neither -- if they are careful.
1844                            */
1845                 p->replacement = NULL;
1846                 clear_bit(WantReplacement, &rdev->flags);
1847         } else
1848                 /* We might have just remove the Replacement as faulty
1849                  * Clear the flag just in case
1850                  */
1851                 clear_bit(WantReplacement, &rdev->flags);
1852
1853         err = md_integrity_register(mddev);
1854
1855 abort:
1856
1857         print_conf(conf);
1858         return err;
1859 }
1860
1861
1862 static void end_sync_read(struct bio *bio, int error)
1863 {
1864         struct r10bio *r10_bio = bio->bi_private;
1865         struct r10conf *conf = r10_bio->mddev->private;
1866         int d;
1867
1868         if (bio == r10_bio->master_bio) {
1869                 /* this is a reshape read */
1870                 d = r10_bio->read_slot; /* really the read dev */
1871         } else
1872                 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1873
1874         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1875                 set_bit(R10BIO_Uptodate, &r10_bio->state);
1876         else
1877                 /* The write handler will notice the lack of
1878                  * R10BIO_Uptodate and record any errors etc
1879                  */
1880                 atomic_add(r10_bio->sectors,
1881                            &conf->mirrors[d].rdev->corrected_errors);
1882
1883         /* for reconstruct, we always reschedule after a read.
1884          * for resync, only after all reads
1885          */
1886         rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1887         if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1888             atomic_dec_and_test(&r10_bio->remaining)) {
1889                 /* we have read all the blocks,
1890                  * do the comparison in process context in raid10d
1891                  */
1892                 reschedule_retry(r10_bio);
1893         }
1894 }
1895
1896 static void end_sync_request(struct r10bio *r10_bio)
1897 {
1898         struct mddev *mddev = r10_bio->mddev;
1899
1900         while (atomic_dec_and_test(&r10_bio->remaining)) {
1901                 if (r10_bio->master_bio == NULL) {
1902                         /* the primary of several recovery bios */
1903                         sector_t s = r10_bio->sectors;
1904                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1905                             test_bit(R10BIO_WriteError, &r10_bio->state))
1906                                 reschedule_retry(r10_bio);
1907                         else
1908                                 put_buf(r10_bio);
1909                         md_done_sync(mddev, s, 1);
1910                         break;
1911                 } else {
1912                         struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1913                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1914                             test_bit(R10BIO_WriteError, &r10_bio->state))
1915                                 reschedule_retry(r10_bio);
1916                         else
1917                                 put_buf(r10_bio);
1918                         r10_bio = r10_bio2;
1919                 }
1920         }
1921 }
1922
1923 static void end_sync_write(struct bio *bio, int error)
1924 {
1925         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1926         struct r10bio *r10_bio = bio->bi_private;
1927         struct mddev *mddev = r10_bio->mddev;
1928         struct r10conf *conf = mddev->private;
1929         int d;
1930         sector_t first_bad;
1931         int bad_sectors;
1932         int slot;
1933         int repl;
1934         struct md_rdev *rdev = NULL;
1935
1936         d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
1937         if (repl)
1938                 rdev = conf->mirrors[d].replacement;
1939         else
1940                 rdev = conf->mirrors[d].rdev;
1941
1942         if (!uptodate) {
1943                 if (repl)
1944                         md_error(mddev, rdev);
1945                 else {
1946                         set_bit(WriteErrorSeen, &rdev->flags);
1947                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
1948                                 set_bit(MD_RECOVERY_NEEDED,
1949                                         &rdev->mddev->recovery);
1950                         set_bit(R10BIO_WriteError, &r10_bio->state);
1951                 }
1952         } else if (is_badblock(rdev,
1953                              r10_bio->devs[slot].addr,
1954                              r10_bio->sectors,
1955                              &first_bad, &bad_sectors))
1956                 set_bit(R10BIO_MadeGood, &r10_bio->state);
1957
1958         rdev_dec_pending(rdev, mddev);
1959
1960         end_sync_request(r10_bio);
1961 }
1962
1963 /*
1964  * Note: sync and recover and handled very differently for raid10
1965  * This code is for resync.
1966  * For resync, we read through virtual addresses and read all blocks.
1967  * If there is any error, we schedule a write.  The lowest numbered
1968  * drive is authoritative.
1969  * However requests come for physical address, so we need to map.
1970  * For every physical address there are raid_disks/copies virtual addresses,
1971  * which is always are least one, but is not necessarly an integer.
1972  * This means that a physical address can span multiple chunks, so we may
1973  * have to submit multiple io requests for a single sync request.
1974  */
1975 /*
1976  * We check if all blocks are in-sync and only write to blocks that
1977  * aren't in sync
1978  */
1979 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
1980 {
1981         struct r10conf *conf = mddev->private;
1982         int i, first;
1983         struct bio *tbio, *fbio;
1984         int vcnt;
1985
1986         atomic_set(&r10_bio->remaining, 1);
1987
1988         /* find the first device with a block */
1989         for (i=0; i<conf->copies; i++)
1990                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1991                         break;
1992
1993         if (i == conf->copies)
1994                 goto done;
1995
1996         first = i;
1997         fbio = r10_bio->devs[i].bio;
1998
1999         vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2000         /* now find blocks with errors */
2001         for (i=0 ; i < conf->copies ; i++) {
2002                 int  j, d;
2003
2004                 tbio = r10_bio->devs[i].bio;
2005
2006                 if (tbio->bi_end_io != end_sync_read)
2007                         continue;
2008                 if (i == first)
2009                         continue;
2010                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
2011                         /* We know that the bi_io_vec layout is the same for
2012                          * both 'first' and 'i', so we just compare them.
2013                          * All vec entries are PAGE_SIZE;
2014                          */
2015                         for (j = 0; j < vcnt; j++)
2016                                 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
2017                                            page_address(tbio->bi_io_vec[j].bv_page),
2018                                            fbio->bi_io_vec[j].bv_len))
2019                                         break;
2020                         if (j == vcnt)
2021                                 continue;
2022                         atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2023                         if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2024                                 /* Don't fix anything. */
2025                                 continue;
2026                 }
2027                 /* Ok, we need to write this bio, either to correct an
2028                  * inconsistency or to correct an unreadable block.
2029                  * First we need to fixup bv_offset, bv_len and
2030                  * bi_vecs, as the read request might have corrupted these
2031                  */
2032                 tbio->bi_vcnt = vcnt;
2033                 tbio->bi_size = r10_bio->sectors << 9;
2034                 tbio->bi_idx = 0;
2035                 tbio->bi_phys_segments = 0;
2036                 tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
2037                 tbio->bi_flags |= 1 << BIO_UPTODATE;
2038                 tbio->bi_next = NULL;
2039                 tbio->bi_rw = WRITE;
2040                 tbio->bi_private = r10_bio;
2041                 tbio->bi_sector = r10_bio->devs[i].addr;
2042
2043                 for (j=0; j < vcnt ; j++) {
2044                         tbio->bi_io_vec[j].bv_offset = 0;
2045                         tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
2046
2047                         memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2048                                page_address(fbio->bi_io_vec[j].bv_page),
2049                                PAGE_SIZE);
2050                 }
2051                 tbio->bi_end_io = end_sync_write;
2052
2053                 d = r10_bio->devs[i].devnum;
2054                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2055                 atomic_inc(&r10_bio->remaining);
2056                 md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
2057
2058                 tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
2059                 tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
2060                 generic_make_request(tbio);
2061         }
2062
2063         /* Now write out to any replacement devices
2064          * that are active
2065          */
2066         for (i = 0; i < conf->copies; i++) {
2067                 int j, d;
2068
2069                 tbio = r10_bio->devs[i].repl_bio;
2070                 if (!tbio || !tbio->bi_end_io)
2071                         continue;
2072                 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2073                     && r10_bio->devs[i].bio != fbio)
2074                         for (j = 0; j < vcnt; j++)
2075                                 memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2076                                        page_address(fbio->bi_io_vec[j].bv_page),
2077                                        PAGE_SIZE);
2078                 d = r10_bio->devs[i].devnum;
2079                 atomic_inc(&r10_bio->remaining);
2080                 md_sync_acct(conf->mirrors[d].replacement->bdev,
2081                              tbio->bi_size >> 9);
2082                 generic_make_request(tbio);
2083         }
2084
2085 done:
2086         if (atomic_dec_and_test(&r10_bio->remaining)) {
2087                 md_done_sync(mddev, r10_bio->sectors, 1);
2088                 put_buf(r10_bio);
2089         }
2090 }
2091
2092 /*
2093  * Now for the recovery code.
2094  * Recovery happens across physical sectors.
2095  * We recover all non-is_sync drives by finding the virtual address of
2096  * each, and then choose a working drive that also has that virt address.
2097  * There is a separate r10_bio for each non-in_sync drive.
2098  * Only the first two slots are in use. The first for reading,
2099  * The second for writing.
2100  *
2101  */
2102 static void fix_recovery_read_error(struct r10bio *r10_bio)
2103 {
2104         /* We got a read error during recovery.
2105          * We repeat the read in smaller page-sized sections.
2106          * If a read succeeds, write it to the new device or record
2107          * a bad block if we cannot.
2108          * If a read fails, record a bad block on both old and
2109          * new devices.
2110          */
2111         struct mddev *mddev = r10_bio->mddev;
2112         struct r10conf *conf = mddev->private;
2113         struct bio *bio = r10_bio->devs[0].bio;
2114         sector_t sect = 0;
2115         int sectors = r10_bio->sectors;
2116         int idx = 0;
2117         int dr = r10_bio->devs[0].devnum;
2118         int dw = r10_bio->devs[1].devnum;
2119
2120         while (sectors) {
2121                 int s = sectors;
2122                 struct md_rdev *rdev;
2123                 sector_t addr;
2124                 int ok;
2125
2126                 if (s > (PAGE_SIZE>>9))
2127                         s = PAGE_SIZE >> 9;
2128
2129                 rdev = conf->mirrors[dr].rdev;
2130                 addr = r10_bio->devs[0].addr + sect,
2131                 ok = sync_page_io(rdev,
2132                                   addr,
2133                                   s << 9,
2134                                   bio->bi_io_vec[idx].bv_page,
2135                                   READ, false);
2136                 if (ok) {
2137                         rdev = conf->mirrors[dw].rdev;
2138                         addr = r10_bio->devs[1].addr + sect;
2139                         ok = sync_page_io(rdev,
2140                                           addr,
2141                                           s << 9,
2142                                           bio->bi_io_vec[idx].bv_page,
2143                                           WRITE, false);
2144                         if (!ok) {
2145                                 set_bit(WriteErrorSeen, &rdev->flags);
2146                                 if (!test_and_set_bit(WantReplacement,
2147                                                       &rdev->flags))
2148                                         set_bit(MD_RECOVERY_NEEDED,
2149                                                 &rdev->mddev->recovery);
2150                         }
2151                 }
2152                 if (!ok) {
2153                         /* We don't worry if we cannot set a bad block -
2154                          * it really is bad so there is no loss in not
2155                          * recording it yet
2156                          */
2157                         rdev_set_badblocks(rdev, addr, s, 0);
2158
2159                         if (rdev != conf->mirrors[dw].rdev) {
2160                                 /* need bad block on destination too */
2161                                 struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2162                                 addr = r10_bio->devs[1].addr + sect;
2163                                 ok = rdev_set_badblocks(rdev2, addr, s, 0);
2164                                 if (!ok) {
2165                                         /* just abort the recovery */
2166                                         printk(KERN_NOTICE
2167                                                "md/raid10:%s: recovery aborted"
2168                                                " due to read error\n",
2169                                                mdname(mddev));
2170
2171                                         conf->mirrors[dw].recovery_disabled
2172                                                 = mddev->recovery_disabled;
2173                                         set_bit(MD_RECOVERY_INTR,
2174                                                 &mddev->recovery);
2175                                         break;
2176                                 }
2177                         }
2178                 }
2179
2180                 sectors -= s;
2181                 sect += s;
2182                 idx++;
2183         }
2184 }
2185
2186 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2187 {
2188         struct r10conf *conf = mddev->private;
2189         int d;
2190         struct bio *wbio, *wbio2;
2191
2192         if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2193                 fix_recovery_read_error(r10_bio);
2194                 end_sync_request(r10_bio);
2195                 return;
2196         }
2197
2198         /*
2199          * share the pages with the first bio
2200          * and submit the write request
2201          */
2202         d = r10_bio->devs[1].devnum;
2203         wbio = r10_bio->devs[1].bio;
2204         wbio2 = r10_bio->devs[1].repl_bio;
2205         if (wbio->bi_end_io) {
2206                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2207                 md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
2208                 generic_make_request(wbio);
2209         }
2210         if (wbio2 && wbio2->bi_end_io) {
2211                 atomic_inc(&conf->mirrors[d].replacement->nr_pending);
2212                 md_sync_acct(conf->mirrors[d].replacement->bdev,
2213                              wbio2->bi_size >> 9);
2214                 generic_make_request(wbio2);
2215         }
2216 }
2217
2218
2219 /*
2220  * Used by fix_read_error() to decay the per rdev read_errors.
2221  * We halve the read error count for every hour that has elapsed
2222  * since the last recorded read error.
2223  *
2224  */
2225 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2226 {
2227         struct timespec cur_time_mon;
2228         unsigned long hours_since_last;
2229         unsigned int read_errors = atomic_read(&rdev->read_errors);
2230
2231         ktime_get_ts(&cur_time_mon);
2232
2233         if (rdev->last_read_error.tv_sec == 0 &&
2234             rdev->last_read_error.tv_nsec == 0) {
2235                 /* first time we've seen a read error */
2236                 rdev->last_read_error = cur_time_mon;
2237                 return;
2238         }
2239
2240         hours_since_last = (cur_time_mon.tv_sec -
2241                             rdev->last_read_error.tv_sec) / 3600;
2242
2243         rdev->last_read_error = cur_time_mon;
2244
2245         /*
2246          * if hours_since_last is > the number of bits in read_errors
2247          * just set read errors to 0. We do this to avoid
2248          * overflowing the shift of read_errors by hours_since_last.
2249          */
2250         if (hours_since_last >= 8 * sizeof(read_errors))
2251                 atomic_set(&rdev->read_errors, 0);
2252         else
2253                 atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
2254 }
2255
2256 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2257                             int sectors, struct page *page, int rw)
2258 {
2259         sector_t first_bad;
2260         int bad_sectors;
2261
2262         if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
2263             && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
2264                 return -1;
2265         if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
2266                 /* success */
2267                 return 1;
2268         if (rw == WRITE) {
2269                 set_bit(WriteErrorSeen, &rdev->flags);
2270                 if (!test_and_set_bit(WantReplacement, &rdev->flags))
2271                         set_bit(MD_RECOVERY_NEEDED,
2272                                 &rdev->mddev->recovery);
2273         }
2274         /* need to record an error - either for the block or the device */
2275         if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2276                 md_error(rdev->mddev, rdev);
2277         return 0;
2278 }
2279
2280 /*
2281  * This is a kernel thread which:
2282  *
2283  *      1.      Retries failed read operations on working mirrors.
2284  *      2.      Updates the raid superblock when problems encounter.
2285  *      3.      Performs writes following reads for array synchronising.
2286  */
2287
2288 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2289 {
2290         int sect = 0; /* Offset from r10_bio->sector */
2291         int sectors = r10_bio->sectors;
2292         struct md_rdev*rdev;
2293         int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2294         int d = r10_bio->devs[r10_bio->read_slot].devnum;
2295
2296         /* still own a reference to this rdev, so it cannot
2297          * have been cleared recently.
2298          */
2299         rdev = conf->mirrors[d].rdev;
2300
2301         if (test_bit(Faulty, &rdev->flags))
2302                 /* drive has already been failed, just ignore any
2303                    more fix_read_error() attempts */
2304                 return;
2305
2306         check_decay_read_errors(mddev, rdev);
2307         atomic_inc(&rdev->read_errors);
2308         if (atomic_read(&rdev->read_errors) > max_read_errors) {
2309                 char b[BDEVNAME_SIZE];
2310                 bdevname(rdev->bdev, b);
2311
2312                 printk(KERN_NOTICE
2313                        "md/raid10:%s: %s: Raid device exceeded "
2314                        "read_error threshold [cur %d:max %d]\n",
2315                        mdname(mddev), b,
2316                        atomic_read(&rdev->read_errors), max_read_errors);
2317                 printk(KERN_NOTICE
2318                        "md/raid10:%s: %s: Failing raid device\n",
2319                        mdname(mddev), b);
2320                 md_error(mddev, conf->mirrors[d].rdev);
2321                 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2322                 return;
2323         }
2324
2325         while(sectors) {
2326                 int s = sectors;
2327                 int sl = r10_bio->read_slot;
2328                 int success = 0;
2329                 int start;
2330
2331                 if (s > (PAGE_SIZE>>9))
2332                         s = PAGE_SIZE >> 9;
2333
2334                 rcu_read_lock();
2335                 do {
2336                         sector_t first_bad;
2337                         int bad_sectors;
2338
2339                         d = r10_bio->devs[sl].devnum;
2340                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2341                         if (rdev &&
2342                             !test_bit(Unmerged, &rdev->flags) &&
2343                             test_bit(In_sync, &rdev->flags) &&
2344                             is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2345                                         &first_bad, &bad_sectors) == 0) {
2346                                 atomic_inc(&rdev->nr_pending);
2347                                 rcu_read_unlock();
2348                                 success = sync_page_io(rdev,
2349                                                        r10_bio->devs[sl].addr +
2350                                                        sect,
2351                                                        s<<9,
2352                                                        conf->tmppage, READ, false);
2353                                 rdev_dec_pending(rdev, mddev);
2354                                 rcu_read_lock();
2355                                 if (success)
2356                                         break;
2357                         }
2358                         sl++;
2359                         if (sl == conf->copies)
2360                                 sl = 0;
2361                 } while (!success && sl != r10_bio->read_slot);
2362                 rcu_read_unlock();
2363
2364                 if (!success) {
2365                         /* Cannot read from anywhere, just mark the block
2366                          * as bad on the first device to discourage future
2367                          * reads.
2368                          */
2369                         int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2370                         rdev = conf->mirrors[dn].rdev;
2371
2372                         if (!rdev_set_badblocks(
2373                                     rdev,
2374                                     r10_bio->devs[r10_bio->read_slot].addr
2375                                     + sect,
2376                                     s, 0)) {
2377                                 md_error(mddev, rdev);
2378                                 r10_bio->devs[r10_bio->read_slot].bio
2379                                         = IO_BLOCKED;
2380                         }
2381                         break;
2382                 }
2383
2384                 start = sl;
2385                 /* write it back and re-read */
2386                 rcu_read_lock();
2387                 while (sl != r10_bio->read_slot) {
2388                         char b[BDEVNAME_SIZE];
2389
2390                         if (sl==0)
2391                                 sl = conf->copies;
2392                         sl--;
2393                         d = r10_bio->devs[sl].devnum;
2394                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2395                         if (!rdev ||
2396                             test_bit(Unmerged, &rdev->flags) ||
2397                             !test_bit(In_sync, &rdev->flags))
2398                                 continue;
2399
2400                         atomic_inc(&rdev->nr_pending);
2401                         rcu_read_unlock();
2402                         if (r10_sync_page_io(rdev,
2403                                              r10_bio->devs[sl].addr +
2404                                              sect,
2405                                              s, conf->tmppage, WRITE)
2406                             == 0) {
2407                                 /* Well, this device is dead */
2408                                 printk(KERN_NOTICE
2409                                        "md/raid10:%s: read correction "
2410                                        "write failed"
2411                                        " (%d sectors at %llu on %s)\n",
2412                                        mdname(mddev), s,
2413                                        (unsigned long long)(
2414                                                sect +
2415                                                choose_data_offset(r10_bio,
2416                                                                   rdev)),
2417                                        bdevname(rdev->bdev, b));
2418                                 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2419                                        "drive\n",
2420                                        mdname(mddev),
2421                                        bdevname(rdev->bdev, b));
2422                         }
2423                         rdev_dec_pending(rdev, mddev);
2424                         rcu_read_lock();
2425                 }
2426                 sl = start;
2427                 while (sl != r10_bio->read_slot) {
2428                         char b[BDEVNAME_SIZE];
2429
2430                         if (sl==0)
2431                                 sl = conf->copies;
2432                         sl--;
2433                         d = r10_bio->devs[sl].devnum;
2434                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2435                         if (!rdev ||
2436                             !test_bit(In_sync, &rdev->flags))
2437                                 continue;
2438
2439                         atomic_inc(&rdev->nr_pending);
2440                         rcu_read_unlock();
2441                         switch (r10_sync_page_io(rdev,
2442                                              r10_bio->devs[sl].addr +
2443                                              sect,
2444                                              s, conf->tmppage,
2445                                                  READ)) {
2446                         case 0:
2447                                 /* Well, this device is dead */
2448                                 printk(KERN_NOTICE
2449                                        "md/raid10:%s: unable to read back "
2450                                        "corrected sectors"
2451                                        " (%d sectors at %llu on %s)\n",
2452                                        mdname(mddev), s,
2453                                        (unsigned long long)(
2454                                                sect +
2455                                                choose_data_offset(r10_bio, rdev)),
2456                                        bdevname(rdev->bdev, b));
2457                                 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2458                                        "drive\n",
2459                                        mdname(mddev),
2460                                        bdevname(rdev->bdev, b));
2461                                 break;
2462                         case 1:
2463                                 printk(KERN_INFO
2464                                        "md/raid10:%s: read error corrected"
2465                                        " (%d sectors at %llu on %s)\n",
2466                                        mdname(mddev), s,
2467                                        (unsigned long long)(
2468                                                sect +
2469                                                choose_data_offset(r10_bio, rdev)),
2470                                        bdevname(rdev->bdev, b));
2471                                 atomic_add(s, &rdev->corrected_errors);
2472                         }
2473
2474                         rdev_dec_pending(rdev, mddev);
2475                         rcu_read_lock();
2476                 }
2477                 rcu_read_unlock();
2478
2479                 sectors -= s;
2480                 sect += s;
2481         }
2482 }
2483
2484 static void bi_complete(struct bio *bio, int error)
2485 {
2486         complete((struct completion *)bio->bi_private);
2487 }
2488
2489 static int submit_bio_wait(int rw, struct bio *bio)
2490 {
2491         struct completion event;
2492         rw |= REQ_SYNC;
2493
2494         init_completion(&event);
2495         bio->bi_private = &event;
2496         bio->bi_end_io = bi_complete;
2497         submit_bio(rw, bio);
2498         wait_for_completion(&event);
2499
2500         return test_bit(BIO_UPTODATE, &bio->bi_flags);
2501 }
2502
2503 static int narrow_write_error(struct r10bio *r10_bio, int i)
2504 {
2505         struct bio *bio = r10_bio->master_bio;
2506         struct mddev *mddev = r10_bio->mddev;
2507         struct r10conf *conf = mddev->private;
2508         struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2509         /* bio has the data to be written to slot 'i' where
2510          * we just recently had a write error.
2511          * We repeatedly clone the bio and trim down to one block,
2512          * then try the write.  Where the write fails we record
2513          * a bad block.
2514          * It is conceivable that the bio doesn't exactly align with
2515          * blocks.  We must handle this.
2516          *
2517          * We currently own a reference to the rdev.
2518          */
2519
2520         int block_sectors;
2521         sector_t sector;
2522         int sectors;
2523         int sect_to_write = r10_bio->sectors;
2524         int ok = 1;
2525
2526         if (rdev->badblocks.shift < 0)
2527                 return 0;
2528
2529         block_sectors = 1 << rdev->badblocks.shift;
2530         sector = r10_bio->sector;
2531         sectors = ((r10_bio->sector + block_sectors)
2532                    & ~(sector_t)(block_sectors - 1))
2533                 - sector;
2534
2535         while (sect_to_write) {
2536                 struct bio *wbio;
2537                 if (sectors > sect_to_write)
2538                         sectors = sect_to_write;
2539                 /* Write at 'sector' for 'sectors' */
2540                 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2541                 md_trim_bio(wbio, sector - bio->bi_sector, sectors);
2542                 wbio->bi_sector = (r10_bio->devs[i].addr+
2543                                    choose_data_offset(r10_bio, rdev) +
2544                                    (sector - r10_bio->sector));
2545                 wbio->bi_bdev = rdev->bdev;
2546                 if (submit_bio_wait(WRITE, wbio) == 0)
2547                         /* Failure! */
2548                         ok = rdev_set_badblocks(rdev, sector,
2549                                                 sectors, 0)
2550                                 && ok;
2551
2552                 bio_put(wbio);
2553                 sect_to_write -= sectors;
2554                 sector += sectors;
2555                 sectors = block_sectors;
2556         }
2557         return ok;
2558 }
2559
2560 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2561 {
2562         int slot = r10_bio->read_slot;
2563         struct bio *bio;
2564         struct r10conf *conf = mddev->private;
2565         struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2566         char b[BDEVNAME_SIZE];
2567         unsigned long do_sync;
2568         int max_sectors;
2569
2570         /* we got a read error. Maybe the drive is bad.  Maybe just
2571          * the block and we can fix it.
2572          * We freeze all other IO, and try reading the block from
2573          * other devices.  When we find one, we re-write
2574          * and check it that fixes the read error.
2575          * This is all done synchronously while the array is
2576          * frozen.
2577          */
2578         bio = r10_bio->devs[slot].bio;
2579         bdevname(bio->bi_bdev, b);
2580         bio_put(bio);
2581         r10_bio->devs[slot].bio = NULL;
2582
2583         if (mddev->ro == 0) {
2584                 freeze_array(conf);
2585                 fix_read_error(conf, mddev, r10_bio);
2586                 unfreeze_array(conf);
2587         } else
2588                 r10_bio->devs[slot].bio = IO_BLOCKED;
2589
2590         rdev_dec_pending(rdev, mddev);
2591
2592 read_more:
2593         rdev = read_balance(conf, r10_bio, &max_sectors);
2594         if (rdev == NULL) {
2595                 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
2596                        " read error for block %llu\n",
2597                        mdname(mddev), b,
2598                        (unsigned long long)r10_bio->sector);
2599                 raid_end_bio_io(r10_bio);
2600                 return;
2601         }
2602
2603         do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
2604         slot = r10_bio->read_slot;
2605         printk_ratelimited(
2606                 KERN_ERR
2607                 "md/raid10:%s: %s: redirecting "
2608                 "sector %llu to another mirror\n",
2609                 mdname(mddev),
2610                 bdevname(rdev->bdev, b),
2611                 (unsigned long long)r10_bio->sector);
2612         bio = bio_clone_mddev(r10_bio->master_bio,
2613                               GFP_NOIO, mddev);
2614         md_trim_bio(bio,
2615                     r10_bio->sector - bio->bi_sector,
2616                     max_sectors);
2617         r10_bio->devs[slot].bio = bio;
2618         r10_bio->devs[slot].rdev = rdev;
2619         bio->bi_sector = r10_bio->devs[slot].addr
2620                 + choose_data_offset(r10_bio, rdev);
2621         bio->bi_bdev = rdev->bdev;
2622         bio->bi_rw = READ | do_sync;
2623         bio->bi_private = r10_bio;
2624         bio->bi_end_io = raid10_end_read_request;
2625         if (max_sectors < r10_bio->sectors) {
2626                 /* Drat - have to split this up more */
2627                 struct bio *mbio = r10_bio->master_bio;
2628                 int sectors_handled =
2629                         r10_bio->sector + max_sectors
2630                         - mbio->bi_sector;
2631                 r10_bio->sectors = max_sectors;
2632                 spin_lock_irq(&conf->device_lock);
2633                 if (mbio->bi_phys_segments == 0)
2634                         mbio->bi_phys_segments = 2;
2635                 else
2636                         mbio->bi_phys_segments++;
2637                 spin_unlock_irq(&conf->device_lock);
2638                 generic_make_request(bio);
2639
2640                 r10_bio = mempool_alloc(conf->r10bio_pool,
2641                                         GFP_NOIO);
2642                 r10_bio->master_bio = mbio;
2643                 r10_bio->sectors = (mbio->bi_size >> 9)
2644                         - sectors_handled;
2645                 r10_bio->state = 0;
2646                 set_bit(R10BIO_ReadError,
2647                         &r10_bio->state);
2648                 r10_bio->mddev = mddev;
2649                 r10_bio->sector = mbio->bi_sector
2650                         + sectors_handled;
2651
2652                 goto read_more;
2653         } else
2654                 generic_make_request(bio);
2655 }
2656
2657 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2658 {
2659         /* Some sort of write request has finished and it
2660          * succeeded in writing where we thought there was a
2661          * bad block.  So forget the bad block.
2662          * Or possibly if failed and we need to record
2663          * a bad block.
2664          */
2665         int m;
2666         struct md_rdev *rdev;
2667
2668         if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2669             test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2670                 for (m = 0; m < conf->copies; m++) {
2671                         int dev = r10_bio->devs[m].devnum;
2672                         rdev = conf->mirrors[dev].rdev;
2673                         if (r10_bio->devs[m].bio == NULL)
2674                                 continue;
2675                         if (test_bit(BIO_UPTODATE,
2676                                      &r10_bio->devs[m].bio->bi_flags)) {
2677                                 rdev_clear_badblocks(
2678                                         rdev,
2679                                         r10_bio->devs[m].addr,
2680                                         r10_bio->sectors, 0);
2681                         } else {
2682                                 if (!rdev_set_badblocks(
2683                                             rdev,
2684                                             r10_bio->devs[m].addr,
2685                                             r10_bio->sectors, 0))
2686                                         md_error(conf->mddev, rdev);
2687                         }
2688                         rdev = conf->mirrors[dev].replacement;
2689                         if (r10_bio->devs[m].repl_bio == NULL)
2690                                 continue;
2691                         if (test_bit(BIO_UPTODATE,
2692                                      &r10_bio->devs[m].repl_bio->bi_flags)) {
2693                                 rdev_clear_badblocks(
2694                                         rdev,
2695                                         r10_bio->devs[m].addr,
2696                                         r10_bio->sectors, 0);
2697                         } else {
2698                                 if (!rdev_set_badblocks(
2699                                             rdev,
2700                                             r10_bio->devs[m].addr,
2701                                             r10_bio->sectors, 0))
2702                                         md_error(conf->mddev, rdev);
2703                         }
2704                 }
2705                 put_buf(r10_bio);
2706         } else {
2707                 for (m = 0; m < conf->copies; m++) {
2708                         int dev = r10_bio->devs[m].devnum;
2709                         struct bio *bio = r10_bio->devs[m].bio;
2710                         rdev = conf->mirrors[dev].rdev;
2711                         if (bio == IO_MADE_GOOD) {
2712                                 rdev_clear_badblocks(
2713                                         rdev,
2714                                         r10_bio->devs[m].addr,
2715                                         r10_bio->sectors, 0);
2716                                 rdev_dec_pending(rdev, conf->mddev);
2717                         } else if (bio != NULL &&
2718                                    !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2719                                 if (!narrow_write_error(r10_bio, m)) {
2720                                         md_error(conf->mddev, rdev);
2721                                         set_bit(R10BIO_Degraded,
2722                                                 &r10_bio->state);
2723                                 }
2724                                 rdev_dec_pending(rdev, conf->mddev);
2725                         }
2726                         bio = r10_bio->devs[m].repl_bio;
2727                         rdev = conf->mirrors[dev].replacement;
2728                         if (rdev && bio == IO_MADE_GOOD) {
2729                                 rdev_clear_badblocks(
2730                                         rdev,
2731                                         r10_bio->devs[m].addr,
2732                                         r10_bio->sectors, 0);
2733                                 rdev_dec_pending(rdev, conf->mddev);
2734                         }
2735                 }
2736                 if (test_bit(R10BIO_WriteError,
2737                              &r10_bio->state))
2738                         close_write(r10_bio);
2739                 raid_end_bio_io(r10_bio);
2740         }
2741 }
2742
2743 static void raid10d(struct md_thread *thread)
2744 {
2745         struct mddev *mddev = thread->mddev;
2746         struct r10bio *r10_bio;
2747         unsigned long flags;
2748         struct r10conf *conf = mddev->private;
2749         struct list_head *head = &conf->retry_list;
2750         struct blk_plug plug;
2751
2752         md_check_recovery(mddev);
2753
2754         blk_start_plug(&plug);
2755         for (;;) {
2756
2757                 flush_pending_writes(conf);
2758
2759                 spin_lock_irqsave(&conf->device_lock, flags);
2760                 if (list_empty(head)) {
2761                         spin_unlock_irqrestore(&conf->device_lock, flags);
2762                         break;
2763                 }
2764                 r10_bio = list_entry(head->prev, struct r10bio, retry_list);
2765                 list_del(head->prev);
2766                 conf->nr_queued--;
2767                 spin_unlock_irqrestore(&conf->device_lock, flags);
2768
2769                 mddev = r10_bio->mddev;
2770                 conf = mddev->private;
2771                 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2772                     test_bit(R10BIO_WriteError, &r10_bio->state))
2773                         handle_write_completed(conf, r10_bio);
2774                 else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
2775                         reshape_request_write(mddev, r10_bio);
2776                 else if (test_bit(R10BIO_IsSync, &r10_bio->state))
2777                         sync_request_write(mddev, r10_bio);
2778                 else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
2779                         recovery_request_write(mddev, r10_bio);
2780                 else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2781                         handle_read_error(mddev, r10_bio);
2782                 else {
2783                         /* just a partial read to be scheduled from a
2784                          * separate context
2785                          */
2786                         int slot = r10_bio->read_slot;
2787                         generic_make_request(r10_bio->devs[slot].bio);
2788                 }
2789
2790                 cond_resched();
2791                 if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
2792                         md_check_recovery(mddev);
2793         }
2794         blk_finish_plug(&plug);
2795 }
2796
2797
2798 static int init_resync(struct r10conf *conf)
2799 {
2800         int buffs;
2801         int i;
2802
2803         buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2804         BUG_ON(conf->r10buf_pool);
2805         conf->have_replacement = 0;
2806         for (i = 0; i < conf->geo.raid_disks; i++)
2807                 if (conf->mirrors[i].replacement)
2808                         conf->have_replacement = 1;
2809         conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
2810         if (!conf->r10buf_pool)
2811                 return -ENOMEM;
2812         conf->next_resync = 0;
2813         return 0;
2814 }
2815
2816 /*
2817  * perform a "sync" on one "block"
2818  *
2819  * We need to make sure that no normal I/O request - particularly write
2820  * requests - conflict with active sync requests.
2821  *
2822  * This is achieved by tracking pending requests and a 'barrier' concept
2823  * that can be installed to exclude normal IO requests.
2824  *
2825  * Resync and recovery are handled very differently.
2826  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
2827  *
2828  * For resync, we iterate over virtual addresses, read all copies,
2829  * and update if there are differences.  If only one copy is live,
2830  * skip it.
2831  * For recovery, we iterate over physical addresses, read a good
2832  * value for each non-in_sync drive, and over-write.
2833  *
2834  * So, for recovery we may have several outstanding complex requests for a
2835  * given address, one for each out-of-sync device.  We model this by allocating
2836  * a number of r10_bio structures, one for each out-of-sync device.
2837  * As we setup these structures, we collect all bio's together into a list
2838  * which we then process collectively to add pages, and then process again
2839  * to pass to generic_make_request.
2840  *
2841  * The r10_bio structures are linked using a borrowed master_bio pointer.
2842  * This link is counted in ->remaining.  When the r10_bio that points to NULL
2843  * has its remaining count decremented to 0, the whole complex operation
2844  * is complete.
2845  *
2846  */
2847
2848 static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
2849                              int *skipped, int go_faster)
2850 {
2851         struct r10conf *conf = mddev->private;
2852         struct r10bio *r10_bio;
2853         struct bio *biolist = NULL, *bio;
2854         sector_t max_sector, nr_sectors;
2855         int i;
2856         int max_sync;
2857         sector_t sync_blocks;
2858         sector_t sectors_skipped = 0;
2859         int chunks_skipped = 0;
2860         sector_t chunk_mask = conf->geo.chunk_mask;
2861
2862         if (!conf->r10buf_pool)
2863                 if (init_resync(conf))
2864                         return 0;
2865
2866  skipped:
2867         max_sector = mddev->dev_sectors;
2868         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
2869             test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2870                 max_sector = mddev->resync_max_sectors;
2871         if (sector_nr >= max_sector) {
2872                 /* If we aborted, we need to abort the
2873                  * sync on the 'current' bitmap chucks (there can
2874                  * be several when recovering multiple devices).
2875                  * as we may have started syncing it but not finished.
2876                  * We can find the current address in
2877                  * mddev->curr_resync, but for recovery,
2878                  * we need to convert that to several
2879                  * virtual addresses.
2880                  */
2881                 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
2882                         end_reshape(conf);
2883                         return 0;
2884                 }
2885
2886                 if (mddev->curr_resync < max_sector) { /* aborted */
2887                         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2888                                 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2889                                                 &sync_blocks, 1);
2890                         else for (i = 0; i < conf->geo.raid_disks; i++) {
2891                                 sector_t sect =
2892                                         raid10_find_virt(conf, mddev->curr_resync, i);
2893                                 bitmap_end_sync(mddev->bitmap, sect,
2894                                                 &sync_blocks, 1);
2895                         }
2896                 } else {
2897                         /* completed sync */
2898                         if ((!mddev->bitmap || conf->fullsync)
2899                             && conf->have_replacement
2900                             && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2901                                 /* Completed a full sync so the replacements
2902                                  * are now fully recovered.
2903                                  */
2904                                 for (i = 0; i < conf->geo.raid_disks; i++)
2905                                         if (conf->mirrors[i].replacement)
2906                                                 conf->mirrors[i].replacement
2907                                                         ->recovery_offset
2908                                                         = MaxSector;
2909                         }
2910                         conf->fullsync = 0;
2911                 }
2912                 bitmap_close_sync(mddev->bitmap);
2913                 close_sync(conf);
2914                 *skipped = 1;
2915                 return sectors_skipped;
2916         }
2917
2918         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2919                 return reshape_request(mddev, sector_nr, skipped);
2920
2921         if (chunks_skipped >= conf->geo.raid_disks) {
2922                 /* if there has been nothing to do on any drive,
2923                  * then there is nothing to do at all..
2924                  */
2925                 *skipped = 1;
2926                 return (max_sector - sector_nr) + sectors_skipped;
2927         }
2928
2929         if (max_sector > mddev->resync_max)
2930                 max_sector = mddev->resync_max; /* Don't do IO beyond here */
2931
2932         /* make sure whole request will fit in a chunk - if chunks
2933          * are meaningful
2934          */
2935         if (conf->geo.near_copies < conf->geo.raid_disks &&
2936             max_sector > (sector_nr | chunk_mask))
2937                 max_sector = (sector_nr | chunk_mask) + 1;
2938         /*
2939          * If there is non-resync activity waiting for us then
2940          * put in a delay to throttle resync.
2941          */
2942         if (!go_faster && conf->nr_waiting)
2943                 msleep_interruptible(1000);
2944
2945         /* Again, very different code for resync and recovery.
2946          * Both must result in an r10bio with a list of bios that
2947          * have bi_end_io, bi_sector, bi_bdev set,
2948          * and bi_private set to the r10bio.
2949          * For recovery, we may actually create several r10bios
2950          * with 2 bios in each, that correspond to the bios in the main one.
2951          * In this case, the subordinate r10bios link back through a
2952          * borrowed master_bio pointer, and the counter in the master
2953          * includes a ref from each subordinate.
2954          */
2955         /* First, we decide what to do and set ->bi_end_io
2956          * To end_sync_read if we want to read, and
2957          * end_sync_write if we will want to write.
2958          */
2959
2960         max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
2961         if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2962                 /* recovery... the complicated one */
2963                 int j;
2964                 r10_bio = NULL;
2965
2966                 for (i = 0 ; i < conf->geo.raid_disks; i++) {
2967                         int still_degraded;
2968                         struct r10bio *rb2;
2969                         sector_t sect;
2970                         int must_sync;
2971                         int any_working;
2972                         struct raid10_info *mirror = &conf->mirrors[i];
2973
2974                         if ((mirror->rdev == NULL ||
2975                              test_bit(In_sync, &mirror->rdev->flags))
2976                             &&
2977                             (mirror->replacement == NULL ||
2978                              test_bit(Faulty,
2979                                       &mirror->replacement->flags)))
2980                                 continue;
2981
2982                         still_degraded = 0;
2983                         /* want to reconstruct this device */
2984                         rb2 = r10_bio;
2985                         sect = raid10_find_virt(conf, sector_nr, i);
2986                         if (sect >= mddev->resync_max_sectors) {
2987                                 /* last stripe is not complete - don't
2988                                  * try to recover this sector.
2989                                  */
2990                                 continue;
2991                         }
2992                         /* Unless we are doing a full sync, or a replacement
2993                          * we only need to recover the block if it is set in
2994                          * the bitmap
2995                          */
2996                         must_sync = bitmap_start_sync(mddev->bitmap, sect,
2997                                                       &sync_blocks, 1);
2998                         if (sync_blocks < max_sync)
2999                                 max_sync = sync_blocks;
3000                         if (!must_sync &&
3001                             mirror->replacement == NULL &&
3002                             !conf->fullsync) {
3003                                 /* yep, skip the sync_blocks here, but don't assume
3004                                  * that there will never be anything to do here
3005                                  */
3006                                 chunks_skipped = -1;
3007                                 continue;
3008                         }
3009
3010                         r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3011                         raise_barrier(conf, rb2 != NULL);
3012                         atomic_set(&r10_bio->remaining, 0);
3013
3014                         r10_bio->master_bio = (struct bio*)rb2;
3015                         if (rb2)
3016                                 atomic_inc(&rb2->remaining);
3017                         r10_bio->mddev = mddev;
3018                         set_bit(R10BIO_IsRecover, &r10_bio->state);
3019                         r10_bio->sector = sect;
3020
3021                         raid10_find_phys(conf, r10_bio);
3022
3023                         /* Need to check if the array will still be
3024                          * degraded
3025                          */
3026                         for (j = 0; j < conf->geo.raid_disks; j++)
3027                                 if (conf->mirrors[j].rdev == NULL ||
3028                                     test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
3029                                         still_degraded = 1;
3030                                         break;
3031                                 }
3032
3033                         must_sync = bitmap_start_sync(mddev->bitmap, sect,
3034                                                       &sync_blocks, still_degraded);
3035
3036                         any_working = 0;
3037                         for (j=0; j<conf->copies;j++) {
3038                                 int k;
3039                                 int d = r10_bio->devs[j].devnum;
3040                                 sector_t from_addr, to_addr;
3041                                 struct md_rdev *rdev;
3042                                 sector_t sector, first_bad;
3043                                 int bad_sectors;
3044                                 if (!conf->mirrors[d].rdev ||
3045                                     !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
3046                                         continue;
3047                                 /* This is where we read from */
3048                                 any_working = 1;
3049                                 rdev = conf->mirrors[d].rdev;
3050                                 sector = r10_bio->devs[j].addr;
3051
3052                                 if (is_badblock(rdev, sector, max_sync,
3053                                                 &first_bad, &bad_sectors)) {
3054                                         if (first_bad > sector)
3055                                                 max_sync = first_bad - sector;
3056                                         else {
3057                                                 bad_sectors -= (sector
3058                                                                 - first_bad);
3059                                                 if (max_sync > bad_sectors)
3060                                                         max_sync = bad_sectors;
3061                                                 continue;
3062                                         }
3063                                 }
3064                                 bio = r10_bio->devs[0].bio;
3065                                 bio->bi_next = biolist;
3066