]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - drivers/md/raid10.c
d1295aff41739eea048f0e43513dfba6ade4c8f1
[~shefty/rdma-dev.git] / drivers / md / raid10.c
1 /*
2  * raid10.c : Multiple Devices driver for Linux
3  *
4  * Copyright (C) 2000-2004 Neil Brown
5  *
6  * RAID-10 support for md.
7  *
8  * Base on code in raid1.c.  See raid1.c for further copyright information.
9  *
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/delay.h>
23 #include <linux/blkdev.h>
24 #include <linux/module.h>
25 #include <linux/seq_file.h>
26 #include <linux/ratelimit.h>
27 #include <linux/kthread.h>
28 #include "md.h"
29 #include "raid10.h"
30 #include "raid0.h"
31 #include "bitmap.h"
32
33 /*
34  * RAID10 provides a combination of RAID0 and RAID1 functionality.
35  * The layout of data is defined by
36  *    chunk_size
37  *    raid_disks
38  *    near_copies (stored in low byte of layout)
39  *    far_copies (stored in second byte of layout)
40  *    far_offset (stored in bit 16 of layout )
41  *
42  * The data to be stored is divided into chunks using chunksize.
43  * Each device is divided into far_copies sections.
44  * In each section, chunks are laid out in a style similar to raid0, but
45  * near_copies copies of each chunk is stored (each on a different drive).
46  * The starting device for each section is offset near_copies from the starting
47  * device of the previous section.
48  * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
49  * drive.
50  * near_copies and far_copies must be at least one, and their product is at most
51  * raid_disks.
52  *
53  * If far_offset is true, then the far_copies are handled a bit differently.
54  * The copies are still in different stripes, but instead of be very far apart
55  * on disk, there are adjacent stripes.
56  */
57
58 /*
59  * Number of guaranteed r10bios in case of extreme VM load:
60  */
61 #define NR_RAID10_BIOS 256
62
63 /* when we get a read error on a read-only array, we redirect to another
64  * device without failing the first device, or trying to over-write to
65  * correct the read error.  To keep track of bad blocks on a per-bio
66  * level, we store IO_BLOCKED in the appropriate 'bios' pointer
67  */
68 #define IO_BLOCKED ((struct bio *)1)
69 /* When we successfully write to a known bad-block, we need to remove the
70  * bad-block marking which must be done from process context.  So we record
71  * the success by setting devs[n].bio to IO_MADE_GOOD
72  */
73 #define IO_MADE_GOOD ((struct bio *)2)
74
75 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
76
77 /* When there are this many requests queued to be written by
78  * the raid10 thread, we become 'congested' to provide back-pressure
79  * for writeback.
80  */
81 static int max_queued_requests = 1024;
82
83 static void allow_barrier(struct r10conf *conf);
84 static void lower_barrier(struct r10conf *conf);
85 static int enough(struct r10conf *conf, int ignore);
86 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
87                                 int *skipped);
88 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
89 static void end_reshape_write(struct bio *bio, int error);
90 static void end_reshape(struct r10conf *conf);
91
92 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
93 {
94         struct r10conf *conf = data;
95         int size = offsetof(struct r10bio, devs[conf->copies]);
96
97         /* allocate a r10bio with room for raid_disks entries in the
98          * bios array */
99         return kzalloc(size, gfp_flags);
100 }
101
102 static void r10bio_pool_free(void *r10_bio, void *data)
103 {
104         kfree(r10_bio);
105 }
106
107 /* Maximum size of each resync request */
108 #define RESYNC_BLOCK_SIZE (64*1024)
109 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
110 /* amount of memory to reserve for resync requests */
111 #define RESYNC_WINDOW (1024*1024)
112 /* maximum number of concurrent requests, memory permitting */
113 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
114
115 /*
116  * When performing a resync, we need to read and compare, so
117  * we need as many pages are there are copies.
118  * When performing a recovery, we need 2 bios, one for read,
119  * one for write (we recover only one drive per r10buf)
120  *
121  */
122 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
123 {
124         struct r10conf *conf = data;
125         struct page *page;
126         struct r10bio *r10_bio;
127         struct bio *bio;
128         int i, j;
129         int nalloc;
130
131         r10_bio = r10bio_pool_alloc(gfp_flags, conf);
132         if (!r10_bio)
133                 return NULL;
134
135         if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
136             test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
137                 nalloc = conf->copies; /* resync */
138         else
139                 nalloc = 2; /* recovery */
140
141         /*
142          * Allocate bios.
143          */
144         for (j = nalloc ; j-- ; ) {
145                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
146                 if (!bio)
147                         goto out_free_bio;
148                 r10_bio->devs[j].bio = bio;
149                 if (!conf->have_replacement)
150                         continue;
151                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
152                 if (!bio)
153                         goto out_free_bio;
154                 r10_bio->devs[j].repl_bio = bio;
155         }
156         /*
157          * Allocate RESYNC_PAGES data pages and attach them
158          * where needed.
159          */
160         for (j = 0 ; j < nalloc; j++) {
161                 struct bio *rbio = r10_bio->devs[j].repl_bio;
162                 bio = r10_bio->devs[j].bio;
163                 for (i = 0; i < RESYNC_PAGES; i++) {
164                         if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
165                                                &conf->mddev->recovery)) {
166                                 /* we can share bv_page's during recovery
167                                  * and reshape */
168                                 struct bio *rbio = r10_bio->devs[0].bio;
169                                 page = rbio->bi_io_vec[i].bv_page;
170                                 get_page(page);
171                         } else
172                                 page = alloc_page(gfp_flags);
173                         if (unlikely(!page))
174                                 goto out_free_pages;
175
176                         bio->bi_io_vec[i].bv_page = page;
177                         if (rbio)
178                                 rbio->bi_io_vec[i].bv_page = page;
179                 }
180         }
181
182         return r10_bio;
183
184 out_free_pages:
185         for ( ; i > 0 ; i--)
186                 safe_put_page(bio->bi_io_vec[i-1].bv_page);
187         while (j--)
188                 for (i = 0; i < RESYNC_PAGES ; i++)
189                         safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
190         j = 0;
191 out_free_bio:
192         for ( ; j < nalloc; j++) {
193                 if (r10_bio->devs[j].bio)
194                         bio_put(r10_bio->devs[j].bio);
195                 if (r10_bio->devs[j].repl_bio)
196                         bio_put(r10_bio->devs[j].repl_bio);
197         }
198         r10bio_pool_free(r10_bio, conf);
199         return NULL;
200 }
201
202 static void r10buf_pool_free(void *__r10_bio, void *data)
203 {
204         int i;
205         struct r10conf *conf = data;
206         struct r10bio *r10bio = __r10_bio;
207         int j;
208
209         for (j=0; j < conf->copies; j++) {
210                 struct bio *bio = r10bio->devs[j].bio;
211                 if (bio) {
212                         for (i = 0; i < RESYNC_PAGES; i++) {
213                                 safe_put_page(bio->bi_io_vec[i].bv_page);
214                                 bio->bi_io_vec[i].bv_page = NULL;
215                         }
216                         bio_put(bio);
217                 }
218                 bio = r10bio->devs[j].repl_bio;
219                 if (bio)
220                         bio_put(bio);
221         }
222         r10bio_pool_free(r10bio, conf);
223 }
224
225 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
226 {
227         int i;
228
229         for (i = 0; i < conf->copies; i++) {
230                 struct bio **bio = & r10_bio->devs[i].bio;
231                 if (!BIO_SPECIAL(*bio))
232                         bio_put(*bio);
233                 *bio = NULL;
234                 bio = &r10_bio->devs[i].repl_bio;
235                 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
236                         bio_put(*bio);
237                 *bio = NULL;
238         }
239 }
240
241 static void free_r10bio(struct r10bio *r10_bio)
242 {
243         struct r10conf *conf = r10_bio->mddev->private;
244
245         put_all_bios(conf, r10_bio);
246         mempool_free(r10_bio, conf->r10bio_pool);
247 }
248
249 static void put_buf(struct r10bio *r10_bio)
250 {
251         struct r10conf *conf = r10_bio->mddev->private;
252
253         mempool_free(r10_bio, conf->r10buf_pool);
254
255         lower_barrier(conf);
256 }
257
258 static void reschedule_retry(struct r10bio *r10_bio)
259 {
260         unsigned long flags;
261         struct mddev *mddev = r10_bio->mddev;
262         struct r10conf *conf = mddev->private;
263
264         spin_lock_irqsave(&conf->device_lock, flags);
265         list_add(&r10_bio->retry_list, &conf->retry_list);
266         conf->nr_queued ++;
267         spin_unlock_irqrestore(&conf->device_lock, flags);
268
269         /* wake up frozen array... */
270         wake_up(&conf->wait_barrier);
271
272         md_wakeup_thread(mddev->thread);
273 }
274
275 /*
276  * raid_end_bio_io() is called when we have finished servicing a mirrored
277  * operation and are ready to return a success/failure code to the buffer
278  * cache layer.
279  */
280 static void raid_end_bio_io(struct r10bio *r10_bio)
281 {
282         struct bio *bio = r10_bio->master_bio;
283         int done;
284         struct r10conf *conf = r10_bio->mddev->private;
285
286         if (bio->bi_phys_segments) {
287                 unsigned long flags;
288                 spin_lock_irqsave(&conf->device_lock, flags);
289                 bio->bi_phys_segments--;
290                 done = (bio->bi_phys_segments == 0);
291                 spin_unlock_irqrestore(&conf->device_lock, flags);
292         } else
293                 done = 1;
294         if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
295                 clear_bit(BIO_UPTODATE, &bio->bi_flags);
296         if (done) {
297                 bio_endio(bio, 0);
298                 /*
299                  * Wake up any possible resync thread that waits for the device
300                  * to go idle.
301                  */
302                 allow_barrier(conf);
303         }
304         free_r10bio(r10_bio);
305 }
306
307 /*
308  * Update disk head position estimator based on IRQ completion info.
309  */
310 static inline void update_head_pos(int slot, struct r10bio *r10_bio)
311 {
312         struct r10conf *conf = r10_bio->mddev->private;
313
314         conf->mirrors[r10_bio->devs[slot].devnum].head_position =
315                 r10_bio->devs[slot].addr + (r10_bio->sectors);
316 }
317
318 /*
319  * Find the disk number which triggered given bio
320  */
321 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
322                          struct bio *bio, int *slotp, int *replp)
323 {
324         int slot;
325         int repl = 0;
326
327         for (slot = 0; slot < conf->copies; slot++) {
328                 if (r10_bio->devs[slot].bio == bio)
329                         break;
330                 if (r10_bio->devs[slot].repl_bio == bio) {
331                         repl = 1;
332                         break;
333                 }
334         }
335
336         BUG_ON(slot == conf->copies);
337         update_head_pos(slot, r10_bio);
338
339         if (slotp)
340                 *slotp = slot;
341         if (replp)
342                 *replp = repl;
343         return r10_bio->devs[slot].devnum;
344 }
345
346 static void raid10_end_read_request(struct bio *bio, int error)
347 {
348         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
349         struct r10bio *r10_bio = bio->bi_private;
350         int slot, dev;
351         struct md_rdev *rdev;
352         struct r10conf *conf = r10_bio->mddev->private;
353
354
355         slot = r10_bio->read_slot;
356         dev = r10_bio->devs[slot].devnum;
357         rdev = r10_bio->devs[slot].rdev;
358         /*
359          * this branch is our 'one mirror IO has finished' event handler:
360          */
361         update_head_pos(slot, r10_bio);
362
363         if (uptodate) {
364                 /*
365                  * Set R10BIO_Uptodate in our master bio, so that
366                  * we will return a good error code to the higher
367                  * levels even if IO on some other mirrored buffer fails.
368                  *
369                  * The 'master' represents the composite IO operation to
370                  * user-side. So if something waits for IO, then it will
371                  * wait for the 'master' bio.
372                  */
373                 set_bit(R10BIO_Uptodate, &r10_bio->state);
374         } else {
375                 /* If all other devices that store this block have
376                  * failed, we want to return the error upwards rather
377                  * than fail the last device.  Here we redefine
378                  * "uptodate" to mean "Don't want to retry"
379                  */
380                 unsigned long flags;
381                 spin_lock_irqsave(&conf->device_lock, flags);
382                 if (!enough(conf, rdev->raid_disk))
383                         uptodate = 1;
384                 spin_unlock_irqrestore(&conf->device_lock, flags);
385         }
386         if (uptodate) {
387                 raid_end_bio_io(r10_bio);
388                 rdev_dec_pending(rdev, conf->mddev);
389         } else {
390                 /*
391                  * oops, read error - keep the refcount on the rdev
392                  */
393                 char b[BDEVNAME_SIZE];
394                 printk_ratelimited(KERN_ERR
395                                    "md/raid10:%s: %s: rescheduling sector %llu\n",
396                                    mdname(conf->mddev),
397                                    bdevname(rdev->bdev, b),
398                                    (unsigned long long)r10_bio->sector);
399                 set_bit(R10BIO_ReadError, &r10_bio->state);
400                 reschedule_retry(r10_bio);
401         }
402 }
403
404 static void close_write(struct r10bio *r10_bio)
405 {
406         /* clear the bitmap if all writes complete successfully */
407         bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
408                         r10_bio->sectors,
409                         !test_bit(R10BIO_Degraded, &r10_bio->state),
410                         0);
411         md_write_end(r10_bio->mddev);
412 }
413
414 static void one_write_done(struct r10bio *r10_bio)
415 {
416         if (atomic_dec_and_test(&r10_bio->remaining)) {
417                 if (test_bit(R10BIO_WriteError, &r10_bio->state))
418                         reschedule_retry(r10_bio);
419                 else {
420                         close_write(r10_bio);
421                         if (test_bit(R10BIO_MadeGood, &r10_bio->state))
422                                 reschedule_retry(r10_bio);
423                         else
424                                 raid_end_bio_io(r10_bio);
425                 }
426         }
427 }
428
429 static void raid10_end_write_request(struct bio *bio, int error)
430 {
431         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
432         struct r10bio *r10_bio = bio->bi_private;
433         int dev;
434         int dec_rdev = 1;
435         struct r10conf *conf = r10_bio->mddev->private;
436         int slot, repl;
437         struct md_rdev *rdev = NULL;
438
439         dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
440
441         if (repl)
442                 rdev = conf->mirrors[dev].replacement;
443         if (!rdev) {
444                 smp_rmb();
445                 repl = 0;
446                 rdev = conf->mirrors[dev].rdev;
447         }
448         /*
449          * this branch is our 'one mirror IO has finished' event handler:
450          */
451         if (!uptodate) {
452                 if (repl)
453                         /* Never record new bad blocks to replacement,
454                          * just fail it.
455                          */
456                         md_error(rdev->mddev, rdev);
457                 else {
458                         set_bit(WriteErrorSeen, &rdev->flags);
459                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
460                                 set_bit(MD_RECOVERY_NEEDED,
461                                         &rdev->mddev->recovery);
462                         set_bit(R10BIO_WriteError, &r10_bio->state);
463                         dec_rdev = 0;
464                 }
465         } else {
466                 /*
467                  * Set R10BIO_Uptodate in our master bio, so that
468                  * we will return a good error code for to the higher
469                  * levels even if IO on some other mirrored buffer fails.
470                  *
471                  * The 'master' represents the composite IO operation to
472                  * user-side. So if something waits for IO, then it will
473                  * wait for the 'master' bio.
474                  */
475                 sector_t first_bad;
476                 int bad_sectors;
477
478                 set_bit(R10BIO_Uptodate, &r10_bio->state);
479
480                 /* Maybe we can clear some bad blocks. */
481                 if (is_badblock(rdev,
482                                 r10_bio->devs[slot].addr,
483                                 r10_bio->sectors,
484                                 &first_bad, &bad_sectors)) {
485                         bio_put(bio);
486                         if (repl)
487                                 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
488                         else
489                                 r10_bio->devs[slot].bio = IO_MADE_GOOD;
490                         dec_rdev = 0;
491                         set_bit(R10BIO_MadeGood, &r10_bio->state);
492                 }
493         }
494
495         /*
496          *
497          * Let's see if all mirrored write operations have finished
498          * already.
499          */
500         one_write_done(r10_bio);
501         if (dec_rdev)
502                 rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
503 }
504
505 /*
506  * RAID10 layout manager
507  * As well as the chunksize and raid_disks count, there are two
508  * parameters: near_copies and far_copies.
509  * near_copies * far_copies must be <= raid_disks.
510  * Normally one of these will be 1.
511  * If both are 1, we get raid0.
512  * If near_copies == raid_disks, we get raid1.
513  *
514  * Chunks are laid out in raid0 style with near_copies copies of the
515  * first chunk, followed by near_copies copies of the next chunk and
516  * so on.
517  * If far_copies > 1, then after 1/far_copies of the array has been assigned
518  * as described above, we start again with a device offset of near_copies.
519  * So we effectively have another copy of the whole array further down all
520  * the drives, but with blocks on different drives.
521  * With this layout, and block is never stored twice on the one device.
522  *
523  * raid10_find_phys finds the sector offset of a given virtual sector
524  * on each device that it is on.
525  *
526  * raid10_find_virt does the reverse mapping, from a device and a
527  * sector offset to a virtual address
528  */
529
530 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
531 {
532         int n,f;
533         sector_t sector;
534         sector_t chunk;
535         sector_t stripe;
536         int dev;
537         int slot = 0;
538
539         /* now calculate first sector/dev */
540         chunk = r10bio->sector >> geo->chunk_shift;
541         sector = r10bio->sector & geo->chunk_mask;
542
543         chunk *= geo->near_copies;
544         stripe = chunk;
545         dev = sector_div(stripe, geo->raid_disks);
546         if (geo->far_offset)
547                 stripe *= geo->far_copies;
548
549         sector += stripe << geo->chunk_shift;
550
551         /* and calculate all the others */
552         for (n = 0; n < geo->near_copies; n++) {
553                 int d = dev;
554                 sector_t s = sector;
555                 r10bio->devs[slot].addr = sector;
556                 r10bio->devs[slot].devnum = d;
557                 slot++;
558
559                 for (f = 1; f < geo->far_copies; f++) {
560                         d += geo->near_copies;
561                         if (d >= geo->raid_disks)
562                                 d -= geo->raid_disks;
563                         s += geo->stride;
564                         r10bio->devs[slot].devnum = d;
565                         r10bio->devs[slot].addr = s;
566                         slot++;
567                 }
568                 dev++;
569                 if (dev >= geo->raid_disks) {
570                         dev = 0;
571                         sector += (geo->chunk_mask + 1);
572                 }
573         }
574 }
575
576 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
577 {
578         struct geom *geo = &conf->geo;
579
580         if (conf->reshape_progress != MaxSector &&
581             ((r10bio->sector >= conf->reshape_progress) !=
582              conf->mddev->reshape_backwards)) {
583                 set_bit(R10BIO_Previous, &r10bio->state);
584                 geo = &conf->prev;
585         } else
586                 clear_bit(R10BIO_Previous, &r10bio->state);
587
588         __raid10_find_phys(geo, r10bio);
589 }
590
591 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
592 {
593         sector_t offset, chunk, vchunk;
594         /* Never use conf->prev as this is only called during resync
595          * or recovery, so reshape isn't happening
596          */
597         struct geom *geo = &conf->geo;
598
599         offset = sector & geo->chunk_mask;
600         if (geo->far_offset) {
601                 int fc;
602                 chunk = sector >> geo->chunk_shift;
603                 fc = sector_div(chunk, geo->far_copies);
604                 dev -= fc * geo->near_copies;
605                 if (dev < 0)
606                         dev += geo->raid_disks;
607         } else {
608                 while (sector >= geo->stride) {
609                         sector -= geo->stride;
610                         if (dev < geo->near_copies)
611                                 dev += geo->raid_disks - geo->near_copies;
612                         else
613                                 dev -= geo->near_copies;
614                 }
615                 chunk = sector >> geo->chunk_shift;
616         }
617         vchunk = chunk * geo->raid_disks + dev;
618         sector_div(vchunk, geo->near_copies);
619         return (vchunk << geo->chunk_shift) + offset;
620 }
621
622 /**
623  *      raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
624  *      @q: request queue
625  *      @bvm: properties of new bio
626  *      @biovec: the request that could be merged to it.
627  *
628  *      Return amount of bytes we can accept at this offset
629  *      This requires checking for end-of-chunk if near_copies != raid_disks,
630  *      and for subordinate merge_bvec_fns if merge_check_needed.
631  */
632 static int raid10_mergeable_bvec(struct request_queue *q,
633                                  struct bvec_merge_data *bvm,
634                                  struct bio_vec *biovec)
635 {
636         struct mddev *mddev = q->queuedata;
637         struct r10conf *conf = mddev->private;
638         sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
639         int max;
640         unsigned int chunk_sectors;
641         unsigned int bio_sectors = bvm->bi_size >> 9;
642         struct geom *geo = &conf->geo;
643
644         chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
645         if (conf->reshape_progress != MaxSector &&
646             ((sector >= conf->reshape_progress) !=
647              conf->mddev->reshape_backwards))
648                 geo = &conf->prev;
649
650         if (geo->near_copies < geo->raid_disks) {
651                 max = (chunk_sectors - ((sector & (chunk_sectors - 1))
652                                         + bio_sectors)) << 9;
653                 if (max < 0)
654                         /* bio_add cannot handle a negative return */
655                         max = 0;
656                 if (max <= biovec->bv_len && bio_sectors == 0)
657                         return biovec->bv_len;
658         } else
659                 max = biovec->bv_len;
660
661         if (mddev->merge_check_needed) {
662                 struct {
663                         struct r10bio r10_bio;
664                         struct r10dev devs[conf->copies];
665                 } on_stack;
666                 struct r10bio *r10_bio = &on_stack.r10_bio;
667                 int s;
668                 if (conf->reshape_progress != MaxSector) {
669                         /* Cannot give any guidance during reshape */
670                         if (max <= biovec->bv_len && bio_sectors == 0)
671                                 return biovec->bv_len;
672                         return 0;
673                 }
674                 r10_bio->sector = sector;
675                 raid10_find_phys(conf, r10_bio);
676                 rcu_read_lock();
677                 for (s = 0; s < conf->copies; s++) {
678                         int disk = r10_bio->devs[s].devnum;
679                         struct md_rdev *rdev = rcu_dereference(
680                                 conf->mirrors[disk].rdev);
681                         if (rdev && !test_bit(Faulty, &rdev->flags)) {
682                                 struct request_queue *q =
683                                         bdev_get_queue(rdev->bdev);
684                                 if (q->merge_bvec_fn) {
685                                         bvm->bi_sector = r10_bio->devs[s].addr
686                                                 + rdev->data_offset;
687                                         bvm->bi_bdev = rdev->bdev;
688                                         max = min(max, q->merge_bvec_fn(
689                                                           q, bvm, biovec));
690                                 }
691                         }
692                         rdev = rcu_dereference(conf->mirrors[disk].replacement);
693                         if (rdev && !test_bit(Faulty, &rdev->flags)) {
694                                 struct request_queue *q =
695                                         bdev_get_queue(rdev->bdev);
696                                 if (q->merge_bvec_fn) {
697                                         bvm->bi_sector = r10_bio->devs[s].addr
698                                                 + rdev->data_offset;
699                                         bvm->bi_bdev = rdev->bdev;
700                                         max = min(max, q->merge_bvec_fn(
701                                                           q, bvm, biovec));
702                                 }
703                         }
704                 }
705                 rcu_read_unlock();
706         }
707         return max;
708 }
709
710 /*
711  * This routine returns the disk from which the requested read should
712  * be done. There is a per-array 'next expected sequential IO' sector
713  * number - if this matches on the next IO then we use the last disk.
714  * There is also a per-disk 'last know head position' sector that is
715  * maintained from IRQ contexts, both the normal and the resync IO
716  * completion handlers update this position correctly. If there is no
717  * perfect sequential match then we pick the disk whose head is closest.
718  *
719  * If there are 2 mirrors in the same 2 devices, performance degrades
720  * because position is mirror, not device based.
721  *
722  * The rdev for the device selected will have nr_pending incremented.
723  */
724
725 /*
726  * FIXME: possibly should rethink readbalancing and do it differently
727  * depending on near_copies / far_copies geometry.
728  */
729 static struct md_rdev *read_balance(struct r10conf *conf,
730                                     struct r10bio *r10_bio,
731                                     int *max_sectors)
732 {
733         const sector_t this_sector = r10_bio->sector;
734         int disk, slot;
735         int sectors = r10_bio->sectors;
736         int best_good_sectors;
737         sector_t new_distance, best_dist;
738         struct md_rdev *best_rdev, *rdev = NULL;
739         int do_balance;
740         int best_slot;
741         struct geom *geo = &conf->geo;
742
743         raid10_find_phys(conf, r10_bio);
744         rcu_read_lock();
745 retry:
746         sectors = r10_bio->sectors;
747         best_slot = -1;
748         best_rdev = NULL;
749         best_dist = MaxSector;
750         best_good_sectors = 0;
751         do_balance = 1;
752         /*
753          * Check if we can balance. We can balance on the whole
754          * device if no resync is going on (recovery is ok), or below
755          * the resync window. We take the first readable disk when
756          * above the resync window.
757          */
758         if (conf->mddev->recovery_cp < MaxSector
759             && (this_sector + sectors >= conf->next_resync))
760                 do_balance = 0;
761
762         for (slot = 0; slot < conf->copies ; slot++) {
763                 sector_t first_bad;
764                 int bad_sectors;
765                 sector_t dev_sector;
766
767                 if (r10_bio->devs[slot].bio == IO_BLOCKED)
768                         continue;
769                 disk = r10_bio->devs[slot].devnum;
770                 rdev = rcu_dereference(conf->mirrors[disk].replacement);
771                 if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
772                     test_bit(Unmerged, &rdev->flags) ||
773                     r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
774                         rdev = rcu_dereference(conf->mirrors[disk].rdev);
775                 if (rdev == NULL ||
776                     test_bit(Faulty, &rdev->flags) ||
777                     test_bit(Unmerged, &rdev->flags))
778                         continue;
779                 if (!test_bit(In_sync, &rdev->flags) &&
780                     r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
781                         continue;
782
783                 dev_sector = r10_bio->devs[slot].addr;
784                 if (is_badblock(rdev, dev_sector, sectors,
785                                 &first_bad, &bad_sectors)) {
786                         if (best_dist < MaxSector)
787                                 /* Already have a better slot */
788                                 continue;
789                         if (first_bad <= dev_sector) {
790                                 /* Cannot read here.  If this is the
791                                  * 'primary' device, then we must not read
792                                  * beyond 'bad_sectors' from another device.
793                                  */
794                                 bad_sectors -= (dev_sector - first_bad);
795                                 if (!do_balance && sectors > bad_sectors)
796                                         sectors = bad_sectors;
797                                 if (best_good_sectors > sectors)
798                                         best_good_sectors = sectors;
799                         } else {
800                                 sector_t good_sectors =
801                                         first_bad - dev_sector;
802                                 if (good_sectors > best_good_sectors) {
803                                         best_good_sectors = good_sectors;
804                                         best_slot = slot;
805                                         best_rdev = rdev;
806                                 }
807                                 if (!do_balance)
808                                         /* Must read from here */
809                                         break;
810                         }
811                         continue;
812                 } else
813                         best_good_sectors = sectors;
814
815                 if (!do_balance)
816                         break;
817
818                 /* This optimisation is debatable, and completely destroys
819                  * sequential read speed for 'far copies' arrays.  So only
820                  * keep it for 'near' arrays, and review those later.
821                  */
822                 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
823                         break;
824
825                 /* for far > 1 always use the lowest address */
826                 if (geo->far_copies > 1)
827                         new_distance = r10_bio->devs[slot].addr;
828                 else
829                         new_distance = abs(r10_bio->devs[slot].addr -
830                                            conf->mirrors[disk].head_position);
831                 if (new_distance < best_dist) {
832                         best_dist = new_distance;
833                         best_slot = slot;
834                         best_rdev = rdev;
835                 }
836         }
837         if (slot >= conf->copies) {
838                 slot = best_slot;
839                 rdev = best_rdev;
840         }
841
842         if (slot >= 0) {
843                 atomic_inc(&rdev->nr_pending);
844                 if (test_bit(Faulty, &rdev->flags)) {
845                         /* Cannot risk returning a device that failed
846                          * before we inc'ed nr_pending
847                          */
848                         rdev_dec_pending(rdev, conf->mddev);
849                         goto retry;
850                 }
851                 r10_bio->read_slot = slot;
852         } else
853                 rdev = NULL;
854         rcu_read_unlock();
855         *max_sectors = best_good_sectors;
856
857         return rdev;
858 }
859
860 int md_raid10_congested(struct mddev *mddev, int bits)
861 {
862         struct r10conf *conf = mddev->private;
863         int i, ret = 0;
864
865         if ((bits & (1 << BDI_async_congested)) &&
866             conf->pending_count >= max_queued_requests)
867                 return 1;
868
869         rcu_read_lock();
870         for (i = 0;
871              (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
872                      && ret == 0;
873              i++) {
874                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
875                 if (rdev && !test_bit(Faulty, &rdev->flags)) {
876                         struct request_queue *q = bdev_get_queue(rdev->bdev);
877
878                         ret |= bdi_congested(&q->backing_dev_info, bits);
879                 }
880         }
881         rcu_read_unlock();
882         return ret;
883 }
884 EXPORT_SYMBOL_GPL(md_raid10_congested);
885
886 static int raid10_congested(void *data, int bits)
887 {
888         struct mddev *mddev = data;
889
890         return mddev_congested(mddev, bits) ||
891                 md_raid10_congested(mddev, bits);
892 }
893
894 static void flush_pending_writes(struct r10conf *conf)
895 {
896         /* Any writes that have been queued but are awaiting
897          * bitmap updates get flushed here.
898          */
899         spin_lock_irq(&conf->device_lock);
900
901         if (conf->pending_bio_list.head) {
902                 struct bio *bio;
903                 bio = bio_list_get(&conf->pending_bio_list);
904                 conf->pending_count = 0;
905                 spin_unlock_irq(&conf->device_lock);
906                 /* flush any pending bitmap writes to disk
907                  * before proceeding w/ I/O */
908                 bitmap_unplug(conf->mddev->bitmap);
909                 wake_up(&conf->wait_barrier);
910
911                 while (bio) { /* submit pending writes */
912                         struct bio *next = bio->bi_next;
913                         bio->bi_next = NULL;
914                         if (unlikely((bio->bi_rw & REQ_DISCARD) &&
915                             !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
916                                 /* Just ignore it */
917                                 bio_endio(bio, 0);
918                         else
919                                 generic_make_request(bio);
920                         bio = next;
921                 }
922         } else
923                 spin_unlock_irq(&conf->device_lock);
924 }
925
926 /* Barriers....
927  * Sometimes we need to suspend IO while we do something else,
928  * either some resync/recovery, or reconfigure the array.
929  * To do this we raise a 'barrier'.
930  * The 'barrier' is a counter that can be raised multiple times
931  * to count how many activities are happening which preclude
932  * normal IO.
933  * We can only raise the barrier if there is no pending IO.
934  * i.e. if nr_pending == 0.
935  * We choose only to raise the barrier if no-one is waiting for the
936  * barrier to go down.  This means that as soon as an IO request
937  * is ready, no other operations which require a barrier will start
938  * until the IO request has had a chance.
939  *
940  * So: regular IO calls 'wait_barrier'.  When that returns there
941  *    is no backgroup IO happening,  It must arrange to call
942  *    allow_barrier when it has finished its IO.
943  * backgroup IO calls must call raise_barrier.  Once that returns
944  *    there is no normal IO happeing.  It must arrange to call
945  *    lower_barrier when the particular background IO completes.
946  */
947
948 static void raise_barrier(struct r10conf *conf, int force)
949 {
950         BUG_ON(force && !conf->barrier);
951         spin_lock_irq(&conf->resync_lock);
952
953         /* Wait until no block IO is waiting (unless 'force') */
954         wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
955                             conf->resync_lock, );
956
957         /* block any new IO from starting */
958         conf->barrier++;
959
960         /* Now wait for all pending IO to complete */
961         wait_event_lock_irq(conf->wait_barrier,
962                             !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
963                             conf->resync_lock, );
964
965         spin_unlock_irq(&conf->resync_lock);
966 }
967
968 static void lower_barrier(struct r10conf *conf)
969 {
970         unsigned long flags;
971         spin_lock_irqsave(&conf->resync_lock, flags);
972         conf->barrier--;
973         spin_unlock_irqrestore(&conf->resync_lock, flags);
974         wake_up(&conf->wait_barrier);
975 }
976
977 static void wait_barrier(struct r10conf *conf)
978 {
979         spin_lock_irq(&conf->resync_lock);
980         if (conf->barrier) {
981                 conf->nr_waiting++;
982                 /* Wait for the barrier to drop.
983                  * However if there are already pending
984                  * requests (preventing the barrier from
985                  * rising completely), and the
986                  * pre-process bio queue isn't empty,
987                  * then don't wait, as we need to empty
988                  * that queue to get the nr_pending
989                  * count down.
990                  */
991                 wait_event_lock_irq(conf->wait_barrier,
992                                     !conf->barrier ||
993                                     (conf->nr_pending &&
994                                      current->bio_list &&
995                                      !bio_list_empty(current->bio_list)),
996                                     conf->resync_lock,
997                         );
998                 conf->nr_waiting--;
999         }
1000         conf->nr_pending++;
1001         spin_unlock_irq(&conf->resync_lock);
1002 }
1003
1004 static void allow_barrier(struct r10conf *conf)
1005 {
1006         unsigned long flags;
1007         spin_lock_irqsave(&conf->resync_lock, flags);
1008         conf->nr_pending--;
1009         spin_unlock_irqrestore(&conf->resync_lock, flags);
1010         wake_up(&conf->wait_barrier);
1011 }
1012
1013 static void freeze_array(struct r10conf *conf)
1014 {
1015         /* stop syncio and normal IO and wait for everything to
1016          * go quiet.
1017          * We increment barrier and nr_waiting, and then
1018          * wait until nr_pending match nr_queued+1
1019          * This is called in the context of one normal IO request
1020          * that has failed. Thus any sync request that might be pending
1021          * will be blocked by nr_pending, and we need to wait for
1022          * pending IO requests to complete or be queued for re-try.
1023          * Thus the number queued (nr_queued) plus this request (1)
1024          * must match the number of pending IOs (nr_pending) before
1025          * we continue.
1026          */
1027         spin_lock_irq(&conf->resync_lock);
1028         conf->barrier++;
1029         conf->nr_waiting++;
1030         wait_event_lock_irq(conf->wait_barrier,
1031                             conf->nr_pending == conf->nr_queued+1,
1032                             conf->resync_lock,
1033                             flush_pending_writes(conf));
1034
1035         spin_unlock_irq(&conf->resync_lock);
1036 }
1037
1038 static void unfreeze_array(struct r10conf *conf)
1039 {
1040         /* reverse the effect of the freeze */
1041         spin_lock_irq(&conf->resync_lock);
1042         conf->barrier--;
1043         conf->nr_waiting--;
1044         wake_up(&conf->wait_barrier);
1045         spin_unlock_irq(&conf->resync_lock);
1046 }
1047
1048 static sector_t choose_data_offset(struct r10bio *r10_bio,
1049                                    struct md_rdev *rdev)
1050 {
1051         if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1052             test_bit(R10BIO_Previous, &r10_bio->state))
1053                 return rdev->data_offset;
1054         else
1055                 return rdev->new_data_offset;
1056 }
1057
1058 struct raid10_plug_cb {
1059         struct blk_plug_cb      cb;
1060         struct bio_list         pending;
1061         int                     pending_cnt;
1062 };
1063
1064 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1065 {
1066         struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
1067                                                    cb);
1068         struct mddev *mddev = plug->cb.data;
1069         struct r10conf *conf = mddev->private;
1070         struct bio *bio;
1071
1072         if (from_schedule) {
1073                 spin_lock_irq(&conf->device_lock);
1074                 bio_list_merge(&conf->pending_bio_list, &plug->pending);
1075                 conf->pending_count += plug->pending_cnt;
1076                 spin_unlock_irq(&conf->device_lock);
1077                 md_wakeup_thread(mddev->thread);
1078                 kfree(plug);
1079                 return;
1080         }
1081
1082         /* we aren't scheduling, so we can do the write-out directly. */
1083         bio = bio_list_get(&plug->pending);
1084         bitmap_unplug(mddev->bitmap);
1085         wake_up(&conf->wait_barrier);
1086
1087         while (bio) { /* submit pending writes */
1088                 struct bio *next = bio->bi_next;
1089                 bio->bi_next = NULL;
1090                 generic_make_request(bio);
1091                 bio = next;
1092         }
1093         kfree(plug);
1094 }
1095
1096 static void make_request(struct mddev *mddev, struct bio * bio)
1097 {
1098         struct r10conf *conf = mddev->private;
1099         struct r10bio *r10_bio;
1100         struct bio *read_bio;
1101         int i;
1102         sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1103         int chunk_sects = chunk_mask + 1;
1104         const int rw = bio_data_dir(bio);
1105         const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
1106         const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
1107         const unsigned long do_discard = (bio->bi_rw
1108                                           & (REQ_DISCARD | REQ_SECURE));
1109         unsigned long flags;
1110         struct md_rdev *blocked_rdev;
1111         struct blk_plug_cb *cb;
1112         struct raid10_plug_cb *plug = NULL;
1113         int sectors_handled;
1114         int max_sectors;
1115         int sectors;
1116
1117         if (unlikely(bio->bi_rw & REQ_FLUSH)) {
1118                 md_flush_request(mddev, bio);
1119                 return;
1120         }
1121
1122         /* If this request crosses a chunk boundary, we need to
1123          * split it.  This will only happen for 1 PAGE (or less) requests.
1124          */
1125         if (unlikely((bio->bi_sector & chunk_mask) + (bio->bi_size >> 9)
1126                      > chunk_sects
1127                      && (conf->geo.near_copies < conf->geo.raid_disks
1128                          || conf->prev.near_copies < conf->prev.raid_disks))) {
1129                 struct bio_pair *bp;
1130                 /* Sanity check -- queue functions should prevent this happening */
1131                 if ((bio->bi_vcnt != 1 && bio->bi_vcnt != 0) ||
1132                     bio->bi_idx != 0)
1133                         goto bad_map;
1134                 /* This is a one page bio that upper layers
1135                  * refuse to split for us, so we need to split it.
1136                  */
1137                 bp = bio_split(bio,
1138                                chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
1139
1140                 /* Each of these 'make_request' calls will call 'wait_barrier'.
1141                  * If the first succeeds but the second blocks due to the resync
1142                  * thread raising the barrier, we will deadlock because the
1143                  * IO to the underlying device will be queued in generic_make_request
1144                  * and will never complete, so will never reduce nr_pending.
1145                  * So increment nr_waiting here so no new raise_barriers will
1146                  * succeed, and so the second wait_barrier cannot block.
1147                  */
1148                 spin_lock_irq(&conf->resync_lock);
1149                 conf->nr_waiting++;
1150                 spin_unlock_irq(&conf->resync_lock);
1151
1152                 make_request(mddev, &bp->bio1);
1153                 make_request(mddev, &bp->bio2);
1154
1155                 spin_lock_irq(&conf->resync_lock);
1156                 conf->nr_waiting--;
1157                 wake_up(&conf->wait_barrier);
1158                 spin_unlock_irq(&conf->resync_lock);
1159
1160                 bio_pair_release(bp);
1161                 return;
1162         bad_map:
1163                 printk("md/raid10:%s: make_request bug: can't convert block across chunks"
1164                        " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
1165                        (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
1166
1167                 bio_io_error(bio);
1168                 return;
1169         }
1170
1171         md_write_start(mddev, bio);
1172
1173         /*
1174          * Register the new request and wait if the reconstruction
1175          * thread has put up a bar for new requests.
1176          * Continue immediately if no resync is active currently.
1177          */
1178         wait_barrier(conf);
1179
1180         sectors = bio->bi_size >> 9;
1181         while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1182             bio->bi_sector < conf->reshape_progress &&
1183             bio->bi_sector + sectors > conf->reshape_progress) {
1184                 /* IO spans the reshape position.  Need to wait for
1185                  * reshape to pass
1186                  */
1187                 allow_barrier(conf);
1188                 wait_event(conf->wait_barrier,
1189                            conf->reshape_progress <= bio->bi_sector ||
1190                            conf->reshape_progress >= bio->bi_sector + sectors);
1191                 wait_barrier(conf);
1192         }
1193         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1194             bio_data_dir(bio) == WRITE &&
1195             (mddev->reshape_backwards
1196              ? (bio->bi_sector < conf->reshape_safe &&
1197                 bio->bi_sector + sectors > conf->reshape_progress)
1198              : (bio->bi_sector + sectors > conf->reshape_safe &&
1199                 bio->bi_sector < conf->reshape_progress))) {
1200                 /* Need to update reshape_position in metadata */
1201                 mddev->reshape_position = conf->reshape_progress;
1202                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
1203                 set_bit(MD_CHANGE_PENDING, &mddev->flags);
1204                 md_wakeup_thread(mddev->thread);
1205                 wait_event(mddev->sb_wait,
1206                            !test_bit(MD_CHANGE_PENDING, &mddev->flags));
1207
1208                 conf->reshape_safe = mddev->reshape_position;
1209         }
1210
1211         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1212
1213         r10_bio->master_bio = bio;
1214         r10_bio->sectors = sectors;
1215
1216         r10_bio->mddev = mddev;
1217         r10_bio->sector = bio->bi_sector;
1218         r10_bio->state = 0;
1219
1220         /* We might need to issue multiple reads to different
1221          * devices if there are bad blocks around, so we keep
1222          * track of the number of reads in bio->bi_phys_segments.
1223          * If this is 0, there is only one r10_bio and no locking
1224          * will be needed when the request completes.  If it is
1225          * non-zero, then it is the number of not-completed requests.
1226          */
1227         bio->bi_phys_segments = 0;
1228         clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1229
1230         if (rw == READ) {
1231                 /*
1232                  * read balancing logic:
1233                  */
1234                 struct md_rdev *rdev;
1235                 int slot;
1236
1237 read_again:
1238                 rdev = read_balance(conf, r10_bio, &max_sectors);
1239                 if (!rdev) {
1240                         raid_end_bio_io(r10_bio);
1241                         return;
1242                 }
1243                 slot = r10_bio->read_slot;
1244
1245                 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1246                 md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector,
1247                             max_sectors);
1248
1249                 r10_bio->devs[slot].bio = read_bio;
1250                 r10_bio->devs[slot].rdev = rdev;
1251
1252                 read_bio->bi_sector = r10_bio->devs[slot].addr +
1253                         choose_data_offset(r10_bio, rdev);
1254                 read_bio->bi_bdev = rdev->bdev;
1255                 read_bio->bi_end_io = raid10_end_read_request;
1256                 read_bio->bi_rw = READ | do_sync;
1257                 read_bio->bi_private = r10_bio;
1258
1259                 if (max_sectors < r10_bio->sectors) {
1260                         /* Could not read all from this device, so we will
1261                          * need another r10_bio.
1262                          */
1263                         sectors_handled = (r10_bio->sectors + max_sectors
1264                                            - bio->bi_sector);
1265                         r10_bio->sectors = max_sectors;
1266                         spin_lock_irq(&conf->device_lock);
1267                         if (bio->bi_phys_segments == 0)
1268                                 bio->bi_phys_segments = 2;
1269                         else
1270                                 bio->bi_phys_segments++;
1271                         spin_unlock(&conf->device_lock);
1272                         /* Cannot call generic_make_request directly
1273                          * as that will be queued in __generic_make_request
1274                          * and subsequent mempool_alloc might block
1275                          * waiting for it.  so hand bio over to raid10d.
1276                          */
1277                         reschedule_retry(r10_bio);
1278
1279                         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1280
1281                         r10_bio->master_bio = bio;
1282                         r10_bio->sectors = ((bio->bi_size >> 9)
1283                                             - sectors_handled);
1284                         r10_bio->state = 0;
1285                         r10_bio->mddev = mddev;
1286                         r10_bio->sector = bio->bi_sector + sectors_handled;
1287                         goto read_again;
1288                 } else
1289                         generic_make_request(read_bio);
1290                 return;
1291         }
1292
1293         /*
1294          * WRITE:
1295          */
1296         if (conf->pending_count >= max_queued_requests) {
1297                 md_wakeup_thread(mddev->thread);
1298                 wait_event(conf->wait_barrier,
1299                            conf->pending_count < max_queued_requests);
1300         }
1301         /* first select target devices under rcu_lock and
1302          * inc refcount on their rdev.  Record them by setting
1303          * bios[x] to bio
1304          * If there are known/acknowledged bad blocks on any device
1305          * on which we have seen a write error, we want to avoid
1306          * writing to those blocks.  This potentially requires several
1307          * writes to write around the bad blocks.  Each set of writes
1308          * gets its own r10_bio with a set of bios attached.  The number
1309          * of r10_bios is recored in bio->bi_phys_segments just as with
1310          * the read case.
1311          */
1312
1313         r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1314         raid10_find_phys(conf, r10_bio);
1315 retry_write:
1316         blocked_rdev = NULL;
1317         rcu_read_lock();
1318         max_sectors = r10_bio->sectors;
1319
1320         for (i = 0;  i < conf->copies; i++) {
1321                 int d = r10_bio->devs[i].devnum;
1322                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1323                 struct md_rdev *rrdev = rcu_dereference(
1324                         conf->mirrors[d].replacement);
1325                 if (rdev == rrdev)
1326                         rrdev = NULL;
1327                 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1328                         atomic_inc(&rdev->nr_pending);
1329                         blocked_rdev = rdev;
1330                         break;
1331                 }
1332                 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1333                         atomic_inc(&rrdev->nr_pending);
1334                         blocked_rdev = rrdev;
1335                         break;
1336                 }
1337                 if (rrdev && (test_bit(Faulty, &rrdev->flags)
1338                               || test_bit(Unmerged, &rrdev->flags)))
1339                         rrdev = NULL;
1340
1341                 r10_bio->devs[i].bio = NULL;
1342                 r10_bio->devs[i].repl_bio = NULL;
1343                 if (!rdev || test_bit(Faulty, &rdev->flags) ||
1344                     test_bit(Unmerged, &rdev->flags)) {
1345                         set_bit(R10BIO_Degraded, &r10_bio->state);
1346                         continue;
1347                 }
1348                 if (test_bit(WriteErrorSeen, &rdev->flags)) {
1349                         sector_t first_bad;
1350                         sector_t dev_sector = r10_bio->devs[i].addr;
1351                         int bad_sectors;
1352                         int is_bad;
1353
1354                         is_bad = is_badblock(rdev, dev_sector,
1355                                              max_sectors,
1356                                              &first_bad, &bad_sectors);
1357                         if (is_bad < 0) {
1358                                 /* Mustn't write here until the bad block
1359                                  * is acknowledged
1360                                  */
1361                                 atomic_inc(&rdev->nr_pending);
1362                                 set_bit(BlockedBadBlocks, &rdev->flags);
1363                                 blocked_rdev = rdev;
1364                                 break;
1365                         }
1366                         if (is_bad && first_bad <= dev_sector) {
1367                                 /* Cannot write here at all */
1368                                 bad_sectors -= (dev_sector - first_bad);
1369                                 if (bad_sectors < max_sectors)
1370                                         /* Mustn't write more than bad_sectors
1371                                          * to other devices yet
1372                                          */
1373                                         max_sectors = bad_sectors;
1374                                 /* We don't set R10BIO_Degraded as that
1375                                  * only applies if the disk is missing,
1376                                  * so it might be re-added, and we want to
1377                                  * know to recover this chunk.
1378                                  * In this case the device is here, and the
1379                                  * fact that this chunk is not in-sync is
1380                                  * recorded in the bad block log.
1381                                  */
1382                                 continue;
1383                         }
1384                         if (is_bad) {
1385                                 int good_sectors = first_bad - dev_sector;
1386                                 if (good_sectors < max_sectors)
1387                                         max_sectors = good_sectors;
1388                         }
1389                 }
1390                 r10_bio->devs[i].bio = bio;
1391                 atomic_inc(&rdev->nr_pending);
1392                 if (rrdev) {
1393                         r10_bio->devs[i].repl_bio = bio;
1394                         atomic_inc(&rrdev->nr_pending);
1395                 }
1396         }
1397         rcu_read_unlock();
1398
1399         if (unlikely(blocked_rdev)) {
1400                 /* Have to wait for this device to get unblocked, then retry */
1401                 int j;
1402                 int d;
1403
1404                 for (j = 0; j < i; j++) {
1405                         if (r10_bio->devs[j].bio) {
1406                                 d = r10_bio->devs[j].devnum;
1407                                 rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1408                         }
1409                         if (r10_bio->devs[j].repl_bio) {
1410                                 struct md_rdev *rdev;
1411                                 d = r10_bio->devs[j].devnum;
1412                                 rdev = conf->mirrors[d].replacement;
1413                                 if (!rdev) {
1414                                         /* Race with remove_disk */
1415                                         smp_mb();
1416                                         rdev = conf->mirrors[d].rdev;
1417                                 }
1418                                 rdev_dec_pending(rdev, mddev);
1419                         }
1420                 }
1421                 allow_barrier(conf);
1422                 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1423                 wait_barrier(conf);
1424                 goto retry_write;
1425         }
1426
1427         if (max_sectors < r10_bio->sectors) {
1428                 /* We are splitting this into multiple parts, so
1429                  * we need to prepare for allocating another r10_bio.
1430                  */
1431                 r10_bio->sectors = max_sectors;
1432                 spin_lock_irq(&conf->device_lock);
1433                 if (bio->bi_phys_segments == 0)
1434                         bio->bi_phys_segments = 2;
1435                 else
1436                         bio->bi_phys_segments++;
1437                 spin_unlock_irq(&conf->device_lock);
1438         }
1439         sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector;
1440
1441         atomic_set(&r10_bio->remaining, 1);
1442         bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1443
1444         for (i = 0; i < conf->copies; i++) {
1445                 struct bio *mbio;
1446                 int d = r10_bio->devs[i].devnum;
1447                 if (!r10_bio->devs[i].bio)
1448                         continue;
1449
1450                 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1451                 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1452                             max_sectors);
1453                 r10_bio->devs[i].bio = mbio;
1454
1455                 mbio->bi_sector = (r10_bio->devs[i].addr+
1456                                    choose_data_offset(r10_bio,
1457                                                       conf->mirrors[d].rdev));
1458                 mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1459                 mbio->bi_end_io = raid10_end_write_request;
1460                 mbio->bi_rw = WRITE | do_sync | do_fua | do_discard;
1461                 mbio->bi_private = r10_bio;
1462
1463                 atomic_inc(&r10_bio->remaining);
1464
1465                 cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
1466                 if (cb)
1467                         plug = container_of(cb, struct raid10_plug_cb, cb);
1468                 else
1469                         plug = NULL;
1470                 spin_lock_irqsave(&conf->device_lock, flags);
1471                 if (plug) {
1472                         bio_list_add(&plug->pending, mbio);
1473                         plug->pending_cnt++;
1474                 } else {
1475                         bio_list_add(&conf->pending_bio_list, mbio);
1476                         conf->pending_count++;
1477                 }
1478                 spin_unlock_irqrestore(&conf->device_lock, flags);
1479                 if (!plug)
1480                         md_wakeup_thread(mddev->thread);
1481
1482                 if (!r10_bio->devs[i].repl_bio)
1483                         continue;
1484
1485                 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1486                 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1487                             max_sectors);
1488                 r10_bio->devs[i].repl_bio = mbio;
1489
1490                 /* We are actively writing to the original device
1491                  * so it cannot disappear, so the replacement cannot
1492                  * become NULL here
1493                  */
1494                 mbio->bi_sector = (r10_bio->devs[i].addr +
1495                                    choose_data_offset(
1496                                            r10_bio,
1497                                            conf->mirrors[d].replacement));
1498                 mbio->bi_bdev = conf->mirrors[d].replacement->bdev;
1499                 mbio->bi_end_io = raid10_end_write_request;
1500                 mbio->bi_rw = WRITE | do_sync | do_fua | do_discard;
1501                 mbio->bi_private = r10_bio;
1502
1503                 atomic_inc(&r10_bio->remaining);
1504                 spin_lock_irqsave(&conf->device_lock, flags);
1505                 bio_list_add(&conf->pending_bio_list, mbio);
1506                 conf->pending_count++;
1507                 spin_unlock_irqrestore(&conf->device_lock, flags);
1508                 if (!mddev_check_plugged(mddev))
1509                         md_wakeup_thread(mddev->thread);
1510         }
1511
1512         /* Don't remove the bias on 'remaining' (one_write_done) until
1513          * after checking if we need to go around again.
1514          */
1515
1516         if (sectors_handled < (bio->bi_size >> 9)) {
1517                 one_write_done(r10_bio);
1518                 /* We need another r10_bio.  It has already been counted
1519                  * in bio->bi_phys_segments.
1520                  */
1521                 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1522
1523                 r10_bio->master_bio = bio;
1524                 r10_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
1525
1526                 r10_bio->mddev = mddev;
1527                 r10_bio->sector = bio->bi_sector + sectors_handled;
1528                 r10_bio->state = 0;
1529                 goto retry_write;
1530         }
1531         one_write_done(r10_bio);
1532
1533         /* In case raid10d snuck in to freeze_array */
1534         wake_up(&conf->wait_barrier);
1535 }
1536
1537 static void status(struct seq_file *seq, struct mddev *mddev)
1538 {
1539         struct r10conf *conf = mddev->private;
1540         int i;
1541
1542         if (conf->geo.near_copies < conf->geo.raid_disks)
1543                 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1544         if (conf->geo.near_copies > 1)
1545                 seq_printf(seq, " %d near-copies", conf->geo.near_copies);
1546         if (conf->geo.far_copies > 1) {
1547                 if (conf->geo.far_offset)
1548                         seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1549                 else
1550                         seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1551         }
1552         seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
1553                                         conf->geo.raid_disks - mddev->degraded);
1554         for (i = 0; i < conf->geo.raid_disks; i++)
1555                 seq_printf(seq, "%s",
1556                               conf->mirrors[i].rdev &&
1557                               test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
1558         seq_printf(seq, "]");
1559 }
1560
1561 /* check if there are enough drives for
1562  * every block to appear on atleast one.
1563  * Don't consider the device numbered 'ignore'
1564  * as we might be about to remove it.
1565  */
1566 static int _enough(struct r10conf *conf, struct geom *geo, int ignore)
1567 {
1568         int first = 0;
1569
1570         do {
1571                 int n = conf->copies;
1572                 int cnt = 0;
1573                 int this = first;
1574                 while (n--) {
1575                         if (conf->mirrors[this].rdev &&
1576                             this != ignore)
1577                                 cnt++;
1578                         this = (this+1) % geo->raid_disks;
1579                 }
1580                 if (cnt == 0)
1581                         return 0;
1582                 first = (first + geo->near_copies) % geo->raid_disks;
1583         } while (first != 0);
1584         return 1;
1585 }
1586
1587 static int enough(struct r10conf *conf, int ignore)
1588 {
1589         return _enough(conf, &conf->geo, ignore) &&
1590                 _enough(conf, &conf->prev, ignore);
1591 }
1592
1593 static void error(struct mddev *mddev, struct md_rdev *rdev)
1594 {
1595         char b[BDEVNAME_SIZE];
1596         struct r10conf *conf = mddev->private;
1597
1598         /*
1599          * If it is not operational, then we have already marked it as dead
1600          * else if it is the last working disks, ignore the error, let the
1601          * next level up know.
1602          * else mark the drive as failed
1603          */
1604         if (test_bit(In_sync, &rdev->flags)
1605             && !enough(conf, rdev->raid_disk))
1606                 /*
1607                  * Don't fail the drive, just return an IO error.
1608                  */
1609                 return;
1610         if (test_and_clear_bit(In_sync, &rdev->flags)) {
1611                 unsigned long flags;
1612                 spin_lock_irqsave(&conf->device_lock, flags);
1613                 mddev->degraded++;
1614                 spin_unlock_irqrestore(&conf->device_lock, flags);
1615                 /*
1616                  * if recovery is running, make sure it aborts.
1617                  */
1618                 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1619         }
1620         set_bit(Blocked, &rdev->flags);
1621         set_bit(Faulty, &rdev->flags);
1622         set_bit(MD_CHANGE_DEVS, &mddev->flags);
1623         printk(KERN_ALERT
1624                "md/raid10:%s: Disk failure on %s, disabling device.\n"
1625                "md/raid10:%s: Operation continuing on %d devices.\n",
1626                mdname(mddev), bdevname(rdev->bdev, b),
1627                mdname(mddev), conf->geo.raid_disks - mddev->degraded);
1628 }
1629
1630 static void print_conf(struct r10conf *conf)
1631 {
1632         int i;
1633         struct raid10_info *tmp;
1634
1635         printk(KERN_DEBUG "RAID10 conf printout:\n");
1636         if (!conf) {
1637                 printk(KERN_DEBUG "(!conf)\n");
1638                 return;
1639         }
1640         printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
1641                 conf->geo.raid_disks);
1642
1643         for (i = 0; i < conf->geo.raid_disks; i++) {
1644                 char b[BDEVNAME_SIZE];
1645                 tmp = conf->mirrors + i;
1646                 if (tmp->rdev)
1647                         printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1648                                 i, !test_bit(In_sync, &tmp->rdev->flags),
1649                                 !test_bit(Faulty, &tmp->rdev->flags),
1650                                 bdevname(tmp->rdev->bdev,b));
1651         }
1652 }
1653
1654 static void close_sync(struct r10conf *conf)
1655 {
1656         wait_barrier(conf);
1657         allow_barrier(conf);
1658
1659         mempool_destroy(conf->r10buf_pool);
1660         conf->r10buf_pool = NULL;
1661 }
1662
1663 static int raid10_spare_active(struct mddev *mddev)
1664 {
1665         int i;
1666         struct r10conf *conf = mddev->private;
1667         struct raid10_info *tmp;
1668         int count = 0;
1669         unsigned long flags;
1670
1671         /*
1672          * Find all non-in_sync disks within the RAID10 configuration
1673          * and mark them in_sync
1674          */
1675         for (i = 0; i < conf->geo.raid_disks; i++) {
1676                 tmp = conf->mirrors + i;
1677                 if (tmp->replacement
1678                     && tmp->replacement->recovery_offset == MaxSector
1679                     && !test_bit(Faulty, &tmp->replacement->flags)
1680                     && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
1681                         /* Replacement has just become active */
1682                         if (!tmp->rdev
1683                             || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
1684                                 count++;
1685                         if (tmp->rdev) {
1686                                 /* Replaced device not technically faulty,
1687                                  * but we need to be sure it gets removed
1688                                  * and never re-added.
1689                                  */
1690                                 set_bit(Faulty, &tmp->rdev->flags);
1691                                 sysfs_notify_dirent_safe(
1692                                         tmp->rdev->sysfs_state);
1693                         }
1694                         sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
1695                 } else if (tmp->rdev
1696                            && !test_bit(Faulty, &tmp->rdev->flags)
1697                            && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1698                         count++;
1699                         sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
1700                 }
1701         }
1702         spin_lock_irqsave(&conf->device_lock, flags);
1703         mddev->degraded -= count;
1704         spin_unlock_irqrestore(&conf->device_lock, flags);
1705
1706         print_conf(conf);
1707         return count;
1708 }
1709
1710
1711 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1712 {
1713         struct r10conf *conf = mddev->private;
1714         int err = -EEXIST;
1715         int mirror;
1716         int first = 0;
1717         int last = conf->geo.raid_disks - 1;
1718         struct request_queue *q = bdev_get_queue(rdev->bdev);
1719
1720         if (mddev->recovery_cp < MaxSector)
1721                 /* only hot-add to in-sync arrays, as recovery is
1722                  * very different from resync
1723                  */
1724                 return -EBUSY;
1725         if (rdev->saved_raid_disk < 0 && !_enough(conf, &conf->prev, -1))
1726                 return -EINVAL;
1727
1728         if (rdev->raid_disk >= 0)
1729                 first = last = rdev->raid_disk;
1730
1731         if (q->merge_bvec_fn) {
1732                 set_bit(Unmerged, &rdev->flags);
1733                 mddev->merge_check_needed = 1;
1734         }
1735
1736         if (rdev->saved_raid_disk >= first &&
1737             conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1738                 mirror = rdev->saved_raid_disk;
1739         else
1740                 mirror = first;
1741         for ( ; mirror <= last ; mirror++) {
1742                 struct raid10_info *p = &conf->mirrors[mirror];
1743                 if (p->recovery_disabled == mddev->recovery_disabled)
1744                         continue;
1745                 if (p->rdev) {
1746                         if (!test_bit(WantReplacement, &p->rdev->flags) ||
1747                             p->replacement != NULL)
1748                                 continue;
1749                         clear_bit(In_sync, &rdev->flags);
1750                         set_bit(Replacement, &rdev->flags);
1751                         rdev->raid_disk = mirror;
1752                         err = 0;
1753                         disk_stack_limits(mddev->gendisk, rdev->bdev,
1754                                           rdev->data_offset << 9);
1755                         conf->fullsync = 1;
1756                         rcu_assign_pointer(p->replacement, rdev);
1757                         break;
1758                 }
1759
1760                 disk_stack_limits(mddev->gendisk, rdev->bdev,
1761                                   rdev->data_offset << 9);
1762
1763                 p->head_position = 0;
1764                 p->recovery_disabled = mddev->recovery_disabled - 1;
1765                 rdev->raid_disk = mirror;
1766                 err = 0;
1767                 if (rdev->saved_raid_disk != mirror)
1768                         conf->fullsync = 1;
1769                 rcu_assign_pointer(p->rdev, rdev);
1770                 break;
1771         }
1772         if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
1773                 /* Some requests might not have seen this new
1774                  * merge_bvec_fn.  We must wait for them to complete
1775                  * before merging the device fully.
1776                  * First we make sure any code which has tested
1777                  * our function has submitted the request, then
1778                  * we wait for all outstanding requests to complete.
1779                  */
1780                 synchronize_sched();
1781                 raise_barrier(conf, 0);
1782                 lower_barrier(conf);
1783                 clear_bit(Unmerged, &rdev->flags);
1784         }
1785         md_integrity_add_rdev(rdev, mddev);
1786         if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1787                 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
1788
1789         print_conf(conf);
1790         return err;
1791 }
1792
1793 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1794 {
1795         struct r10conf *conf = mddev->private;
1796         int err = 0;
1797         int number = rdev->raid_disk;
1798         struct md_rdev **rdevp;
1799         struct raid10_info *p = conf->mirrors + number;
1800
1801         print_conf(conf);
1802         if (rdev == p->rdev)
1803                 rdevp = &p->rdev;
1804         else if (rdev == p->replacement)
1805                 rdevp = &p->replacement;
1806         else
1807                 return 0;
1808
1809         if (test_bit(In_sync, &rdev->flags) ||
1810             atomic_read(&rdev->nr_pending)) {
1811                 err = -EBUSY;
1812                 goto abort;
1813         }
1814         /* Only remove faulty devices if recovery
1815          * is not possible.
1816          */
1817         if (!test_bit(Faulty, &rdev->flags) &&
1818             mddev->recovery_disabled != p->recovery_disabled &&
1819             (!p->replacement || p->replacement == rdev) &&
1820             number < conf->geo.raid_disks &&
1821             enough(conf, -1)) {
1822                 err = -EBUSY;
1823                 goto abort;
1824         }
1825         *rdevp = NULL;
1826         synchronize_rcu();
1827         if (atomic_read(&rdev->nr_pending)) {
1828                 /* lost the race, try later */
1829                 err = -EBUSY;
1830                 *rdevp = rdev;
1831                 goto abort;
1832         } else if (p->replacement) {
1833                 /* We must have just cleared 'rdev' */
1834                 p->rdev = p->replacement;
1835                 clear_bit(Replacement, &p->replacement->flags);
1836                 smp_mb(); /* Make sure other CPUs may see both as identical
1837                            * but will never see neither -- if they are careful.
1838                            */
1839                 p->replacement = NULL;
1840                 clear_bit(WantReplacement, &rdev->flags);
1841         } else
1842                 /* We might have just remove the Replacement as faulty
1843                  * Clear the flag just in case
1844                  */
1845                 clear_bit(WantReplacement, &rdev->flags);
1846
1847         err = md_integrity_register(mddev);
1848
1849 abort:
1850
1851         print_conf(conf);
1852         return err;
1853 }
1854
1855
1856 static void end_sync_read(struct bio *bio, int error)
1857 {
1858         struct r10bio *r10_bio = bio->bi_private;
1859         struct r10conf *conf = r10_bio->mddev->private;
1860         int d;
1861
1862         if (bio == r10_bio->master_bio) {
1863                 /* this is a reshape read */
1864                 d = r10_bio->read_slot; /* really the read dev */
1865         } else
1866                 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1867
1868         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1869                 set_bit(R10BIO_Uptodate, &r10_bio->state);
1870         else
1871                 /* The write handler will notice the lack of
1872                  * R10BIO_Uptodate and record any errors etc
1873                  */
1874                 atomic_add(r10_bio->sectors,
1875                            &conf->mirrors[d].rdev->corrected_errors);
1876
1877         /* for reconstruct, we always reschedule after a read.
1878          * for resync, only after all reads
1879          */
1880         rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1881         if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1882             atomic_dec_and_test(&r10_bio->remaining)) {
1883                 /* we have read all the blocks,
1884                  * do the comparison in process context in raid10d
1885                  */
1886                 reschedule_retry(r10_bio);
1887         }
1888 }
1889
1890 static void end_sync_request(struct r10bio *r10_bio)
1891 {
1892         struct mddev *mddev = r10_bio->mddev;
1893
1894         while (atomic_dec_and_test(&r10_bio->remaining)) {
1895                 if (r10_bio->master_bio == NULL) {
1896                         /* the primary of several recovery bios */
1897                         sector_t s = r10_bio->sectors;
1898                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1899                             test_bit(R10BIO_WriteError, &r10_bio->state))
1900                                 reschedule_retry(r10_bio);
1901                         else
1902                                 put_buf(r10_bio);
1903                         md_done_sync(mddev, s, 1);
1904                         break;
1905                 } else {
1906                         struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1907                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1908                             test_bit(R10BIO_WriteError, &r10_bio->state))
1909                                 reschedule_retry(r10_bio);
1910                         else
1911                                 put_buf(r10_bio);
1912                         r10_bio = r10_bio2;
1913                 }
1914         }
1915 }
1916
1917 static void end_sync_write(struct bio *bio, int error)
1918 {
1919         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1920         struct r10bio *r10_bio = bio->bi_private;
1921         struct mddev *mddev = r10_bio->mddev;
1922         struct r10conf *conf = mddev->private;
1923         int d;
1924         sector_t first_bad;
1925         int bad_sectors;
1926         int slot;
1927         int repl;
1928         struct md_rdev *rdev = NULL;
1929
1930         d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
1931         if (repl)
1932                 rdev = conf->mirrors[d].replacement;
1933         else
1934                 rdev = conf->mirrors[d].rdev;
1935
1936         if (!uptodate) {
1937                 if (repl)
1938                         md_error(mddev, rdev);
1939                 else {
1940                         set_bit(WriteErrorSeen, &rdev->flags);
1941                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
1942                                 set_bit(MD_RECOVERY_NEEDED,
1943                                         &rdev->mddev->recovery);
1944                         set_bit(R10BIO_WriteError, &r10_bio->state);
1945                 }
1946         } else if (is_badblock(rdev,
1947                              r10_bio->devs[slot].addr,
1948                              r10_bio->sectors,
1949                              &first_bad, &bad_sectors))
1950                 set_bit(R10BIO_MadeGood, &r10_bio->state);
1951
1952         rdev_dec_pending(rdev, mddev);
1953
1954         end_sync_request(r10_bio);
1955 }
1956
1957 /*
1958  * Note: sync and recover and handled very differently for raid10
1959  * This code is for resync.
1960  * For resync, we read through virtual addresses and read all blocks.
1961  * If there is any error, we schedule a write.  The lowest numbered
1962  * drive is authoritative.
1963  * However requests come for physical address, so we need to map.
1964  * For every physical address there are raid_disks/copies virtual addresses,
1965  * which is always are least one, but is not necessarly an integer.
1966  * This means that a physical address can span multiple chunks, so we may
1967  * have to submit multiple io requests for a single sync request.
1968  */
1969 /*
1970  * We check if all blocks are in-sync and only write to blocks that
1971  * aren't in sync
1972  */
1973 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
1974 {
1975         struct r10conf *conf = mddev->private;
1976         int i, first;
1977         struct bio *tbio, *fbio;
1978         int vcnt;
1979
1980         atomic_set(&r10_bio->remaining, 1);
1981
1982         /* find the first device with a block */
1983         for (i=0; i<conf->copies; i++)
1984                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1985                         break;
1986
1987         if (i == conf->copies)
1988                 goto done;
1989
1990         first = i;
1991         fbio = r10_bio->devs[i].bio;
1992
1993         vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
1994         /* now find blocks with errors */
1995         for (i=0 ; i < conf->copies ; i++) {
1996                 int  j, d;
1997
1998                 tbio = r10_bio->devs[i].bio;
1999
2000                 if (tbio->bi_end_io != end_sync_read)
2001                         continue;
2002                 if (i == first)
2003                         continue;
2004                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
2005                         /* We know that the bi_io_vec layout is the same for
2006                          * both 'first' and 'i', so we just compare them.
2007                          * All vec entries are PAGE_SIZE;
2008                          */
2009                         for (j = 0; j < vcnt; j++)
2010                                 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
2011                                            page_address(tbio->bi_io_vec[j].bv_page),
2012                                            fbio->bi_io_vec[j].bv_len))
2013                                         break;
2014                         if (j == vcnt)
2015                                 continue;
2016                         atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2017                         if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2018                                 /* Don't fix anything. */
2019                                 continue;
2020                 }
2021                 /* Ok, we need to write this bio, either to correct an
2022                  * inconsistency or to correct an unreadable block.
2023                  * First we need to fixup bv_offset, bv_len and
2024                  * bi_vecs, as the read request might have corrupted these
2025                  */
2026                 tbio->bi_vcnt = vcnt;
2027                 tbio->bi_size = r10_bio->sectors << 9;
2028                 tbio->bi_idx = 0;
2029                 tbio->bi_phys_segments = 0;
2030                 tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
2031                 tbio->bi_flags |= 1 << BIO_UPTODATE;
2032                 tbio->bi_next = NULL;
2033                 tbio->bi_rw = WRITE;
2034                 tbio->bi_private = r10_bio;
2035                 tbio->bi_sector = r10_bio->devs[i].addr;
2036
2037                 for (j=0; j < vcnt ; j++) {
2038                         tbio->bi_io_vec[j].bv_offset = 0;
2039                         tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
2040
2041                         memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2042                                page_address(fbio->bi_io_vec[j].bv_page),
2043                                PAGE_SIZE);
2044                 }
2045                 tbio->bi_end_io = end_sync_write;
2046
2047                 d = r10_bio->devs[i].devnum;
2048                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2049                 atomic_inc(&r10_bio->remaining);
2050                 md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
2051
2052                 tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
2053                 tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
2054                 generic_make_request(tbio);
2055         }
2056
2057         /* Now write out to any replacement devices
2058          * that are active
2059          */
2060         for (i = 0; i < conf->copies; i++) {
2061                 int j, d;
2062
2063                 tbio = r10_bio->devs[i].repl_bio;
2064                 if (!tbio || !tbio->bi_end_io)
2065                         continue;
2066                 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2067                     && r10_bio->devs[i].bio != fbio)
2068                         for (j = 0; j < vcnt; j++)
2069                                 memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2070                                        page_address(fbio->bi_io_vec[j].bv_page),
2071                                        PAGE_SIZE);
2072                 d = r10_bio->devs[i].devnum;
2073                 atomic_inc(&r10_bio->remaining);
2074                 md_sync_acct(conf->mirrors[d].replacement->bdev,
2075                              tbio->bi_size >> 9);
2076                 generic_make_request(tbio);
2077         }
2078
2079 done:
2080         if (atomic_dec_and_test(&r10_bio->remaining)) {
2081                 md_done_sync(mddev, r10_bio->sectors, 1);
2082                 put_buf(r10_bio);
2083         }
2084 }
2085
2086 /*
2087  * Now for the recovery code.
2088  * Recovery happens across physical sectors.
2089  * We recover all non-is_sync drives by finding the virtual address of
2090  * each, and then choose a working drive that also has that virt address.
2091  * There is a separate r10_bio for each non-in_sync drive.
2092  * Only the first two slots are in use. The first for reading,
2093  * The second for writing.
2094  *
2095  */
2096 static void fix_recovery_read_error(struct r10bio *r10_bio)
2097 {
2098         /* We got a read error during recovery.
2099          * We repeat the read in smaller page-sized sections.
2100          * If a read succeeds, write it to the new device or record
2101          * a bad block if we cannot.
2102          * If a read fails, record a bad block on both old and
2103          * new devices.
2104          */
2105         struct mddev *mddev = r10_bio->mddev;
2106         struct r10conf *conf = mddev->private;
2107         struct bio *bio = r10_bio->devs[0].bio;
2108         sector_t sect = 0;
2109         int sectors = r10_bio->sectors;
2110         int idx = 0;
2111         int dr = r10_bio->devs[0].devnum;
2112         int dw = r10_bio->devs[1].devnum;
2113
2114         while (sectors) {
2115                 int s = sectors;
2116                 struct md_rdev *rdev;
2117                 sector_t addr;
2118                 int ok;
2119
2120                 if (s > (PAGE_SIZE>>9))
2121                         s = PAGE_SIZE >> 9;
2122
2123                 rdev = conf->mirrors[dr].rdev;
2124                 addr = r10_bio->devs[0].addr + sect,
2125                 ok = sync_page_io(rdev,
2126                                   addr,
2127                                   s << 9,
2128                                   bio->bi_io_vec[idx].bv_page,
2129                                   READ, false);
2130                 if (ok) {
2131                         rdev = conf->mirrors[dw].rdev;
2132                         addr = r10_bio->devs[1].addr + sect;
2133                         ok = sync_page_io(rdev,
2134                                           addr,
2135                                           s << 9,
2136                                           bio->bi_io_vec[idx].bv_page,
2137                                           WRITE, false);
2138                         if (!ok) {
2139                                 set_bit(WriteErrorSeen, &rdev->flags);
2140                                 if (!test_and_set_bit(WantReplacement,
2141                                                       &rdev->flags))
2142                                         set_bit(MD_RECOVERY_NEEDED,
2143                                                 &rdev->mddev->recovery);
2144                         }
2145                 }
2146                 if (!ok) {
2147                         /* We don't worry if we cannot set a bad block -
2148                          * it really is bad so there is no loss in not
2149                          * recording it yet
2150                          */
2151                         rdev_set_badblocks(rdev, addr, s, 0);
2152
2153                         if (rdev != conf->mirrors[dw].rdev) {
2154                                 /* need bad block on destination too */
2155                                 struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2156                                 addr = r10_bio->devs[1].addr + sect;
2157                                 ok = rdev_set_badblocks(rdev2, addr, s, 0);
2158                                 if (!ok) {
2159                                         /* just abort the recovery */
2160                                         printk(KERN_NOTICE
2161                                                "md/raid10:%s: recovery aborted"
2162                                                " due to read error\n",
2163                                                mdname(mddev));
2164
2165                                         conf->mirrors[dw].recovery_disabled
2166                                                 = mddev->recovery_disabled;
2167                                         set_bit(MD_RECOVERY_INTR,
2168                                                 &mddev->recovery);
2169                                         break;
2170                                 }
2171                         }
2172                 }
2173
2174                 sectors -= s;
2175                 sect += s;
2176                 idx++;
2177         }
2178 }
2179
2180 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2181 {
2182         struct r10conf *conf = mddev->private;
2183         int d;
2184         struct bio *wbio, *wbio2;
2185
2186         if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2187                 fix_recovery_read_error(r10_bio);
2188                 end_sync_request(r10_bio);
2189                 return;
2190         }
2191
2192         /*
2193          * share the pages with the first bio
2194          * and submit the write request
2195          */
2196         d = r10_bio->devs[1].devnum;
2197         wbio = r10_bio->devs[1].bio;
2198         wbio2 = r10_bio->devs[1].repl_bio;
2199         if (wbio->bi_end_io) {
2200                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2201                 md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
2202                 generic_make_request(wbio);
2203         }
2204         if (wbio2 && wbio2->bi_end_io) {
2205                 atomic_inc(&conf->mirrors[d].replacement->nr_pending);
2206                 md_sync_acct(conf->mirrors[d].replacement->bdev,
2207                              wbio2->bi_size >> 9);
2208                 generic_make_request(wbio2);
2209         }
2210 }
2211
2212
2213 /*
2214  * Used by fix_read_error() to decay the per rdev read_errors.
2215  * We halve the read error count for every hour that has elapsed
2216  * since the last recorded read error.
2217  *
2218  */
2219 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2220 {
2221         struct timespec cur_time_mon;
2222         unsigned long hours_since_last;
2223         unsigned int read_errors = atomic_read(&rdev->read_errors);
2224
2225         ktime_get_ts(&cur_time_mon);
2226
2227         if (rdev->last_read_error.tv_sec == 0 &&
2228             rdev->last_read_error.tv_nsec == 0) {
2229                 /* first time we've seen a read error */
2230                 rdev->last_read_error = cur_time_mon;
2231                 return;
2232         }
2233
2234         hours_since_last = (cur_time_mon.tv_sec -
2235                             rdev->last_read_error.tv_sec) / 3600;
2236
2237         rdev->last_read_error = cur_time_mon;
2238
2239         /*
2240          * if hours_since_last is > the number of bits in read_errors
2241          * just set read errors to 0. We do this to avoid
2242          * overflowing the shift of read_errors by hours_since_last.
2243          */
2244         if (hours_since_last >= 8 * sizeof(read_errors))
2245                 atomic_set(&rdev->read_errors, 0);
2246         else
2247                 atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
2248 }
2249
2250 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2251                             int sectors, struct page *page, int rw)
2252 {
2253         sector_t first_bad;
2254         int bad_sectors;
2255
2256         if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
2257             && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
2258                 return -1;
2259         if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
2260                 /* success */
2261                 return 1;
2262         if (rw == WRITE) {
2263                 set_bit(WriteErrorSeen, &rdev->flags);
2264                 if (!test_and_set_bit(WantReplacement, &rdev->flags))
2265                         set_bit(MD_RECOVERY_NEEDED,
2266                                 &rdev->mddev->recovery);
2267         }
2268         /* need to record an error - either for the block or the device */
2269         if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2270                 md_error(rdev->mddev, rdev);
2271         return 0;
2272 }
2273
2274 /*
2275  * This is a kernel thread which:
2276  *
2277  *      1.      Retries failed read operations on working mirrors.
2278  *      2.      Updates the raid superblock when problems encounter.
2279  *      3.      Performs writes following reads for array synchronising.
2280  */
2281
2282 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2283 {
2284         int sect = 0; /* Offset from r10_bio->sector */
2285         int sectors = r10_bio->sectors;
2286         struct md_rdev*rdev;
2287         int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2288         int d = r10_bio->devs[r10_bio->read_slot].devnum;
2289
2290         /* still own a reference to this rdev, so it cannot
2291          * have been cleared recently.
2292          */
2293         rdev = conf->mirrors[d].rdev;
2294
2295         if (test_bit(Faulty, &rdev->flags))
2296                 /* drive has already been failed, just ignore any
2297                    more fix_read_error() attempts */
2298                 return;
2299
2300         check_decay_read_errors(mddev, rdev);
2301         atomic_inc(&rdev->read_errors);
2302         if (atomic_read(&rdev->read_errors) > max_read_errors) {
2303                 char b[BDEVNAME_SIZE];
2304                 bdevname(rdev->bdev, b);
2305
2306                 printk(KERN_NOTICE
2307                        "md/raid10:%s: %s: Raid device exceeded "
2308                        "read_error threshold [cur %d:max %d]\n",
2309                        mdname(mddev), b,
2310                        atomic_read(&rdev->read_errors), max_read_errors);
2311                 printk(KERN_NOTICE
2312                        "md/raid10:%s: %s: Failing raid device\n",
2313                        mdname(mddev), b);
2314                 md_error(mddev, conf->mirrors[d].rdev);
2315                 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2316                 return;
2317         }
2318
2319         while(sectors) {
2320                 int s = sectors;
2321                 int sl = r10_bio->read_slot;
2322                 int success = 0;
2323                 int start;
2324
2325                 if (s > (PAGE_SIZE>>9))
2326                         s = PAGE_SIZE >> 9;
2327
2328                 rcu_read_lock();
2329                 do {
2330                         sector_t first_bad;
2331                         int bad_sectors;
2332
2333                         d = r10_bio->devs[sl].devnum;
2334                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2335                         if (rdev &&
2336                             !test_bit(Unmerged, &rdev->flags) &&
2337                             test_bit(In_sync, &rdev->flags) &&
2338                             is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2339                                         &first_bad, &bad_sectors) == 0) {
2340                                 atomic_inc(&rdev->nr_pending);
2341                                 rcu_read_unlock();
2342                                 success = sync_page_io(rdev,
2343                                                        r10_bio->devs[sl].addr +
2344                                                        sect,
2345                                                        s<<9,
2346                                                        conf->tmppage, READ, false);
2347                                 rdev_dec_pending(rdev, mddev);
2348                                 rcu_read_lock();
2349                                 if (success)
2350                                         break;
2351                         }
2352                         sl++;
2353                         if (sl == conf->copies)
2354                                 sl = 0;
2355                 } while (!success && sl != r10_bio->read_slot);
2356                 rcu_read_unlock();
2357
2358                 if (!success) {
2359                         /* Cannot read from anywhere, just mark the block
2360                          * as bad on the first device to discourage future
2361                          * reads.
2362                          */
2363                         int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2364                         rdev = conf->mirrors[dn].rdev;
2365
2366                         if (!rdev_set_badblocks(
2367                                     rdev,
2368                                     r10_bio->devs[r10_bio->read_slot].addr
2369                                     + sect,
2370                                     s, 0)) {
2371                                 md_error(mddev, rdev);
2372                                 r10_bio->devs[r10_bio->read_slot].bio
2373                                         = IO_BLOCKED;
2374                         }
2375                         break;
2376                 }
2377
2378                 start = sl;
2379                 /* write it back and re-read */
2380                 rcu_read_lock();
2381                 while (sl != r10_bio->read_slot) {
2382                         char b[BDEVNAME_SIZE];
2383
2384                         if (sl==0)
2385                                 sl = conf->copies;
2386                         sl--;
2387                         d = r10_bio->devs[sl].devnum;
2388                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2389                         if (!rdev ||
2390                             test_bit(Unmerged, &rdev->flags) ||
2391                             !test_bit(In_sync, &rdev->flags))
2392                                 continue;
2393
2394                         atomic_inc(&rdev->nr_pending);
2395                         rcu_read_unlock();
2396                         if (r10_sync_page_io(rdev,
2397                                              r10_bio->devs[sl].addr +
2398                                              sect,
2399                                              s, conf->tmppage, WRITE)
2400                             == 0) {
2401                                 /* Well, this device is dead */
2402                                 printk(KERN_NOTICE
2403                                        "md/raid10:%s: read correction "
2404                                        "write failed"
2405                                        " (%d sectors at %llu on %s)\n",
2406                                        mdname(mddev), s,
2407                                        (unsigned long long)(
2408                                                sect +
2409                                                choose_data_offset(r10_bio,
2410                                                                   rdev)),
2411                                        bdevname(rdev->bdev, b));
2412                                 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2413                                        "drive\n",
2414                                        mdname(mddev),
2415                                        bdevname(rdev->bdev, b));
2416                         }
2417                         rdev_dec_pending(rdev, mddev);
2418                         rcu_read_lock();
2419                 }
2420                 sl = start;
2421                 while (sl != r10_bio->read_slot) {
2422                         char b[BDEVNAME_SIZE];
2423
2424                         if (sl==0)
2425                                 sl = conf->copies;
2426                         sl--;
2427                         d = r10_bio->devs[sl].devnum;
2428                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2429                         if (!rdev ||
2430                             !test_bit(In_sync, &rdev->flags))
2431                                 continue;
2432
2433                         atomic_inc(&rdev->nr_pending);
2434                         rcu_read_unlock();
2435                         switch (r10_sync_page_io(rdev,
2436                                              r10_bio->devs[sl].addr +
2437                                              sect,
2438                                              s, conf->tmppage,
2439                                                  READ)) {
2440                         case 0:
2441                                 /* Well, this device is dead */
2442                                 printk(KERN_NOTICE
2443                                        "md/raid10:%s: unable to read back "
2444                                        "corrected sectors"
2445                                        " (%d sectors at %llu on %s)\n",
2446                                        mdname(mddev), s,
2447                                        (unsigned long long)(
2448                                                sect +
2449                                                choose_data_offset(r10_bio, rdev)),
2450                                        bdevname(rdev->bdev, b));
2451                                 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2452                                        "drive\n",
2453                                        mdname(mddev),
2454                                        bdevname(rdev->bdev, b));
2455                                 break;
2456                         case 1:
2457                                 printk(KERN_INFO
2458                                        "md/raid10:%s: read error corrected"
2459                                        " (%d sectors at %llu on %s)\n",
2460                                        mdname(mddev), s,
2461                                        (unsigned long long)(
2462                                                sect +
2463                                                choose_data_offset(r10_bio, rdev)),
2464                                        bdevname(rdev->bdev, b));
2465                                 atomic_add(s, &rdev->corrected_errors);
2466                         }
2467
2468                         rdev_dec_pending(rdev, mddev);
2469                         rcu_read_lock();
2470                 }
2471                 rcu_read_unlock();
2472
2473                 sectors -= s;
2474                 sect += s;
2475         }
2476 }
2477
2478 static void bi_complete(struct bio *bio, int error)
2479 {
2480         complete((struct completion *)bio->bi_private);
2481 }
2482
2483 static int submit_bio_wait(int rw, struct bio *bio)
2484 {
2485         struct completion event;
2486         rw |= REQ_SYNC;
2487
2488         init_completion(&event);
2489         bio->bi_private = &event;
2490         bio->bi_end_io = bi_complete;
2491         submit_bio(rw, bio);
2492         wait_for_completion(&event);
2493
2494         return test_bit(BIO_UPTODATE, &bio->bi_flags);
2495 }
2496
2497 static int narrow_write_error(struct r10bio *r10_bio, int i)
2498 {
2499         struct bio *bio = r10_bio->master_bio;
2500         struct mddev *mddev = r10_bio->mddev;
2501         struct r10conf *conf = mddev->private;
2502         struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2503         /* bio has the data to be written to slot 'i' where
2504          * we just recently had a write error.
2505          * We repeatedly clone the bio and trim down to one block,
2506          * then try the write.  Where the write fails we record
2507          * a bad block.
2508          * It is conceivable that the bio doesn't exactly align with
2509          * blocks.  We must handle this.
2510          *
2511          * We currently own a reference to the rdev.
2512          */
2513
2514         int block_sectors;
2515         sector_t sector;
2516         int sectors;
2517         int sect_to_write = r10_bio->sectors;
2518         int ok = 1;
2519
2520         if (rdev->badblocks.shift < 0)
2521                 return 0;
2522
2523         block_sectors = 1 << rdev->badblocks.shift;
2524         sector = r10_bio->sector;
2525         sectors = ((r10_bio->sector + block_sectors)
2526                    & ~(sector_t)(block_sectors - 1))
2527                 - sector;
2528
2529         while (sect_to_write) {
2530                 struct bio *wbio;
2531                 if (sectors > sect_to_write)
2532                         sectors = sect_to_write;
2533                 /* Write at 'sector' for 'sectors' */
2534                 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2535                 md_trim_bio(wbio, sector - bio->bi_sector, sectors);
2536                 wbio->bi_sector = (r10_bio->devs[i].addr+
2537                                    choose_data_offset(r10_bio, rdev) +
2538                                    (sector - r10_bio->sector));
2539                 wbio->bi_bdev = rdev->bdev;
2540                 if (submit_bio_wait(WRITE, wbio) == 0)
2541                         /* Failure! */
2542                         ok = rdev_set_badblocks(rdev, sector,
2543                                                 sectors, 0)
2544                                 && ok;
2545
2546                 bio_put(wbio);
2547                 sect_to_write -= sectors;
2548                 sector += sectors;
2549                 sectors = block_sectors;
2550         }
2551         return ok;
2552 }
2553
2554 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2555 {
2556         int slot = r10_bio->read_slot;
2557         struct bio *bio;
2558         struct r10conf *conf = mddev->private;
2559         struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2560         char b[BDEVNAME_SIZE];
2561         unsigned long do_sync;
2562         int max_sectors;
2563
2564         /* we got a read error. Maybe the drive is bad.  Maybe just
2565          * the block and we can fix it.
2566          * We freeze all other IO, and try reading the block from
2567          * other devices.  When we find one, we re-write
2568          * and check it that fixes the read error.
2569          * This is all done synchronously while the array is
2570          * frozen.
2571          */
2572         bio = r10_bio->devs[slot].bio;
2573         bdevname(bio->bi_bdev, b);
2574         bio_put(bio);
2575         r10_bio->devs[slot].bio = NULL;
2576
2577         if (mddev->ro == 0) {
2578                 freeze_array(conf);
2579                 fix_read_error(conf, mddev, r10_bio);
2580                 unfreeze_array(conf);
2581         } else
2582                 r10_bio->devs[slot].bio = IO_BLOCKED;
2583
2584         rdev_dec_pending(rdev, mddev);
2585
2586 read_more:
2587         rdev = read_balance(conf, r10_bio, &max_sectors);
2588         if (rdev == NULL) {
2589                 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
2590                        " read error for block %llu\n",
2591                        mdname(mddev), b,
2592                        (unsigned long long)r10_bio->sector);
2593                 raid_end_bio_io(r10_bio);
2594                 return;
2595         }
2596
2597         do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
2598         slot = r10_bio->read_slot;
2599         printk_ratelimited(
2600                 KERN_ERR
2601                 "md/raid10:%s: %s: redirecting "
2602                 "sector %llu to another mirror\n",
2603                 mdname(mddev),
2604                 bdevname(rdev->bdev, b),
2605                 (unsigned long long)r10_bio->sector);
2606         bio = bio_clone_mddev(r10_bio->master_bio,
2607                               GFP_NOIO, mddev);
2608         md_trim_bio(bio,
2609                     r10_bio->sector - bio->bi_sector,
2610                     max_sectors);
2611         r10_bio->devs[slot].bio = bio;
2612         r10_bio->devs[slot].rdev = rdev;
2613         bio->bi_sector = r10_bio->devs[slot].addr
2614                 + choose_data_offset(r10_bio, rdev);
2615         bio->bi_bdev = rdev->bdev;
2616         bio->bi_rw = READ | do_sync;
2617         bio->bi_private = r10_bio;
2618         bio->bi_end_io = raid10_end_read_request;
2619         if (max_sectors < r10_bio->sectors) {
2620                 /* Drat - have to split this up more */
2621                 struct bio *mbio = r10_bio->master_bio;
2622                 int sectors_handled =
2623                         r10_bio->sector + max_sectors
2624                         - mbio->bi_sector;
2625                 r10_bio->sectors = max_sectors;
2626                 spin_lock_irq(&conf->device_lock);
2627                 if (mbio->bi_phys_segments == 0)
2628                         mbio->bi_phys_segments = 2;
2629                 else
2630                         mbio->bi_phys_segments++;
2631                 spin_unlock_irq(&conf->device_lock);
2632                 generic_make_request(bio);
2633
2634                 r10_bio = mempool_alloc(conf->r10bio_pool,
2635                                         GFP_NOIO);
2636                 r10_bio->master_bio = mbio;
2637                 r10_bio->sectors = (mbio->bi_size >> 9)
2638                         - sectors_handled;
2639                 r10_bio->state = 0;
2640                 set_bit(R10BIO_ReadError,
2641                         &r10_bio->state);
2642                 r10_bio->mddev = mddev;
2643                 r10_bio->sector = mbio->bi_sector
2644                         + sectors_handled;
2645
2646                 goto read_more;
2647         } else
2648                 generic_make_request(bio);
2649 }
2650
2651 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2652 {
2653         /* Some sort of write request has finished and it
2654          * succeeded in writing where we thought there was a
2655          * bad block.  So forget the bad block.
2656          * Or possibly if failed and we need to record
2657          * a bad block.
2658          */
2659         int m;
2660         struct md_rdev *rdev;
2661
2662         if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2663             test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2664                 for (m = 0; m < conf->copies; m++) {
2665                         int dev = r10_bio->devs[m].devnum;
2666                         rdev = conf->mirrors[dev].rdev;
2667                         if (r10_bio->devs[m].bio == NULL)
2668                                 continue;
2669                         if (test_bit(BIO_UPTODATE,
2670                                      &r10_bio->devs[m].bio->bi_flags)) {
2671                                 rdev_clear_badblocks(
2672                                         rdev,
2673                                         r10_bio->devs[m].addr,
2674                                         r10_bio->sectors, 0);
2675                         } else {
2676                                 if (!rdev_set_badblocks(
2677                                             rdev,
2678                                             r10_bio->devs[m].addr,
2679                                             r10_bio->sectors, 0))
2680                                         md_error(conf->mddev, rdev);
2681                         }
2682                         rdev = conf->mirrors[dev].replacement;
2683                         if (r10_bio->devs[m].repl_bio == NULL)
2684                                 continue;
2685                         if (test_bit(BIO_UPTODATE,
2686                                      &r10_bio->devs[m].repl_bio->bi_flags)) {
2687                                 rdev_clear_badblocks(
2688                                         rdev,
2689                                         r10_bio->devs[m].addr,
2690                                         r10_bio->sectors, 0);
2691                         } else {
2692                                 if (!rdev_set_badblocks(
2693                                             rdev,
2694                                             r10_bio->devs[m].addr,
2695                                             r10_bio->sectors, 0))
2696                                         md_error(conf->mddev, rdev);
2697                         }
2698                 }
2699                 put_buf(r10_bio);
2700         } else {
2701                 for (m = 0; m < conf->copies; m++) {
2702                         int dev = r10_bio->devs[m].devnum;
2703                         struct bio *bio = r10_bio->devs[m].bio;
2704                         rdev = conf->mirrors[dev].rdev;
2705                         if (bio == IO_MADE_GOOD) {
2706                                 rdev_clear_badblocks(
2707                                         rdev,
2708                                         r10_bio->devs[m].addr,
2709                                         r10_bio->sectors, 0);
2710                                 rdev_dec_pending(rdev, conf->mddev);
2711                         } else if (bio != NULL &&
2712                                    !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2713                                 if (!narrow_write_error(r10_bio, m)) {
2714                                         md_error(conf->mddev, rdev);
2715                                         set_bit(R10BIO_Degraded,
2716                                                 &r10_bio->state);
2717                                 }
2718                                 rdev_dec_pending(rdev, conf->mddev);
2719                         }
2720                         bio = r10_bio->devs[m].repl_bio;
2721                         rdev = conf->mirrors[dev].replacement;
2722                         if (rdev && bio == IO_MADE_GOOD) {
2723                                 rdev_clear_badblocks(
2724                                         rdev,
2725                                         r10_bio->devs[m].addr,
2726                                         r10_bio->sectors, 0);
2727                                 rdev_dec_pending(rdev, conf->mddev);
2728                         }
2729                 }
2730                 if (test_bit(R10BIO_WriteError,
2731                              &r10_bio->state))
2732                         close_write(r10_bio);
2733                 raid_end_bio_io(r10_bio);
2734         }
2735 }
2736
2737 static void raid10d(struct md_thread *thread)
2738 {
2739         struct mddev *mddev = thread->mddev;
2740         struct r10bio *r10_bio;
2741         unsigned long flags;
2742         struct r10conf *conf = mddev->private;
2743         struct list_head *head = &conf->retry_list;
2744         struct blk_plug plug;
2745
2746         md_check_recovery(mddev);
2747
2748         blk_start_plug(&plug);
2749         for (;;) {
2750
2751                 flush_pending_writes(conf);
2752
2753                 spin_lock_irqsave(&conf->device_lock, flags);
2754                 if (list_empty(head)) {
2755                         spin_unlock_irqrestore(&conf->device_lock, flags);
2756                         break;
2757                 }
2758                 r10_bio = list_entry(head->prev, struct r10bio, retry_list);
2759                 list_del(head->prev);
2760                 conf->nr_queued--;
2761                 spin_unlock_irqrestore(&conf->device_lock, flags);
2762
2763                 mddev = r10_bio->mddev;
2764                 conf = mddev->private;
2765                 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2766                     test_bit(R10BIO_WriteError, &r10_bio->state))
2767                         handle_write_completed(conf, r10_bio);
2768                 else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
2769                         reshape_request_write(mddev, r10_bio);
2770                 else if (test_bit(R10BIO_IsSync, &r10_bio->state))
2771                         sync_request_write(mddev, r10_bio);
2772                 else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
2773                         recovery_request_write(mddev, r10_bio);
2774                 else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2775                         handle_read_error(mddev, r10_bio);
2776                 else {
2777                         /* just a partial read to be scheduled from a
2778                          * separate context
2779                          */
2780                         int slot = r10_bio->read_slot;
2781                         generic_make_request(r10_bio->devs[slot].bio);
2782                 }
2783
2784                 cond_resched();
2785                 if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
2786                         md_check_recovery(mddev);
2787         }
2788         blk_finish_plug(&plug);
2789 }
2790
2791
2792 static int init_resync(struct r10conf *conf)
2793 {
2794         int buffs;
2795         int i;
2796
2797         buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2798         BUG_ON(conf->r10buf_pool);
2799         conf->have_replacement = 0;
2800         for (i = 0; i < conf->geo.raid_disks; i++)
2801                 if (conf->mirrors[i].replacement)
2802                         conf->have_replacement = 1;
2803         conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
2804         if (!conf->r10buf_pool)
2805                 return -ENOMEM;
2806         conf->next_resync = 0;
2807         return 0;
2808 }
2809
2810 /*
2811  * perform a "sync" on one "block"
2812  *
2813  * We need to make sure that no normal I/O request - particularly write
2814  * requests - conflict with active sync requests.
2815  *
2816  * This is achieved by tracking pending requests and a 'barrier' concept
2817  * that can be installed to exclude normal IO requests.
2818  *
2819  * Resync and recovery are handled very differently.
2820  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
2821  *
2822  * For resync, we iterate over virtual addresses, read all copies,
2823  * and update if there are differences.  If only one copy is live,
2824  * skip it.
2825  * For recovery, we iterate over physical addresses, read a good
2826  * value for each non-in_sync drive, and over-write.
2827  *
2828  * So, for recovery we may have several outstanding complex requests for a
2829  * given address, one for each out-of-sync device.  We model this by allocating
2830  * a number of r10_bio structures, one for each out-of-sync device.
2831  * As we setup these structures, we collect all bio's together into a list
2832  * which we then process collectively to add pages, and then process again
2833  * to pass to generic_make_request.
2834  *
2835  * The r10_bio structures are linked using a borrowed master_bio pointer.
2836  * This link is counted in ->remaining.  When the r10_bio that points to NULL
2837  * has its remaining count decremented to 0, the whole complex operation
2838  * is complete.
2839  *
2840  */
2841
2842 static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
2843                              int *skipped, int go_faster)
2844 {
2845         struct r10conf *conf = mddev->private;
2846         struct r10bio *r10_bio;
2847         struct bio *biolist = NULL, *bio;
2848         sector_t max_sector, nr_sectors;
2849         int i;
2850         int max_sync;
2851         sector_t sync_blocks;
2852         sector_t sectors_skipped = 0;
2853         int chunks_skipped = 0;
2854         sector_t chunk_mask = conf->geo.chunk_mask;
2855
2856         if (!conf->r10buf_pool)
2857                 if (init_resync(conf))
2858                         return 0;
2859
2860  skipped:
2861         max_sector = mddev->dev_sectors;
2862         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
2863             test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2864                 max_sector = mddev->resync_max_sectors;
2865         if (sector_nr >= max_sector) {
2866                 /* If we aborted, we need to abort the
2867                  * sync on the 'current' bitmap chucks (there can
2868                  * be several when recovering multiple devices).
2869                  * as we may have started syncing it but not finished.
2870                  * We can find the current address in
2871                  * mddev->curr_resync, but for recovery,
2872                  * we need to convert that to several
2873                  * virtual addresses.
2874                  */
2875                 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
2876                         end_reshape(conf);
2877                         return 0;
2878                 }
2879
2880                 if (mddev->curr_resync < max_sector) { /* aborted */
2881                         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2882                                 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2883                                                 &sync_blocks, 1);
2884                         else for (i = 0; i < conf->geo.raid_disks; i++) {
2885                                 sector_t sect =
2886                                         raid10_find_virt(conf, mddev->curr_resync, i);
2887                                 bitmap_end_sync(mddev->bitmap, sect,
2888                                                 &sync_blocks, 1);
2889                         }
2890                 } else {
2891                         /* completed sync */
2892                         if ((!mddev->bitmap || conf->fullsync)
2893                             && conf->have_replacement
2894                             && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2895                                 /* Completed a full sync so the replacements
2896                                  * are now fully recovered.
2897                                  */
2898                                 for (i = 0; i < conf->geo.raid_disks; i++)
2899                                         if (conf->mirrors[i].replacement)
2900                                                 conf->mirrors[i].replacement
2901                                                         ->recovery_offset
2902                                                         = MaxSector;
2903                         }
2904                         conf->fullsync = 0;
2905                 }
2906                 bitmap_close_sync(mddev->bitmap);
2907                 close_sync(conf);
2908                 *skipped = 1;
2909                 return sectors_skipped;
2910         }
2911
2912         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2913                 return reshape_request(mddev, sector_nr, skipped);
2914
2915         if (chunks_skipped >= conf->geo.raid_disks) {
2916                 /* if there has been nothing to do on any drive,
2917                  * then there is nothing to do at all..
2918                  */
2919                 *skipped = 1;
2920                 return (max_sector - sector_nr) + sectors_skipped;
2921         }
2922
2923         if (max_sector > mddev->resync_max)
2924                 max_sector = mddev->resync_max; /* Don't do IO beyond here */
2925
2926         /* make sure whole request will fit in a chunk - if chunks
2927          * are meaningful
2928          */
2929         if (conf->geo.near_copies < conf->geo.raid_disks &&
2930             max_sector > (sector_nr | chunk_mask))
2931                 max_sector = (sector_nr | chunk_mask) + 1;
2932         /*
2933          * If there is non-resync activity waiting for us then
2934          * put in a delay to throttle resync.
2935          */
2936         if (!go_faster && conf->nr_waiting)
2937                 msleep_interruptible(1000);
2938
2939         /* Again, very different code for resync and recovery.
2940          * Both must result in an r10bio with a list of bios that
2941          * have bi_end_io, bi_sector, bi_bdev set,
2942          * and bi_private set to the r10bio.
2943          * For recovery, we may actually create several r10bios
2944          * with 2 bios in each, that correspond to the bios in the main one.
2945          * In this case, the subordinate r10bios link back through a
2946          * borrowed master_bio pointer, and the counter in the master
2947          * includes a ref from each subordinate.
2948          */
2949         /* First, we decide what to do and set ->bi_end_io
2950          * To end_sync_read if we want to read, and
2951          * end_sync_write if we will want to write.
2952          */
2953
2954         max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
2955         if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2956                 /* recovery... the complicated one */
2957                 int j;
2958                 r10_bio = NULL;
2959
2960                 for (i = 0 ; i < conf->geo.raid_disks; i++) {
2961                         int still_degraded;
2962                         struct r10bio *rb2;
2963                         sector_t sect;
2964                         int must_sync;
2965                         int any_working;
2966                         struct raid10_info *mirror = &conf->mirrors[i];
2967
2968                         if ((mirror->rdev == NULL ||
2969                              test_bit(In_sync, &mirror->rdev->flags))
2970                             &&
2971                             (mirror->replacement == NULL ||
2972                              test_bit(Faulty,
2973                                       &mirror->replacement->flags)))
2974                                 continue;
2975
2976                         still_degraded = 0;
2977                         /* want to reconstruct this device */
2978                         rb2 = r10_bio;
2979                         sect = raid10_find_virt(conf, sector_nr, i);
2980                         if (sect >= mddev->resync_max_sectors) {
2981                                 /* last stripe is not complete - don't
2982                                  * try to recover this sector.
2983                                  */
2984                                 continue;
2985                         }
2986                         /* Unless we are doing a full sync, or a replacement
2987                          * we only need to recover the block if it is set in
2988                          * the bitmap
2989                          */
2990                         must_sync = bitmap_start_sync(mddev->bitmap, sect,
2991                                                       &sync_blocks, 1);
2992                         if (sync_blocks < max_sync)
2993                                 max_sync = sync_blocks;
2994                         if (!must_sync &&
2995                             mirror->replacement == NULL &&
2996                             !conf->fullsync) {
2997                                 /* yep, skip the sync_blocks here, but don't assume
2998                                  * that there will never be anything to do here
2999                                  */
3000                                 chunks_skipped = -1;
3001                                 continue;
3002                         }
3003
3004                         r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3005                         raise_barrier(conf, rb2 != NULL);
3006                         atomic_set(&r10_bio->remaining, 0);
3007
3008                         r10_bio->master_bio = (struct bio*)rb2;
3009                         if (rb2)
3010                                 atomic_inc(&rb2->remaining);
3011                         r10_bio->mddev = mddev;
3012                         set_bit(R10BIO_IsRecover, &r10_bio->state);
3013                         r10_bio->sector = sect;
3014
3015                         raid10_find_phys(conf, r10_bio);
3016
3017                         /* Need to check if the array will still be
3018                          * degraded
3019                          */
3020                         for (j = 0; j < conf->geo.raid_disks; j++)
3021                                 if (conf->mirrors[j].rdev == NULL ||
3022                                     test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
3023                                         still_degraded = 1;
3024                                         break;
3025                                 }
3026
3027                         must_sync = bitmap_start_sync(mddev->bitmap, sect,
3028                                                       &sync_blocks, still_degraded);
3029
3030                         any_working = 0;
3031                         for (j=0; j<conf->copies;j++) {
3032                                 int k;
3033                                 int d = r10_bio->devs[j].devnum;
3034                                 sector_t from_addr, to_addr;
3035                                 struct md_rdev *rdev;
3036                                 sector_t sector, first_bad;
3037                                 int bad_sectors;
3038                                 if (!conf->mirrors[d].rdev ||
3039                                     !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
3040                                         continue;
3041                                 /* This is where we read from */
3042                                 any_working = 1;
3043                                 rdev = conf->mirrors[d].rdev;
3044                                 sector = r10_bio->devs[j].addr;
3045
3046                                 if (is_badblock(rdev, sector, max_sync,
3047                                                 &first_bad, &bad_sectors)) {
3048                                         if (first_bad > sector)
3049                                                 max_sync = first_bad - sector;
3050                                         else {
3051                                                 bad_sectors -= (sector
3052                                                                 - first_bad);
3053                                                 if (max_sync > bad_sectors)
3054                                                         max_sync = bad_sectors;
3055                                                 continue;
3056                                         }
3057                                 }
3058                                 bio = r10_bio->devs[0].bio;
3059                                 bio->bi_next = biolist;
3060                                 biolist = bio;
3061                                 bio->bi_private = r10_bio;
3062                                 bio->bi_end_io = end_sync_read;
3063                                 bio->bi_rw = READ;
3064                                 from_addr = r10_bio->devs[j].addr;
3065                                 bio->bi_sector = from_addr + rdev->data_offset;
3066                                 bio->bi_bdev = rdev->bdev;
3067                                 atomic_inc(&rdev->nr_pending);
3068                                 /* and we write to 'i' (if not in_sync) */
3069
3070                                 for (k=0; k<conf->copies; k++)
3071                                         if (r10_bio->devs[k].devnum == i)
3072                                                 break;
3073                                 BUG_ON(k == conf->copies);
3074                                 to_addr = r10_bio->devs[k].addr;
3075                                 r10_bio->devs[0].devnum = d;
3076                                 r10_bio->devs[0].addr = from_addr;
3077                                 r10_bio->devs[1].devnum = i;
3078                                 r10_bio->devs[1].addr = to_addr;
3079
3080                                 rdev = mirror->rdev;
3081                                 if (!test_bit(In_sync, &rdev->flags)) {
3082                                         bio = r10_bio->devs[1].bio;
3083                                         bio->bi_next = biolist;
3084                                         biolist = bio;
3085                                         bio->bi_private = r10_bio;
3086                                         bio->bi_end_io = end_sync_write;
3087                                         bio->bi_rw = WRITE;
3088                                         bio->bi_sector = to_addr
3089                                                 + rdev->data_offset;
3090                                         bio->bi_bdev = rdev->bdev;
3091                                         atomic_inc(&r10_bio->remaining);
3092                                 } else
3093                                         r10_bio->devs[1].bio->bi_end_io = NULL;
3094
3095                                 /* and maybe write to replacement */
3096                                 bio = r10_bio->devs[1].repl_bio;
3097                                 if (bio)
3098                                         bio->bi_end_io = NULL;
3099                                 rdev = mirror->replacement;
3100                                 /* Note: if rdev != NULL, then bio
3101                                  * cannot be NULL as r10buf_pool_alloc will
3102                                  * have allocated it.
3103                                  * So the second test here is pointless.
3104                                  * But it keeps semantic-checkers happy, and
3105                                  * this comment keeps human reviewers
3106                                  * happy.
3107                                  */
3108                                 if (rdev == NULL || bio == NULL ||
3109                                     test_bit(Faulty, &rdev->flags))
3110                                         break;
3111                                 bio->bi_next = biolist;
3112                                 biolist = bio;
3113                                 bio->bi_private = r10_bio;
3114                                 bio->bi_end_io = end_sync_write;
3115                                 bio->bi_rw = WRITE;
3116                                 bio->bi_sector = to_addr + rdev->data_offset;
3117                                 bio->bi_bdev = rdev->bdev;
3118                                 atomic_inc(&r10_bio->remaining);
3119                                 break;
3120                         }
3121                         if (j == conf->copies) {
3122                                 /* Cannot recover, so abort the recovery or
3123                                  * record a bad block */
3124                                 put_buf(r10_bio);
3125                                 if (rb2)
3126                                         atomic_dec(&rb2->remaining);
3127                                 r10_bio = rb2;
3128                                 if (any_working) {
3129                                         /* problem is that there are bad blocks
3130                                          * on other device(s)
3131                                          */
3132                                         int k;
3133                                         for (k = 0; k < conf->copies; k++)
3134                                                 if (r10_bio->devs[k].devnum == i)
3135                                                         break;
3136                                         if (!test_bit(In_sync,
3137                                                       &mirror->rdev->flags)
3138                                             && !rdev_set_badblocks(
3139                                                     mirror->rdev,
3140                                                     r10_bio->devs[k].addr,
3141                                                     max_sync, 0))
3142                                                 any_working = 0;
3143                                         if (mirror->replacement &&
3144                                             !rdev_set_badblocks(
3145                                                     mirror->replacement,
3146                                                     r10_bio->devs[k].addr,
3147                                                     max_sync, 0))
3148                                                 any_working = 0;
3149                                 }
3150                                 if (!any_working)  {
3151                                         if (!test_and_set_bit(MD_RECOVERY_INTR,
3152                                                               &mddev->recovery))
3153                                                 printk(KERN_INFO "md/raid10:%s: insufficient "
3154                                                        "working devices for recovery.\n",
3155                                                        mdname(mddev));
3156                                         mirror->recovery_disabled
3157                                                 = mddev->recovery_disabled;
3158                                 }
3159                                 break;
3160                         }
3161                 }
3162                 if (biolist == NULL) {
3163                         while (r10_bio) {
3164                                 struct r10bio *rb2 = r10_bio;
3165                                 r10_bio = (struct r10bio*) rb2->master_bio;
3166                                 rb2->master_bio = NULL;
3167                                 put_buf(rb2);
3168                         }
3169                         goto giveup;
3170                 }
3171         } else {
3172                 /* resync. Schedule a read for every block at this virt offset */
3173                 int count = 0;
3174
3175                 bitmap_cond_end_sync(mddev->bitmap, sector_nr);
3176
3177                 if (!bitmap_start_sync(mddev->bitmap, sector_nr,
3178                                        &sync_blocks, mddev->degraded) &&
3179                     !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3180                                                  &mddev->recovery)) {
3181                         /* We can skip this block */
3182                         *skipped = 1;
3183                         return sync_blocks + sectors_skipped;
3184                 }
3185                 if (sync_blocks < max_sync)
3186                         max_sync = sync_blocks;
3187                 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3188
3189                 r10_bio->mddev = mddev;
3190                 atomic_set(&r10_bio->remaining, 0);
3191                 raise_barrier(conf, 0);
3192                 conf->next_resync = sector_nr;
3193
3194                 r10_bio->master_bio = NULL;
3195                 r10_bio->sector = sector_nr;
3196                 set_bit(R10BIO_IsSync, &r10_bio->state);
3197                 raid10_find_phys(conf, r10_bio);
3198                 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
3199
3200                 for (i = 0; i < conf->copies; i++) {
3201                         int d = r10_bio->devs[i].devnum;
3202                         sector_t first_bad, sector;
3203                         int bad_sectors;
3204
3205                         if (r10_bio->devs[i].repl_bio)
3206                                 r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3207
3208                         bio = r10_bio->devs[i].bio;
3209                         bio->bi_end_io = NULL;
3210                         clear_bit(BIO_UPTODATE, &bio->bi_flags);
3211                         if (conf->mirrors[d].rdev == NULL ||
3212                             test_bit(Faulty, &conf->mirrors[d].rdev->flags))
3213                                 continue;
3214                         sector = r10_bio->devs[i].addr;
3215                         if (is_badblock(conf->mirrors[d].rdev,
3216                                         sector, max_sync,
3217                                         &first_bad, &bad_sectors)) {
3218                                 if (first_bad > sector)
3219                                         max_sync = first_bad - sector;
3220                                 else {
3221                                         bad_sectors -= (sector - first_bad);
3222                                         if (max_sync > bad_sectors)
3223                                                 max_sync = bad_sectors;
3224                                         continue;
3225                                 }
3226                         }
3227                         atomic_inc(&conf->mirrors[d].rdev->nr_pending);
3228                         atomic_inc(&r10_bio->remaining);
3229                         bio->bi_next = biolist;
3230                         biolist = bio;
3231                         bio->bi_private = r10_bio;
3232                         bio->bi_end_io = end_sync_read;
3233                         bio->bi_rw = READ;
3234                         bio->bi_sector = sector +
3235                                 conf->mirrors[d].rdev->data_offset;
3236                         bio->bi_bdev = conf->mirrors[d].rdev->bdev;
3237                         count++;
3238
3239                         if (conf->mirrors[d].replacement == NULL ||
3240                             test_bit(Faulty,
3241                                      &conf->mirrors[d].replacement->flags))
3242                                 continue;
3243
3244                         /* Need to set up for writing to the replacement */
3245                         bio = r10_bio->devs[i].repl_bio;
3246                         clear_bit(BIO_UPTODATE, &bio->bi_flags);
3247
3248                         sector = r10_bio->devs[i].addr;
3249                         atomic_inc(&conf->mirrors[d].rdev->nr_pending);
3250                         bio->bi_next = biolist;
3251                         biolist = bio;
3252                         bio->bi_private = r10_bio;
3253                         bio->bi_end_io = end_sync_write;
3254                         bio->bi_rw = WRITE;
3255                         bio->bi_sector = sector +
3256                                 conf->mirrors[d].replacement->data_offset;
3257                         bio->bi_bdev = conf->mirrors[d].replacement->bdev;
3258                         count++;
3259                 }
3260
3261                 if (count < 2) {
3262                         for (i=0; i<conf->copies; i++) {
3263                                 int d = r10_bio->devs[i].devnum;
3264                                 if (r10_bio->devs[i].bio->bi_end_io)
3265                                         rdev_dec_pending(conf->mirrors[d].rdev,
3266                                                          mddev);
3267                                 if (r10_bio->devs[i].repl_bio &&
3268                                     r10_bio->devs[i].repl_bio->bi_end_io)
3269                                         rdev_dec_pending(
3270                                                 conf->mirrors[d].replacement,
3271                                                 mddev);
3272                         }
3273                         put_buf(r10_bio);
3274                         biolist = NULL;
3275                         goto giveup;
3276                 }
3277         }
3278
3279         for (bio = biolist; bio ; bio=bio->bi_next) {
3280
3281                 bio->bi_flags &= ~(BIO_POOL_MASK - 1);
3282                 if (bio->bi_end_io)
3283                         bio->bi_flags |= 1 << BIO_UPTODATE;
3284                 bio->bi_vcnt = 0;
3285                 bio->bi_idx = 0;
3286                 bio->bi_phys_segments = 0;
3287                 bio->bi_size = 0;
3288         }
3289
3290         nr_sectors = 0;
3291         if (sector_nr + max_sync < max_sector)
3292                 max_sector = sector_nr + max_sync;
3293         do {
3294                 struct page *page;
3295                 int len = PAGE_SIZE;
3296                 if (sector_nr + (len>>9) > max_sector)
3297                         len = (max_sector - sector_nr) << 9;
3298                 if (len == 0)
3299                         break;
3300                 for (bio= biolist ; bio ; bio=bio->bi_next) {
3301                         struct bio *bio2;
3302                         page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
3303                         if (bio_add_page(bio, page, len, 0))
3304                                 continue;
3305
3306                         /* stop here */
3307                         bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
3308                         for (bio2 = biolist;
3309                              bio2 && bio2 != bio;
3310                              bio2 = bio2->bi_next) {
3311                                 /* remove last page from this bio */
3312                                 bio2->bi_vcnt--;
3313                                 bio2->bi_size -= len;
3314                                 bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
3315                         }
3316                         goto bio_full;
3317                 }
3318                 nr_sectors += len>>9;
3319                 sector_nr += len>>9;
3320         } while (biolist->bi_vcnt < RESYNC_PAGES);
3321  bio_full:
3322         r10_bio->sectors = nr_sectors;
3323
3324         while (biolist) {
3325                 bio = biolist;
3326                 biolist = biolist->bi_next;
3327
3328                 bio->bi_next = NULL;
3329                 r10_bio = bio->bi_private;
3330                 r10_bio->sectors = nr_sectors;
3331
3332                 if (bio->bi_end_io == end_sync_read) {
3333                         md_sync_acct(bio->bi_bdev, nr_sectors);
3334                         generic_make_request(bio);
3335                 }
3336         }
3337
3338         if (sectors_skipped)
3339                 /* pretend they weren't skipped, it makes
3340                  * no important difference in this case
3341                  */
3342                 md_done_sync(mddev, sectors_skipped, 1);
3343
3344         return sectors_skipped + nr_sectors;
3345  giveup:
3346         /* There is nowhere to write, so all non-sync
3347          * drives must be failed or in resync, all drives
3348          * have a bad block, so try the next chunk...
3349          */
3350         if (sector_nr + max_sync < max_sector)
3351                 max_sector = sector_nr + max_sync;
3352
3353         sectors_skipped += (max_sector - sector_nr);
3354         chunks_skipped ++;
3355         sector_nr = max_sector;
3356         goto skipped;
3357 }
3358
3359 static sector_t
3360 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3361 {
3362         sector_t size;
3363         struct r10conf *conf = mddev->private;
3364
3365         if (!raid_disks)
3366                 raid_disks = min(conf->geo.raid_disks,
3367                                  conf->prev.raid_disks);
3368         if (!sectors)
3369                 sectors = conf->dev_sectors;
3370
3371         size = sectors >> conf->geo.chunk_shift;
3372         sector_div(size, conf->geo.far_copies);
3373         size = size * raid_disks;
3374         sector_div(size, conf->geo.near_copies);
3375
3376         return size << conf->geo.chunk_shift;
3377 }
3378
3379 static void calc_sectors(struct r10conf *conf, sector_t size)
3380 {
3381         /* Calculate the number of sectors-per-device that will
3382          * actually be used, and set conf->dev_sectors and
3383          * conf->stride
3384          */
3385
3386         size = size >> conf->geo.chunk_shift;
3387         sector_div(size, conf->geo.far_copies);
3388         size = size * conf->geo.raid_disks;
3389         sector_div(size, conf->geo.near_copies);
3390         /* 'size' is now the number of chunks in the array */
3391         /* calculate "used chunks per device" */
3392         size = size * conf->copies;
3393
3394         /* We need to round up when dividing by raid_disks to
3395          * get the stride size.
3396          */
3397         size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3398
3399         conf->dev_sectors = size << conf->geo.chunk_shift;
3400
3401         if (conf->geo.far_offset)
3402                 conf->geo.stride = 1 << conf->geo.chunk_shift;
3403         else {
3404                 sector_div(size, conf->geo.far_copies);
3405                 conf->geo.stride = size << conf->geo.chunk_shift;
3406         }
3407 }
3408
3409 enum geo_type {geo_new, geo_old, geo_start};
3410 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3411 {
3412         int nc, fc, fo;
3413         i