]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - kernel/power/swap.c
PM / Hibernate: Use async I/O when reading compressed hibernation image
[~shefty/rdma-dev.git] / kernel / power / swap.c
1 /*
2  * linux/kernel/power/swap.c
3  *
4  * This file provides functions for reading the suspend image from
5  * and writing it to a swap partition.
6  *
7  * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
8  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9  * Copyright (C) 2010 Bojan Smojver <bojan@rexursive.com>
10  *
11  * This file is released under the GPLv2.
12  *
13  */
14
15 #include <linux/module.h>
16 #include <linux/file.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/genhd.h>
20 #include <linux/device.h>
21 #include <linux/buffer_head.h>
22 #include <linux/bio.h>
23 #include <linux/blkdev.h>
24 #include <linux/swap.h>
25 #include <linux/swapops.h>
26 #include <linux/pm.h>
27 #include <linux/slab.h>
28 #include <linux/lzo.h>
29 #include <linux/vmalloc.h>
30
31 #include "power.h"
32
33 #define HIBERNATE_SIG   "LINHIB0001"
34
35 /*
36  *      The swap map is a data structure used for keeping track of each page
37  *      written to a swap partition.  It consists of many swap_map_page
38  *      structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
39  *      These structures are stored on the swap and linked together with the
40  *      help of the .next_swap member.
41  *
42  *      The swap map is created during suspend.  The swap map pages are
43  *      allocated and populated one at a time, so we only need one memory
44  *      page to set up the entire structure.
45  *
46  *      During resume we also only need to use one swap_map_page structure
47  *      at a time.
48  */
49
50 #define MAP_PAGE_ENTRIES        (PAGE_SIZE / sizeof(sector_t) - 1)
51
52 struct swap_map_page {
53         sector_t entries[MAP_PAGE_ENTRIES];
54         sector_t next_swap;
55 };
56
57 /**
58  *      The swap_map_handle structure is used for handling swap in
59  *      a file-alike way
60  */
61
62 struct swap_map_handle {
63         struct swap_map_page *cur;
64         sector_t cur_swap;
65         sector_t first_sector;
66         unsigned int k;
67 };
68
69 struct swsusp_header {
70         char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int)];
71         sector_t image;
72         unsigned int flags;     /* Flags to pass to the "boot" kernel */
73         char    orig_sig[10];
74         char    sig[10];
75 } __attribute__((packed));
76
77 static struct swsusp_header *swsusp_header;
78
79 /**
80  *      The following functions are used for tracing the allocated
81  *      swap pages, so that they can be freed in case of an error.
82  */
83
84 struct swsusp_extent {
85         struct rb_node node;
86         unsigned long start;
87         unsigned long end;
88 };
89
90 static struct rb_root swsusp_extents = RB_ROOT;
91
92 static int swsusp_extents_insert(unsigned long swap_offset)
93 {
94         struct rb_node **new = &(swsusp_extents.rb_node);
95         struct rb_node *parent = NULL;
96         struct swsusp_extent *ext;
97
98         /* Figure out where to put the new node */
99         while (*new) {
100                 ext = container_of(*new, struct swsusp_extent, node);
101                 parent = *new;
102                 if (swap_offset < ext->start) {
103                         /* Try to merge */
104                         if (swap_offset == ext->start - 1) {
105                                 ext->start--;
106                                 return 0;
107                         }
108                         new = &((*new)->rb_left);
109                 } else if (swap_offset > ext->end) {
110                         /* Try to merge */
111                         if (swap_offset == ext->end + 1) {
112                                 ext->end++;
113                                 return 0;
114                         }
115                         new = &((*new)->rb_right);
116                 } else {
117                         /* It already is in the tree */
118                         return -EINVAL;
119                 }
120         }
121         /* Add the new node and rebalance the tree. */
122         ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
123         if (!ext)
124                 return -ENOMEM;
125
126         ext->start = swap_offset;
127         ext->end = swap_offset;
128         rb_link_node(&ext->node, parent, new);
129         rb_insert_color(&ext->node, &swsusp_extents);
130         return 0;
131 }
132
133 /**
134  *      alloc_swapdev_block - allocate a swap page and register that it has
135  *      been allocated, so that it can be freed in case of an error.
136  */
137
138 sector_t alloc_swapdev_block(int swap)
139 {
140         unsigned long offset;
141
142         offset = swp_offset(get_swap_page_of_type(swap));
143         if (offset) {
144                 if (swsusp_extents_insert(offset))
145                         swap_free(swp_entry(swap, offset));
146                 else
147                         return swapdev_block(swap, offset);
148         }
149         return 0;
150 }
151
152 /**
153  *      free_all_swap_pages - free swap pages allocated for saving image data.
154  *      It also frees the extents used to register which swap entries had been
155  *      allocated.
156  */
157
158 void free_all_swap_pages(int swap)
159 {
160         struct rb_node *node;
161
162         while ((node = swsusp_extents.rb_node)) {
163                 struct swsusp_extent *ext;
164                 unsigned long offset;
165
166                 ext = container_of(node, struct swsusp_extent, node);
167                 rb_erase(node, &swsusp_extents);
168                 for (offset = ext->start; offset <= ext->end; offset++)
169                         swap_free(swp_entry(swap, offset));
170
171                 kfree(ext);
172         }
173 }
174
175 int swsusp_swap_in_use(void)
176 {
177         return (swsusp_extents.rb_node != NULL);
178 }
179
180 /*
181  * General things
182  */
183
184 static unsigned short root_swap = 0xffff;
185 struct block_device *hib_resume_bdev;
186
187 /*
188  * Saving part
189  */
190
191 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
192 {
193         int error;
194
195         hib_bio_read_page(swsusp_resume_block, swsusp_header, NULL);
196         if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
197             !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
198                 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
199                 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
200                 swsusp_header->image = handle->first_sector;
201                 swsusp_header->flags = flags;
202                 error = hib_bio_write_page(swsusp_resume_block,
203                                         swsusp_header, NULL);
204         } else {
205                 printk(KERN_ERR "PM: Swap header not found!\n");
206                 error = -ENODEV;
207         }
208         return error;
209 }
210
211 /**
212  *      swsusp_swap_check - check if the resume device is a swap device
213  *      and get its index (if so)
214  *
215  *      This is called before saving image
216  */
217 static int swsusp_swap_check(void)
218 {
219         int res;
220
221         res = swap_type_of(swsusp_resume_device, swsusp_resume_block,
222                         &hib_resume_bdev);
223         if (res < 0)
224                 return res;
225
226         root_swap = res;
227         res = blkdev_get(hib_resume_bdev, FMODE_WRITE);
228         if (res)
229                 return res;
230
231         res = set_blocksize(hib_resume_bdev, PAGE_SIZE);
232         if (res < 0)
233                 blkdev_put(hib_resume_bdev, FMODE_WRITE);
234
235         return res;
236 }
237
238 /**
239  *      write_page - Write one page to given swap location.
240  *      @buf:           Address we're writing.
241  *      @offset:        Offset of the swap page we're writing to.
242  *      @bio_chain:     Link the next write BIO here
243  */
244
245 static int write_page(void *buf, sector_t offset, struct bio **bio_chain)
246 {
247         void *src;
248
249         if (!offset)
250                 return -ENOSPC;
251
252         if (bio_chain) {
253                 src = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
254                 if (src) {
255                         copy_page(src, buf);
256                 } else {
257                         WARN_ON_ONCE(1);
258                         bio_chain = NULL;       /* Go synchronous */
259                         src = buf;
260                 }
261         } else {
262                 src = buf;
263         }
264         return hib_bio_write_page(offset, src, bio_chain);
265 }
266
267 static void release_swap_writer(struct swap_map_handle *handle)
268 {
269         if (handle->cur)
270                 free_page((unsigned long)handle->cur);
271         handle->cur = NULL;
272 }
273
274 static int get_swap_writer(struct swap_map_handle *handle)
275 {
276         int ret;
277
278         ret = swsusp_swap_check();
279         if (ret) {
280                 if (ret != -ENOSPC)
281                         printk(KERN_ERR "PM: Cannot find swap device, try "
282                                         "swapon -a.\n");
283                 return ret;
284         }
285         handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
286         if (!handle->cur) {
287                 ret = -ENOMEM;
288                 goto err_close;
289         }
290         handle->cur_swap = alloc_swapdev_block(root_swap);
291         if (!handle->cur_swap) {
292                 ret = -ENOSPC;
293                 goto err_rel;
294         }
295         handle->k = 0;
296         handle->first_sector = handle->cur_swap;
297         return 0;
298 err_rel:
299         release_swap_writer(handle);
300 err_close:
301         swsusp_close(FMODE_WRITE);
302         return ret;
303 }
304
305 static int swap_write_page(struct swap_map_handle *handle, void *buf,
306                                 struct bio **bio_chain)
307 {
308         int error = 0;
309         sector_t offset;
310
311         if (!handle->cur)
312                 return -EINVAL;
313         offset = alloc_swapdev_block(root_swap);
314         error = write_page(buf, offset, bio_chain);
315         if (error)
316                 return error;
317         handle->cur->entries[handle->k++] = offset;
318         if (handle->k >= MAP_PAGE_ENTRIES) {
319                 error = hib_wait_on_bio_chain(bio_chain);
320                 if (error)
321                         goto out;
322                 offset = alloc_swapdev_block(root_swap);
323                 if (!offset)
324                         return -ENOSPC;
325                 handle->cur->next_swap = offset;
326                 error = write_page(handle->cur, handle->cur_swap, NULL);
327                 if (error)
328                         goto out;
329                 clear_page(handle->cur);
330                 handle->cur_swap = offset;
331                 handle->k = 0;
332         }
333  out:
334         return error;
335 }
336
337 static int flush_swap_writer(struct swap_map_handle *handle)
338 {
339         if (handle->cur && handle->cur_swap)
340                 return write_page(handle->cur, handle->cur_swap, NULL);
341         else
342                 return -EINVAL;
343 }
344
345 static int swap_writer_finish(struct swap_map_handle *handle,
346                 unsigned int flags, int error)
347 {
348         if (!error) {
349                 flush_swap_writer(handle);
350                 printk(KERN_INFO "PM: S");
351                 error = mark_swapfiles(handle, flags);
352                 printk("|\n");
353         }
354
355         if (error)
356                 free_all_swap_pages(root_swap);
357         release_swap_writer(handle);
358         swsusp_close(FMODE_WRITE);
359
360         return error;
361 }
362
363 /* We need to remember how much compressed data we need to read. */
364 #define LZO_HEADER      sizeof(size_t)
365
366 /* Number of pages/bytes we'll compress at one time. */
367 #define LZO_UNC_PAGES   32
368 #define LZO_UNC_SIZE    (LZO_UNC_PAGES * PAGE_SIZE)
369
370 /* Number of pages/bytes we need for compressed data (worst case). */
371 #define LZO_CMP_PAGES   DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
372                                      LZO_HEADER, PAGE_SIZE)
373 #define LZO_CMP_SIZE    (LZO_CMP_PAGES * PAGE_SIZE)
374
375 /**
376  *      save_image - save the suspend image data
377  */
378
379 static int save_image(struct swap_map_handle *handle,
380                       struct snapshot_handle *snapshot,
381                       unsigned int nr_to_write)
382 {
383         unsigned int m;
384         int ret;
385         int nr_pages;
386         int err2;
387         struct bio *bio;
388         struct timeval start;
389         struct timeval stop;
390
391         printk(KERN_INFO "PM: Saving image data pages (%u pages) ...     ",
392                 nr_to_write);
393         m = nr_to_write / 100;
394         if (!m)
395                 m = 1;
396         nr_pages = 0;
397         bio = NULL;
398         do_gettimeofday(&start);
399         while (1) {
400                 ret = snapshot_read_next(snapshot);
401                 if (ret <= 0)
402                         break;
403                 ret = swap_write_page(handle, data_of(*snapshot), &bio);
404                 if (ret)
405                         break;
406                 if (!(nr_pages % m))
407                         printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m);
408                 nr_pages++;
409         }
410         err2 = hib_wait_on_bio_chain(&bio);
411         do_gettimeofday(&stop);
412         if (!ret)
413                 ret = err2;
414         if (!ret)
415                 printk(KERN_CONT "\b\b\b\bdone\n");
416         else
417                 printk(KERN_CONT "\n");
418         swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
419         return ret;
420 }
421
422
423 /**
424  * save_image_lzo - Save the suspend image data compressed with LZO.
425  * @handle: Swap mam handle to use for saving the image.
426  * @snapshot: Image to read data from.
427  * @nr_to_write: Number of pages to save.
428  */
429 static int save_image_lzo(struct swap_map_handle *handle,
430                           struct snapshot_handle *snapshot,
431                           unsigned int nr_to_write)
432 {
433         unsigned int m;
434         int ret = 0;
435         int nr_pages;
436         int err2;
437         struct bio *bio;
438         struct timeval start;
439         struct timeval stop;
440         size_t off, unc_len, cmp_len;
441         unsigned char *unc, *cmp, *wrk, *page;
442
443         page = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
444         if (!page) {
445                 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
446                 return -ENOMEM;
447         }
448
449         wrk = vmalloc(LZO1X_1_MEM_COMPRESS);
450         if (!wrk) {
451                 printk(KERN_ERR "PM: Failed to allocate LZO workspace\n");
452                 free_page((unsigned long)page);
453                 return -ENOMEM;
454         }
455
456         unc = vmalloc(LZO_UNC_SIZE);
457         if (!unc) {
458                 printk(KERN_ERR "PM: Failed to allocate LZO uncompressed\n");
459                 vfree(wrk);
460                 free_page((unsigned long)page);
461                 return -ENOMEM;
462         }
463
464         cmp = vmalloc(LZO_CMP_SIZE);
465         if (!cmp) {
466                 printk(KERN_ERR "PM: Failed to allocate LZO compressed\n");
467                 vfree(unc);
468                 vfree(wrk);
469                 free_page((unsigned long)page);
470                 return -ENOMEM;
471         }
472
473         printk(KERN_INFO
474                 "PM: Compressing and saving image data (%u pages) ...     ",
475                 nr_to_write);
476         m = nr_to_write / 100;
477         if (!m)
478                 m = 1;
479         nr_pages = 0;
480         bio = NULL;
481         do_gettimeofday(&start);
482         for (;;) {
483                 for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) {
484                         ret = snapshot_read_next(snapshot);
485                         if (ret < 0)
486                                 goto out_finish;
487
488                         if (!ret)
489                                 break;
490
491                         memcpy(unc + off, data_of(*snapshot), PAGE_SIZE);
492
493                         if (!(nr_pages % m))
494                                 printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m);
495                         nr_pages++;
496                 }
497
498                 if (!off)
499                         break;
500
501                 unc_len = off;
502                 ret = lzo1x_1_compress(unc, unc_len,
503                                        cmp + LZO_HEADER, &cmp_len, wrk);
504                 if (ret < 0) {
505                         printk(KERN_ERR "PM: LZO compression failed\n");
506                         break;
507                 }
508
509                 if (unlikely(!cmp_len ||
510                              cmp_len > lzo1x_worst_compress(unc_len))) {
511                         printk(KERN_ERR "PM: Invalid LZO compressed length\n");
512                         ret = -1;
513                         break;
514                 }
515
516                 *(size_t *)cmp = cmp_len;
517
518                 /*
519                  * Given we are writing one page at a time to disk, we copy
520                  * that much from the buffer, although the last bit will likely
521                  * be smaller than full page. This is OK - we saved the length
522                  * of the compressed data, so any garbage at the end will be
523                  * discarded when we read it.
524                  */
525                 for (off = 0; off < LZO_HEADER + cmp_len; off += PAGE_SIZE) {
526                         memcpy(page, cmp + off, PAGE_SIZE);
527
528                         ret = swap_write_page(handle, page, &bio);
529                         if (ret)
530                                 goto out_finish;
531                 }
532         }
533
534 out_finish:
535         err2 = hib_wait_on_bio_chain(&bio);
536         do_gettimeofday(&stop);
537         if (!ret)
538                 ret = err2;
539         if (!ret)
540                 printk(KERN_CONT "\b\b\b\bdone\n");
541         else
542                 printk(KERN_CONT "\n");
543         swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
544
545         vfree(cmp);
546         vfree(unc);
547         vfree(wrk);
548         free_page((unsigned long)page);
549
550         return ret;
551 }
552
553 /**
554  *      enough_swap - Make sure we have enough swap to save the image.
555  *
556  *      Returns TRUE or FALSE after checking the total amount of swap
557  *      space avaiable from the resume partition.
558  */
559
560 static int enough_swap(unsigned int nr_pages, unsigned int flags)
561 {
562         unsigned int free_swap = count_swap_pages(root_swap, 1);
563         unsigned int required;
564
565         pr_debug("PM: Free swap pages: %u\n", free_swap);
566
567         required = PAGES_FOR_IO + ((flags & SF_NOCOMPRESS_MODE) ?
568                 nr_pages : (nr_pages * LZO_CMP_PAGES) / LZO_UNC_PAGES + 1);
569         return free_swap > required;
570 }
571
572 /**
573  *      swsusp_write - Write entire image and metadata.
574  *      @flags: flags to pass to the "boot" kernel in the image header
575  *
576  *      It is important _NOT_ to umount filesystems at this point. We want
577  *      them synced (in case something goes wrong) but we DO not want to mark
578  *      filesystem clean: it is not. (And it does not matter, if we resume
579  *      correctly, we'll mark system clean, anyway.)
580  */
581
582 int swsusp_write(unsigned int flags)
583 {
584         struct swap_map_handle handle;
585         struct snapshot_handle snapshot;
586         struct swsusp_info *header;
587         unsigned long pages;
588         int error;
589
590         pages = snapshot_get_image_size();
591         error = get_swap_writer(&handle);
592         if (error) {
593                 printk(KERN_ERR "PM: Cannot get swap writer\n");
594                 return error;
595         }
596         if (!enough_swap(pages, flags)) {
597                 printk(KERN_ERR "PM: Not enough free swap\n");
598                 error = -ENOSPC;
599                 goto out_finish;
600         }
601         memset(&snapshot, 0, sizeof(struct snapshot_handle));
602         error = snapshot_read_next(&snapshot);
603         if (error < PAGE_SIZE) {
604                 if (error >= 0)
605                         error = -EFAULT;
606
607                 goto out_finish;
608         }
609         header = (struct swsusp_info *)data_of(snapshot);
610         error = swap_write_page(&handle, header, NULL);
611         if (!error) {
612                 error = (flags & SF_NOCOMPRESS_MODE) ?
613                         save_image(&handle, &snapshot, pages - 1) :
614                         save_image_lzo(&handle, &snapshot, pages - 1);
615         }
616 out_finish:
617         error = swap_writer_finish(&handle, flags, error);
618         return error;
619 }
620
621 /**
622  *      The following functions allow us to read data using a swap map
623  *      in a file-alike way
624  */
625
626 static void release_swap_reader(struct swap_map_handle *handle)
627 {
628         if (handle->cur)
629                 free_page((unsigned long)handle->cur);
630         handle->cur = NULL;
631 }
632
633 static int get_swap_reader(struct swap_map_handle *handle,
634                 unsigned int *flags_p)
635 {
636         int error;
637
638         *flags_p = swsusp_header->flags;
639
640         if (!swsusp_header->image) /* how can this happen? */
641                 return -EINVAL;
642
643         handle->cur = (struct swap_map_page *)get_zeroed_page(__GFP_WAIT | __GFP_HIGH);
644         if (!handle->cur)
645                 return -ENOMEM;
646
647         error = hib_bio_read_page(swsusp_header->image, handle->cur, NULL);
648         if (error) {
649                 release_swap_reader(handle);
650                 return error;
651         }
652         handle->k = 0;
653         return 0;
654 }
655
656 static int swap_read_page(struct swap_map_handle *handle, void *buf,
657                                 struct bio **bio_chain)
658 {
659         sector_t offset;
660         int error;
661
662         if (!handle->cur)
663                 return -EINVAL;
664         offset = handle->cur->entries[handle->k];
665         if (!offset)
666                 return -EFAULT;
667         error = hib_bio_read_page(offset, buf, bio_chain);
668         if (error)
669                 return error;
670         if (++handle->k >= MAP_PAGE_ENTRIES) {
671                 error = hib_wait_on_bio_chain(bio_chain);
672                 handle->k = 0;
673                 offset = handle->cur->next_swap;
674                 if (!offset)
675                         release_swap_reader(handle);
676                 else if (!error)
677                         error = hib_bio_read_page(offset, handle->cur, NULL);
678         }
679         return error;
680 }
681
682 static int swap_reader_finish(struct swap_map_handle *handle)
683 {
684         release_swap_reader(handle);
685
686         return 0;
687 }
688
689 /**
690  *      load_image - load the image using the swap map handle
691  *      @handle and the snapshot handle @snapshot
692  *      (assume there are @nr_pages pages to load)
693  */
694
695 static int load_image(struct swap_map_handle *handle,
696                       struct snapshot_handle *snapshot,
697                       unsigned int nr_to_read)
698 {
699         unsigned int m;
700         int error = 0;
701         struct timeval start;
702         struct timeval stop;
703         struct bio *bio;
704         int err2;
705         unsigned nr_pages;
706
707         printk(KERN_INFO "PM: Loading image data pages (%u pages) ...     ",
708                 nr_to_read);
709         m = nr_to_read / 100;
710         if (!m)
711                 m = 1;
712         nr_pages = 0;
713         bio = NULL;
714         do_gettimeofday(&start);
715         for ( ; ; ) {
716                 error = snapshot_write_next(snapshot);
717                 if (error <= 0)
718                         break;
719                 error = swap_read_page(handle, data_of(*snapshot), &bio);
720                 if (error)
721                         break;
722                 if (snapshot->sync_read)
723                         error = hib_wait_on_bio_chain(&bio);
724                 if (error)
725                         break;
726                 if (!(nr_pages % m))
727                         printk("\b\b\b\b%3d%%", nr_pages / m);
728                 nr_pages++;
729         }
730         err2 = hib_wait_on_bio_chain(&bio);
731         do_gettimeofday(&stop);
732         if (!error)
733                 error = err2;
734         if (!error) {
735                 printk("\b\b\b\bdone\n");
736                 snapshot_write_finalize(snapshot);
737                 if (!snapshot_image_loaded(snapshot))
738                         error = -ENODATA;
739         } else
740                 printk("\n");
741         swsusp_show_speed(&start, &stop, nr_to_read, "Read");
742         return error;
743 }
744
745 /**
746  * load_image_lzo - Load compressed image data and decompress them with LZO.
747  * @handle: Swap map handle to use for loading data.
748  * @snapshot: Image to copy uncompressed data into.
749  * @nr_to_read: Number of pages to load.
750  */
751 static int load_image_lzo(struct swap_map_handle *handle,
752                           struct snapshot_handle *snapshot,
753                           unsigned int nr_to_read)
754 {
755         unsigned int m;
756         int error = 0;
757         struct bio *bio;
758         struct timeval start;
759         struct timeval stop;
760         unsigned nr_pages;
761         size_t i, off, unc_len, cmp_len;
762         unsigned char *unc, *cmp, *page[LZO_CMP_PAGES];
763
764         for (i = 0; i < LZO_CMP_PAGES; i++) {
765                 page[i] = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
766                 if (!page[i]) {
767                         printk(KERN_ERR "PM: Failed to allocate LZO page\n");
768
769                         while (i)
770                                 free_page((unsigned long)page[--i]);
771
772                         return -ENOMEM;
773                 }
774         }
775
776         unc = vmalloc(LZO_UNC_SIZE);
777         if (!unc) {
778                 printk(KERN_ERR "PM: Failed to allocate LZO uncompressed\n");
779
780                 for (i = 0; i < LZO_CMP_PAGES; i++)
781                         free_page((unsigned long)page[i]);
782
783                 return -ENOMEM;
784         }
785
786         cmp = vmalloc(LZO_CMP_SIZE);
787         if (!cmp) {
788                 printk(KERN_ERR "PM: Failed to allocate LZO compressed\n");
789
790                 vfree(unc);
791                 for (i = 0; i < LZO_CMP_PAGES; i++)
792                         free_page((unsigned long)page[i]);
793
794                 return -ENOMEM;
795         }
796
797         printk(KERN_INFO
798                 "PM: Loading and decompressing image data (%u pages) ...     ",
799                 nr_to_read);
800         m = nr_to_read / 100;
801         if (!m)
802                 m = 1;
803         nr_pages = 0;
804         bio = NULL;
805         do_gettimeofday(&start);
806
807         error = snapshot_write_next(snapshot);
808         if (error <= 0)
809                 goto out_finish;
810
811         for (;;) {
812                 error = swap_read_page(handle, page[0], NULL); /* sync */
813                 if (error)
814                         break;
815
816                 cmp_len = *(size_t *)page[0];
817                 if (unlikely(!cmp_len ||
818                              cmp_len > lzo1x_worst_compress(LZO_UNC_SIZE))) {
819                         printk(KERN_ERR "PM: Invalid LZO compressed length\n");
820                         error = -1;
821                         break;
822                 }
823
824                 for (off = PAGE_SIZE, i = 1;
825                      off < LZO_HEADER + cmp_len; off += PAGE_SIZE, i++) {
826                         error = swap_read_page(handle, page[i], &bio);
827                         if (error)
828                                 goto out_finish;
829                 }
830
831                 error = hib_wait_on_bio_chain(&bio); /* need all data now */
832                 if (error)
833                         goto out_finish;
834
835                 for (off = 0, i = 0;
836                      off < LZO_HEADER + cmp_len; off += PAGE_SIZE, i++) {
837                         memcpy(cmp + off, page[i], PAGE_SIZE);
838                 }
839
840                 unc_len = LZO_UNC_SIZE;
841                 error = lzo1x_decompress_safe(cmp + LZO_HEADER, cmp_len,
842                                               unc, &unc_len);
843                 if (error < 0) {
844                         printk(KERN_ERR "PM: LZO decompression failed\n");
845                         break;
846                 }
847
848                 if (unlikely(!unc_len ||
849                              unc_len > LZO_UNC_SIZE ||
850                              unc_len & (PAGE_SIZE - 1))) {
851                         printk(KERN_ERR "PM: Invalid LZO uncompressed length\n");
852                         error = -1;
853                         break;
854                 }
855
856                 for (off = 0; off < unc_len; off += PAGE_SIZE) {
857                         memcpy(data_of(*snapshot), unc + off, PAGE_SIZE);
858
859                         if (!(nr_pages % m))
860                                 printk("\b\b\b\b%3d%%", nr_pages / m);
861                         nr_pages++;
862
863                         error = snapshot_write_next(snapshot);
864                         if (error <= 0)
865                                 goto out_finish;
866                 }
867         }
868
869 out_finish:
870         do_gettimeofday(&stop);
871         if (!error) {
872                 printk("\b\b\b\bdone\n");
873                 snapshot_write_finalize(snapshot);
874                 if (!snapshot_image_loaded(snapshot))
875                         error = -ENODATA;
876         } else
877                 printk("\n");
878         swsusp_show_speed(&start, &stop, nr_to_read, "Read");
879
880         vfree(cmp);
881         vfree(unc);
882         for (i = 0; i < LZO_CMP_PAGES; i++)
883                 free_page((unsigned long)page[i]);
884
885         return error;
886 }
887
888 /**
889  *      swsusp_read - read the hibernation image.
890  *      @flags_p: flags passed by the "frozen" kernel in the image header should
891  *                be written into this memeory location
892  */
893
894 int swsusp_read(unsigned int *flags_p)
895 {
896         int error;
897         struct swap_map_handle handle;
898         struct snapshot_handle snapshot;
899         struct swsusp_info *header;
900
901         memset(&snapshot, 0, sizeof(struct snapshot_handle));
902         error = snapshot_write_next(&snapshot);
903         if (error < PAGE_SIZE)
904                 return error < 0 ? error : -EFAULT;
905         header = (struct swsusp_info *)data_of(snapshot);
906         error = get_swap_reader(&handle, flags_p);
907         if (error)
908                 goto end;
909         if (!error)
910                 error = swap_read_page(&handle, header, NULL);
911         if (!error) {
912                 error = (*flags_p & SF_NOCOMPRESS_MODE) ?
913                         load_image(&handle, &snapshot, header->pages - 1) :
914                         load_image_lzo(&handle, &snapshot, header->pages - 1);
915         }
916         swap_reader_finish(&handle);
917 end:
918         if (!error)
919                 pr_debug("PM: Image successfully loaded\n");
920         else
921                 pr_debug("PM: Error %d resuming\n", error);
922         return error;
923 }
924
925 /**
926  *      swsusp_check - Check for swsusp signature in the resume device
927  */
928
929 int swsusp_check(void)
930 {
931         int error;
932
933         hib_resume_bdev = open_by_devnum(swsusp_resume_device, FMODE_READ);
934         if (!IS_ERR(hib_resume_bdev)) {
935                 set_blocksize(hib_resume_bdev, PAGE_SIZE);
936                 clear_page(swsusp_header);
937                 error = hib_bio_read_page(swsusp_resume_block,
938                                         swsusp_header, NULL);
939                 if (error)
940                         goto put;
941
942                 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
943                         memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
944                         /* Reset swap signature now */
945                         error = hib_bio_write_page(swsusp_resume_block,
946                                                 swsusp_header, NULL);
947                 } else {
948                         error = -EINVAL;
949                 }
950
951 put:
952                 if (error)
953                         blkdev_put(hib_resume_bdev, FMODE_READ);
954                 else
955                         pr_debug("PM: Image signature found, resuming\n");
956         } else {
957                 error = PTR_ERR(hib_resume_bdev);
958         }
959
960         if (error)
961                 pr_debug("PM: Image not found (code %d)\n", error);
962
963         return error;
964 }
965
966 /**
967  *      swsusp_close - close swap device.
968  */
969
970 void swsusp_close(fmode_t mode)
971 {
972         if (IS_ERR(hib_resume_bdev)) {
973                 pr_debug("PM: Image device not initialised\n");
974                 return;
975         }
976
977         blkdev_put(hib_resume_bdev, mode);
978 }
979
980 static int swsusp_header_init(void)
981 {
982         swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
983         if (!swsusp_header)
984                 panic("Could not allocate memory for swsusp_header\n");
985         return 0;
986 }
987
988 core_initcall(swsusp_header_init);