]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - drivers/staging/zram/zram_drv.c
8c3c057aa8478fb2a436edfd571843513544ca71
[~shefty/rdma-dev.git] / drivers / staging / zram / zram_drv.c
1 /*
2  * Compressed RAM block device
3  *
4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
5  *
6  * This code is released using a dual license strategy: BSD/GPL
7  * You can choose the licence that better fits your requirements.
8  *
9  * Released under the terms of 3-clause BSD License
10  * Released under the terms of GNU General Public License Version 2.0
11  *
12  * Project home: http://compcache.googlecode.com
13  */
14
15 #define KMSG_COMPONENT "zram"
16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/bitops.h>
22 #include <linux/blkdev.h>
23 #include <linux/buffer_head.h>
24 #include <linux/device.h>
25 #include <linux/genhd.h>
26 #include <linux/highmem.h>
27 #include <linux/slab.h>
28 #include <linux/lzo.h>
29 #include <linux/string.h>
30 #include <linux/vmalloc.h>
31
32 #include "zram_drv.h"
33
34 /* Globals */
35 static int zram_major;
36 struct zram *devices;
37
38 /* Module params (documentation at end) */
39 unsigned int num_devices;
40
41 static void zram_stat_inc(u32 *v)
42 {
43         *v = *v + 1;
44 }
45
46 static void zram_stat_dec(u32 *v)
47 {
48         *v = *v - 1;
49 }
50
51 static void zram_stat64_add(struct zram *zram, u64 *v, u64 inc)
52 {
53         spin_lock(&zram->stat64_lock);
54         *v = *v + inc;
55         spin_unlock(&zram->stat64_lock);
56 }
57
58 static void zram_stat64_sub(struct zram *zram, u64 *v, u64 dec)
59 {
60         spin_lock(&zram->stat64_lock);
61         *v = *v - dec;
62         spin_unlock(&zram->stat64_lock);
63 }
64
65 static void zram_stat64_inc(struct zram *zram, u64 *v)
66 {
67         zram_stat64_add(zram, v, 1);
68 }
69
70 static int zram_test_flag(struct zram *zram, u32 index,
71                         enum zram_pageflags flag)
72 {
73         return zram->table[index].flags & BIT(flag);
74 }
75
76 static void zram_set_flag(struct zram *zram, u32 index,
77                         enum zram_pageflags flag)
78 {
79         zram->table[index].flags |= BIT(flag);
80 }
81
82 static void zram_clear_flag(struct zram *zram, u32 index,
83                         enum zram_pageflags flag)
84 {
85         zram->table[index].flags &= ~BIT(flag);
86 }
87
88 static int page_zero_filled(void *ptr)
89 {
90         unsigned int pos;
91         unsigned long *page;
92
93         page = (unsigned long *)ptr;
94
95         for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
96                 if (page[pos])
97                         return 0;
98         }
99
100         return 1;
101 }
102
103 static void zram_set_disksize(struct zram *zram, size_t totalram_bytes)
104 {
105         if (!zram->disksize) {
106                 pr_info(
107                 "disk size not provided. You can use disksize_kb module "
108                 "param to specify size.\nUsing default: (%u%% of RAM).\n",
109                 default_disksize_perc_ram
110                 );
111                 zram->disksize = default_disksize_perc_ram *
112                                         (totalram_bytes / 100);
113         }
114
115         if (zram->disksize > 2 * (totalram_bytes)) {
116                 pr_info(
117                 "There is little point creating a zram of greater than "
118                 "twice the size of memory since we expect a 2:1 compression "
119                 "ratio. Note that zram uses about 0.1%% of the size of "
120                 "the disk when not in use so a huge zram is "
121                 "wasteful.\n"
122                 "\tMemory Size: %zu kB\n"
123                 "\tSize you selected: %llu kB\n"
124                 "Continuing anyway ...\n",
125                 totalram_bytes >> 10, zram->disksize
126                 );
127         }
128
129         zram->disksize &= PAGE_MASK;
130 }
131
132 static void zram_free_page(struct zram *zram, size_t index)
133 {
134         u32 clen;
135         void *obj;
136
137         struct page *page = zram->table[index].page;
138         u32 offset = zram->table[index].offset;
139
140         if (unlikely(!page)) {
141                 /*
142                  * No memory is allocated for zero filled pages.
143                  * Simply clear zero page flag.
144                  */
145                 if (zram_test_flag(zram, index, ZRAM_ZERO)) {
146                         zram_clear_flag(zram, index, ZRAM_ZERO);
147                         zram_stat_dec(&zram->stats.pages_zero);
148                 }
149                 return;
150         }
151
152         if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
153                 clen = PAGE_SIZE;
154                 __free_page(page);
155                 zram_clear_flag(zram, index, ZRAM_UNCOMPRESSED);
156                 zram_stat_dec(&zram->stats.pages_expand);
157                 goto out;
158         }
159
160         obj = kmap_atomic(page, KM_USER0) + offset;
161         clen = xv_get_object_size(obj) - sizeof(struct zobj_header);
162         kunmap_atomic(obj, KM_USER0);
163
164         xv_free(zram->mem_pool, page, offset);
165         if (clen <= PAGE_SIZE / 2)
166                 zram_stat_dec(&zram->stats.good_compress);
167
168 out:
169         zram_stat64_sub(zram, &zram->stats.compr_size, clen);
170         zram_stat_dec(&zram->stats.pages_stored);
171
172         zram->table[index].page = NULL;
173         zram->table[index].offset = 0;
174 }
175
176 static void handle_zero_page(struct page *page)
177 {
178         void *user_mem;
179
180         user_mem = kmap_atomic(page, KM_USER0);
181         memset(user_mem, 0, PAGE_SIZE);
182         kunmap_atomic(user_mem, KM_USER0);
183
184         flush_dcache_page(page);
185 }
186
187 static void handle_uncompressed_page(struct zram *zram,
188                                 struct page *page, u32 index)
189 {
190         unsigned char *user_mem, *cmem;
191
192         user_mem = kmap_atomic(page, KM_USER0);
193         cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
194                         zram->table[index].offset;
195
196         memcpy(user_mem, cmem, PAGE_SIZE);
197         kunmap_atomic(user_mem, KM_USER0);
198         kunmap_atomic(cmem, KM_USER1);
199
200         flush_dcache_page(page);
201 }
202
203 static int zram_read(struct zram *zram, struct bio *bio)
204 {
205
206         int i;
207         u32 index;
208         struct bio_vec *bvec;
209
210         if (unlikely(!zram->init_done)) {
211                 set_bit(BIO_UPTODATE, &bio->bi_flags);
212                 bio_endio(bio, 0);
213                 return 0;
214         }
215
216         zram_stat64_inc(zram, &zram->stats.num_reads);
217         index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
218
219         bio_for_each_segment(bvec, bio, i) {
220                 int ret;
221                 size_t clen;
222                 struct page *page;
223                 struct zobj_header *zheader;
224                 unsigned char *user_mem, *cmem;
225
226                 page = bvec->bv_page;
227
228                 if (zram_test_flag(zram, index, ZRAM_ZERO)) {
229                         handle_zero_page(page);
230                         continue;
231                 }
232
233                 /* Requested page is not present in compressed area */
234                 if (unlikely(!zram->table[index].page)) {
235                         pr_debug("Read before write: sector=%lu, size=%u",
236                                 (ulong)(bio->bi_sector), bio->bi_size);
237                         /* Do nothing */
238                         continue;
239                 }
240
241                 /* Page is stored uncompressed since it's incompressible */
242                 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
243                         handle_uncompressed_page(zram, page, index);
244                         continue;
245                 }
246
247                 user_mem = kmap_atomic(page, KM_USER0);
248                 clen = PAGE_SIZE;
249
250                 cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
251                                 zram->table[index].offset;
252
253                 ret = lzo1x_decompress_safe(
254                         cmem + sizeof(*zheader),
255                         xv_get_object_size(cmem) - sizeof(*zheader),
256                         user_mem, &clen);
257
258                 kunmap_atomic(user_mem, KM_USER0);
259                 kunmap_atomic(cmem, KM_USER1);
260
261                 /* Should NEVER happen. Return bio error if it does. */
262                 if (unlikely(ret != LZO_E_OK)) {
263                         pr_err("Decompression failed! err=%d, page=%u\n",
264                                 ret, index);
265                         zram_stat64_inc(zram, &zram->stats.failed_reads);
266                         goto out;
267                 }
268
269                 flush_dcache_page(page);
270                 index++;
271         }
272
273         set_bit(BIO_UPTODATE, &bio->bi_flags);
274         bio_endio(bio, 0);
275         return 0;
276
277 out:
278         bio_io_error(bio);
279         return 0;
280 }
281
282 static int zram_write(struct zram *zram, struct bio *bio)
283 {
284         int i, ret;
285         u32 index;
286         struct bio_vec *bvec;
287
288         if (unlikely(!zram->init_done)) {
289                 ret = zram_init_device(zram);
290                 if (ret)
291                         goto out;
292         }
293
294         zram_stat64_inc(zram, &zram->stats.num_writes);
295         index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
296
297         bio_for_each_segment(bvec, bio, i) {
298                 u32 offset;
299                 size_t clen;
300                 struct zobj_header *zheader;
301                 struct page *page, *page_store;
302                 unsigned char *user_mem, *cmem, *src;
303
304                 page = bvec->bv_page;
305                 src = zram->compress_buffer;
306
307                 /*
308                  * System overwrites unused sectors. Free memory associated
309                  * with this sector now.
310                  */
311                 if (zram->table[index].page ||
312                                 zram_test_flag(zram, index, ZRAM_ZERO))
313                         zram_free_page(zram, index);
314
315                 mutex_lock(&zram->lock);
316
317                 user_mem = kmap_atomic(page, KM_USER0);
318                 if (page_zero_filled(user_mem)) {
319                         kunmap_atomic(user_mem, KM_USER0);
320                         mutex_unlock(&zram->lock);
321                         zram_stat_inc(&zram->stats.pages_zero);
322                         zram_set_flag(zram, index, ZRAM_ZERO);
323                         continue;
324                 }
325
326                 ret = lzo1x_1_compress(user_mem, PAGE_SIZE, src, &clen,
327                                         zram->compress_workmem);
328
329                 kunmap_atomic(user_mem, KM_USER0);
330
331                 if (unlikely(ret != LZO_E_OK)) {
332                         mutex_unlock(&zram->lock);
333                         pr_err("Compression failed! err=%d\n", ret);
334                         zram_stat64_inc(zram, &zram->stats.failed_writes);
335                         goto out;
336                 }
337
338                 /*
339                  * Page is incompressible. Store it as-is (uncompressed)
340                  * since we do not want to return too many disk write
341                  * errors which has side effect of hanging the system.
342                  */
343                 if (unlikely(clen > max_zpage_size)) {
344                         clen = PAGE_SIZE;
345                         page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
346                         if (unlikely(!page_store)) {
347                                 mutex_unlock(&zram->lock);
348                                 pr_info("Error allocating memory for "
349                                         "incompressible page: %u\n", index);
350                                 zram_stat64_inc(zram,
351                                         &zram->stats.failed_writes);
352                                 goto out;
353                         }
354
355                         offset = 0;
356                         zram_set_flag(zram, index, ZRAM_UNCOMPRESSED);
357                         zram_stat_inc(&zram->stats.pages_expand);
358                         zram->table[index].page = page_store;
359                         src = kmap_atomic(page, KM_USER0);
360                         goto memstore;
361                 }
362
363                 if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader),
364                                 &zram->table[index].page, &offset,
365                                 GFP_NOIO | __GFP_HIGHMEM)) {
366                         mutex_unlock(&zram->lock);
367                         pr_info("Error allocating memory for compressed "
368                                 "page: %u, size=%zu\n", index, clen);
369                         zram_stat64_inc(zram, &zram->stats.failed_writes);
370                         goto out;
371                 }
372
373 memstore:
374                 zram->table[index].offset = offset;
375
376                 cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
377                                 zram->table[index].offset;
378
379 #if 0
380                 /* Back-reference needed for memory defragmentation */
381                 if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) {
382                         zheader = (struct zobj_header *)cmem;
383                         zheader->table_idx = index;
384                         cmem += sizeof(*zheader);
385                 }
386 #endif
387
388                 memcpy(cmem, src, clen);
389
390                 kunmap_atomic(cmem, KM_USER1);
391                 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)))
392                         kunmap_atomic(src, KM_USER0);
393
394                 /* Update stats */
395                 zram_stat64_add(zram, &zram->stats.compr_size, clen);
396                 zram_stat_inc(&zram->stats.pages_stored);
397                 if (clen <= PAGE_SIZE / 2)
398                         zram_stat_inc(&zram->stats.good_compress);
399
400                 mutex_unlock(&zram->lock);
401                 index++;
402         }
403
404         set_bit(BIO_UPTODATE, &bio->bi_flags);
405         bio_endio(bio, 0);
406         return 0;
407
408 out:
409         bio_io_error(bio);
410         return 0;
411 }
412
413 /*
414  * Check if request is within bounds and page aligned.
415  */
416 static inline int valid_io_request(struct zram *zram, struct bio *bio)
417 {
418         if (unlikely(
419                 (bio->bi_sector >= (zram->disksize >> SECTOR_SHIFT)) ||
420                 (bio->bi_sector & (SECTORS_PER_PAGE - 1)) ||
421                 (bio->bi_size & (PAGE_SIZE - 1)))) {
422
423                 return 0;
424         }
425
426         /* I/O request is valid */
427         return 1;
428 }
429
430 /*
431  * Handler function for all zram I/O requests.
432  */
433 static int zram_make_request(struct request_queue *queue, struct bio *bio)
434 {
435         int ret = 0;
436         struct zram *zram = queue->queuedata;
437
438         if (unlikely(!zram->init_done)) {
439                 set_bit(BIO_UPTODATE, &bio->bi_flags);
440                 bio_endio(bio, 0);
441                 return 0;
442         }
443
444         if (!valid_io_request(zram, bio)) {
445                 zram_stat64_inc(zram, &zram->stats.invalid_io);
446                 bio_io_error(bio);
447                 return 0;
448         }
449
450         switch (bio_data_dir(bio)) {
451         case READ:
452                 ret = zram_read(zram, bio);
453                 break;
454
455         case WRITE:
456                 ret = zram_write(zram, bio);
457                 break;
458         }
459
460         return ret;
461 }
462
463 void zram_reset_device(struct zram *zram)
464 {
465         size_t index;
466
467         mutex_lock(&zram->init_lock);
468         zram->init_done = 0;
469
470         /* Free various per-device buffers */
471         kfree(zram->compress_workmem);
472         free_pages((unsigned long)zram->compress_buffer, 1);
473
474         zram->compress_workmem = NULL;
475         zram->compress_buffer = NULL;
476
477         /* Free all pages that are still in this zram device */
478         for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
479                 struct page *page;
480                 u16 offset;
481
482                 page = zram->table[index].page;
483                 offset = zram->table[index].offset;
484
485                 if (!page)
486                         continue;
487
488                 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)))
489                         __free_page(page);
490                 else
491                         xv_free(zram->mem_pool, page, offset);
492         }
493
494         vfree(zram->table);
495         zram->table = NULL;
496
497         xv_destroy_pool(zram->mem_pool);
498         zram->mem_pool = NULL;
499
500         /* Reset stats */
501         memset(&zram->stats, 0, sizeof(zram->stats));
502
503         zram->disksize = 0;
504         mutex_unlock(&zram->init_lock);
505 }
506
507 int zram_init_device(struct zram *zram)
508 {
509         int ret;
510         size_t num_pages;
511
512         mutex_lock(&zram->init_lock);
513
514         if (zram->init_done) {
515                 mutex_unlock(&zram->init_lock);
516                 return 0;
517         }
518
519         zram_set_disksize(zram, totalram_pages << PAGE_SHIFT);
520
521         zram->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
522         if (!zram->compress_workmem) {
523                 pr_err("Error allocating compressor working memory!\n");
524                 ret = -ENOMEM;
525                 goto fail;
526         }
527
528         zram->compress_buffer = (void *)__get_free_pages(__GFP_ZERO, 1);
529         if (!zram->compress_buffer) {
530                 pr_err("Error allocating compressor buffer space\n");
531                 ret = -ENOMEM;
532                 goto fail;
533         }
534
535         num_pages = zram->disksize >> PAGE_SHIFT;
536         zram->table = vmalloc(num_pages * sizeof(*zram->table));
537         if (!zram->table) {
538                 pr_err("Error allocating zram address table\n");
539                 /* To prevent accessing table entries during cleanup */
540                 zram->disksize = 0;
541                 ret = -ENOMEM;
542                 goto fail;
543         }
544         memset(zram->table, 0, num_pages * sizeof(*zram->table));
545
546         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
547
548         /* zram devices sort of resembles non-rotational disks */
549         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
550
551         zram->mem_pool = xv_create_pool();
552         if (!zram->mem_pool) {
553                 pr_err("Error creating memory pool\n");
554                 ret = -ENOMEM;
555                 goto fail;
556         }
557
558         zram->init_done = 1;
559         mutex_unlock(&zram->init_lock);
560
561         pr_debug("Initialization done!\n");
562         return 0;
563
564 fail:
565         mutex_unlock(&zram->init_lock);
566         zram_reset_device(zram);
567
568         pr_err("Initialization failed: err=%d\n", ret);
569         return ret;
570 }
571
572 void zram_slot_free_notify(struct block_device *bdev, unsigned long index)
573 {
574         struct zram *zram;
575
576         zram = bdev->bd_disk->private_data;
577         zram_free_page(zram, index);
578         zram_stat64_inc(zram, &zram->stats.notify_free);
579 }
580
581 static const struct block_device_operations zram_devops = {
582         .swap_slot_free_notify = zram_slot_free_notify,
583         .owner = THIS_MODULE
584 };
585
586 static int create_device(struct zram *zram, int device_id)
587 {
588         int ret = 0;
589
590         mutex_init(&zram->lock);
591         mutex_init(&zram->init_lock);
592         spin_lock_init(&zram->stat64_lock);
593
594         zram->queue = blk_alloc_queue(GFP_KERNEL);
595         if (!zram->queue) {
596                 pr_err("Error allocating disk queue for device %d\n",
597                         device_id);
598                 ret = -ENOMEM;
599                 goto out;
600         }
601
602         blk_queue_make_request(zram->queue, zram_make_request);
603         zram->queue->queuedata = zram;
604
605          /* gendisk structure */
606         zram->disk = alloc_disk(1);
607         if (!zram->disk) {
608                 blk_cleanup_queue(zram->queue);
609                 pr_warning("Error allocating disk structure for device %d\n",
610                         device_id);
611                 ret = -ENOMEM;
612                 goto out;
613         }
614
615         zram->disk->major = zram_major;
616         zram->disk->first_minor = device_id;
617         zram->disk->fops = &zram_devops;
618         zram->disk->queue = zram->queue;
619         zram->disk->private_data = zram;
620         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
621
622         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
623         set_capacity(zram->disk, 0);
624
625         /*
626          * To ensure that we always get PAGE_SIZE aligned
627          * and n*PAGE_SIZED sized I/O requests.
628          */
629         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
630         blk_queue_logical_block_size(zram->disk->queue, PAGE_SIZE);
631         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
632         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
633
634         add_disk(zram->disk);
635
636 #ifdef CONFIG_SYSFS
637         ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
638                                 &zram_disk_attr_group);
639         if (ret < 0) {
640                 pr_warning("Error creating sysfs group");
641                 goto out;
642         }
643 #endif
644
645         zram->init_done = 0;
646
647 out:
648         return ret;
649 }
650
651 static void destroy_device(struct zram *zram)
652 {
653 #ifdef CONFIG_SYSFS
654         sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
655                         &zram_disk_attr_group);
656 #endif
657
658         if (zram->disk) {
659                 del_gendisk(zram->disk);
660                 put_disk(zram->disk);
661         }
662
663         if (zram->queue)
664                 blk_cleanup_queue(zram->queue);
665 }
666
667 static int __init zram_init(void)
668 {
669         int ret, dev_id;
670
671         if (num_devices > max_num_devices) {
672                 pr_warning("Invalid value for num_devices: %u\n",
673                                 num_devices);
674                 ret = -EINVAL;
675                 goto out;
676         }
677
678         zram_major = register_blkdev(0, "zram");
679         if (zram_major <= 0) {
680                 pr_warning("Unable to get major number\n");
681                 ret = -EBUSY;
682                 goto out;
683         }
684
685         if (!num_devices) {
686                 pr_info("num_devices not specified. Using default: 1\n");
687                 num_devices = 1;
688         }
689
690         /* Allocate the device array and initialize each one */
691         pr_info("Creating %u devices ...\n", num_devices);
692         devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
693         if (!devices) {
694                 ret = -ENOMEM;
695                 goto unregister;
696         }
697
698         for (dev_id = 0; dev_id < num_devices; dev_id++) {
699                 ret = create_device(&devices[dev_id], dev_id);
700                 if (ret)
701                         goto free_devices;
702         }
703
704         return 0;
705
706 free_devices:
707         while (dev_id)
708                 destroy_device(&devices[--dev_id]);
709         kfree(devices);
710 unregister:
711         unregister_blkdev(zram_major, "zram");
712 out:
713         return ret;
714 }
715
716 static void __exit zram_exit(void)
717 {
718         int i;
719         struct zram *zram;
720
721         for (i = 0; i < num_devices; i++) {
722                 zram = &devices[i];
723
724                 destroy_device(zram);
725                 if (zram->init_done)
726                         zram_reset_device(zram);
727         }
728
729         unregister_blkdev(zram_major, "zram");
730
731         kfree(devices);
732         pr_debug("Cleanup done!\n");
733 }
734
735 module_param(num_devices, uint, 0);
736 MODULE_PARM_DESC(num_devices, "Number of zram devices");
737
738 module_init(zram_init);
739 module_exit(zram_exit);
740
741 MODULE_LICENSE("Dual BSD/GPL");
742 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
743 MODULE_DESCRIPTION("Compressed RAM Block Device");