]> Pileus Git - ~andy/linux/blob - drivers/staging/zram/zram_drv.c
Staging: zram: make zram_read return a bio error if the device is not initialized
[~andy/linux] / drivers / staging / zram / zram_drv.c
1 /*
2  * Compressed RAM block device
3  *
4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
5  *
6  * This code is released using a dual license strategy: BSD/GPL
7  * You can choose the licence that better fits your requirements.
8  *
9  * Released under the terms of 3-clause BSD License
10  * Released under the terms of GNU General Public License Version 2.0
11  *
12  * Project home: http://compcache.googlecode.com
13  */
14
15 #define KMSG_COMPONENT "zram"
16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/bitops.h>
22 #include <linux/blkdev.h>
23 #include <linux/buffer_head.h>
24 #include <linux/device.h>
25 #include <linux/genhd.h>
26 #include <linux/highmem.h>
27 #include <linux/slab.h>
28 #include <linux/lzo.h>
29 #include <linux/string.h>
30 #include <linux/vmalloc.h>
31
32 #include "zram_drv.h"
33
34 /* Globals */
35 static int zram_major;
36 struct zram *devices;
37
38 /* Module params (documentation at end) */
39 unsigned int num_devices;
40
41 static void zram_stat_inc(u32 *v)
42 {
43         *v = *v + 1;
44 }
45
46 static void zram_stat_dec(u32 *v)
47 {
48         *v = *v - 1;
49 }
50
51 static void zram_stat64_add(struct zram *zram, u64 *v, u64 inc)
52 {
53         spin_lock(&zram->stat64_lock);
54         *v = *v + inc;
55         spin_unlock(&zram->stat64_lock);
56 }
57
58 static void zram_stat64_sub(struct zram *zram, u64 *v, u64 dec)
59 {
60         spin_lock(&zram->stat64_lock);
61         *v = *v - dec;
62         spin_unlock(&zram->stat64_lock);
63 }
64
65 static void zram_stat64_inc(struct zram *zram, u64 *v)
66 {
67         zram_stat64_add(zram, v, 1);
68 }
69
70 static int zram_test_flag(struct zram *zram, u32 index,
71                         enum zram_pageflags flag)
72 {
73         return zram->table[index].flags & BIT(flag);
74 }
75
76 static void zram_set_flag(struct zram *zram, u32 index,
77                         enum zram_pageflags flag)
78 {
79         zram->table[index].flags |= BIT(flag);
80 }
81
82 static void zram_clear_flag(struct zram *zram, u32 index,
83                         enum zram_pageflags flag)
84 {
85         zram->table[index].flags &= ~BIT(flag);
86 }
87
88 static int page_zero_filled(void *ptr)
89 {
90         unsigned int pos;
91         unsigned long *page;
92
93         page = (unsigned long *)ptr;
94
95         for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
96                 if (page[pos])
97                         return 0;
98         }
99
100         return 1;
101 }
102
103 static void zram_set_disksize(struct zram *zram, size_t totalram_bytes)
104 {
105         if (!zram->disksize) {
106                 pr_info(
107                 "disk size not provided. You can use disksize_kb module "
108                 "param to specify size.\nUsing default: (%u%% of RAM).\n",
109                 default_disksize_perc_ram
110                 );
111                 zram->disksize = default_disksize_perc_ram *
112                                         (totalram_bytes / 100);
113         }
114
115         if (zram->disksize > 2 * (totalram_bytes)) {
116                 pr_info(
117                 "There is little point creating a zram of greater than "
118                 "twice the size of memory since we expect a 2:1 compression "
119                 "ratio. Note that zram uses about 0.1%% of the size of "
120                 "the disk when not in use so a huge zram is "
121                 "wasteful.\n"
122                 "\tMemory Size: %zu kB\n"
123                 "\tSize you selected: %llu kB\n"
124                 "Continuing anyway ...\n",
125                 totalram_bytes >> 10, zram->disksize
126                 );
127         }
128
129         zram->disksize &= PAGE_MASK;
130 }
131
132 static void zram_free_page(struct zram *zram, size_t index)
133 {
134         u32 clen;
135         void *obj;
136
137         struct page *page = zram->table[index].page;
138         u32 offset = zram->table[index].offset;
139
140         if (unlikely(!page)) {
141                 /*
142                  * No memory is allocated for zero filled pages.
143                  * Simply clear zero page flag.
144                  */
145                 if (zram_test_flag(zram, index, ZRAM_ZERO)) {
146                         zram_clear_flag(zram, index, ZRAM_ZERO);
147                         zram_stat_dec(&zram->stats.pages_zero);
148                 }
149                 return;
150         }
151
152         if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
153                 clen = PAGE_SIZE;
154                 __free_page(page);
155                 zram_clear_flag(zram, index, ZRAM_UNCOMPRESSED);
156                 zram_stat_dec(&zram->stats.pages_expand);
157                 goto out;
158         }
159
160         obj = kmap_atomic(page, KM_USER0) + offset;
161         clen = xv_get_object_size(obj) - sizeof(struct zobj_header);
162         kunmap_atomic(obj, KM_USER0);
163
164         xv_free(zram->mem_pool, page, offset);
165         if (clen <= PAGE_SIZE / 2)
166                 zram_stat_dec(&zram->stats.good_compress);
167
168 out:
169         zram_stat64_sub(zram, &zram->stats.compr_size, clen);
170         zram_stat_dec(&zram->stats.pages_stored);
171
172         zram->table[index].page = NULL;
173         zram->table[index].offset = 0;
174 }
175
176 static void handle_zero_page(struct page *page)
177 {
178         void *user_mem;
179
180         user_mem = kmap_atomic(page, KM_USER0);
181         memset(user_mem, 0, PAGE_SIZE);
182         kunmap_atomic(user_mem, KM_USER0);
183
184         flush_dcache_page(page);
185 }
186
187 static void handle_uncompressed_page(struct zram *zram,
188                                 struct page *page, u32 index)
189 {
190         unsigned char *user_mem, *cmem;
191
192         user_mem = kmap_atomic(page, KM_USER0);
193         cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
194                         zram->table[index].offset;
195
196         memcpy(user_mem, cmem, PAGE_SIZE);
197         kunmap_atomic(user_mem, KM_USER0);
198         kunmap_atomic(cmem, KM_USER1);
199
200         flush_dcache_page(page);
201 }
202
203 static int zram_read(struct zram *zram, struct bio *bio)
204 {
205
206         int i;
207         u32 index;
208         struct bio_vec *bvec;
209
210         if (unlikely(!zram->init_done)) {
211                 bio_endio(bio, -ENXIO);
212                 return 0;
213         }
214
215         zram_stat64_inc(zram, &zram->stats.num_reads);
216         index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
217
218         bio_for_each_segment(bvec, bio, i) {
219                 int ret;
220                 size_t clen;
221                 struct page *page;
222                 struct zobj_header *zheader;
223                 unsigned char *user_mem, *cmem;
224
225                 page = bvec->bv_page;
226
227                 if (zram_test_flag(zram, index, ZRAM_ZERO)) {
228                         handle_zero_page(page);
229                         continue;
230                 }
231
232                 /* Requested page is not present in compressed area */
233                 if (unlikely(!zram->table[index].page)) {
234                         pr_debug("Read before write: sector=%lu, size=%u",
235                                 (ulong)(bio->bi_sector), bio->bi_size);
236                         /* Do nothing */
237                         continue;
238                 }
239
240                 /* Page is stored uncompressed since it's incompressible */
241                 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
242                         handle_uncompressed_page(zram, page, index);
243                         continue;
244                 }
245
246                 user_mem = kmap_atomic(page, KM_USER0);
247                 clen = PAGE_SIZE;
248
249                 cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
250                                 zram->table[index].offset;
251
252                 ret = lzo1x_decompress_safe(
253                         cmem + sizeof(*zheader),
254                         xv_get_object_size(cmem) - sizeof(*zheader),
255                         user_mem, &clen);
256
257                 kunmap_atomic(user_mem, KM_USER0);
258                 kunmap_atomic(cmem, KM_USER1);
259
260                 /* Should NEVER happen. Return bio error if it does. */
261                 if (unlikely(ret != LZO_E_OK)) {
262                         pr_err("Decompression failed! err=%d, page=%u\n",
263                                 ret, index);
264                         zram_stat64_inc(zram, &zram->stats.failed_reads);
265                         goto out;
266                 }
267
268                 flush_dcache_page(page);
269                 index++;
270         }
271
272         set_bit(BIO_UPTODATE, &bio->bi_flags);
273         bio_endio(bio, 0);
274         return 0;
275
276 out:
277         bio_io_error(bio);
278         return 0;
279 }
280
281 static int zram_write(struct zram *zram, struct bio *bio)
282 {
283         int i, ret;
284         u32 index;
285         struct bio_vec *bvec;
286
287         if (unlikely(!zram->init_done)) {
288                 ret = zram_init_device(zram);
289                 if (ret)
290                         goto out;
291         }
292
293         zram_stat64_inc(zram, &zram->stats.num_writes);
294         index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
295
296         bio_for_each_segment(bvec, bio, i) {
297                 u32 offset;
298                 size_t clen;
299                 struct zobj_header *zheader;
300                 struct page *page, *page_store;
301                 unsigned char *user_mem, *cmem, *src;
302
303                 page = bvec->bv_page;
304                 src = zram->compress_buffer;
305
306                 /*
307                  * System overwrites unused sectors. Free memory associated
308                  * with this sector now.
309                  */
310                 if (zram->table[index].page ||
311                                 zram_test_flag(zram, index, ZRAM_ZERO))
312                         zram_free_page(zram, index);
313
314                 mutex_lock(&zram->lock);
315
316                 user_mem = kmap_atomic(page, KM_USER0);
317                 if (page_zero_filled(user_mem)) {
318                         kunmap_atomic(user_mem, KM_USER0);
319                         mutex_unlock(&zram->lock);
320                         zram_stat_inc(&zram->stats.pages_zero);
321                         zram_set_flag(zram, index, ZRAM_ZERO);
322                         continue;
323                 }
324
325                 ret = lzo1x_1_compress(user_mem, PAGE_SIZE, src, &clen,
326                                         zram->compress_workmem);
327
328                 kunmap_atomic(user_mem, KM_USER0);
329
330                 if (unlikely(ret != LZO_E_OK)) {
331                         mutex_unlock(&zram->lock);
332                         pr_err("Compression failed! err=%d\n", ret);
333                         zram_stat64_inc(zram, &zram->stats.failed_writes);
334                         goto out;
335                 }
336
337                 /*
338                  * Page is incompressible. Store it as-is (uncompressed)
339                  * since we do not want to return too many disk write
340                  * errors which has side effect of hanging the system.
341                  */
342                 if (unlikely(clen > max_zpage_size)) {
343                         clen = PAGE_SIZE;
344                         page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
345                         if (unlikely(!page_store)) {
346                                 mutex_unlock(&zram->lock);
347                                 pr_info("Error allocating memory for "
348                                         "incompressible page: %u\n", index);
349                                 zram_stat64_inc(zram,
350                                         &zram->stats.failed_writes);
351                                 goto out;
352                         }
353
354                         offset = 0;
355                         zram_set_flag(zram, index, ZRAM_UNCOMPRESSED);
356                         zram_stat_inc(&zram->stats.pages_expand);
357                         zram->table[index].page = page_store;
358                         src = kmap_atomic(page, KM_USER0);
359                         goto memstore;
360                 }
361
362                 if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader),
363                                 &zram->table[index].page, &offset,
364                                 GFP_NOIO | __GFP_HIGHMEM)) {
365                         mutex_unlock(&zram->lock);
366                         pr_info("Error allocating memory for compressed "
367                                 "page: %u, size=%zu\n", index, clen);
368                         zram_stat64_inc(zram, &zram->stats.failed_writes);
369                         goto out;
370                 }
371
372 memstore:
373                 zram->table[index].offset = offset;
374
375                 cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
376                                 zram->table[index].offset;
377
378 #if 0
379                 /* Back-reference needed for memory defragmentation */
380                 if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) {
381                         zheader = (struct zobj_header *)cmem;
382                         zheader->table_idx = index;
383                         cmem += sizeof(*zheader);
384                 }
385 #endif
386
387                 memcpy(cmem, src, clen);
388
389                 kunmap_atomic(cmem, KM_USER1);
390                 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)))
391                         kunmap_atomic(src, KM_USER0);
392
393                 /* Update stats */
394                 zram_stat64_add(zram, &zram->stats.compr_size, clen);
395                 zram_stat_inc(&zram->stats.pages_stored);
396                 if (clen <= PAGE_SIZE / 2)
397                         zram_stat_inc(&zram->stats.good_compress);
398
399                 mutex_unlock(&zram->lock);
400                 index++;
401         }
402
403         set_bit(BIO_UPTODATE, &bio->bi_flags);
404         bio_endio(bio, 0);
405         return 0;
406
407 out:
408         bio_io_error(bio);
409         return 0;
410 }
411
412 /*
413  * Check if request is within bounds and page aligned.
414  */
415 static inline int valid_io_request(struct zram *zram, struct bio *bio)
416 {
417         if (unlikely(
418                 (bio->bi_sector >= (zram->disksize >> SECTOR_SHIFT)) ||
419                 (bio->bi_sector & (SECTORS_PER_PAGE - 1)) ||
420                 (bio->bi_size & (PAGE_SIZE - 1)))) {
421
422                 return 0;
423         }
424
425         /* I/O request is valid */
426         return 1;
427 }
428
429 /*
430  * Handler function for all zram I/O requests.
431  */
432 static int zram_make_request(struct request_queue *queue, struct bio *bio)
433 {
434         int ret = 0;
435         struct zram *zram = queue->queuedata;
436
437         if (!valid_io_request(zram, bio)) {
438                 zram_stat64_inc(zram, &zram->stats.invalid_io);
439                 bio_io_error(bio);
440                 return 0;
441         }
442
443         switch (bio_data_dir(bio)) {
444         case READ:
445                 ret = zram_read(zram, bio);
446                 break;
447
448         case WRITE:
449                 ret = zram_write(zram, bio);
450                 break;
451         }
452
453         return ret;
454 }
455
456 void zram_reset_device(struct zram *zram)
457 {
458         size_t index;
459
460         mutex_lock(&zram->init_lock);
461         zram->init_done = 0;
462
463         /* Free various per-device buffers */
464         kfree(zram->compress_workmem);
465         free_pages((unsigned long)zram->compress_buffer, 1);
466
467         zram->compress_workmem = NULL;
468         zram->compress_buffer = NULL;
469
470         /* Free all pages that are still in this zram device */
471         for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
472                 struct page *page;
473                 u16 offset;
474
475                 page = zram->table[index].page;
476                 offset = zram->table[index].offset;
477
478                 if (!page)
479                         continue;
480
481                 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)))
482                         __free_page(page);
483                 else
484                         xv_free(zram->mem_pool, page, offset);
485         }
486
487         vfree(zram->table);
488         zram->table = NULL;
489
490         xv_destroy_pool(zram->mem_pool);
491         zram->mem_pool = NULL;
492
493         /* Reset stats */
494         memset(&zram->stats, 0, sizeof(zram->stats));
495
496         zram->disksize = 0;
497         mutex_unlock(&zram->init_lock);
498 }
499
500 int zram_init_device(struct zram *zram)
501 {
502         int ret;
503         size_t num_pages;
504
505         mutex_lock(&zram->init_lock);
506
507         if (zram->init_done) {
508                 mutex_unlock(&zram->init_lock);
509                 return 0;
510         }
511
512         zram_set_disksize(zram, totalram_pages << PAGE_SHIFT);
513
514         zram->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
515         if (!zram->compress_workmem) {
516                 pr_err("Error allocating compressor working memory!\n");
517                 ret = -ENOMEM;
518                 goto fail;
519         }
520
521         zram->compress_buffer = (void *)__get_free_pages(__GFP_ZERO, 1);
522         if (!zram->compress_buffer) {
523                 pr_err("Error allocating compressor buffer space\n");
524                 ret = -ENOMEM;
525                 goto fail;
526         }
527
528         num_pages = zram->disksize >> PAGE_SHIFT;
529         zram->table = vzalloc(num_pages * sizeof(*zram->table));
530         if (!zram->table) {
531                 pr_err("Error allocating zram address table\n");
532                 /* To prevent accessing table entries during cleanup */
533                 zram->disksize = 0;
534                 ret = -ENOMEM;
535                 goto fail;
536         }
537
538         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
539
540         /* zram devices sort of resembles non-rotational disks */
541         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
542
543         zram->mem_pool = xv_create_pool();
544         if (!zram->mem_pool) {
545                 pr_err("Error creating memory pool\n");
546                 ret = -ENOMEM;
547                 goto fail;
548         }
549
550         zram->init_done = 1;
551         mutex_unlock(&zram->init_lock);
552
553         pr_debug("Initialization done!\n");
554         return 0;
555
556 fail:
557         mutex_unlock(&zram->init_lock);
558         zram_reset_device(zram);
559
560         pr_err("Initialization failed: err=%d\n", ret);
561         return ret;
562 }
563
564 void zram_slot_free_notify(struct block_device *bdev, unsigned long index)
565 {
566         struct zram *zram;
567
568         zram = bdev->bd_disk->private_data;
569         zram_free_page(zram, index);
570         zram_stat64_inc(zram, &zram->stats.notify_free);
571 }
572
573 static const struct block_device_operations zram_devops = {
574         .swap_slot_free_notify = zram_slot_free_notify,
575         .owner = THIS_MODULE
576 };
577
578 static int create_device(struct zram *zram, int device_id)
579 {
580         int ret = 0;
581
582         mutex_init(&zram->lock);
583         mutex_init(&zram->init_lock);
584         spin_lock_init(&zram->stat64_lock);
585
586         zram->queue = blk_alloc_queue(GFP_KERNEL);
587         if (!zram->queue) {
588                 pr_err("Error allocating disk queue for device %d\n",
589                         device_id);
590                 ret = -ENOMEM;
591                 goto out;
592         }
593
594         blk_queue_make_request(zram->queue, zram_make_request);
595         zram->queue->queuedata = zram;
596
597          /* gendisk structure */
598         zram->disk = alloc_disk(1);
599         if (!zram->disk) {
600                 blk_cleanup_queue(zram->queue);
601                 pr_warning("Error allocating disk structure for device %d\n",
602                         device_id);
603                 ret = -ENOMEM;
604                 goto out;
605         }
606
607         zram->disk->major = zram_major;
608         zram->disk->first_minor = device_id;
609         zram->disk->fops = &zram_devops;
610         zram->disk->queue = zram->queue;
611         zram->disk->private_data = zram;
612         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
613
614         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
615         set_capacity(zram->disk, 0);
616
617         /*
618          * To ensure that we always get PAGE_SIZE aligned
619          * and n*PAGE_SIZED sized I/O requests.
620          */
621         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
622         blk_queue_logical_block_size(zram->disk->queue, PAGE_SIZE);
623         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
624         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
625
626         add_disk(zram->disk);
627
628         ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
629                                 &zram_disk_attr_group);
630         if (ret < 0) {
631                 pr_warning("Error creating sysfs group");
632                 goto out;
633         }
634
635         zram->init_done = 0;
636
637 out:
638         return ret;
639 }
640
641 static void destroy_device(struct zram *zram)
642 {
643         sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
644                         &zram_disk_attr_group);
645
646         if (zram->disk) {
647                 del_gendisk(zram->disk);
648                 put_disk(zram->disk);
649         }
650
651         if (zram->queue)
652                 blk_cleanup_queue(zram->queue);
653 }
654
655 static int __init zram_init(void)
656 {
657         int ret, dev_id;
658
659         if (num_devices > max_num_devices) {
660                 pr_warning("Invalid value for num_devices: %u\n",
661                                 num_devices);
662                 ret = -EINVAL;
663                 goto out;
664         }
665
666         zram_major = register_blkdev(0, "zram");
667         if (zram_major <= 0) {
668                 pr_warning("Unable to get major number\n");
669                 ret = -EBUSY;
670                 goto out;
671         }
672
673         if (!num_devices) {
674                 pr_info("num_devices not specified. Using default: 1\n");
675                 num_devices = 1;
676         }
677
678         /* Allocate the device array and initialize each one */
679         pr_info("Creating %u devices ...\n", num_devices);
680         devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
681         if (!devices) {
682                 ret = -ENOMEM;
683                 goto unregister;
684         }
685
686         for (dev_id = 0; dev_id < num_devices; dev_id++) {
687                 ret = create_device(&devices[dev_id], dev_id);
688                 if (ret)
689                         goto free_devices;
690         }
691
692         return 0;
693
694 free_devices:
695         while (dev_id)
696                 destroy_device(&devices[--dev_id]);
697         kfree(devices);
698 unregister:
699         unregister_blkdev(zram_major, "zram");
700 out:
701         return ret;
702 }
703
704 static void __exit zram_exit(void)
705 {
706         int i;
707         struct zram *zram;
708
709         for (i = 0; i < num_devices; i++) {
710                 zram = &devices[i];
711
712                 destroy_device(zram);
713                 if (zram->init_done)
714                         zram_reset_device(zram);
715         }
716
717         unregister_blkdev(zram_major, "zram");
718
719         kfree(devices);
720         pr_debug("Cleanup done!\n");
721 }
722
723 module_param(num_devices, uint, 0);
724 MODULE_PARM_DESC(num_devices, "Number of zram devices");
725
726 module_init(zram_init);
727 module_exit(zram_exit);
728
729 MODULE_LICENSE("Dual BSD/GPL");
730 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
731 MODULE_DESCRIPTION("Compressed RAM Block Device");