2 * Compressed RAM block device
4 * Copyright (C) 2008, 2009, 2010 Nitin Gupta
5 * 2012, 2013 Minchan Kim
7 * This code is released using a dual license strategy: BSD/GPL
8 * You can choose the licence that better fits your requirements.
10 * Released under the terms of 3-clause BSD License
11 * Released under the terms of GNU General Public License Version 2.0
15 #define KMSG_COMPONENT "zram"
16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/bitops.h>
22 #include <linux/blkdev.h>
23 #include <linux/buffer_head.h>
24 #include <linux/device.h>
25 #include <linux/genhd.h>
26 #include <linux/highmem.h>
27 #include <linux/slab.h>
28 #include <linux/string.h>
29 #include <linux/vmalloc.h>
30 #include <linux/err.h>
31 #include <linux/idr.h>
32 #include <linux/sysfs.h>
36 static DEFINE_IDR(zram_index_idr);
37 /* idr index must be protected */
38 static DEFINE_MUTEX(zram_index_mutex);
40 static int zram_major;
41 static const char *default_compressor = "lzo";
43 /* Module params (documentation at end) */
44 static unsigned int num_devices = 1;
46 static inline void deprecated_attr_warn(const char *name)
48 pr_warn_once("%d (%s) Attribute %s (and others) will be removed. %s\n",
52 "See zram documentation.");
55 #define ZRAM_ATTR_RO(name) \
56 static ssize_t name##_show(struct device *d, \
57 struct device_attribute *attr, char *b) \
59 struct zram *zram = dev_to_zram(d); \
61 deprecated_attr_warn(__stringify(name)); \
62 return scnprintf(b, PAGE_SIZE, "%llu\n", \
63 (u64)atomic64_read(&zram->stats.name)); \
65 static DEVICE_ATTR_RO(name);
67 static inline bool init_done(struct zram *zram)
69 return zram->disksize;
72 static inline struct zram *dev_to_zram(struct device *dev)
74 return (struct zram *)dev_to_disk(dev)->private_data;
77 /* flag operations require table entry bit_spin_lock() being held */
78 static int zram_test_flag(struct zram_meta *meta, u32 index,
79 enum zram_pageflags flag)
81 return meta->table[index].value & BIT(flag);
84 static void zram_set_flag(struct zram_meta *meta, u32 index,
85 enum zram_pageflags flag)
87 meta->table[index].value |= BIT(flag);
90 static void zram_clear_flag(struct zram_meta *meta, u32 index,
91 enum zram_pageflags flag)
93 meta->table[index].value &= ~BIT(flag);
96 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
98 return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
101 static void zram_set_obj_size(struct zram_meta *meta,
102 u32 index, size_t size)
104 unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
106 meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
109 static inline int is_partial_io(struct bio_vec *bvec)
111 return bvec->bv_len != PAGE_SIZE;
115 * Check if request is within bounds and aligned on zram logical blocks.
117 static inline int valid_io_request(struct zram *zram,
118 sector_t start, unsigned int size)
122 /* unaligned request */
123 if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
125 if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
128 end = start + (size >> SECTOR_SHIFT);
129 bound = zram->disksize >> SECTOR_SHIFT;
130 /* out of range range */
131 if (unlikely(start >= bound || end > bound || start > end))
134 /* I/O request is valid */
138 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
140 if (*offset + bvec->bv_len >= PAGE_SIZE)
142 *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
145 static inline void update_used_max(struct zram *zram,
146 const unsigned long pages)
148 unsigned long old_max, cur_max;
150 old_max = atomic_long_read(&zram->stats.max_used_pages);
155 old_max = atomic_long_cmpxchg(
156 &zram->stats.max_used_pages, cur_max, pages);
157 } while (old_max != cur_max);
160 static int page_zero_filled(void *ptr)
165 page = (unsigned long *)ptr;
167 for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
175 static void handle_zero_page(struct bio_vec *bvec)
177 struct page *page = bvec->bv_page;
180 user_mem = kmap_atomic(page);
181 if (is_partial_io(bvec))
182 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
184 clear_page(user_mem);
185 kunmap_atomic(user_mem);
187 flush_dcache_page(page);
190 static ssize_t initstate_show(struct device *dev,
191 struct device_attribute *attr, char *buf)
194 struct zram *zram = dev_to_zram(dev);
196 down_read(&zram->init_lock);
197 val = init_done(zram);
198 up_read(&zram->init_lock);
200 return scnprintf(buf, PAGE_SIZE, "%u\n", val);
203 static ssize_t disksize_show(struct device *dev,
204 struct device_attribute *attr, char *buf)
206 struct zram *zram = dev_to_zram(dev);
208 return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
211 static ssize_t orig_data_size_show(struct device *dev,
212 struct device_attribute *attr, char *buf)
214 struct zram *zram = dev_to_zram(dev);
216 deprecated_attr_warn("orig_data_size");
217 return scnprintf(buf, PAGE_SIZE, "%llu\n",
218 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
221 static ssize_t mem_used_total_show(struct device *dev,
222 struct device_attribute *attr, char *buf)
225 struct zram *zram = dev_to_zram(dev);
227 deprecated_attr_warn("mem_used_total");
228 down_read(&zram->init_lock);
229 if (init_done(zram)) {
230 struct zram_meta *meta = zram->meta;
231 val = zs_get_total_pages(meta->mem_pool);
233 up_read(&zram->init_lock);
235 return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
238 static ssize_t mem_limit_show(struct device *dev,
239 struct device_attribute *attr, char *buf)
242 struct zram *zram = dev_to_zram(dev);
244 deprecated_attr_warn("mem_limit");
245 down_read(&zram->init_lock);
246 val = zram->limit_pages;
247 up_read(&zram->init_lock);
249 return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
252 static ssize_t mem_limit_store(struct device *dev,
253 struct device_attribute *attr, const char *buf, size_t len)
257 struct zram *zram = dev_to_zram(dev);
259 limit = memparse(buf, &tmp);
260 if (buf == tmp) /* no chars parsed, invalid input */
263 down_write(&zram->init_lock);
264 zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
265 up_write(&zram->init_lock);
270 static ssize_t mem_used_max_show(struct device *dev,
271 struct device_attribute *attr, char *buf)
274 struct zram *zram = dev_to_zram(dev);
276 deprecated_attr_warn("mem_used_max");
277 down_read(&zram->init_lock);
279 val = atomic_long_read(&zram->stats.max_used_pages);
280 up_read(&zram->init_lock);
282 return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
285 static ssize_t mem_used_max_store(struct device *dev,
286 struct device_attribute *attr, const char *buf, size_t len)
290 struct zram *zram = dev_to_zram(dev);
292 err = kstrtoul(buf, 10, &val);
296 down_read(&zram->init_lock);
297 if (init_done(zram)) {
298 struct zram_meta *meta = zram->meta;
299 atomic_long_set(&zram->stats.max_used_pages,
300 zs_get_total_pages(meta->mem_pool));
302 up_read(&zram->init_lock);
307 static ssize_t max_comp_streams_show(struct device *dev,
308 struct device_attribute *attr, char *buf)
311 struct zram *zram = dev_to_zram(dev);
313 down_read(&zram->init_lock);
314 val = zram->max_comp_streams;
315 up_read(&zram->init_lock);
317 return scnprintf(buf, PAGE_SIZE, "%d\n", val);
320 static ssize_t max_comp_streams_store(struct device *dev,
321 struct device_attribute *attr, const char *buf, size_t len)
324 struct zram *zram = dev_to_zram(dev);
327 ret = kstrtoint(buf, 0, &num);
333 down_write(&zram->init_lock);
334 if (init_done(zram)) {
335 if (!zcomp_set_max_streams(zram->comp, num)) {
336 pr_info("Cannot change max compression streams\n");
342 zram->max_comp_streams = num;
345 up_write(&zram->init_lock);
349 static ssize_t comp_algorithm_show(struct device *dev,
350 struct device_attribute *attr, char *buf)
353 struct zram *zram = dev_to_zram(dev);
355 down_read(&zram->init_lock);
356 sz = zcomp_available_show(zram->compressor, buf);
357 up_read(&zram->init_lock);
362 static ssize_t comp_algorithm_store(struct device *dev,
363 struct device_attribute *attr, const char *buf, size_t len)
365 struct zram *zram = dev_to_zram(dev);
366 down_write(&zram->init_lock);
367 if (init_done(zram)) {
368 up_write(&zram->init_lock);
369 pr_info("Can't change algorithm for initialized device\n");
372 strlcpy(zram->compressor, buf, sizeof(zram->compressor));
373 up_write(&zram->init_lock);
377 static ssize_t compact_store(struct device *dev,
378 struct device_attribute *attr, const char *buf, size_t len)
380 unsigned long nr_migrated;
381 struct zram *zram = dev_to_zram(dev);
382 struct zram_meta *meta;
384 down_read(&zram->init_lock);
385 if (!init_done(zram)) {
386 up_read(&zram->init_lock);
391 nr_migrated = zs_compact(meta->mem_pool);
392 atomic64_add(nr_migrated, &zram->stats.num_migrated);
393 up_read(&zram->init_lock);
398 static ssize_t io_stat_show(struct device *dev,
399 struct device_attribute *attr, char *buf)
401 struct zram *zram = dev_to_zram(dev);
404 down_read(&zram->init_lock);
405 ret = scnprintf(buf, PAGE_SIZE,
406 "%8llu %8llu %8llu %8llu\n",
407 (u64)atomic64_read(&zram->stats.failed_reads),
408 (u64)atomic64_read(&zram->stats.failed_writes),
409 (u64)atomic64_read(&zram->stats.invalid_io),
410 (u64)atomic64_read(&zram->stats.notify_free));
411 up_read(&zram->init_lock);
416 static ssize_t mm_stat_show(struct device *dev,
417 struct device_attribute *attr, char *buf)
419 struct zram *zram = dev_to_zram(dev);
420 u64 orig_size, mem_used = 0;
424 down_read(&zram->init_lock);
426 mem_used = zs_get_total_pages(zram->meta->mem_pool);
428 orig_size = atomic64_read(&zram->stats.pages_stored);
429 max_used = atomic_long_read(&zram->stats.max_used_pages);
431 ret = scnprintf(buf, PAGE_SIZE,
432 "%8llu %8llu %8llu %8lu %8ld %8llu %8llu\n",
433 orig_size << PAGE_SHIFT,
434 (u64)atomic64_read(&zram->stats.compr_data_size),
435 mem_used << PAGE_SHIFT,
436 zram->limit_pages << PAGE_SHIFT,
437 max_used << PAGE_SHIFT,
438 (u64)atomic64_read(&zram->stats.zero_pages),
439 (u64)atomic64_read(&zram->stats.num_migrated));
440 up_read(&zram->init_lock);
445 static DEVICE_ATTR_RO(io_stat);
446 static DEVICE_ATTR_RO(mm_stat);
447 ZRAM_ATTR_RO(num_reads);
448 ZRAM_ATTR_RO(num_writes);
449 ZRAM_ATTR_RO(failed_reads);
450 ZRAM_ATTR_RO(failed_writes);
451 ZRAM_ATTR_RO(invalid_io);
452 ZRAM_ATTR_RO(notify_free);
453 ZRAM_ATTR_RO(zero_pages);
454 ZRAM_ATTR_RO(compr_data_size);
456 static inline bool zram_meta_get(struct zram *zram)
458 if (atomic_inc_not_zero(&zram->refcount))
463 static inline void zram_meta_put(struct zram *zram)
465 atomic_dec(&zram->refcount);
468 static void zram_meta_free(struct zram_meta *meta, u64 disksize)
470 size_t num_pages = disksize >> PAGE_SHIFT;
473 /* Free all pages that are still in this zram device */
474 for (index = 0; index < num_pages; index++) {
475 unsigned long handle = meta->table[index].handle;
480 zs_free(meta->mem_pool, handle);
483 zs_destroy_pool(meta->mem_pool);
488 static struct zram_meta *zram_meta_alloc(int device_id, u64 disksize)
492 struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
497 num_pages = disksize >> PAGE_SHIFT;
498 meta->table = vzalloc(num_pages * sizeof(*meta->table));
500 pr_err("Error allocating zram address table\n");
504 snprintf(pool_name, sizeof(pool_name), "zram%d", device_id);
505 meta->mem_pool = zs_create_pool(pool_name, GFP_NOIO | __GFP_HIGHMEM);
506 if (!meta->mem_pool) {
507 pr_err("Error creating memory pool\n");
520 * To protect concurrent access to the same index entry,
521 * caller should hold this table index entry's bit_spinlock to
522 * indicate this index entry is accessing.
524 static void zram_free_page(struct zram *zram, size_t index)
526 struct zram_meta *meta = zram->meta;
527 unsigned long handle = meta->table[index].handle;
529 if (unlikely(!handle)) {
531 * No memory is allocated for zero filled pages.
532 * Simply clear zero page flag.
534 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
535 zram_clear_flag(meta, index, ZRAM_ZERO);
536 atomic64_dec(&zram->stats.zero_pages);
541 zs_free(meta->mem_pool, handle);
543 atomic64_sub(zram_get_obj_size(meta, index),
544 &zram->stats.compr_data_size);
545 atomic64_dec(&zram->stats.pages_stored);
547 meta->table[index].handle = 0;
548 zram_set_obj_size(meta, index, 0);
551 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
555 struct zram_meta *meta = zram->meta;
556 unsigned long handle;
559 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
560 handle = meta->table[index].handle;
561 size = zram_get_obj_size(meta, index);
563 if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
564 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
569 cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
570 if (size == PAGE_SIZE)
571 copy_page(mem, cmem);
573 ret = zcomp_decompress(zram->comp, cmem, size, mem);
574 zs_unmap_object(meta->mem_pool, handle);
575 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
577 /* Should NEVER happen. Return bio error if it does. */
579 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
586 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
587 u32 index, int offset)
591 unsigned char *user_mem, *uncmem = NULL;
592 struct zram_meta *meta = zram->meta;
593 page = bvec->bv_page;
595 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
596 if (unlikely(!meta->table[index].handle) ||
597 zram_test_flag(meta, index, ZRAM_ZERO)) {
598 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
599 handle_zero_page(bvec);
602 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
604 if (is_partial_io(bvec))
605 /* Use a temporary buffer to decompress the page */
606 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
608 user_mem = kmap_atomic(page);
609 if (!is_partial_io(bvec))
613 pr_info("Unable to allocate temp memory\n");
618 ret = zram_decompress_page(zram, uncmem, index);
619 /* Should NEVER happen. Return bio error if it does. */
623 if (is_partial_io(bvec))
624 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
627 flush_dcache_page(page);
630 kunmap_atomic(user_mem);
631 if (is_partial_io(bvec))
636 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
641 unsigned long handle;
643 unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
644 struct zram_meta *meta = zram->meta;
645 struct zcomp_strm *zstrm;
647 unsigned long alloced_pages;
649 page = bvec->bv_page;
650 if (is_partial_io(bvec)) {
652 * This is a partial IO. We need to read the full page
653 * before to write the changes.
655 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
660 ret = zram_decompress_page(zram, uncmem, index);
665 zstrm = zcomp_strm_find(zram->comp);
667 user_mem = kmap_atomic(page);
669 if (is_partial_io(bvec)) {
670 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
672 kunmap_atomic(user_mem);
678 if (page_zero_filled(uncmem)) {
680 kunmap_atomic(user_mem);
681 /* Free memory associated with this sector now. */
682 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
683 zram_free_page(zram, index);
684 zram_set_flag(meta, index, ZRAM_ZERO);
685 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
687 atomic64_inc(&zram->stats.zero_pages);
692 ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
693 if (!is_partial_io(bvec)) {
694 kunmap_atomic(user_mem);
700 pr_err("Compression failed! err=%d\n", ret);
704 if (unlikely(clen > max_zpage_size)) {
706 if (is_partial_io(bvec))
710 handle = zs_malloc(meta->mem_pool, clen);
712 pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
718 alloced_pages = zs_get_total_pages(meta->mem_pool);
719 if (zram->limit_pages && alloced_pages > zram->limit_pages) {
720 zs_free(meta->mem_pool, handle);
725 update_used_max(zram, alloced_pages);
727 cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
729 if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
730 src = kmap_atomic(page);
731 copy_page(cmem, src);
734 memcpy(cmem, src, clen);
737 zcomp_strm_release(zram->comp, zstrm);
739 zs_unmap_object(meta->mem_pool, handle);
742 * Free memory associated with this sector
743 * before overwriting unused sectors.
745 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
746 zram_free_page(zram, index);
748 meta->table[index].handle = handle;
749 zram_set_obj_size(meta, index, clen);
750 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
753 atomic64_add(clen, &zram->stats.compr_data_size);
754 atomic64_inc(&zram->stats.pages_stored);
757 zcomp_strm_release(zram->comp, zstrm);
758 if (is_partial_io(bvec))
764 * zram_bio_discard - handler on discard request
765 * @index: physical block index in PAGE_SIZE units
766 * @offset: byte offset within physical block
768 static void zram_bio_discard(struct zram *zram, u32 index,
769 int offset, struct bio *bio)
771 size_t n = bio->bi_iter.bi_size;
772 struct zram_meta *meta = zram->meta;
775 * zram manages data in physical block size units. Because logical block
776 * size isn't identical with physical block size on some arch, we
777 * could get a discard request pointing to a specific offset within a
778 * certain physical block. Although we can handle this request by
779 * reading that physiclal block and decompressing and partially zeroing
780 * and re-compressing and then re-storing it, this isn't reasonable
781 * because our intent with a discard request is to save memory. So
782 * skipping this logical block is appropriate here.
785 if (n <= (PAGE_SIZE - offset))
788 n -= (PAGE_SIZE - offset);
792 while (n >= PAGE_SIZE) {
793 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
794 zram_free_page(zram, index);
795 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
796 atomic64_inc(&zram->stats.notify_free);
802 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
805 unsigned long start_time = jiffies;
808 generic_start_io_acct(rw, bvec->bv_len >> SECTOR_SHIFT,
812 atomic64_inc(&zram->stats.num_reads);
813 ret = zram_bvec_read(zram, bvec, index, offset);
815 atomic64_inc(&zram->stats.num_writes);
816 ret = zram_bvec_write(zram, bvec, index, offset);
819 generic_end_io_acct(rw, &zram->disk->part0, start_time);
823 atomic64_inc(&zram->stats.failed_reads);
825 atomic64_inc(&zram->stats.failed_writes);
831 static void __zram_make_request(struct zram *zram, struct bio *bio)
836 struct bvec_iter iter;
838 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
839 offset = (bio->bi_iter.bi_sector &
840 (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
842 if (unlikely(bio->bi_rw & REQ_DISCARD)) {
843 zram_bio_discard(zram, index, offset, bio);
848 rw = bio_data_dir(bio);
849 bio_for_each_segment(bvec, bio, iter) {
850 int max_transfer_size = PAGE_SIZE - offset;
852 if (bvec.bv_len > max_transfer_size) {
854 * zram_bvec_rw() can only make operation on a single
855 * zram page. Split the bio vector.
859 bv.bv_page = bvec.bv_page;
860 bv.bv_len = max_transfer_size;
861 bv.bv_offset = bvec.bv_offset;
863 if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)
866 bv.bv_len = bvec.bv_len - max_transfer_size;
867 bv.bv_offset += max_transfer_size;
868 if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)
871 if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)
874 update_position(&index, &offset, &bvec);
877 set_bit(BIO_UPTODATE, &bio->bi_flags);
886 * Handler function for all zram I/O requests.
888 static void zram_make_request(struct request_queue *queue, struct bio *bio)
890 struct zram *zram = queue->queuedata;
892 if (unlikely(!zram_meta_get(zram)))
895 if (!valid_io_request(zram, bio->bi_iter.bi_sector,
896 bio->bi_iter.bi_size)) {
897 atomic64_inc(&zram->stats.invalid_io);
901 __zram_make_request(zram, bio);
910 static void zram_slot_free_notify(struct block_device *bdev,
914 struct zram_meta *meta;
916 zram = bdev->bd_disk->private_data;
919 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
920 zram_free_page(zram, index);
921 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
922 atomic64_inc(&zram->stats.notify_free);
925 static int zram_rw_page(struct block_device *bdev, sector_t sector,
926 struct page *page, int rw)
928 int offset, err = -EIO;
933 zram = bdev->bd_disk->private_data;
934 if (unlikely(!zram_meta_get(zram)))
937 if (!valid_io_request(zram, sector, PAGE_SIZE)) {
938 atomic64_inc(&zram->stats.invalid_io);
943 index = sector >> SECTORS_PER_PAGE_SHIFT;
944 offset = sector & (SECTORS_PER_PAGE - 1) << SECTOR_SHIFT;
947 bv.bv_len = PAGE_SIZE;
950 err = zram_bvec_rw(zram, &bv, index, offset, rw);
955 * If I/O fails, just return error(ie, non-zero) without
956 * calling page_endio.
957 * It causes resubmit the I/O with bio request by upper functions
958 * of rw_page(e.g., swap_readpage, __swap_writepage) and
959 * bio->bi_end_io does things to handle the error
960 * (e.g., SetPageError, set_page_dirty and extra works).
963 page_endio(page, rw, 0);
967 static void zram_reset_device(struct zram *zram)
969 struct zram_meta *meta;
973 down_write(&zram->init_lock);
975 zram->limit_pages = 0;
977 if (!init_done(zram)) {
978 up_write(&zram->init_lock);
984 disksize = zram->disksize;
986 * Refcount will go down to 0 eventually and r/w handler
987 * cannot handle further I/O so it will bail out by
988 * check zram_meta_get.
992 * We want to free zram_meta in process context to avoid
993 * deadlock between reclaim path and any other locks.
995 wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);
998 memset(&zram->stats, 0, sizeof(zram->stats));
1000 zram->max_comp_streams = 1;
1002 set_capacity(zram->disk, 0);
1003 part_stat_set_all(&zram->disk->part0, 0);
1005 up_write(&zram->init_lock);
1006 /* I/O operation under all of CPU are done so let's free */
1007 zram_meta_free(meta, disksize);
1008 zcomp_destroy(comp);
1011 static ssize_t disksize_store(struct device *dev,
1012 struct device_attribute *attr, const char *buf, size_t len)
1016 struct zram_meta *meta;
1017 struct zram *zram = dev_to_zram(dev);
1020 disksize = memparse(buf, NULL);
1024 disksize = PAGE_ALIGN(disksize);
1025 meta = zram_meta_alloc(zram->disk->first_minor, disksize);
1029 comp = zcomp_create(zram->compressor, zram->max_comp_streams);
1031 pr_info("Cannot initialise %s compressing backend\n",
1033 err = PTR_ERR(comp);
1037 down_write(&zram->init_lock);
1038 if (init_done(zram)) {
1039 pr_info("Cannot change disksize for initialized device\n");
1041 goto out_destroy_comp;
1044 init_waitqueue_head(&zram->io_done);
1045 atomic_set(&zram->refcount, 1);
1048 zram->disksize = disksize;
1049 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
1050 up_write(&zram->init_lock);
1053 * Revalidate disk out of the init_lock to avoid lockdep splat.
1054 * It's okay because disk's capacity is protected by init_lock
1055 * so that revalidate_disk always sees up-to-date capacity.
1057 revalidate_disk(zram->disk);
1062 up_write(&zram->init_lock);
1063 zcomp_destroy(comp);
1065 zram_meta_free(meta, disksize);
1069 static ssize_t reset_store(struct device *dev,
1070 struct device_attribute *attr, const char *buf, size_t len)
1073 unsigned short do_reset;
1075 struct block_device *bdev;
1077 ret = kstrtou16(buf, 10, &do_reset);
1084 zram = dev_to_zram(dev);
1085 bdev = bdget_disk(zram->disk, 0);
1089 mutex_lock(&bdev->bd_mutex);
1090 /* Do not reset an active device or claimed device */
1091 if (bdev->bd_openers || zram->claim) {
1092 mutex_unlock(&bdev->bd_mutex);
1097 /* From now on, anyone can't open /dev/zram[0-9] */
1099 mutex_unlock(&bdev->bd_mutex);
1101 /* Make sure all the pending I/O are finished */
1103 zram_reset_device(zram);
1104 revalidate_disk(zram->disk);
1107 mutex_lock(&bdev->bd_mutex);
1108 zram->claim = false;
1109 mutex_unlock(&bdev->bd_mutex);
1114 static int zram_open(struct block_device *bdev, fmode_t mode)
1119 WARN_ON(!mutex_is_locked(&bdev->bd_mutex));
1121 zram = bdev->bd_disk->private_data;
1122 /* zram was claimed to reset so open request fails */
1129 static const struct block_device_operations zram_devops = {
1131 .swap_slot_free_notify = zram_slot_free_notify,
1132 .rw_page = zram_rw_page,
1133 .owner = THIS_MODULE
1136 static DEVICE_ATTR_WO(compact);
1137 static DEVICE_ATTR_RW(disksize);
1138 static DEVICE_ATTR_RO(initstate);
1139 static DEVICE_ATTR_WO(reset);
1140 static DEVICE_ATTR_RO(orig_data_size);
1141 static DEVICE_ATTR_RO(mem_used_total);
1142 static DEVICE_ATTR_RW(mem_limit);
1143 static DEVICE_ATTR_RW(mem_used_max);
1144 static DEVICE_ATTR_RW(max_comp_streams);
1145 static DEVICE_ATTR_RW(comp_algorithm);
1147 static struct attribute *zram_disk_attrs[] = {
1148 &dev_attr_disksize.attr,
1149 &dev_attr_initstate.attr,
1150 &dev_attr_reset.attr,
1151 &dev_attr_num_reads.attr,
1152 &dev_attr_num_writes.attr,
1153 &dev_attr_failed_reads.attr,
1154 &dev_attr_failed_writes.attr,
1155 &dev_attr_compact.attr,
1156 &dev_attr_invalid_io.attr,
1157 &dev_attr_notify_free.attr,
1158 &dev_attr_zero_pages.attr,
1159 &dev_attr_orig_data_size.attr,
1160 &dev_attr_compr_data_size.attr,
1161 &dev_attr_mem_used_total.attr,
1162 &dev_attr_mem_limit.attr,
1163 &dev_attr_mem_used_max.attr,
1164 &dev_attr_max_comp_streams.attr,
1165 &dev_attr_comp_algorithm.attr,
1166 &dev_attr_io_stat.attr,
1167 &dev_attr_mm_stat.attr,
1171 static struct attribute_group zram_disk_attr_group = {
1172 .attrs = zram_disk_attrs,
1176 * Allocate and initialize new zram device. the function returns
1177 * '>= 0' device_id upon success, and negative value otherwise.
1179 static int zram_add(void)
1182 struct request_queue *queue;
1185 zram = kzalloc(sizeof(struct zram), GFP_KERNEL);
1189 ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
1194 init_rwsem(&zram->init_lock);
1196 queue = blk_alloc_queue(GFP_KERNEL);
1198 pr_err("Error allocating disk queue for device %d\n",
1204 blk_queue_make_request(queue, zram_make_request);
1206 /* gendisk structure */
1207 zram->disk = alloc_disk(1);
1209 pr_warn("Error allocating disk structure for device %d\n",
1212 goto out_free_queue;
1215 zram->disk->major = zram_major;
1216 zram->disk->first_minor = device_id;
1217 zram->disk->fops = &zram_devops;
1218 zram->disk->queue = queue;
1219 zram->disk->queue->queuedata = zram;
1220 zram->disk->private_data = zram;
1221 snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
1223 /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
1224 set_capacity(zram->disk, 0);
1225 /* zram devices sort of resembles non-rotational disks */
1226 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
1227 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
1229 * To ensure that we always get PAGE_SIZE aligned
1230 * and n*PAGE_SIZED sized I/O requests.
1232 blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
1233 blk_queue_logical_block_size(zram->disk->queue,
1234 ZRAM_LOGICAL_BLOCK_SIZE);
1235 blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1236 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
1237 zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1238 zram->disk->queue->limits.max_discard_sectors = UINT_MAX;
1240 * zram_bio_discard() will clear all logical blocks if logical block
1241 * size is identical with physical block size(PAGE_SIZE). But if it is
1242 * different, we will skip discarding some parts of logical blocks in
1243 * the part of the request range which isn't aligned to physical block
1244 * size. So we can't ensure that all discarded logical blocks are
1247 if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
1248 zram->disk->queue->limits.discard_zeroes_data = 1;
1250 zram->disk->queue->limits.discard_zeroes_data = 0;
1251 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
1253 add_disk(zram->disk);
1255 ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
1256 &zram_disk_attr_group);
1258 pr_warn("Error creating sysfs group");
1261 strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
1263 zram->max_comp_streams = 1;
1265 pr_info("Added device: %s\n", zram->disk->disk_name);
1269 del_gendisk(zram->disk);
1270 put_disk(zram->disk);
1272 blk_cleanup_queue(queue);
1274 idr_remove(&zram_index_idr, device_id);
1280 static int zram_remove(struct zram *zram)
1282 struct block_device *bdev;
1284 bdev = bdget_disk(zram->disk, 0);
1288 mutex_lock(&bdev->bd_mutex);
1289 if (bdev->bd_openers || zram->claim) {
1290 mutex_unlock(&bdev->bd_mutex);
1296 mutex_unlock(&bdev->bd_mutex);
1299 * Remove sysfs first, so no one will perform a disksize
1300 * store while we destroy the devices. This also helps during
1301 * hot_remove -- zram_reset_device() is the last holder of
1302 * ->init_lock, no later/concurrent disksize_store() or any
1303 * other sysfs handlers are possible.
1305 sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
1306 &zram_disk_attr_group);
1308 /* Make sure all the pending I/O are finished */
1310 zram_reset_device(zram);
1313 pr_info("Removed device: %s\n", zram->disk->disk_name);
1315 idr_remove(&zram_index_idr, zram->disk->first_minor);
1316 blk_cleanup_queue(zram->disk->queue);
1317 del_gendisk(zram->disk);
1318 put_disk(zram->disk);
1323 /* zram-control sysfs attributes */
1324 static ssize_t hot_add_show(struct class *class,
1325 struct class_attribute *attr,
1330 mutex_lock(&zram_index_mutex);
1332 mutex_unlock(&zram_index_mutex);
1336 return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
1339 static ssize_t hot_remove_store(struct class *class,
1340 struct class_attribute *attr,
1347 /* dev_id is gendisk->first_minor, which is `int' */
1348 ret = kstrtoint(buf, 10, &dev_id);
1354 mutex_lock(&zram_index_mutex);
1356 zram = idr_find(&zram_index_idr, dev_id);
1358 ret = zram_remove(zram);
1362 mutex_unlock(&zram_index_mutex);
1363 return ret ? ret : count;
1366 static struct class_attribute zram_control_class_attrs[] = {
1368 __ATTR_WO(hot_remove),
1372 static struct class zram_control_class = {
1373 .name = "zram-control",
1374 .owner = THIS_MODULE,
1375 .class_attrs = zram_control_class_attrs,
1378 static int zram_remove_cb(int id, void *ptr, void *data)
1384 static void destroy_devices(void)
1386 class_unregister(&zram_control_class);
1387 idr_for_each(&zram_index_idr, &zram_remove_cb, NULL);
1388 idr_destroy(&zram_index_idr);
1389 unregister_blkdev(zram_major, "zram");
1392 static int __init zram_init(void)
1396 ret = class_register(&zram_control_class);
1398 pr_warn("Unable to register zram-control class\n");
1402 zram_major = register_blkdev(0, "zram");
1403 if (zram_major <= 0) {
1404 pr_warn("Unable to get major number\n");
1405 class_unregister(&zram_control_class);
1409 while (num_devices != 0) {
1410 mutex_lock(&zram_index_mutex);
1412 mutex_unlock(&zram_index_mutex);
1425 static void __exit zram_exit(void)
1430 module_init(zram_init);
1431 module_exit(zram_exit);
1433 module_param(num_devices, uint, 0);
1434 MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices");
1436 MODULE_LICENSE("Dual BSD/GPL");
1437 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
1438 MODULE_DESCRIPTION("Compressed RAM Block Device");