2 * Compressed RAM based swap device
4 * Copyright (C) 2008, 2009, 2010 Nitin Gupta
6 * This code is released using a dual license strategy: BSD/GPL
7 * You can choose the licence that better fits your requirements.
9 * Released under the terms of 3-clause BSD License
10 * Released under the terms of GNU General Public License Version 2.0
12 * Project home: http://compcache.googlecode.com
15 #define KMSG_COMPONENT "ramzswap"
16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/bitops.h>
21 #include <linux/blkdev.h>
22 #include <linux/buffer_head.h>
23 #include <linux/device.h>
24 #include <linux/genhd.h>
25 #include <linux/highmem.h>
26 #include <linux/lzo.h>
27 #include <linux/string.h>
28 #include <linux/swap.h>
29 #include <linux/swapops.h>
30 #include <linux/vmalloc.h>
32 #include "ramzswap_drv.h"
35 static int ramzswap_major;
36 static struct ramzswap *devices;
39 * Pages that compress to larger than this size are
40 * forwarded to backing swap, if present or stored
41 * uncompressed in memory otherwise.
43 static unsigned int max_zpage_size;
45 /* Module params (documentation at end) */
46 static unsigned int num_devices;
48 static int rzs_test_flag(struct ramzswap *rzs, u32 index,
49 enum rzs_pageflags flag)
51 return rzs->table[index].flags & BIT(flag);
54 static void rzs_set_flag(struct ramzswap *rzs, u32 index,
55 enum rzs_pageflags flag)
57 rzs->table[index].flags |= BIT(flag);
60 static void rzs_clear_flag(struct ramzswap *rzs, u32 index,
61 enum rzs_pageflags flag)
63 rzs->table[index].flags &= ~BIT(flag);
66 static int page_zero_filled(void *ptr)
71 page = (unsigned long *)ptr;
73 for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
82 * memlimit cannot be greater than backing disk size.
84 static void ramzswap_set_memlimit(struct ramzswap *rzs, size_t totalram_bytes)
86 int memlimit_valid = 1;
89 pr_info("Memory limit not set.\n");
93 if (rzs->memlimit > rzs->disksize) {
94 pr_info("Memory limit cannot be greater than "
95 "disksize: limit=%zu, disksize=%zu\n",
96 rzs->memlimit, rzs->disksize);
100 if (!memlimit_valid) {
101 size_t mempart, disksize;
102 pr_info("Using default: smaller of (%u%% of RAM) and "
103 "(backing disk size).\n",
104 default_memlimit_perc_ram);
105 mempart = default_memlimit_perc_ram * (totalram_bytes / 100);
106 disksize = rzs->disksize;
107 rzs->memlimit = mempart > disksize ? disksize : mempart;
110 if (rzs->memlimit > totalram_bytes / 2) {
112 "Its not advisable setting limit more than half of "
113 "size of memory since we expect a 2:1 compression ratio. "
114 "Limit represents amount of *compressed* data we can keep "
116 "\tMemory Size: %zu kB\n"
117 "\tLimit you selected: %zu kB\n"
118 "Continuing anyway ...\n",
119 totalram_bytes >> 10, rzs->memlimit >> 10
123 rzs->memlimit &= PAGE_MASK;
124 BUG_ON(!rzs->memlimit);
127 static void ramzswap_set_disksize(struct ramzswap *rzs, size_t totalram_bytes)
129 if (!rzs->disksize) {
131 "disk size not provided. You can use disksize_kb module "
132 "param to specify size.\nUsing default: (%u%% of RAM).\n",
133 default_disksize_perc_ram
135 rzs->disksize = default_disksize_perc_ram *
136 (totalram_bytes / 100);
139 if (rzs->disksize > 2 * (totalram_bytes)) {
141 "There is little point creating a ramzswap of greater than "
142 "twice the size of memory since we expect a 2:1 compression "
143 "ratio. Note that ramzswap uses about 0.1%% of the size of "
144 "the swap device when not in use so a huge ramzswap is "
146 "\tMemory Size: %zu kB\n"
147 "\tSize you selected: %zu kB\n"
148 "Continuing anyway ...\n",
149 totalram_bytes >> 10, rzs->disksize
153 rzs->disksize &= PAGE_MASK;
157 * Swap header (1st page of swap device) contains information
158 * to indentify it as a swap partition. Prepare such a header
159 * for ramzswap device (ramzswap0) so that swapon can identify
160 * it as swap partition. In case backing swap device is provided,
161 * copy its swap header.
163 static int setup_swap_header(struct ramzswap *rzs, union swap_header *s)
167 struct address_space *mapping;
168 union swap_header *backing_swap_header;
171 * There is no backing swap device. Create a swap header
172 * that is acceptable by swapon.
174 if (!rzs->backing_swap) {
176 s->info.last_page = (rzs->disksize >> PAGE_SHIFT) - 1;
177 s->info.nr_badpages = 0;
178 memcpy(s->magic.magic, "SWAPSPACE2", 10);
183 * We have a backing swap device. Copy its swap header
184 * to ramzswap device header. If this header contains
185 * invalid information (backing device not a swap
186 * partition, etc.), swapon will fail for ramzswap
187 * which is correct behavior - we don't want to swap
188 * over filesystem partition!
191 /* Read the backing swap header (code from sys_swapon) */
192 mapping = rzs->swap_file->f_mapping;
193 if (!mapping->a_ops->readpage) {
198 page = read_mapping_page(mapping, 0, rzs->swap_file);
204 backing_swap_header = kmap(page);
205 memcpy(s, backing_swap_header, sizeof(*s));
206 if (s->info.nr_badpages) {
207 pr_info("Cannot use backing swap with bad pages (%u)\n",
208 s->info.nr_badpages);
212 * ramzswap disksize equals number of usable pages in backing
213 * swap. Set last_page in swap header to match this disksize
214 * ('last_page' means 0-based index of last usable swap page).
216 s->info.last_page = (rzs->disksize >> PAGE_SHIFT) - 1;
223 static void ramzswap_ioctl_get_stats(struct ramzswap *rzs,
224 struct ramzswap_ioctl_stats *s)
226 strncpy(s->backing_swap_name, rzs->backing_swap_name,
227 MAX_SWAP_NAME_LEN - 1);
228 s->backing_swap_name[MAX_SWAP_NAME_LEN - 1] = '\0';
230 s->disksize = rzs->disksize;
231 s->memlimit = rzs->memlimit;
233 #if defined(CONFIG_RAMZSWAP_STATS)
235 struct ramzswap_stats *rs = &rzs->stats;
236 size_t succ_writes, mem_used;
237 unsigned int good_compress_perc = 0, no_compress_perc = 0;
239 mem_used = xv_get_total_size_bytes(rzs->mem_pool)
240 + (rs->pages_expand << PAGE_SHIFT);
241 succ_writes = rzs_stat64_read(rzs, &rs->num_writes) -
242 rzs_stat64_read(rzs, &rs->failed_writes);
244 if (succ_writes && rs->pages_stored) {
245 good_compress_perc = rs->good_compress * 100
247 no_compress_perc = rs->pages_expand * 100
251 s->num_reads = rzs_stat64_read(rzs, &rs->num_reads);
252 s->num_writes = rzs_stat64_read(rzs, &rs->num_writes);
253 s->failed_reads = rzs_stat64_read(rzs, &rs->failed_reads);
254 s->failed_writes = rzs_stat64_read(rzs, &rs->failed_writes);
255 s->invalid_io = rzs_stat64_read(rzs, &rs->invalid_io);
256 s->notify_free = rzs_stat64_read(rzs, &rs->notify_free);
257 s->pages_zero = rs->pages_zero;
259 s->good_compress_pct = good_compress_perc;
260 s->pages_expand_pct = no_compress_perc;
262 s->pages_stored = rs->pages_stored;
263 s->pages_used = mem_used >> PAGE_SHIFT;
264 s->orig_data_size = rs->pages_stored << PAGE_SHIFT;
265 s->compr_data_size = rs->compr_size;
266 s->mem_used_total = mem_used;
268 s->bdev_num_reads = rzs_stat64_read(rzs, &rs->bdev_num_reads);
269 s->bdev_num_writes = rzs_stat64_read(rzs, &rs->bdev_num_writes);
271 #endif /* CONFIG_RAMZSWAP_STATS */
274 static int add_backing_swap_extent(struct ramzswap *rzs,
279 struct list_head *head;
280 struct page *curr_page, *new_page;
281 unsigned int extents_per_page = PAGE_SIZE /
282 sizeof(struct ramzswap_backing_extent);
284 idx = rzs->num_extents % extents_per_page;
286 new_page = alloc_page(__GFP_ZERO);
290 if (rzs->num_extents) {
291 curr_page = virt_to_page(rzs->curr_extent);
292 head = &curr_page->lru;
294 head = &rzs->backing_swap_extent_list;
297 list_add(&new_page->lru, head);
298 rzs->curr_extent = page_address(new_page);
301 rzs->curr_extent->phy_pagenum = phy_pagenum;
302 rzs->curr_extent->num_pages = num_pages;
304 pr_debug("add_extent: idx=%u, phy_pgnum=%lu, num_pgs=%lu, "
305 "pg_last=%lu, curr_ext=%p\n", idx, phy_pagenum, num_pages,
306 phy_pagenum + num_pages - 1, rzs->curr_extent);
308 if (idx != extents_per_page - 1)
314 static int setup_backing_swap_extents(struct ramzswap *rzs,
315 struct inode *inode, unsigned long *num_pages)
319 unsigned blocks_per_page;
320 pgoff_t contig_pages = 0, total_pages = 0;
321 pgoff_t pagenum = 0, prev_pagenum = 0;
322 sector_t probe_block = 0;
325 blkbits = inode->i_blkbits;
326 blocks_per_page = PAGE_SIZE >> blkbits;
328 last_block = i_size_read(inode) >> blkbits;
329 while (probe_block + blocks_per_page <= last_block) {
330 unsigned block_in_page;
331 sector_t first_block;
333 first_block = bmap(inode, probe_block);
334 if (first_block == 0)
337 /* It must be PAGE_SIZE aligned on-disk */
338 if (first_block & (blocks_per_page - 1)) {
343 /* All blocks within this page must be contiguous on disk */
344 for (block_in_page = 1; block_in_page < blocks_per_page;
348 block = bmap(inode, probe_block + block_in_page);
351 if (block != first_block + block_in_page) {
359 * We found a PAGE_SIZE length, PAGE_SIZE aligned
362 pagenum = first_block >> (PAGE_SHIFT - blkbits);
364 if (total_pages && (pagenum != prev_pagenum + 1)) {
365 ret = add_backing_swap_extent(rzs, prev_pagenum -
366 (contig_pages - 1), contig_pages);
374 prev_pagenum = pagenum;
375 probe_block += blocks_per_page;
382 pr_debug("adding last extent: pagenum=%lu, "
383 "contig_pages=%lu\n", pagenum, contig_pages);
384 ret = add_backing_swap_extent(rzs,
385 prev_pagenum - (contig_pages - 1), contig_pages);
390 if (!rzs->num_extents) {
391 pr_err("No swap extents found!\n");
396 *num_pages = total_pages;
397 pr_info("Found %lu extents containing %luk\n",
398 rzs->num_extents, *num_pages << (PAGE_SHIFT - 10));
403 pr_err("Backing swapfile has holes\n");
406 while (ret && !list_empty(&rzs->backing_swap_extent_list)) {
408 struct list_head *entry = rzs->backing_swap_extent_list.next;
409 page = list_entry(entry, struct page, lru);
416 static void map_backing_swap_extents(struct ramzswap *rzs)
418 struct ramzswap_backing_extent *se;
419 struct page *table_page, *se_page;
420 unsigned long num_pages, num_table_pages, entry;
421 unsigned long se_idx, span;
422 unsigned entries_per_page = PAGE_SIZE / sizeof(*rzs->table);
423 unsigned extents_per_page = PAGE_SIZE / sizeof(*se);
425 /* True for block device */
426 if (!rzs->num_extents)
429 se_page = list_entry(rzs->backing_swap_extent_list.next,
431 se = page_address(se_page);
432 span = se->num_pages;
433 num_pages = rzs->disksize >> PAGE_SHIFT;
434 num_table_pages = DIV_ROUND_UP(num_pages * sizeof(*rzs->table),
439 while (num_table_pages--) {
440 table_page = vmalloc_to_page(&rzs->table[entry]);
441 while (span <= entry) {
443 if (se_idx == rzs->num_extents)
446 if (!(se_idx % extents_per_page)) {
447 se_page = list_entry(se_page->lru.next,
449 se = page_address(se_page);
453 span += se->num_pages;
455 table_page->mapping = (struct address_space *)se;
456 table_page->private = se->num_pages - (span - entry);
457 pr_debug("map_table: entry=%lu, span=%lu, map=%p, priv=%lu\n",
458 entry, span, table_page->mapping, table_page->private);
459 entry += entries_per_page;
464 * Check if value of backing_swap module param is sane.
465 * Claim this device and set ramzswap size equal to
466 * size of this block device.
468 static int setup_backing_swap(struct ramzswap *rzs)
472 unsigned long num_pages = 0;
474 struct file *swap_file;
475 struct address_space *mapping;
476 struct block_device *bdev = NULL;
478 if (!rzs->backing_swap_name[0]) {
479 pr_debug("backing_swap param not given\n");
483 pr_info("Using backing swap device: %s\n", rzs->backing_swap_name);
485 swap_file = filp_open(rzs->backing_swap_name,
486 O_RDWR | O_LARGEFILE, 0);
487 if (IS_ERR(swap_file)) {
488 pr_err("Error opening backing device: %s\n",
489 rzs->backing_swap_name);
494 mapping = swap_file->f_mapping;
495 inode = mapping->host;
497 if (S_ISBLK(inode->i_mode)) {
498 bdev = I_BDEV(inode);
499 ret = bd_claim(bdev, setup_backing_swap);
504 disksize = i_size_read(inode);
506 * Can happen if user gives an extended partition as
507 * backing swap or simply a bad disk.
510 pr_err("Error reading backing swap size.\n");
513 } else if (S_ISREG(inode->i_mode)) {
514 bdev = inode->i_sb->s_bdev;
515 if (IS_SWAPFILE(inode)) {
519 ret = setup_backing_swap_extents(rzs, inode, &num_pages);
522 disksize = num_pages << PAGE_SHIFT;
527 rzs->swap_file = swap_file;
528 rzs->backing_swap = bdev;
529 rzs->disksize = disksize;
536 filp_close(swap_file, NULL);
539 rzs->backing_swap = NULL;
544 * Map logical page number 'pagenum' to physical page number
545 * on backing swap device. For block device, this is a nop.
547 static u32 map_backing_swap_page(struct ramzswap *rzs, u32 pagenum)
549 u32 skip_pages, entries_per_page;
550 size_t delta, se_offset, skipped;
551 struct page *table_page, *se_page;
552 struct ramzswap_backing_extent *se;
554 if (!rzs->num_extents)
557 entries_per_page = PAGE_SIZE / sizeof(*rzs->table);
559 table_page = vmalloc_to_page(&rzs->table[pagenum]);
560 se = (struct ramzswap_backing_extent *)table_page->mapping;
561 se_page = virt_to_page(se);
563 skip_pages = pagenum - (pagenum / entries_per_page * entries_per_page);
564 se_offset = table_page->private + skip_pages;
566 if (se_offset < se->num_pages)
567 return se->phy_pagenum + se_offset;
569 skipped = se->num_pages - table_page->private;
571 struct ramzswap_backing_extent *se_base;
572 u32 se_entries_per_page = PAGE_SIZE / sizeof(*se);
574 /* Get next swap extent */
575 se_base = (struct ramzswap_backing_extent *)
576 page_address(se_page);
577 if (se - se_base == se_entries_per_page - 1) {
578 se_page = list_entry(se_page->lru.next,
580 se = page_address(se_page);
585 skipped += se->num_pages;
586 } while (skipped < skip_pages);
588 delta = skipped - skip_pages;
589 se_offset = se->num_pages - delta;
591 return se->phy_pagenum + se_offset;
594 static void ramzswap_free_page(struct ramzswap *rzs, size_t index)
599 struct page *page = rzs->table[index].page;
600 u32 offset = rzs->table[index].offset;
602 if (unlikely(!page)) {
604 * No memory is allocated for zero filled pages.
605 * Simply clear zero page flag.
607 if (rzs_test_flag(rzs, index, RZS_ZERO)) {
608 rzs_clear_flag(rzs, index, RZS_ZERO);
609 rzs_stat_dec(&rzs->stats.pages_zero);
614 if (unlikely(rzs_test_flag(rzs, index, RZS_UNCOMPRESSED))) {
617 rzs_clear_flag(rzs, index, RZS_UNCOMPRESSED);
618 rzs_stat_dec(&rzs->stats.pages_expand);
622 obj = kmap_atomic(page, KM_USER0) + offset;
623 clen = xv_get_object_size(obj) - sizeof(struct zobj_header);
624 kunmap_atomic(obj, KM_USER0);
626 xv_free(rzs->mem_pool, page, offset);
627 if (clen <= PAGE_SIZE / 2)
628 rzs_stat_dec(&rzs->stats.good_compress);
631 rzs->stats.compr_size -= clen;
632 rzs_stat_dec(&rzs->stats.pages_stored);
634 rzs->table[index].page = NULL;
635 rzs->table[index].offset = 0;
638 static int handle_zero_page(struct bio *bio)
641 struct page *page = bio->bi_io_vec[0].bv_page;
643 user_mem = kmap_atomic(page, KM_USER0);
644 memset(user_mem, 0, PAGE_SIZE);
645 kunmap_atomic(user_mem, KM_USER0);
647 flush_dcache_page(page);
649 set_bit(BIO_UPTODATE, &bio->bi_flags);
654 static int handle_uncompressed_page(struct ramzswap *rzs, struct bio *bio)
658 unsigned char *user_mem, *cmem;
660 page = bio->bi_io_vec[0].bv_page;
661 index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
663 user_mem = kmap_atomic(page, KM_USER0);
664 cmem = kmap_atomic(rzs->table[index].page, KM_USER1) +
665 rzs->table[index].offset;
667 memcpy(user_mem, cmem, PAGE_SIZE);
668 kunmap_atomic(user_mem, KM_USER0);
669 kunmap_atomic(cmem, KM_USER1);
671 flush_dcache_page(page);
673 set_bit(BIO_UPTODATE, &bio->bi_flags);
679 * Called when request page is not present in ramzswap.
680 * Its either in backing swap device (if present) or
681 * this is an attempt to read before any previous write
682 * to this location - this happens due to readahead when
683 * swap device is read from user-space (e.g. during swapon)
685 static int handle_ramzswap_fault(struct ramzswap *rzs, struct bio *bio)
688 * Always forward such requests to backing swap
689 * device (if present)
691 if (rzs->backing_swap) {
693 rzs_stat64_dec(rzs, &rzs->stats.num_reads);
694 rzs_stat64_inc(rzs, &rzs->stats.bdev_num_reads);
695 bio->bi_bdev = rzs->backing_swap;
698 * In case backing swap is a file, find the right offset within
699 * the file corresponding to logical position 'index'. For block
700 * device, this is a nop.
702 pagenum = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
703 bio->bi_sector = map_backing_swap_page(rzs, pagenum)
704 << SECTORS_PER_PAGE_SHIFT;
709 * Its unlikely event in case backing dev is
712 pr_debug("Read before write on swap device: "
713 "sector=%lu, size=%u, offset=%u\n",
714 (ulong)(bio->bi_sector), bio->bi_size,
715 bio->bi_io_vec[0].bv_offset);
717 /* Do nothing. Just return success */
718 set_bit(BIO_UPTODATE, &bio->bi_flags);
723 static int ramzswap_read(struct ramzswap *rzs, struct bio *bio)
729 struct zobj_header *zheader;
730 unsigned char *user_mem, *cmem;
732 rzs_stat64_inc(rzs, &rzs->stats.num_reads);
734 page = bio->bi_io_vec[0].bv_page;
735 index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
737 if (rzs_test_flag(rzs, index, RZS_ZERO))
738 return handle_zero_page(bio);
740 /* Requested page is not present in compressed area */
741 if (!rzs->table[index].page)
742 return handle_ramzswap_fault(rzs, bio);
744 /* Page is stored uncompressed since it's incompressible */
745 if (unlikely(rzs_test_flag(rzs, index, RZS_UNCOMPRESSED)))
746 return handle_uncompressed_page(rzs, bio);
748 user_mem = kmap_atomic(page, KM_USER0);
751 cmem = kmap_atomic(rzs->table[index].page, KM_USER1) +
752 rzs->table[index].offset;
754 ret = lzo1x_decompress_safe(
755 cmem + sizeof(*zheader),
756 xv_get_object_size(cmem) - sizeof(*zheader),
759 kunmap_atomic(user_mem, KM_USER0);
760 kunmap_atomic(cmem, KM_USER1);
762 /* should NEVER happen */
763 if (unlikely(ret != LZO_E_OK)) {
764 pr_err("Decompression failed! err=%d, page=%u\n",
766 rzs_stat64_inc(rzs, &rzs->stats.failed_reads);
770 flush_dcache_page(page);
772 set_bit(BIO_UPTODATE, &bio->bi_flags);
781 static int ramzswap_write(struct ramzswap *rzs, struct bio *bio)
783 int ret, fwd_write_request = 0;
786 struct zobj_header *zheader;
787 struct page *page, *page_store;
788 unsigned char *user_mem, *cmem, *src;
790 rzs_stat64_inc(rzs, &rzs->stats.num_writes);
792 page = bio->bi_io_vec[0].bv_page;
793 index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
795 src = rzs->compress_buffer;
798 * System swaps to same sector again when the stored page
799 * is no longer referenced by any process. So, its now safe
800 * to free the memory that was allocated for this page.
802 if (rzs->table[index].page || rzs_test_flag(rzs, index, RZS_ZERO))
803 ramzswap_free_page(rzs, index);
805 mutex_lock(&rzs->lock);
807 user_mem = kmap_atomic(page, KM_USER0);
808 if (page_zero_filled(user_mem)) {
809 kunmap_atomic(user_mem, KM_USER0);
810 mutex_unlock(&rzs->lock);
811 rzs_stat_inc(&rzs->stats.pages_zero);
812 rzs_set_flag(rzs, index, RZS_ZERO);
814 set_bit(BIO_UPTODATE, &bio->bi_flags);
819 if (rzs->backing_swap &&
820 (rzs->stats.compr_size > rzs->memlimit - PAGE_SIZE)) {
821 kunmap_atomic(user_mem, KM_USER0);
822 mutex_unlock(&rzs->lock);
823 fwd_write_request = 1;
827 ret = lzo1x_1_compress(user_mem, PAGE_SIZE, src, &clen,
828 rzs->compress_workmem);
830 kunmap_atomic(user_mem, KM_USER0);
832 if (unlikely(ret != LZO_E_OK)) {
833 mutex_unlock(&rzs->lock);
834 pr_err("Compression failed! err=%d\n", ret);
835 rzs_stat64_inc(rzs, &rzs->stats.failed_writes);
840 * Page is incompressible. Forward it to backing swap
841 * if present. Otherwise, store it as-is (uncompressed)
842 * since we do not want to return too many swap write
843 * errors which has side effect of hanging the system.
845 if (unlikely(clen > max_zpage_size)) {
846 if (rzs->backing_swap) {
847 mutex_unlock(&rzs->lock);
848 fwd_write_request = 1;
853 page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
854 if (unlikely(!page_store)) {
855 mutex_unlock(&rzs->lock);
856 pr_info("Error allocating memory for incompressible "
857 "page: %u\n", index);
858 rzs_stat64_inc(rzs, &rzs->stats.failed_writes);
863 rzs_set_flag(rzs, index, RZS_UNCOMPRESSED);
864 rzs_stat_inc(&rzs->stats.pages_expand);
865 rzs->table[index].page = page_store;
866 src = kmap_atomic(page, KM_USER0);
870 if (xv_malloc(rzs->mem_pool, clen + sizeof(*zheader),
871 &rzs->table[index].page, &offset,
872 GFP_NOIO | __GFP_HIGHMEM)) {
873 mutex_unlock(&rzs->lock);
874 pr_info("Error allocating memory for compressed "
875 "page: %u, size=%zu\n", index, clen);
876 rzs_stat64_inc(rzs, &rzs->stats.failed_writes);
877 if (rzs->backing_swap)
878 fwd_write_request = 1;
883 rzs->table[index].offset = offset;
885 cmem = kmap_atomic(rzs->table[index].page, KM_USER1) +
886 rzs->table[index].offset;
889 /* Back-reference needed for memory defragmentation */
890 if (!rzs_test_flag(rzs, index, RZS_UNCOMPRESSED)) {
891 zheader = (struct zobj_header *)cmem;
892 zheader->table_idx = index;
893 cmem += sizeof(*zheader);
897 memcpy(cmem, src, clen);
899 kunmap_atomic(cmem, KM_USER1);
900 if (unlikely(rzs_test_flag(rzs, index, RZS_UNCOMPRESSED)))
901 kunmap_atomic(src, KM_USER0);
904 rzs->stats.compr_size += clen;
905 rzs_stat_inc(&rzs->stats.pages_stored);
906 if (clen <= PAGE_SIZE / 2)
907 rzs_stat_inc(&rzs->stats.good_compress);
909 mutex_unlock(&rzs->lock);
911 set_bit(BIO_UPTODATE, &bio->bi_flags);
916 if (fwd_write_request) {
917 rzs_stat64_inc(rzs, &rzs->stats.bdev_num_writes);
918 bio->bi_bdev = rzs->backing_swap;
921 * TODO: We currently have linear mapping of ramzswap and
922 * backing swap sectors. This is not desired since we want
923 * to optimize writes to backing swap to minimize disk seeks
924 * or have effective wear leveling (for SSDs). Also, a
925 * non-linear mapping is required to implement compressed
928 bio->bi_sector = get_backing_swap_page()
929 << SECTORS_PER_PAGE_SHIFT;
932 * In case backing swap is a file, find the right offset within
933 * the file corresponding to logical position 'index'. For block
934 * device, this is a nop.
936 bio->bi_sector = map_backing_swap_page(rzs, index)
937 << SECTORS_PER_PAGE_SHIFT;
946 * Check if request is within bounds and page aligned.
948 static inline int valid_swap_request(struct ramzswap *rzs, struct bio *bio)
951 (bio->bi_sector >= (rzs->disksize >> SECTOR_SHIFT)) ||
952 (bio->bi_sector & (SECTORS_PER_PAGE - 1)) ||
953 (bio->bi_vcnt != 1) ||
954 (bio->bi_size != PAGE_SIZE) ||
955 (bio->bi_io_vec[0].bv_offset != 0))) {
960 /* swap request is valid */
965 * Handler function for all ramzswap I/O requests.
967 static int ramzswap_make_request(struct request_queue *queue, struct bio *bio)
970 struct ramzswap *rzs = queue->queuedata;
972 if (unlikely(!rzs->init_done)) {
977 if (!valid_swap_request(rzs, bio)) {
978 rzs_stat64_inc(rzs, &rzs->stats.invalid_io);
983 switch (bio_data_dir(bio)) {
985 ret = ramzswap_read(rzs, bio);
989 ret = ramzswap_write(rzs, bio);
996 static void reset_device(struct ramzswap *rzs)
998 int is_backing_blkdev = 0;
999 size_t index, num_pages;
1000 unsigned entries_per_page;
1001 unsigned long num_table_pages, entry = 0;
1003 /* Do not accept any new I/O request */
1006 if (rzs->backing_swap && !rzs->num_extents)
1007 is_backing_blkdev = 1;
1009 num_pages = rzs->disksize >> PAGE_SHIFT;
1011 /* Free various per-device buffers */
1012 kfree(rzs->compress_workmem);
1013 free_pages((unsigned long)rzs->compress_buffer, 1);
1015 rzs->compress_workmem = NULL;
1016 rzs->compress_buffer = NULL;
1018 /* Free all pages that are still in this ramzswap device */
1019 for (index = 0; index < num_pages; index++) {
1023 page = rzs->table[index].page;
1024 offset = rzs->table[index].offset;
1029 if (unlikely(rzs_test_flag(rzs, index, RZS_UNCOMPRESSED)))
1032 xv_free(rzs->mem_pool, page, offset);
1035 entries_per_page = PAGE_SIZE / sizeof(*rzs->table);
1036 num_table_pages = DIV_ROUND_UP(num_pages * sizeof(*rzs->table),
1039 * Set page->mapping to NULL for every table page.
1040 * Otherwise, we will hit bad_page() during free.
1042 while (rzs->num_extents && num_table_pages--) {
1044 page = vmalloc_to_page(&rzs->table[entry]);
1045 page->mapping = NULL;
1046 entry += entries_per_page;
1051 xv_destroy_pool(rzs->mem_pool);
1052 rzs->mem_pool = NULL;
1054 /* Free all swap extent pages */
1055 while (!list_empty(&rzs->backing_swap_extent_list)) {
1057 struct list_head *entry;
1058 entry = rzs->backing_swap_extent_list.next;
1059 page = list_entry(entry, struct page, lru);
1063 INIT_LIST_HEAD(&rzs->backing_swap_extent_list);
1064 rzs->num_extents = 0;
1066 /* Close backing swap device, if present */
1067 if (rzs->backing_swap) {
1068 if (is_backing_blkdev)
1069 bd_release(rzs->backing_swap);
1070 filp_close(rzs->swap_file, NULL);
1071 rzs->backing_swap = NULL;
1072 memset(rzs->backing_swap_name, 0, MAX_SWAP_NAME_LEN);
1076 memset(&rzs->stats, 0, sizeof(rzs->stats));
1082 static int ramzswap_ioctl_init_device(struct ramzswap *rzs)
1087 union swap_header *swap_header;
1089 if (rzs->init_done) {
1090 pr_info("Device already initialized!\n");
1094 ret = setup_backing_swap(rzs);
1098 if (rzs->backing_swap)
1099 ramzswap_set_memlimit(rzs, totalram_pages << PAGE_SHIFT);
1101 ramzswap_set_disksize(rzs, totalram_pages << PAGE_SHIFT);
1103 rzs->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
1104 if (!rzs->compress_workmem) {
1105 pr_err("Error allocating compressor working memory!\n");
1110 rzs->compress_buffer = (void *)__get_free_pages(__GFP_ZERO, 1);
1111 if (!rzs->compress_buffer) {
1112 pr_err("Error allocating compressor buffer space\n");
1117 num_pages = rzs->disksize >> PAGE_SHIFT;
1118 rzs->table = vmalloc(num_pages * sizeof(*rzs->table));
1120 pr_err("Error allocating ramzswap address table\n");
1121 /* To prevent accessing table entries during cleanup */
1126 memset(rzs->table, 0, num_pages * sizeof(*rzs->table));
1128 map_backing_swap_extents(rzs);
1130 page = alloc_page(__GFP_ZERO);
1132 pr_err("Error allocating swap header page\n");
1136 rzs->table[0].page = page;
1137 rzs_set_flag(rzs, 0, RZS_UNCOMPRESSED);
1139 swap_header = kmap(page);
1140 ret = setup_swap_header(rzs, swap_header);
1143 pr_err("Error setting swap header\n");
1147 set_capacity(rzs->disk, rzs->disksize >> SECTOR_SHIFT);
1150 * We have ident mapping of sectors for ramzswap and
1151 * and the backing swap device. So, this queue flag
1152 * should be according to backing dev.
1154 if (!rzs->backing_swap ||
1155 blk_queue_nonrot(rzs->backing_swap->bd_disk->queue))
1156 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rzs->disk->queue);
1158 rzs->mem_pool = xv_create_pool();
1159 if (!rzs->mem_pool) {
1160 pr_err("Error creating memory pool\n");
1166 * Pages that compress to size greater than this are forwarded
1167 * to physical swap disk (if backing dev is provided)
1168 * TODO: make this configurable
1170 if (rzs->backing_swap)
1171 max_zpage_size = max_zpage_size_bdev;
1173 max_zpage_size = max_zpage_size_nobdev;
1174 pr_debug("Max compressed page size: %u bytes\n", max_zpage_size);
1178 pr_debug("Initialization done!\n");
1184 pr_err("Initialization failed: err=%d\n", ret);
1188 static int ramzswap_ioctl_reset_device(struct ramzswap *rzs)
1196 static int ramzswap_ioctl(struct block_device *bdev, fmode_t mode,
1197 unsigned int cmd, unsigned long arg)
1200 size_t disksize_kb, memlimit_kb;
1202 struct ramzswap *rzs = bdev->bd_disk->private_data;
1205 case RZSIO_SET_DISKSIZE_KB:
1206 if (rzs->init_done) {
1210 if (copy_from_user(&disksize_kb, (void *)arg,
1215 rzs->disksize = disksize_kb << 10;
1216 pr_info("Disk size set to %zu kB\n", disksize_kb);
1219 case RZSIO_SET_MEMLIMIT_KB:
1220 if (rzs->init_done) {
1221 /* TODO: allow changing memlimit */
1225 if (copy_from_user(&memlimit_kb, (void *)arg,
1230 rzs->memlimit = memlimit_kb << 10;
1231 pr_info("Memory limit set to %zu kB\n", memlimit_kb);
1234 case RZSIO_SET_BACKING_SWAP:
1235 if (rzs->init_done) {
1240 if (copy_from_user(&rzs->backing_swap_name, (void *)arg,
1245 rzs->backing_swap_name[MAX_SWAP_NAME_LEN - 1] = '\0';
1246 pr_info("Backing swap set to %s\n", rzs->backing_swap_name);
1249 case RZSIO_GET_STATS:
1251 struct ramzswap_ioctl_stats *stats;
1252 if (!rzs->init_done) {
1256 stats = kzalloc(sizeof(*stats), GFP_KERNEL);
1261 ramzswap_ioctl_get_stats(rzs, stats);
1262 if (copy_to_user((void *)arg, stats, sizeof(*stats))) {
1271 ret = ramzswap_ioctl_init_device(rzs);
1275 /* Do not reset an active device! */
1276 if (bdev->bd_holders) {
1281 /* Make sure all pending I/O is finished */
1285 ret = ramzswap_ioctl_reset_device(rzs);
1289 pr_info("Invalid ioctl %u\n", cmd);
1297 static struct block_device_operations ramzswap_devops = {
1298 .ioctl = ramzswap_ioctl,
1299 .owner = THIS_MODULE,
1302 static int create_device(struct ramzswap *rzs, int device_id)
1306 mutex_init(&rzs->lock);
1307 spin_lock_init(&rzs->stat64_lock);
1308 INIT_LIST_HEAD(&rzs->backing_swap_extent_list);
1310 rzs->queue = blk_alloc_queue(GFP_KERNEL);
1312 pr_err("Error allocating disk queue for device %d\n",
1318 blk_queue_make_request(rzs->queue, ramzswap_make_request);
1319 rzs->queue->queuedata = rzs;
1321 /* gendisk structure */
1322 rzs->disk = alloc_disk(1);
1324 blk_cleanup_queue(rzs->queue);
1325 pr_warning("Error allocating disk structure for device %d\n",
1331 rzs->disk->major = ramzswap_major;
1332 rzs->disk->first_minor = device_id;
1333 rzs->disk->fops = &ramzswap_devops;
1334 rzs->disk->queue = rzs->queue;
1335 rzs->disk->private_data = rzs;
1336 snprintf(rzs->disk->disk_name, 16, "ramzswap%d", device_id);
1339 * Actual capacity set using RZSIO_SET_DISKSIZE_KB ioctl
1340 * or set equal to backing swap device (if provided)
1342 set_capacity(rzs->disk, 0);
1344 blk_queue_physical_block_size(rzs->disk->queue, PAGE_SIZE);
1345 blk_queue_logical_block_size(rzs->disk->queue, PAGE_SIZE);
1347 add_disk(rzs->disk);
1355 static void destroy_device(struct ramzswap *rzs)
1358 del_gendisk(rzs->disk);
1359 put_disk(rzs->disk);
1363 blk_cleanup_queue(rzs->queue);
1366 static int __init ramzswap_init(void)
1370 if (num_devices > max_num_devices) {
1371 pr_warning("Invalid value for num_devices: %u\n",
1377 ramzswap_major = register_blkdev(0, "ramzswap");
1378 if (ramzswap_major <= 0) {
1379 pr_warning("Unable to get major number\n");
1385 pr_info("num_devices not specified. Using default: 1\n");
1389 /* Allocate the device array and initialize each one */
1390 pr_info("Creating %u devices ...\n", num_devices);
1391 devices = kzalloc(num_devices * sizeof(struct ramzswap), GFP_KERNEL);
1397 for (dev_id = 0; dev_id < num_devices; dev_id++) {
1398 ret = create_device(&devices[dev_id], dev_id);
1407 destroy_device(&devices[--dev_id]);
1409 unregister_blkdev(ramzswap_major, "ramzswap");
1414 static void __exit ramzswap_exit(void)
1417 struct ramzswap *rzs;
1419 for (i = 0; i < num_devices; i++) {
1422 destroy_device(rzs);
1427 unregister_blkdev(ramzswap_major, "ramzswap");
1430 pr_debug("Cleanup done!\n");
1433 module_param(num_devices, uint, 0);
1434 MODULE_PARM_DESC(num_devices, "Number of ramzswap devices");
1436 module_init(ramzswap_init);
1437 module_exit(ramzswap_exit);
1439 MODULE_LICENSE("Dual BSD/GPL");
1440 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
1441 MODULE_DESCRIPTION("Compressed RAM Based Swap Device");