]> git.karo-electronics.de Git - karo-tx-linux.git/blob - drivers/block/zram/zram_drv.c
Merge branches 'acpi-soc', 'acpi-bus', 'acpi-pmic' and 'acpi-power'
[karo-tx-linux.git] / drivers / block / zram / zram_drv.c
1 /*
2  * Compressed RAM block device
3  *
4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
5  *               2012, 2013 Minchan Kim
6  *
7  * This code is released using a dual license strategy: BSD/GPL
8  * You can choose the licence that better fits your requirements.
9  *
10  * Released under the terms of 3-clause BSD License
11  * Released under the terms of GNU General Public License Version 2.0
12  *
13  */
14
15 #define KMSG_COMPONENT "zram"
16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/bitops.h>
22 #include <linux/blkdev.h>
23 #include <linux/buffer_head.h>
24 #include <linux/device.h>
25 #include <linux/genhd.h>
26 #include <linux/highmem.h>
27 #include <linux/slab.h>
28 #include <linux/backing-dev.h>
29 #include <linux/string.h>
30 #include <linux/vmalloc.h>
31 #include <linux/err.h>
32 #include <linux/idr.h>
33 #include <linux/sysfs.h>
34 #include <linux/cpuhotplug.h>
35
36 #include "zram_drv.h"
37
38 static DEFINE_IDR(zram_index_idr);
39 /* idr index must be protected */
40 static DEFINE_MUTEX(zram_index_mutex);
41
42 static int zram_major;
43 static const char *default_compressor = "lzo";
44
45 /* Module params (documentation at end) */
46 static unsigned int num_devices = 1;
47
48 static inline bool init_done(struct zram *zram)
49 {
50         return zram->disksize;
51 }
52
53 static inline struct zram *dev_to_zram(struct device *dev)
54 {
55         return (struct zram *)dev_to_disk(dev)->private_data;
56 }
57
58 /* flag operations require table entry bit_spin_lock() being held */
59 static int zram_test_flag(struct zram_meta *meta, u32 index,
60                         enum zram_pageflags flag)
61 {
62         return meta->table[index].value & BIT(flag);
63 }
64
65 static void zram_set_flag(struct zram_meta *meta, u32 index,
66                         enum zram_pageflags flag)
67 {
68         meta->table[index].value |= BIT(flag);
69 }
70
71 static void zram_clear_flag(struct zram_meta *meta, u32 index,
72                         enum zram_pageflags flag)
73 {
74         meta->table[index].value &= ~BIT(flag);
75 }
76
77 static inline void zram_set_element(struct zram_meta *meta, u32 index,
78                         unsigned long element)
79 {
80         meta->table[index].element = element;
81 }
82
83 static inline void zram_clear_element(struct zram_meta *meta, u32 index)
84 {
85         meta->table[index].element = 0;
86 }
87
88 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
89 {
90         return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
91 }
92
93 static void zram_set_obj_size(struct zram_meta *meta,
94                                         u32 index, size_t size)
95 {
96         unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
97
98         meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
99 }
100
101 static inline bool is_partial_io(struct bio_vec *bvec)
102 {
103         return bvec->bv_len != PAGE_SIZE;
104 }
105
106 static void zram_revalidate_disk(struct zram *zram)
107 {
108         revalidate_disk(zram->disk);
109         /* revalidate_disk reset the BDI_CAP_STABLE_WRITES so set again */
110         zram->disk->queue->backing_dev_info->capabilities |=
111                 BDI_CAP_STABLE_WRITES;
112 }
113
114 /*
115  * Check if request is within bounds and aligned on zram logical blocks.
116  */
117 static inline bool valid_io_request(struct zram *zram,
118                 sector_t start, unsigned int size)
119 {
120         u64 end, bound;
121
122         /* unaligned request */
123         if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
124                 return false;
125         if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
126                 return false;
127
128         end = start + (size >> SECTOR_SHIFT);
129         bound = zram->disksize >> SECTOR_SHIFT;
130         /* out of range range */
131         if (unlikely(start >= bound || end > bound || start > end))
132                 return false;
133
134         /* I/O request is valid */
135         return true;
136 }
137
138 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
139 {
140         if (*offset + bvec->bv_len >= PAGE_SIZE)
141                 (*index)++;
142         *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
143 }
144
145 static inline void update_used_max(struct zram *zram,
146                                         const unsigned long pages)
147 {
148         unsigned long old_max, cur_max;
149
150         old_max = atomic_long_read(&zram->stats.max_used_pages);
151
152         do {
153                 cur_max = old_max;
154                 if (pages > cur_max)
155                         old_max = atomic_long_cmpxchg(
156                                 &zram->stats.max_used_pages, cur_max, pages);
157         } while (old_max != cur_max);
158 }
159
160 static inline void zram_fill_page(char *ptr, unsigned long len,
161                                         unsigned long value)
162 {
163         int i;
164         unsigned long *page = (unsigned long *)ptr;
165
166         WARN_ON_ONCE(!IS_ALIGNED(len, sizeof(unsigned long)));
167
168         if (likely(value == 0)) {
169                 memset(ptr, 0, len);
170         } else {
171                 for (i = 0; i < len / sizeof(*page); i++)
172                         page[i] = value;
173         }
174 }
175
176 static bool page_same_filled(void *ptr, unsigned long *element)
177 {
178         unsigned int pos;
179         unsigned long *page;
180
181         page = (unsigned long *)ptr;
182
183         for (pos = 0; pos < PAGE_SIZE / sizeof(*page) - 1; pos++) {
184                 if (page[pos] != page[pos + 1])
185                         return false;
186         }
187
188         *element = page[pos];
189
190         return true;
191 }
192
193 static void handle_same_page(struct bio_vec *bvec, unsigned long element)
194 {
195         struct page *page = bvec->bv_page;
196         void *user_mem;
197
198         user_mem = kmap_atomic(page);
199         zram_fill_page(user_mem + bvec->bv_offset, bvec->bv_len, element);
200         kunmap_atomic(user_mem);
201
202         flush_dcache_page(page);
203 }
204
205 static ssize_t initstate_show(struct device *dev,
206                 struct device_attribute *attr, char *buf)
207 {
208         u32 val;
209         struct zram *zram = dev_to_zram(dev);
210
211         down_read(&zram->init_lock);
212         val = init_done(zram);
213         up_read(&zram->init_lock);
214
215         return scnprintf(buf, PAGE_SIZE, "%u\n", val);
216 }
217
218 static ssize_t disksize_show(struct device *dev,
219                 struct device_attribute *attr, char *buf)
220 {
221         struct zram *zram = dev_to_zram(dev);
222
223         return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
224 }
225
226 static ssize_t mem_limit_store(struct device *dev,
227                 struct device_attribute *attr, const char *buf, size_t len)
228 {
229         u64 limit;
230         char *tmp;
231         struct zram *zram = dev_to_zram(dev);
232
233         limit = memparse(buf, &tmp);
234         if (buf == tmp) /* no chars parsed, invalid input */
235                 return -EINVAL;
236
237         down_write(&zram->init_lock);
238         zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
239         up_write(&zram->init_lock);
240
241         return len;
242 }
243
244 static ssize_t mem_used_max_store(struct device *dev,
245                 struct device_attribute *attr, const char *buf, size_t len)
246 {
247         int err;
248         unsigned long val;
249         struct zram *zram = dev_to_zram(dev);
250
251         err = kstrtoul(buf, 10, &val);
252         if (err || val != 0)
253                 return -EINVAL;
254
255         down_read(&zram->init_lock);
256         if (init_done(zram)) {
257                 struct zram_meta *meta = zram->meta;
258                 atomic_long_set(&zram->stats.max_used_pages,
259                                 zs_get_total_pages(meta->mem_pool));
260         }
261         up_read(&zram->init_lock);
262
263         return len;
264 }
265
266 /*
267  * We switched to per-cpu streams and this attr is not needed anymore.
268  * However, we will keep it around for some time, because:
269  * a) we may revert per-cpu streams in the future
270  * b) it's visible to user space and we need to follow our 2 years
271  *    retirement rule; but we already have a number of 'soon to be
272  *    altered' attrs, so max_comp_streams need to wait for the next
273  *    layoff cycle.
274  */
275 static ssize_t max_comp_streams_show(struct device *dev,
276                 struct device_attribute *attr, char *buf)
277 {
278         return scnprintf(buf, PAGE_SIZE, "%d\n", num_online_cpus());
279 }
280
281 static ssize_t max_comp_streams_store(struct device *dev,
282                 struct device_attribute *attr, const char *buf, size_t len)
283 {
284         return len;
285 }
286
287 static ssize_t comp_algorithm_show(struct device *dev,
288                 struct device_attribute *attr, char *buf)
289 {
290         size_t sz;
291         struct zram *zram = dev_to_zram(dev);
292
293         down_read(&zram->init_lock);
294         sz = zcomp_available_show(zram->compressor, buf);
295         up_read(&zram->init_lock);
296
297         return sz;
298 }
299
300 static ssize_t comp_algorithm_store(struct device *dev,
301                 struct device_attribute *attr, const char *buf, size_t len)
302 {
303         struct zram *zram = dev_to_zram(dev);
304         char compressor[CRYPTO_MAX_ALG_NAME];
305         size_t sz;
306
307         strlcpy(compressor, buf, sizeof(compressor));
308         /* ignore trailing newline */
309         sz = strlen(compressor);
310         if (sz > 0 && compressor[sz - 1] == '\n')
311                 compressor[sz - 1] = 0x00;
312
313         if (!zcomp_available_algorithm(compressor))
314                 return -EINVAL;
315
316         down_write(&zram->init_lock);
317         if (init_done(zram)) {
318                 up_write(&zram->init_lock);
319                 pr_info("Can't change algorithm for initialized device\n");
320                 return -EBUSY;
321         }
322
323         strlcpy(zram->compressor, compressor, sizeof(compressor));
324         up_write(&zram->init_lock);
325         return len;
326 }
327
328 static ssize_t compact_store(struct device *dev,
329                 struct device_attribute *attr, const char *buf, size_t len)
330 {
331         struct zram *zram = dev_to_zram(dev);
332         struct zram_meta *meta;
333
334         down_read(&zram->init_lock);
335         if (!init_done(zram)) {
336                 up_read(&zram->init_lock);
337                 return -EINVAL;
338         }
339
340         meta = zram->meta;
341         zs_compact(meta->mem_pool);
342         up_read(&zram->init_lock);
343
344         return len;
345 }
346
347 static ssize_t io_stat_show(struct device *dev,
348                 struct device_attribute *attr, char *buf)
349 {
350         struct zram *zram = dev_to_zram(dev);
351         ssize_t ret;
352
353         down_read(&zram->init_lock);
354         ret = scnprintf(buf, PAGE_SIZE,
355                         "%8llu %8llu %8llu %8llu\n",
356                         (u64)atomic64_read(&zram->stats.failed_reads),
357                         (u64)atomic64_read(&zram->stats.failed_writes),
358                         (u64)atomic64_read(&zram->stats.invalid_io),
359                         (u64)atomic64_read(&zram->stats.notify_free));
360         up_read(&zram->init_lock);
361
362         return ret;
363 }
364
365 static ssize_t mm_stat_show(struct device *dev,
366                 struct device_attribute *attr, char *buf)
367 {
368         struct zram *zram = dev_to_zram(dev);
369         struct zs_pool_stats pool_stats;
370         u64 orig_size, mem_used = 0;
371         long max_used;
372         ssize_t ret;
373
374         memset(&pool_stats, 0x00, sizeof(struct zs_pool_stats));
375
376         down_read(&zram->init_lock);
377         if (init_done(zram)) {
378                 mem_used = zs_get_total_pages(zram->meta->mem_pool);
379                 zs_pool_stats(zram->meta->mem_pool, &pool_stats);
380         }
381
382         orig_size = atomic64_read(&zram->stats.pages_stored);
383         max_used = atomic_long_read(&zram->stats.max_used_pages);
384
385         ret = scnprintf(buf, PAGE_SIZE,
386                         "%8llu %8llu %8llu %8lu %8ld %8llu %8lu\n",
387                         orig_size << PAGE_SHIFT,
388                         (u64)atomic64_read(&zram->stats.compr_data_size),
389                         mem_used << PAGE_SHIFT,
390                         zram->limit_pages << PAGE_SHIFT,
391                         max_used << PAGE_SHIFT,
392                         (u64)atomic64_read(&zram->stats.same_pages),
393                         pool_stats.pages_compacted);
394         up_read(&zram->init_lock);
395
396         return ret;
397 }
398
399 static ssize_t debug_stat_show(struct device *dev,
400                 struct device_attribute *attr, char *buf)
401 {
402         int version = 1;
403         struct zram *zram = dev_to_zram(dev);
404         ssize_t ret;
405
406         down_read(&zram->init_lock);
407         ret = scnprintf(buf, PAGE_SIZE,
408                         "version: %d\n%8llu\n",
409                         version,
410                         (u64)atomic64_read(&zram->stats.writestall));
411         up_read(&zram->init_lock);
412
413         return ret;
414 }
415
416 static DEVICE_ATTR_RO(io_stat);
417 static DEVICE_ATTR_RO(mm_stat);
418 static DEVICE_ATTR_RO(debug_stat);
419
420 static void zram_meta_free(struct zram_meta *meta, u64 disksize)
421 {
422         size_t num_pages = disksize >> PAGE_SHIFT;
423         size_t index;
424
425         /* Free all pages that are still in this zram device */
426         for (index = 0; index < num_pages; index++) {
427                 unsigned long handle = meta->table[index].handle;
428                 /*
429                  * No memory is allocated for same element filled pages.
430                  * Simply clear same page flag.
431                  */
432                 if (!handle || zram_test_flag(meta, index, ZRAM_SAME))
433                         continue;
434
435                 zs_free(meta->mem_pool, handle);
436         }
437
438         zs_destroy_pool(meta->mem_pool);
439         vfree(meta->table);
440         kfree(meta);
441 }
442
443 static struct zram_meta *zram_meta_alloc(char *pool_name, u64 disksize)
444 {
445         size_t num_pages;
446         struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
447
448         if (!meta)
449                 return NULL;
450
451         num_pages = disksize >> PAGE_SHIFT;
452         meta->table = vzalloc(num_pages * sizeof(*meta->table));
453         if (!meta->table) {
454                 pr_err("Error allocating zram address table\n");
455                 goto out_error;
456         }
457
458         meta->mem_pool = zs_create_pool(pool_name);
459         if (!meta->mem_pool) {
460                 pr_err("Error creating memory pool\n");
461                 goto out_error;
462         }
463
464         return meta;
465
466 out_error:
467         vfree(meta->table);
468         kfree(meta);
469         return NULL;
470 }
471
472 /*
473  * To protect concurrent access to the same index entry,
474  * caller should hold this table index entry's bit_spinlock to
475  * indicate this index entry is accessing.
476  */
477 static void zram_free_page(struct zram *zram, size_t index)
478 {
479         struct zram_meta *meta = zram->meta;
480         unsigned long handle = meta->table[index].handle;
481
482         /*
483          * No memory is allocated for same element filled pages.
484          * Simply clear same page flag.
485          */
486         if (zram_test_flag(meta, index, ZRAM_SAME)) {
487                 zram_clear_flag(meta, index, ZRAM_SAME);
488                 zram_clear_element(meta, index);
489                 atomic64_dec(&zram->stats.same_pages);
490                 return;
491         }
492
493         if (!handle)
494                 return;
495
496         zs_free(meta->mem_pool, handle);
497
498         atomic64_sub(zram_get_obj_size(meta, index),
499                         &zram->stats.compr_data_size);
500         atomic64_dec(&zram->stats.pages_stored);
501
502         meta->table[index].handle = 0;
503         zram_set_obj_size(meta, index, 0);
504 }
505
506 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
507 {
508         int ret = 0;
509         unsigned char *cmem;
510         struct zram_meta *meta = zram->meta;
511         unsigned long handle;
512         unsigned int size;
513
514         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
515         handle = meta->table[index].handle;
516         size = zram_get_obj_size(meta, index);
517
518         if (!handle || zram_test_flag(meta, index, ZRAM_SAME)) {
519                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
520                 zram_fill_page(mem, PAGE_SIZE, meta->table[index].element);
521                 return 0;
522         }
523
524         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
525         if (size == PAGE_SIZE) {
526                 memcpy(mem, cmem, PAGE_SIZE);
527         } else {
528                 struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp);
529
530                 ret = zcomp_decompress(zstrm, cmem, size, mem);
531                 zcomp_stream_put(zram->comp);
532         }
533         zs_unmap_object(meta->mem_pool, handle);
534         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
535
536         /* Should NEVER happen. Return bio error if it does. */
537         if (unlikely(ret)) {
538                 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
539                 return ret;
540         }
541
542         return 0;
543 }
544
545 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
546                           u32 index, int offset)
547 {
548         int ret;
549         struct page *page;
550         unsigned char *user_mem, *uncmem = NULL;
551         struct zram_meta *meta = zram->meta;
552         page = bvec->bv_page;
553
554         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
555         if (unlikely(!meta->table[index].handle) ||
556                         zram_test_flag(meta, index, ZRAM_SAME)) {
557                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
558                 handle_same_page(bvec, meta->table[index].element);
559                 return 0;
560         }
561         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
562
563         if (is_partial_io(bvec))
564                 /* Use  a temporary buffer to decompress the page */
565                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
566
567         user_mem = kmap_atomic(page);
568         if (!is_partial_io(bvec))
569                 uncmem = user_mem;
570
571         if (!uncmem) {
572                 pr_err("Unable to allocate temp memory\n");
573                 ret = -ENOMEM;
574                 goto out_cleanup;
575         }
576
577         ret = zram_decompress_page(zram, uncmem, index);
578         /* Should NEVER happen. Return bio error if it does. */
579         if (unlikely(ret))
580                 goto out_cleanup;
581
582         if (is_partial_io(bvec))
583                 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
584                                 bvec->bv_len);
585
586         flush_dcache_page(page);
587         ret = 0;
588 out_cleanup:
589         kunmap_atomic(user_mem);
590         if (is_partial_io(bvec))
591                 kfree(uncmem);
592         return ret;
593 }
594
595 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
596                            int offset)
597 {
598         int ret = 0;
599         unsigned int clen;
600         unsigned long handle = 0;
601         struct page *page;
602         unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
603         struct zram_meta *meta = zram->meta;
604         struct zcomp_strm *zstrm = NULL;
605         unsigned long alloced_pages;
606         unsigned long element;
607
608         page = bvec->bv_page;
609         if (is_partial_io(bvec)) {
610                 /*
611                  * This is a partial IO. We need to read the full page
612                  * before to write the changes.
613                  */
614                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
615                 if (!uncmem) {
616                         ret = -ENOMEM;
617                         goto out;
618                 }
619                 ret = zram_decompress_page(zram, uncmem, index);
620                 if (ret)
621                         goto out;
622         }
623
624 compress_again:
625         user_mem = kmap_atomic(page);
626         if (is_partial_io(bvec)) {
627                 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
628                        bvec->bv_len);
629                 kunmap_atomic(user_mem);
630                 user_mem = NULL;
631         } else {
632                 uncmem = user_mem;
633         }
634
635         if (page_same_filled(uncmem, &element)) {
636                 if (user_mem)
637                         kunmap_atomic(user_mem);
638                 /* Free memory associated with this sector now. */
639                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
640                 zram_free_page(zram, index);
641                 zram_set_flag(meta, index, ZRAM_SAME);
642                 zram_set_element(meta, index, element);
643                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
644
645                 atomic64_inc(&zram->stats.same_pages);
646                 ret = 0;
647                 goto out;
648         }
649
650         zstrm = zcomp_stream_get(zram->comp);
651         ret = zcomp_compress(zstrm, uncmem, &clen);
652         if (!is_partial_io(bvec)) {
653                 kunmap_atomic(user_mem);
654                 user_mem = NULL;
655                 uncmem = NULL;
656         }
657
658         if (unlikely(ret)) {
659                 pr_err("Compression failed! err=%d\n", ret);
660                 goto out;
661         }
662
663         src = zstrm->buffer;
664         if (unlikely(clen > max_zpage_size)) {
665                 clen = PAGE_SIZE;
666                 if (is_partial_io(bvec))
667                         src = uncmem;
668         }
669
670         /*
671          * handle allocation has 2 paths:
672          * a) fast path is executed with preemption disabled (for
673          *  per-cpu streams) and has __GFP_DIRECT_RECLAIM bit clear,
674          *  since we can't sleep;
675          * b) slow path enables preemption and attempts to allocate
676          *  the page with __GFP_DIRECT_RECLAIM bit set. we have to
677          *  put per-cpu compression stream and, thus, to re-do
678          *  the compression once handle is allocated.
679          *
680          * if we have a 'non-null' handle here then we are coming
681          * from the slow path and handle has already been allocated.
682          */
683         if (!handle)
684                 handle = zs_malloc(meta->mem_pool, clen,
685                                 __GFP_KSWAPD_RECLAIM |
686                                 __GFP_NOWARN |
687                                 __GFP_HIGHMEM |
688                                 __GFP_MOVABLE);
689         if (!handle) {
690                 zcomp_stream_put(zram->comp);
691                 zstrm = NULL;
692
693                 atomic64_inc(&zram->stats.writestall);
694
695                 handle = zs_malloc(meta->mem_pool, clen,
696                                 GFP_NOIO | __GFP_HIGHMEM |
697                                 __GFP_MOVABLE);
698                 if (handle)
699                         goto compress_again;
700
701                 pr_err("Error allocating memory for compressed page: %u, size=%u\n",
702                         index, clen);
703                 ret = -ENOMEM;
704                 goto out;
705         }
706
707         alloced_pages = zs_get_total_pages(meta->mem_pool);
708         update_used_max(zram, alloced_pages);
709
710         if (zram->limit_pages && alloced_pages > zram->limit_pages) {
711                 zs_free(meta->mem_pool, handle);
712                 ret = -ENOMEM;
713                 goto out;
714         }
715
716         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
717
718         if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
719                 src = kmap_atomic(page);
720                 memcpy(cmem, src, PAGE_SIZE);
721                 kunmap_atomic(src);
722         } else {
723                 memcpy(cmem, src, clen);
724         }
725
726         zcomp_stream_put(zram->comp);
727         zstrm = NULL;
728         zs_unmap_object(meta->mem_pool, handle);
729
730         /*
731          * Free memory associated with this sector
732          * before overwriting unused sectors.
733          */
734         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
735         zram_free_page(zram, index);
736
737         meta->table[index].handle = handle;
738         zram_set_obj_size(meta, index, clen);
739         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
740
741         /* Update stats */
742         atomic64_add(clen, &zram->stats.compr_data_size);
743         atomic64_inc(&zram->stats.pages_stored);
744 out:
745         if (zstrm)
746                 zcomp_stream_put(zram->comp);
747         if (is_partial_io(bvec))
748                 kfree(uncmem);
749         return ret;
750 }
751
752 /*
753  * zram_bio_discard - handler on discard request
754  * @index: physical block index in PAGE_SIZE units
755  * @offset: byte offset within physical block
756  */
757 static void zram_bio_discard(struct zram *zram, u32 index,
758                              int offset, struct bio *bio)
759 {
760         size_t n = bio->bi_iter.bi_size;
761         struct zram_meta *meta = zram->meta;
762
763         /*
764          * zram manages data in physical block size units. Because logical block
765          * size isn't identical with physical block size on some arch, we
766          * could get a discard request pointing to a specific offset within a
767          * certain physical block.  Although we can handle this request by
768          * reading that physiclal block and decompressing and partially zeroing
769          * and re-compressing and then re-storing it, this isn't reasonable
770          * because our intent with a discard request is to save memory.  So
771          * skipping this logical block is appropriate here.
772          */
773         if (offset) {
774                 if (n <= (PAGE_SIZE - offset))
775                         return;
776
777                 n -= (PAGE_SIZE - offset);
778                 index++;
779         }
780
781         while (n >= PAGE_SIZE) {
782                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
783                 zram_free_page(zram, index);
784                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
785                 atomic64_inc(&zram->stats.notify_free);
786                 index++;
787                 n -= PAGE_SIZE;
788         }
789 }
790
791 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
792                         int offset, bool is_write)
793 {
794         unsigned long start_time = jiffies;
795         int rw_acct = is_write ? REQ_OP_WRITE : REQ_OP_READ;
796         int ret;
797
798         generic_start_io_acct(rw_acct, bvec->bv_len >> SECTOR_SHIFT,
799                         &zram->disk->part0);
800
801         if (!is_write) {
802                 atomic64_inc(&zram->stats.num_reads);
803                 ret = zram_bvec_read(zram, bvec, index, offset);
804         } else {
805                 atomic64_inc(&zram->stats.num_writes);
806                 ret = zram_bvec_write(zram, bvec, index, offset);
807         }
808
809         generic_end_io_acct(rw_acct, &zram->disk->part0, start_time);
810
811         if (unlikely(ret)) {
812                 if (!is_write)
813                         atomic64_inc(&zram->stats.failed_reads);
814                 else
815                         atomic64_inc(&zram->stats.failed_writes);
816         }
817
818         return ret;
819 }
820
821 static void __zram_make_request(struct zram *zram, struct bio *bio)
822 {
823         int offset;
824         u32 index;
825         struct bio_vec bvec;
826         struct bvec_iter iter;
827
828         index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
829         offset = (bio->bi_iter.bi_sector &
830                   (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
831
832         switch (bio_op(bio)) {
833         case REQ_OP_DISCARD:
834         case REQ_OP_WRITE_ZEROES:
835                 zram_bio_discard(zram, index, offset, bio);
836                 bio_endio(bio);
837                 return;
838         default:
839                 break;
840         }
841
842         bio_for_each_segment(bvec, bio, iter) {
843                 int max_transfer_size = PAGE_SIZE - offset;
844
845                 if (bvec.bv_len > max_transfer_size) {
846                         /*
847                          * zram_bvec_rw() can only make operation on a single
848                          * zram page. Split the bio vector.
849                          */
850                         struct bio_vec bv;
851
852                         bv.bv_page = bvec.bv_page;
853                         bv.bv_len = max_transfer_size;
854                         bv.bv_offset = bvec.bv_offset;
855
856                         if (zram_bvec_rw(zram, &bv, index, offset,
857                                          op_is_write(bio_op(bio))) < 0)
858                                 goto out;
859
860                         bv.bv_len = bvec.bv_len - max_transfer_size;
861                         bv.bv_offset += max_transfer_size;
862                         if (zram_bvec_rw(zram, &bv, index + 1, 0,
863                                          op_is_write(bio_op(bio))) < 0)
864                                 goto out;
865                 } else
866                         if (zram_bvec_rw(zram, &bvec, index, offset,
867                                          op_is_write(bio_op(bio))) < 0)
868                                 goto out;
869
870                 update_position(&index, &offset, &bvec);
871         }
872
873         bio_endio(bio);
874         return;
875
876 out:
877         bio_io_error(bio);
878 }
879
880 /*
881  * Handler function for all zram I/O requests.
882  */
883 static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
884 {
885         struct zram *zram = queue->queuedata;
886
887         blk_queue_split(queue, &bio, queue->bio_split);
888
889         if (!valid_io_request(zram, bio->bi_iter.bi_sector,
890                                         bio->bi_iter.bi_size)) {
891                 atomic64_inc(&zram->stats.invalid_io);
892                 goto error;
893         }
894
895         __zram_make_request(zram, bio);
896         return BLK_QC_T_NONE;
897
898 error:
899         bio_io_error(bio);
900         return BLK_QC_T_NONE;
901 }
902
903 static void zram_slot_free_notify(struct block_device *bdev,
904                                 unsigned long index)
905 {
906         struct zram *zram;
907         struct zram_meta *meta;
908
909         zram = bdev->bd_disk->private_data;
910         meta = zram->meta;
911
912         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
913         zram_free_page(zram, index);
914         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
915         atomic64_inc(&zram->stats.notify_free);
916 }
917
918 static int zram_rw_page(struct block_device *bdev, sector_t sector,
919                        struct page *page, bool is_write)
920 {
921         int offset, err = -EIO;
922         u32 index;
923         struct zram *zram;
924         struct bio_vec bv;
925
926         zram = bdev->bd_disk->private_data;
927
928         if (!valid_io_request(zram, sector, PAGE_SIZE)) {
929                 atomic64_inc(&zram->stats.invalid_io);
930                 err = -EINVAL;
931                 goto out;
932         }
933
934         index = sector >> SECTORS_PER_PAGE_SHIFT;
935         offset = (sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
936
937         bv.bv_page = page;
938         bv.bv_len = PAGE_SIZE;
939         bv.bv_offset = 0;
940
941         err = zram_bvec_rw(zram, &bv, index, offset, is_write);
942 out:
943         /*
944          * If I/O fails, just return error(ie, non-zero) without
945          * calling page_endio.
946          * It causes resubmit the I/O with bio request by upper functions
947          * of rw_page(e.g., swap_readpage, __swap_writepage) and
948          * bio->bi_end_io does things to handle the error
949          * (e.g., SetPageError, set_page_dirty and extra works).
950          */
951         if (err == 0)
952                 page_endio(page, is_write, 0);
953         return err;
954 }
955
956 static void zram_reset_device(struct zram *zram)
957 {
958         struct zram_meta *meta;
959         struct zcomp *comp;
960         u64 disksize;
961
962         down_write(&zram->init_lock);
963
964         zram->limit_pages = 0;
965
966         if (!init_done(zram)) {
967                 up_write(&zram->init_lock);
968                 return;
969         }
970
971         meta = zram->meta;
972         comp = zram->comp;
973         disksize = zram->disksize;
974
975         /* Reset stats */
976         memset(&zram->stats, 0, sizeof(zram->stats));
977         zram->disksize = 0;
978
979         set_capacity(zram->disk, 0);
980         part_stat_set_all(&zram->disk->part0, 0);
981
982         up_write(&zram->init_lock);
983         /* I/O operation under all of CPU are done so let's free */
984         zram_meta_free(meta, disksize);
985         zcomp_destroy(comp);
986 }
987
988 static ssize_t disksize_store(struct device *dev,
989                 struct device_attribute *attr, const char *buf, size_t len)
990 {
991         u64 disksize;
992         struct zcomp *comp;
993         struct zram_meta *meta;
994         struct zram *zram = dev_to_zram(dev);
995         int err;
996
997         disksize = memparse(buf, NULL);
998         if (!disksize)
999                 return -EINVAL;
1000
1001         disksize = PAGE_ALIGN(disksize);
1002         meta = zram_meta_alloc(zram->disk->disk_name, disksize);
1003         if (!meta)
1004                 return -ENOMEM;
1005
1006         comp = zcomp_create(zram->compressor);
1007         if (IS_ERR(comp)) {
1008                 pr_err("Cannot initialise %s compressing backend\n",
1009                                 zram->compressor);
1010                 err = PTR_ERR(comp);
1011                 goto out_free_meta;
1012         }
1013
1014         down_write(&zram->init_lock);
1015         if (init_done(zram)) {
1016                 pr_info("Cannot change disksize for initialized device\n");
1017                 err = -EBUSY;
1018                 goto out_destroy_comp;
1019         }
1020
1021         zram->meta = meta;
1022         zram->comp = comp;
1023         zram->disksize = disksize;
1024         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
1025         zram_revalidate_disk(zram);
1026         up_write(&zram->init_lock);
1027
1028         return len;
1029
1030 out_destroy_comp:
1031         up_write(&zram->init_lock);
1032         zcomp_destroy(comp);
1033 out_free_meta:
1034         zram_meta_free(meta, disksize);
1035         return err;
1036 }
1037
1038 static ssize_t reset_store(struct device *dev,
1039                 struct device_attribute *attr, const char *buf, size_t len)
1040 {
1041         int ret;
1042         unsigned short do_reset;
1043         struct zram *zram;
1044         struct block_device *bdev;
1045
1046         ret = kstrtou16(buf, 10, &do_reset);
1047         if (ret)
1048                 return ret;
1049
1050         if (!do_reset)
1051                 return -EINVAL;
1052
1053         zram = dev_to_zram(dev);
1054         bdev = bdget_disk(zram->disk, 0);
1055         if (!bdev)
1056                 return -ENOMEM;
1057
1058         mutex_lock(&bdev->bd_mutex);
1059         /* Do not reset an active device or claimed device */
1060         if (bdev->bd_openers || zram->claim) {
1061                 mutex_unlock(&bdev->bd_mutex);
1062                 bdput(bdev);
1063                 return -EBUSY;
1064         }
1065
1066         /* From now on, anyone can't open /dev/zram[0-9] */
1067         zram->claim = true;
1068         mutex_unlock(&bdev->bd_mutex);
1069
1070         /* Make sure all the pending I/O are finished */
1071         fsync_bdev(bdev);
1072         zram_reset_device(zram);
1073         zram_revalidate_disk(zram);
1074         bdput(bdev);
1075
1076         mutex_lock(&bdev->bd_mutex);
1077         zram->claim = false;
1078         mutex_unlock(&bdev->bd_mutex);
1079
1080         return len;
1081 }
1082
1083 static int zram_open(struct block_device *bdev, fmode_t mode)
1084 {
1085         int ret = 0;
1086         struct zram *zram;
1087
1088         WARN_ON(!mutex_is_locked(&bdev->bd_mutex));
1089
1090         zram = bdev->bd_disk->private_data;
1091         /* zram was claimed to reset so open request fails */
1092         if (zram->claim)
1093                 ret = -EBUSY;
1094
1095         return ret;
1096 }
1097
1098 static const struct block_device_operations zram_devops = {
1099         .open = zram_open,
1100         .swap_slot_free_notify = zram_slot_free_notify,
1101         .rw_page = zram_rw_page,
1102         .owner = THIS_MODULE
1103 };
1104
1105 static DEVICE_ATTR_WO(compact);
1106 static DEVICE_ATTR_RW(disksize);
1107 static DEVICE_ATTR_RO(initstate);
1108 static DEVICE_ATTR_WO(reset);
1109 static DEVICE_ATTR_WO(mem_limit);
1110 static DEVICE_ATTR_WO(mem_used_max);
1111 static DEVICE_ATTR_RW(max_comp_streams);
1112 static DEVICE_ATTR_RW(comp_algorithm);
1113
1114 static struct attribute *zram_disk_attrs[] = {
1115         &dev_attr_disksize.attr,
1116         &dev_attr_initstate.attr,
1117         &dev_attr_reset.attr,
1118         &dev_attr_compact.attr,
1119         &dev_attr_mem_limit.attr,
1120         &dev_attr_mem_used_max.attr,
1121         &dev_attr_max_comp_streams.attr,
1122         &dev_attr_comp_algorithm.attr,
1123         &dev_attr_io_stat.attr,
1124         &dev_attr_mm_stat.attr,
1125         &dev_attr_debug_stat.attr,
1126         NULL,
1127 };
1128
1129 static struct attribute_group zram_disk_attr_group = {
1130         .attrs = zram_disk_attrs,
1131 };
1132
1133 /*
1134  * Allocate and initialize new zram device. the function returns
1135  * '>= 0' device_id upon success, and negative value otherwise.
1136  */
1137 static int zram_add(void)
1138 {
1139         struct zram *zram;
1140         struct request_queue *queue;
1141         int ret, device_id;
1142
1143         zram = kzalloc(sizeof(struct zram), GFP_KERNEL);
1144         if (!zram)
1145                 return -ENOMEM;
1146
1147         ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
1148         if (ret < 0)
1149                 goto out_free_dev;
1150         device_id = ret;
1151
1152         init_rwsem(&zram->init_lock);
1153
1154         queue = blk_alloc_queue(GFP_KERNEL);
1155         if (!queue) {
1156                 pr_err("Error allocating disk queue for device %d\n",
1157                         device_id);
1158                 ret = -ENOMEM;
1159                 goto out_free_idr;
1160         }
1161
1162         blk_queue_make_request(queue, zram_make_request);
1163
1164         /* gendisk structure */
1165         zram->disk = alloc_disk(1);
1166         if (!zram->disk) {
1167                 pr_err("Error allocating disk structure for device %d\n",
1168                         device_id);
1169                 ret = -ENOMEM;
1170                 goto out_free_queue;
1171         }
1172
1173         zram->disk->major = zram_major;
1174         zram->disk->first_minor = device_id;
1175         zram->disk->fops = &zram_devops;
1176         zram->disk->queue = queue;
1177         zram->disk->queue->queuedata = zram;
1178         zram->disk->private_data = zram;
1179         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
1180
1181         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
1182         set_capacity(zram->disk, 0);
1183         /* zram devices sort of resembles non-rotational disks */
1184         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
1185         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
1186         /*
1187          * To ensure that we always get PAGE_SIZE aligned
1188          * and n*PAGE_SIZED sized I/O requests.
1189          */
1190         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
1191         blk_queue_logical_block_size(zram->disk->queue,
1192                                         ZRAM_LOGICAL_BLOCK_SIZE);
1193         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1194         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
1195         zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1196         zram->disk->queue->limits.max_sectors = SECTORS_PER_PAGE;
1197         zram->disk->queue->limits.chunk_sectors = 0;
1198         blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX);
1199         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
1200
1201         /*
1202          * zram_bio_discard() will clear all logical blocks if logical block
1203          * size is identical with physical block size(PAGE_SIZE). But if it is
1204          * different, we will skip discarding some parts of logical blocks in
1205          * the part of the request range which isn't aligned to physical block
1206          * size.  So we can't ensure that all discarded logical blocks are
1207          * zeroed.
1208          */
1209         if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
1210                 blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX);
1211
1212         add_disk(zram->disk);
1213
1214         ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
1215                                 &zram_disk_attr_group);
1216         if (ret < 0) {
1217                 pr_err("Error creating sysfs group for device %d\n",
1218                                 device_id);
1219                 goto out_free_disk;
1220         }
1221         strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
1222         zram->meta = NULL;
1223
1224         pr_info("Added device: %s\n", zram->disk->disk_name);
1225         return device_id;
1226
1227 out_free_disk:
1228         del_gendisk(zram->disk);
1229         put_disk(zram->disk);
1230 out_free_queue:
1231         blk_cleanup_queue(queue);
1232 out_free_idr:
1233         idr_remove(&zram_index_idr, device_id);
1234 out_free_dev:
1235         kfree(zram);
1236         return ret;
1237 }
1238
1239 static int zram_remove(struct zram *zram)
1240 {
1241         struct block_device *bdev;
1242
1243         bdev = bdget_disk(zram->disk, 0);
1244         if (!bdev)
1245                 return -ENOMEM;
1246
1247         mutex_lock(&bdev->bd_mutex);
1248         if (bdev->bd_openers || zram->claim) {
1249                 mutex_unlock(&bdev->bd_mutex);
1250                 bdput(bdev);
1251                 return -EBUSY;
1252         }
1253
1254         zram->claim = true;
1255         mutex_unlock(&bdev->bd_mutex);
1256
1257         /*
1258          * Remove sysfs first, so no one will perform a disksize
1259          * store while we destroy the devices. This also helps during
1260          * hot_remove -- zram_reset_device() is the last holder of
1261          * ->init_lock, no later/concurrent disksize_store() or any
1262          * other sysfs handlers are possible.
1263          */
1264         sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
1265                         &zram_disk_attr_group);
1266
1267         /* Make sure all the pending I/O are finished */
1268         fsync_bdev(bdev);
1269         zram_reset_device(zram);
1270         bdput(bdev);
1271
1272         pr_info("Removed device: %s\n", zram->disk->disk_name);
1273
1274         blk_cleanup_queue(zram->disk->queue);
1275         del_gendisk(zram->disk);
1276         put_disk(zram->disk);
1277         kfree(zram);
1278         return 0;
1279 }
1280
1281 /* zram-control sysfs attributes */
1282 static ssize_t hot_add_show(struct class *class,
1283                         struct class_attribute *attr,
1284                         char *buf)
1285 {
1286         int ret;
1287
1288         mutex_lock(&zram_index_mutex);
1289         ret = zram_add();
1290         mutex_unlock(&zram_index_mutex);
1291
1292         if (ret < 0)
1293                 return ret;
1294         return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
1295 }
1296
1297 static ssize_t hot_remove_store(struct class *class,
1298                         struct class_attribute *attr,
1299                         const char *buf,
1300                         size_t count)
1301 {
1302         struct zram *zram;
1303         int ret, dev_id;
1304
1305         /* dev_id is gendisk->first_minor, which is `int' */
1306         ret = kstrtoint(buf, 10, &dev_id);
1307         if (ret)
1308                 return ret;
1309         if (dev_id < 0)
1310                 return -EINVAL;
1311
1312         mutex_lock(&zram_index_mutex);
1313
1314         zram = idr_find(&zram_index_idr, dev_id);
1315         if (zram) {
1316                 ret = zram_remove(zram);
1317                 if (!ret)
1318                         idr_remove(&zram_index_idr, dev_id);
1319         } else {
1320                 ret = -ENODEV;
1321         }
1322
1323         mutex_unlock(&zram_index_mutex);
1324         return ret ? ret : count;
1325 }
1326
1327 /*
1328  * NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a
1329  * sense that reading from this file does alter the state of your system -- it
1330  * creates a new un-initialized zram device and returns back this device's
1331  * device_id (or an error code if it fails to create a new device).
1332  */
1333 static struct class_attribute zram_control_class_attrs[] = {
1334         __ATTR(hot_add, 0400, hot_add_show, NULL),
1335         __ATTR_WO(hot_remove),
1336         __ATTR_NULL,
1337 };
1338
1339 static struct class zram_control_class = {
1340         .name           = "zram-control",
1341         .owner          = THIS_MODULE,
1342         .class_attrs    = zram_control_class_attrs,
1343 };
1344
1345 static int zram_remove_cb(int id, void *ptr, void *data)
1346 {
1347         zram_remove(ptr);
1348         return 0;
1349 }
1350
1351 static void destroy_devices(void)
1352 {
1353         class_unregister(&zram_control_class);
1354         idr_for_each(&zram_index_idr, &zram_remove_cb, NULL);
1355         idr_destroy(&zram_index_idr);
1356         unregister_blkdev(zram_major, "zram");
1357         cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
1358 }
1359
1360 static int __init zram_init(void)
1361 {
1362         int ret;
1363
1364         ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare",
1365                                       zcomp_cpu_up_prepare, zcomp_cpu_dead);
1366         if (ret < 0)
1367                 return ret;
1368
1369         ret = class_register(&zram_control_class);
1370         if (ret) {
1371                 pr_err("Unable to register zram-control class\n");
1372                 cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
1373                 return ret;
1374         }
1375
1376         zram_major = register_blkdev(0, "zram");
1377         if (zram_major <= 0) {
1378                 pr_err("Unable to get major number\n");
1379                 class_unregister(&zram_control_class);
1380                 cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
1381                 return -EBUSY;
1382         }
1383
1384         while (num_devices != 0) {
1385                 mutex_lock(&zram_index_mutex);
1386                 ret = zram_add();
1387                 mutex_unlock(&zram_index_mutex);
1388                 if (ret < 0)
1389                         goto out_error;
1390                 num_devices--;
1391         }
1392
1393         return 0;
1394
1395 out_error:
1396         destroy_devices();
1397         return ret;
1398 }
1399
1400 static void __exit zram_exit(void)
1401 {
1402         destroy_devices();
1403 }
1404
1405 module_init(zram_init);
1406 module_exit(zram_exit);
1407
1408 module_param(num_devices, uint, 0);
1409 MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices");
1410
1411 MODULE_LICENSE("Dual BSD/GPL");
1412 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
1413 MODULE_DESCRIPTION("Compressed RAM Block Device");