drivers/block/zram/zram_drv.c

   1 /*
   2  * Compressed RAM block device
   3  *
   4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
   5  *               2012, 2013 Minchan Kim
   6  *
   7  * This code is released using a dual license strategy: BSD/GPL
   8  * You can choose the licence that better fits your requirements.
   9  *
  10  * Released under the terms of 3-clause BSD License
  11  * Released under the terms of GNU General Public License Version 2.0
  12  *
  13  */
  14
  15 #define KMSG_COMPONENT "zram"
  16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  17
  18 #ifdef CONFIG_ZRAM_DEBUG
  19 #define DEBUG
  20 #endif
  21
  22 #include <linux/module.h>
  23 #include <linux/kernel.h>
  24 #include <linux/bio.h>
  25 #include <linux/bitops.h>
  26 #include <linux/blkdev.h>
  27 #include <linux/buffer_head.h>
  28 #include <linux/device.h>
  29 #include <linux/genhd.h>
  30 #include <linux/highmem.h>
  31 #include <linux/slab.h>
  32 #include <linux/string.h>
  33 #include <linux/vmalloc.h>
  34 #include <linux/err.h>
  35
  36 #include "zram_drv.h"
  37
  38 /* Globals */
  39 static int zram_major;
  40 static struct zram *zram_devices;
  41 static const char *default_compressor = "lzo";
  42
  43 /* Module params (documentation at end) */
  44 static unsigned int num_devices = 1;
  45
  46 #define ZRAM_ATTR_RO(name)                                              \
  47 static ssize_t name##_show(struct device *d,            \
  48                                 struct device_attribute *attr, char *b) \
  49 {                                                                       \
  50         struct zram *zram = dev_to_zram(d);                             \
  51         return scnprintf(b, PAGE_SIZE, "%llu\n",                        \
  52                 (u64)atomic64_read(&zram->stats.name));                 \
  53 }                                                                       \
  54 static DEVICE_ATTR_RO(name);
  55
  56 static inline int init_done(struct zram *zram)
  57 {
  58         return zram->meta != NULL;
  59 }
  60
  61 static inline struct zram *dev_to_zram(struct device *dev)
  62 {
  63         return (struct zram *)dev_to_disk(dev)->private_data;
  64 }
  65
  66 static ssize_t disksize_show(struct device *dev,
  67                 struct device_attribute *attr, char *buf)
  68 {
  69         struct zram *zram = dev_to_zram(dev);
  70
  71         return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
  72 }
  73
  74 static ssize_t initstate_show(struct device *dev,
  75                 struct device_attribute *attr, char *buf)
  76 {
  77         u32 val;
  78         struct zram *zram = dev_to_zram(dev);
  79
  80         down_read(&zram->init_lock);
  81         val = init_done(zram);
  82         up_read(&zram->init_lock);
  83
  84         return scnprintf(buf, PAGE_SIZE, "%u\n", val);
  85 }
  86
  87 static ssize_t orig_data_size_show(struct device *dev,
  88                 struct device_attribute *attr, char *buf)
  89 {
  90         struct zram *zram = dev_to_zram(dev);
  91
  92         return scnprintf(buf, PAGE_SIZE, "%llu\n",
  93                 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
  94 }
  95
  96 static ssize_t mem_used_total_show(struct device *dev,
  97                 struct device_attribute *attr, char *buf)
  98 {
  99         u64 val = 0;
 100         struct zram *zram = dev_to_zram(dev);
 101
 102         down_read(&zram->init_lock);
 103         if (init_done(zram)) {
 104                 struct zram_meta *meta = zram->meta;
 105                 val = zs_get_total_pages(meta->mem_pool);
 106         }
 107         up_read(&zram->init_lock);
 108
 109         return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
 110 }
 111
 112 static ssize_t max_comp_streams_show(struct device *dev,
 113                 struct device_attribute *attr, char *buf)
 114 {
 115         int val;
 116         struct zram *zram = dev_to_zram(dev);
 117
 118         down_read(&zram->init_lock);
 119         val = zram->max_comp_streams;
 120         up_read(&zram->init_lock);
 121
 122         return scnprintf(buf, PAGE_SIZE, "%d\n", val);
 123 }
 124
 125 static ssize_t mem_limit_show(struct device *dev,
 126                 struct device_attribute *attr, char *buf)
 127 {
 128         u64 val;
 129         struct zram *zram = dev_to_zram(dev);
 130
 131         down_read(&zram->init_lock);
 132         val = zram->limit_pages;
 133         up_read(&zram->init_lock);
 134
 135         return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
 136 }
 137
 138 static ssize_t mem_limit_store(struct device *dev,
 139                 struct device_attribute *attr, const char *buf, size_t len)
 140 {
 141         u64 limit;
 142         char *tmp;
 143         struct zram *zram = dev_to_zram(dev);
 144
 145         limit = memparse(buf, &tmp);
 146         if (buf == tmp) /* no chars parsed, invalid input */
 147                 return -EINVAL;
 148
 149         down_write(&zram->init_lock);
 150         zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
 151         up_write(&zram->init_lock);
 152
 153         return len;
 154 }
 155
 156 static ssize_t mem_used_max_show(struct device *dev,
 157                 struct device_attribute *attr, char *buf)
 158 {
 159         u64 val = 0;
 160         struct zram *zram = dev_to_zram(dev);
 161
 162         down_read(&zram->init_lock);
 163         if (init_done(zram))
 164                 val = atomic_long_read(&zram->stats.max_used_pages);
 165         up_read(&zram->init_lock);
 166
 167         return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
 168 }
 169
 170 static ssize_t mem_used_max_store(struct device *dev,
 171                 struct device_attribute *attr, const char *buf, size_t len)
 172 {
 173         int err;
 174         unsigned long val;
 175         struct zram *zram = dev_to_zram(dev);
 176
 177         err = kstrtoul(buf, 10, &val);
 178         if (err || val != 0)
 179                 return -EINVAL;
 180
 181         down_read(&zram->init_lock);
 182         if (init_done(zram)) {
 183                 struct zram_meta *meta = zram->meta;
 184                 atomic_long_set(&zram->stats.max_used_pages,
 185                                 zs_get_total_pages(meta->mem_pool));
 186         }
 187         up_read(&zram->init_lock);
 188
 189         return len;
 190 }
 191
 192 static ssize_t max_comp_streams_store(struct device *dev,
 193                 struct device_attribute *attr, const char *buf, size_t len)
 194 {
 195         int num;
 196         struct zram *zram = dev_to_zram(dev);
 197         int ret;
 198
 199         ret = kstrtoint(buf, 0, &num);
 200         if (ret < 0)
 201                 return ret;
 202         if (num < 1)
 203                 return -EINVAL;
 204
 205         down_write(&zram->init_lock);
 206         if (init_done(zram)) {
 207                 if (!zcomp_set_max_streams(zram->comp, num)) {
 208                         pr_info("Cannot change max compression streams\n");
 209                         ret = -EINVAL;
 210                         goto out;
 211                 }
 212         }
 213
 214         zram->max_comp_streams = num;
 215         ret = len;
 216 out:
 217         up_write(&zram->init_lock);
 218         return ret;
 219 }
 220
 221 static ssize_t comp_algorithm_show(struct device *dev,
 222                 struct device_attribute *attr, char *buf)
 223 {
 224         size_t sz;
 225         struct zram *zram = dev_to_zram(dev);
 226
 227         down_read(&zram->init_lock);
 228         sz = zcomp_available_show(zram->compressor, buf);
 229         up_read(&zram->init_lock);
 230
 231         return sz;
 232 }
 233
 234 static ssize_t comp_algorithm_store(struct device *dev,
 235                 struct device_attribute *attr, const char *buf, size_t len)
 236 {
 237         struct zram *zram = dev_to_zram(dev);
 238         down_write(&zram->init_lock);
 239         if (init_done(zram)) {
 240                 up_write(&zram->init_lock);
 241                 pr_info("Can't change algorithm for initialized device\n");
 242                 return -EBUSY;
 243         }
 244         strlcpy(zram->compressor, buf, sizeof(zram->compressor));
 245         up_write(&zram->init_lock);
 246         return len;
 247 }
 248
 249 /* flag operations needs meta->tb_lock */
 250 static int zram_test_flag(struct zram_meta *meta, u32 index,
 251                         enum zram_pageflags flag)
 252 {
 253         return meta->table[index].value & BIT(flag);
 254 }
 255
 256 static void zram_set_flag(struct zram_meta *meta, u32 index,
 257                         enum zram_pageflags flag)
 258 {
 259         meta->table[index].value |= BIT(flag);
 260 }
 261
 262 static void zram_clear_flag(struct zram_meta *meta, u32 index,
 263                         enum zram_pageflags flag)
 264 {
 265         meta->table[index].value &= ~BIT(flag);
 266 }
 267
 268 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
 269 {
 270         return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
 271 }
 272
 273 static void zram_set_obj_size(struct zram_meta *meta,
 274                                         u32 index, size_t size)
 275 {
 276         unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
 277
 278         meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
 279 }
 280
 281 static inline int is_partial_io(struct bio_vec *bvec)
 282 {
 283         return bvec->bv_len != PAGE_SIZE;
 284 }
 285
 286 /*
 287  * Check if request is within bounds and aligned on zram logical blocks.
 288  */
 289 static inline int valid_io_request(struct zram *zram,
 290                 sector_t start, unsigned int size)
 291 {
 292         u64 end, bound;
 293
 294         /* unaligned request */
 295         if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
 296                 return 0;
 297         if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
 298                 return 0;
 299
 300         end = start + (size >> SECTOR_SHIFT);
 301         bound = zram->disksize >> SECTOR_SHIFT;
 302         /* out of range range */
 303         if (unlikely(start >= bound || end > bound || start > end))
 304                 return 0;
 305
 306         /* I/O request is valid */
 307         return 1;
 308 }
 309
 310 static void zram_meta_free(struct zram_meta *meta, u64 disksize)
 311 {
 312         size_t num_pages = disksize >> PAGE_SHIFT;
 313         size_t index;
 314
 315         /* Free all pages that are still in this zram device */
 316         for (index = 0; index < num_pages; index++) {
 317                 unsigned long handle = meta->table[index].handle;
 318
 319                 if (!handle)
 320                         continue;
 321
 322                 zs_free(meta->mem_pool, handle);
 323         }
 324
 325         zs_destroy_pool(meta->mem_pool);
 326         vfree(meta->table);
 327         kfree(meta);
 328 }
 329
 330 static struct zram_meta *zram_meta_alloc(u64 disksize)
 331 {
 332         size_t num_pages;
 333         struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
 334
 335         if (!meta)
 336                 return NULL;
 337
 338         num_pages = disksize >> PAGE_SHIFT;
 339         meta->table = vzalloc(num_pages * sizeof(*meta->table));
 340         if (!meta->table) {
 341                 pr_err("Error allocating zram address table\n");
 342                 goto out_error;
 343         }
 344
 345         meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
 346         if (!meta->mem_pool) {
 347                 pr_err("Error creating memory pool\n");
 348                 goto out_error;
 349         }
 350
 351         return meta;
 352
 353 out_error:
 354         vfree(meta->table);
 355         kfree(meta);
 356         return NULL;
 357 }
 358
 359 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
 360 {
 361         if (*offset + bvec->bv_len >= PAGE_SIZE)
 362                 (*index)++;
 363         *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
 364 }
 365
 366 static int page_zero_filled(void *ptr)
 367 {
 368         unsigned int pos;
 369         unsigned long *page;
 370
 371         page = (unsigned long *)ptr;
 372
 373         for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
 374                 if (page[pos])
 375                         return 0;
 376         }
 377
 378         return 1;
 379 }
 380
 381 static void handle_zero_page(struct bio_vec *bvec)
 382 {
 383         struct page *page = bvec->bv_page;
 384         void *user_mem;
 385
 386         user_mem = kmap_atomic(page);
 387         if (is_partial_io(bvec))
 388                 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
 389         else
 390                 clear_page(user_mem);
 391         kunmap_atomic(user_mem);
 392
 393         flush_dcache_page(page);
 394 }
 395
 396
 397 /*
 398  * To protect concurrent access to the same index entry,
 399  * caller should hold this table index entry's bit_spinlock to
 400  * indicate this index entry is accessing.
 401  */
 402 static void zram_free_page(struct zram *zram, size_t index)
 403 {
 404         struct zram_meta *meta = zram->meta;
 405         unsigned long handle = meta->table[index].handle;
 406
 407         if (unlikely(!handle)) {
 408                 /*
 409                  * No memory is allocated for zero filled pages.
 410                  * Simply clear zero page flag.
 411                  */
 412                 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
 413                         zram_clear_flag(meta, index, ZRAM_ZERO);
 414                         atomic64_dec(&zram->stats.zero_pages);
 415                 }
 416                 return;
 417         }
 418
 419         zs_free(meta->mem_pool, handle);
 420
 421         atomic64_sub(zram_get_obj_size(meta, index),
 422                         &zram->stats.compr_data_size);
 423         atomic64_dec(&zram->stats.pages_stored);
 424
 425         meta->table[index].handle = 0;
 426         zram_set_obj_size(meta, index, 0);
 427 }
 428
 429 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
 430 {
 431         int ret = 0;
 432         unsigned char *cmem;
 433         struct zram_meta *meta = zram->meta;
 434         unsigned long handle;
 435         size_t size;
 436
 437         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 438         handle = meta->table[index].handle;
 439         size = zram_get_obj_size(meta, index);
 440
 441         if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
 442                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 443                 clear_page(mem);
 444                 return 0;
 445         }
 446
 447         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
 448         if (size == PAGE_SIZE)
 449                 copy_page(mem, cmem);
 450         else
 451                 ret = zcomp_decompress(zram->comp, cmem, size, mem);
 452         zs_unmap_object(meta->mem_pool, handle);
 453         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 454
 455         /* Should NEVER happen. Return bio error if it does. */
 456         if (unlikely(ret)) {
 457                 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
 458                 return ret;
 459         }
 460
 461         return 0;
 462 }
 463
 464 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
 465                           u32 index, int offset)
 466 {
 467         int ret;
 468         struct page *page;
 469         unsigned char *user_mem, *uncmem = NULL;
 470         struct zram_meta *meta = zram->meta;
 471         page = bvec->bv_page;
 472
 473         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 474         if (unlikely(!meta->table[index].handle) ||
 475                         zram_test_flag(meta, index, ZRAM_ZERO)) {
 476                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 477                 handle_zero_page(bvec);
 478                 return 0;
 479         }
 480         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 481
 482         if (is_partial_io(bvec))
 483                 /* Use  a temporary buffer to decompress the page */
 484                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 485
 486         user_mem = kmap_atomic(page);
 487         if (!is_partial_io(bvec))
 488                 uncmem = user_mem;
 489
 490         if (!uncmem) {
 491                 pr_info("Unable to allocate temp memory\n");
 492                 ret = -ENOMEM;
 493                 goto out_cleanup;
 494         }
 495
 496         ret = zram_decompress_page(zram, uncmem, index);
 497         /* Should NEVER happen. Return bio error if it does. */
 498         if (unlikely(ret))
 499                 goto out_cleanup;
 500
 501         if (is_partial_io(bvec))
 502                 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
 503                                 bvec->bv_len);
 504
 505         flush_dcache_page(page);
 506         ret = 0;
 507 out_cleanup:
 508         kunmap_atomic(user_mem);
 509         if (is_partial_io(bvec))
 510                 kfree(uncmem);
 511         return ret;
 512 }
 513
 514 static inline void update_used_max(struct zram *zram,
 515                                         const unsigned long pages)
 516 {
 517         int old_max, cur_max;
 518
 519         old_max = atomic_long_read(&zram->stats.max_used_pages);
 520
 521         do {
 522                 cur_max = old_max;
 523                 if (pages > cur_max)
 524                         old_max = atomic_long_cmpxchg(
 525                                 &zram->stats.max_used_pages, cur_max, pages);
 526         } while (old_max != cur_max);
 527 }
 528
 529 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
 530                            int offset)
 531 {
 532         int ret = 0;
 533         size_t clen;
 534         unsigned long handle;
 535         struct page *page;
 536         unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
 537         struct zram_meta *meta = zram->meta;
 538         struct zcomp_strm *zstrm;
 539         bool locked = false;
 540         unsigned long alloced_pages;
 541
 542         page = bvec->bv_page;
 543         if (is_partial_io(bvec)) {
 544                 /*
 545                  * This is a partial IO. We need to read the full page
 546                  * before to write the changes.
 547                  */
 548                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 549                 if (!uncmem) {
 550                         ret = -ENOMEM;
 551                         goto out;
 552                 }
 553                 ret = zram_decompress_page(zram, uncmem, index);
 554                 if (ret)
 555                         goto out;
 556         }
 557
 558         zstrm = zcomp_strm_find(zram->comp);
 559         locked = true;
 560         user_mem = kmap_atomic(page);
 561
 562         if (is_partial_io(bvec)) {
 563                 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
 564                        bvec->bv_len);
 565                 kunmap_atomic(user_mem);
 566                 user_mem = NULL;
 567         } else {
 568                 uncmem = user_mem;
 569         }
 570
 571         if (page_zero_filled(uncmem)) {
 572                 if (user_mem)
 573                         kunmap_atomic(user_mem);
 574                 /* Free memory associated with this sector now. */
 575                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 576                 zram_free_page(zram, index);
 577                 zram_set_flag(meta, index, ZRAM_ZERO);
 578                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 579
 580                 atomic64_inc(&zram->stats.zero_pages);
 581                 ret = 0;
 582                 goto out;
 583         }
 584
 585         ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
 586         if (!is_partial_io(bvec)) {
 587                 kunmap_atomic(user_mem);
 588                 user_mem = NULL;
 589                 uncmem = NULL;
 590         }
 591
 592         if (unlikely(ret)) {
 593                 pr_err("Compression failed! err=%d\n", ret);
 594                 goto out;
 595         }
 596         src = zstrm->buffer;
 597         if (unlikely(clen > max_zpage_size)) {
 598                 clen = PAGE_SIZE;
 599                 if (is_partial_io(bvec))
 600                         src = uncmem;
 601         }
 602
 603         handle = zs_malloc(meta->mem_pool, clen);
 604         if (!handle) {
 605                 pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
 606                         index, clen);
 607                 ret = -ENOMEM;
 608                 goto out;
 609         }
 610
 611         alloced_pages = zs_get_total_pages(meta->mem_pool);
 612         if (zram->limit_pages && alloced_pages > zram->limit_pages) {
 613                 zs_free(meta->mem_pool, handle);
 614                 ret = -ENOMEM;
 615                 goto out;
 616         }
 617
 618         update_used_max(zram, alloced_pages);
 619
 620         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
 621
 622         if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
 623                 src = kmap_atomic(page);
 624                 copy_page(cmem, src);
 625                 kunmap_atomic(src);
 626         } else {
 627                 memcpy(cmem, src, clen);
 628         }
 629
 630         zcomp_strm_release(zram->comp, zstrm);
 631         locked = false;
 632         zs_unmap_object(meta->mem_pool, handle);
 633
 634         /*
 635          * Free memory associated with this sector
 636          * before overwriting unused sectors.
 637          */
 638         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 639         zram_free_page(zram, index);
 640
 641         meta->table[index].handle = handle;
 642         zram_set_obj_size(meta, index, clen);
 643         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 644
 645         /* Update stats */
 646         atomic64_add(clen, &zram->stats.compr_data_size);
 647         atomic64_inc(&zram->stats.pages_stored);
 648 out:
 649         if (locked)
 650                 zcomp_strm_release(zram->comp, zstrm);
 651         if (is_partial_io(bvec))
 652                 kfree(uncmem);
 653         return ret;
 654 }
 655
 656 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
 657                         int offset, int rw)
 658 {
 659         int ret;
 660
 661         if (rw == READ) {
 662                 atomic64_inc(&zram->stats.num_reads);
 663                 ret = zram_bvec_read(zram, bvec, index, offset);
 664         } else {
 665                 atomic64_inc(&zram->stats.num_writes);
 666                 ret = zram_bvec_write(zram, bvec, index, offset);
 667         }
 668
 669         if (unlikely(ret)) {
 670                 if (rw == READ)
 671                         atomic64_inc(&zram->stats.failed_reads);
 672                 else
 673                         atomic64_inc(&zram->stats.failed_writes);
 674         }
 675
 676         return ret;
 677 }
 678
 679 /*
 680  * zram_bio_discard - handler on discard request
 681  * @index: physical block index in PAGE_SIZE units
 682  * @offset: byte offset within physical block
 683  */
 684 static void zram_bio_discard(struct zram *zram, u32 index,
 685                              int offset, struct bio *bio)
 686 {
 687         size_t n = bio->bi_iter.bi_size;
 688         struct zram_meta *meta = zram->meta;
 689
 690         /*
 691          * zram manages data in physical block size units. Because logical block
 692          * size isn't identical with physical block size on some arch, we
 693          * could get a discard request pointing to a specific offset within a
 694          * certain physical block.  Although we can handle this request by
 695          * reading that physiclal block and decompressing and partially zeroing
 696          * and re-compressing and then re-storing it, this isn't reasonable
 697          * because our intent with a discard request is to save memory.  So
 698          * skipping this logical block is appropriate here.
 699          */
 700         if (offset) {
 701                 if (n <= (PAGE_SIZE - offset))
 702                         return;
 703
 704                 n -= (PAGE_SIZE - offset);
 705                 index++;
 706         }
 707
 708         while (n >= PAGE_SIZE) {
 709                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 710                 zram_free_page(zram, index);
 711                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 712                 atomic64_inc(&zram->stats.notify_free);
 713                 index++;
 714                 n -= PAGE_SIZE;
 715         }
 716 }
 717
 718 static void zram_reset_device(struct zram *zram)
 719 {
 720         down_write(&zram->init_lock);
 721
 722         zram->limit_pages = 0;
 723
 724         if (!init_done(zram)) {
 725                 up_write(&zram->init_lock);
 726                 return;
 727         }
 728
 729         zcomp_destroy(zram->comp);
 730         zram->max_comp_streams = 1;
 731         zram_meta_free(zram->meta, zram->disksize);
 732         zram->meta = NULL;
 733         /* Reset stats */
 734         memset(&zram->stats, 0, sizeof(zram->stats));
 735
 736         zram->disksize = 0;
 737         up_write(&zram->init_lock);
 738 }
 739
 740 static ssize_t disksize_store(struct device *dev,
 741                 struct device_attribute *attr, const char *buf, size_t len)
 742 {
 743         u64 disksize;
 744         struct zcomp *comp;
 745         struct zram_meta *meta;
 746         struct zram *zram = dev_to_zram(dev);
 747         int err;
 748
 749         disksize = memparse(buf, NULL);
 750         if (!disksize)
 751                 return -EINVAL;
 752
 753         disksize = PAGE_ALIGN(disksize);
 754         meta = zram_meta_alloc(disksize);
 755         if (!meta)
 756                 return -ENOMEM;
 757
 758         comp = zcomp_create(zram->compressor, zram->max_comp_streams);
 759         if (IS_ERR(comp)) {
 760                 pr_info("Cannot initialise %s compressing backend\n",
 761                                 zram->compressor);
 762                 err = PTR_ERR(comp);
 763                 goto out_free_meta;
 764         }
 765
 766         down_write(&zram->init_lock);
 767         if (init_done(zram)) {
 768                 pr_info("Cannot change disksize for initialized device\n");
 769                 err = -EBUSY;
 770                 goto out_destroy_comp;
 771         }
 772
 773         zram->meta = meta;
 774         zram->comp = comp;
 775         zram->disksize = disksize;
 776         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
 777         up_write(&zram->init_lock);
 778
 779         /*
 780          * Revalidate disk out of the init_lock to avoid lockdep splat.
 781          * It's okay because disk's capacity is protected by init_lock
 782          * so that revalidate_disk always sees up-to-date capacity.
 783          */
 784         revalidate_disk(zram->disk);
 785
 786         return len;
 787
 788 out_destroy_comp:
 789         up_write(&zram->init_lock);
 790         zcomp_destroy(comp);
 791 out_free_meta:
 792         zram_meta_free(meta, disksize);
 793         return err;
 794 }
 795
 796 static ssize_t reset_store(struct device *dev,
 797                 struct device_attribute *attr, const char *buf, size_t len)
 798 {
 799         int ret;
 800         unsigned short do_reset;
 801         struct zram *zram;
 802         struct block_device *bdev;
 803
 804         zram = dev_to_zram(dev);
 805         bdev = bdget_disk(zram->disk, 0);
 806
 807         if (!bdev)
 808                 return -ENOMEM;
 809
 810         mutex_lock(&bdev->bd_mutex);
 811         /* Do not reset an active device! */
 812         if (bdev->bd_holders) {
 813                 ret = -EBUSY;
 814                 goto out;
 815         }
 816
 817         ret = kstrtou16(buf, 10, &do_reset);
 818         if (ret)
 819                 goto out;
 820
 821         if (!do_reset) {
 822                 ret = -EINVAL;
 823                 goto out;
 824         }
 825
 826         /* Make sure all pending I/O is finished */
 827         fsync_bdev(bdev);
 828         zram_reset_device(zram);
 829         set_capacity(zram->disk, 0);
 830
 831         mutex_unlock(&bdev->bd_mutex);
 832         revalidate_disk(zram->disk);
 833         bdput(bdev);
 834
 835         return len;
 836
 837 out:
 838         mutex_unlock(&bdev->bd_mutex);
 839         bdput(bdev);
 840         return ret;
 841 }
 842
 843 static void __zram_make_request(struct zram *zram, struct bio *bio)
 844 {
 845         int offset, rw;
 846         u32 index;
 847         struct bio_vec bvec;
 848         struct bvec_iter iter;
 849
 850         index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 851         offset = (bio->bi_iter.bi_sector &
 852                   (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
 853
 854         if (unlikely(bio->bi_rw & REQ_DISCARD)) {
 855                 zram_bio_discard(zram, index, offset, bio);
 856                 bio_endio(bio, 0);
 857                 return;
 858         }
 859
 860         rw = bio_data_dir(bio);
 861         bio_for_each_segment(bvec, bio, iter) {
 862                 int max_transfer_size = PAGE_SIZE - offset;
 863
 864                 if (bvec.bv_len > max_transfer_size) {
 865                         /*
 866                          * zram_bvec_rw() can only make operation on a single
 867                          * zram page. Split the bio vector.
 868                          */
 869                         struct bio_vec bv;
 870
 871                         bv.bv_page = bvec.bv_page;
 872                         bv.bv_len = max_transfer_size;
 873                         bv.bv_offset = bvec.bv_offset;
 874
 875                         if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)
 876                                 goto out;
 877
 878                         bv.bv_len = bvec.bv_len - max_transfer_size;
 879                         bv.bv_offset += max_transfer_size;
 880                         if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)
 881                                 goto out;
 882                 } else
 883                         if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)
 884                                 goto out;
 885
 886                 update_position(&index, &offset, &bvec);
 887         }
 888
 889         set_bit(BIO_UPTODATE, &bio->bi_flags);
 890         bio_endio(bio, 0);
 891         return;
 892
 893 out:
 894         bio_io_error(bio);
 895 }
 896
 897 /*
 898  * Handler function for all zram I/O requests.
 899  */
 900 static void zram_make_request(struct request_queue *queue, struct bio *bio)
 901 {
 902         struct zram *zram = queue->queuedata;
 903
 904         down_read(&zram->init_lock);
 905         if (unlikely(!init_done(zram)))
 906                 goto error;
 907
 908         if (!valid_io_request(zram, bio->bi_iter.bi_sector,
 909                                         bio->bi_iter.bi_size)) {
 910                 atomic64_inc(&zram->stats.invalid_io);
 911                 goto error;
 912         }
 913
 914         __zram_make_request(zram, bio);
 915         up_read(&zram->init_lock);
 916
 917         return;
 918
 919 error:
 920         up_read(&zram->init_lock);
 921         bio_io_error(bio);
 922 }
 923
 924 static void zram_slot_free_notify(struct block_device *bdev,
 925                                 unsigned long index)
 926 {
 927         struct zram *zram;
 928         struct zram_meta *meta;
 929
 930         zram = bdev->bd_disk->private_data;
 931         meta = zram->meta;
 932
 933         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 934         zram_free_page(zram, index);
 935         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 936         atomic64_inc(&zram->stats.notify_free);
 937 }
 938
 939 static int zram_rw_page(struct block_device *bdev, sector_t sector,
 940                        struct page *page, int rw)
 941 {
 942         int offset, err;
 943         u32 index;
 944         struct zram *zram;
 945         struct bio_vec bv;
 946
 947         zram = bdev->bd_disk->private_data;
 948         if (!valid_io_request(zram, sector, PAGE_SIZE)) {
 949                 atomic64_inc(&zram->stats.invalid_io);
 950                 return -EINVAL;
 951         }
 952
 953         down_read(&zram->init_lock);
 954         if (unlikely(!init_done(zram))) {
 955                 err = -EIO;
 956                 goto out_unlock;
 957         }
 958
 959         index = sector >> SECTORS_PER_PAGE_SHIFT;
 960         offset = sector & (SECTORS_PER_PAGE - 1) << SECTOR_SHIFT;
 961
 962         bv.bv_page = page;
 963         bv.bv_len = PAGE_SIZE;
 964         bv.bv_offset = 0;
 965
 966         err = zram_bvec_rw(zram, &bv, index, offset, rw);
 967 out_unlock:
 968         up_read(&zram->init_lock);
 969         /*
 970          * If I/O fails, just return error(ie, non-zero) without
 971          * calling page_endio.
 972          * It causes resubmit the I/O with bio request by upper functions
 973          * of rw_page(e.g., swap_readpage, __swap_writepage) and
 974          * bio->bi_end_io does things to handle the error
 975          * (e.g., SetPageError, set_page_dirty and extra works).
 976          */
 977         if (err == 0)
 978                 page_endio(page, rw, 0);
 979         return err;
 980 }
 981
 982 static const struct block_device_operations zram_devops = {
 983         .swap_slot_free_notify = zram_slot_free_notify,
 984         .rw_page = zram_rw_page,
 985         .owner = THIS_MODULE
 986 };
 987
 988 static DEVICE_ATTR_RW(disksize);
 989 static DEVICE_ATTR_RO(initstate);
 990 static DEVICE_ATTR_WO(reset);
 991 static DEVICE_ATTR_RO(orig_data_size);
 992 static DEVICE_ATTR_RO(mem_used_total);
 993 static DEVICE_ATTR_RW(mem_limit);
 994 static DEVICE_ATTR_RW(mem_used_max);
 995 static DEVICE_ATTR_RW(max_comp_streams);
 996 static DEVICE_ATTR_RW(comp_algorithm);
 997
 998 ZRAM_ATTR_RO(num_reads);
 999 ZRAM_ATTR_RO(num_writes);
1000 ZRAM_ATTR_RO(failed_reads);
1001 ZRAM_ATTR_RO(failed_writes);
1002 ZRAM_ATTR_RO(invalid_io);
1003 ZRAM_ATTR_RO(notify_free);
1004 ZRAM_ATTR_RO(zero_pages);
1005 ZRAM_ATTR_RO(compr_data_size);
1006
1007 static struct attribute *zram_disk_attrs[] = {
1008         &dev_attr_disksize.attr,
1009         &dev_attr_initstate.attr,
1010         &dev_attr_reset.attr,
1011         &dev_attr_num_reads.attr,
1012         &dev_attr_num_writes.attr,
1013         &dev_attr_failed_reads.attr,
1014         &dev_attr_failed_writes.attr,
1015         &dev_attr_invalid_io.attr,
1016         &dev_attr_notify_free.attr,
1017         &dev_attr_zero_pages.attr,
1018         &dev_attr_orig_data_size.attr,
1019         &dev_attr_compr_data_size.attr,
1020         &dev_attr_mem_used_total.attr,
1021         &dev_attr_mem_limit.attr,
1022         &dev_attr_mem_used_max.attr,
1023         &dev_attr_max_comp_streams.attr,
1024         &dev_attr_comp_algorithm.attr,
1025         NULL,
1026 };
1027
1028 static struct attribute_group zram_disk_attr_group = {
1029         .attrs = zram_disk_attrs,
1030 };
1031
1032 static int create_device(struct zram *zram, int device_id)
1033 {
1034         int ret = -ENOMEM;
1035
1036         init_rwsem(&zram->init_lock);
1037
1038         zram->queue = blk_alloc_queue(GFP_KERNEL);
1039         if (!zram->queue) {
1040                 pr_err("Error allocating disk queue for device %d\n",
1041                         device_id);
1042                 goto out;
1043         }
1044
1045         blk_queue_make_request(zram->queue, zram_make_request);
1046         zram->queue->queuedata = zram;
1047
1048          /* gendisk structure */
1049         zram->disk = alloc_disk(1);
1050         if (!zram->disk) {
1051                 pr_warn("Error allocating disk structure for device %d\n",
1052                         device_id);
1053                 goto out_free_queue;
1054         }
1055
1056         zram->disk->major = zram_major;
1057         zram->disk->first_minor = device_id;
1058         zram->disk->fops = &zram_devops;
1059         zram->disk->queue = zram->queue;
1060         zram->disk->private_data = zram;
1061         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
1062
1063         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
1064         set_capacity(zram->disk, 0);
1065         /* zram devices sort of resembles non-rotational disks */
1066         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
1067         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
1068         /*
1069          * To ensure that we always get PAGE_SIZE aligned
1070          * and n*PAGE_SIZED sized I/O requests.
1071          */
1072         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
1073         blk_queue_logical_block_size(zram->disk->queue,
1074                                         ZRAM_LOGICAL_BLOCK_SIZE);
1075         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1076         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
1077         zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1078         zram->disk->queue->limits.max_discard_sectors = UINT_MAX;
1079         /*
1080          * zram_bio_discard() will clear all logical blocks if logical block
1081          * size is identical with physical block size(PAGE_SIZE). But if it is
1082          * different, we will skip discarding some parts of logical blocks in
1083          * the part of the request range which isn't aligned to physical block
1084          * size.  So we can't ensure that all discarded logical blocks are
1085          * zeroed.
1086          */
1087         if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
1088                 zram->disk->queue->limits.discard_zeroes_data = 1;
1089         else
1090                 zram->disk->queue->limits.discard_zeroes_data = 0;
1091         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
1092
1093         add_disk(zram->disk);
1094
1095         ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
1096                                 &zram_disk_attr_group);
1097         if (ret < 0) {
1098                 pr_warn("Error creating sysfs group");
1099                 goto out_free_disk;
1100         }
1101         strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
1102         zram->meta = NULL;
1103         zram->max_comp_streams = 1;
1104         return 0;
1105
1106 out_free_disk:
1107         del_gendisk(zram->disk);
1108         put_disk(zram->disk);
1109 out_free_queue:
1110         blk_cleanup_queue(zram->queue);
1111 out:
1112         return ret;
1113 }
1114
1115 static void destroy_device(struct zram *zram)
1116 {
1117         sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
1118                         &zram_disk_attr_group);
1119
1120         del_gendisk(zram->disk);
1121         put_disk(zram->disk);
1122
1123         blk_cleanup_queue(zram->queue);
1124 }
1125
1126 static int __init zram_init(void)
1127 {
1128         int ret, dev_id;
1129
1130         if (num_devices > max_num_devices) {
1131                 pr_warn("Invalid value for num_devices: %u\n",
1132                                 num_devices);
1133                 ret = -EINVAL;
1134                 goto out;
1135         }
1136
1137         zram_major = register_blkdev(0, "zram");
1138         if (zram_major <= 0) {
1139                 pr_warn("Unable to get major number\n");
1140                 ret = -EBUSY;
1141                 goto out;
1142         }
1143
1144         /* Allocate the device array and initialize each one */
1145         zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
1146         if (!zram_devices) {
1147                 ret = -ENOMEM;
1148                 goto unregister;
1149         }
1150
1151         for (dev_id = 0; dev_id < num_devices; dev_id++) {
1152                 ret = create_device(&zram_devices[dev_id], dev_id);
1153                 if (ret)
1154                         goto free_devices;
1155         }
1156
1157         pr_info("Created %u device(s) ...\n", num_devices);
1158
1159         return 0;
1160
1161 free_devices:
1162         while (dev_id)
1163                 destroy_device(&zram_devices[--dev_id]);
1164         kfree(zram_devices);
1165 unregister:
1166         unregister_blkdev(zram_major, "zram");
1167 out:
1168         return ret;
1169 }
1170
1171 static void __exit zram_exit(void)
1172 {
1173         int i;
1174         struct zram *zram;
1175
1176         for (i = 0; i < num_devices; i++) {
1177                 zram = &zram_devices[i];
1178
1179                 destroy_device(zram);
1180                 /*
1181                  * Shouldn't access zram->disk after destroy_device
1182                  * because destroy_device already released zram->disk.
1183                  */
1184                 zram_reset_device(zram);
1185         }
1186
1187         unregister_blkdev(zram_major, "zram");
1188
1189         kfree(zram_devices);
1190         pr_debug("Cleanup done!\n");
1191 }
1192
1193 module_init(zram_init);
1194 module_exit(zram_exit);
1195
1196 module_param(num_devices, uint, 0);
1197 MODULE_PARM_DESC(num_devices, "Number of zram devices");
1198
1199 MODULE_LICENSE("Dual BSD/GPL");
1200 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
1201 MODULE_DESCRIPTION("Compressed RAM Block Device");