drivers/nvdimm/core.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/libnvdimm.h>
  14 #include <linux/badblocks.h>
  15 #include <linux/export.h>
  16 #include <linux/module.h>
  17 #include <linux/blkdev.h>
  18 #include <linux/device.h>
  19 #include <linux/ctype.h>
  20 #include <linux/ndctl.h>
  21 #include <linux/mutex.h>
  22 #include <linux/slab.h>
  23 #include "nd-core.h"
  24 #include "nd.h"
  25
  26 LIST_HEAD(nvdimm_bus_list);
  27 DEFINE_MUTEX(nvdimm_bus_list_mutex);
  28 static DEFINE_IDA(nd_ida);
  29
  30 void nvdimm_bus_lock(struct device *dev)
  31 {
  32         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  33
  34         if (!nvdimm_bus)
  35                 return;
  36         mutex_lock(&nvdimm_bus->reconfig_mutex);
  37 }
  38 EXPORT_SYMBOL(nvdimm_bus_lock);
  39
  40 void nvdimm_bus_unlock(struct device *dev)
  41 {
  42         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  43
  44         if (!nvdimm_bus)
  45                 return;
  46         mutex_unlock(&nvdimm_bus->reconfig_mutex);
  47 }
  48 EXPORT_SYMBOL(nvdimm_bus_unlock);
  49
  50 bool is_nvdimm_bus_locked(struct device *dev)
  51 {
  52         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  53
  54         if (!nvdimm_bus)
  55                 return false;
  56         return mutex_is_locked(&nvdimm_bus->reconfig_mutex);
  57 }
  58 EXPORT_SYMBOL(is_nvdimm_bus_locked);
  59
  60 u64 nd_fletcher64(void *addr, size_t len, bool le)
  61 {
  62         u32 *buf = addr;
  63         u32 lo32 = 0;
  64         u64 hi32 = 0;
  65         int i;
  66
  67         for (i = 0; i < len / sizeof(u32); i++) {
  68                 lo32 += le ? le32_to_cpu((__le32) buf[i]) : buf[i];
  69                 hi32 += lo32;
  70         }
  71
  72         return hi32 << 32 | lo32;
  73 }
  74 EXPORT_SYMBOL_GPL(nd_fletcher64);
  75
  76 static void nvdimm_bus_release(struct device *dev)
  77 {
  78         struct nvdimm_bus *nvdimm_bus;
  79
  80         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  81         ida_simple_remove(&nd_ida, nvdimm_bus->id);
  82         kfree(nvdimm_bus);
  83 }
  84
  85 struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
  86 {
  87         struct nvdimm_bus *nvdimm_bus;
  88
  89         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  90         WARN_ON(nvdimm_bus->dev.release != nvdimm_bus_release);
  91         return nvdimm_bus;
  92 }
  93 EXPORT_SYMBOL_GPL(to_nvdimm_bus);
  94
  95 struct nvdimm_bus_descriptor *to_nd_desc(struct nvdimm_bus *nvdimm_bus)
  96 {
  97         /* struct nvdimm_bus definition is private to libnvdimm */
  98         return nvdimm_bus->nd_desc;
  99 }
 100 EXPORT_SYMBOL_GPL(to_nd_desc);
 101
 102 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
 103 {
 104         struct device *dev;
 105
 106         for (dev = nd_dev; dev; dev = dev->parent)
 107                 if (dev->release == nvdimm_bus_release)
 108                         break;
 109         dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
 110         if (dev)
 111                 return to_nvdimm_bus(dev);
 112         return NULL;
 113 }
 114
 115 static bool is_uuid_sep(char sep)
 116 {
 117         if (sep == '\n' || sep == '-' || sep == ':' || sep == '\0')
 118                 return true;
 119         return false;
 120 }
 121
 122 static int nd_uuid_parse(struct device *dev, u8 *uuid_out, const char *buf,
 123                 size_t len)
 124 {
 125         const char *str = buf;
 126         u8 uuid[16];
 127         int i;
 128
 129         for (i = 0; i < 16; i++) {
 130                 if (!isxdigit(str[0]) || !isxdigit(str[1])) {
 131                         dev_dbg(dev, "%s: pos: %d buf[%zd]: %c buf[%zd]: %c\n",
 132                                         __func__, i, str - buf, str[0],
 133                                         str + 1 - buf, str[1]);
 134                         return -EINVAL;
 135                 }
 136
 137                 uuid[i] = (hex_to_bin(str[0]) << 4) | hex_to_bin(str[1]);
 138                 str += 2;
 139                 if (is_uuid_sep(*str))
 140                         str++;
 141         }
 142
 143         memcpy(uuid_out, uuid, sizeof(uuid));
 144         return 0;
 145 }
 146
 147 /**
 148  * nd_uuid_store: common implementation for writing 'uuid' sysfs attributes
 149  * @dev: container device for the uuid property
 150  * @uuid_out: uuid buffer to replace
 151  * @buf: raw sysfs buffer to parse
 152  *
 153  * Enforce that uuids can only be changed while the device is disabled
 154  * (driver detached)
 155  * LOCKING: expects device_lock() is held on entry
 156  */
 157 int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
 158                 size_t len)
 159 {
 160         u8 uuid[16];
 161         int rc;
 162
 163         if (dev->driver)
 164                 return -EBUSY;
 165
 166         rc = nd_uuid_parse(dev, uuid, buf, len);
 167         if (rc)
 168                 return rc;
 169
 170         kfree(*uuid_out);
 171         *uuid_out = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
 172         if (!(*uuid_out))
 173                 return -ENOMEM;
 174
 175         return 0;
 176 }
 177
 178 ssize_t nd_sector_size_show(unsigned long current_lbasize,
 179                 const unsigned long *supported, char *buf)
 180 {
 181         ssize_t len = 0;
 182         int i;
 183
 184         for (i = 0; supported[i]; i++)
 185                 if (current_lbasize == supported[i])
 186                         len += sprintf(buf + len, "[%ld] ", supported[i]);
 187                 else
 188                         len += sprintf(buf + len, "%ld ", supported[i]);
 189         len += sprintf(buf + len, "\n");
 190         return len;
 191 }
 192
 193 ssize_t nd_sector_size_store(struct device *dev, const char *buf,
 194                 unsigned long *current_lbasize, const unsigned long *supported)
 195 {
 196         unsigned long lbasize;
 197         int rc, i;
 198
 199         if (dev->driver)
 200                 return -EBUSY;
 201
 202         rc = kstrtoul(buf, 0, &lbasize);
 203         if (rc)
 204                 return rc;
 205
 206         for (i = 0; supported[i]; i++)
 207                 if (lbasize == supported[i])
 208                         break;
 209
 210         if (supported[i]) {
 211                 *current_lbasize = lbasize;
 212                 return 0;
 213         } else {
 214                 return -EINVAL;
 215         }
 216 }
 217
 218 void __nd_iostat_start(struct bio *bio, unsigned long *start)
 219 {
 220         struct gendisk *disk = bio->bi_bdev->bd_disk;
 221         const int rw = bio_data_dir(bio);
 222         int cpu = part_stat_lock();
 223
 224         *start = jiffies;
 225         part_round_stats(cpu, &disk->part0);
 226         part_stat_inc(cpu, &disk->part0, ios[rw]);
 227         part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio));
 228         part_inc_in_flight(&disk->part0, rw);
 229         part_stat_unlock();
 230 }
 231 EXPORT_SYMBOL(__nd_iostat_start);
 232
 233 void nd_iostat_end(struct bio *bio, unsigned long start)
 234 {
 235         struct gendisk *disk = bio->bi_bdev->bd_disk;
 236         unsigned long duration = jiffies - start;
 237         const int rw = bio_data_dir(bio);
 238         int cpu = part_stat_lock();
 239
 240         part_stat_add(cpu, &disk->part0, ticks[rw], duration);
 241         part_round_stats(cpu, &disk->part0);
 242         part_dec_in_flight(&disk->part0, rw);
 243         part_stat_unlock();
 244 }
 245 EXPORT_SYMBOL(nd_iostat_end);
 246
 247 static ssize_t commands_show(struct device *dev,
 248                 struct device_attribute *attr, char *buf)
 249 {
 250         int cmd, len = 0;
 251         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 252         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 253
 254         for_each_set_bit(cmd, &nd_desc->dsm_mask, BITS_PER_LONG)
 255                 len += sprintf(buf + len, "%s ", nvdimm_bus_cmd_name(cmd));
 256         len += sprintf(buf + len, "\n");
 257         return len;
 258 }
 259 static DEVICE_ATTR_RO(commands);
 260
 261 static const char *nvdimm_bus_provider(struct nvdimm_bus *nvdimm_bus)
 262 {
 263         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 264         struct device *parent = nvdimm_bus->dev.parent;
 265
 266         if (nd_desc->provider_name)
 267                 return nd_desc->provider_name;
 268         else if (parent)
 269                 return dev_name(parent);
 270         else
 271                 return "unknown";
 272 }
 273
 274 static ssize_t provider_show(struct device *dev,
 275                 struct device_attribute *attr, char *buf)
 276 {
 277         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 278
 279         return sprintf(buf, "%s\n", nvdimm_bus_provider(nvdimm_bus));
 280 }
 281 static DEVICE_ATTR_RO(provider);
 282
 283 static int flush_namespaces(struct device *dev, void *data)
 284 {
 285         device_lock(dev);
 286         device_unlock(dev);
 287         return 0;
 288 }
 289
 290 static int flush_regions_dimms(struct device *dev, void *data)
 291 {
 292         device_lock(dev);
 293         device_unlock(dev);
 294         device_for_each_child(dev, NULL, flush_namespaces);
 295         return 0;
 296 }
 297
 298 static ssize_t wait_probe_show(struct device *dev,
 299                 struct device_attribute *attr, char *buf)
 300 {
 301         nd_synchronize();
 302         device_for_each_child(dev, NULL, flush_regions_dimms);
 303         return sprintf(buf, "1\n");
 304 }
 305 static DEVICE_ATTR_RO(wait_probe);
 306
 307 static struct attribute *nvdimm_bus_attributes[] = {
 308         &dev_attr_commands.attr,
 309         &dev_attr_wait_probe.attr,
 310         &dev_attr_provider.attr,
 311         NULL,
 312 };
 313
 314 struct attribute_group nvdimm_bus_attribute_group = {
 315         .attrs = nvdimm_bus_attributes,
 316 };
 317 EXPORT_SYMBOL_GPL(nvdimm_bus_attribute_group);
 318
 319 struct nvdimm_bus *__nvdimm_bus_register(struct device *parent,
 320                 struct nvdimm_bus_descriptor *nd_desc, struct module *module)
 321 {
 322         struct nvdimm_bus *nvdimm_bus;
 323         int rc;
 324
 325         nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
 326         if (!nvdimm_bus)
 327                 return NULL;
 328         INIT_LIST_HEAD(&nvdimm_bus->list);
 329         INIT_LIST_HEAD(&nvdimm_bus->poison_list);
 330         init_waitqueue_head(&nvdimm_bus->probe_wait);
 331         nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
 332         mutex_init(&nvdimm_bus->reconfig_mutex);
 333         if (nvdimm_bus->id < 0) {
 334                 kfree(nvdimm_bus);
 335                 return NULL;
 336         }
 337         nvdimm_bus->nd_desc = nd_desc;
 338         nvdimm_bus->module = module;
 339         nvdimm_bus->dev.parent = parent;
 340         nvdimm_bus->dev.release = nvdimm_bus_release;
 341         nvdimm_bus->dev.groups = nd_desc->attr_groups;
 342         dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
 343         rc = device_register(&nvdimm_bus->dev);
 344         if (rc) {
 345                 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
 346                 goto err;
 347         }
 348
 349         rc = nvdimm_bus_create_ndctl(nvdimm_bus);
 350         if (rc)
 351                 goto err;
 352
 353         mutex_lock(&nvdimm_bus_list_mutex);
 354         list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
 355         mutex_unlock(&nvdimm_bus_list_mutex);
 356
 357         return nvdimm_bus;
 358  err:
 359         put_device(&nvdimm_bus->dev);
 360         return NULL;
 361 }
 362 EXPORT_SYMBOL_GPL(__nvdimm_bus_register);
 363
 364 static void set_badblock(struct badblocks *bb, sector_t s, int num)
 365 {
 366         dev_dbg(bb->dev, "Found a poison range (0x%llx, 0x%llx)\n",
 367                         (u64) s * 512, (u64) num * 512);
 368         /* this isn't an error as the hardware will still throw an exception */
 369         if (badblocks_set(bb, s, num, 1))
 370                 dev_info_once(bb->dev, "%s: failed for sector %llx\n",
 371                                 __func__, (u64) s);
 372 }
 373
 374 /**
 375  * __add_badblock_range() - Convert a physical address range to bad sectors
 376  * @bb:         badblocks instance to populate
 377  * @ns_offset:  namespace offset where the error range begins (in bytes)
 378  * @len:        number of bytes of poison to be added
 379  *
 380  * This assumes that the range provided with (ns_offset, len) is within
 381  * the bounds of physical addresses for this namespace, i.e. lies in the
 382  * interval [ns_start, ns_start + ns_size)
 383  */
 384 static void __add_badblock_range(struct badblocks *bb, u64 ns_offset, u64 len)
 385 {
 386         const unsigned int sector_size = 512;
 387         sector_t start_sector;
 388         u64 num_sectors;
 389         u32 rem;
 390
 391         start_sector = div_u64(ns_offset, sector_size);
 392         num_sectors = div_u64_rem(len, sector_size, &rem);
 393         if (rem)
 394                 num_sectors++;
 395
 396         if (unlikely(num_sectors > (u64)INT_MAX)) {
 397                 u64 remaining = num_sectors;
 398                 sector_t s = start_sector;
 399
 400                 while (remaining) {
 401                         int done = min_t(u64, remaining, INT_MAX);
 402
 403                         set_badblock(bb, s, done);
 404                         remaining -= done;
 405                         s += done;
 406                 }
 407         } else
 408                 set_badblock(bb, start_sector, num_sectors);
 409 }
 410
 411 static void namespace_add_poison(struct list_head *poison_list,
 412                 struct badblocks *bb, struct resource *res)
 413 {
 414         struct nd_poison *pl;
 415
 416         if (list_empty(poison_list))
 417                 return;
 418
 419         list_for_each_entry(pl, poison_list, list) {
 420                 u64 pl_end = pl->start + pl->length - 1;
 421
 422                 /* Discard intervals with no intersection */
 423                 if (pl_end < res->start)
 424                         continue;
 425                 if (pl->start >  res->end)
 426                         continue;
 427                 /* Deal with any overlap after start of the namespace */
 428                 if (pl->start >= res->start) {
 429                         u64 start = pl->start;
 430                         u64 len;
 431
 432                         if (pl_end <= res->end)
 433                                 len = pl->length;
 434                         else
 435                                 len = res->start + resource_size(res)
 436                                         - pl->start;
 437                         __add_badblock_range(bb, start - res->start, len);
 438                         continue;
 439                 }
 440                 /* Deal with overlap for poison starting before the namespace */
 441                 if (pl->start < res->start) {
 442                         u64 len;
 443
 444                         if (pl_end < res->end)
 445                                 len = pl->start + pl->length - res->start;
 446                         else
 447                                 len = resource_size(res);
 448                         __add_badblock_range(bb, 0, len);
 449                 }
 450         }
 451 }
 452
 453 /**
 454  * nvdimm_namespace_add_poison() - Convert a list of poison ranges to badblocks
 455  * @ndns:       the namespace containing poison ranges
 456  * @bb:         badblocks instance to populate
 457  * @offset:     offset at the start of the namespace before 'sector 0'
 458  *
 459  * The poison list generated during NFIT initialization may contain multiple,
 460  * possibly overlapping ranges in the SPA (System Physical Address) space.
 461  * Compare each of these ranges to the namespace currently being initialized,
 462  * and add badblocks to the gendisk for all matching sub-ranges
 463  */
 464 void nvdimm_namespace_add_poison(struct nd_namespace_common *ndns,
 465                 struct badblocks *bb, resource_size_t offset)
 466 {
 467         struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
 468         struct nd_region *nd_region = to_nd_region(ndns->dev.parent);
 469         struct nvdimm_bus *nvdimm_bus;
 470         struct list_head *poison_list;
 471         struct resource res = {
 472                 .start = nsio->res.start + offset,
 473                 .end = nsio->res.end,
 474         };
 475
 476         nvdimm_bus = to_nvdimm_bus(nd_region->dev.parent);
 477         poison_list = &nvdimm_bus->poison_list;
 478
 479         nvdimm_bus_lock(&nvdimm_bus->dev);
 480         namespace_add_poison(poison_list, bb, &res);
 481         nvdimm_bus_unlock(&nvdimm_bus->dev);
 482 }
 483 EXPORT_SYMBOL_GPL(nvdimm_namespace_add_poison);
 484
 485 static int add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 486 {
 487         struct nd_poison *pl;
 488
 489         pl = kzalloc(sizeof(*pl), GFP_KERNEL);
 490         if (!pl)
 491                 return -ENOMEM;
 492
 493         pl->start = addr;
 494         pl->length = length;
 495         list_add_tail(&pl->list, &nvdimm_bus->poison_list);
 496
 497         return 0;
 498 }
 499
 500 static int bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 501 {
 502         struct nd_poison *pl;
 503
 504         if (list_empty(&nvdimm_bus->poison_list))
 505                 return add_poison(nvdimm_bus, addr, length);
 506
 507         /*
 508          * There is a chance this is a duplicate, check for those first.
 509          * This will be the common case as ARS_STATUS returns all known
 510          * errors in the SPA space, and we can't query it per region
 511          */
 512         list_for_each_entry(pl, &nvdimm_bus->poison_list, list)
 513                 if (pl->start == addr) {
 514                         /* If length has changed, update this list entry */
 515                         if (pl->length != length)
 516                                 pl->length = length;
 517                         return 0;
 518                 }
 519
 520         /*
 521          * If not a duplicate or a simple length update, add the entry as is,
 522          * as any overlapping ranges will get resolved when the list is consumed
 523          * and converted to badblocks
 524          */
 525         return add_poison(nvdimm_bus, addr, length);
 526 }
 527
 528 int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 529 {
 530         int rc;
 531
 532         nvdimm_bus_lock(&nvdimm_bus->dev);
 533         rc = bus_add_poison(nvdimm_bus, addr, length);
 534         nvdimm_bus_unlock(&nvdimm_bus->dev);
 535
 536         return rc;
 537 }
 538 EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison);
 539
 540 static void free_poison_list(struct list_head *poison_list)
 541 {
 542         struct nd_poison *pl, *next;
 543
 544         list_for_each_entry_safe(pl, next, poison_list, list) {
 545                 list_del(&pl->list);
 546                 kfree(pl);
 547         }
 548         list_del_init(poison_list);
 549 }
 550
 551 static int child_unregister(struct device *dev, void *data)
 552 {
 553         /*
 554          * the singular ndctl class device per bus needs to be
 555          * "device_destroy"ed, so skip it here
 556          *
 557          * i.e. remove classless children
 558          */
 559         if (dev->class)
 560                 /* pass */;
 561         else
 562                 nd_device_unregister(dev, ND_SYNC);
 563         return 0;
 564 }
 565
 566 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
 567 {
 568         if (!nvdimm_bus)
 569                 return;
 570
 571         mutex_lock(&nvdimm_bus_list_mutex);
 572         list_del_init(&nvdimm_bus->list);
 573         mutex_unlock(&nvdimm_bus_list_mutex);
 574
 575         nd_synchronize();
 576         device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
 577
 578         nvdimm_bus_lock(&nvdimm_bus->dev);
 579         free_poison_list(&nvdimm_bus->poison_list);
 580         nvdimm_bus_unlock(&nvdimm_bus->dev);
 581
 582         nvdimm_bus_destroy_ndctl(nvdimm_bus);
 583
 584         device_unregister(&nvdimm_bus->dev);
 585 }
 586 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
 587
 588 #ifdef CONFIG_BLK_DEV_INTEGRITY
 589 int nd_integrity_init(struct gendisk *disk, unsigned long meta_size)
 590 {
 591         struct blk_integrity bi;
 592
 593         if (meta_size == 0)
 594                 return 0;
 595
 596         bi.profile = NULL;
 597         bi.tuple_size = meta_size;
 598         bi.tag_size = meta_size;
 599
 600         blk_integrity_register(disk, &bi);
 601         blk_queue_max_integrity_segments(disk->queue, 1);
 602
 603         return 0;
 604 }
 605 EXPORT_SYMBOL(nd_integrity_init);
 606
 607 #else /* CONFIG_BLK_DEV_INTEGRITY */
 608 int nd_integrity_init(struct gendisk *disk, unsigned long meta_size)
 609 {
 610         return 0;
 611 }
 612 EXPORT_SYMBOL(nd_integrity_init);
 613
 614 #endif
 615
 616 static __init int libnvdimm_init(void)
 617 {
 618         int rc;
 619
 620         rc = nvdimm_bus_init();
 621         if (rc)
 622                 return rc;
 623         rc = nvdimm_init();
 624         if (rc)
 625                 goto err_dimm;
 626         rc = nd_region_init();
 627         if (rc)
 628                 goto err_region;
 629         return 0;
 630  err_region:
 631         nvdimm_exit();
 632  err_dimm:
 633         nvdimm_bus_exit();
 634         return rc;
 635 }
 636
 637 static __exit void libnvdimm_exit(void)
 638 {
 639         WARN_ON(!list_empty(&nvdimm_bus_list));
 640         nd_region_exit();
 641         nvdimm_exit();
 642         nvdimm_bus_exit();
 643 }
 644
 645 MODULE_LICENSE("GPL v2");
 646 MODULE_AUTHOR("Intel Corporation");
 647 subsys_initcall(libnvdimm_init);
 648 module_exit(libnvdimm_exit);