drivers/nvdimm/core.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/libnvdimm.h>
  14 #include <linux/export.h>
  15 #include <linux/module.h>
  16 #include <linux/blkdev.h>
  17 #include <linux/device.h>
  18 #include <linux/ctype.h>
  19 #include <linux/ndctl.h>
  20 #include <linux/mutex.h>
  21 #include <linux/slab.h>
  22 #include "nd-core.h"
  23 #include "nd.h"
  24
  25 LIST_HEAD(nvdimm_bus_list);
  26 DEFINE_MUTEX(nvdimm_bus_list_mutex);
  27 static DEFINE_IDA(nd_ida);
  28
  29 void nvdimm_bus_lock(struct device *dev)
  30 {
  31         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  32
  33         if (!nvdimm_bus)
  34                 return;
  35         mutex_lock(&nvdimm_bus->reconfig_mutex);
  36 }
  37 EXPORT_SYMBOL(nvdimm_bus_lock);
  38
  39 void nvdimm_bus_unlock(struct device *dev)
  40 {
  41         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  42
  43         if (!nvdimm_bus)
  44                 return;
  45         mutex_unlock(&nvdimm_bus->reconfig_mutex);
  46 }
  47 EXPORT_SYMBOL(nvdimm_bus_unlock);
  48
  49 bool is_nvdimm_bus_locked(struct device *dev)
  50 {
  51         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  52
  53         if (!nvdimm_bus)
  54                 return false;
  55         return mutex_is_locked(&nvdimm_bus->reconfig_mutex);
  56 }
  57 EXPORT_SYMBOL(is_nvdimm_bus_locked);
  58
  59 u64 nd_fletcher64(void *addr, size_t len, bool le)
  60 {
  61         u32 *buf = addr;
  62         u32 lo32 = 0;
  63         u64 hi32 = 0;
  64         int i;
  65
  66         for (i = 0; i < len / sizeof(u32); i++) {
  67                 lo32 += le ? le32_to_cpu((__le32) buf[i]) : buf[i];
  68                 hi32 += lo32;
  69         }
  70
  71         return hi32 << 32 | lo32;
  72 }
  73 EXPORT_SYMBOL_GPL(nd_fletcher64);
  74
  75 static void nvdimm_bus_release(struct device *dev)
  76 {
  77         struct nvdimm_bus *nvdimm_bus;
  78
  79         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  80         ida_simple_remove(&nd_ida, nvdimm_bus->id);
  81         kfree(nvdimm_bus);
  82 }
  83
  84 struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
  85 {
  86         struct nvdimm_bus *nvdimm_bus;
  87
  88         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  89         WARN_ON(nvdimm_bus->dev.release != nvdimm_bus_release);
  90         return nvdimm_bus;
  91 }
  92 EXPORT_SYMBOL_GPL(to_nvdimm_bus);
  93
  94 struct nvdimm_bus_descriptor *to_nd_desc(struct nvdimm_bus *nvdimm_bus)
  95 {
  96         /* struct nvdimm_bus definition is private to libnvdimm */
  97         return nvdimm_bus->nd_desc;
  98 }
  99 EXPORT_SYMBOL_GPL(to_nd_desc);
 100
 101 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
 102 {
 103         struct device *dev;
 104
 105         for (dev = nd_dev; dev; dev = dev->parent)
 106                 if (dev->release == nvdimm_bus_release)
 107                         break;
 108         dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
 109         if (dev)
 110                 return to_nvdimm_bus(dev);
 111         return NULL;
 112 }
 113
 114 static bool is_uuid_sep(char sep)
 115 {
 116         if (sep == '\n' || sep == '-' || sep == ':' || sep == '\0')
 117                 return true;
 118         return false;
 119 }
 120
 121 static int nd_uuid_parse(struct device *dev, u8 *uuid_out, const char *buf,
 122                 size_t len)
 123 {
 124         const char *str = buf;
 125         u8 uuid[16];
 126         int i;
 127
 128         for (i = 0; i < 16; i++) {
 129                 if (!isxdigit(str[0]) || !isxdigit(str[1])) {
 130                         dev_dbg(dev, "%s: pos: %d buf[%zd]: %c buf[%zd]: %c\n",
 131                                         __func__, i, str - buf, str[0],
 132                                         str + 1 - buf, str[1]);
 133                         return -EINVAL;
 134                 }
 135
 136                 uuid[i] = (hex_to_bin(str[0]) << 4) | hex_to_bin(str[1]);
 137                 str += 2;
 138                 if (is_uuid_sep(*str))
 139                         str++;
 140         }
 141
 142         memcpy(uuid_out, uuid, sizeof(uuid));
 143         return 0;
 144 }
 145
 146 /**
 147  * nd_uuid_store: common implementation for writing 'uuid' sysfs attributes
 148  * @dev: container device for the uuid property
 149  * @uuid_out: uuid buffer to replace
 150  * @buf: raw sysfs buffer to parse
 151  *
 152  * Enforce that uuids can only be changed while the device is disabled
 153  * (driver detached)
 154  * LOCKING: expects device_lock() is held on entry
 155  */
 156 int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
 157                 size_t len)
 158 {
 159         u8 uuid[16];
 160         int rc;
 161
 162         if (dev->driver)
 163                 return -EBUSY;
 164
 165         rc = nd_uuid_parse(dev, uuid, buf, len);
 166         if (rc)
 167                 return rc;
 168
 169         kfree(*uuid_out);
 170         *uuid_out = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
 171         if (!(*uuid_out))
 172                 return -ENOMEM;
 173
 174         return 0;
 175 }
 176
 177 ssize_t nd_sector_size_show(unsigned long current_lbasize,
 178                 const unsigned long *supported, char *buf)
 179 {
 180         ssize_t len = 0;
 181         int i;
 182
 183         for (i = 0; supported[i]; i++)
 184                 if (current_lbasize == supported[i])
 185                         len += sprintf(buf + len, "[%ld] ", supported[i]);
 186                 else
 187                         len += sprintf(buf + len, "%ld ", supported[i]);
 188         len += sprintf(buf + len, "\n");
 189         return len;
 190 }
 191
 192 ssize_t nd_sector_size_store(struct device *dev, const char *buf,
 193                 unsigned long *current_lbasize, const unsigned long *supported)
 194 {
 195         unsigned long lbasize;
 196         int rc, i;
 197
 198         if (dev->driver)
 199                 return -EBUSY;
 200
 201         rc = kstrtoul(buf, 0, &lbasize);
 202         if (rc)
 203                 return rc;
 204
 205         for (i = 0; supported[i]; i++)
 206                 if (lbasize == supported[i])
 207                         break;
 208
 209         if (supported[i]) {
 210                 *current_lbasize = lbasize;
 211                 return 0;
 212         } else {
 213                 return -EINVAL;
 214         }
 215 }
 216
 217 void __nd_iostat_start(struct bio *bio, unsigned long *start)
 218 {
 219         struct gendisk *disk = bio->bi_bdev->bd_disk;
 220         const int rw = bio_data_dir(bio);
 221         int cpu = part_stat_lock();
 222
 223         *start = jiffies;
 224         part_round_stats(cpu, &disk->part0);
 225         part_stat_inc(cpu, &disk->part0, ios[rw]);
 226         part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio));
 227         part_inc_in_flight(&disk->part0, rw);
 228         part_stat_unlock();
 229 }
 230 EXPORT_SYMBOL(__nd_iostat_start);
 231
 232 void nd_iostat_end(struct bio *bio, unsigned long start)
 233 {
 234         struct gendisk *disk = bio->bi_bdev->bd_disk;
 235         unsigned long duration = jiffies - start;
 236         const int rw = bio_data_dir(bio);
 237         int cpu = part_stat_lock();
 238
 239         part_stat_add(cpu, &disk->part0, ticks[rw], duration);
 240         part_round_stats(cpu, &disk->part0);
 241         part_dec_in_flight(&disk->part0, rw);
 242         part_stat_unlock();
 243 }
 244 EXPORT_SYMBOL(nd_iostat_end);
 245
 246 static ssize_t commands_show(struct device *dev,
 247                 struct device_attribute *attr, char *buf)
 248 {
 249         int cmd, len = 0;
 250         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 251         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 252
 253         for_each_set_bit(cmd, &nd_desc->dsm_mask, BITS_PER_LONG)
 254                 len += sprintf(buf + len, "%s ", nvdimm_bus_cmd_name(cmd));
 255         len += sprintf(buf + len, "\n");
 256         return len;
 257 }
 258 static DEVICE_ATTR_RO(commands);
 259
 260 static const char *nvdimm_bus_provider(struct nvdimm_bus *nvdimm_bus)
 261 {
 262         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 263         struct device *parent = nvdimm_bus->dev.parent;
 264
 265         if (nd_desc->provider_name)
 266                 return nd_desc->provider_name;
 267         else if (parent)
 268                 return dev_name(parent);
 269         else
 270                 return "unknown";
 271 }
 272
 273 static ssize_t provider_show(struct device *dev,
 274                 struct device_attribute *attr, char *buf)
 275 {
 276         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 277
 278         return sprintf(buf, "%s\n", nvdimm_bus_provider(nvdimm_bus));
 279 }
 280 static DEVICE_ATTR_RO(provider);
 281
 282 static int flush_namespaces(struct device *dev, void *data)
 283 {
 284         device_lock(dev);
 285         device_unlock(dev);
 286         return 0;
 287 }
 288
 289 static int flush_regions_dimms(struct device *dev, void *data)
 290 {
 291         device_lock(dev);
 292         device_unlock(dev);
 293         device_for_each_child(dev, NULL, flush_namespaces);
 294         return 0;
 295 }
 296
 297 static ssize_t wait_probe_show(struct device *dev,
 298                 struct device_attribute *attr, char *buf)
 299 {
 300         nd_synchronize();
 301         device_for_each_child(dev, NULL, flush_regions_dimms);
 302         return sprintf(buf, "1\n");
 303 }
 304 static DEVICE_ATTR_RO(wait_probe);
 305
 306 static struct attribute *nvdimm_bus_attributes[] = {
 307         &dev_attr_commands.attr,
 308         &dev_attr_wait_probe.attr,
 309         &dev_attr_provider.attr,
 310         NULL,
 311 };
 312
 313 struct attribute_group nvdimm_bus_attribute_group = {
 314         .attrs = nvdimm_bus_attributes,
 315 };
 316 EXPORT_SYMBOL_GPL(nvdimm_bus_attribute_group);
 317
 318 struct nvdimm_bus *__nvdimm_bus_register(struct device *parent,
 319                 struct nvdimm_bus_descriptor *nd_desc, struct module *module)
 320 {
 321         struct nvdimm_bus *nvdimm_bus;
 322         int rc;
 323
 324         nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
 325         if (!nvdimm_bus)
 326                 return NULL;
 327         INIT_LIST_HEAD(&nvdimm_bus->list);
 328         INIT_LIST_HEAD(&nvdimm_bus->poison_list);
 329         init_waitqueue_head(&nvdimm_bus->probe_wait);
 330         nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
 331         mutex_init(&nvdimm_bus->reconfig_mutex);
 332         if (nvdimm_bus->id < 0) {
 333                 kfree(nvdimm_bus);
 334                 return NULL;
 335         }
 336         nvdimm_bus->nd_desc = nd_desc;
 337         nvdimm_bus->module = module;
 338         nvdimm_bus->dev.parent = parent;
 339         nvdimm_bus->dev.release = nvdimm_bus_release;
 340         nvdimm_bus->dev.groups = nd_desc->attr_groups;
 341         dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
 342         rc = device_register(&nvdimm_bus->dev);
 343         if (rc) {
 344                 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
 345                 goto err;
 346         }
 347
 348         rc = nvdimm_bus_create_ndctl(nvdimm_bus);
 349         if (rc)
 350                 goto err;
 351
 352         mutex_lock(&nvdimm_bus_list_mutex);
 353         list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
 354         mutex_unlock(&nvdimm_bus_list_mutex);
 355
 356         return nvdimm_bus;
 357  err:
 358         put_device(&nvdimm_bus->dev);
 359         return NULL;
 360 }
 361 EXPORT_SYMBOL_GPL(__nvdimm_bus_register);
 362
 363 static void set_badblock(struct gendisk *disk, sector_t s, int num)
 364 {
 365         struct device *dev = disk->driverfs_dev;
 366
 367         dev_dbg(dev, "Found a poison range (0x%llx, 0x%llx)\n",
 368                         (u64) s * 512, (u64) num * 512);
 369         /* this isn't an error as the hardware will still throw an exception */
 370         if (disk_set_badblocks(disk, s, num))
 371                 dev_info_once(dev, "%s: failed for sector %llx\n",
 372                                 __func__, (u64) s);
 373 }
 374
 375 /**
 376  * __add_badblock_range() - Convert a physical address range to bad sectors
 377  * @disk:       the disk associated with the namespace
 378  * @ns_offset:  namespace offset where the error range begins (in bytes)
 379  * @len:        number of bytes of poison to be added
 380  *
 381  * This assumes that the range provided with (ns_offset, len) is within
 382  * the bounds of physical addresses for this namespace, i.e. lies in the
 383  * interval [ns_start, ns_start + ns_size)
 384  */
 385 static int __add_badblock_range(struct gendisk *disk, u64 ns_offset, u64 len)
 386 {
 387         unsigned int sector_size = queue_logical_block_size(disk->queue);
 388         sector_t start_sector;
 389         u64 num_sectors;
 390         u32 rem;
 391         int rc;
 392
 393         start_sector = div_u64(ns_offset, sector_size);
 394         num_sectors = div_u64_rem(len, sector_size, &rem);
 395         if (rem)
 396                 num_sectors++;
 397
 398         if (!disk->bb) {
 399                 rc = disk_alloc_badblocks(disk);
 400                 if (rc)
 401                         return rc;
 402         }
 403
 404         if (unlikely(num_sectors > (u64)INT_MAX)) {
 405                 u64 remaining = num_sectors;
 406                 sector_t s = start_sector;
 407
 408                 while (remaining) {
 409                         int done = min_t(u64, remaining, INT_MAX);
 410
 411                         set_badblock(disk, s, done);
 412                         remaining -= done;
 413                         s += done;
 414                 }
 415         } else
 416                 set_badblock(disk, start_sector, num_sectors);
 417
 418         return 0;
 419 }
 420
 421 /**
 422  * nvdimm_namespace_add_poison() - Convert a list of poison ranges to badblocks
 423  * @disk:       the gendisk associated with the namespace where badblocks
 424  *              will be stored
 425  * @offset:     offset at the start of the namespace before 'sector 0'
 426  * @ndns:       the namespace containing poison ranges
 427  *
 428  * The poison list generated during NFIT initialization may contain multiple,
 429  * possibly overlapping ranges in the SPA (System Physical Address) space.
 430  * Compare each of these ranges to the namespace currently being initialized,
 431  * and add badblocks to the gendisk for all matching sub-ranges
 432  *
 433  * Return:
 434  * 0 - Success
 435  */
 436 int nvdimm_namespace_add_poison(struct gendisk *disk, resource_size_t offset,
 437                 struct nd_namespace_common *ndns)
 438 {
 439         struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
 440         struct nd_region *nd_region = to_nd_region(ndns->dev.parent);
 441         struct nvdimm_bus *nvdimm_bus;
 442         struct list_head *poison_list;
 443         u64 ns_start, ns_end, ns_size;
 444         struct nd_poison *pl;
 445         int rc;
 446
 447         ns_size = nvdimm_namespace_capacity(ndns) - offset;
 448         ns_start = nsio->res.start + offset;
 449         ns_end = nsio->res.end;
 450
 451         nvdimm_bus = to_nvdimm_bus(nd_region->dev.parent);
 452         poison_list = &nvdimm_bus->poison_list;
 453         if (list_empty(poison_list))
 454                 return 0;
 455
 456         list_for_each_entry(pl, poison_list, list) {
 457                 u64 pl_end = pl->start + pl->length - 1;
 458
 459                 /* Discard intervals with no intersection */
 460                 if (pl_end < ns_start)
 461                         continue;
 462                 if (pl->start > ns_end)
 463                         continue;
 464                 /* Deal with any overlap after start of the namespace */
 465                 if (pl->start >= ns_start) {
 466                         u64 start = pl->start;
 467                         u64 len;
 468
 469                         if (pl_end <= ns_end)
 470                                 len = pl->length;
 471                         else
 472                                 len = ns_start + ns_size - pl->start;
 473
 474                         rc = __add_badblock_range(disk, start - ns_start, len);
 475                         if (rc)
 476                                 return rc;
 477                         continue;
 478                 }
 479                 /* Deal with overlap for poison starting before the namespace */
 480                 if (pl->start < ns_start) {
 481                         u64 len;
 482
 483                         if (pl_end < ns_end)
 484                                 len = pl->start + pl->length - ns_start;
 485                         else
 486                                 len = ns_size;
 487
 488                         rc = __add_badblock_range(disk, 0, len);
 489                         if (rc)
 490                                 return rc;
 491                 }
 492         }
 493
 494         return 0;
 495 }
 496 EXPORT_SYMBOL_GPL(nvdimm_namespace_add_poison);
 497
 498 static int __add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 499 {
 500         struct nd_poison *pl;
 501
 502         pl = kzalloc(sizeof(*pl), GFP_KERNEL);
 503         if (!pl)
 504                 return -ENOMEM;
 505
 506         pl->start = addr;
 507         pl->length = length;
 508         list_add_tail(&pl->list, &nvdimm_bus->poison_list);
 509
 510         return 0;
 511 }
 512
 513 int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 514 {
 515         struct nd_poison *pl;
 516
 517         if (list_empty(&nvdimm_bus->poison_list))
 518                 return __add_poison(nvdimm_bus, addr, length);
 519
 520         /*
 521          * There is a chance this is a duplicate, check for those first.
 522          * This will be the common case as ARS_STATUS returns all known
 523          * errors in the SPA space, and we can't query it per region
 524          */
 525         list_for_each_entry(pl, &nvdimm_bus->poison_list, list)
 526                 if (pl->start == addr) {
 527                         /* If length has changed, update this list entry */
 528                         if (pl->length != length)
 529                                 pl->length = length;
 530                         return 0;
 531                 }
 532
 533         /*
 534          * If not a duplicate or a simple length update, add the entry as is,
 535          * as any overlapping ranges will get resolved when the list is consumed
 536          * and converted to badblocks
 537          */
 538         return __add_poison(nvdimm_bus, addr, length);
 539 }
 540 EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison);
 541
 542 static void free_poison_list(struct list_head *poison_list)
 543 {
 544         struct nd_poison *pl, *next;
 545
 546         list_for_each_entry_safe(pl, next, poison_list, list) {
 547                 list_del(&pl->list);
 548                 kfree(pl);
 549         }
 550         list_del_init(poison_list);
 551 }
 552
 553 static int child_unregister(struct device *dev, void *data)
 554 {
 555         /*
 556          * the singular ndctl class device per bus needs to be
 557          * "device_destroy"ed, so skip it here
 558          *
 559          * i.e. remove classless children
 560          */
 561         if (dev->class)
 562                 /* pass */;
 563         else
 564                 nd_device_unregister(dev, ND_SYNC);
 565         return 0;
 566 }
 567
 568 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
 569 {
 570         if (!nvdimm_bus)
 571                 return;
 572
 573         mutex_lock(&nvdimm_bus_list_mutex);
 574         list_del_init(&nvdimm_bus->list);
 575         mutex_unlock(&nvdimm_bus_list_mutex);
 576
 577         nd_synchronize();
 578         device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
 579         free_poison_list(&nvdimm_bus->poison_list);
 580         nvdimm_bus_destroy_ndctl(nvdimm_bus);
 581
 582         device_unregister(&nvdimm_bus->dev);
 583 }
 584 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
 585
 586 #ifdef CONFIG_BLK_DEV_INTEGRITY
 587 int nd_integrity_init(struct gendisk *disk, unsigned long meta_size)
 588 {
 589         struct blk_integrity bi;
 590
 591         if (meta_size == 0)
 592                 return 0;
 593
 594         bi.profile = NULL;
 595         bi.tuple_size = meta_size;
 596         bi.tag_size = meta_size;
 597
 598         blk_integrity_register(disk, &bi);
 599         blk_queue_max_integrity_segments(disk->queue, 1);
 600
 601         return 0;
 602 }
 603 EXPORT_SYMBOL(nd_integrity_init);
 604
 605 #else /* CONFIG_BLK_DEV_INTEGRITY */
 606 int nd_integrity_init(struct gendisk *disk, unsigned long meta_size)
 607 {
 608         return 0;
 609 }
 610 EXPORT_SYMBOL(nd_integrity_init);
 611
 612 #endif
 613
 614 static __init int libnvdimm_init(void)
 615 {
 616         int rc;
 617
 618         rc = nvdimm_bus_init();
 619         if (rc)
 620                 return rc;
 621         rc = nvdimm_init();
 622         if (rc)
 623                 goto err_dimm;
 624         rc = nd_region_init();
 625         if (rc)
 626                 goto err_region;
 627         return 0;
 628  err_region:
 629         nvdimm_exit();
 630  err_dimm:
 631         nvdimm_bus_exit();
 632         return rc;
 633 }
 634
 635 static __exit void libnvdimm_exit(void)
 636 {
 637         WARN_ON(!list_empty(&nvdimm_bus_list));
 638         nd_region_exit();
 639         nvdimm_exit();
 640         nvdimm_bus_exit();
 641 }
 642
 643 MODULE_LICENSE("GPL v2");
 644 MODULE_AUTHOR("Intel Corporation");
 645 subsys_initcall(libnvdimm_init);
 646 module_exit(libnvdimm_exit);