drivers/lightnvm/sysblk.c

   1 /*
   2  * Copyright (C) 2015 Matias Bjorling. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License version
   6  * 2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; see the file COPYING.  If not, write to
  15  * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
  16  * USA.
  17  *
  18  */
  19
  20 #include <linux/lightnvm.h>
  21
  22 #define MAX_SYSBLKS 3   /* remember to update mapping scheme on change */
  23 #define MAX_BLKS_PR_SYSBLK 2 /* 2 blks with 256 pages and 3000 erases
  24                               * enables ~1.5M updates per sysblk unit
  25                               */
  26
  27 struct sysblk_scan {
  28         /* A row is a collection of flash blocks for a system block. */
  29         int nr_rows;
  30         int row;
  31         int act_blk[MAX_SYSBLKS];
  32
  33         int nr_ppas;
  34         struct ppa_addr ppas[MAX_SYSBLKS * MAX_BLKS_PR_SYSBLK];/* all sysblks */
  35 };
  36
  37 static inline int scan_ppa_idx(int row, int blkid)
  38 {
  39         return (row * MAX_BLKS_PR_SYSBLK) + blkid;
  40 }
  41
  42 static void nvm_sysblk_to_cpu(struct nvm_sb_info *info,
  43                               struct nvm_system_block *sb)
  44 {
  45         info->seqnr = be32_to_cpu(sb->seqnr);
  46         info->erase_cnt = be32_to_cpu(sb->erase_cnt);
  47         info->version = be16_to_cpu(sb->version);
  48         strncpy(info->mmtype, sb->mmtype, NVM_MMTYPE_LEN);
  49         info->fs_ppa.ppa = be64_to_cpu(sb->fs_ppa);
  50 }
  51
  52 static void nvm_cpu_to_sysblk(struct nvm_system_block *sb,
  53                               struct nvm_sb_info *info)
  54 {
  55         sb->magic = cpu_to_be32(NVM_SYSBLK_MAGIC);
  56         sb->seqnr = cpu_to_be32(info->seqnr);
  57         sb->erase_cnt = cpu_to_be32(info->erase_cnt);
  58         sb->version = cpu_to_be16(info->version);
  59         strncpy(sb->mmtype, info->mmtype, NVM_MMTYPE_LEN);
  60         sb->fs_ppa = cpu_to_be64(info->fs_ppa.ppa);
  61 }
  62
  63 static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas)
  64 {
  65         int nr_rows = min_t(int, MAX_SYSBLKS, dev->nr_chnls);
  66         int i;
  67
  68         for (i = 0; i < nr_rows; i++)
  69                 sysblk_ppas[i].ppa = 0;
  70
  71         /* if possible, place sysblk at first channel, middle channel and last
  72          * channel of the device. If not, create only one or two sys blocks
  73          */
  74         switch (dev->nr_chnls) {
  75         case 2:
  76                 sysblk_ppas[1].g.ch = 1;
  77                 /* fall-through */
  78         case 1:
  79                 sysblk_ppas[0].g.ch = 0;
  80                 break;
  81         default:
  82                 sysblk_ppas[0].g.ch = 0;
  83                 sysblk_ppas[1].g.ch = dev->nr_chnls / 2;
  84                 sysblk_ppas[2].g.ch = dev->nr_chnls - 1;
  85                 break;
  86         }
  87
  88         return nr_rows;
  89 }
  90
  91 static void nvm_setup_sysblk_scan(struct nvm_dev *dev, struct sysblk_scan *s,
  92                                                 struct ppa_addr *sysblk_ppas)
  93 {
  94         memset(s, 0, sizeof(struct sysblk_scan));
  95         s->nr_rows = nvm_setup_sysblks(dev, sysblk_ppas);
  96 }
  97
  98 static int sysblk_get_free_blks(struct nvm_dev *dev, struct ppa_addr ppa,
  99                                         u8 *blks, int nr_blks,
 100                                         struct sysblk_scan *s)
 101 {
 102         struct ppa_addr *sppa;
 103         int i, blkid = 0;
 104
 105         nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
 106         if (nr_blks < 0)
 107                 return nr_blks;
 108
 109         for (i = 0; i < nr_blks; i++) {
 110                 if (blks[i] == NVM_BLK_T_HOST)
 111                         return -EEXIST;
 112
 113                 if (blks[i] != NVM_BLK_T_FREE)
 114                         continue;
 115
 116                 sppa = &s->ppas[scan_ppa_idx(s->row, blkid)];
 117                 sppa->g.ch = ppa.g.ch;
 118                 sppa->g.lun = ppa.g.lun;
 119                 sppa->g.blk = i;
 120                 s->nr_ppas++;
 121                 blkid++;
 122
 123                 pr_debug("nvm: use (%u %u %u) as sysblk\n",
 124                                         sppa->g.ch, sppa->g.lun, sppa->g.blk);
 125                 if (blkid > MAX_BLKS_PR_SYSBLK - 1)
 126                         return 0;
 127         }
 128
 129         pr_err("nvm: sysblk failed get sysblk\n");
 130         return -EINVAL;
 131 }
 132
 133 static int sysblk_get_host_blks(struct nvm_dev *dev, struct ppa_addr ppa,
 134                                         u8 *blks, int nr_blks,
 135                                         struct sysblk_scan *s)
 136 {
 137         int i, nr_sysblk = 0;
 138
 139         nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
 140         if (nr_blks < 0)
 141                 return nr_blks;
 142
 143         for (i = 0; i < nr_blks; i++) {
 144                 if (blks[i] != NVM_BLK_T_HOST)
 145                         continue;
 146
 147                 if (s->nr_ppas == MAX_BLKS_PR_SYSBLK * MAX_SYSBLKS) {
 148                         pr_err("nvm: too many host blks\n");
 149                         return -EINVAL;
 150                 }
 151
 152                 ppa.g.blk = i;
 153
 154                 s->ppas[scan_ppa_idx(s->row, nr_sysblk)] = ppa;
 155                 s->nr_ppas++;
 156                 nr_sysblk++;
 157         }
 158
 159         return 0;
 160 }
 161
 162 static int nvm_get_all_sysblks(struct nvm_dev *dev, struct sysblk_scan *s,
 163                                 struct ppa_addr *ppas, int get_free)
 164 {
 165         int i, nr_blks, ret = 0;
 166         u8 *blks;
 167
 168         s->nr_ppas = 0;
 169         nr_blks = dev->blks_per_lun * dev->plane_mode;
 170
 171         blks = kmalloc(nr_blks, GFP_KERNEL);
 172         if (!blks)
 173                 return -ENOMEM;
 174
 175         for (i = 0; i < s->nr_rows; i++) {
 176                 s->row = i;
 177
 178                 ret = nvm_get_bb_tbl(dev, ppas[i], blks);
 179                 if (ret) {
 180                         pr_err("nvm: failed bb tbl for ppa (%u %u)\n",
 181                                                         ppas[i].g.ch,
 182                                                         ppas[i].g.blk);
 183                         goto err_get;
 184                 }
 185
 186                 if (get_free)
 187                         ret = sysblk_get_free_blks(dev, ppas[i], blks, nr_blks,
 188                                                                         s);
 189                 else
 190                         ret = sysblk_get_host_blks(dev, ppas[i], blks, nr_blks,
 191                                                                         s);
 192
 193                 if (ret)
 194                         goto err_get;
 195         }
 196
 197 err_get:
 198         kfree(blks);
 199         return ret;
 200 }
 201
 202 /*
 203  * scans a block for latest sysblk.
 204  * Returns:
 205  *      0 - newer sysblk not found. PPA is updated to latest page.
 206  *      1 - newer sysblk found and stored in *cur. PPA is updated to
 207  *          next valid page.
 208  *      <0- error.
 209  */
 210 static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa,
 211                                                 struct nvm_system_block *sblk)
 212 {
 213         struct nvm_system_block *cur;
 214         int pg, ret, found = 0;
 215
 216         /* the full buffer for a flash page is allocated. Only the first of it
 217          * contains the system block information
 218          */
 219         cur = kmalloc(dev->pfpg_size, GFP_KERNEL);
 220         if (!cur)
 221                 return -ENOMEM;
 222
 223         /* perform linear scan through the block */
 224         for (pg = 0; pg < dev->lps_per_blk; pg++) {
 225                 ppa->g.pg = ppa_to_slc(dev, pg);
 226
 227                 ret = nvm_submit_ppa(dev, ppa, 1, NVM_OP_PREAD, NVM_IO_SLC_MODE,
 228                                                         cur, dev->pfpg_size);
 229                 if (ret) {
 230                         if (ret == NVM_RSP_ERR_EMPTYPAGE) {
 231                                 pr_debug("nvm: sysblk scan empty ppa (%u %u %u %u)\n",
 232                                                         ppa->g.ch,
 233                                                         ppa->g.lun,
 234                                                         ppa->g.blk,
 235                                                         ppa->g.pg);
 236                                 break;
 237                         }
 238                         pr_err("nvm: read failed (%x) for ppa (%u %u %u %u)",
 239                                                         ret,
 240                                                         ppa->g.ch,
 241                                                         ppa->g.lun,
 242                                                         ppa->g.blk,
 243                                                         ppa->g.pg);
 244                         break; /* if we can't read a page, continue to the
 245                                 * next blk
 246                                 */
 247                 }
 248
 249                 if (be32_to_cpu(cur->magic) != NVM_SYSBLK_MAGIC) {
 250                         pr_debug("nvm: scan break for ppa (%u %u %u %u)\n",
 251                                                         ppa->g.ch,
 252                                                         ppa->g.lun,
 253                                                         ppa->g.blk,
 254                                                         ppa->g.pg);
 255                         break; /* last valid page already found */
 256                 }
 257
 258                 if (be32_to_cpu(cur->seqnr) < be32_to_cpu(sblk->seqnr))
 259                         continue;
 260
 261                 memcpy(sblk, cur, sizeof(struct nvm_system_block));
 262                 found = 1;
 263         }
 264
 265         kfree(cur);
 266
 267         return found;
 268 }
 269
 270 static int nvm_sysblk_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s,
 271                                                                 int type)
 272 {
 273         return nvm_set_bb_tbl(dev, s->ppas, s->nr_ppas, type);
 274 }
 275
 276 static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
 277                                                         struct sysblk_scan *s)
 278 {
 279         struct nvm_system_block nvmsb;
 280         void *buf;
 281         int i, sect, ret = 0;
 282         struct ppa_addr *ppas;
 283
 284         nvm_cpu_to_sysblk(&nvmsb, info);
 285
 286         buf = kzalloc(dev->pfpg_size, GFP_KERNEL);
 287         if (!buf)
 288                 return -ENOMEM;
 289         memcpy(buf, &nvmsb, sizeof(struct nvm_system_block));
 290
 291         ppas = kcalloc(dev->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL);
 292         if (!ppas) {
 293                 ret = -ENOMEM;
 294                 goto err;
 295         }
 296
 297         /* Write and verify */
 298         for (i = 0; i < s->nr_rows; i++) {
 299                 ppas[0] = s->ppas[scan_ppa_idx(i, s->act_blk[i])];
 300
 301                 pr_debug("nvm: writing sysblk to ppa (%u %u %u %u)\n",
 302                                                         ppas[0].g.ch,
 303                                                         ppas[0].g.lun,
 304                                                         ppas[0].g.blk,
 305                                                         ppas[0].g.pg);
 306
 307                 /* Expand to all sectors within a flash page */
 308                 if (dev->sec_per_pg > 1) {
 309                         for (sect = 1; sect < dev->sec_per_pg; sect++) {
 310                                 ppas[sect].ppa = ppas[0].ppa;
 311                                 ppas[sect].g.sec = sect;
 312                         }
 313                 }
 314
 315                 ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PWRITE,
 316                                         NVM_IO_SLC_MODE, buf, dev->pfpg_size);
 317                 if (ret) {
 318                         pr_err("nvm: sysblk failed program (%u %u %u)\n",
 319                                                         ppas[0].g.ch,
 320                                                         ppas[0].g.lun,
 321                                                         ppas[0].g.blk);
 322                         break;
 323                 }
 324
 325                 ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PREAD,
 326                                         NVM_IO_SLC_MODE, buf, dev->pfpg_size);
 327                 if (ret) {
 328                         pr_err("nvm: sysblk failed read (%u %u %u)\n",
 329                                                         ppas[0].g.ch,
 330                                                         ppas[0].g.lun,
 331                                                         ppas[0].g.blk);
 332                         break;
 333                 }
 334
 335                 if (memcmp(buf, &nvmsb, sizeof(struct nvm_system_block))) {
 336                         pr_err("nvm: sysblk failed verify (%u %u %u)\n",
 337                                                         ppas[0].g.ch,
 338                                                         ppas[0].g.lun,
 339                                                         ppas[0].g.blk);
 340                         ret = -EINVAL;
 341                         break;
 342                 }
 343         }
 344
 345         kfree(ppas);
 346 err:
 347         kfree(buf);
 348
 349         return ret;
 350 }
 351
 352 static int nvm_prepare_new_sysblks(struct nvm_dev *dev, struct sysblk_scan *s)
 353 {
 354         int i, ret;
 355         unsigned long nxt_blk;
 356         struct ppa_addr *ppa;
 357
 358         for (i = 0; i < s->nr_rows; i++) {
 359                 nxt_blk = (s->act_blk[i] + 1) % MAX_BLKS_PR_SYSBLK;
 360                 ppa = &s->ppas[scan_ppa_idx(i, nxt_blk)];
 361                 ppa->g.pg = ppa_to_slc(dev, 0);
 362
 363                 ret = nvm_erase_ppa(dev, ppa, 1, 0);
 364                 if (ret)
 365                         return ret;
 366
 367                 s->act_blk[i] = nxt_blk;
 368         }
 369
 370         return 0;
 371 }
 372
 373 int nvm_get_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
 374 {
 375         struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
 376         struct sysblk_scan s;
 377         struct nvm_system_block *cur;
 378         int i, j, found = 0;
 379         int ret = -ENOMEM;
 380
 381         /*
 382          * 1. setup sysblk locations
 383          * 2. get bad block list
 384          * 3. filter on host-specific (type 3)
 385          * 4. iterate through all and find the highest seq nr.
 386          * 5. return superblock information
 387          */
 388
 389         if (!dev->ops->get_bb_tbl)
 390                 return -EINVAL;
 391
 392         nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
 393
 394         mutex_lock(&dev->mlock);
 395         ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
 396         if (ret)
 397                 goto err_sysblk;
 398
 399         /* no sysblocks initialized */
 400         if (!s.nr_ppas)
 401                 goto err_sysblk;
 402
 403         cur = kzalloc(sizeof(struct nvm_system_block), GFP_KERNEL);
 404         if (!cur)
 405                 goto err_sysblk;
 406
 407         /* find the latest block across all sysblocks */
 408         for (i = 0; i < s.nr_rows; i++) {
 409                 for (j = 0; j < MAX_BLKS_PR_SYSBLK; j++) {
 410                         struct ppa_addr ppa = s.ppas[scan_ppa_idx(i, j)];
 411
 412                         ret = nvm_scan_block(dev, &ppa, cur);
 413                         if (ret > 0)
 414                                 found = 1;
 415                         else if (ret < 0)
 416                                 break;
 417                 }
 418         }
 419
 420         nvm_sysblk_to_cpu(info, cur);
 421
 422         kfree(cur);
 423 err_sysblk:
 424         mutex_unlock(&dev->mlock);
 425
 426         if (found)
 427                 return 1;
 428         return ret;
 429 }
 430
 431 int nvm_update_sysblock(struct nvm_dev *dev, struct nvm_sb_info *new)
 432 {
 433         /* 1. for each latest superblock
 434          * 2. if room
 435          *    a. write new flash page entry with the updated information
 436          * 3. if no room
 437          *    a. find next available block on lun (linear search)
 438          *       if none, continue to next lun
 439          *       if none at all, report error. also report that it wasn't
 440          *       possible to write to all superblocks.
 441          *    c. write data to block.
 442          */
 443         struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
 444         struct sysblk_scan s;
 445         struct nvm_system_block *cur;
 446         int i, j, ppaidx, found = 0;
 447         int ret = -ENOMEM;
 448
 449         if (!dev->ops->get_bb_tbl)
 450                 return -EINVAL;
 451
 452         nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
 453
 454         mutex_lock(&dev->mlock);
 455         ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
 456         if (ret)
 457                 goto err_sysblk;
 458
 459         cur = kzalloc(sizeof(struct nvm_system_block), GFP_KERNEL);
 460         if (!cur)
 461                 goto err_sysblk;
 462
 463         /* Get the latest sysblk for each sysblk row */
 464         for (i = 0; i < s.nr_rows; i++) {
 465                 found = 0;
 466                 for (j = 0; j < MAX_BLKS_PR_SYSBLK; j++) {
 467                         ppaidx = scan_ppa_idx(i, j);
 468                         ret = nvm_scan_block(dev, &s.ppas[ppaidx], cur);
 469                         if (ret > 0) {
 470                                 s.act_blk[i] = j;
 471                                 found = 1;
 472                         } else if (ret < 0)
 473                                 break;
 474                 }
 475         }
 476
 477         if (!found) {
 478                 pr_err("nvm: no valid sysblks found to update\n");
 479                 ret = -EINVAL;
 480                 goto err_cur;
 481         }
 482
 483         /*
 484          * All sysblocks found. Check that they have same page id in their flash
 485          * blocks
 486          */
 487         for (i = 1; i < s.nr_rows; i++) {
 488                 struct ppa_addr l = s.ppas[scan_ppa_idx(0, s.act_blk[0])];
 489                 struct ppa_addr r = s.ppas[scan_ppa_idx(i, s.act_blk[i])];
 490
 491                 if (l.g.pg != r.g.pg) {
 492                         pr_err("nvm: sysblks not on same page. Previous update failed.\n");
 493                         ret = -EINVAL;
 494                         goto err_cur;
 495                 }
 496         }
 497
 498         /*
 499          * Check that there haven't been another update to the seqnr since we
 500          * began
 501          */
 502         if ((new->seqnr - 1) != be32_to_cpu(cur->seqnr)) {
 503                 pr_err("nvm: seq is not sequential\n");
 504                 ret = -EINVAL;
 505                 goto err_cur;
 506         }
 507
 508         /*
 509          * When all pages in a block has been written, a new block is selected
 510          * and writing is performed on the new block.
 511          */
 512         if (s.ppas[scan_ppa_idx(0, s.act_blk[0])].g.pg ==
 513                                                 dev->lps_per_blk - 1) {
 514                 ret = nvm_prepare_new_sysblks(dev, &s);
 515                 if (ret)
 516                         goto err_cur;
 517         }
 518
 519         ret = nvm_write_and_verify(dev, new, &s);
 520 err_cur:
 521         kfree(cur);
 522 err_sysblk:
 523         mutex_unlock(&dev->mlock);
 524
 525         return ret;
 526 }
 527
 528 int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
 529 {
 530         struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
 531         struct sysblk_scan s;
 532         int ret;
 533
 534         /*
 535          * 1. select master blocks and select first available blks
 536          * 2. get bad block list
 537          * 3. mark MAX_SYSBLKS block as host-based device allocated.
 538          * 4. write and verify data to block
 539          */
 540
 541         if (!dev->ops->get_bb_tbl || !dev->ops->set_bb_tbl)
 542                 return -EINVAL;
 543
 544         if (!(dev->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) {
 545                 pr_err("nvm: memory does not support SLC access\n");
 546                 return -EINVAL;
 547         }
 548
 549         /* Index all sysblocks and mark them as host-driven */
 550         nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
 551
 552         mutex_lock(&dev->mlock);
 553         ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 1);
 554         if (ret)
 555                 goto err_mark;
 556
 557         ret = nvm_sysblk_set_bb_tbl(dev, &s, NVM_BLK_T_HOST);
 558         if (ret)
 559                 goto err_mark;
 560
 561         /* Write to the first block of each row */
 562         ret = nvm_write_and_verify(dev, info, &s);
 563 err_mark:
 564         mutex_unlock(&dev->mlock);
 565         return ret;
 566 }
 567
 568 static int factory_nblks(int nblks)
 569 {
 570         /* Round up to nearest BITS_PER_LONG */
 571         return (nblks + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
 572 }
 573
 574 static unsigned int factory_blk_offset(struct nvm_dev *dev, struct ppa_addr ppa)
 575 {
 576         int nblks = factory_nblks(dev->blks_per_lun);
 577
 578         return ((ppa.g.ch * dev->luns_per_chnl * nblks) + (ppa.g.lun * nblks)) /
 579                                                                 BITS_PER_LONG;
 580 }
 581
 582 static int nvm_factory_blks(struct nvm_dev *dev, struct ppa_addr ppa,
 583                                         u8 *blks, int nr_blks,
 584                                         unsigned long *blk_bitmap, int flags)
 585 {
 586         int i, lunoff;
 587
 588         nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
 589         if (nr_blks < 0)
 590                 return nr_blks;
 591
 592         lunoff = factory_blk_offset(dev, ppa);
 593
 594         /* non-set bits correspond to the block must be erased */
 595         for (i = 0; i < nr_blks; i++) {
 596                 switch (blks[i]) {
 597                 case NVM_BLK_T_FREE:
 598                         if (flags & NVM_FACTORY_ERASE_ONLY_USER)
 599                                 set_bit(i, &blk_bitmap[lunoff]);
 600                         break;
 601                 case NVM_BLK_T_HOST:
 602                         if (!(flags & NVM_FACTORY_RESET_HOST_BLKS))
 603                                 set_bit(i, &blk_bitmap[lunoff]);
 604                         break;
 605                 case NVM_BLK_T_GRWN_BAD:
 606                         if (!(flags & NVM_FACTORY_RESET_GRWN_BBLKS))
 607                                 set_bit(i, &blk_bitmap[lunoff]);
 608                         break;
 609                 default:
 610                         set_bit(i, &blk_bitmap[lunoff]);
 611                         break;
 612                 }
 613         }
 614
 615         return 0;
 616 }
 617
 618 static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list,
 619                                         int max_ppas, unsigned long *blk_bitmap)
 620 {
 621         struct ppa_addr ppa;
 622         int ch, lun, blkid, idx, done = 0, ppa_cnt = 0;
 623         unsigned long *offset;
 624
 625         while (!done) {
 626                 done = 1;
 627                 nvm_for_each_lun_ppa(dev, ppa, ch, lun) {
 628                         idx = factory_blk_offset(dev, ppa);
 629                         offset = &blk_bitmap[idx];
 630
 631                         blkid = find_first_zero_bit(offset,
 632                                                 dev->blks_per_lun);
 633                         if (blkid >= dev->blks_per_lun)
 634                                 continue;
 635                         set_bit(blkid, offset);
 636
 637                         ppa.g.blk = blkid;
 638                         pr_debug("nvm: erase ppa (%u %u %u)\n",
 639                                                         ppa.g.ch,
 640                                                         ppa.g.lun,
 641                                                         ppa.g.blk);
 642
 643                         erase_list[ppa_cnt] = ppa;
 644                         ppa_cnt++;
 645                         done = 0;
 646
 647                         if (ppa_cnt == max_ppas)
 648                                 return ppa_cnt;
 649                 }
 650         }
 651
 652         return ppa_cnt;
 653 }
 654
 655 static int nvm_fact_select_blks(struct nvm_dev *dev, unsigned long *blk_bitmap,
 656                                                                 int flags)
 657 {
 658         struct ppa_addr ppa;
 659         int ch, lun, nr_blks, ret = 0;
 660         u8 *blks;
 661
 662         nr_blks = dev->blks_per_lun * dev->plane_mode;
 663         blks = kmalloc(nr_blks, GFP_KERNEL);
 664         if (!blks)
 665                 return -ENOMEM;
 666
 667         nvm_for_each_lun_ppa(dev, ppa, ch, lun) {
 668                 ret = nvm_get_bb_tbl(dev, ppa, blks);
 669                 if (ret)
 670                         pr_err("nvm: failed bb tbl for ch%u lun%u\n",
 671                                                         ppa.g.ch, ppa.g.blk);
 672
 673                 ret = nvm_factory_blks(dev, ppa, blks, nr_blks, blk_bitmap,
 674                                                                         flags);
 675                 if (ret)
 676                         break;
 677         }
 678
 679         kfree(blks);
 680         return ret;
 681 }
 682
 683 int nvm_dev_factory(struct nvm_dev *dev, int flags)
 684 {
 685         struct ppa_addr *ppas;
 686         int ppa_cnt, ret = -ENOMEM;
 687         int max_ppas = dev->ops->max_phys_sect / dev->nr_planes;
 688         struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
 689         struct sysblk_scan s;
 690         unsigned long *blk_bitmap;
 691
 692         blk_bitmap = kzalloc(factory_nblks(dev->blks_per_lun) * dev->nr_luns,
 693                                                                 GFP_KERNEL);
 694         if (!blk_bitmap)
 695                 return ret;
 696
 697         ppas = kcalloc(max_ppas, sizeof(struct ppa_addr), GFP_KERNEL);
 698         if (!ppas)
 699                 goto err_blks;
 700
 701         /* create list of blks to be erased */
 702         ret = nvm_fact_select_blks(dev, blk_bitmap, flags);
 703         if (ret)
 704                 goto err_ppas;
 705
 706         /* continue to erase until list of blks until empty */
 707         while ((ppa_cnt =
 708                         nvm_fact_get_blks(dev, ppas, max_ppas, blk_bitmap)) > 0)
 709                 nvm_erase_ppa(dev, ppas, ppa_cnt, 0);
 710
 711         /* mark host reserved blocks free */
 712         if (flags & NVM_FACTORY_RESET_HOST_BLKS) {
 713                 nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
 714                 mutex_lock(&dev->mlock);
 715                 ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
 716                 if (!ret)
 717                         ret = nvm_sysblk_set_bb_tbl(dev, &s, NVM_BLK_T_FREE);
 718                 mutex_unlock(&dev->mlock);
 719         }
 720 err_ppas:
 721         kfree(ppas);
 722 err_blks:
 723         kfree(blk_bitmap);
 724         return ret;
 725 }
 726 EXPORT_SYMBOL(nvm_dev_factory);