2 * Copyright (C) 2015 Matias Bjorling. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; see the file COPYING. If not, write to
15 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
20 #include <linux/lightnvm.h>
22 #define MAX_SYSBLKS 3 /* remember to update mapping scheme on change */
23 #define MAX_BLKS_PR_SYSBLK 2 /* 2 blks with 256 pages and 3000 erases
24 * enables ~1.5M updates per sysblk unit
28 /* A row is a collection of flash blocks for a system block. */
31 int act_blk[MAX_SYSBLKS];
34 struct ppa_addr ppas[MAX_SYSBLKS * MAX_BLKS_PR_SYSBLK];/* all sysblks */
37 static inline int scan_ppa_idx(int row, int blkid)
39 return (row * MAX_BLKS_PR_SYSBLK) + blkid;
42 static void nvm_sysblk_to_cpu(struct nvm_sb_info *info,
43 struct nvm_system_block *sb)
45 info->seqnr = be32_to_cpu(sb->seqnr);
46 info->erase_cnt = be32_to_cpu(sb->erase_cnt);
47 info->version = be16_to_cpu(sb->version);
48 strncpy(info->mmtype, sb->mmtype, NVM_MMTYPE_LEN);
49 info->fs_ppa.ppa = be64_to_cpu(sb->fs_ppa);
52 static void nvm_cpu_to_sysblk(struct nvm_system_block *sb,
53 struct nvm_sb_info *info)
55 sb->magic = cpu_to_be32(NVM_SYSBLK_MAGIC);
56 sb->seqnr = cpu_to_be32(info->seqnr);
57 sb->erase_cnt = cpu_to_be32(info->erase_cnt);
58 sb->version = cpu_to_be16(info->version);
59 strncpy(sb->mmtype, info->mmtype, NVM_MMTYPE_LEN);
60 sb->fs_ppa = cpu_to_be64(info->fs_ppa.ppa);
63 static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas)
65 int nr_rows = min_t(int, MAX_SYSBLKS, dev->nr_chnls);
68 for (i = 0; i < nr_rows; i++)
69 sysblk_ppas[i].ppa = 0;
71 /* if possible, place sysblk at first channel, middle channel and last
72 * channel of the device. If not, create only one or two sys blocks
74 switch (dev->nr_chnls) {
76 sysblk_ppas[1].g.ch = 1;
79 sysblk_ppas[0].g.ch = 0;
82 sysblk_ppas[0].g.ch = 0;
83 sysblk_ppas[1].g.ch = dev->nr_chnls / 2;
84 sysblk_ppas[2].g.ch = dev->nr_chnls - 1;
91 static void nvm_setup_sysblk_scan(struct nvm_dev *dev, struct sysblk_scan *s,
92 struct ppa_addr *sysblk_ppas)
94 memset(s, 0, sizeof(struct sysblk_scan));
95 s->nr_rows = nvm_setup_sysblks(dev, sysblk_ppas);
98 static int sysblk_get_free_blks(struct nvm_dev *dev, struct ppa_addr ppa,
99 u8 *blks, int nr_blks,
100 struct sysblk_scan *s)
102 struct ppa_addr *sppa;
105 nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
109 for (i = 0; i < nr_blks; i++) {
110 if (blks[i] == NVM_BLK_T_HOST)
113 if (blks[i] != NVM_BLK_T_FREE)
116 sppa = &s->ppas[scan_ppa_idx(s->row, blkid)];
117 sppa->g.ch = ppa.g.ch;
118 sppa->g.lun = ppa.g.lun;
123 pr_debug("nvm: use (%u %u %u) as sysblk\n",
124 sppa->g.ch, sppa->g.lun, sppa->g.blk);
125 if (blkid > MAX_BLKS_PR_SYSBLK - 1)
129 pr_err("nvm: sysblk failed get sysblk\n");
133 static int sysblk_get_host_blks(struct nvm_dev *dev, struct ppa_addr ppa,
134 u8 *blks, int nr_blks,
135 struct sysblk_scan *s)
137 int i, nr_sysblk = 0;
139 nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
143 for (i = 0; i < nr_blks; i++) {
144 if (blks[i] != NVM_BLK_T_HOST)
147 if (s->nr_ppas == MAX_BLKS_PR_SYSBLK * MAX_SYSBLKS) {
148 pr_err("nvm: too many host blks\n");
154 s->ppas[scan_ppa_idx(s->row, nr_sysblk)] = ppa;
162 static int nvm_get_all_sysblks(struct nvm_dev *dev, struct sysblk_scan *s,
163 struct ppa_addr *ppas, int get_free)
165 int i, nr_blks, ret = 0;
169 nr_blks = dev->blks_per_lun * dev->plane_mode;
171 blks = kmalloc(nr_blks, GFP_KERNEL);
175 for (i = 0; i < s->nr_rows; i++) {
178 ret = nvm_get_bb_tbl(dev, ppas[i], blks);
180 pr_err("nvm: failed bb tbl for ppa (%u %u)\n",
187 ret = sysblk_get_free_blks(dev, ppas[i], blks, nr_blks,
190 ret = sysblk_get_host_blks(dev, ppas[i], blks, nr_blks,
203 * scans a block for latest sysblk.
205 * 0 - newer sysblk not found. PPA is updated to latest page.
206 * 1 - newer sysblk found and stored in *cur. PPA is updated to
210 static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa,
211 struct nvm_system_block *sblk)
213 struct nvm_system_block *cur;
214 int pg, ret, found = 0;
216 /* the full buffer for a flash page is allocated. Only the first of it
217 * contains the system block information
219 cur = kmalloc(dev->pfpg_size, GFP_KERNEL);
223 /* perform linear scan through the block */
224 for (pg = 0; pg < dev->lps_per_blk; pg++) {
225 ppa->g.pg = ppa_to_slc(dev, pg);
227 ret = nvm_submit_ppa(dev, ppa, 1, NVM_OP_PREAD, NVM_IO_SLC_MODE,
228 cur, dev->pfpg_size);
230 if (ret == NVM_RSP_ERR_EMPTYPAGE) {
231 pr_debug("nvm: sysblk scan empty ppa (%u %u %u %u)\n",
238 pr_err("nvm: read failed (%x) for ppa (%u %u %u %u)",
244 break; /* if we can't read a page, continue to the
249 if (be32_to_cpu(cur->magic) != NVM_SYSBLK_MAGIC) {
250 pr_debug("nvm: scan break for ppa (%u %u %u %u)\n",
255 break; /* last valid page already found */
258 if (be32_to_cpu(cur->seqnr) < be32_to_cpu(sblk->seqnr))
261 memcpy(sblk, cur, sizeof(struct nvm_system_block));
270 static int nvm_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s, int type)
275 if (s->nr_ppas > dev->ops->max_phys_sect) {
276 pr_err("nvm: unable to update all sysblocks atomically\n");
280 memset(&rqd, 0, sizeof(struct nvm_rq));
282 nvm_set_rqd_ppalist(dev, &rqd, s->ppas, s->nr_ppas, 1);
283 nvm_generic_to_addr_mode(dev, &rqd);
285 ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
286 nvm_free_rqd_ppalist(dev, &rqd);
288 pr_err("nvm: sysblk failed bb mark\n");
295 static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
296 struct sysblk_scan *s)
298 struct nvm_system_block nvmsb;
300 int i, sect, ret = 0;
301 struct ppa_addr *ppas;
303 nvm_cpu_to_sysblk(&nvmsb, info);
305 buf = kzalloc(dev->pfpg_size, GFP_KERNEL);
308 memcpy(buf, &nvmsb, sizeof(struct nvm_system_block));
310 ppas = kcalloc(dev->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL);
316 /* Write and verify */
317 for (i = 0; i < s->nr_rows; i++) {
318 ppas[0] = s->ppas[scan_ppa_idx(i, s->act_blk[i])];
320 pr_debug("nvm: writing sysblk to ppa (%u %u %u %u)\n",
326 /* Expand to all sectors within a flash page */
327 if (dev->sec_per_pg > 1) {
328 for (sect = 1; sect < dev->sec_per_pg; sect++) {
329 ppas[sect].ppa = ppas[0].ppa;
330 ppas[sect].g.sec = sect;
334 ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PWRITE,
335 NVM_IO_SLC_MODE, buf, dev->pfpg_size);
337 pr_err("nvm: sysblk failed program (%u %u %u)\n",
344 ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PREAD,
345 NVM_IO_SLC_MODE, buf, dev->pfpg_size);
347 pr_err("nvm: sysblk failed read (%u %u %u)\n",
354 if (memcmp(buf, &nvmsb, sizeof(struct nvm_system_block))) {
355 pr_err("nvm: sysblk failed verify (%u %u %u)\n",
371 static int nvm_prepare_new_sysblks(struct nvm_dev *dev, struct sysblk_scan *s)
374 unsigned long nxt_blk;
375 struct ppa_addr *ppa;
377 for (i = 0; i < s->nr_rows; i++) {
378 nxt_blk = (s->act_blk[i] + 1) % MAX_BLKS_PR_SYSBLK;
379 ppa = &s->ppas[scan_ppa_idx(i, nxt_blk)];
380 ppa->g.pg = ppa_to_slc(dev, 0);
382 ret = nvm_erase_ppa(dev, ppa, 1);
386 s->act_blk[i] = nxt_blk;
392 int nvm_get_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
394 struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
395 struct sysblk_scan s;
396 struct nvm_system_block *cur;
401 * 1. setup sysblk locations
402 * 2. get bad block list
403 * 3. filter on host-specific (type 3)
404 * 4. iterate through all and find the highest seq nr.
405 * 5. return superblock information
408 if (!dev->ops->get_bb_tbl)
411 nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
413 mutex_lock(&dev->mlock);
414 ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
418 /* no sysblocks initialized */
422 cur = kzalloc(sizeof(struct nvm_system_block), GFP_KERNEL);
426 /* find the latest block across all sysblocks */
427 for (i = 0; i < s.nr_rows; i++) {
428 for (j = 0; j < MAX_BLKS_PR_SYSBLK; j++) {
429 struct ppa_addr ppa = s.ppas[scan_ppa_idx(i, j)];
431 ret = nvm_scan_block(dev, &ppa, cur);
439 nvm_sysblk_to_cpu(info, cur);
443 mutex_unlock(&dev->mlock);
450 int nvm_update_sysblock(struct nvm_dev *dev, struct nvm_sb_info *new)
452 /* 1. for each latest superblock
454 * a. write new flash page entry with the updated information
456 * a. find next available block on lun (linear search)
457 * if none, continue to next lun
458 * if none at all, report error. also report that it wasn't
459 * possible to write to all superblocks.
460 * c. write data to block.
462 struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
463 struct sysblk_scan s;
464 struct nvm_system_block *cur;
465 int i, j, ppaidx, found = 0;
468 if (!dev->ops->get_bb_tbl)
471 nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
473 mutex_lock(&dev->mlock);
474 ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
478 cur = kzalloc(sizeof(struct nvm_system_block), GFP_KERNEL);
482 /* Get the latest sysblk for each sysblk row */
483 for (i = 0; i < s.nr_rows; i++) {
485 for (j = 0; j < MAX_BLKS_PR_SYSBLK; j++) {
486 ppaidx = scan_ppa_idx(i, j);
487 ret = nvm_scan_block(dev, &s.ppas[ppaidx], cur);
497 pr_err("nvm: no valid sysblks found to update\n");
503 * All sysblocks found. Check that they have same page id in their flash
506 for (i = 1; i < s.nr_rows; i++) {
507 struct ppa_addr l = s.ppas[scan_ppa_idx(0, s.act_blk[0])];
508 struct ppa_addr r = s.ppas[scan_ppa_idx(i, s.act_blk[i])];
510 if (l.g.pg != r.g.pg) {
511 pr_err("nvm: sysblks not on same page. Previous update failed.\n");
518 * Check that there haven't been another update to the seqnr since we
521 if ((new->seqnr - 1) != be32_to_cpu(cur->seqnr)) {
522 pr_err("nvm: seq is not sequential\n");
528 * When all pages in a block has been written, a new block is selected
529 * and writing is performed on the new block.
531 if (s.ppas[scan_ppa_idx(0, s.act_blk[0])].g.pg ==
532 dev->lps_per_blk - 1) {
533 ret = nvm_prepare_new_sysblks(dev, &s);
538 ret = nvm_write_and_verify(dev, new, &s);
542 mutex_unlock(&dev->mlock);
547 int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
549 struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
550 struct sysblk_scan s;
554 * 1. select master blocks and select first available blks
555 * 2. get bad block list
556 * 3. mark MAX_SYSBLKS block as host-based device allocated.
557 * 4. write and verify data to block
560 if (!dev->ops->get_bb_tbl || !dev->ops->set_bb_tbl)
563 if (!(dev->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) {
564 pr_err("nvm: memory does not support SLC access\n");
568 /* Index all sysblocks and mark them as host-driven */
569 nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
571 mutex_lock(&dev->mlock);
572 ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 1);
576 ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_HOST);
580 /* Write to the first block of each row */
581 ret = nvm_write_and_verify(dev, info, &s);
583 mutex_unlock(&dev->mlock);
587 static int factory_nblks(int nblks)
589 /* Round up to nearest BITS_PER_LONG */
590 return (nblks + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
593 static unsigned int factory_blk_offset(struct nvm_dev *dev, struct ppa_addr ppa)
595 int nblks = factory_nblks(dev->blks_per_lun);
597 return ((ppa.g.ch * dev->luns_per_chnl * nblks) + (ppa.g.lun * nblks)) /
601 static int nvm_factory_blks(struct nvm_dev *dev, struct ppa_addr ppa,
602 u8 *blks, int nr_blks,
603 unsigned long *blk_bitmap, int flags)
607 nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
611 lunoff = factory_blk_offset(dev, ppa);
613 /* non-set bits correspond to the block must be erased */
614 for (i = 0; i < nr_blks; i++) {
617 if (flags & NVM_FACTORY_ERASE_ONLY_USER)
618 set_bit(i, &blk_bitmap[lunoff]);
621 if (!(flags & NVM_FACTORY_RESET_HOST_BLKS))
622 set_bit(i, &blk_bitmap[lunoff]);
624 case NVM_BLK_T_GRWN_BAD:
625 if (!(flags & NVM_FACTORY_RESET_GRWN_BBLKS))
626 set_bit(i, &blk_bitmap[lunoff]);
629 set_bit(i, &blk_bitmap[lunoff]);
637 static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list,
638 int max_ppas, unsigned long *blk_bitmap)
641 int ch, lun, blkid, idx, done = 0, ppa_cnt = 0;
642 unsigned long *offset;
646 nvm_for_each_lun_ppa(dev, ppa, ch, lun) {
647 idx = factory_blk_offset(dev, ppa);
648 offset = &blk_bitmap[idx];
650 blkid = find_first_zero_bit(offset,
652 if (blkid >= dev->blks_per_lun)
654 set_bit(blkid, offset);
657 pr_debug("nvm: erase ppa (%u %u %u)\n",
662 erase_list[ppa_cnt] = ppa;
666 if (ppa_cnt == max_ppas)
674 static int nvm_fact_select_blks(struct nvm_dev *dev, unsigned long *blk_bitmap,
678 int ch, lun, nr_blks, ret = 0;
681 nr_blks = dev->blks_per_lun * dev->plane_mode;
682 blks = kmalloc(nr_blks, GFP_KERNEL);
686 nvm_for_each_lun_ppa(dev, ppa, ch, lun) {
687 ret = nvm_get_bb_tbl(dev, ppa, blks);
689 pr_err("nvm: failed bb tbl for ch%u lun%u\n",
690 ppa.g.ch, ppa.g.blk);
692 ret = nvm_factory_blks(dev, ppa, blks, nr_blks, blk_bitmap,
702 int nvm_dev_factory(struct nvm_dev *dev, int flags)
704 struct ppa_addr *ppas;
705 int ppa_cnt, ret = -ENOMEM;
706 int max_ppas = dev->ops->max_phys_sect / dev->nr_planes;
707 struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
708 struct sysblk_scan s;
709 unsigned long *blk_bitmap;
711 blk_bitmap = kzalloc(factory_nblks(dev->blks_per_lun) * dev->nr_luns,
716 ppas = kcalloc(max_ppas, sizeof(struct ppa_addr), GFP_KERNEL);
720 /* create list of blks to be erased */
721 ret = nvm_fact_select_blks(dev, blk_bitmap, flags);
725 /* continue to erase until list of blks until empty */
727 nvm_fact_get_blks(dev, ppas, max_ppas, blk_bitmap)) > 0)
728 nvm_erase_ppa(dev, ppas, ppa_cnt);
730 /* mark host reserved blocks free */
731 if (flags & NVM_FACTORY_RESET_HOST_BLKS) {
732 nvm_setup_sysblk_scan(dev, &s, sysblk_ppas);
733 mutex_lock(&dev->mlock);
734 ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
736 ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_FREE);
737 mutex_unlock(&dev->mlock);
745 EXPORT_SYMBOL(nvm_dev_factory);