fs/f2fs/node.c

   1 /*
   2  * fs/f2fs/node.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/mpage.h>
  14 #include <linux/backing-dev.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/pagevec.h>
  17 #include <linux/swap.h>
  18
  19 #include "f2fs.h"
  20 #include "node.h"
  21 #include "segment.h"
  22 #include <trace/events/f2fs.h>
  23
  24 static struct kmem_cache *nat_entry_slab;
  25 static struct kmem_cache *free_nid_slab;
  26
  27 static void clear_node_page_dirty(struct page *page)
  28 {
  29         struct address_space *mapping = page->mapping;
  30         struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
  31         unsigned int long flags;
  32
  33         if (PageDirty(page)) {
  34                 spin_lock_irqsave(&mapping->tree_lock, flags);
  35                 radix_tree_tag_clear(&mapping->page_tree,
  36                                 page_index(page),
  37                                 PAGECACHE_TAG_DIRTY);
  38                 spin_unlock_irqrestore(&mapping->tree_lock, flags);
  39
  40                 clear_page_dirty_for_io(page);
  41                 dec_page_count(sbi, F2FS_DIRTY_NODES);
  42         }
  43         ClearPageUptodate(page);
  44 }
  45
  46 static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
  47 {
  48         pgoff_t index = current_nat_addr(sbi, nid);
  49         return get_meta_page(sbi, index);
  50 }
  51
  52 static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
  53 {
  54         struct page *src_page;
  55         struct page *dst_page;
  56         pgoff_t src_off;
  57         pgoff_t dst_off;
  58         void *src_addr;
  59         void *dst_addr;
  60         struct f2fs_nm_info *nm_i = NM_I(sbi);
  61
  62         src_off = current_nat_addr(sbi, nid);
  63         dst_off = next_nat_addr(sbi, src_off);
  64
  65         /* get current nat block page with lock */
  66         src_page = get_meta_page(sbi, src_off);
  67
  68         /* Dirty src_page means that it is already the new target NAT page. */
  69         if (PageDirty(src_page))
  70                 return src_page;
  71
  72         dst_page = grab_meta_page(sbi, dst_off);
  73
  74         src_addr = page_address(src_page);
  75         dst_addr = page_address(dst_page);
  76         memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
  77         set_page_dirty(dst_page);
  78         f2fs_put_page(src_page, 1);
  79
  80         set_to_next_nat(nm_i, nid);
  81
  82         return dst_page;
  83 }
  84
  85 static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)
  86 {
  87         return radix_tree_lookup(&nm_i->nat_root, n);
  88 }
  89
  90 static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,
  91                 nid_t start, unsigned int nr, struct nat_entry **ep)
  92 {
  93         return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);
  94 }
  95
  96 static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
  97 {
  98         list_del(&e->list);
  99         radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));
 100         nm_i->nat_cnt--;
 101         kmem_cache_free(nat_entry_slab, e);
 102 }
 103
 104 int is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
 105 {
 106         struct f2fs_nm_info *nm_i = NM_I(sbi);
 107         struct nat_entry *e;
 108         int is_cp = 1;
 109
 110         read_lock(&nm_i->nat_tree_lock);
 111         e = __lookup_nat_cache(nm_i, nid);
 112         if (e && !e->checkpointed)
 113                 is_cp = 0;
 114         read_unlock(&nm_i->nat_tree_lock);
 115         return is_cp;
 116 }
 117
 118 static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)
 119 {
 120         struct nat_entry *new;
 121
 122         new = kmem_cache_alloc(nat_entry_slab, GFP_ATOMIC);
 123         if (!new)
 124                 return NULL;
 125         if (radix_tree_insert(&nm_i->nat_root, nid, new)) {
 126                 kmem_cache_free(nat_entry_slab, new);
 127                 return NULL;
 128         }
 129         memset(new, 0, sizeof(struct nat_entry));
 130         nat_set_nid(new, nid);
 131         new->checkpointed = true;
 132         list_add_tail(&new->list, &nm_i->nat_entries);
 133         nm_i->nat_cnt++;
 134         return new;
 135 }
 136
 137 static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,
 138                                                 struct f2fs_nat_entry *ne)
 139 {
 140         struct nat_entry *e;
 141 retry:
 142         write_lock(&nm_i->nat_tree_lock);
 143         e = __lookup_nat_cache(nm_i, nid);
 144         if (!e) {
 145                 e = grab_nat_entry(nm_i, nid);
 146                 if (!e) {
 147                         write_unlock(&nm_i->nat_tree_lock);
 148                         goto retry;
 149                 }
 150                 nat_set_blkaddr(e, le32_to_cpu(ne->block_addr));
 151                 nat_set_ino(e, le32_to_cpu(ne->ino));
 152                 nat_set_version(e, ne->version);
 153         }
 154         write_unlock(&nm_i->nat_tree_lock);
 155 }
 156
 157 static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
 158                         block_t new_blkaddr)
 159 {
 160         struct f2fs_nm_info *nm_i = NM_I(sbi);
 161         struct nat_entry *e;
 162 retry:
 163         write_lock(&nm_i->nat_tree_lock);
 164         e = __lookup_nat_cache(nm_i, ni->nid);
 165         if (!e) {
 166                 e = grab_nat_entry(nm_i, ni->nid);
 167                 if (!e) {
 168                         write_unlock(&nm_i->nat_tree_lock);
 169                         goto retry;
 170                 }
 171                 e->ni = *ni;
 172                 f2fs_bug_on(ni->blk_addr == NEW_ADDR);
 173         } else if (new_blkaddr == NEW_ADDR) {
 174                 /*
 175                  * when nid is reallocated,
 176                  * previous nat entry can be remained in nat cache.
 177                  * So, reinitialize it with new information.
 178                  */
 179                 e->ni = *ni;
 180                 f2fs_bug_on(ni->blk_addr != NULL_ADDR);
 181         }
 182
 183         /* sanity check */
 184         f2fs_bug_on(nat_get_blkaddr(e) != ni->blk_addr);
 185         f2fs_bug_on(nat_get_blkaddr(e) == NULL_ADDR &&
 186                         new_blkaddr == NULL_ADDR);
 187         f2fs_bug_on(nat_get_blkaddr(e) == NEW_ADDR &&
 188                         new_blkaddr == NEW_ADDR);
 189         f2fs_bug_on(nat_get_blkaddr(e) != NEW_ADDR &&
 190                         nat_get_blkaddr(e) != NULL_ADDR &&
 191                         new_blkaddr == NEW_ADDR);
 192
 193         /* increament version no as node is removed */
 194         if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {
 195                 unsigned char version = nat_get_version(e);
 196                 nat_set_version(e, inc_node_version(version));
 197         }
 198
 199         /* change address */
 200         nat_set_blkaddr(e, new_blkaddr);
 201         __set_nat_cache_dirty(nm_i, e);
 202         write_unlock(&nm_i->nat_tree_lock);
 203 }
 204
 205 int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
 206 {
 207         struct f2fs_nm_info *nm_i = NM_I(sbi);
 208
 209         if (nm_i->nat_cnt <= NM_WOUT_THRESHOLD)
 210                 return 0;
 211
 212         write_lock(&nm_i->nat_tree_lock);
 213         while (nr_shrink && !list_empty(&nm_i->nat_entries)) {
 214                 struct nat_entry *ne;
 215                 ne = list_first_entry(&nm_i->nat_entries,
 216                                         struct nat_entry, list);
 217                 __del_from_nat_cache(nm_i, ne);
 218                 nr_shrink--;
 219         }
 220         write_unlock(&nm_i->nat_tree_lock);
 221         return nr_shrink;
 222 }
 223
 224 /*
 225  * This function returns always success
 226  */
 227 void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni)
 228 {
 229         struct f2fs_nm_info *nm_i = NM_I(sbi);
 230         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 231         struct f2fs_summary_block *sum = curseg->sum_blk;
 232         nid_t start_nid = START_NID(nid);
 233         struct f2fs_nat_block *nat_blk;
 234         struct page *page = NULL;
 235         struct f2fs_nat_entry ne;
 236         struct nat_entry *e;
 237         int i;
 238
 239         memset(&ne, 0, sizeof(struct f2fs_nat_entry));
 240         ni->nid = nid;
 241
 242         /* Check nat cache */
 243         read_lock(&nm_i->nat_tree_lock);
 244         e = __lookup_nat_cache(nm_i, nid);
 245         if (e) {
 246                 ni->ino = nat_get_ino(e);
 247                 ni->blk_addr = nat_get_blkaddr(e);
 248                 ni->version = nat_get_version(e);
 249         }
 250         read_unlock(&nm_i->nat_tree_lock);
 251         if (e)
 252                 return;
 253
 254         /* Check current segment summary */
 255         mutex_lock(&curseg->curseg_mutex);
 256         i = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 0);
 257         if (i >= 0) {
 258                 ne = nat_in_journal(sum, i);
 259                 node_info_from_raw_nat(ni, &ne);
 260         }
 261         mutex_unlock(&curseg->curseg_mutex);
 262         if (i >= 0)
 263                 goto cache;
 264
 265         /* Fill node_info from nat page */
 266         page = get_current_nat_page(sbi, start_nid);
 267         nat_blk = (struct f2fs_nat_block *)page_address(page);
 268         ne = nat_blk->entries[nid - start_nid];
 269         node_info_from_raw_nat(ni, &ne);
 270         f2fs_put_page(page, 1);
 271 cache:
 272         /* cache nat entry */
 273         cache_nat_entry(NM_I(sbi), nid, &ne);
 274 }
 275
 276 /*
 277  * The maximum depth is four.
 278  * Offset[0] will have raw inode offset.
 279  */
 280 static int get_node_path(struct f2fs_inode_info *fi, long block,
 281                                 int offset[4], unsigned int noffset[4])
 282 {
 283         const long direct_index = ADDRS_PER_INODE(fi);
 284         const long direct_blks = ADDRS_PER_BLOCK;
 285         const long dptrs_per_blk = NIDS_PER_BLOCK;
 286         const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
 287         const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
 288         int n = 0;
 289         int level = 0;
 290
 291         noffset[0] = 0;
 292
 293         if (block < direct_index) {
 294                 offset[n] = block;
 295                 goto got;
 296         }
 297         block -= direct_index;
 298         if (block < direct_blks) {
 299                 offset[n++] = NODE_DIR1_BLOCK;
 300                 noffset[n] = 1;
 301                 offset[n] = block;
 302                 level = 1;
 303                 goto got;
 304         }
 305         block -= direct_blks;
 306         if (block < direct_blks) {
 307                 offset[n++] = NODE_DIR2_BLOCK;
 308                 noffset[n] = 2;
 309                 offset[n] = block;
 310                 level = 1;
 311                 goto got;
 312         }
 313         block -= direct_blks;
 314         if (block < indirect_blks) {
 315                 offset[n++] = NODE_IND1_BLOCK;
 316                 noffset[n] = 3;
 317                 offset[n++] = block / direct_blks;
 318                 noffset[n] = 4 + offset[n - 1];
 319                 offset[n] = block % direct_blks;
 320                 level = 2;
 321                 goto got;
 322         }
 323         block -= indirect_blks;
 324         if (block < indirect_blks) {
 325                 offset[n++] = NODE_IND2_BLOCK;
 326                 noffset[n] = 4 + dptrs_per_blk;
 327                 offset[n++] = block / direct_blks;
 328                 noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
 329                 offset[n] = block % direct_blks;
 330                 level = 2;
 331                 goto got;
 332         }
 333         block -= indirect_blks;
 334         if (block < dindirect_blks) {
 335                 offset[n++] = NODE_DIND_BLOCK;
 336                 noffset[n] = 5 + (dptrs_per_blk * 2);
 337                 offset[n++] = block / indirect_blks;
 338                 noffset[n] = 6 + (dptrs_per_blk * 2) +
 339                               offset[n - 1] * (dptrs_per_blk + 1);
 340                 offset[n++] = (block / direct_blks) % dptrs_per_blk;
 341                 noffset[n] = 7 + (dptrs_per_blk * 2) +
 342                               offset[n - 2] * (dptrs_per_blk + 1) +
 343                               offset[n - 1];
 344                 offset[n] = block % direct_blks;
 345                 level = 3;
 346                 goto got;
 347         } else {
 348                 BUG();
 349         }
 350 got:
 351         return level;
 352 }
 353
 354 /*
 355  * Caller should call f2fs_put_dnode(dn).
 356  * Also, it should grab and release a rwsem by calling f2fs_lock_op() and
 357  * f2fs_unlock_op() only if ro is not set RDONLY_NODE.
 358  * In the case of RDONLY_NODE, we don't need to care about mutex.
 359  */
 360 int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 361 {
 362         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 363         struct page *npage[4];
 364         struct page *parent;
 365         int offset[4];
 366         unsigned int noffset[4];
 367         nid_t nids[4];
 368         int level, i;
 369         int err = 0;
 370
 371         level = get_node_path(F2FS_I(dn->inode), index, offset, noffset);
 372
 373         nids[0] = dn->inode->i_ino;
 374         npage[0] = dn->inode_page;
 375
 376         if (!npage[0]) {
 377                 npage[0] = get_node_page(sbi, nids[0]);
 378                 if (IS_ERR(npage[0]))
 379                         return PTR_ERR(npage[0]);
 380         }
 381         parent = npage[0];
 382         if (level != 0)
 383                 nids[1] = get_nid(parent, offset[0], true);
 384         dn->inode_page = npage[0];
 385         dn->inode_page_locked = true;
 386
 387         /* get indirect or direct nodes */
 388         for (i = 1; i <= level; i++) {
 389                 bool done = false;
 390
 391                 if (!nids[i] && mode == ALLOC_NODE) {
 392                         /* alloc new node */
 393                         if (!alloc_nid(sbi, &(nids[i]))) {
 394                                 err = -ENOSPC;
 395                                 goto release_pages;
 396                         }
 397
 398                         dn->nid = nids[i];
 399                         npage[i] = new_node_page(dn, noffset[i], NULL);
 400                         if (IS_ERR(npage[i])) {
 401                                 alloc_nid_failed(sbi, nids[i]);
 402                                 err = PTR_ERR(npage[i]);
 403                                 goto release_pages;
 404                         }
 405
 406                         set_nid(parent, offset[i - 1], nids[i], i == 1);
 407                         alloc_nid_done(sbi, nids[i]);
 408                         done = true;
 409                 } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
 410                         npage[i] = get_node_page_ra(parent, offset[i - 1]);
 411                         if (IS_ERR(npage[i])) {
 412                                 err = PTR_ERR(npage[i]);
 413                                 goto release_pages;
 414                         }
 415                         done = true;
 416                 }
 417                 if (i == 1) {
 418                         dn->inode_page_locked = false;
 419                         unlock_page(parent);
 420                 } else {
 421                         f2fs_put_page(parent, 1);
 422                 }
 423
 424                 if (!done) {
 425                         npage[i] = get_node_page(sbi, nids[i]);
 426                         if (IS_ERR(npage[i])) {
 427                                 err = PTR_ERR(npage[i]);
 428                                 f2fs_put_page(npage[0], 0);
 429                                 goto release_out;
 430                         }
 431                 }
 432                 if (i < level) {
 433                         parent = npage[i];
 434                         nids[i + 1] = get_nid(parent, offset[i], false);
 435                 }
 436         }
 437         dn->nid = nids[level];
 438         dn->ofs_in_node = offset[level];
 439         dn->node_page = npage[level];
 440         dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
 441         return 0;
 442
 443 release_pages:
 444         f2fs_put_page(parent, 1);
 445         if (i > 1)
 446                 f2fs_put_page(npage[0], 0);
 447 release_out:
 448         dn->inode_page = NULL;
 449         dn->node_page = NULL;
 450         return err;
 451 }
 452
 453 static void truncate_node(struct dnode_of_data *dn)
 454 {
 455         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 456         struct node_info ni;
 457
 458         get_node_info(sbi, dn->nid, &ni);
 459         if (dn->inode->i_blocks == 0) {
 460                 f2fs_bug_on(ni.blk_addr != NULL_ADDR);
 461                 goto invalidate;
 462         }
 463         f2fs_bug_on(ni.blk_addr == NULL_ADDR);
 464
 465         /* Deallocate node address */
 466         invalidate_blocks(sbi, ni.blk_addr);
 467         dec_valid_node_count(sbi, dn->inode);
 468         set_node_addr(sbi, &ni, NULL_ADDR);
 469
 470         if (dn->nid == dn->inode->i_ino) {
 471                 remove_orphan_inode(sbi, dn->nid);
 472                 dec_valid_inode_count(sbi);
 473         } else {
 474                 sync_inode_page(dn);
 475         }
 476 invalidate:
 477         clear_node_page_dirty(dn->node_page);
 478         F2FS_SET_SB_DIRT(sbi);
 479
 480         f2fs_put_page(dn->node_page, 1);
 481
 482         invalidate_mapping_pages(NODE_MAPPING(sbi),
 483                         dn->node_page->index, dn->node_page->index);
 484
 485         dn->node_page = NULL;
 486         trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
 487 }
 488
 489 static int truncate_dnode(struct dnode_of_data *dn)
 490 {
 491         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 492         struct page *page;
 493
 494         if (dn->nid == 0)
 495                 return 1;
 496
 497         /* get direct node */
 498         page = get_node_page(sbi, dn->nid);
 499         if (IS_ERR(page) && PTR_ERR(page) == -ENOENT)
 500                 return 1;
 501         else if (IS_ERR(page))
 502                 return PTR_ERR(page);
 503
 504         /* Make dnode_of_data for parameter */
 505         dn->node_page = page;
 506         dn->ofs_in_node = 0;
 507         truncate_data_blocks(dn);
 508         truncate_node(dn);
 509         return 1;
 510 }
 511
 512 static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 513                                                 int ofs, int depth)
 514 {
 515         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 516         struct dnode_of_data rdn = *dn;
 517         struct page *page;
 518         struct f2fs_node *rn;
 519         nid_t child_nid;
 520         unsigned int child_nofs;
 521         int freed = 0;
 522         int i, ret;
 523
 524         if (dn->nid == 0)
 525                 return NIDS_PER_BLOCK + 1;
 526
 527         trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);
 528
 529         page = get_node_page(sbi, dn->nid);
 530         if (IS_ERR(page)) {
 531                 trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(page));
 532                 return PTR_ERR(page);
 533         }
 534
 535         rn = F2FS_NODE(page);
 536         if (depth < 3) {
 537                 for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
 538                         child_nid = le32_to_cpu(rn->in.nid[i]);
 539                         if (child_nid == 0)
 540                                 continue;
 541                         rdn.nid = child_nid;
 542                         ret = truncate_dnode(&rdn);
 543                         if (ret < 0)
 544                                 goto out_err;
 545                         set_nid(page, i, 0, false);
 546                 }
 547         } else {
 548                 child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
 549                 for (i = ofs; i < NIDS_PER_BLOCK; i++) {
 550                         child_nid = le32_to_cpu(rn->in.nid[i]);
 551                         if (child_nid == 0) {
 552                                 child_nofs += NIDS_PER_BLOCK + 1;
 553                                 continue;
 554                         }
 555                         rdn.nid = child_nid;
 556                         ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
 557                         if (ret == (NIDS_PER_BLOCK + 1)) {
 558                                 set_nid(page, i, 0, false);
 559                                 child_nofs += ret;
 560                         } else if (ret < 0 && ret != -ENOENT) {
 561                                 goto out_err;
 562                         }
 563                 }
 564                 freed = child_nofs;
 565         }
 566
 567         if (!ofs) {
 568                 /* remove current indirect node */
 569                 dn->node_page = page;
 570                 truncate_node(dn);
 571                 freed++;
 572         } else {
 573                 f2fs_put_page(page, 1);
 574         }
 575         trace_f2fs_truncate_nodes_exit(dn->inode, freed);
 576         return freed;
 577
 578 out_err:
 579         f2fs_put_page(page, 1);
 580         trace_f2fs_truncate_nodes_exit(dn->inode, ret);
 581         return ret;
 582 }
 583
 584 static int truncate_partial_nodes(struct dnode_of_data *dn,
 585                         struct f2fs_inode *ri, int *offset, int depth)
 586 {
 587         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 588         struct page *pages[2];
 589         nid_t nid[3];
 590         nid_t child_nid;
 591         int err = 0;
 592         int i;
 593         int idx = depth - 2;
 594
 595         nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
 596         if (!nid[0])
 597                 return 0;
 598
 599         /* get indirect nodes in the path */
 600         for (i = 0; i < idx + 1; i++) {
 601                 /* refernece count'll be increased */
 602                 pages[i] = get_node_page(sbi, nid[i]);
 603                 if (IS_ERR(pages[i])) {
 604                         err = PTR_ERR(pages[i]);
 605                         idx = i - 1;
 606                         goto fail;
 607                 }
 608                 nid[i + 1] = get_nid(pages[i], offset[i + 1], false);
 609         }
 610
 611         /* free direct nodes linked to a partial indirect node */
 612         for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) {
 613                 child_nid = get_nid(pages[idx], i, false);
 614                 if (!child_nid)
 615                         continue;
 616                 dn->nid = child_nid;
 617                 err = truncate_dnode(dn);
 618                 if (err < 0)
 619                         goto fail;
 620                 set_nid(pages[idx], i, 0, false);
 621         }
 622
 623         if (offset[idx + 1] == 0) {
 624                 dn->node_page = pages[idx];
 625                 dn->nid = nid[idx];
 626                 truncate_node(dn);
 627         } else {
 628                 f2fs_put_page(pages[idx], 1);
 629         }
 630         offset[idx]++;
 631         offset[idx + 1] = 0;
 632         idx--;
 633 fail:
 634         for (i = idx; i >= 0; i--)
 635                 f2fs_put_page(pages[i], 1);
 636
 637         trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);
 638
 639         return err;
 640 }
 641
 642 /*
 643  * All the block addresses of data and nodes should be nullified.
 644  */
 645 int truncate_inode_blocks(struct inode *inode, pgoff_t from)
 646 {
 647         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 648         int err = 0, cont = 1;
 649         int level, offset[4], noffset[4];
 650         unsigned int nofs = 0;
 651         struct f2fs_inode *ri;
 652         struct dnode_of_data dn;
 653         struct page *page;
 654
 655         trace_f2fs_truncate_inode_blocks_enter(inode, from);
 656
 657         level = get_node_path(F2FS_I(inode), from, offset, noffset);
 658 restart:
 659         page = get_node_page(sbi, inode->i_ino);
 660         if (IS_ERR(page)) {
 661                 trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(page));
 662                 return PTR_ERR(page);
 663         }
 664
 665         set_new_dnode(&dn, inode, page, NULL, 0);
 666         unlock_page(page);
 667
 668         ri = F2FS_INODE(page);
 669         switch (level) {
 670         case 0:
 671         case 1:
 672                 nofs = noffset[1];
 673                 break;
 674         case 2:
 675                 nofs = noffset[1];
 676                 if (!offset[level - 1])
 677                         goto skip_partial;
 678                 err = truncate_partial_nodes(&dn, ri, offset, level);
 679                 if (err < 0 && err != -ENOENT)
 680                         goto fail;
 681                 nofs += 1 + NIDS_PER_BLOCK;
 682                 break;
 683         case 3:
 684                 nofs = 5 + 2 * NIDS_PER_BLOCK;
 685                 if (!offset[level - 1])
 686                         goto skip_partial;
 687                 err = truncate_partial_nodes(&dn, ri, offset, level);
 688                 if (err < 0 && err != -ENOENT)
 689                         goto fail;
 690                 break;
 691         default:
 692                 BUG();
 693         }
 694
 695 skip_partial:
 696         while (cont) {
 697                 dn.nid = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
 698                 switch (offset[0]) {
 699                 case NODE_DIR1_BLOCK:
 700                 case NODE_DIR2_BLOCK:
 701                         err = truncate_dnode(&dn);
 702                         break;
 703
 704                 case NODE_IND1_BLOCK:
 705                 case NODE_IND2_BLOCK:
 706                         err = truncate_nodes(&dn, nofs, offset[1], 2);
 707                         break;
 708
 709                 case NODE_DIND_BLOCK:
 710                         err = truncate_nodes(&dn, nofs, offset[1], 3);
 711                         cont = 0;
 712                         break;
 713
 714                 default:
 715                         BUG();
 716                 }
 717                 if (err < 0 && err != -ENOENT)
 718                         goto fail;
 719                 if (offset[1] == 0 &&
 720                                 ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) {
 721                         lock_page(page);
 722                         if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
 723                                 f2fs_put_page(page, 1);
 724                                 goto restart;
 725                         }
 726                         wait_on_page_writeback(page);
 727                         ri->i_nid[offset[0] - NODE_DIR1_BLOCK] = 0;
 728                         set_page_dirty(page);
 729                         unlock_page(page);
 730                 }
 731                 offset[1] = 0;
 732                 offset[0]++;
 733                 nofs += err;
 734         }
 735 fail:
 736         f2fs_put_page(page, 0);
 737         trace_f2fs_truncate_inode_blocks_exit(inode, err);
 738         return err > 0 ? 0 : err;
 739 }
 740
 741 int truncate_xattr_node(struct inode *inode, struct page *page)
 742 {
 743         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 744         nid_t nid = F2FS_I(inode)->i_xattr_nid;
 745         struct dnode_of_data dn;
 746         struct page *npage;
 747
 748         if (!nid)
 749                 return 0;
 750
 751         npage = get_node_page(sbi, nid);
 752         if (IS_ERR(npage))
 753                 return PTR_ERR(npage);
 754
 755         F2FS_I(inode)->i_xattr_nid = 0;
 756
 757         /* need to do checkpoint during fsync */
 758         F2FS_I(inode)->xattr_ver = cur_cp_version(F2FS_CKPT(sbi));
 759
 760         set_new_dnode(&dn, inode, page, npage, nid);
 761
 762         if (page)
 763                 dn.inode_page_locked = true;
 764         truncate_node(&dn);
 765         return 0;
 766 }
 767
 768 /*
 769  * Caller should grab and release a rwsem by calling f2fs_lock_op() and
 770  * f2fs_unlock_op().
 771  */
 772 void remove_inode_page(struct inode *inode)
 773 {
 774         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 775         struct page *page;
 776         nid_t ino = inode->i_ino;
 777         struct dnode_of_data dn;
 778
 779         page = get_node_page(sbi, ino);
 780         if (IS_ERR(page))
 781                 return;
 782
 783         if (truncate_xattr_node(inode, page)) {
 784                 f2fs_put_page(page, 1);
 785                 return;
 786         }
 787         /* 0 is possible, after f2fs_new_inode() is failed */
 788         f2fs_bug_on(inode->i_blocks != 0 && inode->i_blocks != 1);
 789         set_new_dnode(&dn, inode, page, page, ino);
 790         truncate_node(&dn);
 791 }
 792
 793 struct page *new_inode_page(struct inode *inode, const struct qstr *name)
 794 {
 795         struct dnode_of_data dn;
 796
 797         /* allocate inode page for new inode */
 798         set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
 799
 800         /* caller should f2fs_put_page(page, 1); */
 801         return new_node_page(&dn, 0, NULL);
 802 }
 803
 804 struct page *new_node_page(struct dnode_of_data *dn,
 805                                 unsigned int ofs, struct page *ipage)
 806 {
 807         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 808         struct node_info old_ni, new_ni;
 809         struct page *page;
 810         int err;
 811
 812         if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
 813                 return ERR_PTR(-EPERM);
 814
 815         page = grab_cache_page(NODE_MAPPING(sbi), dn->nid);
 816         if (!page)
 817                 return ERR_PTR(-ENOMEM);
 818
 819         if (unlikely(!inc_valid_node_count(sbi, dn->inode))) {
 820                 err = -ENOSPC;
 821                 goto fail;
 822         }
 823
 824         get_node_info(sbi, dn->nid, &old_ni);
 825
 826         /* Reinitialize old_ni with new node page */
 827         f2fs_bug_on(old_ni.blk_addr != NULL_ADDR);
 828         new_ni = old_ni;
 829         new_ni.ino = dn->inode->i_ino;
 830         set_node_addr(sbi, &new_ni, NEW_ADDR);
 831
 832         fill_node_footer(page, dn->nid, dn->inode->i_ino, ofs, true);
 833         set_cold_node(dn->inode, page);
 834         SetPageUptodate(page);
 835         set_page_dirty(page);
 836
 837         if (ofs == XATTR_NODE_OFFSET)
 838                 F2FS_I(dn->inode)->i_xattr_nid = dn->nid;
 839
 840         dn->node_page = page;
 841         if (ipage)
 842                 update_inode(dn->inode, ipage);
 843         else
 844                 sync_inode_page(dn);
 845         if (ofs == 0)
 846                 inc_valid_inode_count(sbi);
 847
 848         return page;
 849
 850 fail:
 851         clear_node_page_dirty(page);
 852         f2fs_put_page(page, 1);
 853         return ERR_PTR(err);
 854 }
 855
 856 /*
 857  * Caller should do after getting the following values.
 858  * 0: f2fs_put_page(page, 0)
 859  * LOCKED_PAGE: f2fs_put_page(page, 1)
 860  * error: nothing
 861  */
 862 static int read_node_page(struct page *page, int rw)
 863 {
 864         struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
 865         struct node_info ni;
 866
 867         get_node_info(sbi, page->index, &ni);
 868
 869         if (unlikely(ni.blk_addr == NULL_ADDR)) {
 870                 f2fs_put_page(page, 1);
 871                 return -ENOENT;
 872         }
 873
 874         if (PageUptodate(page))
 875                 return LOCKED_PAGE;
 876
 877         return f2fs_submit_page_bio(sbi, page, ni.blk_addr, rw);
 878 }
 879
 880 /*
 881  * Readahead a node page
 882  */
 883 void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
 884 {
 885         struct page *apage;
 886         int err;
 887
 888         apage = find_get_page(NODE_MAPPING(sbi), nid);
 889         if (apage && PageUptodate(apage)) {
 890                 f2fs_put_page(apage, 0);
 891                 return;
 892         }
 893         f2fs_put_page(apage, 0);
 894
 895         apage = grab_cache_page(NODE_MAPPING(sbi), nid);
 896         if (!apage)
 897                 return;
 898
 899         err = read_node_page(apage, READA);
 900         if (err == 0)
 901                 f2fs_put_page(apage, 0);
 902         else if (err == LOCKED_PAGE)
 903                 f2fs_put_page(apage, 1);
 904 }
 905
 906 struct page *get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid)
 907 {
 908         struct page *page;
 909         int err;
 910 repeat:
 911         page = grab_cache_page(NODE_MAPPING(sbi), nid);
 912         if (!page)
 913                 return ERR_PTR(-ENOMEM);
 914
 915         err = read_node_page(page, READ_SYNC);
 916         if (err < 0)
 917                 return ERR_PTR(err);
 918         else if (err == LOCKED_PAGE)
 919                 goto got_it;
 920
 921         lock_page(page);
 922         if (unlikely(!PageUptodate(page))) {
 923                 f2fs_put_page(page, 1);
 924                 return ERR_PTR(-EIO);
 925         }
 926         if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
 927                 f2fs_put_page(page, 1);
 928                 goto repeat;
 929         }
 930 got_it:
 931         f2fs_bug_on(nid != nid_of_node(page));
 932         mark_page_accessed(page);
 933         return page;
 934 }
 935
 936 /*
 937  * Return a locked page for the desired node page.
 938  * And, readahead MAX_RA_NODE number of node pages.
 939  */
 940 struct page *get_node_page_ra(struct page *parent, int start)
 941 {
 942         struct f2fs_sb_info *sbi = F2FS_SB(parent->mapping->host->i_sb);
 943         struct blk_plug plug;
 944         struct page *page;
 945         int err, i, end;
 946         nid_t nid;
 947
 948         /* First, try getting the desired direct node. */
 949         nid = get_nid(parent, start, false);
 950         if (!nid)
 951                 return ERR_PTR(-ENOENT);
 952 repeat:
 953         page = grab_cache_page(NODE_MAPPING(sbi), nid);
 954         if (!page)
 955                 return ERR_PTR(-ENOMEM);
 956
 957         err = read_node_page(page, READ_SYNC);
 958         if (err < 0)
 959                 return ERR_PTR(err);
 960         else if (err == LOCKED_PAGE)
 961                 goto page_hit;
 962
 963         blk_start_plug(&plug);
 964
 965         /* Then, try readahead for siblings of the desired node */
 966         end = start + MAX_RA_NODE;
 967         end = min(end, NIDS_PER_BLOCK);
 968         for (i = start + 1; i < end; i++) {
 969                 nid = get_nid(parent, i, false);
 970                 if (!nid)
 971                         continue;
 972                 ra_node_page(sbi, nid);
 973         }
 974
 975         blk_finish_plug(&plug);
 976
 977         lock_page(page);
 978         if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
 979                 f2fs_put_page(page, 1);
 980                 goto repeat;
 981         }
 982 page_hit:
 983         if (unlikely(!PageUptodate(page))) {
 984                 f2fs_put_page(page, 1);
 985                 return ERR_PTR(-EIO);
 986         }
 987         mark_page_accessed(page);
 988         return page;
 989 }
 990
 991 void sync_inode_page(struct dnode_of_data *dn)
 992 {
 993         if (IS_INODE(dn->node_page) || dn->inode_page == dn->node_page) {
 994                 update_inode(dn->inode, dn->node_page);
 995         } else if (dn->inode_page) {
 996                 if (!dn->inode_page_locked)
 997                         lock_page(dn->inode_page);
 998                 update_inode(dn->inode, dn->inode_page);
 999                 if (!dn->inode_page_locked)
1000                         unlock_page(dn->inode_page);
1001         } else {
1002                 update_inode_page(dn->inode);
1003         }
1004 }
1005
1006 int sync_node_pages(struct f2fs_sb_info *sbi, nid_t ino,
1007                                         struct writeback_control *wbc)
1008 {
1009         pgoff_t index, end;
1010         struct pagevec pvec;
1011         int step = ino ? 2 : 0;
1012         int nwritten = 0, wrote = 0;
1013
1014         pagevec_init(&pvec, 0);
1015
1016 next_step:
1017         index = 0;
1018         end = LONG_MAX;
1019
1020         while (index <= end) {
1021                 int i, nr_pages;
1022                 nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
1023                                 PAGECACHE_TAG_DIRTY,
1024                                 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1025                 if (nr_pages == 0)
1026                         break;
1027
1028                 for (i = 0; i < nr_pages; i++) {
1029                         struct page *page = pvec.pages[i];
1030
1031                         /*
1032                          * flushing sequence with step:
1033                          * 0. indirect nodes
1034                          * 1. dentry dnodes
1035                          * 2. file dnodes
1036                          */
1037                         if (step == 0 && IS_DNODE(page))
1038                                 continue;
1039                         if (step == 1 && (!IS_DNODE(page) ||
1040                                                 is_cold_node(page)))
1041                                 continue;
1042                         if (step == 2 && (!IS_DNODE(page) ||
1043                                                 !is_cold_node(page)))
1044                                 continue;
1045
1046                         /*
1047                          * If an fsync mode,
1048                          * we should not skip writing node pages.
1049                          */
1050                         if (ino && ino_of_node(page) == ino)
1051                                 lock_page(page);
1052                         else if (!trylock_page(page))
1053                                 continue;
1054
1055                         if (unlikely(page->mapping != NODE_MAPPING(sbi))) {
1056 continue_unlock:
1057                                 unlock_page(page);
1058                                 continue;
1059                         }
1060                         if (ino && ino_of_node(page) != ino)
1061                                 goto continue_unlock;
1062
1063                         if (!PageDirty(page)) {
1064                                 /* someone wrote it for us */
1065                                 goto continue_unlock;
1066                         }
1067
1068                         if (!clear_page_dirty_for_io(page))
1069                                 goto continue_unlock;
1070
1071                         /* called by fsync() */
1072                         if (ino && IS_DNODE(page)) {
1073                                 int mark = !is_checkpointed_node(sbi, ino);
1074                                 set_fsync_mark(page, 1);
1075                                 if (IS_INODE(page))
1076                                         set_dentry_mark(page, mark);
1077                                 nwritten++;
1078                         } else {
1079                                 set_fsync_mark(page, 0);
1080                                 set_dentry_mark(page, 0);
1081                         }
1082                         NODE_MAPPING(sbi)->a_ops->writepage(page, wbc);
1083                         wrote++;
1084
1085                         if (--wbc->nr_to_write == 0)
1086                                 break;
1087                 }
1088                 pagevec_release(&pvec);
1089                 cond_resched();
1090
1091                 if (wbc->nr_to_write == 0) {
1092                         step = 2;
1093                         break;
1094                 }
1095         }
1096
1097         if (step < 2) {
1098                 step++;
1099                 goto next_step;
1100         }
1101
1102         if (wrote)
1103                 f2fs_submit_merged_bio(sbi, NODE, WRITE);
1104         return nwritten;
1105 }
1106
1107 int wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, nid_t ino)
1108 {
1109         pgoff_t index = 0, end = LONG_MAX;
1110         struct pagevec pvec;
1111         int ret2 = 0, ret = 0;
1112
1113         pagevec_init(&pvec, 0);
1114
1115         while (index <= end) {
1116                 int i, nr_pages;
1117                 nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index,
1118                                 PAGECACHE_TAG_WRITEBACK,
1119                                 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1120                 if (nr_pages == 0)
1121                         break;
1122
1123                 for (i = 0; i < nr_pages; i++) {
1124                         struct page *page = pvec.pages[i];
1125
1126                         /* until radix tree lookup accepts end_index */
1127                         if (unlikely(page->index > end))
1128                                 continue;
1129
1130                         if (ino && ino_of_node(page) == ino) {
1131                                 wait_on_page_writeback(page);
1132                                 if (TestClearPageError(page))
1133                                         ret = -EIO;
1134                         }
1135                 }
1136                 pagevec_release(&pvec);
1137                 cond_resched();
1138         }
1139
1140         if (unlikely(test_and_clear_bit(AS_ENOSPC, &NODE_MAPPING(sbi)->flags)))
1141                 ret2 = -ENOSPC;
1142         if (unlikely(test_and_clear_bit(AS_EIO, &NODE_MAPPING(sbi)->flags)))
1143                 ret2 = -EIO;
1144         if (!ret)
1145                 ret = ret2;
1146         return ret;
1147 }
1148
1149 static int f2fs_write_node_page(struct page *page,
1150                                 struct writeback_control *wbc)
1151 {
1152         struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
1153         nid_t nid;
1154         block_t new_addr;
1155         struct node_info ni;
1156         struct f2fs_io_info fio = {
1157                 .type = NODE,
1158                 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1159         };
1160
1161         if (unlikely(sbi->por_doing))
1162                 goto redirty_out;
1163
1164         wait_on_page_writeback(page);
1165
1166         /* get old block addr of this node page */
1167         nid = nid_of_node(page);
1168         f2fs_bug_on(page->index != nid);
1169
1170         get_node_info(sbi, nid, &ni);
1171
1172         /* This page is already truncated */
1173         if (unlikely(ni.blk_addr == NULL_ADDR)) {
1174                 dec_page_count(sbi, F2FS_DIRTY_NODES);
1175                 unlock_page(page);
1176                 return 0;
1177         }
1178
1179         if (wbc->for_reclaim)
1180                 goto redirty_out;
1181
1182         mutex_lock(&sbi->node_write);
1183         set_page_writeback(page);
1184         write_node_page(sbi, page, &fio, nid, ni.blk_addr, &new_addr);
1185         set_node_addr(sbi, &ni, new_addr);
1186         dec_page_count(sbi, F2FS_DIRTY_NODES);
1187         mutex_unlock(&sbi->node_write);
1188         unlock_page(page);
1189         return 0;
1190
1191 redirty_out:
1192         dec_page_count(sbi, F2FS_DIRTY_NODES);
1193         wbc->pages_skipped++;
1194         set_page_dirty(page);
1195         return AOP_WRITEPAGE_ACTIVATE;
1196 }
1197
1198 /*
1199  * It is very important to gather dirty pages and write at once, so that we can
1200  * submit a big bio without interfering other data writes.
1201  * Be default, 512 pages (2MB) * 3 node types, is more reasonable.
1202  */
1203 #define COLLECT_DIRTY_NODES     1536
1204 static int f2fs_write_node_pages(struct address_space *mapping,
1205                             struct writeback_control *wbc)
1206 {
1207         struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
1208         long nr_to_write = wbc->nr_to_write;
1209
1210         /* balancing f2fs's metadata in background */
1211         f2fs_balance_fs_bg(sbi);
1212
1213         /* collect a number of dirty node pages and write together */
1214         if (get_pages(sbi, F2FS_DIRTY_NODES) < COLLECT_DIRTY_NODES)
1215                 return 0;
1216
1217         /* if mounting is failed, skip writing node pages */
1218         wbc->nr_to_write = 3 * max_hw_blocks(sbi);
1219         wbc->sync_mode = WB_SYNC_NONE;
1220         sync_node_pages(sbi, 0, wbc);
1221         wbc->nr_to_write = nr_to_write - (3 * max_hw_blocks(sbi) -
1222                                                 wbc->nr_to_write);
1223         return 0;
1224 }
1225
1226 static int f2fs_set_node_page_dirty(struct page *page)
1227 {
1228         struct address_space *mapping = page->mapping;
1229         struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
1230
1231         trace_f2fs_set_page_dirty(page, NODE);
1232
1233         SetPageUptodate(page);
1234         if (!PageDirty(page)) {
1235                 __set_page_dirty_nobuffers(page);
1236                 inc_page_count(sbi, F2FS_DIRTY_NODES);
1237                 SetPagePrivate(page);
1238                 return 1;
1239         }
1240         return 0;
1241 }
1242
1243 static void f2fs_invalidate_node_page(struct page *page, unsigned int offset,
1244                                       unsigned int length)
1245 {
1246         struct inode *inode = page->mapping->host;
1247         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
1248         if (PageDirty(page))
1249                 dec_page_count(sbi, F2FS_DIRTY_NODES);
1250         ClearPagePrivate(page);
1251 }
1252
1253 static int f2fs_release_node_page(struct page *page, gfp_t wait)
1254 {
1255         ClearPagePrivate(page);
1256         return 1;
1257 }
1258
1259 /*
1260  * Structure of the f2fs node operations
1261  */
1262 const struct address_space_operations f2fs_node_aops = {
1263         .writepage      = f2fs_write_node_page,
1264         .writepages     = f2fs_write_node_pages,
1265         .set_page_dirty = f2fs_set_node_page_dirty,
1266         .invalidatepage = f2fs_invalidate_node_page,
1267         .releasepage    = f2fs_release_node_page,
1268 };
1269
1270 static struct free_nid *__lookup_free_nid_list(nid_t n, struct list_head *head)
1271 {
1272         struct list_head *this;
1273         struct free_nid *i;
1274         list_for_each(this, head) {
1275                 i = list_entry(this, struct free_nid, list);
1276                 if (i->nid == n)
1277                         return i;
1278         }
1279         return NULL;
1280 }
1281
1282 static void __del_from_free_nid_list(struct free_nid *i)
1283 {
1284         list_del(&i->list);
1285         kmem_cache_free(free_nid_slab, i);
1286 }
1287
1288 static int add_free_nid(struct f2fs_nm_info *nm_i, nid_t nid, bool build)
1289 {
1290         struct free_nid *i;
1291         struct nat_entry *ne;
1292         bool allocated = false;
1293
1294         if (nm_i->fcnt > 2 * MAX_FREE_NIDS)
1295                 return -1;
1296
1297         /* 0 nid should not be used */
1298         if (unlikely(nid == 0))
1299                 return 0;
1300
1301         if (build) {
1302                 /* do not add allocated nids */
1303                 read_lock(&nm_i->nat_tree_lock);
1304                 ne = __lookup_nat_cache(nm_i, nid);
1305                 if (ne && nat_get_blkaddr(ne) != NULL_ADDR)
1306                         allocated = true;
1307                 read_unlock(&nm_i->nat_tree_lock);
1308                 if (allocated)
1309                         return 0;
1310         }
1311
1312         i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS);
1313         i->nid = nid;
1314         i->state = NID_NEW;
1315
1316         spin_lock(&nm_i->free_nid_list_lock);
1317         if (__lookup_free_nid_list(nid, &nm_i->free_nid_list)) {
1318                 spin_unlock(&nm_i->free_nid_list_lock);
1319                 kmem_cache_free(free_nid_slab, i);
1320                 return 0;
1321         }
1322         list_add_tail(&i->list, &nm_i->free_nid_list);
1323         nm_i->fcnt++;
1324         spin_unlock(&nm_i->free_nid_list_lock);
1325         return 1;
1326 }
1327
1328 static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
1329 {
1330         struct free_nid *i;
1331         spin_lock(&nm_i->free_nid_list_lock);
1332         i = __lookup_free_nid_list(nid, &nm_i->free_nid_list);
1333         if (i && i->state == NID_NEW) {
1334                 __del_from_free_nid_list(i);
1335                 nm_i->fcnt--;
1336         }
1337         spin_unlock(&nm_i->free_nid_list_lock);
1338 }
1339
1340 static void scan_nat_page(struct f2fs_nm_info *nm_i,
1341                         struct page *nat_page, nid_t start_nid)
1342 {
1343         struct f2fs_nat_block *nat_blk = page_address(nat_page);
1344         block_t blk_addr;
1345         int i;
1346
1347         i = start_nid % NAT_ENTRY_PER_BLOCK;
1348
1349         for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
1350
1351                 if (unlikely(start_nid >= nm_i->max_nid))
1352                         break;
1353
1354                 blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
1355                 f2fs_bug_on(blk_addr == NEW_ADDR);
1356                 if (blk_addr == NULL_ADDR) {
1357                         if (add_free_nid(nm_i, start_nid, true) < 0)
1358                                 break;
1359                 }
1360         }
1361 }
1362
1363 static void build_free_nids(struct f2fs_sb_info *sbi)
1364 {
1365         struct f2fs_nm_info *nm_i = NM_I(sbi);
1366         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
1367         struct f2fs_summary_block *sum = curseg->sum_blk;
1368         int i = 0;
1369         nid_t nid = nm_i->next_scan_nid;
1370
1371         /* Enough entries */
1372         if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK)
1373                 return;
1374
1375         /* readahead nat pages to be scanned */
1376         ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES, META_NAT);
1377
1378         while (1) {
1379                 struct page *page = get_current_nat_page(sbi, nid);
1380
1381                 scan_nat_page(nm_i, page, nid);
1382                 f2fs_put_page(page, 1);
1383
1384                 nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
1385                 if (unlikely(nid >= nm_i->max_nid))
1386                         nid = 0;
1387
1388                 if (i++ == FREE_NID_PAGES)
1389                         break;
1390         }
1391
1392         /* go to the next free nat pages to find free nids abundantly */
1393         nm_i->next_scan_nid = nid;
1394
1395         /* find free nids from current sum_pages */
1396         mutex_lock(&curseg->curseg_mutex);
1397         for (i = 0; i < nats_in_cursum(sum); i++) {
1398                 block_t addr = le32_to_cpu(nat_in_journal(sum, i).block_addr);
1399                 nid = le32_to_cpu(nid_in_journal(sum, i));
1400                 if (addr == NULL_ADDR)
1401                         add_free_nid(nm_i, nid, true);
1402                 else
1403                         remove_free_nid(nm_i, nid);
1404         }
1405         mutex_unlock(&curseg->curseg_mutex);
1406 }
1407
1408 /*
1409  * If this function returns success, caller can obtain a new nid
1410  * from second parameter of this function.
1411  * The returned nid could be used ino as well as nid when inode is created.
1412  */
1413 bool alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
1414 {
1415         struct f2fs_nm_info *nm_i = NM_I(sbi);
1416         struct free_nid *i = NULL;
1417         struct list_head *this;
1418 retry:
1419         if (unlikely(sbi->total_valid_node_count + 1 >= nm_i->max_nid))
1420                 return false;
1421
1422         spin_lock(&nm_i->free_nid_list_lock);
1423
1424         /* We should not use stale free nids created by build_free_nids */
1425         if (nm_i->fcnt && !sbi->on_build_free_nids) {
1426                 f2fs_bug_on(list_empty(&nm_i->free_nid_list));
1427                 list_for_each(this, &nm_i->free_nid_list) {
1428                         i = list_entry(this, struct free_nid, list);
1429                         if (i->state == NID_NEW)
1430                                 break;
1431                 }
1432
1433                 f2fs_bug_on(i->state != NID_NEW);
1434                 *nid = i->nid;
1435                 i->state = NID_ALLOC;
1436                 nm_i->fcnt--;
1437                 spin_unlock(&nm_i->free_nid_list_lock);
1438                 return true;
1439         }
1440         spin_unlock(&nm_i->free_nid_list_lock);
1441
1442         /* Let's scan nat pages and its caches to get free nids */
1443         mutex_lock(&nm_i->build_lock);
1444         sbi->on_build_free_nids = true;
1445         build_free_nids(sbi);
1446         sbi->on_build_free_nids = false;
1447         mutex_unlock(&nm_i->build_lock);
1448         goto retry;
1449 }
1450
1451 /*
1452  * alloc_nid() should be called prior to this function.
1453  */
1454 void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
1455 {
1456         struct f2fs_nm_info *nm_i = NM_I(sbi);
1457         struct free_nid *i;
1458
1459         spin_lock(&nm_i->free_nid_list_lock);
1460         i = __lookup_free_nid_list(nid, &nm_i->free_nid_list);
1461         f2fs_bug_on(!i || i->state != NID_ALLOC);
1462         __del_from_free_nid_list(i);
1463         spin_unlock(&nm_i->free_nid_list_lock);
1464 }
1465
1466 /*
1467  * alloc_nid() should be called prior to this function.
1468  */
1469 void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
1470 {
1471         struct f2fs_nm_info *nm_i = NM_I(sbi);
1472         struct free_nid *i;
1473
1474         if (!nid)
1475                 return;
1476
1477         spin_lock(&nm_i->free_nid_list_lock);
1478         i = __lookup_free_nid_list(nid, &nm_i->free_nid_list);
1479         f2fs_bug_on(!i || i->state != NID_ALLOC);
1480         if (nm_i->fcnt > 2 * MAX_FREE_NIDS) {
1481                 __del_from_free_nid_list(i);
1482         } else {
1483                 i->state = NID_NEW;
1484                 nm_i->fcnt++;
1485         }
1486         spin_unlock(&nm_i->free_nid_list_lock);
1487 }
1488
1489 void recover_node_page(struct f2fs_sb_info *sbi, struct page *page,
1490                 struct f2fs_summary *sum, struct node_info *ni,
1491                 block_t new_blkaddr)
1492 {
1493         rewrite_node_page(sbi, page, sum, ni->blk_addr, new_blkaddr);
1494         set_node_addr(sbi, ni, new_blkaddr);
1495         clear_node_page_dirty(page);
1496 }
1497
1498 bool recover_xattr_data(struct inode *inode, struct page *page, block_t blkaddr)
1499 {
1500         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
1501         nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid;
1502         nid_t new_xnid = nid_of_node(page);
1503         struct node_info ni;
1504
1505         if (ofs_of_node(page) != XATTR_NODE_OFFSET)
1506                 return false;
1507
1508         /* 1: invalidate the previous xattr nid */
1509         if (!prev_xnid)
1510                 goto recover_xnid;
1511
1512         /* Deallocate node address */
1513         get_node_info(sbi, prev_xnid, &ni);
1514         f2fs_bug_on(ni.blk_addr == NULL_ADDR);
1515         invalidate_blocks(sbi, ni.blk_addr);
1516         dec_valid_node_count(sbi, inode);
1517         set_node_addr(sbi, &ni, NULL_ADDR);
1518
1519 recover_xnid:
1520         /* 2: allocate new xattr nid */
1521         if (unlikely(!inc_valid_node_count(sbi, inode)))
1522                 f2fs_bug_on(1);
1523
1524         remove_free_nid(NM_I(sbi), new_xnid);
1525         get_node_info(sbi, new_xnid, &ni);
1526         ni.ino = inode->i_ino;
1527         set_node_addr(sbi, &ni, NEW_ADDR);
1528         F2FS_I(inode)->i_xattr_nid = new_xnid;
1529
1530         /* 3: update xattr blkaddr */
1531         refresh_sit_entry(sbi, NEW_ADDR, blkaddr);
1532         set_node_addr(sbi, &ni, blkaddr);
1533
1534         update_inode_page(inode);
1535         return true;
1536 }
1537
1538 int recover_inode_page(struct f2fs_sb_info *sbi, struct page *page)
1539 {
1540         struct f2fs_inode *src, *dst;
1541         nid_t ino = ino_of_node(page);
1542         struct node_info old_ni, new_ni;
1543         struct page *ipage;
1544
1545         ipage = grab_cache_page(NODE_MAPPING(sbi), ino);
1546         if (!ipage)
1547                 return -ENOMEM;
1548
1549         /* Should not use this inode  from free nid list */
1550         remove_free_nid(NM_I(sbi), ino);
1551
1552         get_node_info(sbi, ino, &old_ni);
1553         SetPageUptodate(ipage);
1554         fill_node_footer(ipage, ino, ino, 0, true);
1555
1556         src = F2FS_INODE(page);
1557         dst = F2FS_INODE(ipage);
1558
1559         memcpy(dst, src, (unsigned long)&src->i_ext - (unsigned long)src);
1560         dst->i_size = 0;
1561         dst->i_blocks = cpu_to_le64(1);
1562         dst->i_links = cpu_to_le32(1);
1563         dst->i_xattr_nid = 0;
1564
1565         new_ni = old_ni;
1566         new_ni.ino = ino;
1567
1568         if (unlikely(!inc_valid_node_count(sbi, NULL)))
1569                 WARN_ON(1);
1570         set_node_addr(sbi, &new_ni, NEW_ADDR);
1571         inc_valid_inode_count(sbi);
1572         f2fs_put_page(ipage, 1);
1573         return 0;
1574 }
1575
1576 /*
1577  * ra_sum_pages() merge contiguous pages into one bio and submit.
1578  * these pre-readed pages are linked in pages list.
1579  */
1580 static int ra_sum_pages(struct f2fs_sb_info *sbi, struct list_head *pages,
1581                                 int start, int nrpages)
1582 {
1583         struct page *page;
1584         int page_idx = start;
1585         struct f2fs_io_info fio = {
1586                 .type = META,
1587                 .rw = READ_SYNC | REQ_META | REQ_PRIO
1588         };
1589
1590         for (; page_idx < start + nrpages; page_idx++) {
1591                 /* alloc temporal page for read node summary info*/
1592                 page = alloc_page(GFP_F2FS_ZERO);
1593                 if (!page) {
1594                         struct page *tmp;
1595                         list_for_each_entry_safe(page, tmp, pages, lru) {
1596                                 list_del(&page->lru);
1597                                 unlock_page(page);
1598                                 __free_pages(page, 0);
1599                         }
1600                         return -ENOMEM;
1601                 }
1602
1603                 lock_page(page);
1604                 page->index = page_idx;
1605                 list_add_tail(&page->lru, pages);
1606         }
1607
1608         list_for_each_entry(page, pages, lru)
1609                 f2fs_submit_page_mbio(sbi, page, page->index, &fio);
1610
1611         f2fs_submit_merged_bio(sbi, META, READ);
1612         return 0;
1613 }
1614
1615 int restore_node_summary(struct f2fs_sb_info *sbi,
1616                         unsigned int segno, struct f2fs_summary_block *sum)
1617 {
1618         struct f2fs_node *rn;
1619         struct f2fs_summary *sum_entry;
1620         struct page *page, *tmp;
1621         block_t addr;
1622         int bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
1623         int i, last_offset, nrpages, err = 0;
1624         LIST_HEAD(page_list);
1625
1626         /* scan the node segment */
1627         last_offset = sbi->blocks_per_seg;
1628         addr = START_BLOCK(sbi, segno);
1629         sum_entry = &sum->entries[0];
1630
1631         for (i = 0; i < last_offset; i += nrpages, addr += nrpages) {
1632                 nrpages = min(last_offset - i, bio_blocks);
1633
1634                 /* read ahead node pages */
1635                 err = ra_sum_pages(sbi, &page_list, addr, nrpages);
1636                 if (err)
1637                         return err;
1638
1639                 list_for_each_entry_safe(page, tmp, &page_list, lru) {
1640
1641                         lock_page(page);
1642                         if (unlikely(!PageUptodate(page))) {
1643                                 err = -EIO;
1644                         } else {
1645                                 rn = F2FS_NODE(page);
1646                                 sum_entry->nid = rn->footer.nid;
1647                                 sum_entry->version = 0;
1648                                 sum_entry->ofs_in_node = 0;
1649                                 sum_entry++;
1650                         }
1651
1652                         list_del(&page->lru);
1653                         unlock_page(page);
1654                         __free_pages(page, 0);
1655                 }
1656         }
1657         return err;
1658 }
1659
1660 static bool flush_nats_in_journal(struct f2fs_sb_info *sbi)
1661 {
1662         struct f2fs_nm_info *nm_i = NM_I(sbi);
1663         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
1664         struct f2fs_summary_block *sum = curseg->sum_blk;
1665         int i;
1666
1667         mutex_lock(&curseg->curseg_mutex);
1668
1669         if (nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) {
1670                 mutex_unlock(&curseg->curseg_mutex);
1671                 return false;
1672         }
1673
1674         for (i = 0; i < nats_in_cursum(sum); i++) {
1675                 struct nat_entry *ne;
1676                 struct f2fs_nat_entry raw_ne;
1677                 nid_t nid = le32_to_cpu(nid_in_journal(sum, i));
1678
1679                 raw_ne = nat_in_journal(sum, i);
1680 retry:
1681                 write_lock(&nm_i->nat_tree_lock);
1682                 ne = __lookup_nat_cache(nm_i, nid);
1683                 if (ne) {
1684                         __set_nat_cache_dirty(nm_i, ne);
1685                         write_unlock(&nm_i->nat_tree_lock);
1686                         continue;
1687                 }
1688                 ne = grab_nat_entry(nm_i, nid);
1689                 if (!ne) {
1690                         write_unlock(&nm_i->nat_tree_lock);
1691                         goto retry;
1692                 }
1693                 nat_set_blkaddr(ne, le32_to_cpu(raw_ne.block_addr));
1694                 nat_set_ino(ne, le32_to_cpu(raw_ne.ino));
1695                 nat_set_version(ne, raw_ne.version);
1696                 __set_nat_cache_dirty(nm_i, ne);
1697                 write_unlock(&nm_i->nat_tree_lock);
1698         }
1699         update_nats_in_cursum(sum, -i);
1700         mutex_unlock(&curseg->curseg_mutex);
1701         return true;
1702 }
1703
1704 /*
1705  * This function is called during the checkpointing process.
1706  */
1707 void flush_nat_entries(struct f2fs_sb_info *sbi)
1708 {
1709         struct f2fs_nm_info *nm_i = NM_I(sbi);
1710         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
1711         struct f2fs_summary_block *sum = curseg->sum_blk;
1712         struct list_head *cur, *n;
1713         struct page *page = NULL;
1714         struct f2fs_nat_block *nat_blk = NULL;
1715         nid_t start_nid = 0, end_nid = 0;
1716         bool flushed;
1717
1718         flushed = flush_nats_in_journal(sbi);
1719
1720         if (!flushed)
1721                 mutex_lock(&curseg->curseg_mutex);
1722
1723         /* 1) flush dirty nat caches */
1724         list_for_each_safe(cur, n, &nm_i->dirty_nat_entries) {
1725                 struct nat_entry *ne;
1726                 nid_t nid;
1727                 struct f2fs_nat_entry raw_ne;
1728                 int offset = -1;
1729                 block_t new_blkaddr;
1730
1731                 ne = list_entry(cur, struct nat_entry, list);
1732                 nid = nat_get_nid(ne);
1733
1734                 if (nat_get_blkaddr(ne) == NEW_ADDR)
1735                         continue;
1736                 if (flushed)
1737                         goto to_nat_page;
1738
1739                 /* if there is room for nat enries in curseg->sumpage */
1740                 offset = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 1);
1741                 if (offset >= 0) {
1742                         raw_ne = nat_in_journal(sum, offset);
1743                         goto flush_now;
1744                 }
1745 to_nat_page:
1746                 if (!page || (start_nid > nid || nid > end_nid)) {
1747                         if (page) {
1748                                 f2fs_put_page(page, 1);
1749                                 page = NULL;
1750                         }
1751                         start_nid = START_NID(nid);
1752                         end_nid = start_nid + NAT_ENTRY_PER_BLOCK - 1;
1753
1754                         /*
1755                          * get nat block with dirty flag, increased reference
1756                          * count, mapped and lock
1757                          */
1758                         page = get_next_nat_page(sbi, start_nid);
1759                         nat_blk = page_address(page);
1760                 }
1761
1762                 f2fs_bug_on(!nat_blk);
1763                 raw_ne = nat_blk->entries[nid - start_nid];
1764 flush_now:
1765                 new_blkaddr = nat_get_blkaddr(ne);
1766
1767                 raw_ne.ino = cpu_to_le32(nat_get_ino(ne));
1768                 raw_ne.block_addr = cpu_to_le32(new_blkaddr);
1769                 raw_ne.version = nat_get_version(ne);
1770
1771                 if (offset < 0) {
1772                         nat_blk->entries[nid - start_nid] = raw_ne;
1773                 } else {
1774                         nat_in_journal(sum, offset) = raw_ne;
1775                         nid_in_journal(sum, offset) = cpu_to_le32(nid);
1776                 }
1777
1778                 if (nat_get_blkaddr(ne) == NULL_ADDR &&
1779                                 add_free_nid(NM_I(sbi), nid, false) <= 0) {
1780                         write_lock(&nm_i->nat_tree_lock);
1781                         __del_from_nat_cache(nm_i, ne);
1782                         write_unlock(&nm_i->nat_tree_lock);
1783                 } else {
1784                         write_lock(&nm_i->nat_tree_lock);
1785                         __clear_nat_cache_dirty(nm_i, ne);
1786                         write_unlock(&nm_i->nat_tree_lock);
1787                 }
1788         }
1789         if (!flushed)
1790                 mutex_unlock(&curseg->curseg_mutex);
1791         f2fs_put_page(page, 1);
1792
1793         /* 2) shrink nat caches if necessary */
1794         try_to_free_nats(sbi, nm_i->nat_cnt - NM_WOUT_THRESHOLD);
1795 }
1796
1797 static int init_node_manager(struct f2fs_sb_info *sbi)
1798 {
1799         struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi);
1800         struct f2fs_nm_info *nm_i = NM_I(sbi);
1801         unsigned char *version_bitmap;
1802         unsigned int nat_segs, nat_blocks;
1803
1804         nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);
1805
1806         /* segment_count_nat includes pair segment so divide to 2. */
1807         nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
1808         nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
1809
1810         /* not used nids: 0, node, meta, (and root counted as valid node) */
1811         nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks - 3;
1812         nm_i->fcnt = 0;
1813         nm_i->nat_cnt = 0;
1814
1815         INIT_LIST_HEAD(&nm_i->free_nid_list);
1816         INIT_RADIX_TREE(&nm_i->nat_root, GFP_ATOMIC);
1817         INIT_LIST_HEAD(&nm_i->nat_entries);
1818         INIT_LIST_HEAD(&nm_i->dirty_nat_entries);
1819
1820         mutex_init(&nm_i->build_lock);
1821         spin_lock_init(&nm_i->free_nid_list_lock);
1822         rwlock_init(&nm_i->nat_tree_lock);
1823
1824         nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
1825         nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
1826         version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
1827         if (!version_bitmap)
1828                 return -EFAULT;
1829
1830         nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
1831                                         GFP_KERNEL);
1832         if (!nm_i->nat_bitmap)
1833                 return -ENOMEM;
1834         return 0;
1835 }
1836
1837 int build_node_manager(struct f2fs_sb_info *sbi)
1838 {
1839         int err;
1840
1841         sbi->nm_info = kzalloc(sizeof(struct f2fs_nm_info), GFP_KERNEL);
1842         if (!sbi->nm_info)
1843                 return -ENOMEM;
1844
1845         err = init_node_manager(sbi);
1846         if (err)
1847                 return err;
1848
1849         build_free_nids(sbi);
1850         return 0;
1851 }
1852
1853 void destroy_node_manager(struct f2fs_sb_info *sbi)
1854 {
1855         struct f2fs_nm_info *nm_i = NM_I(sbi);
1856         struct free_nid *i, *next_i;
1857         struct nat_entry *natvec[NATVEC_SIZE];
1858         nid_t nid = 0;
1859         unsigned int found;
1860
1861         if (!nm_i)
1862                 return;
1863
1864         /* destroy free nid list */
1865         spin_lock(&nm_i->free_nid_list_lock);
1866         list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
1867                 f2fs_bug_on(i->state == NID_ALLOC);
1868                 __del_from_free_nid_list(i);
1869                 nm_i->fcnt--;
1870         }
1871         f2fs_bug_on(nm_i->fcnt);
1872         spin_unlock(&nm_i->free_nid_list_lock);
1873
1874         /* destroy nat cache */
1875         write_lock(&nm_i->nat_tree_lock);
1876         while ((found = __gang_lookup_nat_cache(nm_i,
1877                                         nid, NATVEC_SIZE, natvec))) {
1878                 unsigned idx;
1879                 for (idx = 0; idx < found; idx++) {
1880                         struct nat_entry *e = natvec[idx];
1881                         nid = nat_get_nid(e) + 1;
1882                         __del_from_nat_cache(nm_i, e);
1883                 }
1884         }
1885         f2fs_bug_on(nm_i->nat_cnt);
1886         write_unlock(&nm_i->nat_tree_lock);
1887
1888         kfree(nm_i->nat_bitmap);
1889         sbi->nm_info = NULL;
1890         kfree(nm_i);
1891 }
1892
1893 int __init create_node_manager_caches(void)
1894 {
1895         nat_entry_slab = f2fs_kmem_cache_create("nat_entry",
1896                         sizeof(struct nat_entry), NULL);
1897         if (!nat_entry_slab)
1898                 return -ENOMEM;
1899
1900         free_nid_slab = f2fs_kmem_cache_create("free_nid",
1901                         sizeof(struct free_nid), NULL);
1902         if (!free_nid_slab) {
1903                 kmem_cache_destroy(nat_entry_slab);
1904                 return -ENOMEM;
1905         }
1906         return 0;
1907 }
1908
1909 void destroy_node_manager_caches(void)
1910 {
1911         kmem_cache_destroy(free_nid_slab);
1912         kmem_cache_destroy(nat_entry_slab);
1913 }