fs/f2fs/data.c

   1 /*
   2  * fs/f2fs/data.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/buffer_head.h>
  14 #include <linux/mpage.h>
  15 #include <linux/writeback.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/pagevec.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/bio.h>
  20 #include <linux/prefetch.h>
  21 #include <linux/uio.h>
  22 #include <linux/cleancache.h>
  23
  24 #include "f2fs.h"
  25 #include "node.h"
  26 #include "segment.h"
  27 #include "trace.h"
  28 #include <trace/events/f2fs.h>
  29
  30 static void f2fs_read_end_io(struct bio *bio)
  31 {
  32         struct bio_vec *bvec;
  33         int i;
  34
  35         if (f2fs_bio_encrypted(bio)) {
  36                 if (bio->bi_error) {
  37                         f2fs_release_crypto_ctx(bio->bi_private);
  38                 } else {
  39                         f2fs_end_io_crypto_work(bio->bi_private, bio);
  40                         return;
  41                 }
  42         }
  43
  44         bio_for_each_segment_all(bvec, bio, i) {
  45                 struct page *page = bvec->bv_page;
  46
  47                 if (!bio->bi_error) {
  48                         SetPageUptodate(page);
  49                 } else {
  50                         ClearPageUptodate(page);
  51                         SetPageError(page);
  52                 }
  53                 unlock_page(page);
  54         }
  55         bio_put(bio);
  56 }
  57
  58 static void f2fs_write_end_io(struct bio *bio)
  59 {
  60         struct f2fs_sb_info *sbi = bio->bi_private;
  61         struct bio_vec *bvec;
  62         int i;
  63
  64         bio_for_each_segment_all(bvec, bio, i) {
  65                 struct page *page = bvec->bv_page;
  66
  67                 f2fs_restore_and_release_control_page(&page);
  68
  69                 if (unlikely(bio->bi_error)) {
  70                         set_bit(AS_EIO, &page->mapping->flags);
  71                         f2fs_stop_checkpoint(sbi);
  72                 }
  73                 end_page_writeback(page);
  74                 dec_page_count(sbi, F2FS_WRITEBACK);
  75         }
  76
  77         if (!get_pages(sbi, F2FS_WRITEBACK) &&
  78                         !list_empty(&sbi->cp_wait.task_list))
  79                 wake_up(&sbi->cp_wait);
  80
  81         bio_put(bio);
  82 }
  83
  84 /*
  85  * Low-level block read/write IO operations.
  86  */
  87 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
  88                                 int npages, bool is_read)
  89 {
  90         struct bio *bio;
  91
  92         bio = f2fs_bio_alloc(npages);
  93
  94         bio->bi_bdev = sbi->sb->s_bdev;
  95         bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
  96         bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
  97         bio->bi_private = is_read ? NULL : sbi;
  98
  99         return bio;
 100 }
 101
 102 static void __submit_merged_bio(struct f2fs_bio_info *io)
 103 {
 104         struct f2fs_io_info *fio = &io->fio;
 105
 106         if (!io->bio)
 107                 return;
 108
 109         if (is_read_io(fio->rw))
 110                 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);
 111         else
 112                 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);
 113
 114         submit_bio(fio->rw, io->bio);
 115         io->bio = NULL;
 116 }
 117
 118 bool is_merged_page(struct f2fs_sb_info *sbi, struct page *page,
 119                                                         enum page_type type)
 120 {
 121         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 122         struct f2fs_bio_info *io = &sbi->write_io[btype];
 123         struct bio_vec *bvec;
 124         struct page *target;
 125         int i;
 126
 127         down_read(&io->io_rwsem);
 128         if (!io->bio) {
 129                 up_read(&io->io_rwsem);
 130                 return false;
 131         }
 132
 133         bio_for_each_segment_all(bvec, io->bio, i) {
 134
 135                 if (bvec->bv_page->mapping) {
 136                         target = bvec->bv_page;
 137                 } else {
 138                         struct f2fs_crypto_ctx *ctx;
 139
 140                         /* encrypted page */
 141                         ctx = (struct f2fs_crypto_ctx *)page_private(
 142                                                                 bvec->bv_page);
 143                         target = ctx->w.control_page;
 144                 }
 145
 146                 if (page == target) {
 147                         up_read(&io->io_rwsem);
 148                         return true;
 149                 }
 150         }
 151
 152         up_read(&io->io_rwsem);
 153         return false;
 154 }
 155
 156 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
 157                                 enum page_type type, int rw)
 158 {
 159         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 160         struct f2fs_bio_info *io;
 161
 162         io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];
 163
 164         down_write(&io->io_rwsem);
 165
 166         /* change META to META_FLUSH in the checkpoint procedure */
 167         if (type >= META_FLUSH) {
 168                 io->fio.type = META_FLUSH;
 169                 if (test_opt(sbi, NOBARRIER))
 170                         io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
 171                 else
 172                         io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
 173         }
 174         __submit_merged_bio(io);
 175         up_write(&io->io_rwsem);
 176 }
 177
 178 /*
 179  * Fill the locked page with data located in the block address.
 180  * Return unlocked page.
 181  */
 182 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
 183 {
 184         struct bio *bio;
 185         struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page;
 186
 187         trace_f2fs_submit_page_bio(page, fio);
 188         f2fs_trace_ios(fio, 0);
 189
 190         /* Allocate a new bio */
 191         bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw));
 192
 193         if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
 194                 bio_put(bio);
 195                 return -EFAULT;
 196         }
 197
 198         submit_bio(fio->rw, bio);
 199         return 0;
 200 }
 201
 202 void f2fs_submit_page_mbio(struct f2fs_io_info *fio)
 203 {
 204         struct f2fs_sb_info *sbi = fio->sbi;
 205         enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
 206         struct f2fs_bio_info *io;
 207         bool is_read = is_read_io(fio->rw);
 208         struct page *bio_page;
 209
 210         io = is_read ? &sbi->read_io : &sbi->write_io[btype];
 211
 212         verify_block_addr(sbi, fio->blk_addr);
 213
 214         down_write(&io->io_rwsem);
 215
 216         if (!is_read)
 217                 inc_page_count(sbi, F2FS_WRITEBACK);
 218
 219         if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 ||
 220                                                 io->fio.rw != fio->rw))
 221                 __submit_merged_bio(io);
 222 alloc_new:
 223         if (io->bio == NULL) {
 224                 int bio_blocks = MAX_BIO_BLOCKS(sbi);
 225
 226                 io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read);
 227                 io->fio = *fio;
 228         }
 229
 230         bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
 231
 232         if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) <
 233                                                         PAGE_CACHE_SIZE) {
 234                 __submit_merged_bio(io);
 235                 goto alloc_new;
 236         }
 237
 238         io->last_block_in_bio = fio->blk_addr;
 239         f2fs_trace_ios(fio, 0);
 240
 241         up_write(&io->io_rwsem);
 242         trace_f2fs_submit_page_mbio(fio->page, fio);
 243 }
 244
 245 /*
 246  * Lock ordering for the change of data block address:
 247  * ->data_page
 248  *  ->node_page
 249  *    update block addresses in the node page
 250  */
 251 void set_data_blkaddr(struct dnode_of_data *dn)
 252 {
 253         struct f2fs_node *rn;
 254         __le32 *addr_array;
 255         struct page *node_page = dn->node_page;
 256         unsigned int ofs_in_node = dn->ofs_in_node;
 257
 258         f2fs_wait_on_page_writeback(node_page, NODE, true);
 259
 260         rn = F2FS_NODE(node_page);
 261
 262         /* Get physical address of data block */
 263         addr_array = blkaddr_in_node(rn);
 264         addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
 265         if (set_page_dirty(node_page))
 266                 dn->node_changed = true;
 267 }
 268
 269 int reserve_new_block(struct dnode_of_data *dn)
 270 {
 271         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 272
 273         if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
 274                 return -EPERM;
 275         if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
 276                 return -ENOSPC;
 277
 278         trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
 279
 280         dn->data_blkaddr = NEW_ADDR;
 281         set_data_blkaddr(dn);
 282         mark_inode_dirty(dn->inode);
 283         sync_inode_page(dn);
 284         return 0;
 285 }
 286
 287 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
 288 {
 289         bool need_put = dn->inode_page ? false : true;
 290         int err;
 291
 292         err = get_dnode_of_data(dn, index, ALLOC_NODE);
 293         if (err)
 294                 return err;
 295
 296         if (dn->data_blkaddr == NULL_ADDR)
 297                 err = reserve_new_block(dn);
 298         if (err || need_put)
 299                 f2fs_put_dnode(dn);
 300         return err;
 301 }
 302
 303 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
 304 {
 305         struct extent_info ei;
 306         struct inode *inode = dn->inode;
 307
 308         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 309                 dn->data_blkaddr = ei.blk + index - ei.fofs;
 310                 return 0;
 311         }
 312
 313         return f2fs_reserve_block(dn, index);
 314 }
 315
 316 struct page *get_read_data_page(struct inode *inode, pgoff_t index,
 317                                                 int rw, bool for_write)
 318 {
 319         struct address_space *mapping = inode->i_mapping;
 320         struct dnode_of_data dn;
 321         struct page *page;
 322         struct extent_info ei;
 323         int err;
 324         struct f2fs_io_info fio = {
 325                 .sbi = F2FS_I_SB(inode),
 326                 .type = DATA,
 327                 .rw = rw,
 328                 .encrypted_page = NULL,
 329         };
 330
 331         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
 332                 return read_mapping_page(mapping, index, NULL);
 333
 334         page = f2fs_grab_cache_page(mapping, index, for_write);
 335         if (!page)
 336                 return ERR_PTR(-ENOMEM);
 337
 338         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 339                 dn.data_blkaddr = ei.blk + index - ei.fofs;
 340                 goto got_it;
 341         }
 342
 343         set_new_dnode(&dn, inode, NULL, NULL, 0);
 344         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
 345         if (err)
 346                 goto put_err;
 347         f2fs_put_dnode(&dn);
 348
 349         if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
 350                 err = -ENOENT;
 351                 goto put_err;
 352         }
 353 got_it:
 354         if (PageUptodate(page)) {
 355                 unlock_page(page);
 356                 return page;
 357         }
 358
 359         /*
 360          * A new dentry page is allocated but not able to be written, since its
 361          * new inode page couldn't be allocated due to -ENOSPC.
 362          * In such the case, its blkaddr can be remained as NEW_ADDR.
 363          * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
 364          */
 365         if (dn.data_blkaddr == NEW_ADDR) {
 366                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
 367                 SetPageUptodate(page);
 368                 unlock_page(page);
 369                 return page;
 370         }
 371
 372         fio.blk_addr = dn.data_blkaddr;
 373         fio.page = page;
 374         err = f2fs_submit_page_bio(&fio);
 375         if (err)
 376                 goto put_err;
 377         return page;
 378
 379 put_err:
 380         f2fs_put_page(page, 1);
 381         return ERR_PTR(err);
 382 }
 383
 384 struct page *find_data_page(struct inode *inode, pgoff_t index)
 385 {
 386         struct address_space *mapping = inode->i_mapping;
 387         struct page *page;
 388
 389         page = find_get_page(mapping, index);
 390         if (page && PageUptodate(page))
 391                 return page;
 392         f2fs_put_page(page, 0);
 393
 394         page = get_read_data_page(inode, index, READ_SYNC, false);
 395         if (IS_ERR(page))
 396                 return page;
 397
 398         if (PageUptodate(page))
 399                 return page;
 400
 401         wait_on_page_locked(page);
 402         if (unlikely(!PageUptodate(page))) {
 403                 f2fs_put_page(page, 0);
 404                 return ERR_PTR(-EIO);
 405         }
 406         return page;
 407 }
 408
 409 /*
 410  * If it tries to access a hole, return an error.
 411  * Because, the callers, functions in dir.c and GC, should be able to know
 412  * whether this page exists or not.
 413  */
 414 struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
 415                                                         bool for_write)
 416 {
 417         struct address_space *mapping = inode->i_mapping;
 418         struct page *page;
 419 repeat:
 420         page = get_read_data_page(inode, index, READ_SYNC, for_write);
 421         if (IS_ERR(page))
 422                 return page;
 423
 424         /* wait for read completion */
 425         lock_page(page);
 426         if (unlikely(!PageUptodate(page))) {
 427                 f2fs_put_page(page, 1);
 428                 return ERR_PTR(-EIO);
 429         }
 430         if (unlikely(page->mapping != mapping)) {
 431                 f2fs_put_page(page, 1);
 432                 goto repeat;
 433         }
 434         return page;
 435 }
 436
 437 /*
 438  * Caller ensures that this data page is never allocated.
 439  * A new zero-filled data page is allocated in the page cache.
 440  *
 441  * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 442  * f2fs_unlock_op().
 443  * Note that, ipage is set only by make_empty_dir, and if any error occur,
 444  * ipage should be released by this function.
 445  */
 446 struct page *get_new_data_page(struct inode *inode,
 447                 struct page *ipage, pgoff_t index, bool new_i_size)
 448 {
 449         struct address_space *mapping = inode->i_mapping;
 450         struct page *page;
 451         struct dnode_of_data dn;
 452         int err;
 453
 454         page = f2fs_grab_cache_page(mapping, index, true);
 455         if (!page) {
 456                 /*
 457                  * before exiting, we should make sure ipage will be released
 458                  * if any error occur.
 459                  */
 460                 f2fs_put_page(ipage, 1);
 461                 return ERR_PTR(-ENOMEM);
 462         }
 463
 464         set_new_dnode(&dn, inode, ipage, NULL, 0);
 465         err = f2fs_reserve_block(&dn, index);
 466         if (err) {
 467                 f2fs_put_page(page, 1);
 468                 return ERR_PTR(err);
 469         }
 470         if (!ipage)
 471                 f2fs_put_dnode(&dn);
 472
 473         if (PageUptodate(page))
 474                 goto got_it;
 475
 476         if (dn.data_blkaddr == NEW_ADDR) {
 477                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
 478                 SetPageUptodate(page);
 479         } else {
 480                 f2fs_put_page(page, 1);
 481
 482                 /* if ipage exists, blkaddr should be NEW_ADDR */
 483                 f2fs_bug_on(F2FS_I_SB(inode), ipage);
 484                 page = get_lock_data_page(inode, index, true);
 485                 if (IS_ERR(page))
 486                         return page;
 487         }
 488 got_it:
 489         if (new_i_size && i_size_read(inode) <
 490                                 ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) {
 491                 i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT));
 492                 /* Only the directory inode sets new_i_size */
 493                 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
 494         }
 495         return page;
 496 }
 497
 498 static int __allocate_data_block(struct dnode_of_data *dn)
 499 {
 500         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 501         struct f2fs_inode_info *fi = F2FS_I(dn->inode);
 502         struct f2fs_summary sum;
 503         struct node_info ni;
 504         int seg = CURSEG_WARM_DATA;
 505         pgoff_t fofs;
 506
 507         if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
 508                 return -EPERM;
 509
 510         dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
 511         if (dn->data_blkaddr == NEW_ADDR)
 512                 goto alloc;
 513
 514         if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
 515                 return -ENOSPC;
 516
 517 alloc:
 518         get_node_info(sbi, dn->nid, &ni);
 519         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 520
 521         if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page)
 522                 seg = CURSEG_DIRECT_IO;
 523
 524         allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
 525                                                                 &sum, seg);
 526         set_data_blkaddr(dn);
 527
 528         /* update i_size */
 529         fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
 530                                                         dn->ofs_in_node;
 531         if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT))
 532                 i_size_write(dn->inode,
 533                                 ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT));
 534         return 0;
 535 }
 536
 537 static int __allocate_data_blocks(struct inode *inode, loff_t offset,
 538                                                         size_t count)
 539 {
 540         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 541         struct dnode_of_data dn;
 542         u64 start = F2FS_BYTES_TO_BLK(offset);
 543         u64 len = F2FS_BYTES_TO_BLK(count);
 544         bool allocated = false;
 545         u64 end_offset;
 546         int err = 0;
 547
 548         while (len) {
 549                 f2fs_lock_op(sbi);
 550
 551                 /* When reading holes, we need its node page */
 552                 set_new_dnode(&dn, inode, NULL, NULL, 0);
 553                 err = get_dnode_of_data(&dn, start, ALLOC_NODE);
 554                 if (err)
 555                         goto out;
 556
 557                 allocated = false;
 558                 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
 559
 560                 while (dn.ofs_in_node < end_offset && len) {
 561                         block_t blkaddr;
 562
 563                         if (unlikely(f2fs_cp_error(sbi))) {
 564                                 err = -EIO;
 565                                 goto sync_out;
 566                         }
 567
 568                         blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
 569                         if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
 570                                 err = __allocate_data_block(&dn);
 571                                 if (err)
 572                                         goto sync_out;
 573                                 allocated = true;
 574                         }
 575                         len--;
 576                         start++;
 577                         dn.ofs_in_node++;
 578                 }
 579
 580                 if (allocated)
 581                         sync_inode_page(&dn);
 582
 583                 f2fs_put_dnode(&dn);
 584                 f2fs_unlock_op(sbi);
 585
 586                 f2fs_balance_fs(sbi, allocated);
 587         }
 588         return err;
 589
 590 sync_out:
 591         if (allocated)
 592                 sync_inode_page(&dn);
 593         f2fs_put_dnode(&dn);
 594 out:
 595         f2fs_unlock_op(sbi);
 596         f2fs_balance_fs(sbi, allocated);
 597         return err;
 598 }
 599
 600 /*
 601  * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
 602  * f2fs_map_blocks structure.
 603  * If original data blocks are allocated, then give them to blockdev.
 604  * Otherwise,
 605  *     a. preallocate requested block addresses
 606  *     b. do not use extent cache for better performance
 607  *     c. give the block addresses to blockdev
 608  */
 609 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
 610                                                 int create, int flag)
 611 {
 612         unsigned int maxblocks = map->m_len;
 613         struct dnode_of_data dn;
 614         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 615         int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
 616         pgoff_t pgofs, end_offset;
 617         int err = 0, ofs = 1;
 618         struct extent_info ei;
 619         bool allocated = false;
 620         block_t blkaddr;
 621
 622         map->m_len = 0;
 623         map->m_flags = 0;
 624
 625         /* it only supports block size == page size */
 626         pgofs = (pgoff_t)map->m_lblk;
 627
 628         if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
 629                 map->m_pblk = ei.blk + pgofs - ei.fofs;
 630                 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
 631                 map->m_flags = F2FS_MAP_MAPPED;
 632                 goto out;
 633         }
 634
 635         if (create)
 636                 f2fs_lock_op(sbi);
 637
 638         /* When reading holes, we need its node page */
 639         set_new_dnode(&dn, inode, NULL, NULL, 0);
 640         err = get_dnode_of_data(&dn, pgofs, mode);
 641         if (err) {
 642                 if (err == -ENOENT)
 643                         err = 0;
 644                 goto unlock_out;
 645         }
 646
 647         if (dn.data_blkaddr == NEW_ADDR || dn.data_blkaddr == NULL_ADDR) {
 648                 if (create) {
 649                         if (unlikely(f2fs_cp_error(sbi))) {
 650                                 err = -EIO;
 651                                 goto put_out;
 652                         }
 653                         err = __allocate_data_block(&dn);
 654                         if (err)
 655                                 goto put_out;
 656                         allocated = true;
 657                         map->m_flags = F2FS_MAP_NEW;
 658                 } else {
 659                         if (flag != F2FS_GET_BLOCK_FIEMAP ||
 660                                                 dn.data_blkaddr != NEW_ADDR) {
 661                                 if (flag == F2FS_GET_BLOCK_BMAP)
 662                                         err = -ENOENT;
 663                                 goto put_out;
 664                         }
 665
 666                         /*
 667                          * preallocated unwritten block should be mapped
 668                          * for fiemap.
 669                          */
 670                         if (dn.data_blkaddr == NEW_ADDR)
 671                                 map->m_flags = F2FS_MAP_UNWRITTEN;
 672                 }
 673         }
 674
 675         map->m_flags |= F2FS_MAP_MAPPED;
 676         map->m_pblk = dn.data_blkaddr;
 677         map->m_len = 1;
 678
 679         end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
 680         dn.ofs_in_node++;
 681         pgofs++;
 682
 683 get_next:
 684         if (map->m_len >= maxblocks)
 685                 goto sync_out;
 686
 687         if (dn.ofs_in_node >= end_offset) {
 688                 if (allocated)
 689                         sync_inode_page(&dn);
 690                 f2fs_put_dnode(&dn);
 691
 692                 if (create) {
 693                         f2fs_unlock_op(sbi);
 694                         f2fs_balance_fs(sbi, allocated);
 695                         f2fs_lock_op(sbi);
 696                 }
 697                 allocated = false;
 698
 699                 set_new_dnode(&dn, inode, NULL, NULL, 0);
 700                 err = get_dnode_of_data(&dn, pgofs, mode);
 701                 if (err) {
 702                         if (err == -ENOENT)
 703                                 err = 0;
 704                         goto unlock_out;
 705                 }
 706
 707                 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
 708         }
 709
 710         blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
 711
 712         if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
 713                 if (create) {
 714                         if (unlikely(f2fs_cp_error(sbi))) {
 715                                 err = -EIO;
 716                                 goto sync_out;
 717                         }
 718                         err = __allocate_data_block(&dn);
 719                         if (err)
 720                                 goto sync_out;
 721                         allocated = true;
 722                         map->m_flags |= F2FS_MAP_NEW;
 723                         blkaddr = dn.data_blkaddr;
 724                 } else {
 725                         /*
 726                          * we only merge preallocated unwritten blocks
 727                          * for fiemap.
 728                          */
 729                         if (flag != F2FS_GET_BLOCK_FIEMAP ||
 730                                         blkaddr != NEW_ADDR)
 731                                 goto sync_out;
 732                 }
 733         }
 734
 735         /* Give more consecutive addresses for the readahead */
 736         if ((map->m_pblk != NEW_ADDR &&
 737                         blkaddr == (map->m_pblk + ofs)) ||
 738                         (map->m_pblk == NEW_ADDR &&
 739                         blkaddr == NEW_ADDR)) {
 740                 ofs++;
 741                 dn.ofs_in_node++;
 742                 pgofs++;
 743                 map->m_len++;
 744                 goto get_next;
 745         }
 746
 747 sync_out:
 748         if (allocated)
 749                 sync_inode_page(&dn);
 750 put_out:
 751         f2fs_put_dnode(&dn);
 752 unlock_out:
 753         if (create) {
 754                 f2fs_unlock_op(sbi);
 755                 f2fs_balance_fs(sbi, allocated);
 756         }
 757 out:
 758         trace_f2fs_map_blocks(inode, map, err);
 759         return err;
 760 }
 761
 762 static int __get_data_block(struct inode *inode, sector_t iblock,
 763                         struct buffer_head *bh, int create, int flag)
 764 {
 765         struct f2fs_map_blocks map;
 766         int ret;
 767
 768         map.m_lblk = iblock;
 769         map.m_len = bh->b_size >> inode->i_blkbits;
 770
 771         ret = f2fs_map_blocks(inode, &map, create, flag);
 772         if (!ret) {
 773                 map_bh(bh, inode->i_sb, map.m_pblk);
 774                 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
 775                 bh->b_size = map.m_len << inode->i_blkbits;
 776         }
 777         return ret;
 778 }
 779
 780 static int get_data_block(struct inode *inode, sector_t iblock,
 781                         struct buffer_head *bh_result, int create, int flag)
 782 {
 783         return __get_data_block(inode, iblock, bh_result, create, flag);
 784 }
 785
 786 static int get_data_block_dio(struct inode *inode, sector_t iblock,
 787                         struct buffer_head *bh_result, int create)
 788 {
 789         return __get_data_block(inode, iblock, bh_result, create,
 790                                                 F2FS_GET_BLOCK_DIO);
 791 }
 792
 793 static int get_data_block_bmap(struct inode *inode, sector_t iblock,
 794                         struct buffer_head *bh_result, int create)
 795 {
 796         /* Block number less than F2FS MAX BLOCKS */
 797         if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
 798                 return -EFBIG;
 799
 800         return __get_data_block(inode, iblock, bh_result, create,
 801                                                 F2FS_GET_BLOCK_BMAP);
 802 }
 803
 804 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
 805 {
 806         return (offset >> inode->i_blkbits);
 807 }
 808
 809 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
 810 {
 811         return (blk << inode->i_blkbits);
 812 }
 813
 814 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 815                 u64 start, u64 len)
 816 {
 817         struct buffer_head map_bh;
 818         sector_t start_blk, last_blk;
 819         loff_t isize;
 820         u64 logical = 0, phys = 0, size = 0;
 821         u32 flags = 0;
 822         int ret = 0;
 823
 824         ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
 825         if (ret)
 826                 return ret;
 827
 828         if (f2fs_has_inline_data(inode)) {
 829                 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
 830                 if (ret != -EAGAIN)
 831                         return ret;
 832         }
 833
 834         inode_lock(inode);
 835
 836         isize = i_size_read(inode);
 837         if (start >= isize)
 838                 goto out;
 839
 840         if (start + len > isize)
 841                 len = isize - start;
 842
 843         if (logical_to_blk(inode, len) == 0)
 844                 len = blk_to_logical(inode, 1);
 845
 846         start_blk = logical_to_blk(inode, start);
 847         last_blk = logical_to_blk(inode, start + len - 1);
 848
 849 next:
 850         memset(&map_bh, 0, sizeof(struct buffer_head));
 851         map_bh.b_size = len;
 852
 853         ret = get_data_block(inode, start_blk, &map_bh, 0,
 854                                         F2FS_GET_BLOCK_FIEMAP);
 855         if (ret)
 856                 goto out;
 857
 858         /* HOLE */
 859         if (!buffer_mapped(&map_bh)) {
 860                 /* Go through holes util pass the EOF */
 861                 if (blk_to_logical(inode, start_blk++) < isize)
 862                         goto prep_next;
 863                 /* Found a hole beyond isize means no more extents.
 864                  * Note that the premise is that filesystems don't
 865                  * punch holes beyond isize and keep size unchanged.
 866                  */
 867                 flags |= FIEMAP_EXTENT_LAST;
 868         }
 869
 870         if (size) {
 871                 if (f2fs_encrypted_inode(inode))
 872                         flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
 873
 874                 ret = fiemap_fill_next_extent(fieinfo, logical,
 875                                 phys, size, flags);
 876         }
 877
 878         if (start_blk > last_blk || ret)
 879                 goto out;
 880
 881         logical = blk_to_logical(inode, start_blk);
 882         phys = blk_to_logical(inode, map_bh.b_blocknr);
 883         size = map_bh.b_size;
 884         flags = 0;
 885         if (buffer_unwritten(&map_bh))
 886                 flags = FIEMAP_EXTENT_UNWRITTEN;
 887
 888         start_blk += logical_to_blk(inode, size);
 889
 890 prep_next:
 891         cond_resched();
 892         if (fatal_signal_pending(current))
 893                 ret = -EINTR;
 894         else
 895                 goto next;
 896 out:
 897         if (ret == 1)
 898                 ret = 0;
 899
 900         inode_unlock(inode);
 901         return ret;
 902 }
 903
 904 /*
 905  * This function was originally taken from fs/mpage.c, and customized for f2fs.
 906  * Major change was from block_size == page_size in f2fs by default.
 907  */
 908 static int f2fs_mpage_readpages(struct address_space *mapping,
 909                         struct list_head *pages, struct page *page,
 910                         unsigned nr_pages)
 911 {
 912         struct bio *bio = NULL;
 913         unsigned page_idx;
 914         sector_t last_block_in_bio = 0;
 915         struct inode *inode = mapping->host;
 916         const unsigned blkbits = inode->i_blkbits;
 917         const unsigned blocksize = 1 << blkbits;
 918         sector_t block_in_file;
 919         sector_t last_block;
 920         sector_t last_block_in_file;
 921         sector_t block_nr;
 922         struct block_device *bdev = inode->i_sb->s_bdev;
 923         struct f2fs_map_blocks map;
 924
 925         map.m_pblk = 0;
 926         map.m_lblk = 0;
 927         map.m_len = 0;
 928         map.m_flags = 0;
 929
 930         for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
 931
 932                 prefetchw(&page->flags);
 933                 if (pages) {
 934                         page = list_entry(pages->prev, struct page, lru);
 935                         list_del(&page->lru);
 936                         if (add_to_page_cache_lru(page, mapping,
 937                                                   page->index, GFP_KERNEL))
 938                                 goto next_page;
 939                 }
 940
 941                 block_in_file = (sector_t)page->index;
 942                 last_block = block_in_file + nr_pages;
 943                 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
 944                                                                 blkbits;
 945                 if (last_block > last_block_in_file)
 946                         last_block = last_block_in_file;
 947
 948                 /*
 949                  * Map blocks using the previous result first.
 950                  */
 951                 if ((map.m_flags & F2FS_MAP_MAPPED) &&
 952                                 block_in_file > map.m_lblk &&
 953                                 block_in_file < (map.m_lblk + map.m_len))
 954                         goto got_it;
 955
 956                 /*
 957                  * Then do more f2fs_map_blocks() calls until we are
 958                  * done with this page.
 959                  */
 960                 map.m_flags = 0;
 961
 962                 if (block_in_file < last_block) {
 963                         map.m_lblk = block_in_file;
 964                         map.m_len = last_block - block_in_file;
 965
 966                         if (f2fs_map_blocks(inode, &map, 0,
 967                                                         F2FS_GET_BLOCK_READ))
 968                                 goto set_error_page;
 969                 }
 970 got_it:
 971                 if ((map.m_flags & F2FS_MAP_MAPPED)) {
 972                         block_nr = map.m_pblk + block_in_file - map.m_lblk;
 973                         SetPageMappedToDisk(page);
 974
 975                         if (!PageUptodate(page) && !cleancache_get_page(page)) {
 976                                 SetPageUptodate(page);
 977                                 goto confused;
 978                         }
 979                 } else {
 980                         zero_user_segment(page, 0, PAGE_CACHE_SIZE);
 981                         SetPageUptodate(page);
 982                         unlock_page(page);
 983                         goto next_page;
 984                 }
 985
 986                 /*
 987                  * This page will go to BIO.  Do we need to send this
 988                  * BIO off first?
 989                  */
 990                 if (bio && (last_block_in_bio != block_nr - 1)) {
 991 submit_and_realloc:
 992                         submit_bio(READ, bio);
 993                         bio = NULL;
 994                 }
 995                 if (bio == NULL) {
 996                         struct f2fs_crypto_ctx *ctx = NULL;
 997
 998                         if (f2fs_encrypted_inode(inode) &&
 999                                         S_ISREG(inode->i_mode)) {
1000
1001                                 ctx = f2fs_get_crypto_ctx(inode);
1002                                 if (IS_ERR(ctx))
1003                                         goto set_error_page;
1004
1005                                 /* wait the page to be moved by cleaning */
1006                                 f2fs_wait_on_encrypted_page_writeback(
1007                                                 F2FS_I_SB(inode), block_nr);
1008                         }
1009
1010                         bio = bio_alloc(GFP_KERNEL,
1011                                 min_t(int, nr_pages, BIO_MAX_PAGES));
1012                         if (!bio) {
1013                                 if (ctx)
1014                                         f2fs_release_crypto_ctx(ctx);
1015                                 goto set_error_page;
1016                         }
1017                         bio->bi_bdev = bdev;
1018                         bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr);
1019                         bio->bi_end_io = f2fs_read_end_io;
1020                         bio->bi_private = ctx;
1021                 }
1022
1023                 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1024                         goto submit_and_realloc;
1025
1026                 last_block_in_bio = block_nr;
1027                 goto next_page;
1028 set_error_page:
1029                 SetPageError(page);
1030                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
1031                 unlock_page(page);
1032                 goto next_page;
1033 confused:
1034                 if (bio) {
1035                         submit_bio(READ, bio);
1036                         bio = NULL;
1037                 }
1038                 unlock_page(page);
1039 next_page:
1040                 if (pages)
1041                         page_cache_release(page);
1042         }
1043         BUG_ON(pages && !list_empty(pages));
1044         if (bio)
1045                 submit_bio(READ, bio);
1046         return 0;
1047 }
1048
1049 static int f2fs_read_data_page(struct file *file, struct page *page)
1050 {
1051         struct inode *inode = page->mapping->host;
1052         int ret = -EAGAIN;
1053
1054         trace_f2fs_readpage(page, DATA);
1055
1056         /* If the file has inline data, try to read it directly */
1057         if (f2fs_has_inline_data(inode))
1058                 ret = f2fs_read_inline_data(inode, page);
1059         if (ret == -EAGAIN)
1060                 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1061         return ret;
1062 }
1063
1064 static int f2fs_read_data_pages(struct file *file,
1065                         struct address_space *mapping,
1066                         struct list_head *pages, unsigned nr_pages)
1067 {
1068         struct inode *inode = file->f_mapping->host;
1069         struct page *page = list_entry(pages->prev, struct page, lru);
1070
1071         trace_f2fs_readpages(inode, page, nr_pages);
1072
1073         /* If the file has inline data, skip readpages */
1074         if (f2fs_has_inline_data(inode))
1075                 return 0;
1076
1077         return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1078 }
1079
1080 int do_write_data_page(struct f2fs_io_info *fio)
1081 {
1082         struct page *page = fio->page;
1083         struct inode *inode = page->mapping->host;
1084         struct dnode_of_data dn;
1085         int err = 0;
1086
1087         set_new_dnode(&dn, inode, NULL, NULL, 0);
1088         err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1089         if (err)
1090                 return err;
1091
1092         fio->blk_addr = dn.data_blkaddr;
1093
1094         /* This page is already truncated */
1095         if (fio->blk_addr == NULL_ADDR) {
1096                 ClearPageUptodate(page);
1097                 goto out_writepage;
1098         }
1099
1100         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1101
1102                 /* wait for GCed encrypted page writeback */
1103                 f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode),
1104                                                         fio->blk_addr);
1105
1106                 fio->encrypted_page = f2fs_encrypt(inode, fio->page);
1107                 if (IS_ERR(fio->encrypted_page)) {
1108                         err = PTR_ERR(fio->encrypted_page);
1109                         goto out_writepage;
1110                 }
1111         }
1112
1113         set_page_writeback(page);
1114
1115         /*
1116          * If current allocation needs SSR,
1117          * it had better in-place writes for updated data.
1118          */
1119         if (unlikely(fio->blk_addr != NEW_ADDR &&
1120                         !is_cold_data(page) &&
1121                         !IS_ATOMIC_WRITTEN_PAGE(page) &&
1122                         need_inplace_update(inode))) {
1123                 rewrite_data_page(fio);
1124                 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
1125                 trace_f2fs_do_write_data_page(page, IPU);
1126         } else {
1127                 write_data_page(&dn, fio);
1128                 set_data_blkaddr(&dn);
1129                 f2fs_update_extent_cache(&dn);
1130                 trace_f2fs_do_write_data_page(page, OPU);
1131                 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
1132                 if (page->index == 0)
1133                         set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN);
1134         }
1135 out_writepage:
1136         f2fs_put_dnode(&dn);
1137         return err;
1138 }
1139
1140 static int f2fs_write_data_page(struct page *page,
1141                                         struct writeback_control *wbc)
1142 {
1143         struct inode *inode = page->mapping->host;
1144         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1145         loff_t i_size = i_size_read(inode);
1146         const pgoff_t end_index = ((unsigned long long) i_size)
1147                                                         >> PAGE_CACHE_SHIFT;
1148         unsigned offset = 0;
1149         bool need_balance_fs = false;
1150         int err = 0;
1151         struct f2fs_io_info fio = {
1152                 .sbi = sbi,
1153                 .type = DATA,
1154                 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
1155                 .page = page,
1156                 .encrypted_page = NULL,
1157         };
1158
1159         trace_f2fs_writepage(page, DATA);
1160
1161         if (page->index < end_index)
1162                 goto write;
1163
1164         /*
1165          * If the offset is out-of-range of file size,
1166          * this page does not have to be written to disk.
1167          */
1168         offset = i_size & (PAGE_CACHE_SIZE - 1);
1169         if ((page->index >= end_index + 1) || !offset)
1170                 goto out;
1171
1172         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
1173 write:
1174         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1175                 goto redirty_out;
1176         if (f2fs_is_drop_cache(inode))
1177                 goto out;
1178         if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim &&
1179                         available_free_memory(sbi, BASE_CHECK))
1180                 goto redirty_out;
1181
1182         /* Dentry blocks are controlled by checkpoint */
1183         if (S_ISDIR(inode->i_mode)) {
1184                 if (unlikely(f2fs_cp_error(sbi)))
1185                         goto redirty_out;
1186                 err = do_write_data_page(&fio);
1187                 goto done;
1188         }
1189
1190         /* we should bypass data pages to proceed the kworkder jobs */
1191         if (unlikely(f2fs_cp_error(sbi))) {
1192                 SetPageError(page);
1193                 goto out;
1194         }
1195
1196         if (!wbc->for_reclaim)
1197                 need_balance_fs = true;
1198         else if (has_not_enough_free_secs(sbi, 0))
1199                 goto redirty_out;
1200
1201         err = -EAGAIN;
1202         f2fs_lock_op(sbi);
1203         if (f2fs_has_inline_data(inode))
1204                 err = f2fs_write_inline_data(inode, page);
1205         if (err == -EAGAIN)
1206                 err = do_write_data_page(&fio);
1207         f2fs_unlock_op(sbi);
1208 done:
1209         if (err && err != -ENOENT)
1210                 goto redirty_out;
1211
1212         clear_cold_data(page);
1213 out:
1214         inode_dec_dirty_pages(inode);
1215         if (err)
1216                 ClearPageUptodate(page);
1217         unlock_page(page);
1218         f2fs_balance_fs(sbi, need_balance_fs);
1219         if (wbc->for_reclaim || unlikely(f2fs_cp_error(sbi))) {
1220                 f2fs_submit_merged_bio(sbi, DATA, WRITE);
1221                 remove_dirty_inode(inode);
1222         }
1223         return 0;
1224
1225 redirty_out:
1226         redirty_page_for_writepage(wbc, page);
1227         return AOP_WRITEPAGE_ACTIVATE;
1228 }
1229
1230 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
1231                         void *data)
1232 {
1233         struct address_space *mapping = data;
1234         int ret = mapping->a_ops->writepage(page, wbc);
1235         mapping_set_error(mapping, ret);
1236         return ret;
1237 }
1238
1239 /*
1240  * This function was copied from write_cche_pages from mm/page-writeback.c.
1241  * The major change is making write step of cold data page separately from
1242  * warm/hot data page.
1243  */
1244 static int f2fs_write_cache_pages(struct address_space *mapping,
1245                         struct writeback_control *wbc, writepage_t writepage,
1246                         void *data)
1247 {
1248         int ret = 0;
1249         int done = 0;
1250         struct pagevec pvec;
1251         int nr_pages;
1252         pgoff_t uninitialized_var(writeback_index);
1253         pgoff_t index;
1254         pgoff_t end;            /* Inclusive */
1255         pgoff_t done_index;
1256         int cycled;
1257         int range_whole = 0;
1258         int tag;
1259         int step = 0;
1260
1261         pagevec_init(&pvec, 0);
1262 next:
1263         if (wbc->range_cyclic) {
1264                 writeback_index = mapping->writeback_index; /* prev offset */
1265                 index = writeback_index;
1266                 if (index == 0)
1267                         cycled = 1;
1268                 else
1269                         cycled = 0;
1270                 end = -1;
1271         } else {
1272                 index = wbc->range_start >> PAGE_CACHE_SHIFT;
1273                 end = wbc->range_end >> PAGE_CACHE_SHIFT;
1274                 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1275                         range_whole = 1;
1276                 cycled = 1; /* ignore range_cyclic tests */
1277         }
1278         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1279                 tag = PAGECACHE_TAG_TOWRITE;
1280         else
1281                 tag = PAGECACHE_TAG_DIRTY;
1282 retry:
1283         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1284                 tag_pages_for_writeback(mapping, index, end);
1285         done_index = index;
1286         while (!done && (index <= end)) {
1287                 int i;
1288
1289                 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1290                               min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
1291                 if (nr_pages == 0)
1292                         break;
1293
1294                 for (i = 0; i < nr_pages; i++) {
1295                         struct page *page = pvec.pages[i];
1296
1297                         if (page->index > end) {
1298                                 done = 1;
1299                                 break;
1300                         }
1301
1302                         done_index = page->index;
1303
1304                         lock_page(page);
1305
1306                         if (unlikely(page->mapping != mapping)) {
1307 continue_unlock:
1308                                 unlock_page(page);
1309                                 continue;
1310                         }
1311
1312                         if (!PageDirty(page)) {
1313                                 /* someone wrote it for us */
1314                                 goto continue_unlock;
1315                         }
1316
1317                         if (step == is_cold_data(page))
1318                                 goto continue_unlock;
1319
1320                         if (PageWriteback(page)) {
1321                                 if (wbc->sync_mode != WB_SYNC_NONE)
1322                                         f2fs_wait_on_page_writeback(page,
1323                                                                 DATA, true);
1324                                 else
1325                                         goto continue_unlock;
1326                         }
1327
1328                         BUG_ON(PageWriteback(page));
1329                         if (!clear_page_dirty_for_io(page))
1330                                 goto continue_unlock;
1331
1332                         ret = (*writepage)(page, wbc, data);
1333                         if (unlikely(ret)) {
1334                                 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1335                                         unlock_page(page);
1336                                         ret = 0;
1337                                 } else {
1338                                         done_index = page->index + 1;
1339                                         done = 1;
1340                                         break;
1341                                 }
1342                         }
1343
1344                         if (--wbc->nr_to_write <= 0 &&
1345                             wbc->sync_mode == WB_SYNC_NONE) {
1346                                 done = 1;
1347                                 break;
1348                         }
1349                 }
1350                 pagevec_release(&pvec);
1351                 cond_resched();
1352         }
1353
1354         if (step < 1) {
1355                 step++;
1356                 goto next;
1357         }
1358
1359         if (!cycled && !done) {
1360                 cycled = 1;
1361                 index = 0;
1362                 end = writeback_index - 1;
1363                 goto retry;
1364         }
1365         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1366                 mapping->writeback_index = done_index;
1367
1368         return ret;
1369 }
1370
1371 static int f2fs_write_data_pages(struct address_space *mapping,
1372                             struct writeback_control *wbc)
1373 {
1374         struct inode *inode = mapping->host;
1375         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1376         bool locked = false;
1377         int ret;
1378         long diff;
1379
1380         trace_f2fs_writepages(mapping->host, wbc, DATA);
1381
1382         /* deal with chardevs and other special file */
1383         if (!mapping->a_ops->writepage)
1384                 return 0;
1385
1386         /* skip writing if there is no dirty page in this inode */
1387         if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
1388                 return 0;
1389
1390         if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
1391                         get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
1392                         available_free_memory(sbi, DIRTY_DENTS))
1393                 goto skip_write;
1394
1395         /* skip writing during file defragment */
1396         if (is_inode_flag_set(F2FS_I(inode), FI_DO_DEFRAG))
1397                 goto skip_write;
1398
1399         /* during POR, we don't need to trigger writepage at all. */
1400         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1401                 goto skip_write;
1402
1403         diff = nr_pages_to_write(sbi, DATA, wbc);
1404
1405         if (!S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_ALL) {
1406                 mutex_lock(&sbi->writepages);
1407                 locked = true;
1408         }
1409         ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
1410         f2fs_submit_merged_bio(sbi, DATA, WRITE);
1411         if (locked)
1412                 mutex_unlock(&sbi->writepages);
1413
1414         remove_dirty_inode(inode);
1415
1416         wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
1417         return ret;
1418
1419 skip_write:
1420         wbc->pages_skipped += get_dirty_pages(inode);
1421         return 0;
1422 }
1423
1424 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
1425 {
1426         struct inode *inode = mapping->host;
1427         loff_t i_size = i_size_read(inode);
1428
1429         if (to > i_size) {
1430                 truncate_pagecache(inode, i_size);
1431                 truncate_blocks(inode, i_size, true);
1432         }
1433 }
1434
1435 static int prepare_write_begin(struct f2fs_sb_info *sbi,
1436                         struct page *page, loff_t pos, unsigned len,
1437                         block_t *blk_addr, bool *node_changed)
1438 {
1439         struct inode *inode = page->mapping->host;
1440         pgoff_t index = page->index;
1441         struct dnode_of_data dn;
1442         struct page *ipage;
1443         bool locked = false;
1444         struct extent_info ei;
1445         int err = 0;
1446
1447         if (f2fs_has_inline_data(inode) ||
1448                         (pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
1449                 f2fs_lock_op(sbi);
1450                 locked = true;
1451         }
1452 restart:
1453         /* check inline_data */
1454         ipage = get_node_page(sbi, inode->i_ino);
1455         if (IS_ERR(ipage)) {
1456                 err = PTR_ERR(ipage);
1457                 goto unlock_out;
1458         }
1459
1460         set_new_dnode(&dn, inode, ipage, ipage, 0);
1461
1462         if (f2fs_has_inline_data(inode)) {
1463                 if (pos + len <= MAX_INLINE_DATA) {
1464                         read_inline_data(page, ipage);
1465                         set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
1466                         set_inline_node(ipage);
1467                 } else {
1468                         err = f2fs_convert_inline_page(&dn, page);
1469                         if (err)
1470                                 goto out;
1471                         if (dn.data_blkaddr == NULL_ADDR)
1472                                 err = f2fs_get_block(&dn, index);
1473                 }
1474         } else if (locked) {
1475                 err = f2fs_get_block(&dn, index);
1476         } else {
1477                 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1478                         dn.data_blkaddr = ei.blk + index - ei.fofs;
1479                 } else {
1480                         /* hole case */
1481                         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
1482                         if (err || (!err && dn.data_blkaddr == NULL_ADDR)) {
1483                                 f2fs_put_dnode(&dn);
1484                                 f2fs_lock_op(sbi);
1485                                 locked = true;
1486                                 goto restart;
1487                         }
1488                 }
1489         }
1490
1491         /* convert_inline_page can make node_changed */
1492         *blk_addr = dn.data_blkaddr;
1493         *node_changed = dn.node_changed;
1494 out:
1495         f2fs_put_dnode(&dn);
1496 unlock_out:
1497         if (locked)
1498                 f2fs_unlock_op(sbi);
1499         return err;
1500 }
1501
1502 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
1503                 loff_t pos, unsigned len, unsigned flags,
1504                 struct page **pagep, void **fsdata)
1505 {
1506         struct inode *inode = mapping->host;
1507         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1508         struct page *page = NULL;
1509         pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
1510         bool need_balance = false;
1511         block_t blkaddr = NULL_ADDR;
1512         int err = 0;
1513
1514         trace_f2fs_write_begin(inode, pos, len, flags);
1515
1516         /*
1517          * We should check this at this moment to avoid deadlock on inode page
1518          * and #0 page. The locking rule for inline_data conversion should be:
1519          * lock_page(page #0) -> lock_page(inode_page)
1520          */
1521         if (index != 0) {
1522                 err = f2fs_convert_inline_inode(inode);
1523                 if (err)
1524                         goto fail;
1525         }
1526 repeat:
1527         page = grab_cache_page_write_begin(mapping, index, flags);
1528         if (!page) {
1529                 err = -ENOMEM;
1530                 goto fail;
1531         }
1532
1533         *pagep = page;
1534
1535         err = prepare_write_begin(sbi, page, pos, len,
1536                                         &blkaddr, &need_balance);
1537         if (err)
1538                 goto fail;
1539
1540         if (need_balance && has_not_enough_free_secs(sbi, 0)) {
1541                 unlock_page(page);
1542                 f2fs_balance_fs(sbi, true);
1543                 lock_page(page);
1544                 if (page->mapping != mapping) {
1545                         /* The page got truncated from under us */
1546                         f2fs_put_page(page, 1);
1547                         goto repeat;
1548                 }
1549         }
1550
1551         f2fs_wait_on_page_writeback(page, DATA, false);
1552
1553         /* wait for GCed encrypted page writeback */
1554         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1555                 f2fs_wait_on_encrypted_page_writeback(sbi, blkaddr);
1556
1557         if (len == PAGE_CACHE_SIZE)
1558                 goto out_update;
1559         if (PageUptodate(page))
1560                 goto out_clear;
1561
1562         if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
1563                 unsigned start = pos & (PAGE_CACHE_SIZE - 1);
1564                 unsigned end = start + len;
1565
1566                 /* Reading beyond i_size is simple: memset to zero */
1567                 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
1568                 goto out_update;
1569         }
1570
1571         if (blkaddr == NEW_ADDR) {
1572                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
1573         } else {
1574                 struct f2fs_io_info fio = {
1575                         .sbi = sbi,
1576                         .type = DATA,
1577                         .rw = READ_SYNC,
1578                         .blk_addr = blkaddr,
1579                         .page = page,
1580                         .encrypted_page = NULL,
1581                 };
1582                 err = f2fs_submit_page_bio(&fio);
1583                 if (err)
1584                         goto fail;
1585
1586                 lock_page(page);
1587                 if (unlikely(!PageUptodate(page))) {
1588                         err = -EIO;
1589                         goto fail;
1590                 }
1591                 if (unlikely(page->mapping != mapping)) {
1592                         f2fs_put_page(page, 1);
1593                         goto repeat;
1594                 }
1595
1596                 /* avoid symlink page */
1597                 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1598                         err = f2fs_decrypt_one(inode, page);
1599                         if (err)
1600                                 goto fail;
1601                 }
1602         }
1603 out_update:
1604         SetPageUptodate(page);
1605 out_clear:
1606         clear_cold_data(page);
1607         return 0;
1608
1609 fail:
1610         f2fs_put_page(page, 1);
1611         f2fs_write_failed(mapping, pos + len);
1612         return err;
1613 }
1614
1615 static int f2fs_write_end(struct file *file,
1616                         struct address_space *mapping,
1617                         loff_t pos, unsigned len, unsigned copied,
1618                         struct page *page, void *fsdata)
1619 {
1620         struct inode *inode = page->mapping->host;
1621
1622         trace_f2fs_write_end(inode, pos, len, copied);
1623
1624         set_page_dirty(page);
1625
1626         if (pos + copied > i_size_read(inode)) {
1627                 i_size_write(inode, pos + copied);
1628                 mark_inode_dirty(inode);
1629         }
1630
1631         f2fs_put_page(page, 1);
1632         f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
1633         return copied;
1634 }
1635
1636 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
1637                            loff_t offset)
1638 {
1639         unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
1640
1641         if (offset & blocksize_mask)
1642                 return -EINVAL;
1643
1644         if (iov_iter_alignment(iter) & blocksize_mask)
1645                 return -EINVAL;
1646
1647         return 0;
1648 }
1649
1650 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
1651                               loff_t offset)
1652 {
1653         struct file *file = iocb->ki_filp;
1654         struct address_space *mapping = file->f_mapping;
1655         struct inode *inode = mapping->host;
1656         size_t count = iov_iter_count(iter);
1657         int err;
1658
1659         /* we don't need to use inline_data strictly */
1660         err = f2fs_convert_inline_inode(inode);
1661         if (err)
1662                 return err;
1663
1664         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1665                 return 0;
1666
1667         err = check_direct_IO(inode, iter, offset);
1668         if (err)
1669                 return err;
1670
1671         trace_f2fs_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
1672
1673         if (iov_iter_rw(iter) == WRITE) {
1674                 err = __allocate_data_blocks(inode, offset, count);
1675                 if (err)
1676                         goto out;
1677         }
1678
1679         err = blockdev_direct_IO(iocb, inode, iter, offset, get_data_block_dio);
1680 out:
1681         if (err < 0 && iov_iter_rw(iter) == WRITE)
1682                 f2fs_write_failed(mapping, offset + count);
1683
1684         trace_f2fs_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), err);
1685
1686         return err;
1687 }
1688
1689 void f2fs_invalidate_page(struct page *page, unsigned int offset,
1690                                                         unsigned int length)
1691 {
1692         struct inode *inode = page->mapping->host;
1693         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1694
1695         if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
1696                 (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE))
1697                 return;
1698
1699         if (PageDirty(page)) {
1700                 if (inode->i_ino == F2FS_META_INO(sbi))
1701                         dec_page_count(sbi, F2FS_DIRTY_META);
1702                 else if (inode->i_ino == F2FS_NODE_INO(sbi))
1703                         dec_page_count(sbi, F2FS_DIRTY_NODES);
1704                 else
1705                         inode_dec_dirty_pages(inode);
1706         }
1707
1708         /* This is atomic written page, keep Private */
1709         if (IS_ATOMIC_WRITTEN_PAGE(page))
1710                 return;
1711
1712         ClearPagePrivate(page);
1713 }
1714
1715 int f2fs_release_page(struct page *page, gfp_t wait)
1716 {
1717         /* If this is dirty page, keep PagePrivate */
1718         if (PageDirty(page))
1719                 return 0;
1720
1721         /* This is atomic written page, keep Private */
1722         if (IS_ATOMIC_WRITTEN_PAGE(page))
1723                 return 0;
1724
1725         ClearPagePrivate(page);
1726         return 1;
1727 }
1728
1729 static int f2fs_set_data_page_dirty(struct page *page)
1730 {
1731         struct address_space *mapping = page->mapping;
1732         struct inode *inode = mapping->host;
1733
1734         trace_f2fs_set_page_dirty(page, DATA);
1735
1736         SetPageUptodate(page);
1737
1738         if (f2fs_is_atomic_file(inode)) {
1739                 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
1740                         register_inmem_page(inode, page);
1741                         return 1;
1742                 }
1743                 /*
1744                  * Previously, this page has been registered, we just
1745                  * return here.
1746                  */
1747                 return 0;
1748         }
1749
1750         if (!PageDirty(page)) {
1751                 __set_page_dirty_nobuffers(page);
1752                 update_dirty_page(inode, page);
1753                 return 1;
1754         }
1755         return 0;
1756 }
1757
1758 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
1759 {
1760         struct inode *inode = mapping->host;
1761
1762         if (f2fs_has_inline_data(inode))
1763                 return 0;
1764
1765         /* make sure allocating whole blocks */
1766         if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
1767                 filemap_write_and_wait(mapping);
1768
1769         return generic_block_bmap(mapping, block, get_data_block_bmap);
1770 }
1771
1772 const struct address_space_operations f2fs_dblock_aops = {
1773         .readpage       = f2fs_read_data_page,
1774         .readpages      = f2fs_read_data_pages,
1775         .writepage      = f2fs_write_data_page,
1776         .writepages     = f2fs_write_data_pages,
1777         .write_begin    = f2fs_write_begin,
1778         .write_end      = f2fs_write_end,
1779         .set_page_dirty = f2fs_set_data_page_dirty,
1780         .invalidatepage = f2fs_invalidate_page,
1781         .releasepage    = f2fs_release_page,
1782         .direct_IO      = f2fs_direct_IO,
1783         .bmap           = f2fs_bmap,
1784 };