]> git.karo-electronics.de Git - linux-beck.git/blobdiff - fs/f2fs/segment.c
f2fs: improve write performance under frequent fsync calls
[linux-beck.git] / fs / f2fs / segment.c
index 68b8a2bdf6d82253bfc35f50ec09ef0c1f09c497..5f84639354e3adaa17209aa4daebbc5ee785c009 100644 (file)
@@ -14,6 +14,7 @@
 #include <linux/blkdev.h>
 #include <linux/prefetch.h>
 #include <linux/vmalloc.h>
+#include <linux/swap.h>
 
 #include "f2fs.h"
 #include "segment.h"
@@ -269,8 +270,8 @@ static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 static void f2fs_issue_discard(struct f2fs_sb_info *sbi,
                                block_t blkstart, block_t blklen)
 {
-       sector_t start = ((sector_t)blkstart) << sbi->log_sectors_per_block;
-       sector_t len = ((sector_t)blklen) << sbi->log_sectors_per_block;
+       sector_t start = SECTOR_FROM_BLOCK(sbi, blkstart);
+       sector_t len = SECTOR_FROM_BLOCK(sbi, blklen);
        blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
        trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
 }
@@ -786,156 +787,6 @@ static const struct segment_allocation default_salloc_ops = {
        .allocate_segment = allocate_segment_by_default,
 };
 
-static void f2fs_end_io_write(struct bio *bio, int err)
-{
-       const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
-       struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
-       struct bio_private *p = bio->bi_private;
-
-       do {
-               struct page *page = bvec->bv_page;
-
-               if (--bvec >= bio->bi_io_vec)
-                       prefetchw(&bvec->bv_page->flags);
-               if (!uptodate) {
-                       SetPageError(page);
-                       if (page->mapping)
-                               set_bit(AS_EIO, &page->mapping->flags);
-                       set_ckpt_flags(p->sbi->ckpt, CP_ERROR_FLAG);
-                       p->sbi->sb->s_flags |= MS_RDONLY;
-               }
-               end_page_writeback(page);
-               dec_page_count(p->sbi, F2FS_WRITEBACK);
-       } while (bvec >= bio->bi_io_vec);
-
-       if (p->is_sync)
-               complete(p->wait);
-
-       if (!get_pages(p->sbi, F2FS_WRITEBACK) &&
-                       !list_empty(&p->sbi->cp_wait.task_list))
-               wake_up(&p->sbi->cp_wait);
-
-       kfree(p);
-       bio_put(bio);
-}
-
-struct bio *f2fs_bio_alloc(struct block_device *bdev, int npages)
-{
-       struct bio *bio;
-
-       /* No failure on bio allocation */
-       bio = bio_alloc(GFP_NOIO, npages);
-       bio->bi_bdev = bdev;
-       bio->bi_private = NULL;
-
-       return bio;
-}
-
-static void do_submit_bio(struct f2fs_sb_info *sbi,
-                               enum page_type type, bool sync)
-{
-       int rw = sync ? WRITE_SYNC : WRITE;
-       enum page_type btype = PAGE_TYPE_OF_BIO(type);
-       struct bio *bio = sbi->bio[btype];
-       struct bio_private *p;
-
-       if (!bio)
-               return;
-
-       sbi->bio[btype] = NULL;
-
-       if (type >= META_FLUSH)
-               rw = WRITE_FLUSH_FUA;
-       if (btype == META)
-               rw |= REQ_META;
-
-       p = bio->bi_private;
-       p->sbi = sbi;
-       bio->bi_end_io = f2fs_end_io_write;
-
-       trace_f2fs_do_submit_bio(sbi->sb, btype, sync, bio);
-
-       if (type == META_FLUSH) {
-               DECLARE_COMPLETION_ONSTACK(wait);
-               p->is_sync = true;
-               p->wait = &wait;
-               submit_bio(rw, bio);
-               wait_for_completion(&wait);
-       } else {
-               p->is_sync = false;
-               submit_bio(rw, bio);
-       }
-}
-
-void f2fs_submit_bio(struct f2fs_sb_info *sbi, enum page_type type, bool sync)
-{
-       enum page_type btype = PAGE_TYPE_OF_BIO(type);
-
-       if (!sbi->bio[btype])
-               return;
-
-       mutex_lock(&sbi->write_mutex[btype]);
-       do_submit_bio(sbi, type, sync);
-       mutex_unlock(&sbi->write_mutex[btype]);
-}
-
-static void submit_write_page(struct f2fs_sb_info *sbi, struct page *page,
-                               block_t blk_addr, enum page_type type)
-{
-       struct block_device *bdev = sbi->sb->s_bdev;
-       int bio_blocks;
-
-       verify_block_addr(sbi, blk_addr);
-
-       mutex_lock(&sbi->write_mutex[type]);
-
-       inc_page_count(sbi, F2FS_WRITEBACK);
-
-       if (sbi->bio[type] && sbi->last_block_in_bio[type] != blk_addr - 1)
-               do_submit_bio(sbi, type, false);
-alloc_new:
-       if (sbi->bio[type] == NULL) {
-               struct bio_private *priv;
-retry:
-               priv = kmalloc(sizeof(struct bio_private), GFP_NOFS);
-               if (!priv) {
-                       cond_resched();
-                       goto retry;
-               }
-
-               bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
-               sbi->bio[type] = f2fs_bio_alloc(bdev, bio_blocks);
-               sbi->bio[type]->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
-               sbi->bio[type]->bi_private = priv;
-               /*
-                * The end_io will be assigned at the sumbission phase.
-                * Until then, let bio_add_page() merge consecutive IOs as much
-                * as possible.
-                */
-       }
-
-       if (bio_add_page(sbi->bio[type], page, PAGE_CACHE_SIZE, 0) <
-                                                       PAGE_CACHE_SIZE) {
-               do_submit_bio(sbi, type, false);
-               goto alloc_new;
-       }
-
-       sbi->last_block_in_bio[type] = blk_addr;
-
-       mutex_unlock(&sbi->write_mutex[type]);
-       trace_f2fs_submit_write_page(page, blk_addr, type);
-}
-
-void f2fs_wait_on_page_writeback(struct page *page,
-                               enum page_type type, bool sync)
-{
-       struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
-       if (PageWriteback(page)) {
-               f2fs_submit_bio(sbi, type, sync);
-               wait_on_page_writeback(page);
-       }
-}
-
 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
 {
        struct curseg_info *curseg = CURSEG_I(sbi, type);
@@ -1003,16 +854,14 @@ static int __get_segment_type(struct page *page, enum page_type p_type)
        return __get_segment_type_6(page, p_type);
 }
 
-static void do_write_page(struct f2fs_sb_info *sbi, struct page *page,
-                       block_t old_blkaddr, block_t *new_blkaddr,
-                       struct f2fs_summary *sum, enum page_type p_type)
+void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
+               block_t old_blkaddr, block_t *new_blkaddr,
+               struct f2fs_summary *sum, int type)
 {
        struct sit_info *sit_i = SIT_I(sbi);
        struct curseg_info *curseg;
        unsigned int old_cursegno;
-       int type;
 
-       type = __get_segment_type(page, p_type);
        curseg = CURSEG_I(sbi, type);
 
        mutex_lock(&curseg->curseg_mutex);
@@ -1045,49 +894,63 @@ static void do_write_page(struct f2fs_sb_info *sbi, struct page *page,
        locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
        mutex_unlock(&sit_i->sentry_lock);
 
-       if (p_type == NODE)
+       if (page && IS_NODESEG(type))
                fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
 
-       /* writeout dirty page into bdev */
-       submit_write_page(sbi, page, *new_blkaddr, p_type);
-
        mutex_unlock(&curseg->curseg_mutex);
 }
 
+static void do_write_page(struct f2fs_sb_info *sbi, struct page *page,
+                       block_t old_blkaddr, block_t *new_blkaddr,
+                       struct f2fs_summary *sum, struct f2fs_io_info *fio)
+{
+       int type = __get_segment_type(page, fio->type);
+
+       allocate_data_block(sbi, page, old_blkaddr, new_blkaddr, sum, type);
+
+       /* writeout dirty page into bdev */
+       f2fs_submit_page_mbio(sbi, page, *new_blkaddr, fio);
+}
+
 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
 {
+       struct f2fs_io_info fio = {
+               .type = META,
+               .rw = WRITE_SYNC | REQ_META | REQ_PRIO
+       };
+
        set_page_writeback(page);
-       submit_write_page(sbi, page, page->index, META);
+       f2fs_submit_page_mbio(sbi, page, page->index, &fio);
 }
 
 void write_node_page(struct f2fs_sb_info *sbi, struct page *page,
+               struct f2fs_io_info *fio,
                unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr)
 {
        struct f2fs_summary sum;
        set_summary(&sum, nid, 0, 0);
-       do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, NODE);
+       do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, fio);
 }
 
-void write_data_page(struct inode *inode, struct page *page,
-               struct dnode_of_data *dn, block_t old_blkaddr,
-               block_t *new_blkaddr)
+void write_data_page(struct page *page, struct dnode_of_data *dn,
+               block_t *new_blkaddr, struct f2fs_io_info *fio)
 {
-       struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
+       struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
        struct f2fs_summary sum;
        struct node_info ni;
 
-       f2fs_bug_on(old_blkaddr == NULL_ADDR);
+       f2fs_bug_on(dn->data_blkaddr == NULL_ADDR);
        get_node_info(sbi, dn->nid, &ni);
        set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 
-       do_write_page(sbi, page, old_blkaddr,
-                       new_blkaddr, &sum, DATA);
+       do_write_page(sbi, page, dn->data_blkaddr, new_blkaddr, &sum, fio);
 }
 
-void rewrite_data_page(struct f2fs_sb_info *sbi, struct page *page,
-                                       block_t old_blk_addr)
+void rewrite_data_page(struct page *page, block_t old_blkaddr, struct f2fs_io_info *fio)
 {
-       submit_write_page(sbi, page, old_blk_addr, DATA);
+       struct inode *inode = page->mapping->host;
+       struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
+       f2fs_submit_page_mbio(sbi, page, old_blkaddr, fio);
 }
 
 void recover_data_page(struct f2fs_sb_info *sbi,
@@ -1146,6 +1009,10 @@ void rewrite_node_page(struct f2fs_sb_info *sbi,
        unsigned int segno, old_cursegno;
        block_t next_blkaddr = next_blkaddr_of_node(page);
        unsigned int next_segno = GET_SEGNO(sbi, next_blkaddr);
+       struct f2fs_io_info fio = {
+               .type = NODE,
+               .rw = WRITE_SYNC,
+       };
 
        curseg = CURSEG_I(sbi, type);
 
@@ -1174,8 +1041,8 @@ void rewrite_node_page(struct f2fs_sb_info *sbi,
 
        /* rewrite node page */
        set_page_writeback(page);
-       submit_write_page(sbi, page, new_blkaddr, NODE);
-       f2fs_submit_bio(sbi, NODE, true);
+       f2fs_submit_page_mbio(sbi, page, new_blkaddr, &fio);
+       f2fs_submit_merged_bio(sbi, NODE, WRITE);
        refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);
 
        locate_dirty_segment(sbi, old_cursegno);
@@ -1185,6 +1052,16 @@ void rewrite_node_page(struct f2fs_sb_info *sbi,
        mutex_unlock(&curseg->curseg_mutex);
 }
 
+void f2fs_wait_on_page_writeback(struct page *page,
+                               enum page_type type, bool sync)
+{
+       struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
+       if (PageWriteback(page)) {
+               f2fs_submit_merged_bio(sbi, type, WRITE);
+               wait_on_page_writeback(page);
+       }
+}
+
 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
 {
        struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
@@ -1705,41 +1582,93 @@ static int build_curseg(struct f2fs_sb_info *sbi)
        return restore_curseg_summaries(sbi);
 }
 
+static int ra_sit_pages(struct f2fs_sb_info *sbi, int start, int nrpages)
+{
+       struct address_space *mapping = sbi->meta_inode->i_mapping;
+       struct page *page;
+       block_t blk_addr, prev_blk_addr = 0;
+       int sit_blk_cnt = SIT_BLK_CNT(sbi);
+       int blkno = start;
+       struct f2fs_io_info fio = {
+               .type = META,
+               .rw = READ_SYNC | REQ_META | REQ_PRIO
+       };
+
+       for (; blkno < start + nrpages && blkno < sit_blk_cnt; blkno++) {
+
+               blk_addr = current_sit_addr(sbi, blkno * SIT_ENTRY_PER_BLOCK);
+
+               if (blkno != start && prev_blk_addr + 1 != blk_addr)
+                       break;
+               prev_blk_addr = blk_addr;
+repeat:
+               page = grab_cache_page(mapping, blk_addr);
+               if (!page) {
+                       cond_resched();
+                       goto repeat;
+               }
+               if (PageUptodate(page)) {
+                       mark_page_accessed(page);
+                       f2fs_put_page(page, 1);
+                       continue;
+               }
+
+               f2fs_submit_page_mbio(sbi, page, blk_addr, &fio);
+
+               mark_page_accessed(page);
+               f2fs_put_page(page, 0);
+       }
+
+       f2fs_submit_merged_bio(sbi, META, READ);
+       return blkno - start;
+}
+
 static void build_sit_entries(struct f2fs_sb_info *sbi)
 {
        struct sit_info *sit_i = SIT_I(sbi);
        struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
        struct f2fs_summary_block *sum = curseg->sum_blk;
-       unsigned int start;
+       int sit_blk_cnt = SIT_BLK_CNT(sbi);
+       unsigned int i, start, end;
+       unsigned int readed, start_blk = 0;
+       int nrpages = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
 
-       for (start = 0; start < TOTAL_SEGS(sbi); start++) {
-               struct seg_entry *se = &sit_i->sentries[start];
-               struct f2fs_sit_block *sit_blk;
-               struct f2fs_sit_entry sit;
-               struct page *page;
-               int i;
-
-               mutex_lock(&curseg->curseg_mutex);
-               for (i = 0; i < sits_in_cursum(sum); i++) {
-                       if (le32_to_cpu(segno_in_journal(sum, i)) == start) {
-                               sit = sit_in_journal(sum, i);
-                               mutex_unlock(&curseg->curseg_mutex);
-                               goto got_it;
+       do {
+               readed = ra_sit_pages(sbi, start_blk, nrpages);
+
+               start = start_blk * sit_i->sents_per_block;
+               end = (start_blk + readed) * sit_i->sents_per_block;
+
+               for (; start < end && start < TOTAL_SEGS(sbi); start++) {
+                       struct seg_entry *se = &sit_i->sentries[start];
+                       struct f2fs_sit_block *sit_blk;
+                       struct f2fs_sit_entry sit;
+                       struct page *page;
+
+                       mutex_lock(&curseg->curseg_mutex);
+                       for (i = 0; i < sits_in_cursum(sum); i++) {
+                               if (le32_to_cpu(segno_in_journal(sum, i)) == start) {
+                                       sit = sit_in_journal(sum, i);
+                                       mutex_unlock(&curseg->curseg_mutex);
+                                       goto got_it;
+                               }
                        }
-               }
-               mutex_unlock(&curseg->curseg_mutex);
-               page = get_current_sit_page(sbi, start);
-               sit_blk = (struct f2fs_sit_block *)page_address(page);
-               sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
-               f2fs_put_page(page, 1);
+                       mutex_unlock(&curseg->curseg_mutex);
+
+                       page = get_current_sit_page(sbi, start);
+                       sit_blk = (struct f2fs_sit_block *)page_address(page);
+                       sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
+                       f2fs_put_page(page, 1);
 got_it:
-               check_block_count(sbi, start, &sit);
-               seg_info_from_raw_sit(se, &sit);
-               if (sbi->segs_per_sec > 1) {
-                       struct sec_entry *e = get_sec_entry(sbi, start);
-                       e->valid_blocks += se->valid_blocks;
+                       check_block_count(sbi, start, &sit);
+                       seg_info_from_raw_sit(se, &sit);
+                       if (sbi->segs_per_sec > 1) {
+                               struct sec_entry *e = get_sec_entry(sbi, start);
+                               e->valid_blocks += se->valid_blocks;
+                       }
                }
-       }
+               start_blk += readed;
+       } while (start_blk < sit_blk_cnt);
 }
 
 static void init_free_segmap(struct f2fs_sb_info *sbi)
@@ -1869,6 +1798,8 @@ int build_segment_manager(struct f2fs_sb_info *sbi)
        sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
        sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
        sm_info->rec_prefree_segments = DEF_RECLAIM_PREFREE_SEGMENTS;
+       sm_info->ipu_policy = F2FS_IPU_DISABLE;
+       sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
 
        INIT_LIST_HEAD(&sm_info->discard_list);
        sm_info->nr_discards = 0;