4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #define NULL_SEGNO ((unsigned int)(~0))
13 #define NULL_SECNO ((unsigned int)(~0))
15 /* V: Logical segment # in volume, R: Relative segment # in main area */
16 #define GET_L2R_SEGNO(free_i, segno) (segno - free_i->start_segno)
17 #define GET_R2L_SEGNO(free_i, segno) (segno + free_i->start_segno)
19 #define IS_DATASEG(t) \
20 ((t == CURSEG_HOT_DATA) || (t == CURSEG_COLD_DATA) || \
21 (t == CURSEG_WARM_DATA))
23 #define IS_NODESEG(t) \
24 ((t == CURSEG_HOT_NODE) || (t == CURSEG_COLD_NODE) || \
25 (t == CURSEG_WARM_NODE))
27 #define IS_CURSEG(sbi, seg) \
28 ((seg == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno) || \
29 (seg == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno) || \
30 (seg == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno) || \
31 (seg == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno) || \
32 (seg == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno) || \
33 (seg == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno))
35 #define IS_CURSEC(sbi, secno) \
36 ((secno == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno / \
37 sbi->segs_per_sec) || \
38 (secno == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno / \
39 sbi->segs_per_sec) || \
40 (secno == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno / \
41 sbi->segs_per_sec) || \
42 (secno == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno / \
43 sbi->segs_per_sec) || \
44 (secno == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno / \
45 sbi->segs_per_sec) || \
46 (secno == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno / \
49 #define START_BLOCK(sbi, segno) \
50 (SM_I(sbi)->seg0_blkaddr + \
51 (GET_R2L_SEGNO(FREE_I(sbi), segno) << sbi->log_blocks_per_seg))
52 #define NEXT_FREE_BLKADDR(sbi, curseg) \
53 (START_BLOCK(sbi, curseg->segno) + curseg->next_blkoff)
55 #define MAIN_BASE_BLOCK(sbi) (SM_I(sbi)->main_blkaddr)
57 #define GET_SEGOFF_FROM_SEG0(sbi, blk_addr) \
58 ((blk_addr) - SM_I(sbi)->seg0_blkaddr)
59 #define GET_SEGNO_FROM_SEG0(sbi, blk_addr) \
60 (GET_SEGOFF_FROM_SEG0(sbi, blk_addr) >> sbi->log_blocks_per_seg)
61 #define GET_SEGNO(sbi, blk_addr) \
62 (((blk_addr == NULL_ADDR) || (blk_addr == NEW_ADDR)) ? \
63 NULL_SEGNO : GET_L2R_SEGNO(FREE_I(sbi), \
64 GET_SEGNO_FROM_SEG0(sbi, blk_addr)))
65 #define GET_SECNO(sbi, segno) \
66 ((segno) / sbi->segs_per_sec)
67 #define GET_ZONENO_FROM_SEGNO(sbi, segno) \
68 ((segno / sbi->segs_per_sec) / sbi->secs_per_zone)
70 #define GET_SUM_BLOCK(sbi, segno) \
71 ((sbi->sm_info->ssa_blkaddr) + segno)
73 #define GET_SUM_TYPE(footer) ((footer)->entry_type)
74 #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = type)
76 #define SIT_ENTRY_OFFSET(sit_i, segno) \
77 (segno % sit_i->sents_per_block)
78 #define SIT_BLOCK_OFFSET(sit_i, segno) \
79 (segno / SIT_ENTRY_PER_BLOCK)
80 #define START_SEGNO(sit_i, segno) \
81 (SIT_BLOCK_OFFSET(sit_i, segno) * SIT_ENTRY_PER_BLOCK)
82 #define f2fs_bitmap_size(nr) \
83 (BITS_TO_LONGS(nr) * sizeof(unsigned long))
84 #define TOTAL_SEGS(sbi) (SM_I(sbi)->main_segments)
85 #define TOTAL_SECS(sbi) (sbi->total_sections)
87 #define SECTOR_FROM_BLOCK(sbi, blk_addr) \
88 (blk_addr << ((sbi)->log_blocksize - F2FS_LOG_SECTOR_SIZE))
90 /* during checkpoint, bio_private is used to synchronize the last bio */
92 struct f2fs_sb_info *sbi;
98 * indicate a block allocation direction: RIGHT and LEFT.
99 * RIGHT means allocating new sections towards the end of volume.
100 * LEFT means the opposite direction.
108 * In the victim_sel_policy->alloc_mode, there are two block allocation modes.
109 * LFS writes data sequentially with cleaning operations.
110 * SSR (Slack Space Recycle) reuses obsolete space without cleaning operations.
118 * In the victim_sel_policy->gc_mode, there are two gc, aka cleaning, modes.
119 * GC_CB is based on cost-benefit algorithm.
120 * GC_GREEDY is based on greedy algorithm.
128 * BG_GC means the background cleaning job.
129 * FG_GC means the on-demand cleaning job.
136 /* for a function parameter to select a victim segment */
137 struct victim_sel_policy {
138 int alloc_mode; /* LFS or SSR */
139 int gc_mode; /* GC_CB or GC_GREEDY */
140 unsigned long *dirty_segmap; /* dirty segment bitmap */
141 unsigned int offset; /* last scanned bitmap offset */
142 unsigned int ofs_unit; /* bitmap search unit */
143 unsigned int min_cost; /* minimum cost */
144 unsigned int min_segno; /* segment # having min. cost */
148 unsigned short valid_blocks; /* # of valid blocks */
149 unsigned char *cur_valid_map; /* validity bitmap of blocks */
151 * # of valid blocks and the validity bitmap stored in the the last
152 * checkpoint pack. This information is used by the SSR mode.
154 unsigned short ckpt_valid_blocks;
155 unsigned char *ckpt_valid_map;
156 unsigned char type; /* segment type like CURSEG_XXX_TYPE */
157 unsigned long long mtime; /* modification time of the segment */
161 unsigned int valid_blocks; /* # of valid blocks in a section */
164 struct segment_allocation {
165 void (*allocate_segment)(struct f2fs_sb_info *, int, bool);
169 const struct segment_allocation *s_ops;
171 block_t sit_base_addr; /* start block address of SIT area */
172 block_t sit_blocks; /* # of blocks used by SIT area */
173 block_t written_valid_blocks; /* # of valid blocks in main area */
174 char *sit_bitmap; /* SIT bitmap pointer */
175 unsigned int bitmap_size; /* SIT bitmap size */
177 unsigned long *dirty_sentries_bitmap; /* bitmap for dirty sentries */
178 unsigned int dirty_sentries; /* # of dirty sentries */
179 unsigned int sents_per_block; /* # of SIT entries per block */
180 struct mutex sentry_lock; /* to protect SIT cache */
181 struct seg_entry *sentries; /* SIT segment-level cache */
182 struct sec_entry *sec_entries; /* SIT section-level cache */
184 /* for cost-benefit algorithm in cleaning procedure */
185 unsigned long long elapsed_time; /* elapsed time after mount */
186 unsigned long long mounted_time; /* mount time */
187 unsigned long long min_mtime; /* min. modification time */
188 unsigned long long max_mtime; /* max. modification time */
191 struct free_segmap_info {
192 unsigned int start_segno; /* start segment number logically */
193 unsigned int free_segments; /* # of free segments */
194 unsigned int free_sections; /* # of free sections */
195 rwlock_t segmap_lock; /* free segmap lock */
196 unsigned long *free_segmap; /* free segment bitmap */
197 unsigned long *free_secmap; /* free section bitmap */
200 /* Notice: The order of dirty type is same with CURSEG_XXX in f2fs.h */
202 DIRTY_HOT_DATA, /* dirty segments assigned as hot data logs */
203 DIRTY_WARM_DATA, /* dirty segments assigned as warm data logs */
204 DIRTY_COLD_DATA, /* dirty segments assigned as cold data logs */
205 DIRTY_HOT_NODE, /* dirty segments assigned as hot node logs */
206 DIRTY_WARM_NODE, /* dirty segments assigned as warm node logs */
207 DIRTY_COLD_NODE, /* dirty segments assigned as cold node logs */
208 DIRTY, /* to count # of dirty segments */
209 PRE, /* to count # of entirely obsolete segments */
213 struct dirty_seglist_info {
214 const struct victim_selection *v_ops; /* victim selction operation */
215 unsigned long *dirty_segmap[NR_DIRTY_TYPE];
216 struct mutex seglist_lock; /* lock for segment bitmaps */
217 int nr_dirty[NR_DIRTY_TYPE]; /* # of dirty segments */
218 unsigned long *victim_secmap; /* background GC victims */
221 /* victim selection function for cleaning and SSR */
222 struct victim_selection {
223 int (*get_victim)(struct f2fs_sb_info *, unsigned int *,
227 /* for active log information */
229 struct mutex curseg_mutex; /* lock for consistency */
230 struct f2fs_summary_block *sum_blk; /* cached summary block */
231 unsigned char alloc_type; /* current allocation type */
232 unsigned int segno; /* current segment number */
233 unsigned short next_blkoff; /* next block offset to write */
234 unsigned int zone; /* current zone number */
235 unsigned int next_segno; /* preallocated segment */
241 static inline struct curseg_info *CURSEG_I(struct f2fs_sb_info *sbi, int type)
243 return (struct curseg_info *)(SM_I(sbi)->curseg_array + type);
246 static inline struct seg_entry *get_seg_entry(struct f2fs_sb_info *sbi,
249 struct sit_info *sit_i = SIT_I(sbi);
250 return &sit_i->sentries[segno];
253 static inline struct sec_entry *get_sec_entry(struct f2fs_sb_info *sbi,
256 struct sit_info *sit_i = SIT_I(sbi);
257 return &sit_i->sec_entries[GET_SECNO(sbi, segno)];
260 static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi,
261 unsigned int segno, int section)
264 * In order to get # of valid blocks in a section instantly from many
265 * segments, f2fs manages two counting structures separately.
268 return get_sec_entry(sbi, segno)->valid_blocks;
270 return get_seg_entry(sbi, segno)->valid_blocks;
273 static inline void seg_info_from_raw_sit(struct seg_entry *se,
274 struct f2fs_sit_entry *rs)
276 se->valid_blocks = GET_SIT_VBLOCKS(rs);
277 se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs);
278 memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
279 memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
280 se->type = GET_SIT_TYPE(rs);
281 se->mtime = le64_to_cpu(rs->mtime);
284 static inline void seg_info_to_raw_sit(struct seg_entry *se,
285 struct f2fs_sit_entry *rs)
287 unsigned short raw_vblocks = (se->type << SIT_VBLOCKS_SHIFT) |
289 rs->vblocks = cpu_to_le16(raw_vblocks);
290 memcpy(rs->valid_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
291 memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
292 se->ckpt_valid_blocks = se->valid_blocks;
293 rs->mtime = cpu_to_le64(se->mtime);
296 static inline unsigned int find_next_inuse(struct free_segmap_info *free_i,
297 unsigned int max, unsigned int segno)
300 read_lock(&free_i->segmap_lock);
301 ret = find_next_bit(free_i->free_segmap, max, segno);
302 read_unlock(&free_i->segmap_lock);
306 static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno)
308 struct free_segmap_info *free_i = FREE_I(sbi);
309 unsigned int secno = segno / sbi->segs_per_sec;
310 unsigned int start_segno = secno * sbi->segs_per_sec;
313 write_lock(&free_i->segmap_lock);
314 clear_bit(segno, free_i->free_segmap);
315 free_i->free_segments++;
317 next = find_next_bit(free_i->free_segmap, TOTAL_SEGS(sbi), start_segno);
318 if (next >= start_segno + sbi->segs_per_sec) {
319 clear_bit(secno, free_i->free_secmap);
320 free_i->free_sections++;
322 write_unlock(&free_i->segmap_lock);
325 static inline void __set_inuse(struct f2fs_sb_info *sbi,
328 struct free_segmap_info *free_i = FREE_I(sbi);
329 unsigned int secno = segno / sbi->segs_per_sec;
330 set_bit(segno, free_i->free_segmap);
331 free_i->free_segments--;
332 if (!test_and_set_bit(secno, free_i->free_secmap))
333 free_i->free_sections--;
336 static inline void __set_test_and_free(struct f2fs_sb_info *sbi,
339 struct free_segmap_info *free_i = FREE_I(sbi);
340 unsigned int secno = segno / sbi->segs_per_sec;
341 unsigned int start_segno = secno * sbi->segs_per_sec;
344 write_lock(&free_i->segmap_lock);
345 if (test_and_clear_bit(segno, free_i->free_segmap)) {
346 free_i->free_segments++;
348 next = find_next_bit(free_i->free_segmap, TOTAL_SEGS(sbi),
350 if (next >= start_segno + sbi->segs_per_sec) {
351 if (test_and_clear_bit(secno, free_i->free_secmap))
352 free_i->free_sections++;
355 write_unlock(&free_i->segmap_lock);
358 static inline void __set_test_and_inuse(struct f2fs_sb_info *sbi,
361 struct free_segmap_info *free_i = FREE_I(sbi);
362 unsigned int secno = segno / sbi->segs_per_sec;
363 write_lock(&free_i->segmap_lock);
364 if (!test_and_set_bit(segno, free_i->free_segmap)) {
365 free_i->free_segments--;
366 if (!test_and_set_bit(secno, free_i->free_secmap))
367 free_i->free_sections--;
369 write_unlock(&free_i->segmap_lock);
372 static inline void get_sit_bitmap(struct f2fs_sb_info *sbi,
375 struct sit_info *sit_i = SIT_I(sbi);
376 memcpy(dst_addr, sit_i->sit_bitmap, sit_i->bitmap_size);
379 static inline block_t written_block_count(struct f2fs_sb_info *sbi)
381 struct sit_info *sit_i = SIT_I(sbi);
384 mutex_lock(&sit_i->sentry_lock);
385 vblocks = sit_i->written_valid_blocks;
386 mutex_unlock(&sit_i->sentry_lock);
391 static inline unsigned int free_segments(struct f2fs_sb_info *sbi)
393 struct free_segmap_info *free_i = FREE_I(sbi);
394 unsigned int free_segs;
396 read_lock(&free_i->segmap_lock);
397 free_segs = free_i->free_segments;
398 read_unlock(&free_i->segmap_lock);
403 static inline int reserved_segments(struct f2fs_sb_info *sbi)
405 return SM_I(sbi)->reserved_segments;
408 static inline unsigned int free_sections(struct f2fs_sb_info *sbi)
410 struct free_segmap_info *free_i = FREE_I(sbi);
411 unsigned int free_secs;
413 read_lock(&free_i->segmap_lock);
414 free_secs = free_i->free_sections;
415 read_unlock(&free_i->segmap_lock);
420 static inline unsigned int prefree_segments(struct f2fs_sb_info *sbi)
422 return DIRTY_I(sbi)->nr_dirty[PRE];
425 static inline unsigned int dirty_segments(struct f2fs_sb_info *sbi)
427 return DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_DATA] +
428 DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_DATA] +
429 DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_DATA] +
430 DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_NODE] +
431 DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_NODE] +
432 DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_NODE];
435 static inline int overprovision_segments(struct f2fs_sb_info *sbi)
437 return SM_I(sbi)->ovp_segments;
440 static inline int overprovision_sections(struct f2fs_sb_info *sbi)
442 return ((unsigned int) overprovision_segments(sbi)) / sbi->segs_per_sec;
445 static inline int reserved_sections(struct f2fs_sb_info *sbi)
447 return ((unsigned int) reserved_segments(sbi)) / sbi->segs_per_sec;
450 static inline bool need_SSR(struct f2fs_sb_info *sbi)
452 return (free_sections(sbi) < overprovision_sections(sbi));
455 static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, int freed)
457 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
458 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
463 return ((free_sections(sbi) + freed) <= (node_secs + 2 * dent_secs +
464 reserved_sections(sbi)));
467 static inline int utilization(struct f2fs_sb_info *sbi)
469 return div_u64(valid_user_blocks(sbi) * 100, sbi->user_block_count);
473 * Sometimes f2fs may be better to drop out-of-place update policy.
474 * So, if fs utilization is over MIN_IPU_UTIL, then f2fs tries to write
475 * data in the original place likewise other traditional file systems.
476 * But, currently set 100 in percentage, which means it is disabled.
477 * See below need_inplace_update().
479 #define MIN_IPU_UTIL 100
480 static inline bool need_inplace_update(struct inode *inode)
482 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
483 if (S_ISDIR(inode->i_mode))
485 if (need_SSR(sbi) && utilization(sbi) > MIN_IPU_UTIL)
490 static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi,
493 struct curseg_info *curseg = CURSEG_I(sbi, type);
494 return curseg->segno;
497 static inline unsigned char curseg_alloc_type(struct f2fs_sb_info *sbi,
500 struct curseg_info *curseg = CURSEG_I(sbi, type);
501 return curseg->alloc_type;
504 static inline unsigned short curseg_blkoff(struct f2fs_sb_info *sbi, int type)
506 struct curseg_info *curseg = CURSEG_I(sbi, type);
507 return curseg->next_blkoff;
510 static inline void check_seg_range(struct f2fs_sb_info *sbi, unsigned int segno)
512 unsigned int end_segno = SM_I(sbi)->segment_count - 1;
513 BUG_ON(segno > end_segno);
517 * This function is used for only debugging.
518 * NOTE: In future, we have to remove this function.
520 static inline void verify_block_addr(struct f2fs_sb_info *sbi, block_t blk_addr)
522 struct f2fs_sm_info *sm_info = SM_I(sbi);
523 block_t total_blks = sm_info->segment_count << sbi->log_blocks_per_seg;
524 block_t start_addr = sm_info->seg0_blkaddr;
525 block_t end_addr = start_addr + total_blks - 1;
526 BUG_ON(blk_addr < start_addr);
527 BUG_ON(blk_addr > end_addr);
531 * Summary block is always treated as invalid block
533 static inline void check_block_count(struct f2fs_sb_info *sbi,
534 int segno, struct f2fs_sit_entry *raw_sit)
536 struct f2fs_sm_info *sm_info = SM_I(sbi);
537 unsigned int end_segno = sm_info->segment_count - 1;
538 int valid_blocks = 0;
541 /* check segment usage */
542 BUG_ON(GET_SIT_VBLOCKS(raw_sit) > sbi->blocks_per_seg);
544 /* check boundary of a given segment number */
545 BUG_ON(segno > end_segno);
547 /* check bitmap with valid block count */
548 for (i = 0; i < sbi->blocks_per_seg; i++)
549 if (f2fs_test_bit(i, raw_sit->valid_map))
551 BUG_ON(GET_SIT_VBLOCKS(raw_sit) != valid_blocks);
554 static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi,
557 struct sit_info *sit_i = SIT_I(sbi);
558 unsigned int offset = SIT_BLOCK_OFFSET(sit_i, start);
559 block_t blk_addr = sit_i->sit_base_addr + offset;
561 check_seg_range(sbi, start);
563 /* calculate sit block address */
564 if (f2fs_test_bit(offset, sit_i->sit_bitmap))
565 blk_addr += sit_i->sit_blocks;
570 static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi,
573 struct sit_info *sit_i = SIT_I(sbi);
574 block_addr -= sit_i->sit_base_addr;
575 if (block_addr < sit_i->sit_blocks)
576 block_addr += sit_i->sit_blocks;
578 block_addr -= sit_i->sit_blocks;
580 return block_addr + sit_i->sit_base_addr;
583 static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start)
585 unsigned int block_off = SIT_BLOCK_OFFSET(sit_i, start);
587 if (f2fs_test_bit(block_off, sit_i->sit_bitmap))
588 f2fs_clear_bit(block_off, sit_i->sit_bitmap);
590 f2fs_set_bit(block_off, sit_i->sit_bitmap);
593 static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi)
595 struct sit_info *sit_i = SIT_I(sbi);
596 return sit_i->elapsed_time + CURRENT_TIME_SEC.tv_sec -
600 static inline void set_summary(struct f2fs_summary *sum, nid_t nid,
601 unsigned int ofs_in_node, unsigned char version)
603 sum->nid = cpu_to_le32(nid);
604 sum->ofs_in_node = cpu_to_le16(ofs_in_node);
605 sum->version = version;
608 static inline block_t start_sum_block(struct f2fs_sb_info *sbi)
610 return __start_cp_addr(sbi) +
611 le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
614 static inline block_t sum_blk_addr(struct f2fs_sb_info *sbi, int base, int type)
616 return __start_cp_addr(sbi) +
617 le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_total_block_count)
621 static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno)
623 if (IS_CURSEC(sbi, secno) || (sbi->cur_victim_sec == secno))