2 #include <linux/wait.h>
3 #include <linux/backing-dev.h>
4 #include <linux/kthread.h>
5 #include <linux/freezer.h>
7 #include <linux/pagemap.h>
9 #include <linux/sched.h>
10 #include <linux/module.h>
11 #include <linux/writeback.h>
12 #include <linux/device.h>
14 void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
17 EXPORT_SYMBOL(default_unplug_io_fn);
19 struct backing_dev_info default_backing_dev_info = {
21 .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
23 .capabilities = BDI_CAP_MAP_COPY,
24 .unplug_io_fn = default_unplug_io_fn,
26 EXPORT_SYMBOL_GPL(default_backing_dev_info);
28 static struct class *bdi_class;
29 DEFINE_SPINLOCK(bdi_lock);
31 LIST_HEAD(bdi_pending_list);
33 static struct task_struct *sync_supers_tsk;
34 static struct timer_list sync_supers_timer;
36 static int bdi_sync_supers(void *);
37 static void sync_supers_timer_fn(unsigned long);
38 static void arm_supers_timer(void);
40 static void bdi_add_default_flusher_task(struct backing_dev_info *bdi);
42 #ifdef CONFIG_DEBUG_FS
43 #include <linux/debugfs.h>
44 #include <linux/seq_file.h>
46 static struct dentry *bdi_debug_root;
48 static void bdi_debug_init(void)
50 bdi_debug_root = debugfs_create_dir("bdi", NULL);
53 static int bdi_debug_stats_show(struct seq_file *m, void *v)
55 struct backing_dev_info *bdi = m->private;
56 struct bdi_writeback *wb;
57 unsigned long background_thresh;
58 unsigned long dirty_thresh;
59 unsigned long bdi_thresh;
60 unsigned long nr_dirty, nr_io, nr_more_io, nr_wb;
64 * inode lock is enough here, the bdi->wb_list is protected by
65 * RCU on the reader side
67 nr_wb = nr_dirty = nr_io = nr_more_io = 0;
68 spin_lock(&inode_lock);
69 list_for_each_entry(wb, &bdi->wb_list, list) {
71 list_for_each_entry(inode, &wb->b_dirty, i_list)
73 list_for_each_entry(inode, &wb->b_io, i_list)
75 list_for_each_entry(inode, &wb->b_more_io, i_list)
78 spin_unlock(&inode_lock);
80 get_dirty_limits(&background_thresh, &dirty_thresh, &bdi_thresh, bdi);
82 #define K(x) ((x) << (PAGE_SHIFT - 10))
84 "BdiWriteback: %8lu kB\n"
85 "BdiReclaimable: %8lu kB\n"
86 "BdiDirtyThresh: %8lu kB\n"
87 "DirtyThresh: %8lu kB\n"
88 "BackgroundThresh: %8lu kB\n"
89 "WriteBack threads:%8lu\n"
98 (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)),
99 (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)),
100 K(bdi_thresh), K(dirty_thresh),
101 K(background_thresh), nr_wb, nr_dirty, nr_io, nr_more_io,
102 !list_empty(&bdi->bdi_list), bdi->state, bdi->wb_mask,
103 !list_empty(&bdi->wb_list), bdi->wb_cnt);
109 static int bdi_debug_stats_open(struct inode *inode, struct file *file)
111 return single_open(file, bdi_debug_stats_show, inode->i_private);
114 static const struct file_operations bdi_debug_stats_fops = {
115 .open = bdi_debug_stats_open,
118 .release = single_release,
121 static void bdi_debug_register(struct backing_dev_info *bdi, const char *name)
123 bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
124 bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
125 bdi, &bdi_debug_stats_fops);
128 static void bdi_debug_unregister(struct backing_dev_info *bdi)
130 debugfs_remove(bdi->debug_stats);
131 debugfs_remove(bdi->debug_dir);
134 static inline void bdi_debug_init(void)
137 static inline void bdi_debug_register(struct backing_dev_info *bdi,
141 static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
146 static ssize_t read_ahead_kb_store(struct device *dev,
147 struct device_attribute *attr,
148 const char *buf, size_t count)
150 struct backing_dev_info *bdi = dev_get_drvdata(dev);
152 unsigned long read_ahead_kb;
153 ssize_t ret = -EINVAL;
155 read_ahead_kb = simple_strtoul(buf, &end, 10);
156 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
157 bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);
163 #define K(pages) ((pages) << (PAGE_SHIFT - 10))
165 #define BDI_SHOW(name, expr) \
166 static ssize_t name##_show(struct device *dev, \
167 struct device_attribute *attr, char *page) \
169 struct backing_dev_info *bdi = dev_get_drvdata(dev); \
171 return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \
174 BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))
176 static ssize_t min_ratio_store(struct device *dev,
177 struct device_attribute *attr, const char *buf, size_t count)
179 struct backing_dev_info *bdi = dev_get_drvdata(dev);
182 ssize_t ret = -EINVAL;
184 ratio = simple_strtoul(buf, &end, 10);
185 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
186 ret = bdi_set_min_ratio(bdi, ratio);
192 BDI_SHOW(min_ratio, bdi->min_ratio)
194 static ssize_t max_ratio_store(struct device *dev,
195 struct device_attribute *attr, const char *buf, size_t count)
197 struct backing_dev_info *bdi = dev_get_drvdata(dev);
200 ssize_t ret = -EINVAL;
202 ratio = simple_strtoul(buf, &end, 10);
203 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
204 ret = bdi_set_max_ratio(bdi, ratio);
210 BDI_SHOW(max_ratio, bdi->max_ratio)
212 #define __ATTR_RW(attr) __ATTR(attr, 0644, attr##_show, attr##_store)
214 static struct device_attribute bdi_dev_attrs[] = {
215 __ATTR_RW(read_ahead_kb),
216 __ATTR_RW(min_ratio),
217 __ATTR_RW(max_ratio),
221 static __init int bdi_class_init(void)
223 bdi_class = class_create(THIS_MODULE, "bdi");
224 bdi_class->dev_attrs = bdi_dev_attrs;
228 postcore_initcall(bdi_class_init);
230 static int __init default_bdi_init(void)
234 sync_supers_tsk = kthread_run(bdi_sync_supers, NULL, "sync_supers");
235 BUG_ON(IS_ERR(sync_supers_tsk));
237 init_timer(&sync_supers_timer);
238 setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0);
241 err = bdi_init(&default_backing_dev_info);
243 bdi_register(&default_backing_dev_info, NULL, "default");
247 subsys_initcall(default_bdi_init);
249 static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi)
251 memset(wb, 0, sizeof(*wb));
254 wb->last_old_flush = jiffies;
255 INIT_LIST_HEAD(&wb->b_dirty);
256 INIT_LIST_HEAD(&wb->b_io);
257 INIT_LIST_HEAD(&wb->b_more_io);
260 static void bdi_task_init(struct backing_dev_info *bdi,
261 struct bdi_writeback *wb)
263 struct task_struct *tsk = current;
265 spin_lock(&bdi->wb_lock);
266 list_add_tail_rcu(&wb->list, &bdi->wb_list);
267 spin_unlock(&bdi->wb_lock);
269 tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
273 * Our parent may run at a different priority, just set us to normal
275 set_user_nice(tsk, 0);
278 static int bdi_start_fn(void *ptr)
280 struct bdi_writeback *wb = ptr;
281 struct backing_dev_info *bdi = wb->bdi;
285 * Add us to the active bdi_list
287 spin_lock(&bdi_lock);
288 list_add(&bdi->bdi_list, &bdi_list);
289 spin_unlock(&bdi_lock);
291 bdi_task_init(bdi, wb);
294 * Clear pending bit and wakeup anybody waiting to tear us down
296 clear_bit(BDI_pending, &bdi->state);
297 smp_mb__after_clear_bit();
298 wake_up_bit(&bdi->state, BDI_pending);
300 ret = bdi_writeback_task(wb);
303 * Remove us from the list
305 spin_lock(&bdi->wb_lock);
306 list_del_rcu(&wb->list);
307 spin_unlock(&bdi->wb_lock);
310 * Flush any work that raced with us exiting. No new work
311 * will be added, since this bdi isn't discoverable anymore.
313 if (!list_empty(&bdi->work_list))
314 wb_do_writeback(wb, 1);
320 int bdi_has_dirty_io(struct backing_dev_info *bdi)
322 return wb_has_dirty_io(&bdi->wb);
325 static void bdi_flush_io(struct backing_dev_info *bdi)
327 struct writeback_control wbc = {
329 .sync_mode = WB_SYNC_NONE,
330 .older_than_this = NULL,
335 writeback_inodes_wbc(&wbc);
339 * kupdated() used to do this. We cannot do it from the bdi_forker_task()
340 * or we risk deadlocking on ->s_umount. The longer term solution would be
341 * to implement sync_supers_bdi() or similar and simply do it from the
342 * bdi writeback tasks individually.
344 static int bdi_sync_supers(void *unused)
346 set_user_nice(current, 0);
348 while (!kthread_should_stop()) {
349 set_current_state(TASK_INTERRUPTIBLE);
353 * Do this periodically, like kupdated() did before.
361 static void arm_supers_timer(void)
365 next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies;
366 mod_timer(&sync_supers_timer, round_jiffies_up(next));
369 static void sync_supers_timer_fn(unsigned long unused)
371 wake_up_process(sync_supers_tsk);
375 static int bdi_forker_task(void *ptr)
377 struct bdi_writeback *me = ptr;
379 bdi_task_init(me->bdi, me);
382 struct backing_dev_info *bdi, *tmp;
383 struct bdi_writeback *wb;
386 * Temporary measure, we want to make sure we don't see
387 * dirty data on the default backing_dev_info
389 if (wb_has_dirty_io(me) || !list_empty(&me->bdi->work_list))
390 wb_do_writeback(me, 0);
392 spin_lock(&bdi_lock);
395 * Check if any existing bdi's have dirty data without
396 * a thread registered. If so, set that up.
398 list_for_each_entry_safe(bdi, tmp, &bdi_list, bdi_list) {
401 if (list_empty(&bdi->work_list) &&
402 !bdi_has_dirty_io(bdi))
405 bdi_add_default_flusher_task(bdi);
408 set_current_state(TASK_INTERRUPTIBLE);
410 if (list_empty(&bdi_pending_list)) {
413 spin_unlock(&bdi_lock);
414 wait = msecs_to_jiffies(dirty_writeback_interval * 10);
415 schedule_timeout(wait);
420 __set_current_state(TASK_RUNNING);
423 * This is our real job - check for pending entries in
424 * bdi_pending_list, and create the tasks that got added
426 bdi = list_entry(bdi_pending_list.next, struct backing_dev_info,
428 list_del_init(&bdi->bdi_list);
429 spin_unlock(&bdi_lock);
432 wb->task = kthread_run(bdi_start_fn, wb, "flush-%s",
435 * If task creation fails, then readd the bdi to
436 * the pending list and force writeout of the bdi
437 * from this forker thread. That will free some memory
438 * and we can try again.
440 if (IS_ERR(wb->task)) {
444 * Add this 'bdi' to the back, so we get
445 * a chance to flush other bdi's to free
448 spin_lock(&bdi_lock);
449 list_add_tail(&bdi->bdi_list, &bdi_pending_list);
450 spin_unlock(&bdi_lock);
460 * Add the default flusher task that gets created for any bdi
461 * that has dirty data pending writeout
463 void static bdi_add_default_flusher_task(struct backing_dev_info *bdi)
465 if (!bdi_cap_writeback_dirty(bdi))
468 if (WARN_ON(!test_bit(BDI_registered, &bdi->state))) {
469 printk(KERN_ERR "bdi %p/%s is not registered!\n",
475 * Check with the helper whether to proceed adding a task. Will only
476 * abort if we two or more simultanous calls to
477 * bdi_add_default_flusher_task() occured, further additions will block
478 * waiting for previous additions to finish.
480 if (!test_and_set_bit(BDI_pending, &bdi->state)) {
481 list_move_tail(&bdi->bdi_list, &bdi_pending_list);
484 * We are now on the pending list, wake up bdi_forker_task()
485 * to finish the job and add us back to the active bdi_list
487 wake_up_process(default_backing_dev_info.wb.task);
491 int bdi_register(struct backing_dev_info *bdi, struct device *parent,
492 const char *fmt, ...)
498 if (bdi->dev) /* The driver needs to use separate queues per device */
502 dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
509 spin_lock(&bdi_lock);
510 list_add_tail(&bdi->bdi_list, &bdi_list);
511 spin_unlock(&bdi_lock);
516 * Just start the forker thread for our default backing_dev_info,
517 * and add other bdi's to the list. They will get a thread created
518 * on-demand when they need it.
520 if (bdi_cap_flush_forker(bdi)) {
521 struct bdi_writeback *wb = &bdi->wb;
523 wb->task = kthread_run(bdi_forker_task, wb, "bdi-%s",
525 if (IS_ERR(wb->task)) {
529 spin_lock(&bdi_lock);
530 list_del(&bdi->bdi_list);
531 spin_unlock(&bdi_lock);
536 bdi_debug_register(bdi, dev_name(dev));
537 set_bit(BDI_registered, &bdi->state);
541 EXPORT_SYMBOL(bdi_register);
543 int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
545 return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev));
547 EXPORT_SYMBOL(bdi_register_dev);
550 * Remove bdi from the global list and shutdown any threads we have running
552 static void bdi_wb_shutdown(struct backing_dev_info *bdi)
554 struct bdi_writeback *wb;
556 if (!bdi_cap_writeback_dirty(bdi))
560 * If setup is pending, wait for that to complete first
562 wait_on_bit(&bdi->state, BDI_pending, bdi_sched_wait,
563 TASK_UNINTERRUPTIBLE);
566 * Make sure nobody finds us on the bdi_list anymore
568 spin_lock(&bdi_lock);
569 list_del(&bdi->bdi_list);
570 spin_unlock(&bdi_lock);
573 * Finally, kill the kernel threads. We don't need to be RCU
574 * safe anymore, since the bdi is gone from visibility.
576 list_for_each_entry(wb, &bdi->wb_list, list)
577 kthread_stop(wb->task);
580 void bdi_unregister(struct backing_dev_info *bdi)
583 if (!bdi_cap_flush_forker(bdi))
584 bdi_wb_shutdown(bdi);
585 bdi_debug_unregister(bdi);
586 device_unregister(bdi->dev);
590 EXPORT_SYMBOL(bdi_unregister);
592 int bdi_init(struct backing_dev_info *bdi)
599 bdi->max_ratio = 100;
600 bdi->max_prop_frac = PROP_FRAC_BASE;
601 spin_lock_init(&bdi->wb_lock);
602 INIT_LIST_HEAD(&bdi->bdi_list);
603 INIT_LIST_HEAD(&bdi->wb_list);
604 INIT_LIST_HEAD(&bdi->work_list);
606 bdi_wb_init(&bdi->wb, bdi);
609 * Just one thread support for now, hard code mask and count
614 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) {
615 err = percpu_counter_init(&bdi->bdi_stat[i], 0);
620 bdi->dirty_exceeded = 0;
621 err = prop_local_init_percpu(&bdi->completions);
626 percpu_counter_destroy(&bdi->bdi_stat[i]);
631 EXPORT_SYMBOL(bdi_init);
633 void bdi_destroy(struct backing_dev_info *bdi)
637 WARN_ON(bdi_has_dirty_io(bdi));
641 for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
642 percpu_counter_destroy(&bdi->bdi_stat[i]);
644 prop_local_destroy_percpu(&bdi->completions);
646 EXPORT_SYMBOL(bdi_destroy);
648 static wait_queue_head_t congestion_wqh[2] = {
649 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
650 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
653 void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
656 wait_queue_head_t *wqh = &congestion_wqh[sync];
658 bit = sync ? BDI_sync_congested : BDI_async_congested;
659 clear_bit(bit, &bdi->state);
660 smp_mb__after_clear_bit();
661 if (waitqueue_active(wqh))
664 EXPORT_SYMBOL(clear_bdi_congested);
666 void set_bdi_congested(struct backing_dev_info *bdi, int sync)
670 bit = sync ? BDI_sync_congested : BDI_async_congested;
671 set_bit(bit, &bdi->state);
673 EXPORT_SYMBOL(set_bdi_congested);
676 * congestion_wait - wait for a backing_dev to become uncongested
677 * @sync: SYNC or ASYNC IO
678 * @timeout: timeout in jiffies
680 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
681 * write congestion. If no backing_devs are congested then just wait for the
682 * next write to be completed.
684 long congestion_wait(int sync, long timeout)
688 wait_queue_head_t *wqh = &congestion_wqh[sync];
690 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
691 ret = io_schedule_timeout(timeout);
692 finish_wait(wqh, &wait);
695 EXPORT_SYMBOL(congestion_wait);