From: David Howells Date: Fri, 10 May 2013 18:50:25 +0000 (+0100) Subject: FS-Cache: Don't sleep in page release if __GFP_FS is not set X-Git-Tag: next-20130607~90^2~7 X-Git-Url: https://git.karo-electronics.de/?a=commitdiff_plain;h=6627f07780484ee058f668b1d0fbeab129ca7911;p=karo-tx-linux.git FS-Cache: Don't sleep in page release if __GFP_FS is not set Don't sleep in __fscache_maybe_release_page() if __GFP_FS is not set. This goes some way towards mitigating fscache deadlocking against ext4 by way of the allocator, eg: INFO: task flush-8:0:24427 blocked for more than 120 seconds. "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. flush-8:0 D ffff88003e2b9fd8 0 24427 2 0x00000000 ffff88003e2b9138 0000000000000046 ffff880012e3a040 ffff88003e2b9fd8 0000000000011c80 ffff88003e2b9fd8 ffffffff81a10400 ffff880012e3a040 0000000000000002 ffff880012e3a040 ffff88003e2b9098 ffffffff8106dcf5 Call Trace: [] ? __lock_is_held+0x31/0x53 [] ? radix_tree_lookup_element+0xf4/0x12a [] schedule+0x60/0x62 [] __fscache_wait_on_page_write+0x8b/0xa5 [fscache] [] ? __init_waitqueue_head+0x4d/0x4d [] __fscache_maybe_release_page+0x30c/0x324 [fscache] [] ? __fscache_maybe_release_page+0x6c/0x324 [fscache] [] ? trace_hardirqs_on_caller+0x114/0x170 [] nfs_fscache_release_page+0x68/0x94 [nfs] [] nfs_release_page+0x7e/0x86 [nfs] [] try_to_release_page+0x32/0x3b [] shrink_page_list+0x535/0x71a [] ? trace_hardirqs_on_caller+0x114/0x170 [] shrink_inactive_list+0x20a/0x2dd [] ? mark_held_locks+0xbe/0xea [] shrink_lruvec+0x34c/0x3eb [] do_try_to_free_pages+0xcf/0x355 [] try_to_free_pages+0x9a/0xa1 [] __alloc_pages_nodemask+0x494/0x6f7 [] kmem_getpages+0x58/0x155 [] fallback_alloc+0x120/0x1f3 [] ? trace_hardirqs_off+0xd/0xf [] ____cache_alloc_node+0x177/0x186 [] ? ext4_init_io_end+0x1c/0x37 [] kmem_cache_alloc+0xf1/0x176 [] ? test_set_page_writeback+0x101/0x113 [] ext4_init_io_end+0x1c/0x37 [] ext4_bio_write_page+0x20f/0x3af [] mpage_da_submit_io+0x26e/0x2f6 [] ? __find_get_block_slow+0x38/0x133 [] mpage_da_map_and_submit+0x3a7/0x3bd [] ext4_da_writepages+0x30d/0x426 [] do_writepages+0x1c/0x2a [] __writeback_single_inode+0x3e/0xe5 [] writeback_sb_inodes+0x1bd/0x2f4 [] __writeback_inodes_wb+0x6f/0xb4 [] wb_writeback+0x101/0x195 [] ? trace_hardirqs_on_caller+0x114/0x170 [] ? wb_do_writeback+0xaa/0x173 [] wb_do_writeback+0x4a/0x173 [] ? trace_hardirqs_on+0xd/0xf [] ? del_timer+0x4b/0x5b [] bdi_writeback_thread+0x6d/0x147 [] ? wb_do_writeback+0x173/0x173 [] kthread+0xd0/0xd8 [] ? _raw_spin_unlock_irq+0x29/0x3e [] ? __init_kthread_worker+0x55/0x55 [] ret_from_fork+0x7c/0xb0 [] ? __init_kthread_worker+0x55/0x55 2 locks held by flush-8:0/24427: #0: (&type->s_umount_key#41){.+.+..}, at: [] grab_super_passive+0x4c/0x76 #1: (jbd2_handle){+.+...}, at: [] start_this_handle+0x475/0x4ea The problem here is that another thread, which is attempting to write the to-be-stored NFS page to the on-ext4 cache file is waiting for the journal lock, eg: INFO: task kworker/u:2:24437 blocked for more than 120 seconds. "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. kworker/u:2 D ffff880039589768 0 24437 2 0x00000000 ffff8800395896d8 0000000000000046 ffff8800283bf040 ffff880039589fd8 0000000000011c80 ffff880039589fd8 ffff880039f0b040 ffff8800283bf040 0000000000000006 ffff8800283bf6b8 ffff880039589658 ffffffff81071a13 Call Trace: [] ? mark_held_locks+0xbe/0xea [] ? _raw_spin_unlock_irqrestore+0x3a/0x50 [] ? trace_hardirqs_on_caller+0x114/0x170 [] ? trace_hardirqs_on+0xd/0xf [] schedule+0x60/0x62 [] start_this_handle+0x317/0x4ea [] ? __init_waitqueue_head+0x4d/0x4d [] jbd2__journal_start+0xb3/0x12e [] __ext4_journal_start_sb+0xb2/0xc6 [] ext4_da_write_begin+0x109/0x233 [] generic_file_buffered_write+0x11a/0x264 [] ? __mark_inode_dirty+0x2d/0x1ee [] __generic_file_aio_write+0x2a5/0x2d5 [] generic_file_aio_write+0x6f/0xd0 [] ext4_file_write+0x38c/0x3c4 [] do_sync_write+0x91/0xd1 [] cachefiles_write_page+0x26f/0x310 [cachefiles] [] fscache_write_op+0x21e/0x37a [fscache] [] ? _raw_spin_unlock_irq+0x29/0x3e [] fscache_op_work_func+0x78/0xd7 [fscache] [] process_one_work+0x232/0x3a8 [] ? process_one_work+0x1d7/0x3a8 [] worker_thread+0x214/0x303 [] ? manage_workers+0x245/0x245 [] kthread+0xd0/0xd8 [] ? _raw_spin_unlock_irq+0x29/0x3e [] ? __init_kthread_worker+0x55/0x55 [] ret_from_fork+0x7c/0xb0 [] ? __init_kthread_worker+0x55/0x55 4 locks held by kworker/u:2/24437: #0: (fscache_operation){.+.+.+}, at: [] process_one_work+0x1d7/0x3a8 #1: ((&op->work)){+.+.+.}, at: [] process_one_work+0x1d7/0x3a8 #2: (sb_writers#14){.+.+.+}, at: [] generic_file_aio_write+0x51/0xd0 #3: (&sb->s_type->i_mutex_key#19){+.+.+.}, at: [] generic_file_aio_write+0x5b/0x fscache already tries to cancel pending stores, but it can't cancel a write for which I/O is already in progress. An alternative would be to accept writing garbage to the cache under extreme circumstances and to kill the afflicted cache object if we have to do this. However, we really need to know how strapped the allocator is before deciding to do that. Signed-off-by: David Howells Tested-By: Milosz Tanski --- diff --git a/fs/fscache/page.c b/fs/fscache/page.c index 4882c806253f..42f8f2d8a197 100644 --- a/fs/fscache/page.c +++ b/fs/fscache/page.c @@ -109,7 +109,7 @@ page_busy: * allocator as the work threads writing to the cache may all end up * sleeping on memory allocation, so we may need to impose a timeout * too. */ - if (!(gfp & __GFP_WAIT)) { + if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) { fscache_stat(&fscache_n_store_vmscan_busy); return false; }