mm: vmscan: fix do_try_to_free_pages() livelock

This patch is based on KOSAKI's work and I add a little more description,
please refer https://lkml.org/lkml/2012/6/14/74.

Currently, I found system can enter a state that there are lots of free
pages in a zone but only order-0 and order-1 pages which means the zone is
heavily fragmented, then high order allocation could make direct reclaim
path's long stall(ex, 60 seconds) especially in no swap and no compaciton
enviroment.  This problem happened on v3.4, but it seems issue still lives
in current tree, the reason is do_try_to_free_pages enter live lock:

kswapd will go to sleep if the zones have been fully scanned and are still
not balanced.  As kswapd thinks there's little point trying all over again
to avoid infinite loop.  Instead it changes order from high-order to
0-order because kswapd think order-0 is the most important.  Look at
73ce02e9 in detail.  If watermarks are ok, kswapd will go back to sleep
and may leave zone->all_unreclaimable =3D 0.  It assume high-order users
can still perform direct reclaim if they wish.

Direct reclaim continue to reclaim for a high order which is not a
COSTLY_ORDER without oom-killer until kswapd turn on
zone->all_unreclaimble= .  This is because to avoid too early oom-kill.
So it means direct_reclaim depends on kswapd to break this loop.

In worst case, direct-reclaim may continue to page reclaim forever when
kswapd sleeps forever until someone like watchdog detect and finally kill
the process.  As described in:
http://thread.gmane.org/gmane.linux.kernel.mm/103737

We can't turn on zone->all_unreclaimable from direct reclaim path because
direct reclaim path don't take any lock and this way is racy.  Thus this
patch removes zone->all_unreclaimable field completely and recalculates
zone reclaimable state every time.

Note: we can't take the idea that direct-reclaim see zone->pages_scanned
directly and kswapd continue to use zone->all_unreclaimable.  Because, it
is racy.  commit 929bea7c71 (vmscan: all_unreclaimable() use
zone->all_unreclaimable as a name) describes the detail.

[akpm@linux-foundation.org: uninline zone_reclaimable_pages() and zone_reclaimable()]
Cc: Aaditya Kumar <aaditya.kumar.30@gmail.com>
Cc: Ying Han <yinghan@google.com>
Cc: Nick Piggin <npiggin@gmail.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Bob Liu <lliubbo@gmail.com>
Cc: Neil Zhang <zhangwm@marvell.com>
Cc: Russell King - ARM Linux <linux@arm.linux.org.uk>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Acked-by: Minchan Kim <minchan@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Lisa Du <cldu@marvell.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 1397ccf81e91f16d937c3f0b2e7d361104dce5f1..cf55945c83fb91d18bdaef4f0d893d6a94fa9034 100644 (file)
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -2,6 +2,7 @@
 #define LINUX_MM_INLINE_H
 
 #include <linux/huge_mm.h>
+#include <linux/swap.h>
 
 /**
  * page_is_file_cache - should the page be on a file LRU or anon LRU?

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index ac1ea796ec0f8d87da37b8edd69d98edcf7692fc..bd791e452ad7a96329d883afba92ffbdf7bbc9bc 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -353,7 +353,6 @@ struct zone {
         * free areas of different sizes
         */
        spinlock_t              lock;
-       int                     all_unreclaimable; /* All pages pinned */
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
        /* Set to true when the PG_migrate_skip bits should be cleared */
        bool                    compact_blockskip_flush;

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 502767f4e4d48556ecab6eaf65b0f2049b6e67ca..e4b948080d20e7a537c7a83da17b8b5b7fec0008 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -143,7 +143,6 @@ static inline unsigned long zone_page_state_snapshot(struct zone *zone,
 }
 
 extern unsigned long global_reclaimable_pages(void);
-extern unsigned long zone_reclaimable_pages(struct zone *zone);
 
 #ifdef CONFIG_NUMA
 /*

diff --git a/mm/internal.h b/mm/internal.h
index 4390ac6c106e6124d653f18948ec296173527521..684f7aa9692aecc9e002a3095468a23c5c5c4ed4 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -85,6 +85,8 @@ extern unsigned long highest_memmap_pfn;
  */
 extern int isolate_lru_page(struct page *page);
 extern void putback_lru_page(struct page *page);
+extern unsigned long zone_reclaimable_pages(struct zone *zone);
+extern bool zone_reclaimable(struct zone *zone);
 
 /*
  * in mm/rmap.c:

diff --git a/mm/migrate.c b/mm/migrate.c
index 61f14a1923fd6044f7435498fd7cc9d0a0eebc9f..b7ded7eafe3a00c998855944270a65c541ebea80 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1471,7 +1471,7 @@ static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
                if (!populated_zone(zone))
                        continue;
 
-               if (zone->all_unreclaimable)
+               if (!zone_reclaimable(zone))
                        continue;
 
                /* Avoid waking kswapd by allocating pages_to_migrate pages. */

diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index d374b29296ddd4c18c9609c8e03940d6debed1cc..3750431b3cd83d295d0961b35602b227fb20c7cc 100644 (file)
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -36,8 +36,11 @@
 #include <linux/pagevec.h>
 #include <linux/timer.h>
 #include <linux/sched/rt.h>
+#include <linux/mm_inline.h>
 #include <trace/events/writeback.h>
 
+#include "internal.h"
+
 /*
  * Sleep at most 200ms at a time in balance_dirty_pages().
  */

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 7b1b706a1ffa2b050e05ab66a7cd9d9b80c726c4..ff2782576e394849735ace584866de62adb73ba9 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -56,6 +56,7 @@
 #include <linux/ftrace_event.h>
 #include <linux/memcontrol.h>
 #include <linux/prefetch.h>
+#include <linux/mm_inline.h>
 #include <linux/migrate.h>
 #include <linux/page-debug-flags.h>
 #include <linux/hugetlb.h>
@@ -647,7 +648,6 @@ static void free_pcppages_bulk(struct zone *zone, int count,
        int to_free = count;
 
        spin_lock(&zone->lock);
-       zone->all_unreclaimable = 0;
        zone->pages_scanned = 0;
 
        while (to_free) {
@@ -696,7 +696,6 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
                                int migratetype)
 {
        spin_lock(&zone->lock);
-       zone->all_unreclaimable = 0;
        zone->pages_scanned = 0;
 
        __free_one_page(page, zone, order, migratetype);
@@ -3164,7 +3163,7 @@ void show_free_areas(unsigned int filter)
                        K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
                        K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
                        zone->pages_scanned,
-                       (zone->all_unreclaimable ? "yes" : "no")
+                       (!zone_reclaimable(zone) ? "yes" : "no")
                        );
                printk("lowmem_reserve[]:");
                for (i = 0; i < MAX_NR_ZONES; i++)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 44c072a7cba2b9e696fdc9bdaae465da8421b868..fe715daeb8bc8288b0991be836a3dbcd60cebc85 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -146,6 +146,25 @@ static bool global_reclaim(struct scan_control *sc)
 }
 #endif
 
+unsigned long zone_reclaimable_pages(struct zone *zone)
+{
+       int nr;
+
+       nr = zone_page_state(zone, NR_ACTIVE_FILE) +
+            zone_page_state(zone, NR_INACTIVE_FILE);
+
+       if (get_nr_swap_pages() > 0)
+               nr += zone_page_state(zone, NR_ACTIVE_ANON) +
+                     zone_page_state(zone, NR_INACTIVE_ANON);
+
+       return nr;
+}
+
+bool zone_reclaimable(struct zone *zone)
+{
+       return zone->pages_scanned < zone_reclaimable_pages(zone) * 6;
+}
+
 static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru)
 {
        if (!mem_cgroup_disabled())
@@ -1789,7 +1808,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
         * latencies, so it's better to scan a minimum amount there as
         * well.
         */
-       if (current_is_kswapd() && zone->all_unreclaimable)
+       if (current_is_kswapd() && !zone_reclaimable(zone))
                force_scan = true;
        if (!global_reclaim(sc))
                force_scan = true;
@@ -2244,8 +2263,8 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
                if (global_reclaim(sc)) {
                        if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
                                continue;
-                       if (zone->all_unreclaimable &&
-                                       sc->priority != DEF_PRIORITY)
+                       if (sc->priority != DEF_PRIORITY &&
+                           !zone_reclaimable(zone))
                                continue;       /* Let kswapd poll it */
                        if (IS_ENABLED(CONFIG_COMPACTION)) {
                                /*
@@ -2283,11 +2302,6 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
        return aborted_reclaim;
 }
 
-static bool zone_reclaimable(struct zone *zone)
-{
-       return zone->pages_scanned < zone_reclaimable_pages(zone) * 6;
-}
-
 /* All zones in zonelist are unreclaimable? */
 static bool all_unreclaimable(struct zonelist *zonelist,
                struct scan_control *sc)
@@ -2301,7 +2315,7 @@ static bool all_unreclaimable(struct zonelist *zonelist,
                        continue;
                if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
                        continue;
-               if (!zone->all_unreclaimable)
+               if (zone_reclaimable(zone))
                        return false;
        }
 
@@ -2712,7 +2726,7 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
                 * DEF_PRIORITY. Effectively, it considers them balanced so
                 * they must be considered balanced here as well!
                 */
-               if (zone->all_unreclaimable) {
+               if (!zone_reclaimable(zone)) {
                        balanced_pages += zone->managed_pages;
                        continue;
                }
@@ -2773,7 +2787,6 @@ static bool kswapd_shrink_zone(struct zone *zone,
                               unsigned long lru_pages,
                               unsigned long *nr_attempted)
 {
-       unsigned long nr_slab;
        int testorder = sc->order;
        unsigned long balance_gap;
        struct reclaim_state *reclaim_state = current->reclaim_state;
@@ -2818,15 +2831,12 @@ static bool kswapd_shrink_zone(struct zone *zone,
        shrink_zone(zone, sc);
 
        reclaim_state->reclaimed_slab = 0;
-       nr_slab = shrink_slab(&shrink, sc->nr_scanned, lru_pages);
+       shrink_slab(&shrink, sc->nr_scanned, lru_pages);
        sc->nr_reclaimed += reclaim_state->reclaimed_slab;
 
        /* Account for the number of pages attempted to reclaim */
        *nr_attempted += sc->nr_to_reclaim;
 
-       if (nr_slab == 0 && !zone_reclaimable(zone))
-               zone->all_unreclaimable = 1;
-
        zone_clear_flag(zone, ZONE_WRITEBACK);
 
        /*
@@ -2835,7 +2845,7 @@ static bool kswapd_shrink_zone(struct zone *zone,
         * BDIs but as pressure is relieved, speculatively avoid congestion
         * waits.
         */
-       if (!zone->all_unreclaimable &&
+       if (zone_reclaimable(zone) &&
            zone_balanced(zone, testorder, 0, classzone_idx)) {
                zone_clear_flag(zone, ZONE_CONGESTED);
                zone_clear_flag(zone, ZONE_TAIL_LRU_DIRTY);
@@ -2901,8 +2911,8 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
                        if (!populated_zone(zone))
                                continue;
 
-                       if (zone->all_unreclaimable &&
-                           sc.priority != DEF_PRIORITY)
+                       if (sc.priority != DEF_PRIORITY &&
+                           !zone_reclaimable(zone))
                                continue;
 
                        /*
@@ -2980,8 +2990,8 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
                        if (!populated_zone(zone))
                                continue;
 
-                       if (zone->all_unreclaimable &&
-                           sc.priority != DEF_PRIORITY)
+                       if (sc.priority != DEF_PRIORITY &&
+                           !zone_reclaimable(zone))
                                continue;
 
                        sc.nr_scanned = 0;
@@ -3265,20 +3275,6 @@ unsigned long global_reclaimable_pages(void)
        return nr;
 }
 
-unsigned long zone_reclaimable_pages(struct zone *zone)
-{
-       int nr;
-
-       nr = zone_page_state(zone, NR_ACTIVE_FILE) +
-            zone_page_state(zone, NR_INACTIVE_FILE);
-
-       if (get_nr_swap_pages() > 0)
-               nr += zone_page_state(zone, NR_ACTIVE_ANON) +
-                     zone_page_state(zone, NR_INACTIVE_ANON);
-
-       return nr;
-}
-
 #ifdef CONFIG_HIBERNATION
 /*
  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
@@ -3576,7 +3572,7 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
            zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
                return ZONE_RECLAIM_FULL;
 
-       if (zone->all_unreclaimable)
+       if (!zone_reclaimable(zone))
                return ZONE_RECLAIM_FULL;
 
        /*

diff --git a/mm/vmstat.c b/mm/vmstat.c
index d57a09143bf9c9ba2358e8b6604fa08a32f7c622..9bb314577911f50c06848373d273b3b993373858 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -19,6 +19,9 @@
 #include <linux/math64.h>
 #include <linux/writeback.h>
 #include <linux/compaction.h>
+#include <linux/mm_inline.h>
+
+#include "internal.h"
 
 #ifdef CONFIG_VM_EVENT_COUNTERS
 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
@@ -1088,7 +1091,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
                   "\n  all_unreclaimable: %u"
                   "\n  start_pfn:         %lu"
                   "\n  inactive_ratio:    %u",
-                  zone->all_unreclaimable,
+                  !zone_reclaimable(zone),
                   zone->zone_start_pfn,
                   zone->inactive_ratio);
        seq_putc(m, '\n');