cma: fix watermark checking

* Add ALLOC_CMA alloc flag and pass it to [__]zone_watermark_ok()
  (from Minchan Kim).

* During watermark check decrease available free pages number by
  free CMA pages number if necessary (unmovable allocations cannot
  use pages from CMA areas).

Signed-off-by: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
Signed-off-by: Kyungmin Park <kyungmin.park@samsung.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9705f2d9dff096a2db4a74c4271830fc94ad39aa..ba627bff0961276d7a24430f68d5505bd6078a65 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -232,6 +232,21 @@ enum zone_watermarks {
 #define low_wmark_pages(z) (z->watermark[WMARK_LOW])
 #define high_wmark_pages(z) (z->watermark[WMARK_HIGH])
 
+/* The ALLOC_WMARK bits are used as an index to zone->watermark */
+#define ALLOC_WMARK_MIN                WMARK_MIN
+#define ALLOC_WMARK_LOW                WMARK_LOW
+#define ALLOC_WMARK_HIGH       WMARK_HIGH
+#define ALLOC_NO_WATERMARKS    0x04 /* don't check watermarks at all */
+
+/* Mask to get the watermark bits */
+#define ALLOC_WMARK_MASK       (ALLOC_NO_WATERMARKS-1)
+
+#define ALLOC_HARDER           0x10 /* try to alloc harder */
+#define ALLOC_HIGH             0x20 /* __GFP_HIGH set */
+#define ALLOC_CPUSET           0x40 /* check for correct cpuset */
+
+#define ALLOC_CMA              0x80
+
 struct per_cpu_pages {
        int count;              /* number of pages in the list */
        int high;               /* high watermark, emptying needed */

diff --git a/mm/compaction.c b/mm/compaction.c
index 0fbc6b73a522c1210cfc3b13b441e2762ec4e2e8..1f61bcbd62627f2df0097f83f0d23135b7c4139a 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -934,6 +934,7 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist,
        struct zoneref *z;
        struct zone *zone;
        int rc = COMPACT_SKIPPED;
+       int alloc_flags = 0;
 
        /* Check if the GFP flags allow compaction */
        if (!order || !may_enter_fs || !may_perform_io)
@@ -941,6 +942,10 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist,
 
        count_vm_event(COMPACTSTALL);
 
+#ifdef CONFIG_CMA
+       if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
+               alloc_flags |= ALLOC_CMA;
+#endif
        /* Compact each zone in the list */
        for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx,
                                                                nodemask) {
@@ -951,7 +956,8 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist,
                rc = max(status, rc);
 
                /* If a normal allocation would succeed, stop compacting */
-               if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0))
+               if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0,
+                                     alloc_flags))
                        break;
        }
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 36c3460f7c5b33e708fa17fd231dea8ed5106f6d..c3fc97e19d316198441d422d6e781af04b074c26 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1540,19 +1540,6 @@ failed:
        return NULL;
 }
 
-/* The ALLOC_WMARK bits are used as an index to zone->watermark */
-#define ALLOC_WMARK_MIN                WMARK_MIN
-#define ALLOC_WMARK_LOW                WMARK_LOW
-#define ALLOC_WMARK_HIGH       WMARK_HIGH
-#define ALLOC_NO_WATERMARKS    0x04 /* don't check watermarks at all */
-
-/* Mask to get the watermark bits */
-#define ALLOC_WMARK_MASK       (ALLOC_NO_WATERMARKS-1)
-
-#define ALLOC_HARDER           0x10 /* try to alloc harder */
-#define ALLOC_HIGH             0x20 /* __GFP_HIGH set */
-#define ALLOC_CPUSET           0x40 /* check for correct cpuset */
-
 #ifdef CONFIG_FAIL_PAGE_ALLOC
 
 static struct {
@@ -1647,7 +1634,10 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
                min -= min / 2;
        if (alloc_flags & ALLOC_HARDER)
                min -= min / 4;
-
+#ifdef CONFIG_CMA
+       if (!(alloc_flags & ALLOC_CMA))
+               free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
+#endif
        if (free_pages <= min + lowmem_reserve)
                return false;
        for (o = 0; o < order; o++) {
@@ -2361,7 +2351,10 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
                                 unlikely(test_thread_flag(TIF_MEMDIE))))
                        alloc_flags |= ALLOC_NO_WATERMARKS;
        }
-
+#ifdef CONFIG_CMA
+       if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
+               alloc_flags |= ALLOC_CMA;
+#endif
        return alloc_flags;
 }
 
@@ -2586,6 +2579,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
        struct page *page = NULL;
        int migratetype = allocflags_to_migratetype(gfp_mask);
        unsigned int cpuset_mems_cookie;
+       int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET;
 
        gfp_mask &= gfp_allowed_mask;
 
@@ -2614,9 +2608,13 @@ retry_cpuset:
        if (!preferred_zone)
                goto out;
 
+#ifdef CONFIG_CMA
+       if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
+               alloc_flags |= ALLOC_CMA;
+#endif
        /* First allocation attempt */
        page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
-                       zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET,
+                       zonelist, high_zoneidx, alloc_flags,
                        preferred_zone, migratetype);
        if (unlikely(!page))
                page = __alloc_pages_slowpath(gfp_mask, order,