powerpc/debug: Add missing warn flag to WARN_ON's non-builtin path

[karo-tx-linux.git] / mm / compaction.c

diff --git a/mm/compaction.c b/mm/compaction.c
index 09c5282ebdd2812fbb758255194d9063cf5dcf97..613c59e928cb5b07075c197126e01b9cdafff992 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -89,11 +89,6 @@ static void map_pages(struct list_head *list)
        list_splice(&tmp_list, list);
 }
 
-static inline bool migrate_async_suitable(int migratetype)
-{
-       return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
-}
-
 #ifdef CONFIG_COMPACTION
 
 int PageMovable(struct page *page)
@@ -988,6 +983,22 @@ isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
 #endif /* CONFIG_COMPACTION || CONFIG_CMA */
 #ifdef CONFIG_COMPACTION
 
+static bool suitable_migration_source(struct compact_control *cc,
+                                                       struct page *page)
+{
+       int block_mt;
+
+       if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
+               return true;
+
+       block_mt = get_pageblock_migratetype(page);
+
+       if (cc->migratetype == MIGRATE_MOVABLE)
+               return is_migrate_movable(block_mt);
+       else
+               return block_mt == cc->migratetype;
+}
+
 /* Returns true if the page is within a block suitable for migration to */
 static bool suitable_migration_target(struct compact_control *cc,
                                                        struct page *page)
@@ -1007,7 +1018,7 @@ static bool suitable_migration_target(struct compact_control *cc,
                return true;
 
        /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
-       if (migrate_async_suitable(get_pageblock_migratetype(page)))
+       if (is_migrate_movable(get_pageblock_migratetype(page)))
                return true;
 
        /* Otherwise skip the block */
@@ -1242,8 +1253,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
                 * Async compaction is optimistic to see if the minimum amount
                 * of work satisfies the allocation.
                 */
-               if (cc->mode == MIGRATE_ASYNC &&
-                   !migrate_async_suitable(get_pageblock_migratetype(page)))
+               if (!suitable_migration_source(cc, page))
                        continue;
 
                /* Perform the isolation */
@@ -1276,11 +1286,11 @@ static inline bool is_via_compact_memory(int order)
        return order == -1;
 }
 
-static enum compact_result __compact_finished(struct zone *zone, struct compact_control *cc,
-                           const int migratetype)
+static enum compact_result __compact_finished(struct zone *zone,
+                                               struct compact_control *cc)
 {
        unsigned int order;
-       unsigned long watermark;
+       const int migratetype = cc->migratetype;
 
        if (cc->contended || fatal_signal_pending(current))
                return COMPACT_CONTENDED;
@@ -1308,12 +1318,16 @@ static enum compact_result __compact_finished(struct zone *zone, struct compact_
        if (is_via_compact_memory(cc->order))
                return COMPACT_CONTINUE;
 
-       /* Compaction run is not finished if the watermark is not met */
-       watermark = zone->watermark[cc->alloc_flags & ALLOC_WMARK_MASK];
-
-       if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx,
-                                                       cc->alloc_flags))
-               return COMPACT_CONTINUE;
+       if (cc->finishing_block) {
+               /*
+                * We have finished the pageblock, but better check again that
+                * we really succeeded.
+                */
+               if (IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
+                       cc->finishing_block = false;
+               else
+                       return COMPACT_CONTINUE;
+       }
 
        /* Direct compactor: Is a suitable page free? */
        for (order = cc->order; order < MAX_ORDER; order++) {
@@ -1335,20 +1349,40 @@ static enum compact_result __compact_finished(struct zone *zone, struct compact_
                 * other migratetype buddy lists.
                 */
                if (find_suitable_fallback(area, order, migratetype,
-                                               true, &can_steal) != -1)
-                       return COMPACT_SUCCESS;
+                                               true, &can_steal) != -1) {
+
+                       /* movable pages are OK in any pageblock */
+                       if (migratetype == MIGRATE_MOVABLE)
+                               return COMPACT_SUCCESS;
+
+                       /*
+                        * We are stealing for a non-movable allocation. Make
+                        * sure we finish compacting the current pageblock
+                        * first so it is as free as possible and we won't
+                        * have to steal another one soon. This only applies
+                        * to sync compaction, as async compaction operates
+                        * on pageblocks of the same migratetype.
+                        */
+                       if (cc->mode == MIGRATE_ASYNC ||
+                                       IS_ALIGNED(cc->migrate_pfn,
+                                                       pageblock_nr_pages)) {
+                               return COMPACT_SUCCESS;
+                       }
+
+                       cc->finishing_block = true;
+                       return COMPACT_CONTINUE;
+               }
        }
 
        return COMPACT_NO_SUITABLE_PAGE;
 }
 
 static enum compact_result compact_finished(struct zone *zone,
-                       struct compact_control *cc,
-                       const int migratetype)
+                       struct compact_control *cc)
 {
        int ret;
 
-       ret = __compact_finished(zone, cc, migratetype);
+       ret = __compact_finished(zone, cc);
        trace_mm_compaction_finished(zone, cc->order, ret);
        if (ret == COMPACT_NO_SUITABLE_PAGE)
                ret = COMPACT_CONTINUE;
@@ -1481,9 +1515,9 @@ static enum compact_result compact_zone(struct zone *zone, struct compact_contro
        enum compact_result ret;
        unsigned long start_pfn = zone->zone_start_pfn;
        unsigned long end_pfn = zone_end_pfn(zone);
-       const int migratetype = gfpflags_to_migratetype(cc->gfp_mask);
        const bool sync = cc->mode != MIGRATE_ASYNC;
 
+       cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
        ret = compaction_suitable(zone, cc->order, cc->alloc_flags,
                                                        cc->classzone_idx);
        /* Compaction is likely to fail */
@@ -1533,8 +1567,7 @@ static enum compact_result compact_zone(struct zone *zone, struct compact_contro
 
        migrate_prep_local();
 
-       while ((ret = compact_finished(zone, cc, migratetype)) ==
-                                               COMPACT_CONTINUE) {
+       while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) {
                int err;
 
                switch (isolate_migratepages(zone, cc)) {