CMA: migrate mlocked pages

Presently CMA cannot migrate mlocked pages so it ends up failing to allocate
contiguous memory space.

This patch makes mlocked pages be migrated out.  Of course, it can affect
realtime processes but in CMA usecase, contiguous memory allocation failing
is far worse than access latency to an mlocked page being variable while
CMA is running.  If someone wants to make the system realtime, he shouldn't
enable CMA because stalls can still happen at random times.

Signed-off-by: Minchan Kim <minchan@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 59067c128beee7624fe5f51cf75d63fd8a9f96aa..4d106b2163c5332b802ebda1c798ada0d0242511 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -217,6 +217,8 @@ struct lruvec {
 #define ISOLATE_UNMAPPED       ((__force isolate_mode_t)0x2)
 /* Isolate for asynchronous migration */
 #define ISOLATE_ASYNC_MIGRATE  ((__force isolate_mode_t)0x4)
+/* Isolate unevictable pages */
+#define ISOLATE_UNEVICTABLE    ((__force isolate_mode_t)0x8)
 
 /* LRU Isolation modes. */
 typedef unsigned __bitwise__ isolate_mode_t;

diff --git a/mm/compaction.c b/mm/compaction.c
index 76d8a5e76a4c2010b37a350177c7efcdd8188904..136debd13996efaa097504388181d3e5d3998814 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -468,6 +468,7 @@ static bool too_many_isolated(struct zone *zone)
  * @cc:                Compaction control structure.
  * @low_pfn:   The first PFN of the range.
  * @end_pfn:   The one-past-the-last PFN of the range.
+ * @unevictable: true if it allows to isolate unevictable pages
  *
  * Isolate all pages that can be migrated from the range specified by
  * [low_pfn, end_pfn).  Returns zero if there is a fatal signal
@@ -483,7 +484,7 @@ static bool too_many_isolated(struct zone *zone)
  */
 unsigned long
 isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
-                          unsigned long low_pfn, unsigned long end_pfn)
+               unsigned long low_pfn, unsigned long end_pfn, bool unevictable)
 {
        unsigned long last_pageblock_nr = 0, pageblock_nr;
        unsigned long nr_scanned = 0, nr_isolated = 0;
@@ -609,6 +610,9 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
                if (!cc->sync)
                        mode |= ISOLATE_ASYNC_MIGRATE;
 
+               if (unevictable)
+                       mode |= ISOLATE_UNEVICTABLE;
+
                lruvec = mem_cgroup_page_lruvec(page, zone);
 
                /* Try isolate the page */
@@ -814,7 +818,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
        }
 
        /* Perform the isolation */
-       low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn);
+       low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn, false);
        if (!low_pfn || cc->contended)
                return ISOLATE_ABORT;
 

diff --git a/mm/internal.h b/mm/internal.h
index d1e84fd6416a9eb7ca6a15cbf01eec591b39c7b1..9d5d276c11b4703720f6d9c91d1496a065ea6bd8 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -138,7 +138,7 @@ unsigned long
 isolate_freepages_range(unsigned long start_pfn, unsigned long end_pfn);
 unsigned long
 isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
-                          unsigned long low_pfn, unsigned long end_pfn);
+       unsigned long low_pfn, unsigned long end_pfn, bool unevictable);
 
 #endif
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6062f8c613743b5ec874b375485ebb7dc3cf0f49..ddc155300a67e512f047d2fd0feb746dfdcd0972 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5697,7 +5697,7 @@ static int __alloc_contig_migrate_range(unsigned long start, unsigned long end)
                if (list_empty(&cc.migratepages)) {
                        cc.nr_migratepages = 0;
                        pfn = isolate_migratepages_range(cc.zone, &cc,
-                                                        pfn, end);
+                                                        pfn, end, true);
                        if (!pfn) {
                                ret = -EINTR;
                                break;

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 8b627309dd4482ff3128d87e0f6fb2bc9173ff26..1e9aa66cb1d48891862166a784f7e1441d902b62 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1009,8 +1009,8 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode)
        if (!PageLRU(page))
                return ret;
 
-       /* Do not give back unevictable pages for compaction */
-       if (PageUnevictable(page))
+       /* Compaction can't handle unevictable pages but CMA can do */
+       if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
                return ret;
 
        ret = -EBUSY;