mm/hwpoison: fix race between soft_offline_page and unpoison_memory

[karo-tx-linux.git] / mm / compaction.c

diff --git a/mm/compaction.c b/mm/compaction.c
index 0dce7e87d771b3d63f05ce054188a07ed4e6fa48..8f64d3533990079d65e821cdd11843eaffbf9b17 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -437,6 +437,24 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
 
                if (!valid_page)
                        valid_page = page;
+
+               /*
+                * For compound pages such as THP and hugetlbfs, we can save
+                * potentially a lot of iterations if we skip them at once.
+                * The check is racy, but we can consider only valid values
+                * and the only danger is skipping too much.
+                */
+               if (PageCompound(page)) {
+                       unsigned int comp_order = compound_order(page);
+
+                       if (likely(comp_order < MAX_ORDER)) {
+                               blockpfn += (1UL << comp_order) - 1;
+                               cursor += (1UL << comp_order) - 1;
+                       }
+
+                       goto isolate_fail;
+               }
+
                if (!PageBuddy(page))
                        goto isolate_fail;
 
@@ -496,6 +514,13 @@ isolate_fail:
 
        }
 
+       /*
+        * There is a tiny chance that we have read bogus compound_order(),
+        * so be careful to not go outside of the pageblock.
+        */
+       if (unlikely(blockpfn > end_pfn))
+               blockpfn = end_pfn;
+
        trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
                                        nr_scanned, total_isolated);
 
@@ -680,6 +705,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
        /* Time to isolate some pages for migration */
        for (; low_pfn < end_pfn; low_pfn++) {
+               bool is_lru;
+
                /*
                 * Periodically drop the lock (if held) regardless of its
                 * contention, to give chance to IRQs. Abort async compaction
@@ -723,36 +750,35 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                 * It's possible to migrate LRU pages and balloon pages
                 * Skip any other type of page
                 */
-               if (!PageLRU(page)) {
+               is_lru = PageLRU(page);
+               if (!is_lru) {
                        if (unlikely(balloon_page_movable(page))) {
                                if (balloon_page_isolate(page)) {
                                        /* Successfully isolated */
                                        goto isolate_success;
                                }
                        }
-                       continue;
                }
 
                /*
-                * PageLRU is set. lru_lock normally excludes isolation
-                * splitting and collapsing (collapsing has already happened
-                * if PageLRU is set) but the lock is not necessarily taken
-                * here and it is wasteful to take it just to check transhuge.
-                * Check TransHuge without lock and skip the whole pageblock if
-                * it's either a transhuge or hugetlbfs page, as calling
-                * compound_order() without preventing THP from splitting the
-                * page underneath us may return surprising results.
+                * Regardless of being on LRU, compound pages such as THP and
+                * hugetlbfs are not to be compacted. We can potentially save
+                * a lot of iterations if we skip them at once. The check is
+                * racy, but we can consider only valid values and the only
+                * danger is skipping too much.
                 */
-               if (PageTransHuge(page)) {
-                       if (!locked)
-                               low_pfn = ALIGN(low_pfn + 1,
-                                               pageblock_nr_pages) - 1;
-                       else
-                               low_pfn += (1 << compound_order(page)) - 1;
+               if (PageCompound(page)) {
+                       unsigned int comp_order = compound_order(page);
+
+                       if (likely(comp_order < MAX_ORDER))
+                               low_pfn += (1UL << comp_order) - 1;
 
                        continue;
                }
 
+               if (!is_lru)
+                       continue;
+
                /*
                 * Migration will fail if an anonymous page is pinned in memory,
                 * so avoid taking lru_lock and isolating it unnecessarily in an
@@ -769,11 +795,17 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        if (!locked)
                                break;
 
-                       /* Recheck PageLRU and PageTransHuge under lock */
+                       /* Recheck PageLRU and PageCompound under lock */
                        if (!PageLRU(page))
                                continue;
-                       if (PageTransHuge(page)) {
-                               low_pfn += (1 << compound_order(page)) - 1;
+
+                       /*
+                        * Page become compound since the non-locked check,
+                        * and it's on LRU. It can only be a THP so the order
+                        * is safe to read and it's 0 for tail pages.
+                        */
+                       if (unlikely(PageCompound(page))) {
+                               low_pfn += (1UL << compound_order(page)) - 1;
                                continue;
                        }
                }
@@ -784,7 +816,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                if (__isolate_lru_page(page, isolate_mode) != 0)
                        continue;
 
-               VM_BUG_ON_PAGE(PageTransCompound(page), page);
+               VM_BUG_ON_PAGE(PageCompound(page), page);
 
                /* Successfully isolated */
                del_page_from_lru_list(page, lruvec, page_lru(page));