mm/huge_memory.c

   1 /*
   2  *  Copyright (C) 2009  Red Hat, Inc.
   3  *
   4  *  This work is licensed under the terms of the GNU GPL, version 2. See
   5  *  the COPYING file in the top-level directory.
   6  */
   7
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/mm.h>
  11 #include <linux/sched.h>
  12 #include <linux/highmem.h>
  13 #include <linux/hugetlb.h>
  14 #include <linux/mmu_notifier.h>
  15 #include <linux/rmap.h>
  16 #include <linux/swap.h>
  17 #include <linux/shrinker.h>
  18 #include <linux/mm_inline.h>
  19 #include <linux/swapops.h>
  20 #include <linux/dax.h>
  21 #include <linux/kthread.h>
  22 #include <linux/khugepaged.h>
  23 #include <linux/freezer.h>
  24 #include <linux/pfn_t.h>
  25 #include <linux/mman.h>
  26 #include <linux/pagemap.h>
  27 #include <linux/debugfs.h>
  28 #include <linux/migrate.h>
  29 #include <linux/hashtable.h>
  30 #include <linux/userfaultfd_k.h>
  31 #include <linux/page_idle.h>
  32
  33 #include <asm/tlb.h>
  34 #include <asm/pgalloc.h>
  35 #include "internal.h"
  36
  37 enum scan_result {
  38         SCAN_FAIL,
  39         SCAN_SUCCEED,
  40         SCAN_PMD_NULL,
  41         SCAN_EXCEED_NONE_PTE,
  42         SCAN_PTE_NON_PRESENT,
  43         SCAN_PAGE_RO,
  44         SCAN_NO_REFERENCED_PAGE,
  45         SCAN_PAGE_NULL,
  46         SCAN_SCAN_ABORT,
  47         SCAN_PAGE_COUNT,
  48         SCAN_PAGE_LRU,
  49         SCAN_PAGE_LOCK,
  50         SCAN_PAGE_ANON,
  51         SCAN_PAGE_COMPOUND,
  52         SCAN_ANY_PROCESS,
  53         SCAN_VMA_NULL,
  54         SCAN_VMA_CHECK,
  55         SCAN_ADDRESS_RANGE,
  56         SCAN_SWAP_CACHE_PAGE,
  57         SCAN_DEL_PAGE_LRU,
  58         SCAN_ALLOC_HUGE_PAGE_FAIL,
  59         SCAN_CGROUP_CHARGE_FAIL
  60 };
  61
  62 #define CREATE_TRACE_POINTS
  63 #include <trace/events/huge_memory.h>
  64
  65 /*
  66  * By default transparent hugepage support is disabled in order that avoid
  67  * to risk increase the memory footprint of applications without a guaranteed
  68  * benefit. When transparent hugepage support is enabled, is for all mappings,
  69  * and khugepaged scans all mappings.
  70  * Defrag is invoked by khugepaged hugepage allocations and by page faults
  71  * for all hugepage allocations.
  72  */
  73 unsigned long transparent_hugepage_flags __read_mostly =
  74 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
  75         (1<<TRANSPARENT_HUGEPAGE_FLAG)|
  76 #endif
  77 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
  78         (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
  79 #endif
  80         (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)|
  81         (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
  82         (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
  83
  84 /* default scan 8*512 pte (or vmas) every 30 second */
  85 static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8;
  86 static unsigned int khugepaged_pages_collapsed;
  87 static unsigned int khugepaged_full_scans;
  88 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
  89 /* during fragmentation poll the hugepage allocator once every minute */
  90 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
  91 static struct task_struct *khugepaged_thread __read_mostly;
  92 static DEFINE_MUTEX(khugepaged_mutex);
  93 static DEFINE_SPINLOCK(khugepaged_mm_lock);
  94 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  95 /*
  96  * default collapse hugepages if there is at least one pte mapped like
  97  * it would have happened if the vma was large enough during page
  98  * fault.
  99  */
 100 static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1;
 101
 102 static int khugepaged(void *none);
 103 static int khugepaged_slab_init(void);
 104 static void khugepaged_slab_exit(void);
 105
 106 #define MM_SLOTS_HASH_BITS 10
 107 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
 108
 109 static struct kmem_cache *mm_slot_cache __read_mostly;
 110
 111 /**
 112  * struct mm_slot - hash lookup from mm to mm_slot
 113  * @hash: hash collision list
 114  * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
 115  * @mm: the mm that this information is valid for
 116  */
 117 struct mm_slot {
 118         struct hlist_node hash;
 119         struct list_head mm_node;
 120         struct mm_struct *mm;
 121 };
 122
 123 /**
 124  * struct khugepaged_scan - cursor for scanning
 125  * @mm_head: the head of the mm list to scan
 126  * @mm_slot: the current mm_slot we are scanning
 127  * @address: the next address inside that to be scanned
 128  *
 129  * There is only the one khugepaged_scan instance of this cursor structure.
 130  */
 131 struct khugepaged_scan {
 132         struct list_head mm_head;
 133         struct mm_slot *mm_slot;
 134         unsigned long address;
 135 };
 136 static struct khugepaged_scan khugepaged_scan = {
 137         .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
 138 };
 139
 140 static DEFINE_SPINLOCK(split_queue_lock);
 141 static LIST_HEAD(split_queue);
 142 static unsigned long split_queue_len;
 143 static struct shrinker deferred_split_shrinker;
 144
 145 static void set_recommended_min_free_kbytes(void)
 146 {
 147         struct zone *zone;
 148         int nr_zones = 0;
 149         unsigned long recommended_min;
 150
 151         for_each_populated_zone(zone)
 152                 nr_zones++;
 153
 154         /* Ensure 2 pageblocks are free to assist fragmentation avoidance */
 155         recommended_min = pageblock_nr_pages * nr_zones * 2;
 156
 157         /*
 158          * Make sure that on average at least two pageblocks are almost free
 159          * of another type, one for a migratetype to fall back to and a
 160          * second to avoid subsequent fallbacks of other types There are 3
 161          * MIGRATE_TYPES we care about.
 162          */
 163         recommended_min += pageblock_nr_pages * nr_zones *
 164                            MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
 165
 166         /* don't ever allow to reserve more than 5% of the lowmem */
 167         recommended_min = min(recommended_min,
 168                               (unsigned long) nr_free_buffer_pages() / 20);
 169         recommended_min <<= (PAGE_SHIFT-10);
 170
 171         if (recommended_min > min_free_kbytes) {
 172                 if (user_min_free_kbytes >= 0)
 173                         pr_info("raising min_free_kbytes from %d to %lu "
 174                                 "to help transparent hugepage allocations\n",
 175                                 min_free_kbytes, recommended_min);
 176
 177                 min_free_kbytes = recommended_min;
 178         }
 179         setup_per_zone_wmarks();
 180 }
 181
 182 static int start_stop_khugepaged(void)
 183 {
 184         int err = 0;
 185         if (khugepaged_enabled()) {
 186                 if (!khugepaged_thread)
 187                         khugepaged_thread = kthread_run(khugepaged, NULL,
 188                                                         "khugepaged");
 189                 if (IS_ERR(khugepaged_thread)) {
 190                         pr_err("khugepaged: kthread_run(khugepaged) failed\n");
 191                         err = PTR_ERR(khugepaged_thread);
 192                         khugepaged_thread = NULL;
 193                         goto fail;
 194                 }
 195
 196                 if (!list_empty(&khugepaged_scan.mm_head))
 197                         wake_up_interruptible(&khugepaged_wait);
 198
 199                 set_recommended_min_free_kbytes();
 200         } else if (khugepaged_thread) {
 201                 kthread_stop(khugepaged_thread);
 202                 khugepaged_thread = NULL;
 203         }
 204 fail:
 205         return err;
 206 }
 207
 208 static atomic_t huge_zero_refcount;
 209 struct page *huge_zero_page __read_mostly;
 210
 211 struct page *get_huge_zero_page(void)
 212 {
 213         struct page *zero_page;
 214 retry:
 215         if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
 216                 return READ_ONCE(huge_zero_page);
 217
 218         zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
 219                         HPAGE_PMD_ORDER);
 220         if (!zero_page) {
 221                 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
 222                 return NULL;
 223         }
 224         count_vm_event(THP_ZERO_PAGE_ALLOC);
 225         preempt_disable();
 226         if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
 227                 preempt_enable();
 228                 __free_pages(zero_page, compound_order(zero_page));
 229                 goto retry;
 230         }
 231
 232         /* We take additional reference here. It will be put back by shrinker */
 233         atomic_set(&huge_zero_refcount, 2);
 234         preempt_enable();
 235         return READ_ONCE(huge_zero_page);
 236 }
 237
 238 static void put_huge_zero_page(void)
 239 {
 240         /*
 241          * Counter should never go to zero here. Only shrinker can put
 242          * last reference.
 243          */
 244         BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
 245 }
 246
 247 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
 248                                         struct shrink_control *sc)
 249 {
 250         /* we can free zero page only if last reference remains */
 251         return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
 252 }
 253
 254 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
 255                                        struct shrink_control *sc)
 256 {
 257         if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
 258                 struct page *zero_page = xchg(&huge_zero_page, NULL);
 259                 BUG_ON(zero_page == NULL);
 260                 __free_pages(zero_page, compound_order(zero_page));
 261                 return HPAGE_PMD_NR;
 262         }
 263
 264         return 0;
 265 }
 266
 267 static struct shrinker huge_zero_page_shrinker = {
 268         .count_objects = shrink_huge_zero_page_count,
 269         .scan_objects = shrink_huge_zero_page_scan,
 270         .seeks = DEFAULT_SEEKS,
 271 };
 272
 273 #ifdef CONFIG_SYSFS
 274
 275 static ssize_t double_flag_show(struct kobject *kobj,
 276                                 struct kobj_attribute *attr, char *buf,
 277                                 enum transparent_hugepage_flag enabled,
 278                                 enum transparent_hugepage_flag req_madv)
 279 {
 280         if (test_bit(enabled, &transparent_hugepage_flags)) {
 281                 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
 282                 return sprintf(buf, "[always] madvise never\n");
 283         } else if (test_bit(req_madv, &transparent_hugepage_flags))
 284                 return sprintf(buf, "always [madvise] never\n");
 285         else
 286                 return sprintf(buf, "always madvise [never]\n");
 287 }
 288 static ssize_t double_flag_store(struct kobject *kobj,
 289                                  struct kobj_attribute *attr,
 290                                  const char *buf, size_t count,
 291                                  enum transparent_hugepage_flag enabled,
 292                                  enum transparent_hugepage_flag req_madv)
 293 {
 294         if (!memcmp("always", buf,
 295                     min(sizeof("always")-1, count))) {
 296                 set_bit(enabled, &transparent_hugepage_flags);
 297                 clear_bit(req_madv, &transparent_hugepage_flags);
 298         } else if (!memcmp("madvise", buf,
 299                            min(sizeof("madvise")-1, count))) {
 300                 clear_bit(enabled, &transparent_hugepage_flags);
 301                 set_bit(req_madv, &transparent_hugepage_flags);
 302         } else if (!memcmp("never", buf,
 303                            min(sizeof("never")-1, count))) {
 304                 clear_bit(enabled, &transparent_hugepage_flags);
 305                 clear_bit(req_madv, &transparent_hugepage_flags);
 306         } else
 307                 return -EINVAL;
 308
 309         return count;
 310 }
 311
 312 static ssize_t enabled_show(struct kobject *kobj,
 313                             struct kobj_attribute *attr, char *buf)
 314 {
 315         return double_flag_show(kobj, attr, buf,
 316                                 TRANSPARENT_HUGEPAGE_FLAG,
 317                                 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
 318 }
 319 static ssize_t enabled_store(struct kobject *kobj,
 320                              struct kobj_attribute *attr,
 321                              const char *buf, size_t count)
 322 {
 323         ssize_t ret;
 324
 325         ret = double_flag_store(kobj, attr, buf, count,
 326                                 TRANSPARENT_HUGEPAGE_FLAG,
 327                                 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
 328
 329         if (ret > 0) {
 330                 int err;
 331
 332                 mutex_lock(&khugepaged_mutex);
 333                 err = start_stop_khugepaged();
 334                 mutex_unlock(&khugepaged_mutex);
 335
 336                 if (err)
 337                         ret = err;
 338         }
 339
 340         return ret;
 341 }
 342 static struct kobj_attribute enabled_attr =
 343         __ATTR(enabled, 0644, enabled_show, enabled_store);
 344
 345 static ssize_t single_flag_show(struct kobject *kobj,
 346                                 struct kobj_attribute *attr, char *buf,
 347                                 enum transparent_hugepage_flag flag)
 348 {
 349         return sprintf(buf, "%d\n",
 350                        !!test_bit(flag, &transparent_hugepage_flags));
 351 }
 352
 353 static ssize_t single_flag_store(struct kobject *kobj,
 354                                  struct kobj_attribute *attr,
 355                                  const char *buf, size_t count,
 356                                  enum transparent_hugepage_flag flag)
 357 {
 358         unsigned long value;
 359         int ret;
 360
 361         ret = kstrtoul(buf, 10, &value);
 362         if (ret < 0)
 363                 return ret;
 364         if (value > 1)
 365                 return -EINVAL;
 366
 367         if (value)
 368                 set_bit(flag, &transparent_hugepage_flags);
 369         else
 370                 clear_bit(flag, &transparent_hugepage_flags);
 371
 372         return count;
 373 }
 374
 375 /*
 376  * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
 377  * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
 378  * memory just to allocate one more hugepage.
 379  */
 380 static ssize_t defrag_show(struct kobject *kobj,
 381                            struct kobj_attribute *attr, char *buf)
 382 {
 383         return double_flag_show(kobj, attr, buf,
 384                                 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
 385                                 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
 386 }
 387 static ssize_t defrag_store(struct kobject *kobj,
 388                             struct kobj_attribute *attr,
 389                             const char *buf, size_t count)
 390 {
 391         return double_flag_store(kobj, attr, buf, count,
 392                                  TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
 393                                  TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
 394 }
 395 static struct kobj_attribute defrag_attr =
 396         __ATTR(defrag, 0644, defrag_show, defrag_store);
 397
 398 static ssize_t use_zero_page_show(struct kobject *kobj,
 399                 struct kobj_attribute *attr, char *buf)
 400 {
 401         return single_flag_show(kobj, attr, buf,
 402                                 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 403 }
 404 static ssize_t use_zero_page_store(struct kobject *kobj,
 405                 struct kobj_attribute *attr, const char *buf, size_t count)
 406 {
 407         return single_flag_store(kobj, attr, buf, count,
 408                                  TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 409 }
 410 static struct kobj_attribute use_zero_page_attr =
 411         __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
 412 #ifdef CONFIG_DEBUG_VM
 413 static ssize_t debug_cow_show(struct kobject *kobj,
 414                                 struct kobj_attribute *attr, char *buf)
 415 {
 416         return single_flag_show(kobj, attr, buf,
 417                                 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
 418 }
 419 static ssize_t debug_cow_store(struct kobject *kobj,
 420                                struct kobj_attribute *attr,
 421                                const char *buf, size_t count)
 422 {
 423         return single_flag_store(kobj, attr, buf, count,
 424                                  TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
 425 }
 426 static struct kobj_attribute debug_cow_attr =
 427         __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
 428 #endif /* CONFIG_DEBUG_VM */
 429
 430 static struct attribute *hugepage_attr[] = {
 431         &enabled_attr.attr,
 432         &defrag_attr.attr,
 433         &use_zero_page_attr.attr,
 434 #ifdef CONFIG_DEBUG_VM
 435         &debug_cow_attr.attr,
 436 #endif
 437         NULL,
 438 };
 439
 440 static struct attribute_group hugepage_attr_group = {
 441         .attrs = hugepage_attr,
 442 };
 443
 444 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
 445                                          struct kobj_attribute *attr,
 446                                          char *buf)
 447 {
 448         return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
 449 }
 450
 451 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
 452                                           struct kobj_attribute *attr,
 453                                           const char *buf, size_t count)
 454 {
 455         unsigned long msecs;
 456         int err;
 457
 458         err = kstrtoul(buf, 10, &msecs);
 459         if (err || msecs > UINT_MAX)
 460                 return -EINVAL;
 461
 462         khugepaged_scan_sleep_millisecs = msecs;
 463         wake_up_interruptible(&khugepaged_wait);
 464
 465         return count;
 466 }
 467 static struct kobj_attribute scan_sleep_millisecs_attr =
 468         __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
 469                scan_sleep_millisecs_store);
 470
 471 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
 472                                           struct kobj_attribute *attr,
 473                                           char *buf)
 474 {
 475         return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
 476 }
 477
 478 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
 479                                            struct kobj_attribute *attr,
 480                                            const char *buf, size_t count)
 481 {
 482         unsigned long msecs;
 483         int err;
 484
 485         err = kstrtoul(buf, 10, &msecs);
 486         if (err || msecs > UINT_MAX)
 487                 return -EINVAL;
 488
 489         khugepaged_alloc_sleep_millisecs = msecs;
 490         wake_up_interruptible(&khugepaged_wait);
 491
 492         return count;
 493 }
 494 static struct kobj_attribute alloc_sleep_millisecs_attr =
 495         __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
 496                alloc_sleep_millisecs_store);
 497
 498 static ssize_t pages_to_scan_show(struct kobject *kobj,
 499                                   struct kobj_attribute *attr,
 500                                   char *buf)
 501 {
 502         return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
 503 }
 504 static ssize_t pages_to_scan_store(struct kobject *kobj,
 505                                    struct kobj_attribute *attr,
 506                                    const char *buf, size_t count)
 507 {
 508         int err;
 509         unsigned long pages;
 510
 511         err = kstrtoul(buf, 10, &pages);
 512         if (err || !pages || pages > UINT_MAX)
 513                 return -EINVAL;
 514
 515         khugepaged_pages_to_scan = pages;
 516
 517         return count;
 518 }
 519 static struct kobj_attribute pages_to_scan_attr =
 520         __ATTR(pages_to_scan, 0644, pages_to_scan_show,
 521                pages_to_scan_store);
 522
 523 static ssize_t pages_collapsed_show(struct kobject *kobj,
 524                                     struct kobj_attribute *attr,
 525                                     char *buf)
 526 {
 527         return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
 528 }
 529 static struct kobj_attribute pages_collapsed_attr =
 530         __ATTR_RO(pages_collapsed);
 531
 532 static ssize_t full_scans_show(struct kobject *kobj,
 533                                struct kobj_attribute *attr,
 534                                char *buf)
 535 {
 536         return sprintf(buf, "%u\n", khugepaged_full_scans);
 537 }
 538 static struct kobj_attribute full_scans_attr =
 539         __ATTR_RO(full_scans);
 540
 541 static ssize_t khugepaged_defrag_show(struct kobject *kobj,
 542                                       struct kobj_attribute *attr, char *buf)
 543 {
 544         return single_flag_show(kobj, attr, buf,
 545                                 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
 546 }
 547 static ssize_t khugepaged_defrag_store(struct kobject *kobj,
 548                                        struct kobj_attribute *attr,
 549                                        const char *buf, size_t count)
 550 {
 551         return single_flag_store(kobj, attr, buf, count,
 552                                  TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
 553 }
 554 static struct kobj_attribute khugepaged_defrag_attr =
 555         __ATTR(defrag, 0644, khugepaged_defrag_show,
 556                khugepaged_defrag_store);
 557
 558 /*
 559  * max_ptes_none controls if khugepaged should collapse hugepages over
 560  * any unmapped ptes in turn potentially increasing the memory
 561  * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
 562  * reduce the available free memory in the system as it
 563  * runs. Increasing max_ptes_none will instead potentially reduce the
 564  * free memory in the system during the khugepaged scan.
 565  */
 566 static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
 567                                              struct kobj_attribute *attr,
 568                                              char *buf)
 569 {
 570         return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
 571 }
 572 static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
 573                                               struct kobj_attribute *attr,
 574                                               const char *buf, size_t count)
 575 {
 576         int err;
 577         unsigned long max_ptes_none;
 578
 579         err = kstrtoul(buf, 10, &max_ptes_none);
 580         if (err || max_ptes_none > HPAGE_PMD_NR-1)
 581                 return -EINVAL;
 582
 583         khugepaged_max_ptes_none = max_ptes_none;
 584
 585         return count;
 586 }
 587 static struct kobj_attribute khugepaged_max_ptes_none_attr =
 588         __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
 589                khugepaged_max_ptes_none_store);
 590
 591 static struct attribute *khugepaged_attr[] = {
 592         &khugepaged_defrag_attr.attr,
 593         &khugepaged_max_ptes_none_attr.attr,
 594         &pages_to_scan_attr.attr,
 595         &pages_collapsed_attr.attr,
 596         &full_scans_attr.attr,
 597         &scan_sleep_millisecs_attr.attr,
 598         &alloc_sleep_millisecs_attr.attr,
 599         NULL,
 600 };
 601
 602 static struct attribute_group khugepaged_attr_group = {
 603         .attrs = khugepaged_attr,
 604         .name = "khugepaged",
 605 };
 606
 607 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
 608 {
 609         int err;
 610
 611         *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
 612         if (unlikely(!*hugepage_kobj)) {
 613                 pr_err("failed to create transparent hugepage kobject\n");
 614                 return -ENOMEM;
 615         }
 616
 617         err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
 618         if (err) {
 619                 pr_err("failed to register transparent hugepage group\n");
 620                 goto delete_obj;
 621         }
 622
 623         err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
 624         if (err) {
 625                 pr_err("failed to register transparent hugepage group\n");
 626                 goto remove_hp_group;
 627         }
 628
 629         return 0;
 630
 631 remove_hp_group:
 632         sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
 633 delete_obj:
 634         kobject_put(*hugepage_kobj);
 635         return err;
 636 }
 637
 638 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
 639 {
 640         sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
 641         sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
 642         kobject_put(hugepage_kobj);
 643 }
 644 #else
 645 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
 646 {
 647         return 0;
 648 }
 649
 650 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
 651 {
 652 }
 653 #endif /* CONFIG_SYSFS */
 654
 655 static int __init hugepage_init(void)
 656 {
 657         int err;
 658         struct kobject *hugepage_kobj;
 659
 660         if (!has_transparent_hugepage()) {
 661                 transparent_hugepage_flags = 0;
 662                 return -EINVAL;
 663         }
 664
 665         err = hugepage_init_sysfs(&hugepage_kobj);
 666         if (err)
 667                 goto err_sysfs;
 668
 669         err = khugepaged_slab_init();
 670         if (err)
 671                 goto err_slab;
 672
 673         err = register_shrinker(&huge_zero_page_shrinker);
 674         if (err)
 675                 goto err_hzp_shrinker;
 676         err = register_shrinker(&deferred_split_shrinker);
 677         if (err)
 678                 goto err_split_shrinker;
 679
 680         /*
 681          * By default disable transparent hugepages on smaller systems,
 682          * where the extra memory used could hurt more than TLB overhead
 683          * is likely to save.  The admin can still enable it through /sys.
 684          */
 685         if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
 686                 transparent_hugepage_flags = 0;
 687                 return 0;
 688         }
 689
 690         err = start_stop_khugepaged();
 691         if (err)
 692                 goto err_khugepaged;
 693
 694         return 0;
 695 err_khugepaged:
 696         unregister_shrinker(&deferred_split_shrinker);
 697 err_split_shrinker:
 698         unregister_shrinker(&huge_zero_page_shrinker);
 699 err_hzp_shrinker:
 700         khugepaged_slab_exit();
 701 err_slab:
 702         hugepage_exit_sysfs(hugepage_kobj);
 703 err_sysfs:
 704         return err;
 705 }
 706 subsys_initcall(hugepage_init);
 707
 708 static int __init setup_transparent_hugepage(char *str)
 709 {
 710         int ret = 0;
 711         if (!str)
 712                 goto out;
 713         if (!strcmp(str, "always")) {
 714                 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
 715                         &transparent_hugepage_flags);
 716                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 717                           &transparent_hugepage_flags);
 718                 ret = 1;
 719         } else if (!strcmp(str, "madvise")) {
 720                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 721                           &transparent_hugepage_flags);
 722                 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 723                         &transparent_hugepage_flags);
 724                 ret = 1;
 725         } else if (!strcmp(str, "never")) {
 726                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 727                           &transparent_hugepage_flags);
 728                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 729                           &transparent_hugepage_flags);
 730                 ret = 1;
 731         }
 732 out:
 733         if (!ret)
 734                 pr_warn("transparent_hugepage= cannot parse, ignored\n");
 735         return ret;
 736 }
 737 __setup("transparent_hugepage=", setup_transparent_hugepage);
 738
 739 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
 740 {
 741         if (likely(vma->vm_flags & VM_WRITE))
 742                 pmd = pmd_mkwrite(pmd);
 743         return pmd;
 744 }
 745
 746 static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
 747 {
 748         pmd_t entry;
 749         entry = mk_pmd(page, prot);
 750         entry = pmd_mkhuge(entry);
 751         return entry;
 752 }
 753
 754 static inline struct list_head *page_deferred_list(struct page *page)
 755 {
 756         /*
 757          * ->lru in the tail pages is occupied by compound_head.
 758          * Let's use ->mapping + ->index in the second tail page as list_head.
 759          */
 760         return (struct list_head *)&page[2].mapping;
 761 }
 762
 763 void prep_transhuge_page(struct page *page)
 764 {
 765         /*
 766          * we use page->mapping and page->indexlru in second tail page
 767          * as list_head: assuming THP order >= 2
 768          */
 769         BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
 770
 771         INIT_LIST_HEAD(page_deferred_list(page));
 772         set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
 773 }
 774
 775 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
 776                                         struct vm_area_struct *vma,
 777                                         unsigned long address, pmd_t *pmd,
 778                                         struct page *page, gfp_t gfp,
 779                                         unsigned int flags)
 780 {
 781         struct mem_cgroup *memcg;
 782         pgtable_t pgtable;
 783         spinlock_t *ptl;
 784         unsigned long haddr = address & HPAGE_PMD_MASK;
 785
 786         VM_BUG_ON_PAGE(!PageCompound(page), page);
 787
 788         if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) {
 789                 put_page(page);
 790                 count_vm_event(THP_FAULT_FALLBACK);
 791                 return VM_FAULT_FALLBACK;
 792         }
 793
 794         pgtable = pte_alloc_one(mm, haddr);
 795         if (unlikely(!pgtable)) {
 796                 mem_cgroup_cancel_charge(page, memcg, true);
 797                 put_page(page);
 798                 return VM_FAULT_OOM;
 799         }
 800
 801         clear_huge_page(page, haddr, HPAGE_PMD_NR);
 802         /*
 803          * The memory barrier inside __SetPageUptodate makes sure that
 804          * clear_huge_page writes become visible before the set_pmd_at()
 805          * write.
 806          */
 807         __SetPageUptodate(page);
 808
 809         ptl = pmd_lock(mm, pmd);
 810         if (unlikely(!pmd_none(*pmd))) {
 811                 spin_unlock(ptl);
 812                 mem_cgroup_cancel_charge(page, memcg, true);
 813                 put_page(page);
 814                 pte_free(mm, pgtable);
 815         } else {
 816                 pmd_t entry;
 817
 818                 /* Deliver the page fault to userland */
 819                 if (userfaultfd_missing(vma)) {
 820                         int ret;
 821
 822                         spin_unlock(ptl);
 823                         mem_cgroup_cancel_charge(page, memcg, true);
 824                         put_page(page);
 825                         pte_free(mm, pgtable);
 826                         ret = handle_userfault(vma, address, flags,
 827                                                VM_UFFD_MISSING);
 828                         VM_BUG_ON(ret & VM_FAULT_FALLBACK);
 829                         return ret;
 830                 }
 831
 832                 entry = mk_huge_pmd(page, vma->vm_page_prot);
 833                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 834                 page_add_new_anon_rmap(page, vma, haddr, true);
 835                 mem_cgroup_commit_charge(page, memcg, false, true);
 836                 lru_cache_add_active_or_unevictable(page, vma);
 837                 pgtable_trans_huge_deposit(mm, pmd, pgtable);
 838                 set_pmd_at(mm, haddr, pmd, entry);
 839                 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
 840                 atomic_long_inc(&mm->nr_ptes);
 841                 spin_unlock(ptl);
 842                 count_vm_event(THP_FAULT_ALLOC);
 843         }
 844
 845         return 0;
 846 }
 847
 848 static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)
 849 {
 850         return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) | extra_gfp;
 851 }
 852
 853 /* Caller must hold page table lock. */
 854 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
 855                 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
 856                 struct page *zero_page)
 857 {
 858         pmd_t entry;
 859         if (!pmd_none(*pmd))
 860                 return false;
 861         entry = mk_pmd(zero_page, vma->vm_page_prot);
 862         entry = pmd_mkhuge(entry);
 863         pgtable_trans_huge_deposit(mm, pmd, pgtable);
 864         set_pmd_at(mm, haddr, pmd, entry);
 865         atomic_long_inc(&mm->nr_ptes);
 866         return true;
 867 }
 868
 869 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
 870                                unsigned long address, pmd_t *pmd,
 871                                unsigned int flags)
 872 {
 873         gfp_t gfp;
 874         struct page *page;
 875         unsigned long haddr = address & HPAGE_PMD_MASK;
 876
 877         if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
 878                 return VM_FAULT_FALLBACK;
 879         if (unlikely(anon_vma_prepare(vma)))
 880                 return VM_FAULT_OOM;
 881         if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
 882                 return VM_FAULT_OOM;
 883         if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) &&
 884                         transparent_hugepage_use_zero_page()) {
 885                 spinlock_t *ptl;
 886                 pgtable_t pgtable;
 887                 struct page *zero_page;
 888                 bool set;
 889                 int ret;
 890                 pgtable = pte_alloc_one(mm, haddr);
 891                 if (unlikely(!pgtable))
 892                         return VM_FAULT_OOM;
 893                 zero_page = get_huge_zero_page();
 894                 if (unlikely(!zero_page)) {
 895                         pte_free(mm, pgtable);
 896                         count_vm_event(THP_FAULT_FALLBACK);
 897                         return VM_FAULT_FALLBACK;
 898                 }
 899                 ptl = pmd_lock(mm, pmd);
 900                 ret = 0;
 901                 set = false;
 902                 if (pmd_none(*pmd)) {
 903                         if (userfaultfd_missing(vma)) {
 904                                 spin_unlock(ptl);
 905                                 ret = handle_userfault(vma, address, flags,
 906                                                        VM_UFFD_MISSING);
 907                                 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
 908                         } else {
 909                                 set_huge_zero_page(pgtable, mm, vma,
 910                                                    haddr, pmd,
 911                                                    zero_page);
 912                                 spin_unlock(ptl);
 913                                 set = true;
 914                         }
 915                 } else
 916                         spin_unlock(ptl);
 917                 if (!set) {
 918                         pte_free(mm, pgtable);
 919                         put_huge_zero_page();
 920                 }
 921                 return ret;
 922         }
 923         gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
 924         page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
 925         if (unlikely(!page)) {
 926                 count_vm_event(THP_FAULT_FALLBACK);
 927                 return VM_FAULT_FALLBACK;
 928         }
 929         prep_transhuge_page(page);
 930         return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp,
 931                                             flags);
 932 }
 933
 934 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
 935                 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write)
 936 {
 937         struct mm_struct *mm = vma->vm_mm;
 938         pmd_t entry;
 939         spinlock_t *ptl;
 940
 941         ptl = pmd_lock(mm, pmd);
 942         entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
 943         if (pfn_t_devmap(pfn))
 944                 entry = pmd_mkdevmap(entry);
 945         if (write) {
 946                 entry = pmd_mkyoung(pmd_mkdirty(entry));
 947                 entry = maybe_pmd_mkwrite(entry, vma);
 948         }
 949         set_pmd_at(mm, addr, pmd, entry);
 950         update_mmu_cache_pmd(vma, addr, pmd);
 951         spin_unlock(ptl);
 952 }
 953
 954 int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
 955                         pmd_t *pmd, pfn_t pfn, bool write)
 956 {
 957         pgprot_t pgprot = vma->vm_page_prot;
 958         /*
 959          * If we had pmd_special, we could avoid all these restrictions,
 960          * but we need to be consistent with PTEs and architectures that
 961          * can't support a 'special' bit.
 962          */
 963         BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
 964         BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
 965                                                 (VM_PFNMAP|VM_MIXEDMAP));
 966         BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
 967         BUG_ON(!pfn_t_devmap(pfn));
 968
 969         if (addr < vma->vm_start || addr >= vma->vm_end)
 970                 return VM_FAULT_SIGBUS;
 971         if (track_pfn_insert(vma, &pgprot, pfn))
 972                 return VM_FAULT_SIGBUS;
 973         insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);
 974         return VM_FAULT_NOPAGE;
 975 }
 976
 977 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 978                   pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
 979                   struct vm_area_struct *vma)
 980 {
 981         spinlock_t *dst_ptl, *src_ptl;
 982         struct page *src_page;
 983         pmd_t pmd;
 984         pgtable_t pgtable;
 985         int ret;
 986
 987         ret = -ENOMEM;
 988         pgtable = pte_alloc_one(dst_mm, addr);
 989         if (unlikely(!pgtable))
 990                 goto out;
 991
 992         dst_ptl = pmd_lock(dst_mm, dst_pmd);
 993         src_ptl = pmd_lockptr(src_mm, src_pmd);
 994         spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
 995
 996         ret = -EAGAIN;
 997         pmd = *src_pmd;
 998         if (unlikely(!pmd_trans_huge(pmd))) {
 999                 pte_free(dst_mm, pgtable);
1000                 goto out_unlock;
1001         }
1002         /*
1003          * When page table lock is held, the huge zero pmd should not be
1004          * under splitting since we don't split the page itself, only pmd to
1005          * a page table.
1006          */
1007         if (is_huge_zero_pmd(pmd)) {
1008                 struct page *zero_page;
1009                 /*
1010                  * get_huge_zero_page() will never allocate a new page here,
1011                  * since we already have a zero page to copy. It just takes a
1012                  * reference.
1013                  */
1014                 zero_page = get_huge_zero_page();
1015                 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
1016                                 zero_page);
1017                 ret = 0;
1018                 goto out_unlock;
1019         }
1020
1021         src_page = pmd_page(pmd);
1022         VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
1023         get_page(src_page);
1024         page_dup_rmap(src_page, true);
1025         add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1026
1027         pmdp_set_wrprotect(src_mm, addr, src_pmd);
1028         pmd = pmd_mkold(pmd_wrprotect(pmd));
1029         pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1030         set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1031         atomic_long_inc(&dst_mm->nr_ptes);
1032
1033         ret = 0;
1034 out_unlock:
1035         spin_unlock(src_ptl);
1036         spin_unlock(dst_ptl);
1037 out:
1038         return ret;
1039 }
1040
1041 void huge_pmd_set_accessed(struct mm_struct *mm,
1042                            struct vm_area_struct *vma,
1043                            unsigned long address,
1044                            pmd_t *pmd, pmd_t orig_pmd,
1045                            int dirty)
1046 {
1047         spinlock_t *ptl;
1048         pmd_t entry;
1049         unsigned long haddr;
1050
1051         ptl = pmd_lock(mm, pmd);
1052         if (unlikely(!pmd_same(*pmd, orig_pmd)))
1053                 goto unlock;
1054
1055         entry = pmd_mkyoung(orig_pmd);
1056         haddr = address & HPAGE_PMD_MASK;
1057         if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty))
1058                 update_mmu_cache_pmd(vma, address, pmd);
1059
1060 unlock:
1061         spin_unlock(ptl);
1062 }
1063
1064 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
1065                                         struct vm_area_struct *vma,
1066                                         unsigned long address,
1067                                         pmd_t *pmd, pmd_t orig_pmd,
1068                                         struct page *page,
1069                                         unsigned long haddr)
1070 {
1071         struct mem_cgroup *memcg;
1072         spinlock_t *ptl;
1073         pgtable_t pgtable;
1074         pmd_t _pmd;
1075         int ret = 0, i;
1076         struct page **pages;
1077         unsigned long mmun_start;       /* For mmu_notifiers */
1078         unsigned long mmun_end;         /* For mmu_notifiers */
1079
1080         pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
1081                         GFP_KERNEL);
1082         if (unlikely(!pages)) {
1083                 ret |= VM_FAULT_OOM;
1084                 goto out;
1085         }
1086
1087         for (i = 0; i < HPAGE_PMD_NR; i++) {
1088                 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
1089                                                __GFP_OTHER_NODE,
1090                                                vma, address, page_to_nid(page));
1091                 if (unlikely(!pages[i] ||
1092                              mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL,
1093                                                    &memcg, false))) {
1094                         if (pages[i])
1095                                 put_page(pages[i]);
1096                         while (--i >= 0) {
1097                                 memcg = (void *)page_private(pages[i]);
1098                                 set_page_private(pages[i], 0);
1099                                 mem_cgroup_cancel_charge(pages[i], memcg,
1100                                                 false);
1101                                 put_page(pages[i]);
1102                         }
1103                         kfree(pages);
1104                         ret |= VM_FAULT_OOM;
1105                         goto out;
1106                 }
1107                 set_page_private(pages[i], (unsigned long)memcg);
1108         }
1109
1110         for (i = 0; i < HPAGE_PMD_NR; i++) {
1111                 copy_user_highpage(pages[i], page + i,
1112                                    haddr + PAGE_SIZE * i, vma);
1113                 __SetPageUptodate(pages[i]);
1114                 cond_resched();
1115         }
1116
1117         mmun_start = haddr;
1118         mmun_end   = haddr + HPAGE_PMD_SIZE;
1119         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1120
1121         ptl = pmd_lock(mm, pmd);
1122         if (unlikely(!pmd_same(*pmd, orig_pmd)))
1123                 goto out_free_pages;
1124         VM_BUG_ON_PAGE(!PageHead(page), page);
1125
1126         pmdp_huge_clear_flush_notify(vma, haddr, pmd);
1127         /* leave pmd empty until pte is filled */
1128
1129         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1130         pmd_populate(mm, &_pmd, pgtable);
1131
1132         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1133                 pte_t *pte, entry;
1134                 entry = mk_pte(pages[i], vma->vm_page_prot);
1135                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1136                 memcg = (void *)page_private(pages[i]);
1137                 set_page_private(pages[i], 0);
1138                 page_add_new_anon_rmap(pages[i], vma, haddr, false);
1139                 mem_cgroup_commit_charge(pages[i], memcg, false, false);
1140                 lru_cache_add_active_or_unevictable(pages[i], vma);
1141                 pte = pte_offset_map(&_pmd, haddr);
1142                 VM_BUG_ON(!pte_none(*pte));
1143                 set_pte_at(mm, haddr, pte, entry);
1144                 pte_unmap(pte);
1145         }
1146         kfree(pages);
1147
1148         smp_wmb(); /* make pte visible before pmd */
1149         pmd_populate(mm, pmd, pgtable);
1150         page_remove_rmap(page, true);
1151         spin_unlock(ptl);
1152
1153         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1154
1155         ret |= VM_FAULT_WRITE;
1156         put_page(page);
1157
1158 out:
1159         return ret;
1160
1161 out_free_pages:
1162         spin_unlock(ptl);
1163         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1164         for (i = 0; i < HPAGE_PMD_NR; i++) {
1165                 memcg = (void *)page_private(pages[i]);
1166                 set_page_private(pages[i], 0);
1167                 mem_cgroup_cancel_charge(pages[i], memcg, false);
1168                 put_page(pages[i]);
1169         }
1170         kfree(pages);
1171         goto out;
1172 }
1173
1174 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
1175                         unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
1176 {
1177         spinlock_t *ptl;
1178         int ret = 0;
1179         struct page *page = NULL, *new_page;
1180         struct mem_cgroup *memcg;
1181         unsigned long haddr;
1182         unsigned long mmun_start;       /* For mmu_notifiers */
1183         unsigned long mmun_end;         /* For mmu_notifiers */
1184         gfp_t huge_gfp;                 /* for allocation and charge */
1185
1186         ptl = pmd_lockptr(mm, pmd);
1187         VM_BUG_ON_VMA(!vma->anon_vma, vma);
1188         haddr = address & HPAGE_PMD_MASK;
1189         if (is_huge_zero_pmd(orig_pmd))
1190                 goto alloc;
1191         spin_lock(ptl);
1192         if (unlikely(!pmd_same(*pmd, orig_pmd)))
1193                 goto out_unlock;
1194
1195         page = pmd_page(orig_pmd);
1196         VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1197         /*
1198          * We can only reuse the page if nobody else maps the huge page or it's
1199          * part. We can do it by checking page_mapcount() on each sub-page, but
1200          * it's expensive.
1201          * The cheaper way is to check page_count() to be equal 1: every
1202          * mapcount takes page reference reference, so this way we can
1203          * guarantee, that the PMD is the only mapping.
1204          * This can give false negative if somebody pinned the page, but that's
1205          * fine.
1206          */
1207         if (page_mapcount(page) == 1 && page_count(page) == 1) {
1208                 pmd_t entry;
1209                 entry = pmd_mkyoung(orig_pmd);
1210                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1211                 if (pmdp_set_access_flags(vma, haddr, pmd, entry,  1))
1212                         update_mmu_cache_pmd(vma, address, pmd);
1213                 ret |= VM_FAULT_WRITE;
1214                 goto out_unlock;
1215         }
1216         get_page(page);
1217         spin_unlock(ptl);
1218 alloc:
1219         if (transparent_hugepage_enabled(vma) &&
1220             !transparent_hugepage_debug_cow()) {
1221                 huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
1222                 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1223         } else
1224                 new_page = NULL;
1225
1226         if (likely(new_page)) {
1227                 prep_transhuge_page(new_page);
1228         } else {
1229                 if (!page) {
1230                         split_huge_pmd(vma, pmd, address);
1231                         ret |= VM_FAULT_FALLBACK;
1232                 } else {
1233                         ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
1234                                         pmd, orig_pmd, page, haddr);
1235                         if (ret & VM_FAULT_OOM) {
1236                                 split_huge_pmd(vma, pmd, address);
1237                                 ret |= VM_FAULT_FALLBACK;
1238                         }
1239                         put_page(page);
1240                 }
1241                 count_vm_event(THP_FAULT_FALLBACK);
1242                 goto out;
1243         }
1244
1245         if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg,
1246                                            true))) {
1247                 put_page(new_page);
1248                 if (page) {
1249                         split_huge_pmd(vma, pmd, address);
1250                         put_page(page);
1251                 } else
1252                         split_huge_pmd(vma, pmd, address);
1253                 ret |= VM_FAULT_FALLBACK;
1254                 count_vm_event(THP_FAULT_FALLBACK);
1255                 goto out;
1256         }
1257
1258         count_vm_event(THP_FAULT_ALLOC);
1259
1260         if (!page)
1261                 clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
1262         else
1263                 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
1264         __SetPageUptodate(new_page);
1265
1266         mmun_start = haddr;
1267         mmun_end   = haddr + HPAGE_PMD_SIZE;
1268         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1269
1270         spin_lock(ptl);
1271         if (page)
1272                 put_page(page);
1273         if (unlikely(!pmd_same(*pmd, orig_pmd))) {
1274                 spin_unlock(ptl);
1275                 mem_cgroup_cancel_charge(new_page, memcg, true);
1276                 put_page(new_page);
1277                 goto out_mn;
1278         } else {
1279                 pmd_t entry;
1280                 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1281                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1282                 pmdp_huge_clear_flush_notify(vma, haddr, pmd);
1283                 page_add_new_anon_rmap(new_page, vma, haddr, true);
1284                 mem_cgroup_commit_charge(new_page, memcg, false, true);
1285                 lru_cache_add_active_or_unevictable(new_page, vma);
1286                 set_pmd_at(mm, haddr, pmd, entry);
1287                 update_mmu_cache_pmd(vma, address, pmd);
1288                 if (!page) {
1289                         add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
1290                         put_huge_zero_page();
1291                 } else {
1292                         VM_BUG_ON_PAGE(!PageHead(page), page);
1293                         page_remove_rmap(page, true);
1294                         put_page(page);
1295                 }
1296                 ret |= VM_FAULT_WRITE;
1297         }
1298         spin_unlock(ptl);
1299 out_mn:
1300         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1301 out:
1302         return ret;
1303 out_unlock:
1304         spin_unlock(ptl);
1305         return ret;
1306 }
1307
1308 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1309                                    unsigned long addr,
1310                                    pmd_t *pmd,
1311                                    unsigned int flags)
1312 {
1313         struct mm_struct *mm = vma->vm_mm;
1314         struct page *page = NULL;
1315
1316         assert_spin_locked(pmd_lockptr(mm, pmd));
1317
1318         if (flags & FOLL_WRITE && !pmd_write(*pmd))
1319                 goto out;
1320
1321         /* Avoid dumping huge zero page */
1322         if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1323                 return ERR_PTR(-EFAULT);
1324
1325         /* Full NUMA hinting faults to serialise migration in fault paths */
1326         if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1327                 goto out;
1328
1329         page = pmd_page(*pmd);
1330         VM_BUG_ON_PAGE(!PageHead(page), page);
1331         if (flags & FOLL_TOUCH) {
1332                 pmd_t _pmd;
1333                 /*
1334                  * We should set the dirty bit only for FOLL_WRITE but
1335                  * for now the dirty bit in the pmd is meaningless.
1336                  * And if the dirty bit will become meaningful and
1337                  * we'll only set it with FOLL_WRITE, an atomic
1338                  * set_bit will be required on the pmd to set the
1339                  * young bit, instead of the current set_pmd_at.
1340                  */
1341                 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
1342                 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
1343                                           pmd, _pmd,  1))
1344                         update_mmu_cache_pmd(vma, addr, pmd);
1345         }
1346         if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1347                 /*
1348                  * We don't mlock() pte-mapped THPs. This way we can avoid
1349                  * leaking mlocked pages into non-VM_LOCKED VMAs.
1350                  *
1351                  * In most cases the pmd is the only mapping of the page as we
1352                  * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
1353                  * writable private mappings in populate_vma_page_range().
1354                  *
1355                  * The only scenario when we have the page shared here is if we
1356                  * mlocking read-only mapping shared over fork(). We skip
1357                  * mlocking such pages.
1358                  */
1359                 if (compound_mapcount(page) == 1 && !PageDoubleMap(page) &&
1360                                 page->mapping && trylock_page(page)) {
1361                         lru_add_drain();
1362                         if (page->mapping)
1363                                 mlock_vma_page(page);
1364                         unlock_page(page);
1365                 }
1366         }
1367         page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1368         VM_BUG_ON_PAGE(!PageCompound(page), page);
1369         if (flags & FOLL_GET)
1370                 get_page(page);
1371
1372 out:
1373         return page;
1374 }
1375
1376 /* NUMA hinting page fault entry point for trans huge pmds */
1377 int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
1378                                 unsigned long addr, pmd_t pmd, pmd_t *pmdp)
1379 {
1380         spinlock_t *ptl;
1381         struct anon_vma *anon_vma = NULL;
1382         struct page *page;
1383         unsigned long haddr = addr & HPAGE_PMD_MASK;
1384         int page_nid = -1, this_nid = numa_node_id();
1385         int target_nid, last_cpupid = -1;
1386         bool page_locked;
1387         bool migrated = false;
1388         bool was_writable;
1389         int flags = 0;
1390
1391         /* A PROT_NONE fault should not end up here */
1392         BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)));
1393
1394         ptl = pmd_lock(mm, pmdp);
1395         if (unlikely(!pmd_same(pmd, *pmdp)))
1396                 goto out_unlock;
1397
1398         /*
1399          * If there are potential migrations, wait for completion and retry
1400          * without disrupting NUMA hinting information. Do not relock and
1401          * check_same as the page may no longer be mapped.
1402          */
1403         if (unlikely(pmd_trans_migrating(*pmdp))) {
1404                 page = pmd_page(*pmdp);
1405                 spin_unlock(ptl);
1406                 wait_on_page_locked(page);
1407                 goto out;
1408         }
1409
1410         page = pmd_page(pmd);
1411         BUG_ON(is_huge_zero_page(page));
1412         page_nid = page_to_nid(page);
1413         last_cpupid = page_cpupid_last(page);
1414         count_vm_numa_event(NUMA_HINT_FAULTS);
1415         if (page_nid == this_nid) {
1416                 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1417                 flags |= TNF_FAULT_LOCAL;
1418         }
1419
1420         /* See similar comment in do_numa_page for explanation */
1421         if (!(vma->vm_flags & VM_WRITE))
1422                 flags |= TNF_NO_GROUP;
1423
1424         /*
1425          * Acquire the page lock to serialise THP migrations but avoid dropping
1426          * page_table_lock if at all possible
1427          */
1428         page_locked = trylock_page(page);
1429         target_nid = mpol_misplaced(page, vma, haddr);
1430         if (target_nid == -1) {
1431                 /* If the page was locked, there are no parallel migrations */
1432                 if (page_locked)
1433                         goto clear_pmdnuma;
1434         }
1435
1436         /* Migration could have started since the pmd_trans_migrating check */
1437         if (!page_locked) {
1438                 spin_unlock(ptl);
1439                 wait_on_page_locked(page);
1440                 page_nid = -1;
1441                 goto out;
1442         }
1443
1444         /*
1445          * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
1446          * to serialises splits
1447          */
1448         get_page(page);
1449         spin_unlock(ptl);
1450         anon_vma = page_lock_anon_vma_read(page);
1451
1452         /* Confirm the PMD did not change while page_table_lock was released */
1453         spin_lock(ptl);
1454         if (unlikely(!pmd_same(pmd, *pmdp))) {
1455                 unlock_page(page);
1456                 put_page(page);
1457                 page_nid = -1;
1458                 goto out_unlock;
1459         }
1460
1461         /* Bail if we fail to protect against THP splits for any reason */
1462         if (unlikely(!anon_vma)) {
1463                 put_page(page);
1464                 page_nid = -1;
1465                 goto clear_pmdnuma;
1466         }
1467
1468         /*
1469          * Migrate the THP to the requested node, returns with page unlocked
1470          * and access rights restored.
1471          */
1472         spin_unlock(ptl);
1473         migrated = migrate_misplaced_transhuge_page(mm, vma,
1474                                 pmdp, pmd, addr, page, target_nid);
1475         if (migrated) {
1476                 flags |= TNF_MIGRATED;
1477                 page_nid = target_nid;
1478         } else
1479                 flags |= TNF_MIGRATE_FAIL;
1480
1481         goto out;
1482 clear_pmdnuma:
1483         BUG_ON(!PageLocked(page));
1484         was_writable = pmd_write(pmd);
1485         pmd = pmd_modify(pmd, vma->vm_page_prot);
1486         pmd = pmd_mkyoung(pmd);
1487         if (was_writable)
1488                 pmd = pmd_mkwrite(pmd);
1489         set_pmd_at(mm, haddr, pmdp, pmd);
1490         update_mmu_cache_pmd(vma, addr, pmdp);
1491         unlock_page(page);
1492 out_unlock:
1493         spin_unlock(ptl);
1494
1495 out:
1496         if (anon_vma)
1497                 page_unlock_anon_vma_read(anon_vma);
1498
1499         if (page_nid != -1)
1500                 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags);
1501
1502         return 0;
1503 }
1504
1505 int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1506                 pmd_t *pmd, unsigned long addr, unsigned long next)
1507
1508 {
1509         spinlock_t *ptl;
1510         pmd_t orig_pmd;
1511         struct page *page;
1512         struct mm_struct *mm = tlb->mm;
1513         int ret = 0;
1514
1515         if (!pmd_trans_huge_lock(pmd, vma, &ptl))
1516                 goto out;
1517
1518         orig_pmd = *pmd;
1519         if (is_huge_zero_pmd(orig_pmd)) {
1520                 ret = 1;
1521                 goto out;
1522         }
1523
1524         page = pmd_page(orig_pmd);
1525         /*
1526          * If other processes are mapping this page, we couldn't discard
1527          * the page unless they all do MADV_FREE so let's skip the page.
1528          */
1529         if (page_mapcount(page) != 1)
1530                 goto out;
1531
1532         if (!trylock_page(page))
1533                 goto out;
1534
1535         /*
1536          * If user want to discard part-pages of THP, split it so MADV_FREE
1537          * will deactivate only them.
1538          */
1539         if (next - addr != HPAGE_PMD_SIZE) {
1540                 get_page(page);
1541                 spin_unlock(ptl);
1542                 if (split_huge_page(page)) {
1543                         put_page(page);
1544                         unlock_page(page);
1545                         goto out_unlocked;
1546                 }
1547                 put_page(page);
1548                 unlock_page(page);
1549                 ret = 1;
1550                 goto out_unlocked;
1551         }
1552
1553         if (PageDirty(page))
1554                 ClearPageDirty(page);
1555         unlock_page(page);
1556
1557         if (PageActive(page))
1558                 deactivate_page(page);
1559
1560         if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1561                 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1562                         tlb->fullmm);
1563                 orig_pmd = pmd_mkold(orig_pmd);
1564                 orig_pmd = pmd_mkclean(orig_pmd);
1565
1566                 set_pmd_at(mm, addr, pmd, orig_pmd);
1567                 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1568         }
1569         ret = 1;
1570 out:
1571         spin_unlock(ptl);
1572 out_unlocked:
1573         return ret;
1574 }
1575
1576 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1577                  pmd_t *pmd, unsigned long addr)
1578 {
1579         pmd_t orig_pmd;
1580         spinlock_t *ptl;
1581
1582         if (!__pmd_trans_huge_lock(pmd, vma, &ptl))
1583                 return 0;
1584         /*
1585          * For architectures like ppc64 we look at deposited pgtable
1586          * when calling pmdp_huge_get_and_clear. So do the
1587          * pgtable_trans_huge_withdraw after finishing pmdp related
1588          * operations.
1589          */
1590         orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1591                         tlb->fullmm);
1592         tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1593         if (vma_is_dax(vma)) {
1594                 spin_unlock(ptl);
1595                 if (is_huge_zero_pmd(orig_pmd))
1596                         put_huge_zero_page();
1597         } else if (is_huge_zero_pmd(orig_pmd)) {
1598                 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1599                 atomic_long_dec(&tlb->mm->nr_ptes);
1600                 spin_unlock(ptl);
1601                 put_huge_zero_page();
1602         } else {
1603                 struct page *page = pmd_page(orig_pmd);
1604                 page_remove_rmap(page, true);
1605                 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1606                 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1607                 VM_BUG_ON_PAGE(!PageHead(page), page);
1608                 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1609                 atomic_long_dec(&tlb->mm->nr_ptes);
1610                 spin_unlock(ptl);
1611                 tlb_remove_page(tlb, page);
1612         }
1613         return 1;
1614 }
1615
1616 bool move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
1617                   unsigned long old_addr,
1618                   unsigned long new_addr, unsigned long old_end,
1619                   pmd_t *old_pmd, pmd_t *new_pmd)
1620 {
1621         spinlock_t *old_ptl, *new_ptl;
1622         pmd_t pmd;
1623
1624         struct mm_struct *mm = vma->vm_mm;
1625
1626         if ((old_addr & ~HPAGE_PMD_MASK) ||
1627             (new_addr & ~HPAGE_PMD_MASK) ||
1628             old_end - old_addr < HPAGE_PMD_SIZE ||
1629             (new_vma->vm_flags & VM_NOHUGEPAGE))
1630                 return false;
1631
1632         /*
1633          * The destination pmd shouldn't be established, free_pgtables()
1634          * should have release it.
1635          */
1636         if (WARN_ON(!pmd_none(*new_pmd))) {
1637                 VM_BUG_ON(pmd_trans_huge(*new_pmd));
1638                 return false;
1639         }
1640
1641         /*
1642          * We don't have to worry about the ordering of src and dst
1643          * ptlocks because exclusive mmap_sem prevents deadlock.
1644          */
1645         if (__pmd_trans_huge_lock(old_pmd, vma, &old_ptl)) {
1646                 new_ptl = pmd_lockptr(mm, new_pmd);
1647                 if (new_ptl != old_ptl)
1648                         spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1649                 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1650                 VM_BUG_ON(!pmd_none(*new_pmd));
1651
1652                 if (pmd_move_must_withdraw(new_ptl, old_ptl)) {
1653                         pgtable_t pgtable;
1654                         pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1655                         pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
1656                 }
1657                 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
1658                 if (new_ptl != old_ptl)
1659                         spin_unlock(new_ptl);
1660                 spin_unlock(old_ptl);
1661                 return true;
1662         }
1663         return false;
1664 }
1665
1666 /*
1667  * Returns
1668  *  - 0 if PMD could not be locked
1669  *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1670  *  - HPAGE_PMD_NR is protections changed and TLB flush necessary
1671  */
1672 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1673                 unsigned long addr, pgprot_t newprot, int prot_numa)
1674 {
1675         struct mm_struct *mm = vma->vm_mm;
1676         spinlock_t *ptl;
1677         int ret = 0;
1678
1679         if (__pmd_trans_huge_lock(pmd, vma, &ptl)) {
1680                 pmd_t entry;
1681                 bool preserve_write = prot_numa && pmd_write(*pmd);
1682                 ret = 1;
1683
1684                 /*
1685                  * Avoid trapping faults against the zero page. The read-only
1686                  * data is likely to be read-cached on the local CPU and
1687                  * local/remote hits to the zero page are not interesting.
1688                  */
1689                 if (prot_numa && is_huge_zero_pmd(*pmd)) {
1690                         spin_unlock(ptl);
1691                         return ret;
1692                 }
1693
1694                 if (!prot_numa || !pmd_protnone(*pmd)) {
1695                         entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd);
1696                         entry = pmd_modify(entry, newprot);
1697                         if (preserve_write)
1698                                 entry = pmd_mkwrite(entry);
1699                         ret = HPAGE_PMD_NR;
1700                         set_pmd_at(mm, addr, pmd, entry);
1701                         BUG_ON(!preserve_write && pmd_write(entry));
1702                 }
1703                 spin_unlock(ptl);
1704         }
1705
1706         return ret;
1707 }
1708
1709 /*
1710  * Returns true if a given pmd maps a thp, false otherwise.
1711  *
1712  * Note that if it returns true, this routine returns without unlocking page
1713  * table lock. So callers must unlock it.
1714  */
1715 bool __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
1716                 spinlock_t **ptl)
1717 {
1718         *ptl = pmd_lock(vma->vm_mm, pmd);
1719         if (likely(pmd_trans_huge(*pmd)))
1720                 return true;
1721         spin_unlock(*ptl);
1722         return false;
1723 }
1724
1725 #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE)
1726
1727 int hugepage_madvise(struct vm_area_struct *vma,
1728                      unsigned long *vm_flags, int advice)
1729 {
1730         switch (advice) {
1731         case MADV_HUGEPAGE:
1732 #ifdef CONFIG_S390
1733                 /*
1734                  * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
1735                  * can't handle this properly after s390_enable_sie, so we simply
1736                  * ignore the madvise to prevent qemu from causing a SIGSEGV.
1737                  */
1738                 if (mm_has_pgste(vma->vm_mm))
1739                         return 0;
1740 #endif
1741                 /*
1742                  * Be somewhat over-protective like KSM for now!
1743                  */
1744                 if (*vm_flags & VM_NO_THP)
1745                         return -EINVAL;
1746                 *vm_flags &= ~VM_NOHUGEPAGE;
1747                 *vm_flags |= VM_HUGEPAGE;
1748                 /*
1749                  * If the vma become good for khugepaged to scan,
1750                  * register it here without waiting a page fault that
1751                  * may not happen any time soon.
1752                  */
1753                 if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags)))
1754                         return -ENOMEM;
1755                 break;
1756         case MADV_NOHUGEPAGE:
1757                 /*
1758                  * Be somewhat over-protective like KSM for now!
1759                  */
1760                 if (*vm_flags & VM_NO_THP)
1761                         return -EINVAL;
1762                 *vm_flags &= ~VM_HUGEPAGE;
1763                 *vm_flags |= VM_NOHUGEPAGE;
1764                 /*
1765                  * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
1766                  * this vma even if we leave the mm registered in khugepaged if
1767                  * it got registered before VM_NOHUGEPAGE was set.
1768                  */
1769                 break;
1770         }
1771
1772         return 0;
1773 }
1774
1775 static int __init khugepaged_slab_init(void)
1776 {
1777         mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
1778                                           sizeof(struct mm_slot),
1779                                           __alignof__(struct mm_slot), 0, NULL);
1780         if (!mm_slot_cache)
1781                 return -ENOMEM;
1782
1783         return 0;
1784 }
1785
1786 static void __init khugepaged_slab_exit(void)
1787 {
1788         kmem_cache_destroy(mm_slot_cache);
1789 }
1790
1791 static inline struct mm_slot *alloc_mm_slot(void)
1792 {
1793         if (!mm_slot_cache)     /* initialization failed */
1794                 return NULL;
1795         return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
1796 }
1797
1798 static inline void free_mm_slot(struct mm_slot *mm_slot)
1799 {
1800         kmem_cache_free(mm_slot_cache, mm_slot);
1801 }
1802
1803 static struct mm_slot *get_mm_slot(struct mm_struct *mm)
1804 {
1805         struct mm_slot *mm_slot;
1806
1807         hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
1808                 if (mm == mm_slot->mm)
1809                         return mm_slot;
1810
1811         return NULL;
1812 }
1813
1814 static void insert_to_mm_slots_hash(struct mm_struct *mm,
1815                                     struct mm_slot *mm_slot)
1816 {
1817         mm_slot->mm = mm;
1818         hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
1819 }
1820
1821 static inline int khugepaged_test_exit(struct mm_struct *mm)
1822 {
1823         return atomic_read(&mm->mm_users) == 0;
1824 }
1825
1826 int __khugepaged_enter(struct mm_struct *mm)
1827 {
1828         struct mm_slot *mm_slot;
1829         int wakeup;
1830
1831         mm_slot = alloc_mm_slot();
1832         if (!mm_slot)
1833                 return -ENOMEM;
1834
1835         /* __khugepaged_exit() must not run from under us */
1836         VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
1837         if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
1838                 free_mm_slot(mm_slot);
1839                 return 0;
1840         }
1841
1842         spin_lock(&khugepaged_mm_lock);
1843         insert_to_mm_slots_hash(mm, mm_slot);
1844         /*
1845          * Insert just behind the scanning cursor, to let the area settle
1846          * down a little.
1847          */
1848         wakeup = list_empty(&khugepaged_scan.mm_head);
1849         list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
1850         spin_unlock(&khugepaged_mm_lock);
1851
1852         atomic_inc(&mm->mm_count);
1853         if (wakeup)
1854                 wake_up_interruptible(&khugepaged_wait);
1855
1856         return 0;
1857 }
1858
1859 int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
1860                                unsigned long vm_flags)
1861 {
1862         unsigned long hstart, hend;
1863         if (!vma->anon_vma)
1864                 /*
1865                  * Not yet faulted in so we will register later in the
1866                  * page fault if needed.
1867                  */
1868                 return 0;
1869         if (vma->vm_ops)
1870                 /* khugepaged not yet working on file or special mappings */
1871                 return 0;
1872         VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma);
1873         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
1874         hend = vma->vm_end & HPAGE_PMD_MASK;
1875         if (hstart < hend)
1876                 return khugepaged_enter(vma, vm_flags);
1877         return 0;
1878 }
1879
1880 void __khugepaged_exit(struct mm_struct *mm)
1881 {
1882         struct mm_slot *mm_slot;
1883         int free = 0;
1884
1885         spin_lock(&khugepaged_mm_lock);
1886         mm_slot = get_mm_slot(mm);
1887         if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
1888                 hash_del(&mm_slot->hash);
1889                 list_del(&mm_slot->mm_node);
1890                 free = 1;
1891         }
1892         spin_unlock(&khugepaged_mm_lock);
1893
1894         if (free) {
1895                 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
1896                 free_mm_slot(mm_slot);
1897                 mmdrop(mm);
1898         } else if (mm_slot) {
1899                 /*
1900                  * This is required to serialize against
1901                  * khugepaged_test_exit() (which is guaranteed to run
1902                  * under mmap sem read mode). Stop here (after we
1903                  * return all pagetables will be destroyed) until
1904                  * khugepaged has finished working on the pagetables
1905                  * under the mmap_sem.
1906                  */
1907                 down_write(&mm->mmap_sem);
1908                 up_write(&mm->mmap_sem);
1909         }
1910 }
1911
1912 static void release_pte_page(struct page *page)
1913 {
1914         /* 0 stands for page_is_file_cache(page) == false */
1915         dec_zone_page_state(page, NR_ISOLATED_ANON + 0);
1916         unlock_page(page);
1917         putback_lru_page(page);
1918 }
1919
1920 static void release_pte_pages(pte_t *pte, pte_t *_pte)
1921 {
1922         while (--_pte >= pte) {
1923                 pte_t pteval = *_pte;
1924                 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
1925                         release_pte_page(pte_page(pteval));
1926         }
1927 }
1928
1929 static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
1930                                         unsigned long address,
1931                                         pte_t *pte)
1932 {
1933         struct page *page = NULL;
1934         pte_t *_pte;
1935         int none_or_zero = 0, result = 0;
1936         bool referenced = false, writable = false;
1937
1938         for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
1939              _pte++, address += PAGE_SIZE) {
1940                 pte_t pteval = *_pte;
1941                 if (pte_none(pteval) || (pte_present(pteval) &&
1942                                 is_zero_pfn(pte_pfn(pteval)))) {
1943                         if (!userfaultfd_armed(vma) &&
1944                             ++none_or_zero <= khugepaged_max_ptes_none) {
1945                                 continue;
1946                         } else {
1947                                 result = SCAN_EXCEED_NONE_PTE;
1948                                 goto out;
1949                         }
1950                 }
1951                 if (!pte_present(pteval)) {
1952                         result = SCAN_PTE_NON_PRESENT;
1953                         goto out;
1954                 }
1955                 page = vm_normal_page(vma, address, pteval);
1956                 if (unlikely(!page)) {
1957                         result = SCAN_PAGE_NULL;
1958                         goto out;
1959                 }
1960
1961                 VM_BUG_ON_PAGE(PageCompound(page), page);
1962                 VM_BUG_ON_PAGE(!PageAnon(page), page);
1963                 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1964
1965                 /*
1966                  * We can do it before isolate_lru_page because the
1967                  * page can't be freed from under us. NOTE: PG_lock
1968                  * is needed to serialize against split_huge_page
1969                  * when invoked from the VM.
1970                  */
1971                 if (!trylock_page(page)) {
1972                         result = SCAN_PAGE_LOCK;
1973                         goto out;
1974                 }
1975
1976                 /*
1977                  * cannot use mapcount: can't collapse if there's a gup pin.
1978                  * The page must only be referenced by the scanned process
1979                  * and page swap cache.
1980                  */
1981                 if (page_count(page) != 1 + !!PageSwapCache(page)) {
1982                         unlock_page(page);
1983                         result = SCAN_PAGE_COUNT;
1984                         goto out;
1985                 }
1986                 if (pte_write(pteval)) {
1987                         writable = true;
1988                 } else {
1989                         if (PageSwapCache(page) && !reuse_swap_page(page)) {
1990                                 unlock_page(page);
1991                                 result = SCAN_SWAP_CACHE_PAGE;
1992                                 goto out;
1993                         }
1994                         /*
1995                          * Page is not in the swap cache. It can be collapsed
1996                          * into a THP.
1997                          */
1998                 }
1999
2000                 /*
2001                  * Isolate the page to avoid collapsing an hugepage
2002                  * currently in use by the VM.
2003                  */
2004                 if (isolate_lru_page(page)) {
2005                         unlock_page(page);
2006                         result = SCAN_DEL_PAGE_LRU;
2007                         goto out;
2008                 }
2009                 /* 0 stands for page_is_file_cache(page) == false */
2010                 inc_zone_page_state(page, NR_ISOLATED_ANON + 0);
2011                 VM_BUG_ON_PAGE(!PageLocked(page), page);
2012                 VM_BUG_ON_PAGE(PageLRU(page), page);
2013
2014                 /* If there is no mapped pte young don't collapse the page */
2015                 if (pte_young(pteval) ||
2016                     page_is_young(page) || PageReferenced(page) ||
2017                     mmu_notifier_test_young(vma->vm_mm, address))
2018                         referenced = true;
2019         }
2020         if (likely(writable)) {
2021                 if (likely(referenced)) {
2022                         result = SCAN_SUCCEED;
2023                         trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero,
2024                                                             referenced, writable, result);
2025                         return 1;
2026                 }
2027         } else {
2028                 result = SCAN_PAGE_RO;
2029         }
2030
2031 out:
2032         release_pte_pages(pte, _pte);
2033         trace_mm_collapse_huge_page_isolate(page_to_pfn(page), none_or_zero,
2034                                             referenced, writable, result);
2035         return 0;
2036 }
2037
2038 static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
2039                                       struct vm_area_struct *vma,
2040                                       unsigned long address,
2041                                       spinlock_t *ptl)
2042 {
2043         pte_t *_pte;
2044         for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) {
2045                 pte_t pteval = *_pte;
2046                 struct page *src_page;
2047
2048                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
2049                         clear_user_highpage(page, address);
2050                         add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
2051                         if (is_zero_pfn(pte_pfn(pteval))) {
2052                                 /*
2053                                  * ptl mostly unnecessary.
2054                                  */
2055                                 spin_lock(ptl);
2056                                 /*
2057                                  * paravirt calls inside pte_clear here are
2058                                  * superfluous.
2059                                  */
2060                                 pte_clear(vma->vm_mm, address, _pte);
2061                                 spin_unlock(ptl);
2062                         }
2063                 } else {
2064                         src_page = pte_page(pteval);
2065                         copy_user_highpage(page, src_page, address, vma);
2066                         VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
2067                         release_pte_page(src_page);
2068                         /*
2069                          * ptl mostly unnecessary, but preempt has to
2070                          * be disabled to update the per-cpu stats
2071                          * inside page_remove_rmap().
2072                          */
2073                         spin_lock(ptl);
2074                         /*
2075                          * paravirt calls inside pte_clear here are
2076                          * superfluous.
2077                          */
2078                         pte_clear(vma->vm_mm, address, _pte);
2079                         page_remove_rmap(src_page, false);
2080                         spin_unlock(ptl);
2081                         free_page_and_swap_cache(src_page);
2082                 }
2083
2084                 address += PAGE_SIZE;
2085                 page++;
2086         }
2087 }
2088
2089 static void khugepaged_alloc_sleep(void)
2090 {
2091         DEFINE_WAIT(wait);
2092
2093         add_wait_queue(&khugepaged_wait, &wait);
2094         freezable_schedule_timeout_interruptible(
2095                 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
2096         remove_wait_queue(&khugepaged_wait, &wait);
2097 }
2098
2099 static int khugepaged_node_load[MAX_NUMNODES];
2100
2101 static bool khugepaged_scan_abort(int nid)
2102 {
2103         int i;
2104
2105         /*
2106          * If zone_reclaim_mode is disabled, then no extra effort is made to
2107          * allocate memory locally.
2108          */
2109         if (!zone_reclaim_mode)
2110                 return false;
2111
2112         /* If there is a count for this node already, it must be acceptable */
2113         if (khugepaged_node_load[nid])
2114                 return false;
2115
2116         for (i = 0; i < MAX_NUMNODES; i++) {
2117                 if (!khugepaged_node_load[i])
2118                         continue;
2119                 if (node_distance(nid, i) > RECLAIM_DISTANCE)
2120                         return true;
2121         }
2122         return false;
2123 }
2124
2125 #ifdef CONFIG_NUMA
2126 static int khugepaged_find_target_node(void)
2127 {
2128         static int last_khugepaged_target_node = NUMA_NO_NODE;
2129         int nid, target_node = 0, max_value = 0;
2130
2131         /* find first node with max normal pages hit */
2132         for (nid = 0; nid < MAX_NUMNODES; nid++)
2133                 if (khugepaged_node_load[nid] > max_value) {
2134                         max_value = khugepaged_node_load[nid];
2135                         target_node = nid;
2136                 }
2137
2138         /* do some balance if several nodes have the same hit record */
2139         if (target_node <= last_khugepaged_target_node)
2140                 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
2141                                 nid++)
2142                         if (max_value == khugepaged_node_load[nid]) {
2143                                 target_node = nid;
2144                                 break;
2145                         }
2146
2147         last_khugepaged_target_node = target_node;
2148         return target_node;
2149 }
2150
2151 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2152 {
2153         if (IS_ERR(*hpage)) {
2154                 if (!*wait)
2155                         return false;
2156
2157                 *wait = false;
2158                 *hpage = NULL;
2159                 khugepaged_alloc_sleep();
2160         } else if (*hpage) {
2161                 put_page(*hpage);
2162                 *hpage = NULL;
2163         }
2164
2165         return true;
2166 }
2167
2168 static struct page *
2169 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
2170                        unsigned long address, int node)
2171 {
2172         VM_BUG_ON_PAGE(*hpage, *hpage);
2173
2174         /*
2175          * Before allocating the hugepage, release the mmap_sem read lock.
2176          * The allocation can take potentially a long time if it involves
2177          * sync compaction, and we do not need to hold the mmap_sem during
2178          * that. We will recheck the vma after taking it again in write mode.
2179          */
2180         up_read(&mm->mmap_sem);
2181
2182         *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
2183         if (unlikely(!*hpage)) {
2184                 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2185                 *hpage = ERR_PTR(-ENOMEM);
2186                 return NULL;
2187         }
2188
2189         prep_transhuge_page(*hpage);
2190         count_vm_event(THP_COLLAPSE_ALLOC);
2191         return *hpage;
2192 }
2193 #else
2194 static int khugepaged_find_target_node(void)
2195 {
2196         return 0;
2197 }
2198
2199 static inline struct page *alloc_hugepage(int defrag)
2200 {
2201         struct page *page;
2202
2203         page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER);
2204         if (page)
2205                 prep_transhuge_page(page);
2206         return page;
2207 }
2208
2209 static struct page *khugepaged_alloc_hugepage(bool *wait)
2210 {
2211         struct page *hpage;
2212
2213         do {
2214                 hpage = alloc_hugepage(khugepaged_defrag());
2215                 if (!hpage) {
2216                         count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2217                         if (!*wait)
2218                                 return NULL;
2219
2220                         *wait = false;
2221                         khugepaged_alloc_sleep();
2222                 } else
2223                         count_vm_event(THP_COLLAPSE_ALLOC);
2224         } while (unlikely(!hpage) && likely(khugepaged_enabled()));
2225
2226         return hpage;
2227 }
2228
2229 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2230 {
2231         if (!*hpage)
2232                 *hpage = khugepaged_alloc_hugepage(wait);
2233
2234         if (unlikely(!*hpage))
2235                 return false;
2236
2237         return true;
2238 }
2239
2240 static struct page *
2241 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
2242                        unsigned long address, int node)
2243 {
2244         up_read(&mm->mmap_sem);
2245         VM_BUG_ON(!*hpage);
2246
2247         return  *hpage;
2248 }
2249 #endif
2250
2251 static bool hugepage_vma_check(struct vm_area_struct *vma)
2252 {
2253         if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
2254             (vma->vm_flags & VM_NOHUGEPAGE))
2255                 return false;
2256         if (!vma->anon_vma || vma->vm_ops)
2257                 return false;
2258         if (is_vma_temporary_stack(vma))
2259                 return false;
2260         VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma);
2261         return true;
2262 }
2263
2264 static void collapse_huge_page(struct mm_struct *mm,
2265                                    unsigned long address,
2266                                    struct page **hpage,
2267                                    struct vm_area_struct *vma,
2268                                    int node)
2269 {
2270         pmd_t *pmd, _pmd;
2271         pte_t *pte;
2272         pgtable_t pgtable;
2273         struct page *new_page;
2274         spinlock_t *pmd_ptl, *pte_ptl;
2275         int isolated, result = 0;
2276         unsigned long hstart, hend;
2277         struct mem_cgroup *memcg;
2278         unsigned long mmun_start;       /* For mmu_notifiers */
2279         unsigned long mmun_end;         /* For mmu_notifiers */
2280         gfp_t gfp;
2281
2282         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2283
2284         /* Only allocate from the target node */
2285         gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) |
2286                 __GFP_THISNODE;
2287
2288         /* release the mmap_sem read lock. */
2289         new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);
2290         if (!new_page) {
2291                 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
2292                 goto out_nolock;
2293         }
2294
2295         if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
2296                 result = SCAN_CGROUP_CHARGE_FAIL;
2297                 goto out_nolock;
2298         }
2299
2300         /*
2301          * Prevent all access to pagetables with the exception of
2302          * gup_fast later hanlded by the ptep_clear_flush and the VM
2303          * handled by the anon_vma lock + PG_lock.
2304          */
2305         down_write(&mm->mmap_sem);
2306         if (unlikely(khugepaged_test_exit(mm))) {
2307                 result = SCAN_ANY_PROCESS;
2308                 goto out;
2309         }
2310
2311         vma = find_vma(mm, address);
2312         if (!vma) {
2313                 result = SCAN_VMA_NULL;
2314                 goto out;
2315         }
2316         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2317         hend = vma->vm_end & HPAGE_PMD_MASK;
2318         if (address < hstart || address + HPAGE_PMD_SIZE > hend) {
2319                 result = SCAN_ADDRESS_RANGE;
2320                 goto out;
2321         }
2322         if (!hugepage_vma_check(vma)) {
2323                 result = SCAN_VMA_CHECK;
2324                 goto out;
2325         }
2326         pmd = mm_find_pmd(mm, address);
2327         if (!pmd) {
2328                 result = SCAN_PMD_NULL;
2329                 goto out;
2330         }
2331
2332         anon_vma_lock_write(vma->anon_vma);
2333
2334         pte = pte_offset_map(pmd, address);
2335         pte_ptl = pte_lockptr(mm, pmd);
2336
2337         mmun_start = address;
2338         mmun_end   = address + HPAGE_PMD_SIZE;
2339         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2340         pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
2341         /*
2342          * After this gup_fast can't run anymore. This also removes
2343          * any huge TLB entry from the CPU so we won't allow
2344          * huge and small TLB entries for the same virtual address
2345          * to avoid the risk of CPU bugs in that area.
2346          */
2347         _pmd = pmdp_collapse_flush(vma, address, pmd);
2348         spin_unlock(pmd_ptl);
2349         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2350
2351         spin_lock(pte_ptl);
2352         isolated = __collapse_huge_page_isolate(vma, address, pte);
2353         spin_unlock(pte_ptl);
2354
2355         if (unlikely(!isolated)) {
2356                 pte_unmap(pte);
2357                 spin_lock(pmd_ptl);
2358                 BUG_ON(!pmd_none(*pmd));
2359                 /*
2360                  * We can only use set_pmd_at when establishing
2361                  * hugepmds and never for establishing regular pmds that
2362                  * points to regular pagetables. Use pmd_populate for that
2363                  */
2364                 pmd_populate(mm, pmd, pmd_pgtable(_pmd));
2365                 spin_unlock(pmd_ptl);
2366                 anon_vma_unlock_write(vma->anon_vma);
2367                 result = SCAN_FAIL;
2368                 goto out;
2369         }
2370
2371         /*
2372          * All pages are isolated and locked so anon_vma rmap
2373          * can't run anymore.
2374          */
2375         anon_vma_unlock_write(vma->anon_vma);
2376
2377         __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
2378         pte_unmap(pte);
2379         __SetPageUptodate(new_page);
2380         pgtable = pmd_pgtable(_pmd);
2381
2382         _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
2383         _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
2384
2385         /*
2386          * spin_lock() below is not the equivalent of smp_wmb(), so
2387          * this is needed to avoid the copy_huge_page writes to become
2388          * visible after the set_pmd_at() write.
2389          */
2390         smp_wmb();
2391
2392         spin_lock(pmd_ptl);
2393         BUG_ON(!pmd_none(*pmd));
2394         page_add_new_anon_rmap(new_page, vma, address, true);
2395         mem_cgroup_commit_charge(new_page, memcg, false, true);
2396         lru_cache_add_active_or_unevictable(new_page, vma);
2397         pgtable_trans_huge_deposit(mm, pmd, pgtable);
2398         set_pmd_at(mm, address, pmd, _pmd);
2399         update_mmu_cache_pmd(vma, address, pmd);
2400         spin_unlock(pmd_ptl);
2401
2402         *hpage = NULL;
2403
2404         khugepaged_pages_collapsed++;
2405         result = SCAN_SUCCEED;
2406 out_up_write:
2407         up_write(&mm->mmap_sem);
2408         trace_mm_collapse_huge_page(mm, isolated, result);
2409         return;
2410
2411 out_nolock:
2412         trace_mm_collapse_huge_page(mm, isolated, result);
2413         return;
2414 out:
2415         mem_cgroup_cancel_charge(new_page, memcg, true);
2416         goto out_up_write;
2417 }
2418
2419 static int khugepaged_scan_pmd(struct mm_struct *mm,
2420                                struct vm_area_struct *vma,
2421                                unsigned long address,
2422                                struct page **hpage)
2423 {
2424         pmd_t *pmd;
2425         pte_t *pte, *_pte;
2426         int ret = 0, none_or_zero = 0, result = 0;
2427         struct page *page = NULL;
2428         unsigned long _address;
2429         spinlock_t *ptl;
2430         int node = NUMA_NO_NODE;
2431         bool writable = false, referenced = false;
2432
2433         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2434
2435         pmd = mm_find_pmd(mm, address);
2436         if (!pmd) {
2437                 result = SCAN_PMD_NULL;
2438                 goto out;
2439         }
2440
2441         memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
2442         pte = pte_offset_map_lock(mm, pmd, address, &ptl);
2443         for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
2444              _pte++, _address += PAGE_SIZE) {
2445                 pte_t pteval = *_pte;
2446                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
2447                         if (!userfaultfd_armed(vma) &&
2448                             ++none_or_zero <= khugepaged_max_ptes_none) {
2449                                 continue;
2450                         } else {
2451                                 result = SCAN_EXCEED_NONE_PTE;
2452                                 goto out_unmap;
2453                         }
2454                 }
2455                 if (!pte_present(pteval)) {
2456                         result = SCAN_PTE_NON_PRESENT;
2457                         goto out_unmap;
2458                 }
2459                 if (pte_write(pteval))
2460                         writable = true;
2461
2462                 page = vm_normal_page(vma, _address, pteval);
2463                 if (unlikely(!page)) {
2464                         result = SCAN_PAGE_NULL;
2465                         goto out_unmap;
2466                 }
2467
2468                 /* TODO: teach khugepaged to collapse THP mapped with pte */
2469                 if (PageCompound(page)) {
2470                         result = SCAN_PAGE_COMPOUND;
2471                         goto out_unmap;
2472                 }
2473
2474                 /*
2475                  * Record which node the original page is from and save this
2476                  * information to khugepaged_node_load[].
2477                  * Khupaged will allocate hugepage from the node has the max
2478                  * hit record.
2479                  */
2480                 node = page_to_nid(page);
2481                 if (khugepaged_scan_abort(node)) {
2482                         result = SCAN_SCAN_ABORT;
2483                         goto out_unmap;
2484                 }
2485                 khugepaged_node_load[node]++;
2486                 if (!PageLRU(page)) {
2487                         result = SCAN_SCAN_ABORT;
2488                         goto out_unmap;
2489                 }
2490                 if (PageLocked(page)) {
2491                         result = SCAN_PAGE_LOCK;
2492                         goto out_unmap;
2493                 }
2494                 if (!PageAnon(page)) {
2495                         result = SCAN_PAGE_ANON;
2496                         goto out_unmap;
2497                 }
2498
2499                 /*
2500                  * cannot use mapcount: can't collapse if there's a gup pin.
2501                  * The page must only be referenced by the scanned process
2502                  * and page swap cache.
2503                  */
2504                 if (page_count(page) != 1 + !!PageSwapCache(page)) {
2505                         result = SCAN_PAGE_COUNT;
2506                         goto out_unmap;
2507                 }
2508                 if (pte_young(pteval) ||
2509                     page_is_young(page) || PageReferenced(page) ||
2510                     mmu_notifier_test_young(vma->vm_mm, address))
2511                         referenced = true;
2512         }
2513         if (writable) {
2514                 if (referenced) {
2515                         result = SCAN_SUCCEED;
2516                         ret = 1;
2517                 } else {
2518                         result = SCAN_NO_REFERENCED_PAGE;
2519                 }
2520         } else {
2521                 result = SCAN_PAGE_RO;
2522         }
2523 out_unmap:
2524         pte_unmap_unlock(pte, ptl);
2525         if (ret) {
2526                 node = khugepaged_find_target_node();
2527                 /* collapse_huge_page will return with the mmap_sem released */
2528                 collapse_huge_page(mm, address, hpage, vma, node);
2529         }
2530 out:
2531         trace_mm_khugepaged_scan_pmd(mm, page_to_pfn(page), writable, referenced,
2532                                      none_or_zero, result);
2533         return ret;
2534 }
2535
2536 static void collect_mm_slot(struct mm_slot *mm_slot)
2537 {
2538         struct mm_struct *mm = mm_slot->mm;
2539
2540         VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
2541
2542         if (khugepaged_test_exit(mm)) {
2543                 /* free mm_slot */
2544                 hash_del(&mm_slot->hash);
2545                 list_del(&mm_slot->mm_node);
2546
2547                 /*
2548                  * Not strictly needed because the mm exited already.
2549                  *
2550                  * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2551                  */
2552
2553                 /* khugepaged_mm_lock actually not necessary for the below */
2554                 free_mm_slot(mm_slot);
2555                 mmdrop(mm);
2556         }
2557 }
2558
2559 static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
2560                                             struct page **hpage)
2561         __releases(&khugepaged_mm_lock)
2562         __acquires(&khugepaged_mm_lock)
2563 {
2564         struct mm_slot *mm_slot;
2565         struct mm_struct *mm;
2566         struct vm_area_struct *vma;
2567         int progress = 0;
2568
2569         VM_BUG_ON(!pages);
2570         VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
2571
2572         if (khugepaged_scan.mm_slot)
2573                 mm_slot = khugepaged_scan.mm_slot;
2574         else {
2575                 mm_slot = list_entry(khugepaged_scan.mm_head.next,
2576                                      struct mm_slot, mm_node);
2577                 khugepaged_scan.address = 0;
2578                 khugepaged_scan.mm_slot = mm_slot;
2579         }
2580         spin_unlock(&khugepaged_mm_lock);
2581
2582         mm = mm_slot->mm;
2583         down_read(&mm->mmap_sem);
2584         if (unlikely(khugepaged_test_exit(mm)))
2585                 vma = NULL;
2586         else
2587                 vma = find_vma(mm, khugepaged_scan.address);
2588
2589         progress++;
2590         for (; vma; vma = vma->vm_next) {
2591                 unsigned long hstart, hend;
2592
2593                 cond_resched();
2594                 if (unlikely(khugepaged_test_exit(mm))) {
2595                         progress++;
2596                         break;
2597                 }
2598                 if (!hugepage_vma_check(vma)) {
2599 skip:
2600                         progress++;
2601                         continue;
2602                 }
2603                 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2604                 hend = vma->vm_end & HPAGE_PMD_MASK;
2605                 if (hstart >= hend)
2606                         goto skip;
2607                 if (khugepaged_scan.address > hend)
2608                         goto skip;
2609                 if (khugepaged_scan.address < hstart)
2610                         khugepaged_scan.address = hstart;
2611                 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
2612
2613                 while (khugepaged_scan.address < hend) {
2614                         int ret;
2615                         cond_resched();
2616                         if (unlikely(khugepaged_test_exit(mm)))
2617                                 goto breakouterloop;
2618
2619                         VM_BUG_ON(khugepaged_scan.address < hstart ||
2620                                   khugepaged_scan.address + HPAGE_PMD_SIZE >
2621                                   hend);
2622                         ret = khugepaged_scan_pmd(mm, vma,
2623                                                   khugepaged_scan.address,
2624                                                   hpage);
2625                         /* move to next address */
2626                         khugepaged_scan.address += HPAGE_PMD_SIZE;
2627                         progress += HPAGE_PMD_NR;
2628                         if (ret)
2629                                 /* we released mmap_sem so break loop */
2630                                 goto breakouterloop_mmap_sem;
2631                         if (progress >= pages)
2632                                 goto breakouterloop;
2633                 }
2634         }
2635 breakouterloop:
2636         up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
2637 breakouterloop_mmap_sem:
2638
2639         spin_lock(&khugepaged_mm_lock);
2640         VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
2641         /*
2642          * Release the current mm_slot if this mm is about to die, or
2643          * if we scanned all vmas of this mm.
2644          */
2645         if (khugepaged_test_exit(mm) || !vma) {
2646                 /*
2647                  * Make sure that if mm_users is reaching zero while
2648                  * khugepaged runs here, khugepaged_exit will find
2649                  * mm_slot not pointing to the exiting mm.
2650                  */
2651                 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
2652                         khugepaged_scan.mm_slot = list_entry(
2653                                 mm_slot->mm_node.next,
2654                                 struct mm_slot, mm_node);
2655                         khugepaged_scan.address = 0;
2656                 } else {
2657                         khugepaged_scan.mm_slot = NULL;
2658                         khugepaged_full_scans++;
2659                 }
2660
2661                 collect_mm_slot(mm_slot);
2662         }
2663
2664         return progress;
2665 }
2666
2667 static int khugepaged_has_work(void)
2668 {
2669         return !list_empty(&khugepaged_scan.mm_head) &&
2670                 khugepaged_enabled();
2671 }
2672
2673 static int khugepaged_wait_event(void)
2674 {
2675         return !list_empty(&khugepaged_scan.mm_head) ||
2676                 kthread_should_stop();
2677 }
2678
2679 static void khugepaged_do_scan(void)
2680 {
2681         struct page *hpage = NULL;
2682         unsigned int progress = 0, pass_through_head = 0;
2683         unsigned int pages = khugepaged_pages_to_scan;
2684         bool wait = true;
2685
2686         barrier(); /* write khugepaged_pages_to_scan to local stack */
2687
2688         while (progress < pages) {
2689                 if (!khugepaged_prealloc_page(&hpage, &wait))
2690                         break;
2691
2692                 cond_resched();
2693
2694                 if (unlikely(kthread_should_stop() || try_to_freeze()))
2695                         break;
2696
2697                 spin_lock(&khugepaged_mm_lock);
2698                 if (!khugepaged_scan.mm_slot)
2699                         pass_through_head++;
2700                 if (khugepaged_has_work() &&
2701                     pass_through_head < 2)
2702                         progress += khugepaged_scan_mm_slot(pages - progress,
2703                                                             &hpage);
2704                 else
2705                         progress = pages;
2706                 spin_unlock(&khugepaged_mm_lock);
2707         }
2708
2709         if (!IS_ERR_OR_NULL(hpage))
2710                 put_page(hpage);
2711 }
2712
2713 static void khugepaged_wait_work(void)
2714 {
2715         if (khugepaged_has_work()) {
2716                 if (!khugepaged_scan_sleep_millisecs)
2717                         return;
2718
2719                 wait_event_freezable_timeout(khugepaged_wait,
2720                                              kthread_should_stop(),
2721                         msecs_to_jiffies(khugepaged_scan_sleep_millisecs));
2722                 return;
2723         }
2724
2725         if (khugepaged_enabled())
2726                 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
2727 }
2728
2729 static int khugepaged(void *none)
2730 {
2731         struct mm_slot *mm_slot;
2732
2733         set_freezable();
2734         set_user_nice(current, MAX_NICE);
2735
2736         while (!kthread_should_stop()) {
2737                 khugepaged_do_scan();
2738                 khugepaged_wait_work();
2739         }
2740
2741         spin_lock(&khugepaged_mm_lock);
2742         mm_slot = khugepaged_scan.mm_slot;
2743         khugepaged_scan.mm_slot = NULL;
2744         if (mm_slot)
2745                 collect_mm_slot(mm_slot);
2746         spin_unlock(&khugepaged_mm_lock);
2747         return 0;
2748 }
2749
2750 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2751                 unsigned long haddr, pmd_t *pmd)
2752 {
2753         struct mm_struct *mm = vma->vm_mm;
2754         pgtable_t pgtable;
2755         pmd_t _pmd;
2756         int i;
2757
2758         /* leave pmd empty until pte is filled */
2759         pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2760
2761         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2762         pmd_populate(mm, &_pmd, pgtable);
2763
2764         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2765                 pte_t *pte, entry;
2766                 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2767                 entry = pte_mkspecial(entry);
2768                 pte = pte_offset_map(&_pmd, haddr);
2769                 VM_BUG_ON(!pte_none(*pte));
2770                 set_pte_at(mm, haddr, pte, entry);
2771                 pte_unmap(pte);
2772         }
2773         smp_wmb(); /* make pte visible before pmd */
2774         pmd_populate(mm, pmd, pgtable);
2775         put_huge_zero_page();
2776 }
2777
2778 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
2779                 unsigned long haddr, bool freeze)
2780 {
2781         struct mm_struct *mm = vma->vm_mm;
2782         struct page *page;
2783         pgtable_t pgtable;
2784         pmd_t _pmd;
2785         bool young, write, dirty;
2786         int i;
2787
2788         VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2789         VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2790         VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
2791         VM_BUG_ON(!pmd_trans_huge(*pmd));
2792
2793         count_vm_event(THP_SPLIT_PMD);
2794
2795         if (vma_is_dax(vma)) {
2796                 pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2797                 if (is_huge_zero_pmd(_pmd))
2798                         put_huge_zero_page();
2799                 return;
2800         } else if (is_huge_zero_pmd(*pmd)) {
2801                 return __split_huge_zero_page_pmd(vma, haddr, pmd);
2802         }
2803
2804         page = pmd_page(*pmd);
2805         VM_BUG_ON_PAGE(!page_count(page), page);
2806         atomic_add(HPAGE_PMD_NR - 1, &page->_count);
2807         write = pmd_write(*pmd);
2808         young = pmd_young(*pmd);
2809         dirty = pmd_dirty(*pmd);
2810
2811         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2812         pmd_populate(mm, &_pmd, pgtable);
2813
2814         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2815                 pte_t entry, *pte;
2816                 /*
2817                  * Note that NUMA hinting access restrictions are not
2818                  * transferred to avoid any possibility of altering
2819                  * permissions across VMAs.
2820                  */
2821                 if (freeze) {
2822                         swp_entry_t swp_entry;
2823                         swp_entry = make_migration_entry(page + i, write);
2824                         entry = swp_entry_to_pte(swp_entry);
2825                 } else {
2826                         entry = mk_pte(page + i, vma->vm_page_prot);
2827                         entry = maybe_mkwrite(entry, vma);
2828                         if (!write)
2829                                 entry = pte_wrprotect(entry);
2830                         if (!young)
2831                                 entry = pte_mkold(entry);
2832                 }
2833                 if (dirty)
2834                         SetPageDirty(page + i);
2835                 pte = pte_offset_map(&_pmd, haddr);
2836                 BUG_ON(!pte_none(*pte));
2837                 set_pte_at(mm, haddr, pte, entry);
2838                 atomic_inc(&page[i]._mapcount);
2839                 pte_unmap(pte);
2840         }
2841
2842         /*
2843          * Set PG_double_map before dropping compound_mapcount to avoid
2844          * false-negative page_mapped().
2845          */
2846         if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
2847                 for (i = 0; i < HPAGE_PMD_NR; i++)
2848                         atomic_inc(&page[i]._mapcount);
2849         }
2850
2851         if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
2852                 /* Last compound_mapcount is gone. */
2853                 __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
2854                 if (TestClearPageDoubleMap(page)) {
2855                         /* No need in mapcount reference anymore */
2856                         for (i = 0; i < HPAGE_PMD_NR; i++)
2857                                 atomic_dec(&page[i]._mapcount);
2858                 }
2859         }
2860
2861         smp_wmb(); /* make pte visible before pmd */
2862         /*
2863          * Up to this point the pmd is present and huge and userland has the
2864          * whole access to the hugepage during the split (which happens in
2865          * place). If we overwrite the pmd with the not-huge version pointing
2866          * to the pte here (which of course we could if all CPUs were bug
2867          * free), userland could trigger a small page size TLB miss on the
2868          * small sized TLB while the hugepage TLB entry is still established in
2869          * the huge TLB. Some CPU doesn't like that.
2870          * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
2871          * 383 on page 93. Intel should be safe but is also warns that it's
2872          * only safe if the permission and cache attributes of the two entries
2873          * loaded in the two TLB is identical (which should be the case here).
2874          * But it is generally safer to never allow small and huge TLB entries
2875          * for the same virtual address to be loaded simultaneously. So instead
2876          * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
2877          * current pmd notpresent (atomically because here the pmd_trans_huge
2878          * and pmd_trans_splitting must remain set at all times on the pmd
2879          * until the split is complete for this pmd), then we flush the SMP TLB
2880          * and finally we write the non-huge version of the pmd entry with
2881          * pmd_populate.
2882          */
2883         pmdp_invalidate(vma, haddr, pmd);
2884         pmd_populate(mm, pmd, pgtable);
2885
2886         if (freeze) {
2887                 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2888                         page_remove_rmap(page + i, false);
2889                         put_page(page + i);
2890                 }
2891         }
2892 }
2893
2894 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2895                 unsigned long address)
2896 {
2897         spinlock_t *ptl;
2898         struct mm_struct *mm = vma->vm_mm;
2899         struct page *page = NULL;
2900         unsigned long haddr = address & HPAGE_PMD_MASK;
2901
2902         mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
2903         ptl = pmd_lock(mm, pmd);
2904         if (unlikely(!pmd_trans_huge(*pmd)))
2905                 goto out;
2906         page = pmd_page(*pmd);
2907         __split_huge_pmd_locked(vma, pmd, haddr, false);
2908         if (PageMlocked(page))
2909                 get_page(page);
2910         else
2911                 page = NULL;
2912 out:
2913         spin_unlock(ptl);
2914         mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
2915         if (page) {
2916                 lock_page(page);
2917                 munlock_vma_page(page);
2918                 unlock_page(page);
2919                 put_page(page);
2920         }
2921 }
2922
2923 static void split_huge_pmd_address(struct vm_area_struct *vma,
2924                                     unsigned long address)
2925 {
2926         pgd_t *pgd;
2927         pud_t *pud;
2928         pmd_t *pmd;
2929
2930         VM_BUG_ON(!(address & ~HPAGE_PMD_MASK));
2931
2932         pgd = pgd_offset(vma->vm_mm, address);
2933         if (!pgd_present(*pgd))
2934                 return;
2935
2936         pud = pud_offset(pgd, address);
2937         if (!pud_present(*pud))
2938                 return;
2939
2940         pmd = pmd_offset(pud, address);
2941         if (!pmd_present(*pmd) || !pmd_trans_huge(*pmd))
2942                 return;
2943         /*
2944          * Caller holds the mmap_sem write mode, so a huge pmd cannot
2945          * materialize from under us.
2946          */
2947         split_huge_pmd(vma, pmd, address);
2948 }
2949
2950 void vma_adjust_trans_huge(struct vm_area_struct *vma,
2951                              unsigned long start,
2952                              unsigned long end,
2953                              long adjust_next)
2954 {
2955         /*
2956          * If the new start address isn't hpage aligned and it could
2957          * previously contain an hugepage: check if we need to split
2958          * an huge pmd.
2959          */
2960         if (start & ~HPAGE_PMD_MASK &&
2961             (start & HPAGE_PMD_MASK) >= vma->vm_start &&
2962             (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2963                 split_huge_pmd_address(vma, start);
2964
2965         /*
2966          * If the new end address isn't hpage aligned and it could
2967          * previously contain an hugepage: check if we need to split
2968          * an huge pmd.
2969          */
2970         if (end & ~HPAGE_PMD_MASK &&
2971             (end & HPAGE_PMD_MASK) >= vma->vm_start &&
2972             (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2973                 split_huge_pmd_address(vma, end);
2974
2975         /*
2976          * If we're also updating the vma->vm_next->vm_start, if the new
2977          * vm_next->vm_start isn't page aligned and it could previously
2978          * contain an hugepage: check if we need to split an huge pmd.
2979          */
2980         if (adjust_next > 0) {
2981                 struct vm_area_struct *next = vma->vm_next;
2982                 unsigned long nstart = next->vm_start;
2983                 nstart += adjust_next << PAGE_SHIFT;
2984                 if (nstart & ~HPAGE_PMD_MASK &&
2985                     (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
2986                     (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
2987                         split_huge_pmd_address(next, nstart);
2988         }
2989 }
2990
2991 static void freeze_page_vma(struct vm_area_struct *vma, struct page *page,
2992                 unsigned long address)
2993 {
2994         unsigned long haddr = address & HPAGE_PMD_MASK;
2995         spinlock_t *ptl;
2996         pgd_t *pgd;
2997         pud_t *pud;
2998         pmd_t *pmd;
2999         pte_t *pte;
3000         int i, nr = HPAGE_PMD_NR;
3001
3002         /* Skip pages which doesn't belong to the VMA */
3003         if (address < vma->vm_start) {
3004                 int off = (vma->vm_start - address) >> PAGE_SHIFT;
3005                 page += off;
3006                 nr -= off;
3007                 address = vma->vm_start;
3008         }
3009
3010         pgd = pgd_offset(vma->vm_mm, address);
3011         if (!pgd_present(*pgd))
3012                 return;
3013         pud = pud_offset(pgd, address);
3014         if (!pud_present(*pud))
3015                 return;
3016         pmd = pmd_offset(pud, address);
3017         ptl = pmd_lock(vma->vm_mm, pmd);
3018         if (!pmd_present(*pmd)) {
3019                 spin_unlock(ptl);
3020                 return;
3021         }
3022         if (pmd_trans_huge(*pmd)) {
3023                 if (page == pmd_page(*pmd))
3024                         __split_huge_pmd_locked(vma, pmd, haddr, true);
3025                 spin_unlock(ptl);
3026                 return;
3027         }
3028         spin_unlock(ptl);
3029
3030         pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
3031         for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3032                 pte_t entry, swp_pte;
3033                 swp_entry_t swp_entry;
3034
3035                 /*
3036                  * We've just crossed page table boundary: need to map next one.
3037                  * It can happen if THP was mremaped to non PMD-aligned address.
3038                  */
3039                 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3040                         pte_unmap_unlock(pte - 1, ptl);
3041                         pmd = mm_find_pmd(vma->vm_mm, address);
3042                         if (!pmd)
3043                                 return;
3044                         pte = pte_offset_map_lock(vma->vm_mm, pmd,
3045                                         address, &ptl);
3046                 }
3047
3048                 if (!pte_present(*pte))
3049                         continue;
3050                 if (page_to_pfn(page) != pte_pfn(*pte))
3051                         continue;
3052                 flush_cache_page(vma, address, page_to_pfn(page));
3053                 entry = ptep_clear_flush(vma, address, pte);
3054                 if (pte_dirty(entry))
3055                         SetPageDirty(page);
3056                 swp_entry = make_migration_entry(page, pte_write(entry));
3057                 swp_pte = swp_entry_to_pte(swp_entry);
3058                 if (pte_soft_dirty(entry))
3059                         swp_pte = pte_swp_mksoft_dirty(swp_pte);
3060                 set_pte_at(vma->vm_mm, address, pte, swp_pte);
3061                 page_remove_rmap(page, false);
3062                 put_page(page);
3063         }
3064         pte_unmap_unlock(pte - 1, ptl);
3065 }
3066
3067 static void freeze_page(struct anon_vma *anon_vma, struct page *page)
3068 {
3069         struct anon_vma_chain *avc;
3070         pgoff_t pgoff = page_to_pgoff(page);
3071
3072         VM_BUG_ON_PAGE(!PageHead(page), page);
3073
3074         anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff,
3075                         pgoff + HPAGE_PMD_NR - 1) {
3076                 unsigned long address = __vma_address(page, avc->vma);
3077
3078                 mmu_notifier_invalidate_range_start(avc->vma->vm_mm,
3079                                 address, address + HPAGE_PMD_SIZE);
3080                 freeze_page_vma(avc->vma, page, address);
3081                 mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3082                                 address, address + HPAGE_PMD_SIZE);
3083         }
3084 }
3085
3086 static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page,
3087                 unsigned long address)
3088 {
3089         spinlock_t *ptl;
3090         pmd_t *pmd;
3091         pte_t *pte, entry;
3092         swp_entry_t swp_entry;
3093         unsigned long haddr = address & HPAGE_PMD_MASK;
3094         int i, nr = HPAGE_PMD_NR;
3095
3096         /* Skip pages which doesn't belong to the VMA */
3097         if (address < vma->vm_start) {
3098                 int off = (vma->vm_start - address) >> PAGE_SHIFT;
3099                 page += off;
3100                 nr -= off;
3101                 address = vma->vm_start;
3102         }
3103
3104         pmd = mm_find_pmd(vma->vm_mm, address);
3105         if (!pmd)
3106                 return;
3107
3108         pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
3109         for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3110                 /*
3111                  * We've just crossed page table boundary: need to map next one.
3112                  * It can happen if THP was mremaped to non-PMD aligned address.
3113                  */
3114                 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3115                         pte_unmap_unlock(pte - 1, ptl);
3116                         pmd = mm_find_pmd(vma->vm_mm, address);
3117                         if (!pmd)
3118                                 return;
3119                         pte = pte_offset_map_lock(vma->vm_mm, pmd,
3120                                         address, &ptl);
3121                 }
3122
3123                 if (!is_swap_pte(*pte))
3124                         continue;
3125
3126                 swp_entry = pte_to_swp_entry(*pte);
3127                 if (!is_migration_entry(swp_entry))
3128                         continue;
3129                 if (migration_entry_to_page(swp_entry) != page)
3130                         continue;
3131
3132                 get_page(page);
3133                 page_add_anon_rmap(page, vma, address, false);
3134
3135                 entry = pte_mkold(mk_pte(page, vma->vm_page_prot));
3136                 if (PageDirty(page))
3137                         entry = pte_mkdirty(entry);
3138                 if (is_write_migration_entry(swp_entry))
3139                         entry = maybe_mkwrite(entry, vma);
3140
3141                 flush_dcache_page(page);
3142                 set_pte_at(vma->vm_mm, address, pte, entry);
3143
3144                 /* No need to invalidate - it was non-present before */
3145                 update_mmu_cache(vma, address, pte);
3146         }
3147         pte_unmap_unlock(pte - 1, ptl);
3148 }
3149
3150 static void unfreeze_page(struct anon_vma *anon_vma, struct page *page)
3151 {
3152         struct anon_vma_chain *avc;
3153         pgoff_t pgoff = page_to_pgoff(page);
3154
3155         anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
3156                         pgoff, pgoff + HPAGE_PMD_NR - 1) {
3157                 unsigned long address = __vma_address(page, avc->vma);
3158
3159                 mmu_notifier_invalidate_range_start(avc->vma->vm_mm,
3160                                 address, address + HPAGE_PMD_SIZE);
3161                 unfreeze_page_vma(avc->vma, page, address);
3162                 mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3163                                 address, address + HPAGE_PMD_SIZE);
3164         }
3165 }
3166
3167 static int __split_huge_page_tail(struct page *head, int tail,
3168                 struct lruvec *lruvec, struct list_head *list)
3169 {
3170         int mapcount;
3171         struct page *page_tail = head + tail;
3172
3173         mapcount = atomic_read(&page_tail->_mapcount) + 1;
3174         VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail);
3175
3176         /*
3177          * tail_page->_count is zero and not changing from under us. But
3178          * get_page_unless_zero() may be running from under us on the
3179          * tail_page. If we used atomic_set() below instead of atomic_add(), we
3180          * would then run atomic_set() concurrently with
3181          * get_page_unless_zero(), and atomic_set() is implemented in C not
3182          * using locked ops. spin_unlock on x86 sometime uses locked ops
3183          * because of PPro errata 66, 92, so unless somebody can guarantee
3184          * atomic_set() here would be safe on all archs (and not only on x86),
3185          * it's safer to use atomic_add().
3186          */
3187         atomic_add(mapcount + 1, &page_tail->_count);
3188
3189
3190         page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
3191         page_tail->flags |= (head->flags &
3192                         ((1L << PG_referenced) |
3193                          (1L << PG_swapbacked) |
3194                          (1L << PG_mlocked) |
3195                          (1L << PG_uptodate) |
3196                          (1L << PG_active) |
3197                          (1L << PG_locked) |
3198                          (1L << PG_unevictable) |
3199                          (1L << PG_dirty)));
3200
3201         /*
3202          * After clearing PageTail the gup refcount can be released.
3203          * Page flags also must be visible before we make the page non-compound.
3204          */
3205         smp_wmb();
3206
3207         clear_compound_head(page_tail);
3208
3209         if (page_is_young(head))
3210                 set_page_young(page_tail);
3211         if (page_is_idle(head))
3212                 set_page_idle(page_tail);
3213
3214         /* ->mapping in first tail page is compound_mapcount */
3215         VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
3216                         page_tail);
3217         page_tail->mapping = head->mapping;
3218
3219         page_tail->index = head->index + tail;
3220         page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
3221         lru_add_page_tail(head, page_tail, lruvec, list);
3222
3223         return mapcount;
3224 }
3225
3226 static void __split_huge_page(struct page *page, struct list_head *list)
3227 {
3228         struct page *head = compound_head(page);
3229         struct zone *zone = page_zone(head);
3230         struct lruvec *lruvec;
3231         int i, tail_mapcount;
3232
3233         /* prevent PageLRU to go away from under us, and freeze lru stats */
3234         spin_lock_irq(&zone->lru_lock);
3235         lruvec = mem_cgroup_page_lruvec(head, zone);
3236
3237         /* complete memcg works before add pages to LRU */
3238         mem_cgroup_split_huge_fixup(head);
3239
3240         tail_mapcount = 0;
3241         for (i = HPAGE_PMD_NR - 1; i >= 1; i--)
3242                 tail_mapcount += __split_huge_page_tail(head, i, lruvec, list);
3243         atomic_sub(tail_mapcount, &head->_count);
3244
3245         ClearPageCompound(head);
3246         spin_unlock_irq(&zone->lru_lock);
3247
3248         unfreeze_page(page_anon_vma(head), head);
3249
3250         for (i = 0; i < HPAGE_PMD_NR; i++) {
3251                 struct page *subpage = head + i;
3252                 if (subpage == page)
3253                         continue;
3254                 unlock_page(subpage);
3255
3256                 /*
3257                  * Subpages may be freed if there wasn't any mapping
3258                  * like if add_to_swap() is running on a lru page that
3259                  * had its mapping zapped. And freeing these pages
3260                  * requires taking the lru_lock so we do the put_page
3261                  * of the tail pages after the split is complete.
3262                  */
3263                 put_page(subpage);
3264         }
3265 }
3266
3267 int total_mapcount(struct page *page)
3268 {
3269         int i, ret;
3270
3271         VM_BUG_ON_PAGE(PageTail(page), page);
3272
3273         if (likely(!PageCompound(page)))
3274                 return atomic_read(&page->_mapcount) + 1;
3275
3276         ret = compound_mapcount(page);
3277         if (PageHuge(page))
3278                 return ret;
3279         for (i = 0; i < HPAGE_PMD_NR; i++)
3280                 ret += atomic_read(&page[i]._mapcount) + 1;
3281         if (PageDoubleMap(page))
3282                 ret -= HPAGE_PMD_NR;
3283         return ret;
3284 }
3285
3286 /*
3287  * This function splits huge page into normal pages. @page can point to any
3288  * subpage of huge page to split. Split doesn't change the position of @page.
3289  *
3290  * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
3291  * The huge page must be locked.
3292  *
3293  * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
3294  *
3295  * Both head page and tail pages will inherit mapping, flags, and so on from
3296  * the hugepage.
3297  *
3298  * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
3299  * they are not mapped.
3300  *
3301  * Returns 0 if the hugepage is split successfully.
3302  * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
3303  * us.
3304  */
3305 int split_huge_page_to_list(struct page *page, struct list_head *list)
3306 {
3307         struct page *head = compound_head(page);
3308         struct anon_vma *anon_vma;
3309         int count, mapcount, ret;
3310         bool mlocked;
3311
3312         VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
3313         VM_BUG_ON_PAGE(!PageAnon(page), page);
3314         VM_BUG_ON_PAGE(!PageLocked(page), page);
3315         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
3316         VM_BUG_ON_PAGE(!PageCompound(page), page);
3317
3318         /*
3319          * The caller does not necessarily hold an mmap_sem that would prevent
3320          * the anon_vma disappearing so we first we take a reference to it
3321          * and then lock the anon_vma for write. This is similar to
3322          * page_lock_anon_vma_read except the write lock is taken to serialise
3323          * against parallel split or collapse operations.
3324          */
3325         anon_vma = page_get_anon_vma(head);
3326         if (!anon_vma) {
3327                 ret = -EBUSY;
3328                 goto out;
3329         }
3330         anon_vma_lock_write(anon_vma);
3331
3332         /*
3333          * Racy check if we can split the page, before freeze_page() will
3334          * split PMDs
3335          */
3336         if (total_mapcount(head) != page_count(head) - 1) {
3337                 ret = -EBUSY;
3338                 goto out_unlock;
3339         }
3340
3341         mlocked = PageMlocked(page);
3342         freeze_page(anon_vma, head);
3343         VM_BUG_ON_PAGE(compound_mapcount(head), head);
3344
3345         /* Make sure the page is not on per-CPU pagevec as it takes pin */
3346         if (mlocked)
3347                 lru_add_drain();
3348
3349         /* Prevent deferred_split_scan() touching ->_count */
3350         spin_lock(&split_queue_lock);
3351         count = page_count(head);
3352         mapcount = total_mapcount(head);
3353         if (!mapcount && count == 1) {
3354                 if (!list_empty(page_deferred_list(head))) {
3355                         split_queue_len--;
3356                         list_del(page_deferred_list(head));
3357                 }
3358                 spin_unlock(&split_queue_lock);
3359                 __split_huge_page(page, list);
3360                 ret = 0;
3361         } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
3362                 spin_unlock(&split_queue_lock);
3363                 pr_alert("total_mapcount: %u, page_count(): %u\n",
3364                                 mapcount, count);
3365                 if (PageTail(page))
3366                         dump_page(head, NULL);
3367                 dump_page(page, "total_mapcount(head) > 0");
3368                 BUG();
3369         } else {
3370                 spin_unlock(&split_queue_lock);
3371                 unfreeze_page(anon_vma, head);
3372                 ret = -EBUSY;
3373         }
3374
3375 out_unlock:
3376         anon_vma_unlock_write(anon_vma);
3377         put_anon_vma(anon_vma);
3378 out:
3379         count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
3380         return ret;
3381 }
3382
3383 void free_transhuge_page(struct page *page)
3384 {
3385         unsigned long flags;
3386
3387         spin_lock_irqsave(&split_queue_lock, flags);
3388         if (!list_empty(page_deferred_list(page))) {
3389                 split_queue_len--;
3390                 list_del(page_deferred_list(page));
3391         }
3392         spin_unlock_irqrestore(&split_queue_lock, flags);
3393         free_compound_page(page);
3394 }
3395
3396 void deferred_split_huge_page(struct page *page)
3397 {
3398         unsigned long flags;
3399
3400         VM_BUG_ON_PAGE(!PageTransHuge(page), page);
3401
3402         spin_lock_irqsave(&split_queue_lock, flags);
3403         if (list_empty(page_deferred_list(page))) {
3404                 list_add_tail(page_deferred_list(page), &split_queue);
3405                 split_queue_len++;
3406         }
3407         spin_unlock_irqrestore(&split_queue_lock, flags);
3408 }
3409
3410 static unsigned long deferred_split_count(struct shrinker *shrink,
3411                 struct shrink_control *sc)
3412 {
3413         /*
3414          * Split a page from split_queue will free up at least one page,
3415          * at most HPAGE_PMD_NR - 1. We don't track exact number.
3416          * Let's use HPAGE_PMD_NR / 2 as ballpark.
3417          */
3418         return ACCESS_ONCE(split_queue_len) * HPAGE_PMD_NR / 2;
3419 }
3420
3421 static unsigned long deferred_split_scan(struct shrinker *shrink,
3422                 struct shrink_control *sc)
3423 {
3424         unsigned long flags;
3425         LIST_HEAD(list), *pos, *next;
3426         struct page *page;
3427         int split = 0;
3428
3429         spin_lock_irqsave(&split_queue_lock, flags);
3430         list_splice_init(&split_queue, &list);
3431
3432         /* Take pin on all head pages to avoid freeing them under us */
3433         list_for_each_safe(pos, next, &list) {
3434                 page = list_entry((void *)pos, struct page, mapping);
3435                 page = compound_head(page);
3436                 /* race with put_compound_page() */
3437                 if (!get_page_unless_zero(page)) {
3438                         list_del_init(page_deferred_list(page));
3439                         split_queue_len--;
3440                 }
3441         }
3442         spin_unlock_irqrestore(&split_queue_lock, flags);
3443
3444         list_for_each_safe(pos, next, &list) {
3445                 page = list_entry((void *)pos, struct page, mapping);
3446                 lock_page(page);
3447                 /* split_huge_page() removes page from list on success */
3448                 if (!split_huge_page(page))
3449                         split++;
3450                 unlock_page(page);
3451                 put_page(page);
3452         }
3453
3454         spin_lock_irqsave(&split_queue_lock, flags);
3455         list_splice_tail(&list, &split_queue);
3456         spin_unlock_irqrestore(&split_queue_lock, flags);
3457
3458         return split * HPAGE_PMD_NR / 2;
3459 }
3460
3461 static struct shrinker deferred_split_shrinker = {
3462         .count_objects = deferred_split_count,
3463         .scan_objects = deferred_split_scan,
3464         .seeks = DEFAULT_SEEKS,
3465 };
3466
3467 #ifdef CONFIG_DEBUG_FS
3468 static int split_huge_pages_set(void *data, u64 val)
3469 {
3470         struct zone *zone;
3471         struct page *page;
3472         unsigned long pfn, max_zone_pfn;
3473         unsigned long total = 0, split = 0;
3474
3475         if (val != 1)
3476                 return -EINVAL;
3477
3478         for_each_populated_zone(zone) {
3479                 max_zone_pfn = zone_end_pfn(zone);
3480                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
3481                         if (!pfn_valid(pfn))
3482                                 continue;
3483
3484                         page = pfn_to_page(pfn);
3485                         if (!get_page_unless_zero(page))
3486                                 continue;
3487
3488                         if (zone != page_zone(page))
3489                                 goto next;
3490
3491                         if (!PageHead(page) || !PageAnon(page) ||
3492                                         PageHuge(page))
3493                                 goto next;
3494
3495                         total++;
3496                         lock_page(page);
3497                         if (!split_huge_page(page))
3498                                 split++;
3499                         unlock_page(page);
3500 next:
3501                         put_page(page);
3502                 }
3503         }
3504
3505         pr_info("%lu of %lu THP split", split, total);
3506
3507         return 0;
3508 }
3509 DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
3510                 "%llu\n");
3511
3512 static int __init split_huge_pages_debugfs(void)
3513 {
3514         void *ret;
3515
3516         ret = debugfs_create_file("split_huge_pages", 0644, NULL, NULL,
3517                         &split_huge_pages_fops);
3518         if (!ret)
3519                 pr_warn("Failed to create split_huge_pages in debugfs");
3520         return 0;
3521 }
3522 late_initcall(split_huge_pages_debugfs);
3523 #endif