]> git.karo-electronics.de Git - karo-tx-linux.git/blob - mm/hugetlb.c
Merge branches 'work.namei', 'work.dcache' and 'work.iov_iter' into for-linus
[karo-tx-linux.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/seq_file.h>
9 #include <linux/sysctl.h>
10 #include <linux/highmem.h>
11 #include <linux/mmu_notifier.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/compiler.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24 #include <linux/page-isolation.h>
25 #include <linux/jhash.h>
26
27 #include <asm/page.h>
28 #include <asm/pgtable.h>
29 #include <asm/tlb.h>
30
31 #include <linux/io.h>
32 #include <linux/hugetlb.h>
33 #include <linux/hugetlb_cgroup.h>
34 #include <linux/node.h>
35 #include "internal.h"
36
37 int hugepages_treat_as_movable;
38
39 int hugetlb_max_hstate __read_mostly;
40 unsigned int default_hstate_idx;
41 struct hstate hstates[HUGE_MAX_HSTATE];
42 /*
43  * Minimum page order among possible hugepage sizes, set to a proper value
44  * at boot time.
45  */
46 static unsigned int minimum_order __read_mostly = UINT_MAX;
47
48 __initdata LIST_HEAD(huge_boot_pages);
49
50 /* for command line parsing */
51 static struct hstate * __initdata parsed_hstate;
52 static unsigned long __initdata default_hstate_max_huge_pages;
53 static unsigned long __initdata default_hstate_size;
54 static bool __initdata parsed_valid_hugepagesz = true;
55
56 /*
57  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
58  * free_huge_pages, and surplus_huge_pages.
59  */
60 DEFINE_SPINLOCK(hugetlb_lock);
61
62 /*
63  * Serializes faults on the same logical page.  This is used to
64  * prevent spurious OOMs when the hugepage pool is fully utilized.
65  */
66 static int num_fault_mutexes;
67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
68
69 /* Forward declaration */
70 static int hugetlb_acct_memory(struct hstate *h, long delta);
71
72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
73 {
74         bool free = (spool->count == 0) && (spool->used_hpages == 0);
75
76         spin_unlock(&spool->lock);
77
78         /* If no pages are used, and no other handles to the subpool
79          * remain, give up any reservations mased on minimum size and
80          * free the subpool */
81         if (free) {
82                 if (spool->min_hpages != -1)
83                         hugetlb_acct_memory(spool->hstate,
84                                                 -spool->min_hpages);
85                 kfree(spool);
86         }
87 }
88
89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
90                                                 long min_hpages)
91 {
92         struct hugepage_subpool *spool;
93
94         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
95         if (!spool)
96                 return NULL;
97
98         spin_lock_init(&spool->lock);
99         spool->count = 1;
100         spool->max_hpages = max_hpages;
101         spool->hstate = h;
102         spool->min_hpages = min_hpages;
103
104         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105                 kfree(spool);
106                 return NULL;
107         }
108         spool->rsv_hpages = min_hpages;
109
110         return spool;
111 }
112
113 void hugepage_put_subpool(struct hugepage_subpool *spool)
114 {
115         spin_lock(&spool->lock);
116         BUG_ON(!spool->count);
117         spool->count--;
118         unlock_or_release_subpool(spool);
119 }
120
121 /*
122  * Subpool accounting for allocating and reserving pages.
123  * Return -ENOMEM if there are not enough resources to satisfy the
124  * the request.  Otherwise, return the number of pages by which the
125  * global pools must be adjusted (upward).  The returned value may
126  * only be different than the passed value (delta) in the case where
127  * a subpool minimum size must be manitained.
128  */
129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130                                       long delta)
131 {
132         long ret = delta;
133
134         if (!spool)
135                 return ret;
136
137         spin_lock(&spool->lock);
138
139         if (spool->max_hpages != -1) {          /* maximum size accounting */
140                 if ((spool->used_hpages + delta) <= spool->max_hpages)
141                         spool->used_hpages += delta;
142                 else {
143                         ret = -ENOMEM;
144                         goto unlock_ret;
145                 }
146         }
147
148         /* minimum size accounting */
149         if (spool->min_hpages != -1 && spool->rsv_hpages) {
150                 if (delta > spool->rsv_hpages) {
151                         /*
152                          * Asking for more reserves than those already taken on
153                          * behalf of subpool.  Return difference.
154                          */
155                         ret = delta - spool->rsv_hpages;
156                         spool->rsv_hpages = 0;
157                 } else {
158                         ret = 0;        /* reserves already accounted for */
159                         spool->rsv_hpages -= delta;
160                 }
161         }
162
163 unlock_ret:
164         spin_unlock(&spool->lock);
165         return ret;
166 }
167
168 /*
169  * Subpool accounting for freeing and unreserving pages.
170  * Return the number of global page reservations that must be dropped.
171  * The return value may only be different than the passed value (delta)
172  * in the case where a subpool minimum size must be maintained.
173  */
174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
175                                        long delta)
176 {
177         long ret = delta;
178
179         if (!spool)
180                 return delta;
181
182         spin_lock(&spool->lock);
183
184         if (spool->max_hpages != -1)            /* maximum size accounting */
185                 spool->used_hpages -= delta;
186
187          /* minimum size accounting */
188         if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
189                 if (spool->rsv_hpages + delta <= spool->min_hpages)
190                         ret = 0;
191                 else
192                         ret = spool->rsv_hpages + delta - spool->min_hpages;
193
194                 spool->rsv_hpages += delta;
195                 if (spool->rsv_hpages > spool->min_hpages)
196                         spool->rsv_hpages = spool->min_hpages;
197         }
198
199         /*
200          * If hugetlbfs_put_super couldn't free spool due to an outstanding
201          * quota reference, free it now.
202          */
203         unlock_or_release_subpool(spool);
204
205         return ret;
206 }
207
208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
209 {
210         return HUGETLBFS_SB(inode->i_sb)->spool;
211 }
212
213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
214 {
215         return subpool_inode(file_inode(vma->vm_file));
216 }
217
218 /*
219  * Region tracking -- allows tracking of reservations and instantiated pages
220  *                    across the pages in a mapping.
221  *
222  * The region data structures are embedded into a resv_map and protected
223  * by a resv_map's lock.  The set of regions within the resv_map represent
224  * reservations for huge pages, or huge pages that have already been
225  * instantiated within the map.  The from and to elements are huge page
226  * indicies into the associated mapping.  from indicates the starting index
227  * of the region.  to represents the first index past the end of  the region.
228  *
229  * For example, a file region structure with from == 0 and to == 4 represents
230  * four huge pages in a mapping.  It is important to note that the to element
231  * represents the first element past the end of the region. This is used in
232  * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
233  *
234  * Interval notation of the form [from, to) will be used to indicate that
235  * the endpoint from is inclusive and to is exclusive.
236  */
237 struct file_region {
238         struct list_head link;
239         long from;
240         long to;
241 };
242
243 /*
244  * Add the huge page range represented by [f, t) to the reserve
245  * map.  In the normal case, existing regions will be expanded
246  * to accommodate the specified range.  Sufficient regions should
247  * exist for expansion due to the previous call to region_chg
248  * with the same range.  However, it is possible that region_del
249  * could have been called after region_chg and modifed the map
250  * in such a way that no region exists to be expanded.  In this
251  * case, pull a region descriptor from the cache associated with
252  * the map and use that for the new range.
253  *
254  * Return the number of new huge pages added to the map.  This
255  * number is greater than or equal to zero.
256  */
257 static long region_add(struct resv_map *resv, long f, long t)
258 {
259         struct list_head *head = &resv->regions;
260         struct file_region *rg, *nrg, *trg;
261         long add = 0;
262
263         spin_lock(&resv->lock);
264         /* Locate the region we are either in or before. */
265         list_for_each_entry(rg, head, link)
266                 if (f <= rg->to)
267                         break;
268
269         /*
270          * If no region exists which can be expanded to include the
271          * specified range, the list must have been modified by an
272          * interleving call to region_del().  Pull a region descriptor
273          * from the cache and use it for this range.
274          */
275         if (&rg->link == head || t < rg->from) {
276                 VM_BUG_ON(resv->region_cache_count <= 0);
277
278                 resv->region_cache_count--;
279                 nrg = list_first_entry(&resv->region_cache, struct file_region,
280                                         link);
281                 list_del(&nrg->link);
282
283                 nrg->from = f;
284                 nrg->to = t;
285                 list_add(&nrg->link, rg->link.prev);
286
287                 add += t - f;
288                 goto out_locked;
289         }
290
291         /* Round our left edge to the current segment if it encloses us. */
292         if (f > rg->from)
293                 f = rg->from;
294
295         /* Check for and consume any regions we now overlap with. */
296         nrg = rg;
297         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
298                 if (&rg->link == head)
299                         break;
300                 if (rg->from > t)
301                         break;
302
303                 /* If this area reaches higher then extend our area to
304                  * include it completely.  If this is not the first area
305                  * which we intend to reuse, free it. */
306                 if (rg->to > t)
307                         t = rg->to;
308                 if (rg != nrg) {
309                         /* Decrement return value by the deleted range.
310                          * Another range will span this area so that by
311                          * end of routine add will be >= zero
312                          */
313                         add -= (rg->to - rg->from);
314                         list_del(&rg->link);
315                         kfree(rg);
316                 }
317         }
318
319         add += (nrg->from - f);         /* Added to beginning of region */
320         nrg->from = f;
321         add += t - nrg->to;             /* Added to end of region */
322         nrg->to = t;
323
324 out_locked:
325         resv->adds_in_progress--;
326         spin_unlock(&resv->lock);
327         VM_BUG_ON(add < 0);
328         return add;
329 }
330
331 /*
332  * Examine the existing reserve map and determine how many
333  * huge pages in the specified range [f, t) are NOT currently
334  * represented.  This routine is called before a subsequent
335  * call to region_add that will actually modify the reserve
336  * map to add the specified range [f, t).  region_chg does
337  * not change the number of huge pages represented by the
338  * map.  However, if the existing regions in the map can not
339  * be expanded to represent the new range, a new file_region
340  * structure is added to the map as a placeholder.  This is
341  * so that the subsequent region_add call will have all the
342  * regions it needs and will not fail.
343  *
344  * Upon entry, region_chg will also examine the cache of region descriptors
345  * associated with the map.  If there are not enough descriptors cached, one
346  * will be allocated for the in progress add operation.
347  *
348  * Returns the number of huge pages that need to be added to the existing
349  * reservation map for the range [f, t).  This number is greater or equal to
350  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
351  * is needed and can not be allocated.
352  */
353 static long region_chg(struct resv_map *resv, long f, long t)
354 {
355         struct list_head *head = &resv->regions;
356         struct file_region *rg, *nrg = NULL;
357         long chg = 0;
358
359 retry:
360         spin_lock(&resv->lock);
361 retry_locked:
362         resv->adds_in_progress++;
363
364         /*
365          * Check for sufficient descriptors in the cache to accommodate
366          * the number of in progress add operations.
367          */
368         if (resv->adds_in_progress > resv->region_cache_count) {
369                 struct file_region *trg;
370
371                 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
372                 /* Must drop lock to allocate a new descriptor. */
373                 resv->adds_in_progress--;
374                 spin_unlock(&resv->lock);
375
376                 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
377                 if (!trg) {
378                         kfree(nrg);
379                         return -ENOMEM;
380                 }
381
382                 spin_lock(&resv->lock);
383                 list_add(&trg->link, &resv->region_cache);
384                 resv->region_cache_count++;
385                 goto retry_locked;
386         }
387
388         /* Locate the region we are before or in. */
389         list_for_each_entry(rg, head, link)
390                 if (f <= rg->to)
391                         break;
392
393         /* If we are below the current region then a new region is required.
394          * Subtle, allocate a new region at the position but make it zero
395          * size such that we can guarantee to record the reservation. */
396         if (&rg->link == head || t < rg->from) {
397                 if (!nrg) {
398                         resv->adds_in_progress--;
399                         spin_unlock(&resv->lock);
400                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
401                         if (!nrg)
402                                 return -ENOMEM;
403
404                         nrg->from = f;
405                         nrg->to   = f;
406                         INIT_LIST_HEAD(&nrg->link);
407                         goto retry;
408                 }
409
410                 list_add(&nrg->link, rg->link.prev);
411                 chg = t - f;
412                 goto out_nrg;
413         }
414
415         /* Round our left edge to the current segment if it encloses us. */
416         if (f > rg->from)
417                 f = rg->from;
418         chg = t - f;
419
420         /* Check for and consume any regions we now overlap with. */
421         list_for_each_entry(rg, rg->link.prev, link) {
422                 if (&rg->link == head)
423                         break;
424                 if (rg->from > t)
425                         goto out;
426
427                 /* We overlap with this area, if it extends further than
428                  * us then we must extend ourselves.  Account for its
429                  * existing reservation. */
430                 if (rg->to > t) {
431                         chg += rg->to - t;
432                         t = rg->to;
433                 }
434                 chg -= rg->to - rg->from;
435         }
436
437 out:
438         spin_unlock(&resv->lock);
439         /*  We already know we raced and no longer need the new region */
440         kfree(nrg);
441         return chg;
442 out_nrg:
443         spin_unlock(&resv->lock);
444         return chg;
445 }
446
447 /*
448  * Abort the in progress add operation.  The adds_in_progress field
449  * of the resv_map keeps track of the operations in progress between
450  * calls to region_chg and region_add.  Operations are sometimes
451  * aborted after the call to region_chg.  In such cases, region_abort
452  * is called to decrement the adds_in_progress counter.
453  *
454  * NOTE: The range arguments [f, t) are not needed or used in this
455  * routine.  They are kept to make reading the calling code easier as
456  * arguments will match the associated region_chg call.
457  */
458 static void region_abort(struct resv_map *resv, long f, long t)
459 {
460         spin_lock(&resv->lock);
461         VM_BUG_ON(!resv->region_cache_count);
462         resv->adds_in_progress--;
463         spin_unlock(&resv->lock);
464 }
465
466 /*
467  * Delete the specified range [f, t) from the reserve map.  If the
468  * t parameter is LONG_MAX, this indicates that ALL regions after f
469  * should be deleted.  Locate the regions which intersect [f, t)
470  * and either trim, delete or split the existing regions.
471  *
472  * Returns the number of huge pages deleted from the reserve map.
473  * In the normal case, the return value is zero or more.  In the
474  * case where a region must be split, a new region descriptor must
475  * be allocated.  If the allocation fails, -ENOMEM will be returned.
476  * NOTE: If the parameter t == LONG_MAX, then we will never split
477  * a region and possibly return -ENOMEM.  Callers specifying
478  * t == LONG_MAX do not need to check for -ENOMEM error.
479  */
480 static long region_del(struct resv_map *resv, long f, long t)
481 {
482         struct list_head *head = &resv->regions;
483         struct file_region *rg, *trg;
484         struct file_region *nrg = NULL;
485         long del = 0;
486
487 retry:
488         spin_lock(&resv->lock);
489         list_for_each_entry_safe(rg, trg, head, link) {
490                 /*
491                  * Skip regions before the range to be deleted.  file_region
492                  * ranges are normally of the form [from, to).  However, there
493                  * may be a "placeholder" entry in the map which is of the form
494                  * (from, to) with from == to.  Check for placeholder entries
495                  * at the beginning of the range to be deleted.
496                  */
497                 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
498                         continue;
499
500                 if (rg->from >= t)
501                         break;
502
503                 if (f > rg->from && t < rg->to) { /* Must split region */
504                         /*
505                          * Check for an entry in the cache before dropping
506                          * lock and attempting allocation.
507                          */
508                         if (!nrg &&
509                             resv->region_cache_count > resv->adds_in_progress) {
510                                 nrg = list_first_entry(&resv->region_cache,
511                                                         struct file_region,
512                                                         link);
513                                 list_del(&nrg->link);
514                                 resv->region_cache_count--;
515                         }
516
517                         if (!nrg) {
518                                 spin_unlock(&resv->lock);
519                                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
520                                 if (!nrg)
521                                         return -ENOMEM;
522                                 goto retry;
523                         }
524
525                         del += t - f;
526
527                         /* New entry for end of split region */
528                         nrg->from = t;
529                         nrg->to = rg->to;
530                         INIT_LIST_HEAD(&nrg->link);
531
532                         /* Original entry is trimmed */
533                         rg->to = f;
534
535                         list_add(&nrg->link, &rg->link);
536                         nrg = NULL;
537                         break;
538                 }
539
540                 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
541                         del += rg->to - rg->from;
542                         list_del(&rg->link);
543                         kfree(rg);
544                         continue;
545                 }
546
547                 if (f <= rg->from) {    /* Trim beginning of region */
548                         del += t - rg->from;
549                         rg->from = t;
550                 } else {                /* Trim end of region */
551                         del += rg->to - f;
552                         rg->to = f;
553                 }
554         }
555
556         spin_unlock(&resv->lock);
557         kfree(nrg);
558         return del;
559 }
560
561 /*
562  * A rare out of memory error was encountered which prevented removal of
563  * the reserve map region for a page.  The huge page itself was free'ed
564  * and removed from the page cache.  This routine will adjust the subpool
565  * usage count, and the global reserve count if needed.  By incrementing
566  * these counts, the reserve map entry which could not be deleted will
567  * appear as a "reserved" entry instead of simply dangling with incorrect
568  * counts.
569  */
570 void hugetlb_fix_reserve_counts(struct inode *inode)
571 {
572         struct hugepage_subpool *spool = subpool_inode(inode);
573         long rsv_adjust;
574
575         rsv_adjust = hugepage_subpool_get_pages(spool, 1);
576         if (rsv_adjust) {
577                 struct hstate *h = hstate_inode(inode);
578
579                 hugetlb_acct_memory(h, 1);
580         }
581 }
582
583 /*
584  * Count and return the number of huge pages in the reserve map
585  * that intersect with the range [f, t).
586  */
587 static long region_count(struct resv_map *resv, long f, long t)
588 {
589         struct list_head *head = &resv->regions;
590         struct file_region *rg;
591         long chg = 0;
592
593         spin_lock(&resv->lock);
594         /* Locate each segment we overlap with, and count that overlap. */
595         list_for_each_entry(rg, head, link) {
596                 long seg_from;
597                 long seg_to;
598
599                 if (rg->to <= f)
600                         continue;
601                 if (rg->from >= t)
602                         break;
603
604                 seg_from = max(rg->from, f);
605                 seg_to = min(rg->to, t);
606
607                 chg += seg_to - seg_from;
608         }
609         spin_unlock(&resv->lock);
610
611         return chg;
612 }
613
614 /*
615  * Convert the address within this vma to the page offset within
616  * the mapping, in pagecache page units; huge pages here.
617  */
618 static pgoff_t vma_hugecache_offset(struct hstate *h,
619                         struct vm_area_struct *vma, unsigned long address)
620 {
621         return ((address - vma->vm_start) >> huge_page_shift(h)) +
622                         (vma->vm_pgoff >> huge_page_order(h));
623 }
624
625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
626                                      unsigned long address)
627 {
628         return vma_hugecache_offset(hstate_vma(vma), vma, address);
629 }
630 EXPORT_SYMBOL_GPL(linear_hugepage_index);
631
632 /*
633  * Return the size of the pages allocated when backing a VMA. In the majority
634  * cases this will be same size as used by the page table entries.
635  */
636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
637 {
638         struct hstate *hstate;
639
640         if (!is_vm_hugetlb_page(vma))
641                 return PAGE_SIZE;
642
643         hstate = hstate_vma(vma);
644
645         return 1UL << huge_page_shift(hstate);
646 }
647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
648
649 /*
650  * Return the page size being used by the MMU to back a VMA. In the majority
651  * of cases, the page size used by the kernel matches the MMU size. On
652  * architectures where it differs, an architecture-specific version of this
653  * function is required.
654  */
655 #ifndef vma_mmu_pagesize
656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
657 {
658         return vma_kernel_pagesize(vma);
659 }
660 #endif
661
662 /*
663  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
664  * bits of the reservation map pointer, which are always clear due to
665  * alignment.
666  */
667 #define HPAGE_RESV_OWNER    (1UL << 0)
668 #define HPAGE_RESV_UNMAPPED (1UL << 1)
669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
670
671 /*
672  * These helpers are used to track how many pages are reserved for
673  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
674  * is guaranteed to have their future faults succeed.
675  *
676  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
677  * the reserve counters are updated with the hugetlb_lock held. It is safe
678  * to reset the VMA at fork() time as it is not in use yet and there is no
679  * chance of the global counters getting corrupted as a result of the values.
680  *
681  * The private mapping reservation is represented in a subtly different
682  * manner to a shared mapping.  A shared mapping has a region map associated
683  * with the underlying file, this region map represents the backing file
684  * pages which have ever had a reservation assigned which this persists even
685  * after the page is instantiated.  A private mapping has a region map
686  * associated with the original mmap which is attached to all VMAs which
687  * reference it, this region map represents those offsets which have consumed
688  * reservation ie. where pages have been instantiated.
689  */
690 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
691 {
692         return (unsigned long)vma->vm_private_data;
693 }
694
695 static void set_vma_private_data(struct vm_area_struct *vma,
696                                                         unsigned long value)
697 {
698         vma->vm_private_data = (void *)value;
699 }
700
701 struct resv_map *resv_map_alloc(void)
702 {
703         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
704         struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
705
706         if (!resv_map || !rg) {
707                 kfree(resv_map);
708                 kfree(rg);
709                 return NULL;
710         }
711
712         kref_init(&resv_map->refs);
713         spin_lock_init(&resv_map->lock);
714         INIT_LIST_HEAD(&resv_map->regions);
715
716         resv_map->adds_in_progress = 0;
717
718         INIT_LIST_HEAD(&resv_map->region_cache);
719         list_add(&rg->link, &resv_map->region_cache);
720         resv_map->region_cache_count = 1;
721
722         return resv_map;
723 }
724
725 void resv_map_release(struct kref *ref)
726 {
727         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
728         struct list_head *head = &resv_map->region_cache;
729         struct file_region *rg, *trg;
730
731         /* Clear out any active regions before we release the map. */
732         region_del(resv_map, 0, LONG_MAX);
733
734         /* ... and any entries left in the cache */
735         list_for_each_entry_safe(rg, trg, head, link) {
736                 list_del(&rg->link);
737                 kfree(rg);
738         }
739
740         VM_BUG_ON(resv_map->adds_in_progress);
741
742         kfree(resv_map);
743 }
744
745 static inline struct resv_map *inode_resv_map(struct inode *inode)
746 {
747         return inode->i_mapping->private_data;
748 }
749
750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
751 {
752         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
753         if (vma->vm_flags & VM_MAYSHARE) {
754                 struct address_space *mapping = vma->vm_file->f_mapping;
755                 struct inode *inode = mapping->host;
756
757                 return inode_resv_map(inode);
758
759         } else {
760                 return (struct resv_map *)(get_vma_private_data(vma) &
761                                                         ~HPAGE_RESV_MASK);
762         }
763 }
764
765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
766 {
767         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
768         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
769
770         set_vma_private_data(vma, (get_vma_private_data(vma) &
771                                 HPAGE_RESV_MASK) | (unsigned long)map);
772 }
773
774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
775 {
776         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
777         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
778
779         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
780 }
781
782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
783 {
784         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
785
786         return (get_vma_private_data(vma) & flag) != 0;
787 }
788
789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
791 {
792         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
793         if (!(vma->vm_flags & VM_MAYSHARE))
794                 vma->vm_private_data = (void *)0;
795 }
796
797 /* Returns true if the VMA has associated reserve pages */
798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
799 {
800         if (vma->vm_flags & VM_NORESERVE) {
801                 /*
802                  * This address is already reserved by other process(chg == 0),
803                  * so, we should decrement reserved count. Without decrementing,
804                  * reserve count remains after releasing inode, because this
805                  * allocated page will go into page cache and is regarded as
806                  * coming from reserved pool in releasing step.  Currently, we
807                  * don't have any other solution to deal with this situation
808                  * properly, so add work-around here.
809                  */
810                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
811                         return true;
812                 else
813                         return false;
814         }
815
816         /* Shared mappings always use reserves */
817         if (vma->vm_flags & VM_MAYSHARE) {
818                 /*
819                  * We know VM_NORESERVE is not set.  Therefore, there SHOULD
820                  * be a region map for all pages.  The only situation where
821                  * there is no region map is if a hole was punched via
822                  * fallocate.  In this case, there really are no reverves to
823                  * use.  This situation is indicated if chg != 0.
824                  */
825                 if (chg)
826                         return false;
827                 else
828                         return true;
829         }
830
831         /*
832          * Only the process that called mmap() has reserves for
833          * private mappings.
834          */
835         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
836                 /*
837                  * Like the shared case above, a hole punch or truncate
838                  * could have been performed on the private mapping.
839                  * Examine the value of chg to determine if reserves
840                  * actually exist or were previously consumed.
841                  * Very Subtle - The value of chg comes from a previous
842                  * call to vma_needs_reserves().  The reserve map for
843                  * private mappings has different (opposite) semantics
844                  * than that of shared mappings.  vma_needs_reserves()
845                  * has already taken this difference in semantics into
846                  * account.  Therefore, the meaning of chg is the same
847                  * as in the shared case above.  Code could easily be
848                  * combined, but keeping it separate draws attention to
849                  * subtle differences.
850                  */
851                 if (chg)
852                         return false;
853                 else
854                         return true;
855         }
856
857         return false;
858 }
859
860 static void enqueue_huge_page(struct hstate *h, struct page *page)
861 {
862         int nid = page_to_nid(page);
863         list_move(&page->lru, &h->hugepage_freelists[nid]);
864         h->free_huge_pages++;
865         h->free_huge_pages_node[nid]++;
866 }
867
868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
869 {
870         struct page *page;
871
872         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
873                 if (!is_migrate_isolate_page(page))
874                         break;
875         /*
876          * if 'non-isolated free hugepage' not found on the list,
877          * the allocation fails.
878          */
879         if (&h->hugepage_freelists[nid] == &page->lru)
880                 return NULL;
881         list_move(&page->lru, &h->hugepage_activelist);
882         set_page_refcounted(page);
883         h->free_huge_pages--;
884         h->free_huge_pages_node[nid]--;
885         return page;
886 }
887
888 /* Movability of hugepages depends on migration support. */
889 static inline gfp_t htlb_alloc_mask(struct hstate *h)
890 {
891         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
892                 return GFP_HIGHUSER_MOVABLE;
893         else
894                 return GFP_HIGHUSER;
895 }
896
897 static struct page *dequeue_huge_page_vma(struct hstate *h,
898                                 struct vm_area_struct *vma,
899                                 unsigned long address, int avoid_reserve,
900                                 long chg)
901 {
902         struct page *page = NULL;
903         struct mempolicy *mpol;
904         nodemask_t *nodemask;
905         struct zonelist *zonelist;
906         struct zone *zone;
907         struct zoneref *z;
908         unsigned int cpuset_mems_cookie;
909
910         /*
911          * A child process with MAP_PRIVATE mappings created by their parent
912          * have no page reserves. This check ensures that reservations are
913          * not "stolen". The child may still get SIGKILLed
914          */
915         if (!vma_has_reserves(vma, chg) &&
916                         h->free_huge_pages - h->resv_huge_pages == 0)
917                 goto err;
918
919         /* If reserves cannot be used, ensure enough pages are in the pool */
920         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
921                 goto err;
922
923 retry_cpuset:
924         cpuset_mems_cookie = read_mems_allowed_begin();
925         zonelist = huge_zonelist(vma, address,
926                                         htlb_alloc_mask(h), &mpol, &nodemask);
927
928         for_each_zone_zonelist_nodemask(zone, z, zonelist,
929                                                 MAX_NR_ZONES - 1, nodemask) {
930                 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
931                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
932                         if (page) {
933                                 if (avoid_reserve)
934                                         break;
935                                 if (!vma_has_reserves(vma, chg))
936                                         break;
937
938                                 SetPagePrivate(page);
939                                 h->resv_huge_pages--;
940                                 break;
941                         }
942                 }
943         }
944
945         mpol_cond_put(mpol);
946         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
947                 goto retry_cpuset;
948         return page;
949
950 err:
951         return NULL;
952 }
953
954 /*
955  * common helper functions for hstate_next_node_to_{alloc|free}.
956  * We may have allocated or freed a huge page based on a different
957  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
958  * be outside of *nodes_allowed.  Ensure that we use an allowed
959  * node for alloc or free.
960  */
961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
962 {
963         nid = next_node_in(nid, *nodes_allowed);
964         VM_BUG_ON(nid >= MAX_NUMNODES);
965
966         return nid;
967 }
968
969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
970 {
971         if (!node_isset(nid, *nodes_allowed))
972                 nid = next_node_allowed(nid, nodes_allowed);
973         return nid;
974 }
975
976 /*
977  * returns the previously saved node ["this node"] from which to
978  * allocate a persistent huge page for the pool and advance the
979  * next node from which to allocate, handling wrap at end of node
980  * mask.
981  */
982 static int hstate_next_node_to_alloc(struct hstate *h,
983                                         nodemask_t *nodes_allowed)
984 {
985         int nid;
986
987         VM_BUG_ON(!nodes_allowed);
988
989         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
990         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
991
992         return nid;
993 }
994
995 /*
996  * helper for free_pool_huge_page() - return the previously saved
997  * node ["this node"] from which to free a huge page.  Advance the
998  * next node id whether or not we find a free huge page to free so
999  * that the next attempt to free addresses the next node.
1000  */
1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1002 {
1003         int nid;
1004
1005         VM_BUG_ON(!nodes_allowed);
1006
1007         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1008         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1009
1010         return nid;
1011 }
1012
1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
1014         for (nr_nodes = nodes_weight(*mask);                            \
1015                 nr_nodes > 0 &&                                         \
1016                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
1017                 nr_nodes--)
1018
1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
1020         for (nr_nodes = nodes_weight(*mask);                            \
1021                 nr_nodes > 0 &&                                         \
1022                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
1023                 nr_nodes--)
1024
1025 #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \
1026         ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \
1027         defined(CONFIG_CMA))
1028 static void destroy_compound_gigantic_page(struct page *page,
1029                                         unsigned int order)
1030 {
1031         int i;
1032         int nr_pages = 1 << order;
1033         struct page *p = page + 1;
1034
1035         atomic_set(compound_mapcount_ptr(page), 0);
1036         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1037                 clear_compound_head(p);
1038                 set_page_refcounted(p);
1039         }
1040
1041         set_compound_order(page, 0);
1042         __ClearPageHead(page);
1043 }
1044
1045 static void free_gigantic_page(struct page *page, unsigned int order)
1046 {
1047         free_contig_range(page_to_pfn(page), 1 << order);
1048 }
1049
1050 static int __alloc_gigantic_page(unsigned long start_pfn,
1051                                 unsigned long nr_pages)
1052 {
1053         unsigned long end_pfn = start_pfn + nr_pages;
1054         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1055 }
1056
1057 static bool pfn_range_valid_gigantic(struct zone *z,
1058                         unsigned long start_pfn, unsigned long nr_pages)
1059 {
1060         unsigned long i, end_pfn = start_pfn + nr_pages;
1061         struct page *page;
1062
1063         for (i = start_pfn; i < end_pfn; i++) {
1064                 if (!pfn_valid(i))
1065                         return false;
1066
1067                 page = pfn_to_page(i);
1068
1069                 if (page_zone(page) != z)
1070                         return false;
1071
1072                 if (PageReserved(page))
1073                         return false;
1074
1075                 if (page_count(page) > 0)
1076                         return false;
1077
1078                 if (PageHuge(page))
1079                         return false;
1080         }
1081
1082         return true;
1083 }
1084
1085 static bool zone_spans_last_pfn(const struct zone *zone,
1086                         unsigned long start_pfn, unsigned long nr_pages)
1087 {
1088         unsigned long last_pfn = start_pfn + nr_pages - 1;
1089         return zone_spans_pfn(zone, last_pfn);
1090 }
1091
1092 static struct page *alloc_gigantic_page(int nid, unsigned int order)
1093 {
1094         unsigned long nr_pages = 1 << order;
1095         unsigned long ret, pfn, flags;
1096         struct zone *z;
1097
1098         z = NODE_DATA(nid)->node_zones;
1099         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1100                 spin_lock_irqsave(&z->lock, flags);
1101
1102                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
1103                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1104                         if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1105                                 /*
1106                                  * We release the zone lock here because
1107                                  * alloc_contig_range() will also lock the zone
1108                                  * at some point. If there's an allocation
1109                                  * spinning on this lock, it may win the race
1110                                  * and cause alloc_contig_range() to fail...
1111                                  */
1112                                 spin_unlock_irqrestore(&z->lock, flags);
1113                                 ret = __alloc_gigantic_page(pfn, nr_pages);
1114                                 if (!ret)
1115                                         return pfn_to_page(pfn);
1116                                 spin_lock_irqsave(&z->lock, flags);
1117                         }
1118                         pfn += nr_pages;
1119                 }
1120
1121                 spin_unlock_irqrestore(&z->lock, flags);
1122         }
1123
1124         return NULL;
1125 }
1126
1127 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1128 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1129
1130 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1131 {
1132         struct page *page;
1133
1134         page = alloc_gigantic_page(nid, huge_page_order(h));
1135         if (page) {
1136                 prep_compound_gigantic_page(page, huge_page_order(h));
1137                 prep_new_huge_page(h, page, nid);
1138         }
1139
1140         return page;
1141 }
1142
1143 static int alloc_fresh_gigantic_page(struct hstate *h,
1144                                 nodemask_t *nodes_allowed)
1145 {
1146         struct page *page = NULL;
1147         int nr_nodes, node;
1148
1149         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1150                 page = alloc_fresh_gigantic_page_node(h, node);
1151                 if (page)
1152                         return 1;
1153         }
1154
1155         return 0;
1156 }
1157
1158 static inline bool gigantic_page_supported(void) { return true; }
1159 #else
1160 static inline bool gigantic_page_supported(void) { return false; }
1161 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1162 static inline void destroy_compound_gigantic_page(struct page *page,
1163                                                 unsigned int order) { }
1164 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1165                                         nodemask_t *nodes_allowed) { return 0; }
1166 #endif
1167
1168 static void update_and_free_page(struct hstate *h, struct page *page)
1169 {
1170         int i;
1171
1172         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1173                 return;
1174
1175         h->nr_huge_pages--;
1176         h->nr_huge_pages_node[page_to_nid(page)]--;
1177         for (i = 0; i < pages_per_huge_page(h); i++) {
1178                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1179                                 1 << PG_referenced | 1 << PG_dirty |
1180                                 1 << PG_active | 1 << PG_private |
1181                                 1 << PG_writeback);
1182         }
1183         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1184         set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1185         set_page_refcounted(page);
1186         if (hstate_is_gigantic(h)) {
1187                 destroy_compound_gigantic_page(page, huge_page_order(h));
1188                 free_gigantic_page(page, huge_page_order(h));
1189         } else {
1190                 __free_pages(page, huge_page_order(h));
1191         }
1192 }
1193
1194 struct hstate *size_to_hstate(unsigned long size)
1195 {
1196         struct hstate *h;
1197
1198         for_each_hstate(h) {
1199                 if (huge_page_size(h) == size)
1200                         return h;
1201         }
1202         return NULL;
1203 }
1204
1205 /*
1206  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1207  * to hstate->hugepage_activelist.)
1208  *
1209  * This function can be called for tail pages, but never returns true for them.
1210  */
1211 bool page_huge_active(struct page *page)
1212 {
1213         VM_BUG_ON_PAGE(!PageHuge(page), page);
1214         return PageHead(page) && PagePrivate(&page[1]);
1215 }
1216
1217 /* never called for tail page */
1218 static void set_page_huge_active(struct page *page)
1219 {
1220         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1221         SetPagePrivate(&page[1]);
1222 }
1223
1224 static void clear_page_huge_active(struct page *page)
1225 {
1226         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1227         ClearPagePrivate(&page[1]);
1228 }
1229
1230 void free_huge_page(struct page *page)
1231 {
1232         /*
1233          * Can't pass hstate in here because it is called from the
1234          * compound page destructor.
1235          */
1236         struct hstate *h = page_hstate(page);
1237         int nid = page_to_nid(page);
1238         struct hugepage_subpool *spool =
1239                 (struct hugepage_subpool *)page_private(page);
1240         bool restore_reserve;
1241
1242         set_page_private(page, 0);
1243         page->mapping = NULL;
1244         VM_BUG_ON_PAGE(page_count(page), page);
1245         VM_BUG_ON_PAGE(page_mapcount(page), page);
1246         restore_reserve = PagePrivate(page);
1247         ClearPagePrivate(page);
1248
1249         /*
1250          * A return code of zero implies that the subpool will be under its
1251          * minimum size if the reservation is not restored after page is free.
1252          * Therefore, force restore_reserve operation.
1253          */
1254         if (hugepage_subpool_put_pages(spool, 1) == 0)
1255                 restore_reserve = true;
1256
1257         spin_lock(&hugetlb_lock);
1258         clear_page_huge_active(page);
1259         hugetlb_cgroup_uncharge_page(hstate_index(h),
1260                                      pages_per_huge_page(h), page);
1261         if (restore_reserve)
1262                 h->resv_huge_pages++;
1263
1264         if (h->surplus_huge_pages_node[nid]) {
1265                 /* remove the page from active list */
1266                 list_del(&page->lru);
1267                 update_and_free_page(h, page);
1268                 h->surplus_huge_pages--;
1269                 h->surplus_huge_pages_node[nid]--;
1270         } else {
1271                 arch_clear_hugepage_flags(page);
1272                 enqueue_huge_page(h, page);
1273         }
1274         spin_unlock(&hugetlb_lock);
1275 }
1276
1277 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1278 {
1279         INIT_LIST_HEAD(&page->lru);
1280         set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1281         spin_lock(&hugetlb_lock);
1282         set_hugetlb_cgroup(page, NULL);
1283         h->nr_huge_pages++;
1284         h->nr_huge_pages_node[nid]++;
1285         spin_unlock(&hugetlb_lock);
1286         put_page(page); /* free it into the hugepage allocator */
1287 }
1288
1289 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1290 {
1291         int i;
1292         int nr_pages = 1 << order;
1293         struct page *p = page + 1;
1294
1295         /* we rely on prep_new_huge_page to set the destructor */
1296         set_compound_order(page, order);
1297         __ClearPageReserved(page);
1298         __SetPageHead(page);
1299         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1300                 /*
1301                  * For gigantic hugepages allocated through bootmem at
1302                  * boot, it's safer to be consistent with the not-gigantic
1303                  * hugepages and clear the PG_reserved bit from all tail pages
1304                  * too.  Otherwse drivers using get_user_pages() to access tail
1305                  * pages may get the reference counting wrong if they see
1306                  * PG_reserved set on a tail page (despite the head page not
1307                  * having PG_reserved set).  Enforcing this consistency between
1308                  * head and tail pages allows drivers to optimize away a check
1309                  * on the head page when they need know if put_page() is needed
1310                  * after get_user_pages().
1311                  */
1312                 __ClearPageReserved(p);
1313                 set_page_count(p, 0);
1314                 set_compound_head(p, page);
1315         }
1316         atomic_set(compound_mapcount_ptr(page), -1);
1317 }
1318
1319 /*
1320  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1321  * transparent huge pages.  See the PageTransHuge() documentation for more
1322  * details.
1323  */
1324 int PageHuge(struct page *page)
1325 {
1326         if (!PageCompound(page))
1327                 return 0;
1328
1329         page = compound_head(page);
1330         return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1331 }
1332 EXPORT_SYMBOL_GPL(PageHuge);
1333
1334 /*
1335  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1336  * normal or transparent huge pages.
1337  */
1338 int PageHeadHuge(struct page *page_head)
1339 {
1340         if (!PageHead(page_head))
1341                 return 0;
1342
1343         return get_compound_page_dtor(page_head) == free_huge_page;
1344 }
1345
1346 pgoff_t __basepage_index(struct page *page)
1347 {
1348         struct page *page_head = compound_head(page);
1349         pgoff_t index = page_index(page_head);
1350         unsigned long compound_idx;
1351
1352         if (!PageHuge(page_head))
1353                 return page_index(page);
1354
1355         if (compound_order(page_head) >= MAX_ORDER)
1356                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1357         else
1358                 compound_idx = page - page_head;
1359
1360         return (index << compound_order(page_head)) + compound_idx;
1361 }
1362
1363 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1364 {
1365         struct page *page;
1366
1367         page = __alloc_pages_node(nid,
1368                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1369                                                 __GFP_REPEAT|__GFP_NOWARN,
1370                 huge_page_order(h));
1371         if (page) {
1372                 prep_new_huge_page(h, page, nid);
1373         }
1374
1375         return page;
1376 }
1377
1378 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1379 {
1380         struct page *page;
1381         int nr_nodes, node;
1382         int ret = 0;
1383
1384         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1385                 page = alloc_fresh_huge_page_node(h, node);
1386                 if (page) {
1387                         ret = 1;
1388                         break;
1389                 }
1390         }
1391
1392         if (ret)
1393                 count_vm_event(HTLB_BUDDY_PGALLOC);
1394         else
1395                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1396
1397         return ret;
1398 }
1399
1400 /*
1401  * Free huge page from pool from next node to free.
1402  * Attempt to keep persistent huge pages more or less
1403  * balanced over allowed nodes.
1404  * Called with hugetlb_lock locked.
1405  */
1406 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1407                                                          bool acct_surplus)
1408 {
1409         int nr_nodes, node;
1410         int ret = 0;
1411
1412         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1413                 /*
1414                  * If we're returning unused surplus pages, only examine
1415                  * nodes with surplus pages.
1416                  */
1417                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1418                     !list_empty(&h->hugepage_freelists[node])) {
1419                         struct page *page =
1420                                 list_entry(h->hugepage_freelists[node].next,
1421                                           struct page, lru);
1422                         list_del(&page->lru);
1423                         h->free_huge_pages--;
1424                         h->free_huge_pages_node[node]--;
1425                         if (acct_surplus) {
1426                                 h->surplus_huge_pages--;
1427                                 h->surplus_huge_pages_node[node]--;
1428                         }
1429                         update_and_free_page(h, page);
1430                         ret = 1;
1431                         break;
1432                 }
1433         }
1434
1435         return ret;
1436 }
1437
1438 /*
1439  * Dissolve a given free hugepage into free buddy pages. This function does
1440  * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the
1441  * number of free hugepages would be reduced below the number of reserved
1442  * hugepages.
1443  */
1444 static int dissolve_free_huge_page(struct page *page)
1445 {
1446         int rc = 0;
1447
1448         spin_lock(&hugetlb_lock);
1449         if (PageHuge(page) && !page_count(page)) {
1450                 struct page *head = compound_head(page);
1451                 struct hstate *h = page_hstate(head);
1452                 int nid = page_to_nid(head);
1453                 if (h->free_huge_pages - h->resv_huge_pages == 0) {
1454                         rc = -EBUSY;
1455                         goto out;
1456                 }
1457                 list_del(&head->lru);
1458                 h->free_huge_pages--;
1459                 h->free_huge_pages_node[nid]--;
1460                 h->max_huge_pages--;
1461                 update_and_free_page(h, head);
1462         }
1463 out:
1464         spin_unlock(&hugetlb_lock);
1465         return rc;
1466 }
1467
1468 /*
1469  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1470  * make specified memory blocks removable from the system.
1471  * Note that this will dissolve a free gigantic hugepage completely, if any
1472  * part of it lies within the given range.
1473  * Also note that if dissolve_free_huge_page() returns with an error, all
1474  * free hugepages that were dissolved before that error are lost.
1475  */
1476 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1477 {
1478         unsigned long pfn;
1479         struct page *page;
1480         int rc = 0;
1481
1482         if (!hugepages_supported())
1483                 return rc;
1484
1485         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1486                 page = pfn_to_page(pfn);
1487                 if (PageHuge(page) && !page_count(page)) {
1488                         rc = dissolve_free_huge_page(page);
1489                         if (rc)
1490                                 break;
1491                 }
1492         }
1493
1494         return rc;
1495 }
1496
1497 /*
1498  * There are 3 ways this can get called:
1499  * 1. With vma+addr: we use the VMA's memory policy
1500  * 2. With !vma, but nid=NUMA_NO_NODE:  We try to allocate a huge
1501  *    page from any node, and let the buddy allocator itself figure
1502  *    it out.
1503  * 3. With !vma, but nid!=NUMA_NO_NODE.  We allocate a huge page
1504  *    strictly from 'nid'
1505  */
1506 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1507                 struct vm_area_struct *vma, unsigned long addr, int nid)
1508 {
1509         int order = huge_page_order(h);
1510         gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1511         unsigned int cpuset_mems_cookie;
1512
1513         /*
1514          * We need a VMA to get a memory policy.  If we do not
1515          * have one, we use the 'nid' argument.
1516          *
1517          * The mempolicy stuff below has some non-inlined bits
1518          * and calls ->vm_ops.  That makes it hard to optimize at
1519          * compile-time, even when NUMA is off and it does
1520          * nothing.  This helps the compiler optimize it out.
1521          */
1522         if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1523                 /*
1524                  * If a specific node is requested, make sure to
1525                  * get memory from there, but only when a node
1526                  * is explicitly specified.
1527                  */
1528                 if (nid != NUMA_NO_NODE)
1529                         gfp |= __GFP_THISNODE;
1530                 /*
1531                  * Make sure to call something that can handle
1532                  * nid=NUMA_NO_NODE
1533                  */
1534                 return alloc_pages_node(nid, gfp, order);
1535         }
1536
1537         /*
1538          * OK, so we have a VMA.  Fetch the mempolicy and try to
1539          * allocate a huge page with it.  We will only reach this
1540          * when CONFIG_NUMA=y.
1541          */
1542         do {
1543                 struct page *page;
1544                 struct mempolicy *mpol;
1545                 struct zonelist *zl;
1546                 nodemask_t *nodemask;
1547
1548                 cpuset_mems_cookie = read_mems_allowed_begin();
1549                 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
1550                 mpol_cond_put(mpol);
1551                 page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
1552                 if (page)
1553                         return page;
1554         } while (read_mems_allowed_retry(cpuset_mems_cookie));
1555
1556         return NULL;
1557 }
1558
1559 /*
1560  * There are two ways to allocate a huge page:
1561  * 1. When you have a VMA and an address (like a fault)
1562  * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1563  *
1564  * 'vma' and 'addr' are only for (1).  'nid' is always NUMA_NO_NODE in
1565  * this case which signifies that the allocation should be done with
1566  * respect for the VMA's memory policy.
1567  *
1568  * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1569  * implies that memory policies will not be taken in to account.
1570  */
1571 static struct page *__alloc_buddy_huge_page(struct hstate *h,
1572                 struct vm_area_struct *vma, unsigned long addr, int nid)
1573 {
1574         struct page *page;
1575         unsigned int r_nid;
1576
1577         if (hstate_is_gigantic(h))
1578                 return NULL;
1579
1580         /*
1581          * Make sure that anyone specifying 'nid' is not also specifying a VMA.
1582          * This makes sure the caller is picking _one_ of the modes with which
1583          * we can call this function, not both.
1584          */
1585         if (vma || (addr != -1)) {
1586                 VM_WARN_ON_ONCE(addr == -1);
1587                 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1588         }
1589         /*
1590          * Assume we will successfully allocate the surplus page to
1591          * prevent racing processes from causing the surplus to exceed
1592          * overcommit
1593          *
1594          * This however introduces a different race, where a process B
1595          * tries to grow the static hugepage pool while alloc_pages() is
1596          * called by process A. B will only examine the per-node
1597          * counters in determining if surplus huge pages can be
1598          * converted to normal huge pages in adjust_pool_surplus(). A
1599          * won't be able to increment the per-node counter, until the
1600          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1601          * no more huge pages can be converted from surplus to normal
1602          * state (and doesn't try to convert again). Thus, we have a
1603          * case where a surplus huge page exists, the pool is grown, and
1604          * the surplus huge page still exists after, even though it
1605          * should just have been converted to a normal huge page. This
1606          * does not leak memory, though, as the hugepage will be freed
1607          * once it is out of use. It also does not allow the counters to
1608          * go out of whack in adjust_pool_surplus() as we don't modify
1609          * the node values until we've gotten the hugepage and only the
1610          * per-node value is checked there.
1611          */
1612         spin_lock(&hugetlb_lock);
1613         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1614                 spin_unlock(&hugetlb_lock);
1615                 return NULL;
1616         } else {
1617                 h->nr_huge_pages++;
1618                 h->surplus_huge_pages++;
1619         }
1620         spin_unlock(&hugetlb_lock);
1621
1622         page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1623
1624         spin_lock(&hugetlb_lock);
1625         if (page) {
1626                 INIT_LIST_HEAD(&page->lru);
1627                 r_nid = page_to_nid(page);
1628                 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1629                 set_hugetlb_cgroup(page, NULL);
1630                 /*
1631                  * We incremented the global counters already
1632                  */
1633                 h->nr_huge_pages_node[r_nid]++;
1634                 h->surplus_huge_pages_node[r_nid]++;
1635                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1636         } else {
1637                 h->nr_huge_pages--;
1638                 h->surplus_huge_pages--;
1639                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1640         }
1641         spin_unlock(&hugetlb_lock);
1642
1643         return page;
1644 }
1645
1646 /*
1647  * Allocate a huge page from 'nid'.  Note, 'nid' may be
1648  * NUMA_NO_NODE, which means that it may be allocated
1649  * anywhere.
1650  */
1651 static
1652 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1653 {
1654         unsigned long addr = -1;
1655
1656         return __alloc_buddy_huge_page(h, NULL, addr, nid);
1657 }
1658
1659 /*
1660  * Use the VMA's mpolicy to allocate a huge page from the buddy.
1661  */
1662 static
1663 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1664                 struct vm_area_struct *vma, unsigned long addr)
1665 {
1666         return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1667 }
1668
1669 /*
1670  * This allocation function is useful in the context where vma is irrelevant.
1671  * E.g. soft-offlining uses this function because it only cares physical
1672  * address of error page.
1673  */
1674 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1675 {
1676         struct page *page = NULL;
1677
1678         spin_lock(&hugetlb_lock);
1679         if (h->free_huge_pages - h->resv_huge_pages > 0)
1680                 page = dequeue_huge_page_node(h, nid);
1681         spin_unlock(&hugetlb_lock);
1682
1683         if (!page)
1684                 page = __alloc_buddy_huge_page_no_mpol(h, nid);
1685
1686         return page;
1687 }
1688
1689 /*
1690  * Increase the hugetlb pool such that it can accommodate a reservation
1691  * of size 'delta'.
1692  */
1693 static int gather_surplus_pages(struct hstate *h, int delta)
1694 {
1695         struct list_head surplus_list;
1696         struct page *page, *tmp;
1697         int ret, i;
1698         int needed, allocated;
1699         bool alloc_ok = true;
1700
1701         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1702         if (needed <= 0) {
1703                 h->resv_huge_pages += delta;
1704                 return 0;
1705         }
1706
1707         allocated = 0;
1708         INIT_LIST_HEAD(&surplus_list);
1709
1710         ret = -ENOMEM;
1711 retry:
1712         spin_unlock(&hugetlb_lock);
1713         for (i = 0; i < needed; i++) {
1714                 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1715                 if (!page) {
1716                         alloc_ok = false;
1717                         break;
1718                 }
1719                 list_add(&page->lru, &surplus_list);
1720         }
1721         allocated += i;
1722
1723         /*
1724          * After retaking hugetlb_lock, we need to recalculate 'needed'
1725          * because either resv_huge_pages or free_huge_pages may have changed.
1726          */
1727         spin_lock(&hugetlb_lock);
1728         needed = (h->resv_huge_pages + delta) -
1729                         (h->free_huge_pages + allocated);
1730         if (needed > 0) {
1731                 if (alloc_ok)
1732                         goto retry;
1733                 /*
1734                  * We were not able to allocate enough pages to
1735                  * satisfy the entire reservation so we free what
1736                  * we've allocated so far.
1737                  */
1738                 goto free;
1739         }
1740         /*
1741          * The surplus_list now contains _at_least_ the number of extra pages
1742          * needed to accommodate the reservation.  Add the appropriate number
1743          * of pages to the hugetlb pool and free the extras back to the buddy
1744          * allocator.  Commit the entire reservation here to prevent another
1745          * process from stealing the pages as they are added to the pool but
1746          * before they are reserved.
1747          */
1748         needed += allocated;
1749         h->resv_huge_pages += delta;
1750         ret = 0;
1751
1752         /* Free the needed pages to the hugetlb pool */
1753         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1754                 if ((--needed) < 0)
1755                         break;
1756                 /*
1757                  * This page is now managed by the hugetlb allocator and has
1758                  * no users -- drop the buddy allocator's reference.
1759                  */
1760                 put_page_testzero(page);
1761                 VM_BUG_ON_PAGE(page_count(page), page);
1762                 enqueue_huge_page(h, page);
1763         }
1764 free:
1765         spin_unlock(&hugetlb_lock);
1766
1767         /* Free unnecessary surplus pages to the buddy allocator */
1768         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1769                 put_page(page);
1770         spin_lock(&hugetlb_lock);
1771
1772         return ret;
1773 }
1774
1775 /*
1776  * When releasing a hugetlb pool reservation, any surplus pages that were
1777  * allocated to satisfy the reservation must be explicitly freed if they were
1778  * never used.
1779  * Called with hugetlb_lock held.
1780  */
1781 static void return_unused_surplus_pages(struct hstate *h,
1782                                         unsigned long unused_resv_pages)
1783 {
1784         unsigned long nr_pages;
1785
1786         /* Uncommit the reservation */
1787         h->resv_huge_pages -= unused_resv_pages;
1788
1789         /* Cannot return gigantic pages currently */
1790         if (hstate_is_gigantic(h))
1791                 return;
1792
1793         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1794
1795         /*
1796          * We want to release as many surplus pages as possible, spread
1797          * evenly across all nodes with memory. Iterate across these nodes
1798          * until we can no longer free unreserved surplus pages. This occurs
1799          * when the nodes with surplus pages have no free pages.
1800          * free_pool_huge_page() will balance the the freed pages across the
1801          * on-line nodes with memory and will handle the hstate accounting.
1802          */
1803         while (nr_pages--) {
1804                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1805                         break;
1806                 cond_resched_lock(&hugetlb_lock);
1807         }
1808 }
1809
1810
1811 /*
1812  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1813  * are used by the huge page allocation routines to manage reservations.
1814  *
1815  * vma_needs_reservation is called to determine if the huge page at addr
1816  * within the vma has an associated reservation.  If a reservation is
1817  * needed, the value 1 is returned.  The caller is then responsible for
1818  * managing the global reservation and subpool usage counts.  After
1819  * the huge page has been allocated, vma_commit_reservation is called
1820  * to add the page to the reservation map.  If the page allocation fails,
1821  * the reservation must be ended instead of committed.  vma_end_reservation
1822  * is called in such cases.
1823  *
1824  * In the normal case, vma_commit_reservation returns the same value
1825  * as the preceding vma_needs_reservation call.  The only time this
1826  * is not the case is if a reserve map was changed between calls.  It
1827  * is the responsibility of the caller to notice the difference and
1828  * take appropriate action.
1829  *
1830  * vma_add_reservation is used in error paths where a reservation must
1831  * be restored when a newly allocated huge page must be freed.  It is
1832  * to be called after calling vma_needs_reservation to determine if a
1833  * reservation exists.
1834  */
1835 enum vma_resv_mode {
1836         VMA_NEEDS_RESV,
1837         VMA_COMMIT_RESV,
1838         VMA_END_RESV,
1839         VMA_ADD_RESV,
1840 };
1841 static long __vma_reservation_common(struct hstate *h,
1842                                 struct vm_area_struct *vma, unsigned long addr,
1843                                 enum vma_resv_mode mode)
1844 {
1845         struct resv_map *resv;
1846         pgoff_t idx;
1847         long ret;
1848
1849         resv = vma_resv_map(vma);
1850         if (!resv)
1851                 return 1;
1852
1853         idx = vma_hugecache_offset(h, vma, addr);
1854         switch (mode) {
1855         case VMA_NEEDS_RESV:
1856                 ret = region_chg(resv, idx, idx + 1);
1857                 break;
1858         case VMA_COMMIT_RESV:
1859                 ret = region_add(resv, idx, idx + 1);
1860                 break;
1861         case VMA_END_RESV:
1862                 region_abort(resv, idx, idx + 1);
1863                 ret = 0;
1864                 break;
1865         case VMA_ADD_RESV:
1866                 if (vma->vm_flags & VM_MAYSHARE)
1867                         ret = region_add(resv, idx, idx + 1);
1868                 else {
1869                         region_abort(resv, idx, idx + 1);
1870                         ret = region_del(resv, idx, idx + 1);
1871                 }
1872                 break;
1873         default:
1874                 BUG();
1875         }
1876
1877         if (vma->vm_flags & VM_MAYSHARE)
1878                 return ret;
1879         else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1880                 /*
1881                  * In most cases, reserves always exist for private mappings.
1882                  * However, a file associated with mapping could have been
1883                  * hole punched or truncated after reserves were consumed.
1884                  * As subsequent fault on such a range will not use reserves.
1885                  * Subtle - The reserve map for private mappings has the
1886                  * opposite meaning than that of shared mappings.  If NO
1887                  * entry is in the reserve map, it means a reservation exists.
1888                  * If an entry exists in the reserve map, it means the
1889                  * reservation has already been consumed.  As a result, the
1890                  * return value of this routine is the opposite of the
1891                  * value returned from reserve map manipulation routines above.
1892                  */
1893                 if (ret)
1894                         return 0;
1895                 else
1896                         return 1;
1897         }
1898         else
1899                 return ret < 0 ? ret : 0;
1900 }
1901
1902 static long vma_needs_reservation(struct hstate *h,
1903                         struct vm_area_struct *vma, unsigned long addr)
1904 {
1905         return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1906 }
1907
1908 static long vma_commit_reservation(struct hstate *h,
1909                         struct vm_area_struct *vma, unsigned long addr)
1910 {
1911         return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1912 }
1913
1914 static void vma_end_reservation(struct hstate *h,
1915                         struct vm_area_struct *vma, unsigned long addr)
1916 {
1917         (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1918 }
1919
1920 static long vma_add_reservation(struct hstate *h,
1921                         struct vm_area_struct *vma, unsigned long addr)
1922 {
1923         return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
1924 }
1925
1926 /*
1927  * This routine is called to restore a reservation on error paths.  In the
1928  * specific error paths, a huge page was allocated (via alloc_huge_page)
1929  * and is about to be freed.  If a reservation for the page existed,
1930  * alloc_huge_page would have consumed the reservation and set PagePrivate
1931  * in the newly allocated page.  When the page is freed via free_huge_page,
1932  * the global reservation count will be incremented if PagePrivate is set.
1933  * However, free_huge_page can not adjust the reserve map.  Adjust the
1934  * reserve map here to be consistent with global reserve count adjustments
1935  * to be made by free_huge_page.
1936  */
1937 static void restore_reserve_on_error(struct hstate *h,
1938                         struct vm_area_struct *vma, unsigned long address,
1939                         struct page *page)
1940 {
1941         if (unlikely(PagePrivate(page))) {
1942                 long rc = vma_needs_reservation(h, vma, address);
1943
1944                 if (unlikely(rc < 0)) {
1945                         /*
1946                          * Rare out of memory condition in reserve map
1947                          * manipulation.  Clear PagePrivate so that
1948                          * global reserve count will not be incremented
1949                          * by free_huge_page.  This will make it appear
1950                          * as though the reservation for this page was
1951                          * consumed.  This may prevent the task from
1952                          * faulting in the page at a later time.  This
1953                          * is better than inconsistent global huge page
1954                          * accounting of reserve counts.
1955                          */
1956                         ClearPagePrivate(page);
1957                 } else if (rc) {
1958                         rc = vma_add_reservation(h, vma, address);
1959                         if (unlikely(rc < 0))
1960                                 /*
1961                                  * See above comment about rare out of
1962                                  * memory condition.
1963                                  */
1964                                 ClearPagePrivate(page);
1965                 } else
1966                         vma_end_reservation(h, vma, address);
1967         }
1968 }
1969
1970 struct page *alloc_huge_page(struct vm_area_struct *vma,
1971                                     unsigned long addr, int avoid_reserve)
1972 {
1973         struct hugepage_subpool *spool = subpool_vma(vma);
1974         struct hstate *h = hstate_vma(vma);
1975         struct page *page;
1976         long map_chg, map_commit;
1977         long gbl_chg;
1978         int ret, idx;
1979         struct hugetlb_cgroup *h_cg;
1980
1981         idx = hstate_index(h);
1982         /*
1983          * Examine the region/reserve map to determine if the process
1984          * has a reservation for the page to be allocated.  A return
1985          * code of zero indicates a reservation exists (no change).
1986          */
1987         map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
1988         if (map_chg < 0)
1989                 return ERR_PTR(-ENOMEM);
1990
1991         /*
1992          * Processes that did not create the mapping will have no
1993          * reserves as indicated by the region/reserve map. Check
1994          * that the allocation will not exceed the subpool limit.
1995          * Allocations for MAP_NORESERVE mappings also need to be
1996          * checked against any subpool limit.
1997          */
1998         if (map_chg || avoid_reserve) {
1999                 gbl_chg = hugepage_subpool_get_pages(spool, 1);
2000                 if (gbl_chg < 0) {
2001                         vma_end_reservation(h, vma, addr);
2002                         return ERR_PTR(-ENOSPC);
2003                 }
2004
2005                 /*
2006                  * Even though there was no reservation in the region/reserve
2007                  * map, there could be reservations associated with the
2008                  * subpool that can be used.  This would be indicated if the
2009                  * return value of hugepage_subpool_get_pages() is zero.
2010                  * However, if avoid_reserve is specified we still avoid even
2011                  * the subpool reservations.
2012                  */
2013                 if (avoid_reserve)
2014                         gbl_chg = 1;
2015         }
2016
2017         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2018         if (ret)
2019                 goto out_subpool_put;
2020
2021         spin_lock(&hugetlb_lock);
2022         /*
2023          * glb_chg is passed to indicate whether or not a page must be taken
2024          * from the global free pool (global change).  gbl_chg == 0 indicates
2025          * a reservation exists for the allocation.
2026          */
2027         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2028         if (!page) {
2029                 spin_unlock(&hugetlb_lock);
2030                 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
2031                 if (!page)
2032                         goto out_uncharge_cgroup;
2033                 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2034                         SetPagePrivate(page);
2035                         h->resv_huge_pages--;
2036                 }
2037                 spin_lock(&hugetlb_lock);
2038                 list_move(&page->lru, &h->hugepage_activelist);
2039                 /* Fall through */
2040         }
2041         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2042         spin_unlock(&hugetlb_lock);
2043
2044         set_page_private(page, (unsigned long)spool);
2045
2046         map_commit = vma_commit_reservation(h, vma, addr);
2047         if (unlikely(map_chg > map_commit)) {
2048                 /*
2049                  * The page was added to the reservation map between
2050                  * vma_needs_reservation and vma_commit_reservation.
2051                  * This indicates a race with hugetlb_reserve_pages.
2052                  * Adjust for the subpool count incremented above AND
2053                  * in hugetlb_reserve_pages for the same page.  Also,
2054                  * the reservation count added in hugetlb_reserve_pages
2055                  * no longer applies.
2056                  */
2057                 long rsv_adjust;
2058
2059                 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2060                 hugetlb_acct_memory(h, -rsv_adjust);
2061         }
2062         return page;
2063
2064 out_uncharge_cgroup:
2065         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2066 out_subpool_put:
2067         if (map_chg || avoid_reserve)
2068                 hugepage_subpool_put_pages(spool, 1);
2069         vma_end_reservation(h, vma, addr);
2070         return ERR_PTR(-ENOSPC);
2071 }
2072
2073 /*
2074  * alloc_huge_page()'s wrapper which simply returns the page if allocation
2075  * succeeds, otherwise NULL. This function is called from new_vma_page(),
2076  * where no ERR_VALUE is expected to be returned.
2077  */
2078 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
2079                                 unsigned long addr, int avoid_reserve)
2080 {
2081         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
2082         if (IS_ERR(page))
2083                 page = NULL;
2084         return page;
2085 }
2086
2087 int __weak alloc_bootmem_huge_page(struct hstate *h)
2088 {
2089         struct huge_bootmem_page *m;
2090         int nr_nodes, node;
2091
2092         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2093                 void *addr;
2094
2095                 addr = memblock_virt_alloc_try_nid_nopanic(
2096                                 huge_page_size(h), huge_page_size(h),
2097                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2098                 if (addr) {
2099                         /*
2100                          * Use the beginning of the huge page to store the
2101                          * huge_bootmem_page struct (until gather_bootmem
2102                          * puts them into the mem_map).
2103                          */
2104                         m = addr;
2105                         goto found;
2106                 }
2107         }
2108         return 0;
2109
2110 found:
2111         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2112         /* Put them into a private list first because mem_map is not up yet */
2113         list_add(&m->list, &huge_boot_pages);
2114         m->hstate = h;
2115         return 1;
2116 }
2117
2118 static void __init prep_compound_huge_page(struct page *page,
2119                 unsigned int order)
2120 {
2121         if (unlikely(order > (MAX_ORDER - 1)))
2122                 prep_compound_gigantic_page(page, order);
2123         else
2124                 prep_compound_page(page, order);
2125 }
2126
2127 /* Put bootmem huge pages into the standard lists after mem_map is up */
2128 static void __init gather_bootmem_prealloc(void)
2129 {
2130         struct huge_bootmem_page *m;
2131
2132         list_for_each_entry(m, &huge_boot_pages, list) {
2133                 struct hstate *h = m->hstate;
2134                 struct page *page;
2135
2136 #ifdef CONFIG_HIGHMEM
2137                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
2138                 memblock_free_late(__pa(m),
2139                                    sizeof(struct huge_bootmem_page));
2140 #else
2141                 page = virt_to_page(m);
2142 #endif
2143                 WARN_ON(page_count(page) != 1);
2144                 prep_compound_huge_page(page, h->order);
2145                 WARN_ON(PageReserved(page));
2146                 prep_new_huge_page(h, page, page_to_nid(page));
2147                 /*
2148                  * If we had gigantic hugepages allocated at boot time, we need
2149                  * to restore the 'stolen' pages to totalram_pages in order to
2150                  * fix confusing memory reports from free(1) and another
2151                  * side-effects, like CommitLimit going negative.
2152                  */
2153                 if (hstate_is_gigantic(h))
2154                         adjust_managed_page_count(page, 1 << h->order);
2155         }
2156 }
2157
2158 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2159 {
2160         unsigned long i;
2161
2162         for (i = 0; i < h->max_huge_pages; ++i) {
2163                 if (hstate_is_gigantic(h)) {
2164                         if (!alloc_bootmem_huge_page(h))
2165                                 break;
2166                 } else if (!alloc_fresh_huge_page(h,
2167                                          &node_states[N_MEMORY]))
2168                         break;
2169         }
2170         h->max_huge_pages = i;
2171 }
2172
2173 static void __init hugetlb_init_hstates(void)
2174 {
2175         struct hstate *h;
2176
2177         for_each_hstate(h) {
2178                 if (minimum_order > huge_page_order(h))
2179                         minimum_order = huge_page_order(h);
2180
2181                 /* oversize hugepages were init'ed in early boot */
2182                 if (!hstate_is_gigantic(h))
2183                         hugetlb_hstate_alloc_pages(h);
2184         }
2185         VM_BUG_ON(minimum_order == UINT_MAX);
2186 }
2187
2188 static char * __init memfmt(char *buf, unsigned long n)
2189 {
2190         if (n >= (1UL << 30))
2191                 sprintf(buf, "%lu GB", n >> 30);
2192         else if (n >= (1UL << 20))
2193                 sprintf(buf, "%lu MB", n >> 20);
2194         else
2195                 sprintf(buf, "%lu KB", n >> 10);
2196         return buf;
2197 }
2198
2199 static void __init report_hugepages(void)
2200 {
2201         struct hstate *h;
2202
2203         for_each_hstate(h) {
2204                 char buf[32];
2205                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2206                         memfmt(buf, huge_page_size(h)),
2207                         h->free_huge_pages);
2208         }
2209 }
2210
2211 #ifdef CONFIG_HIGHMEM
2212 static void try_to_free_low(struct hstate *h, unsigned long count,
2213                                                 nodemask_t *nodes_allowed)
2214 {
2215         int i;
2216
2217         if (hstate_is_gigantic(h))
2218                 return;
2219
2220         for_each_node_mask(i, *nodes_allowed) {
2221                 struct page *page, *next;
2222                 struct list_head *freel = &h->hugepage_freelists[i];
2223                 list_for_each_entry_safe(page, next, freel, lru) {
2224                         if (count >= h->nr_huge_pages)
2225                                 return;
2226                         if (PageHighMem(page))
2227                                 continue;
2228                         list_del(&page->lru);
2229                         update_and_free_page(h, page);
2230                         h->free_huge_pages--;
2231                         h->free_huge_pages_node[page_to_nid(page)]--;
2232                 }
2233         }
2234 }
2235 #else
2236 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2237                                                 nodemask_t *nodes_allowed)
2238 {
2239 }
2240 #endif
2241
2242 /*
2243  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2244  * balanced by operating on them in a round-robin fashion.
2245  * Returns 1 if an adjustment was made.
2246  */
2247 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2248                                 int delta)
2249 {
2250         int nr_nodes, node;
2251
2252         VM_BUG_ON(delta != -1 && delta != 1);
2253
2254         if (delta < 0) {
2255                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2256                         if (h->surplus_huge_pages_node[node])
2257                                 goto found;
2258                 }
2259         } else {
2260                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2261                         if (h->surplus_huge_pages_node[node] <
2262                                         h->nr_huge_pages_node[node])
2263                                 goto found;
2264                 }
2265         }
2266         return 0;
2267
2268 found:
2269         h->surplus_huge_pages += delta;
2270         h->surplus_huge_pages_node[node] += delta;
2271         return 1;
2272 }
2273
2274 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2275 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2276                                                 nodemask_t *nodes_allowed)
2277 {
2278         unsigned long min_count, ret;
2279
2280         if (hstate_is_gigantic(h) && !gigantic_page_supported())
2281                 return h->max_huge_pages;
2282
2283         /*
2284          * Increase the pool size
2285          * First take pages out of surplus state.  Then make up the
2286          * remaining difference by allocating fresh huge pages.
2287          *
2288          * We might race with __alloc_buddy_huge_page() here and be unable
2289          * to convert a surplus huge page to a normal huge page. That is
2290          * not critical, though, it just means the overall size of the
2291          * pool might be one hugepage larger than it needs to be, but
2292          * within all the constraints specified by the sysctls.
2293          */
2294         spin_lock(&hugetlb_lock);
2295         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2296                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2297                         break;
2298         }
2299
2300         while (count > persistent_huge_pages(h)) {
2301                 /*
2302                  * If this allocation races such that we no longer need the
2303                  * page, free_huge_page will handle it by freeing the page
2304                  * and reducing the surplus.
2305                  */
2306                 spin_unlock(&hugetlb_lock);
2307
2308                 /* yield cpu to avoid soft lockup */
2309                 cond_resched();
2310
2311                 if (hstate_is_gigantic(h))
2312                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2313                 else
2314                         ret = alloc_fresh_huge_page(h, nodes_allowed);
2315                 spin_lock(&hugetlb_lock);
2316                 if (!ret)
2317                         goto out;
2318
2319                 /* Bail for signals. Probably ctrl-c from user */
2320                 if (signal_pending(current))
2321                         goto out;
2322         }
2323
2324         /*
2325          * Decrease the pool size
2326          * First return free pages to the buddy allocator (being careful
2327          * to keep enough around to satisfy reservations).  Then place
2328          * pages into surplus state as needed so the pool will shrink
2329          * to the desired size as pages become free.
2330          *
2331          * By placing pages into the surplus state independent of the
2332          * overcommit value, we are allowing the surplus pool size to
2333          * exceed overcommit. There are few sane options here. Since
2334          * __alloc_buddy_huge_page() is checking the global counter,
2335          * though, we'll note that we're not allowed to exceed surplus
2336          * and won't grow the pool anywhere else. Not until one of the
2337          * sysctls are changed, or the surplus pages go out of use.
2338          */
2339         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2340         min_count = max(count, min_count);
2341         try_to_free_low(h, min_count, nodes_allowed);
2342         while (min_count < persistent_huge_pages(h)) {
2343                 if (!free_pool_huge_page(h, nodes_allowed, 0))
2344                         break;
2345                 cond_resched_lock(&hugetlb_lock);
2346         }
2347         while (count < persistent_huge_pages(h)) {
2348                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2349                         break;
2350         }
2351 out:
2352         ret = persistent_huge_pages(h);
2353         spin_unlock(&hugetlb_lock);
2354         return ret;
2355 }
2356
2357 #define HSTATE_ATTR_RO(_name) \
2358         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2359
2360 #define HSTATE_ATTR(_name) \
2361         static struct kobj_attribute _name##_attr = \
2362                 __ATTR(_name, 0644, _name##_show, _name##_store)
2363
2364 static struct kobject *hugepages_kobj;
2365 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2366
2367 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2368
2369 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2370 {
2371         int i;
2372
2373         for (i = 0; i < HUGE_MAX_HSTATE; i++)
2374                 if (hstate_kobjs[i] == kobj) {
2375                         if (nidp)
2376                                 *nidp = NUMA_NO_NODE;
2377                         return &hstates[i];
2378                 }
2379
2380         return kobj_to_node_hstate(kobj, nidp);
2381 }
2382
2383 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2384                                         struct kobj_attribute *attr, char *buf)
2385 {
2386         struct hstate *h;
2387         unsigned long nr_huge_pages;
2388         int nid;
2389
2390         h = kobj_to_hstate(kobj, &nid);
2391         if (nid == NUMA_NO_NODE)
2392                 nr_huge_pages = h->nr_huge_pages;
2393         else
2394                 nr_huge_pages = h->nr_huge_pages_node[nid];
2395
2396         return sprintf(buf, "%lu\n", nr_huge_pages);
2397 }
2398
2399 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2400                                            struct hstate *h, int nid,
2401                                            unsigned long count, size_t len)
2402 {
2403         int err;
2404         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2405
2406         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2407                 err = -EINVAL;
2408                 goto out;
2409         }
2410
2411         if (nid == NUMA_NO_NODE) {
2412                 /*
2413                  * global hstate attribute
2414                  */
2415                 if (!(obey_mempolicy &&
2416                                 init_nodemask_of_mempolicy(nodes_allowed))) {
2417                         NODEMASK_FREE(nodes_allowed);
2418                         nodes_allowed = &node_states[N_MEMORY];
2419                 }
2420         } else if (nodes_allowed) {
2421                 /*
2422                  * per node hstate attribute: adjust count to global,
2423                  * but restrict alloc/free to the specified node.
2424                  */
2425                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2426                 init_nodemask_of_node(nodes_allowed, nid);
2427         } else
2428                 nodes_allowed = &node_states[N_MEMORY];
2429
2430         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2431
2432         if (nodes_allowed != &node_states[N_MEMORY])
2433                 NODEMASK_FREE(nodes_allowed);
2434
2435         return len;
2436 out:
2437         NODEMASK_FREE(nodes_allowed);
2438         return err;
2439 }
2440
2441 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2442                                          struct kobject *kobj, const char *buf,
2443                                          size_t len)
2444 {
2445         struct hstate *h;
2446         unsigned long count;
2447         int nid;
2448         int err;
2449
2450         err = kstrtoul(buf, 10, &count);
2451         if (err)
2452                 return err;
2453
2454         h = kobj_to_hstate(kobj, &nid);
2455         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2456 }
2457
2458 static ssize_t nr_hugepages_show(struct kobject *kobj,
2459                                        struct kobj_attribute *attr, char *buf)
2460 {
2461         return nr_hugepages_show_common(kobj, attr, buf);
2462 }
2463
2464 static ssize_t nr_hugepages_store(struct kobject *kobj,
2465                struct kobj_attribute *attr, const char *buf, size_t len)
2466 {
2467         return nr_hugepages_store_common(false, kobj, buf, len);
2468 }
2469 HSTATE_ATTR(nr_hugepages);
2470
2471 #ifdef CONFIG_NUMA
2472
2473 /*
2474  * hstate attribute for optionally mempolicy-based constraint on persistent
2475  * huge page alloc/free.
2476  */
2477 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2478                                        struct kobj_attribute *attr, char *buf)
2479 {
2480         return nr_hugepages_show_common(kobj, attr, buf);
2481 }
2482
2483 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2484                struct kobj_attribute *attr, const char *buf, size_t len)
2485 {
2486         return nr_hugepages_store_common(true, kobj, buf, len);
2487 }
2488 HSTATE_ATTR(nr_hugepages_mempolicy);
2489 #endif
2490
2491
2492 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2493                                         struct kobj_attribute *attr, char *buf)
2494 {
2495         struct hstate *h = kobj_to_hstate(kobj, NULL);
2496         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2497 }
2498
2499 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2500                 struct kobj_attribute *attr, const char *buf, size_t count)
2501 {
2502         int err;
2503         unsigned long input;
2504         struct hstate *h = kobj_to_hstate(kobj, NULL);
2505
2506         if (hstate_is_gigantic(h))
2507                 return -EINVAL;
2508
2509         err = kstrtoul(buf, 10, &input);
2510         if (err)
2511                 return err;
2512
2513         spin_lock(&hugetlb_lock);
2514         h->nr_overcommit_huge_pages = input;
2515         spin_unlock(&hugetlb_lock);
2516
2517         return count;
2518 }
2519 HSTATE_ATTR(nr_overcommit_hugepages);
2520
2521 static ssize_t free_hugepages_show(struct kobject *kobj,
2522                                         struct kobj_attribute *attr, char *buf)
2523 {
2524         struct hstate *h;
2525         unsigned long free_huge_pages;
2526         int nid;
2527
2528         h = kobj_to_hstate(kobj, &nid);
2529         if (nid == NUMA_NO_NODE)
2530                 free_huge_pages = h->free_huge_pages;
2531         else
2532                 free_huge_pages = h->free_huge_pages_node[nid];
2533
2534         return sprintf(buf, "%lu\n", free_huge_pages);
2535 }
2536 HSTATE_ATTR_RO(free_hugepages);
2537
2538 static ssize_t resv_hugepages_show(struct kobject *kobj,
2539                                         struct kobj_attribute *attr, char *buf)
2540 {
2541         struct hstate *h = kobj_to_hstate(kobj, NULL);
2542         return sprintf(buf, "%lu\n", h->resv_huge_pages);
2543 }
2544 HSTATE_ATTR_RO(resv_hugepages);
2545
2546 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2547                                         struct kobj_attribute *attr, char *buf)
2548 {
2549         struct hstate *h;
2550         unsigned long surplus_huge_pages;
2551         int nid;
2552
2553         h = kobj_to_hstate(kobj, &nid);
2554         if (nid == NUMA_NO_NODE)
2555                 surplus_huge_pages = h->surplus_huge_pages;
2556         else
2557                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2558
2559         return sprintf(buf, "%lu\n", surplus_huge_pages);
2560 }
2561 HSTATE_ATTR_RO(surplus_hugepages);
2562
2563 static struct attribute *hstate_attrs[] = {
2564         &nr_hugepages_attr.attr,
2565         &nr_overcommit_hugepages_attr.attr,
2566         &free_hugepages_attr.attr,
2567         &resv_hugepages_attr.attr,
2568         &surplus_hugepages_attr.attr,
2569 #ifdef CONFIG_NUMA
2570         &nr_hugepages_mempolicy_attr.attr,
2571 #endif
2572         NULL,
2573 };
2574
2575 static struct attribute_group hstate_attr_group = {
2576         .attrs = hstate_attrs,
2577 };
2578
2579 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2580                                     struct kobject **hstate_kobjs,
2581                                     struct attribute_group *hstate_attr_group)
2582 {
2583         int retval;
2584         int hi = hstate_index(h);
2585
2586         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2587         if (!hstate_kobjs[hi])
2588                 return -ENOMEM;
2589
2590         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2591         if (retval)
2592                 kobject_put(hstate_kobjs[hi]);
2593
2594         return retval;
2595 }
2596
2597 static void __init hugetlb_sysfs_init(void)
2598 {
2599         struct hstate *h;
2600         int err;
2601
2602         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2603         if (!hugepages_kobj)
2604                 return;
2605
2606         for_each_hstate(h) {
2607                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2608                                          hstate_kobjs, &hstate_attr_group);
2609                 if (err)
2610                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
2611         }
2612 }
2613
2614 #ifdef CONFIG_NUMA
2615
2616 /*
2617  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2618  * with node devices in node_devices[] using a parallel array.  The array
2619  * index of a node device or _hstate == node id.
2620  * This is here to avoid any static dependency of the node device driver, in
2621  * the base kernel, on the hugetlb module.
2622  */
2623 struct node_hstate {
2624         struct kobject          *hugepages_kobj;
2625         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
2626 };
2627 static struct node_hstate node_hstates[MAX_NUMNODES];
2628
2629 /*
2630  * A subset of global hstate attributes for node devices
2631  */
2632 static struct attribute *per_node_hstate_attrs[] = {
2633         &nr_hugepages_attr.attr,
2634         &free_hugepages_attr.attr,
2635         &surplus_hugepages_attr.attr,
2636         NULL,
2637 };
2638
2639 static struct attribute_group per_node_hstate_attr_group = {
2640         .attrs = per_node_hstate_attrs,
2641 };
2642
2643 /*
2644  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2645  * Returns node id via non-NULL nidp.
2646  */
2647 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2648 {
2649         int nid;
2650
2651         for (nid = 0; nid < nr_node_ids; nid++) {
2652                 struct node_hstate *nhs = &node_hstates[nid];
2653                 int i;
2654                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2655                         if (nhs->hstate_kobjs[i] == kobj) {
2656                                 if (nidp)
2657                                         *nidp = nid;
2658                                 return &hstates[i];
2659                         }
2660         }
2661
2662         BUG();
2663         return NULL;
2664 }
2665
2666 /*
2667  * Unregister hstate attributes from a single node device.
2668  * No-op if no hstate attributes attached.
2669  */
2670 static void hugetlb_unregister_node(struct node *node)
2671 {
2672         struct hstate *h;
2673         struct node_hstate *nhs = &node_hstates[node->dev.id];
2674
2675         if (!nhs->hugepages_kobj)
2676                 return;         /* no hstate attributes */
2677
2678         for_each_hstate(h) {
2679                 int idx = hstate_index(h);
2680                 if (nhs->hstate_kobjs[idx]) {
2681                         kobject_put(nhs->hstate_kobjs[idx]);
2682                         nhs->hstate_kobjs[idx] = NULL;
2683                 }
2684         }
2685
2686         kobject_put(nhs->hugepages_kobj);
2687         nhs->hugepages_kobj = NULL;
2688 }
2689
2690
2691 /*
2692  * Register hstate attributes for a single node device.
2693  * No-op if attributes already registered.
2694  */
2695 static void hugetlb_register_node(struct node *node)
2696 {
2697         struct hstate *h;
2698         struct node_hstate *nhs = &node_hstates[node->dev.id];
2699         int err;
2700
2701         if (nhs->hugepages_kobj)
2702                 return;         /* already allocated */
2703
2704         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2705                                                         &node->dev.kobj);
2706         if (!nhs->hugepages_kobj)
2707                 return;
2708
2709         for_each_hstate(h) {
2710                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2711                                                 nhs->hstate_kobjs,
2712                                                 &per_node_hstate_attr_group);
2713                 if (err) {
2714                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2715                                 h->name, node->dev.id);
2716                         hugetlb_unregister_node(node);
2717                         break;
2718                 }
2719         }
2720 }
2721
2722 /*
2723  * hugetlb init time:  register hstate attributes for all registered node
2724  * devices of nodes that have memory.  All on-line nodes should have
2725  * registered their associated device by this time.
2726  */
2727 static void __init hugetlb_register_all_nodes(void)
2728 {
2729         int nid;
2730
2731         for_each_node_state(nid, N_MEMORY) {
2732                 struct node *node = node_devices[nid];
2733                 if (node->dev.id == nid)
2734                         hugetlb_register_node(node);
2735         }
2736
2737         /*
2738          * Let the node device driver know we're here so it can
2739          * [un]register hstate attributes on node hotplug.
2740          */
2741         register_hugetlbfs_with_node(hugetlb_register_node,
2742                                      hugetlb_unregister_node);
2743 }
2744 #else   /* !CONFIG_NUMA */
2745
2746 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2747 {
2748         BUG();
2749         if (nidp)
2750                 *nidp = -1;
2751         return NULL;
2752 }
2753
2754 static void hugetlb_register_all_nodes(void) { }
2755
2756 #endif
2757
2758 static int __init hugetlb_init(void)
2759 {
2760         int i;
2761
2762         if (!hugepages_supported())
2763                 return 0;
2764
2765         if (!size_to_hstate(default_hstate_size)) {
2766                 default_hstate_size = HPAGE_SIZE;
2767                 if (!size_to_hstate(default_hstate_size))
2768                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2769         }
2770         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2771         if (default_hstate_max_huge_pages) {
2772                 if (!default_hstate.max_huge_pages)
2773                         default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2774         }
2775
2776         hugetlb_init_hstates();
2777         gather_bootmem_prealloc();
2778         report_hugepages();
2779
2780         hugetlb_sysfs_init();
2781         hugetlb_register_all_nodes();
2782         hugetlb_cgroup_file_init();
2783
2784 #ifdef CONFIG_SMP
2785         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2786 #else
2787         num_fault_mutexes = 1;
2788 #endif
2789         hugetlb_fault_mutex_table =
2790                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2791         BUG_ON(!hugetlb_fault_mutex_table);
2792
2793         for (i = 0; i < num_fault_mutexes; i++)
2794                 mutex_init(&hugetlb_fault_mutex_table[i]);
2795         return 0;
2796 }
2797 subsys_initcall(hugetlb_init);
2798
2799 /* Should be called on processing a hugepagesz=... option */
2800 void __init hugetlb_bad_size(void)
2801 {
2802         parsed_valid_hugepagesz = false;
2803 }
2804
2805 void __init hugetlb_add_hstate(unsigned int order)
2806 {
2807         struct hstate *h;
2808         unsigned long i;
2809
2810         if (size_to_hstate(PAGE_SIZE << order)) {
2811                 pr_warn("hugepagesz= specified twice, ignoring\n");
2812                 return;
2813         }
2814         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2815         BUG_ON(order == 0);
2816         h = &hstates[hugetlb_max_hstate++];
2817         h->order = order;
2818         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2819         h->nr_huge_pages = 0;
2820         h->free_huge_pages = 0;
2821         for (i = 0; i < MAX_NUMNODES; ++i)
2822                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2823         INIT_LIST_HEAD(&h->hugepage_activelist);
2824         h->next_nid_to_alloc = first_memory_node;
2825         h->next_nid_to_free = first_memory_node;
2826         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2827                                         huge_page_size(h)/1024);
2828
2829         parsed_hstate = h;
2830 }
2831
2832 static int __init hugetlb_nrpages_setup(char *s)
2833 {
2834         unsigned long *mhp;
2835         static unsigned long *last_mhp;
2836
2837         if (!parsed_valid_hugepagesz) {
2838                 pr_warn("hugepages = %s preceded by "
2839                         "an unsupported hugepagesz, ignoring\n", s);
2840                 parsed_valid_hugepagesz = true;
2841                 return 1;
2842         }
2843         /*
2844          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2845          * so this hugepages= parameter goes to the "default hstate".
2846          */
2847         else if (!hugetlb_max_hstate)
2848                 mhp = &default_hstate_max_huge_pages;
2849         else
2850                 mhp = &parsed_hstate->max_huge_pages;
2851
2852         if (mhp == last_mhp) {
2853                 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2854                 return 1;
2855         }
2856
2857         if (sscanf(s, "%lu", mhp) <= 0)
2858                 *mhp = 0;
2859
2860         /*
2861          * Global state is always initialized later in hugetlb_init.
2862          * But we need to allocate >= MAX_ORDER hstates here early to still
2863          * use the bootmem allocator.
2864          */
2865         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2866                 hugetlb_hstate_alloc_pages(parsed_hstate);
2867
2868         last_mhp = mhp;
2869
2870         return 1;
2871 }
2872 __setup("hugepages=", hugetlb_nrpages_setup);
2873
2874 static int __init hugetlb_default_setup(char *s)
2875 {
2876         default_hstate_size = memparse(s, &s);
2877         return 1;
2878 }
2879 __setup("default_hugepagesz=", hugetlb_default_setup);
2880
2881 static unsigned int cpuset_mems_nr(unsigned int *array)
2882 {
2883         int node;
2884         unsigned int nr = 0;
2885
2886         for_each_node_mask(node, cpuset_current_mems_allowed)
2887                 nr += array[node];
2888
2889         return nr;
2890 }
2891
2892 #ifdef CONFIG_SYSCTL
2893 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2894                          struct ctl_table *table, int write,
2895                          void __user *buffer, size_t *length, loff_t *ppos)
2896 {
2897         struct hstate *h = &default_hstate;
2898         unsigned long tmp = h->max_huge_pages;
2899         int ret;
2900
2901         if (!hugepages_supported())
2902                 return -EOPNOTSUPP;
2903
2904         table->data = &tmp;
2905         table->maxlen = sizeof(unsigned long);
2906         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2907         if (ret)
2908                 goto out;
2909
2910         if (write)
2911                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2912                                                   NUMA_NO_NODE, tmp, *length);
2913 out:
2914         return ret;
2915 }
2916
2917 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2918                           void __user *buffer, size_t *length, loff_t *ppos)
2919 {
2920
2921         return hugetlb_sysctl_handler_common(false, table, write,
2922                                                         buffer, length, ppos);
2923 }
2924
2925 #ifdef CONFIG_NUMA
2926 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2927                           void __user *buffer, size_t *length, loff_t *ppos)
2928 {
2929         return hugetlb_sysctl_handler_common(true, table, write,
2930                                                         buffer, length, ppos);
2931 }
2932 #endif /* CONFIG_NUMA */
2933
2934 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2935                         void __user *buffer,
2936                         size_t *length, loff_t *ppos)
2937 {
2938         struct hstate *h = &default_hstate;
2939         unsigned long tmp;
2940         int ret;
2941
2942         if (!hugepages_supported())
2943                 return -EOPNOTSUPP;
2944
2945         tmp = h->nr_overcommit_huge_pages;
2946
2947         if (write && hstate_is_gigantic(h))
2948                 return -EINVAL;
2949
2950         table->data = &tmp;
2951         table->maxlen = sizeof(unsigned long);
2952         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2953         if (ret)
2954                 goto out;
2955
2956         if (write) {
2957                 spin_lock(&hugetlb_lock);
2958                 h->nr_overcommit_huge_pages = tmp;
2959                 spin_unlock(&hugetlb_lock);
2960         }
2961 out:
2962         return ret;
2963 }
2964
2965 #endif /* CONFIG_SYSCTL */
2966
2967 void hugetlb_report_meminfo(struct seq_file *m)
2968 {
2969         struct hstate *h = &default_hstate;
2970         if (!hugepages_supported())
2971                 return;
2972         seq_printf(m,
2973                         "HugePages_Total:   %5lu\n"
2974                         "HugePages_Free:    %5lu\n"
2975                         "HugePages_Rsvd:    %5lu\n"
2976                         "HugePages_Surp:    %5lu\n"
2977                         "Hugepagesize:   %8lu kB\n",
2978                         h->nr_huge_pages,
2979                         h->free_huge_pages,
2980                         h->resv_huge_pages,
2981                         h->surplus_huge_pages,
2982                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2983 }
2984
2985 int hugetlb_report_node_meminfo(int nid, char *buf)
2986 {
2987         struct hstate *h = &default_hstate;
2988         if (!hugepages_supported())
2989                 return 0;
2990         return sprintf(buf,
2991                 "Node %d HugePages_Total: %5u\n"
2992                 "Node %d HugePages_Free:  %5u\n"
2993                 "Node %d HugePages_Surp:  %5u\n",
2994                 nid, h->nr_huge_pages_node[nid],
2995                 nid, h->free_huge_pages_node[nid],
2996                 nid, h->surplus_huge_pages_node[nid]);
2997 }
2998
2999 void hugetlb_show_meminfo(void)
3000 {
3001         struct hstate *h;
3002         int nid;
3003
3004         if (!hugepages_supported())
3005                 return;
3006
3007         for_each_node_state(nid, N_MEMORY)
3008                 for_each_hstate(h)
3009                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3010                                 nid,
3011                                 h->nr_huge_pages_node[nid],
3012                                 h->free_huge_pages_node[nid],
3013                                 h->surplus_huge_pages_node[nid],
3014                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3015 }
3016
3017 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3018 {
3019         seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3020                    atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3021 }
3022
3023 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3024 unsigned long hugetlb_total_pages(void)
3025 {
3026         struct hstate *h;
3027         unsigned long nr_total_pages = 0;
3028
3029         for_each_hstate(h)
3030                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3031         return nr_total_pages;
3032 }
3033
3034 static int hugetlb_acct_memory(struct hstate *h, long delta)
3035 {
3036         int ret = -ENOMEM;
3037
3038         spin_lock(&hugetlb_lock);
3039         /*
3040          * When cpuset is configured, it breaks the strict hugetlb page
3041          * reservation as the accounting is done on a global variable. Such
3042          * reservation is completely rubbish in the presence of cpuset because
3043          * the reservation is not checked against page availability for the
3044          * current cpuset. Application can still potentially OOM'ed by kernel
3045          * with lack of free htlb page in cpuset that the task is in.
3046          * Attempt to enforce strict accounting with cpuset is almost
3047          * impossible (or too ugly) because cpuset is too fluid that
3048          * task or memory node can be dynamically moved between cpusets.
3049          *
3050          * The change of semantics for shared hugetlb mapping with cpuset is
3051          * undesirable. However, in order to preserve some of the semantics,
3052          * we fall back to check against current free page availability as
3053          * a best attempt and hopefully to minimize the impact of changing
3054          * semantics that cpuset has.
3055          */
3056         if (delta > 0) {
3057                 if (gather_surplus_pages(h, delta) < 0)
3058                         goto out;
3059
3060                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
3061                         return_unused_surplus_pages(h, delta);
3062                         goto out;
3063                 }
3064         }
3065
3066         ret = 0;
3067         if (delta < 0)
3068                 return_unused_surplus_pages(h, (unsigned long) -delta);
3069
3070 out:
3071         spin_unlock(&hugetlb_lock);
3072         return ret;
3073 }
3074
3075 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3076 {
3077         struct resv_map *resv = vma_resv_map(vma);
3078
3079         /*
3080          * This new VMA should share its siblings reservation map if present.
3081          * The VMA will only ever have a valid reservation map pointer where
3082          * it is being copied for another still existing VMA.  As that VMA
3083          * has a reference to the reservation map it cannot disappear until
3084          * after this open call completes.  It is therefore safe to take a
3085          * new reference here without additional locking.
3086          */
3087         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3088                 kref_get(&resv->refs);
3089 }
3090
3091 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3092 {
3093         struct hstate *h = hstate_vma(vma);
3094         struct resv_map *resv = vma_resv_map(vma);
3095         struct hugepage_subpool *spool = subpool_vma(vma);
3096         unsigned long reserve, start, end;
3097         long gbl_reserve;
3098
3099         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3100                 return;
3101
3102         start = vma_hugecache_offset(h, vma, vma->vm_start);
3103         end = vma_hugecache_offset(h, vma, vma->vm_end);
3104
3105         reserve = (end - start) - region_count(resv, start, end);
3106
3107         kref_put(&resv->refs, resv_map_release);
3108
3109         if (reserve) {
3110                 /*
3111                  * Decrement reserve counts.  The global reserve count may be
3112                  * adjusted if the subpool has a minimum size.
3113                  */
3114                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3115                 hugetlb_acct_memory(h, -gbl_reserve);
3116         }
3117 }
3118
3119 /*
3120  * We cannot handle pagefaults against hugetlb pages at all.  They cause
3121  * handle_mm_fault() to try to instantiate regular-sized pages in the
3122  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
3123  * this far.
3124  */
3125 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3126 {
3127         BUG();
3128         return 0;
3129 }
3130
3131 const struct vm_operations_struct hugetlb_vm_ops = {
3132         .fault = hugetlb_vm_op_fault,
3133         .open = hugetlb_vm_op_open,
3134         .close = hugetlb_vm_op_close,
3135 };
3136
3137 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3138                                 int writable)
3139 {
3140         pte_t entry;
3141
3142         if (writable) {
3143                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3144                                          vma->vm_page_prot)));
3145         } else {
3146                 entry = huge_pte_wrprotect(mk_huge_pte(page,
3147                                            vma->vm_page_prot));
3148         }
3149         entry = pte_mkyoung(entry);
3150         entry = pte_mkhuge(entry);
3151         entry = arch_make_huge_pte(entry, vma, page, writable);
3152
3153         return entry;
3154 }
3155
3156 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3157                                    unsigned long address, pte_t *ptep)
3158 {
3159         pte_t entry;
3160
3161         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3162         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3163                 update_mmu_cache(vma, address, ptep);
3164 }
3165
3166 static int is_hugetlb_entry_migration(pte_t pte)
3167 {
3168         swp_entry_t swp;
3169
3170         if (huge_pte_none(pte) || pte_present(pte))
3171                 return 0;
3172         swp = pte_to_swp_entry(pte);
3173         if (non_swap_entry(swp) && is_migration_entry(swp))
3174                 return 1;
3175         else
3176                 return 0;
3177 }
3178
3179 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3180 {
3181         swp_entry_t swp;
3182
3183         if (huge_pte_none(pte) || pte_present(pte))
3184                 return 0;
3185         swp = pte_to_swp_entry(pte);
3186         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3187                 return 1;
3188         else
3189                 return 0;
3190 }
3191
3192 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3193                             struct vm_area_struct *vma)
3194 {
3195         pte_t *src_pte, *dst_pte, entry;
3196         struct page *ptepage;
3197         unsigned long addr;
3198         int cow;
3199         struct hstate *h = hstate_vma(vma);
3200         unsigned long sz = huge_page_size(h);
3201         unsigned long mmun_start;       /* For mmu_notifiers */
3202         unsigned long mmun_end;         /* For mmu_notifiers */
3203         int ret = 0;
3204
3205         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3206
3207         mmun_start = vma->vm_start;
3208         mmun_end = vma->vm_end;
3209         if (cow)
3210                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3211
3212         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3213                 spinlock_t *src_ptl, *dst_ptl;
3214                 src_pte = huge_pte_offset(src, addr);
3215                 if (!src_pte)
3216                         continue;
3217                 dst_pte = huge_pte_alloc(dst, addr, sz);
3218                 if (!dst_pte) {
3219                         ret = -ENOMEM;
3220                         break;
3221                 }
3222
3223                 /* If the pagetables are shared don't copy or take references */
3224                 if (dst_pte == src_pte)
3225                         continue;
3226
3227                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3228                 src_ptl = huge_pte_lockptr(h, src, src_pte);
3229                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3230                 entry = huge_ptep_get(src_pte);
3231                 if (huge_pte_none(entry)) { /* skip none entry */
3232                         ;
3233                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3234                                     is_hugetlb_entry_hwpoisoned(entry))) {
3235                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
3236
3237                         if (is_write_migration_entry(swp_entry) && cow) {
3238                                 /*
3239                                  * COW mappings require pages in both
3240                                  * parent and child to be set to read.
3241                                  */
3242                                 make_migration_entry_read(&swp_entry);
3243                                 entry = swp_entry_to_pte(swp_entry);
3244                                 set_huge_pte_at(src, addr, src_pte, entry);
3245                         }
3246                         set_huge_pte_at(dst, addr, dst_pte, entry);
3247                 } else {
3248                         if (cow) {
3249                                 huge_ptep_set_wrprotect(src, addr, src_pte);
3250                                 mmu_notifier_invalidate_range(src, mmun_start,
3251                                                                    mmun_end);
3252                         }
3253                         entry = huge_ptep_get(src_pte);
3254                         ptepage = pte_page(entry);
3255                         get_page(ptepage);
3256                         page_dup_rmap(ptepage, true);
3257                         set_huge_pte_at(dst, addr, dst_pte, entry);
3258                         hugetlb_count_add(pages_per_huge_page(h), dst);
3259                 }
3260                 spin_unlock(src_ptl);
3261                 spin_unlock(dst_ptl);
3262         }
3263
3264         if (cow)
3265                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3266
3267         return ret;
3268 }
3269
3270 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3271                             unsigned long start, unsigned long end,
3272                             struct page *ref_page)
3273 {
3274         struct mm_struct *mm = vma->vm_mm;
3275         unsigned long address;
3276         pte_t *ptep;
3277         pte_t pte;
3278         spinlock_t *ptl;
3279         struct page *page;
3280         struct hstate *h = hstate_vma(vma);
3281         unsigned long sz = huge_page_size(h);
3282         const unsigned long mmun_start = start; /* For mmu_notifiers */
3283         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
3284
3285         WARN_ON(!is_vm_hugetlb_page(vma));
3286         BUG_ON(start & ~huge_page_mask(h));
3287         BUG_ON(end & ~huge_page_mask(h));
3288
3289         tlb_start_vma(tlb, vma);
3290         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3291         address = start;
3292         for (; address < end; address += sz) {
3293                 ptep = huge_pte_offset(mm, address);
3294                 if (!ptep)
3295                         continue;
3296
3297                 ptl = huge_pte_lock(h, mm, ptep);
3298                 if (huge_pmd_unshare(mm, &address, ptep)) {
3299                         spin_unlock(ptl);
3300                         continue;
3301                 }
3302
3303                 pte = huge_ptep_get(ptep);
3304                 if (huge_pte_none(pte)) {
3305                         spin_unlock(ptl);
3306                         continue;
3307                 }
3308
3309                 /*
3310                  * Migrating hugepage or HWPoisoned hugepage is already
3311                  * unmapped and its refcount is dropped, so just clear pte here.
3312                  */
3313                 if (unlikely(!pte_present(pte))) {
3314                         huge_pte_clear(mm, address, ptep);
3315                         spin_unlock(ptl);
3316                         continue;
3317                 }
3318
3319                 page = pte_page(pte);
3320                 /*
3321                  * If a reference page is supplied, it is because a specific
3322                  * page is being unmapped, not a range. Ensure the page we
3323                  * are about to unmap is the actual page of interest.
3324                  */
3325                 if (ref_page) {
3326                         if (page != ref_page) {
3327                                 spin_unlock(ptl);
3328                                 continue;
3329                         }
3330                         /*
3331                          * Mark the VMA as having unmapped its page so that
3332                          * future faults in this VMA will fail rather than
3333                          * looking like data was lost
3334                          */
3335                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3336                 }
3337
3338                 pte = huge_ptep_get_and_clear(mm, address, ptep);
3339                 tlb_remove_tlb_entry(tlb, ptep, address);
3340                 if (huge_pte_dirty(pte))
3341                         set_page_dirty(page);
3342
3343                 hugetlb_count_sub(pages_per_huge_page(h), mm);
3344                 page_remove_rmap(page, true);
3345
3346                 spin_unlock(ptl);
3347                 tlb_remove_page_size(tlb, page, huge_page_size(h));
3348                 /*
3349                  * Bail out after unmapping reference page if supplied
3350                  */
3351                 if (ref_page)
3352                         break;
3353         }
3354         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3355         tlb_end_vma(tlb, vma);
3356 }
3357
3358 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3359                           struct vm_area_struct *vma, unsigned long start,
3360                           unsigned long end, struct page *ref_page)
3361 {
3362         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3363
3364         /*
3365          * Clear this flag so that x86's huge_pmd_share page_table_shareable
3366          * test will fail on a vma being torn down, and not grab a page table
3367          * on its way out.  We're lucky that the flag has such an appropriate
3368          * name, and can in fact be safely cleared here. We could clear it
3369          * before the __unmap_hugepage_range above, but all that's necessary
3370          * is to clear it before releasing the i_mmap_rwsem. This works
3371          * because in the context this is called, the VMA is about to be
3372          * destroyed and the i_mmap_rwsem is held.
3373          */
3374         vma->vm_flags &= ~VM_MAYSHARE;
3375 }
3376
3377 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3378                           unsigned long end, struct page *ref_page)
3379 {
3380         struct mm_struct *mm;
3381         struct mmu_gather tlb;
3382
3383         mm = vma->vm_mm;
3384
3385         tlb_gather_mmu(&tlb, mm, start, end);
3386         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3387         tlb_finish_mmu(&tlb, start, end);
3388 }
3389
3390 /*
3391  * This is called when the original mapper is failing to COW a MAP_PRIVATE
3392  * mappping it owns the reserve page for. The intention is to unmap the page
3393  * from other VMAs and let the children be SIGKILLed if they are faulting the
3394  * same region.
3395  */
3396 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3397                               struct page *page, unsigned long address)
3398 {
3399         struct hstate *h = hstate_vma(vma);
3400         struct vm_area_struct *iter_vma;
3401         struct address_space *mapping;
3402         pgoff_t pgoff;
3403
3404         /*
3405          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3406          * from page cache lookup which is in HPAGE_SIZE units.
3407          */
3408         address = address & huge_page_mask(h);
3409         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3410                         vma->vm_pgoff;
3411         mapping = vma->vm_file->f_mapping;
3412
3413         /*
3414          * Take the mapping lock for the duration of the table walk. As
3415          * this mapping should be shared between all the VMAs,
3416          * __unmap_hugepage_range() is called as the lock is already held
3417          */
3418         i_mmap_lock_write(mapping);
3419         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3420                 /* Do not unmap the current VMA */
3421                 if (iter_vma == vma)
3422                         continue;
3423
3424                 /*
3425                  * Shared VMAs have their own reserves and do not affect
3426                  * MAP_PRIVATE accounting but it is possible that a shared
3427                  * VMA is using the same page so check and skip such VMAs.
3428                  */
3429                 if (iter_vma->vm_flags & VM_MAYSHARE)
3430                         continue;
3431
3432                 /*
3433                  * Unmap the page from other VMAs without their own reserves.
3434                  * They get marked to be SIGKILLed if they fault in these
3435                  * areas. This is because a future no-page fault on this VMA
3436                  * could insert a zeroed page instead of the data existing
3437                  * from the time of fork. This would look like data corruption
3438                  */
3439                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3440                         unmap_hugepage_range(iter_vma, address,
3441                                              address + huge_page_size(h), page);
3442         }
3443         i_mmap_unlock_write(mapping);
3444 }
3445
3446 /*
3447  * Hugetlb_cow() should be called with page lock of the original hugepage held.
3448  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3449  * cannot race with other handlers or page migration.
3450  * Keep the pte_same checks anyway to make transition from the mutex easier.
3451  */
3452 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3453                         unsigned long address, pte_t *ptep, pte_t pte,
3454                         struct page *pagecache_page, spinlock_t *ptl)
3455 {
3456         struct hstate *h = hstate_vma(vma);
3457         struct page *old_page, *new_page;
3458         int ret = 0, outside_reserve = 0;
3459         unsigned long mmun_start;       /* For mmu_notifiers */
3460         unsigned long mmun_end;         /* For mmu_notifiers */
3461
3462         old_page = pte_page(pte);
3463
3464 retry_avoidcopy:
3465         /* If no-one else is actually using this page, avoid the copy
3466          * and just make the page writable */
3467         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3468                 page_move_anon_rmap(old_page, vma);
3469                 set_huge_ptep_writable(vma, address, ptep);
3470                 return 0;
3471         }
3472
3473         /*
3474          * If the process that created a MAP_PRIVATE mapping is about to
3475          * perform a COW due to a shared page count, attempt to satisfy
3476          * the allocation without using the existing reserves. The pagecache
3477          * page is used to determine if the reserve at this address was
3478          * consumed or not. If reserves were used, a partial faulted mapping
3479          * at the time of fork() could consume its reserves on COW instead
3480          * of the full address range.
3481          */
3482         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3483                         old_page != pagecache_page)
3484                 outside_reserve = 1;
3485
3486         get_page(old_page);
3487
3488         /*
3489          * Drop page table lock as buddy allocator may be called. It will
3490          * be acquired again before returning to the caller, as expected.
3491          */
3492         spin_unlock(ptl);
3493         new_page = alloc_huge_page(vma, address, outside_reserve);
3494
3495         if (IS_ERR(new_page)) {
3496                 /*
3497                  * If a process owning a MAP_PRIVATE mapping fails to COW,
3498                  * it is due to references held by a child and an insufficient
3499                  * huge page pool. To guarantee the original mappers
3500                  * reliability, unmap the page from child processes. The child
3501                  * may get SIGKILLed if it later faults.
3502                  */
3503                 if (outside_reserve) {
3504                         put_page(old_page);
3505                         BUG_ON(huge_pte_none(pte));
3506                         unmap_ref_private(mm, vma, old_page, address);
3507                         BUG_ON(huge_pte_none(pte));
3508                         spin_lock(ptl);
3509                         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3510                         if (likely(ptep &&
3511                                    pte_same(huge_ptep_get(ptep), pte)))
3512                                 goto retry_avoidcopy;
3513                         /*
3514                          * race occurs while re-acquiring page table
3515                          * lock, and our job is done.
3516                          */
3517                         return 0;
3518                 }
3519
3520                 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3521                         VM_FAULT_OOM : VM_FAULT_SIGBUS;
3522                 goto out_release_old;
3523         }
3524
3525         /*
3526          * When the original hugepage is shared one, it does not have
3527          * anon_vma prepared.
3528          */
3529         if (unlikely(anon_vma_prepare(vma))) {
3530                 ret = VM_FAULT_OOM;
3531                 goto out_release_all;
3532         }
3533
3534         copy_user_huge_page(new_page, old_page, address, vma,
3535                             pages_per_huge_page(h));
3536         __SetPageUptodate(new_page);
3537         set_page_huge_active(new_page);
3538
3539         mmun_start = address & huge_page_mask(h);
3540         mmun_end = mmun_start + huge_page_size(h);
3541         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3542
3543         /*
3544          * Retake the page table lock to check for racing updates
3545          * before the page tables are altered
3546          */
3547         spin_lock(ptl);
3548         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3549         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3550                 ClearPagePrivate(new_page);
3551
3552                 /* Break COW */
3553                 huge_ptep_clear_flush(vma, address, ptep);
3554                 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3555                 set_huge_pte_at(mm, address, ptep,
3556                                 make_huge_pte(vma, new_page, 1));
3557                 page_remove_rmap(old_page, true);
3558                 hugepage_add_new_anon_rmap(new_page, vma, address);
3559                 /* Make the old page be freed below */
3560                 new_page = old_page;
3561         }
3562         spin_unlock(ptl);
3563         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3564 out_release_all:
3565         restore_reserve_on_error(h, vma, address, new_page);
3566         put_page(new_page);
3567 out_release_old:
3568         put_page(old_page);
3569
3570         spin_lock(ptl); /* Caller expects lock to be held */
3571         return ret;
3572 }
3573
3574 /* Return the pagecache page at a given address within a VMA */
3575 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3576                         struct vm_area_struct *vma, unsigned long address)
3577 {
3578         struct address_space *mapping;
3579         pgoff_t idx;
3580
3581         mapping = vma->vm_file->f_mapping;
3582         idx = vma_hugecache_offset(h, vma, address);
3583
3584         return find_lock_page(mapping, idx);
3585 }
3586
3587 /*
3588  * Return whether there is a pagecache page to back given address within VMA.
3589  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3590  */
3591 static bool hugetlbfs_pagecache_present(struct hstate *h,
3592                         struct vm_area_struct *vma, unsigned long address)
3593 {
3594         struct address_space *mapping;
3595         pgoff_t idx;
3596         struct page *page;
3597
3598         mapping = vma->vm_file->f_mapping;
3599         idx = vma_hugecache_offset(h, vma, address);
3600
3601         page = find_get_page(mapping, idx);
3602         if (page)
3603                 put_page(page);
3604         return page != NULL;
3605 }
3606
3607 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3608                            pgoff_t idx)
3609 {
3610         struct inode *inode = mapping->host;
3611         struct hstate *h = hstate_inode(inode);
3612         int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3613
3614         if (err)
3615                 return err;
3616         ClearPagePrivate(page);
3617
3618         spin_lock(&inode->i_lock);
3619         inode->i_blocks += blocks_per_huge_page(h);
3620         spin_unlock(&inode->i_lock);
3621         return 0;
3622 }
3623
3624 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3625                            struct address_space *mapping, pgoff_t idx,
3626                            unsigned long address, pte_t *ptep, unsigned int flags)
3627 {
3628         struct hstate *h = hstate_vma(vma);
3629         int ret = VM_FAULT_SIGBUS;
3630         int anon_rmap = 0;
3631         unsigned long size;
3632         struct page *page;
3633         pte_t new_pte;
3634         spinlock_t *ptl;
3635
3636         /*
3637          * Currently, we are forced to kill the process in the event the
3638          * original mapper has unmapped pages from the child due to a failed
3639          * COW. Warn that such a situation has occurred as it may not be obvious
3640          */
3641         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3642                 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3643                            current->pid);
3644                 return ret;
3645         }
3646
3647         /*
3648          * Use page lock to guard against racing truncation
3649          * before we get page_table_lock.
3650          */
3651 retry:
3652         page = find_lock_page(mapping, idx);
3653         if (!page) {
3654                 size = i_size_read(mapping->host) >> huge_page_shift(h);
3655                 if (idx >= size)
3656                         goto out;
3657                 page = alloc_huge_page(vma, address, 0);
3658                 if (IS_ERR(page)) {
3659                         ret = PTR_ERR(page);
3660                         if (ret == -ENOMEM)
3661                                 ret = VM_FAULT_OOM;
3662                         else
3663                                 ret = VM_FAULT_SIGBUS;
3664                         goto out;
3665                 }
3666                 clear_huge_page(page, address, pages_per_huge_page(h));
3667                 __SetPageUptodate(page);
3668                 set_page_huge_active(page);
3669
3670                 if (vma->vm_flags & VM_MAYSHARE) {
3671                         int err = huge_add_to_page_cache(page, mapping, idx);
3672                         if (err) {
3673                                 put_page(page);
3674                                 if (err == -EEXIST)
3675                                         goto retry;
3676                                 goto out;
3677                         }
3678                 } else {
3679                         lock_page(page);
3680                         if (unlikely(anon_vma_prepare(vma))) {
3681                                 ret = VM_FAULT_OOM;
3682                                 goto backout_unlocked;
3683                         }
3684                         anon_rmap = 1;
3685                 }
3686         } else {
3687                 /*
3688                  * If memory error occurs between mmap() and fault, some process
3689                  * don't have hwpoisoned swap entry for errored virtual address.
3690                  * So we need to block hugepage fault by PG_hwpoison bit check.
3691                  */
3692                 if (unlikely(PageHWPoison(page))) {
3693                         ret = VM_FAULT_HWPOISON |
3694                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3695                         goto backout_unlocked;
3696                 }
3697         }
3698
3699         /*
3700          * If we are going to COW a private mapping later, we examine the
3701          * pending reservations for this page now. This will ensure that
3702          * any allocations necessary to record that reservation occur outside
3703          * the spinlock.
3704          */
3705         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3706                 if (vma_needs_reservation(h, vma, address) < 0) {
3707                         ret = VM_FAULT_OOM;
3708                         goto backout_unlocked;
3709                 }
3710                 /* Just decrements count, does not deallocate */
3711                 vma_end_reservation(h, vma, address);
3712         }
3713
3714         ptl = huge_pte_lockptr(h, mm, ptep);
3715         spin_lock(ptl);
3716         size = i_size_read(mapping->host) >> huge_page_shift(h);
3717         if (idx >= size)
3718                 goto backout;
3719
3720         ret = 0;
3721         if (!huge_pte_none(huge_ptep_get(ptep)))
3722                 goto backout;
3723
3724         if (anon_rmap) {
3725                 ClearPagePrivate(page);
3726                 hugepage_add_new_anon_rmap(page, vma, address);
3727         } else
3728                 page_dup_rmap(page, true);
3729         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3730                                 && (vma->vm_flags & VM_SHARED)));
3731         set_huge_pte_at(mm, address, ptep, new_pte);
3732
3733         hugetlb_count_add(pages_per_huge_page(h), mm);
3734         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3735                 /* Optimization, do the COW without a second fault */
3736                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3737         }
3738
3739         spin_unlock(ptl);
3740         unlock_page(page);
3741 out:
3742         return ret;
3743
3744 backout:
3745         spin_unlock(ptl);
3746 backout_unlocked:
3747         unlock_page(page);
3748         restore_reserve_on_error(h, vma, address, page);
3749         put_page(page);
3750         goto out;
3751 }
3752
3753 #ifdef CONFIG_SMP
3754 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3755                             struct vm_area_struct *vma,
3756                             struct address_space *mapping,
3757                             pgoff_t idx, unsigned long address)
3758 {
3759         unsigned long key[2];
3760         u32 hash;
3761
3762         if (vma->vm_flags & VM_SHARED) {
3763                 key[0] = (unsigned long) mapping;
3764                 key[1] = idx;
3765         } else {
3766                 key[0] = (unsigned long) mm;
3767                 key[1] = address >> huge_page_shift(h);
3768         }
3769
3770         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3771
3772         return hash & (num_fault_mutexes - 1);
3773 }
3774 #else
3775 /*
3776  * For uniprocesor systems we always use a single mutex, so just
3777  * return 0 and avoid the hashing overhead.
3778  */
3779 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3780                             struct vm_area_struct *vma,
3781                             struct address_space *mapping,
3782                             pgoff_t idx, unsigned long address)
3783 {
3784         return 0;
3785 }
3786 #endif
3787
3788 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3789                         unsigned long address, unsigned int flags)
3790 {
3791         pte_t *ptep, entry;
3792         spinlock_t *ptl;
3793         int ret;
3794         u32 hash;
3795         pgoff_t idx;
3796         struct page *page = NULL;
3797         struct page *pagecache_page = NULL;
3798         struct hstate *h = hstate_vma(vma);
3799         struct address_space *mapping;
3800         int need_wait_lock = 0;
3801
3802         address &= huge_page_mask(h);
3803
3804         ptep = huge_pte_offset(mm, address);
3805         if (ptep) {
3806                 entry = huge_ptep_get(ptep);
3807                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3808                         migration_entry_wait_huge(vma, mm, ptep);
3809                         return 0;
3810                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3811                         return VM_FAULT_HWPOISON_LARGE |
3812                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3813         } else {
3814                 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3815                 if (!ptep)
3816                         return VM_FAULT_OOM;
3817         }
3818
3819         mapping = vma->vm_file->f_mapping;
3820         idx = vma_hugecache_offset(h, vma, address);
3821
3822         /*
3823          * Serialize hugepage allocation and instantiation, so that we don't
3824          * get spurious allocation failures if two CPUs race to instantiate
3825          * the same page in the page cache.
3826          */
3827         hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
3828         mutex_lock(&hugetlb_fault_mutex_table[hash]);
3829
3830         entry = huge_ptep_get(ptep);
3831         if (huge_pte_none(entry)) {
3832                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3833                 goto out_mutex;
3834         }
3835
3836         ret = 0;
3837
3838         /*
3839          * entry could be a migration/hwpoison entry at this point, so this
3840          * check prevents the kernel from going below assuming that we have
3841          * a active hugepage in pagecache. This goto expects the 2nd page fault,
3842          * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3843          * handle it.
3844          */
3845         if (!pte_present(entry))
3846                 goto out_mutex;
3847
3848         /*
3849          * If we are going to COW the mapping later, we examine the pending
3850          * reservations for this page now. This will ensure that any
3851          * allocations necessary to record that reservation occur outside the
3852          * spinlock. For private mappings, we also lookup the pagecache
3853          * page now as it is used to determine if a reservation has been
3854          * consumed.
3855          */
3856         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3857                 if (vma_needs_reservation(h, vma, address) < 0) {
3858                         ret = VM_FAULT_OOM;
3859                         goto out_mutex;
3860                 }
3861                 /* Just decrements count, does not deallocate */
3862                 vma_end_reservation(h, vma, address);
3863
3864                 if (!(vma->vm_flags & VM_MAYSHARE))
3865                         pagecache_page = hugetlbfs_pagecache_page(h,
3866                                                                 vma, address);
3867         }
3868
3869         ptl = huge_pte_lock(h, mm, ptep);
3870
3871         /* Check for a racing update before calling hugetlb_cow */
3872         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3873                 goto out_ptl;
3874
3875         /*
3876          * hugetlb_cow() requires page locks of pte_page(entry) and
3877          * pagecache_page, so here we need take the former one
3878          * when page != pagecache_page or !pagecache_page.
3879          */
3880         page = pte_page(entry);
3881         if (page != pagecache_page)
3882                 if (!trylock_page(page)) {
3883                         need_wait_lock = 1;
3884                         goto out_ptl;
3885                 }
3886
3887         get_page(page);
3888
3889         if (flags & FAULT_FLAG_WRITE) {
3890                 if (!huge_pte_write(entry)) {
3891                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
3892                                         pagecache_page, ptl);
3893                         goto out_put_page;
3894                 }
3895                 entry = huge_pte_mkdirty(entry);
3896         }
3897         entry = pte_mkyoung(entry);
3898         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3899                                                 flags & FAULT_FLAG_WRITE))
3900                 update_mmu_cache(vma, address, ptep);
3901 out_put_page:
3902         if (page != pagecache_page)
3903                 unlock_page(page);
3904         put_page(page);
3905 out_ptl:
3906         spin_unlock(ptl);
3907
3908         if (pagecache_page) {
3909                 unlock_page(pagecache_page);
3910                 put_page(pagecache_page);
3911         }
3912 out_mutex:
3913         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3914         /*
3915          * Generally it's safe to hold refcount during waiting page lock. But
3916          * here we just wait to defer the next page fault to avoid busy loop and
3917          * the page is not used after unlocked before returning from the current
3918          * page fault. So we are safe from accessing freed page, even if we wait
3919          * here without taking refcount.
3920          */
3921         if (need_wait_lock)
3922                 wait_on_page_locked(page);
3923         return ret;
3924 }
3925
3926 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3927                          struct page **pages, struct vm_area_struct **vmas,
3928                          unsigned long *position, unsigned long *nr_pages,
3929                          long i, unsigned int flags)
3930 {
3931         unsigned long pfn_offset;
3932         unsigned long vaddr = *position;
3933         unsigned long remainder = *nr_pages;
3934         struct hstate *h = hstate_vma(vma);
3935
3936         while (vaddr < vma->vm_end && remainder) {
3937                 pte_t *pte;
3938                 spinlock_t *ptl = NULL;
3939                 int absent;
3940                 struct page *page;
3941
3942                 /*
3943                  * If we have a pending SIGKILL, don't keep faulting pages and
3944                  * potentially allocating memory.
3945                  */
3946                 if (unlikely(fatal_signal_pending(current))) {
3947                         remainder = 0;
3948                         break;
3949                 }
3950
3951                 /*
3952                  * Some archs (sparc64, sh*) have multiple pte_ts to
3953                  * each hugepage.  We have to make sure we get the
3954                  * first, for the page indexing below to work.
3955                  *
3956                  * Note that page table lock is not held when pte is null.
3957                  */
3958                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3959                 if (pte)
3960                         ptl = huge_pte_lock(h, mm, pte);
3961                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3962
3963                 /*
3964                  * When coredumping, it suits get_dump_page if we just return
3965                  * an error where there's an empty slot with no huge pagecache
3966                  * to back it.  This way, we avoid allocating a hugepage, and
3967                  * the sparse dumpfile avoids allocating disk blocks, but its
3968                  * huge holes still show up with zeroes where they need to be.
3969                  */
3970                 if (absent && (flags & FOLL_DUMP) &&
3971                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3972                         if (pte)
3973                                 spin_unlock(ptl);
3974                         remainder = 0;
3975                         break;
3976                 }
3977
3978                 /*
3979                  * We need call hugetlb_fault for both hugepages under migration
3980                  * (in which case hugetlb_fault waits for the migration,) and
3981                  * hwpoisoned hugepages (in which case we need to prevent the
3982                  * caller from accessing to them.) In order to do this, we use
3983                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3984                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3985                  * both cases, and because we can't follow correct pages
3986                  * directly from any kind of swap entries.
3987                  */
3988                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3989                     ((flags & FOLL_WRITE) &&
3990                       !huge_pte_write(huge_ptep_get(pte)))) {
3991                         int ret;
3992
3993                         if (pte)
3994                                 spin_unlock(ptl);
3995                         ret = hugetlb_fault(mm, vma, vaddr,
3996                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3997                         if (!(ret & VM_FAULT_ERROR))
3998                                 continue;
3999
4000                         remainder = 0;
4001                         break;
4002                 }
4003
4004                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
4005                 page = pte_page(huge_ptep_get(pte));
4006 same_page:
4007                 if (pages) {
4008                         pages[i] = mem_map_offset(page, pfn_offset);
4009                         get_page(pages[i]);
4010                 }
4011
4012                 if (vmas)
4013                         vmas[i] = vma;
4014
4015                 vaddr += PAGE_SIZE;
4016                 ++pfn_offset;
4017                 --remainder;
4018                 ++i;
4019                 if (vaddr < vma->vm_end && remainder &&
4020                                 pfn_offset < pages_per_huge_page(h)) {
4021                         /*
4022                          * We use pfn_offset to avoid touching the pageframes
4023                          * of this compound page.
4024                          */
4025                         goto same_page;
4026                 }
4027                 spin_unlock(ptl);
4028         }
4029         *nr_pages = remainder;
4030         *position = vaddr;
4031
4032         return i ? i : -EFAULT;
4033 }
4034
4035 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
4036 /*
4037  * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
4038  * implement this.
4039  */
4040 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
4041 #endif
4042
4043 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
4044                 unsigned long address, unsigned long end, pgprot_t newprot)
4045 {
4046         struct mm_struct *mm = vma->vm_mm;
4047         unsigned long start = address;
4048         pte_t *ptep;
4049         pte_t pte;
4050         struct hstate *h = hstate_vma(vma);
4051         unsigned long pages = 0;
4052
4053         BUG_ON(address >= end);
4054         flush_cache_range(vma, address, end);
4055
4056         mmu_notifier_invalidate_range_start(mm, start, end);
4057         i_mmap_lock_write(vma->vm_file->f_mapping);
4058         for (; address < end; address += huge_page_size(h)) {
4059                 spinlock_t *ptl;
4060                 ptep = huge_pte_offset(mm, address);
4061                 if (!ptep)
4062                         continue;
4063                 ptl = huge_pte_lock(h, mm, ptep);
4064                 if (huge_pmd_unshare(mm, &address, ptep)) {
4065                         pages++;
4066                         spin_unlock(ptl);
4067                         continue;
4068                 }
4069                 pte = huge_ptep_get(ptep);
4070                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
4071                         spin_unlock(ptl);
4072                         continue;
4073                 }
4074                 if (unlikely(is_hugetlb_entry_migration(pte))) {
4075                         swp_entry_t entry = pte_to_swp_entry(pte);
4076
4077                         if (is_write_migration_entry(entry)) {
4078                                 pte_t newpte;
4079
4080                                 make_migration_entry_read(&entry);
4081                                 newpte = swp_entry_to_pte(entry);
4082                                 set_huge_pte_at(mm, address, ptep, newpte);
4083                                 pages++;
4084                         }
4085                         spin_unlock(ptl);
4086                         continue;
4087                 }
4088                 if (!huge_pte_none(pte)) {
4089                         pte = huge_ptep_get_and_clear(mm, address, ptep);
4090                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4091                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
4092                         set_huge_pte_at(mm, address, ptep, pte);
4093                         pages++;
4094                 }
4095                 spin_unlock(ptl);
4096         }
4097         /*
4098          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4099          * may have cleared our pud entry and done put_page on the page table:
4100          * once we release i_mmap_rwsem, another task can do the final put_page
4101          * and that page table be reused and filled with junk.
4102          */
4103         flush_hugetlb_tlb_range(vma, start, end);
4104         mmu_notifier_invalidate_range(mm, start, end);
4105         i_mmap_unlock_write(vma->vm_file->f_mapping);
4106         mmu_notifier_invalidate_range_end(mm, start, end);
4107
4108         return pages << h->order;
4109 }
4110
4111 int hugetlb_reserve_pages(struct inode *inode,
4112                                         long from, long to,
4113                                         struct vm_area_struct *vma,
4114                                         vm_flags_t vm_flags)
4115 {
4116         long ret, chg;
4117         struct hstate *h = hstate_inode(inode);
4118         struct hugepage_subpool *spool = subpool_inode(inode);
4119         struct resv_map *resv_map;
4120         long gbl_reserve;
4121
4122         /*
4123          * Only apply hugepage reservation if asked. At fault time, an
4124          * attempt will be made for VM_NORESERVE to allocate a page
4125          * without using reserves
4126          */
4127         if (vm_flags & VM_NORESERVE)
4128                 return 0;
4129
4130         /*
4131          * Shared mappings base their reservation on the number of pages that
4132          * are already allocated on behalf of the file. Private mappings need
4133          * to reserve the full area even if read-only as mprotect() may be
4134          * called to make the mapping read-write. Assume !vma is a shm mapping
4135          */
4136         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4137                 resv_map = inode_resv_map(inode);
4138
4139                 chg = region_chg(resv_map, from, to);
4140
4141         } else {
4142                 resv_map = resv_map_alloc();
4143                 if (!resv_map)
4144                         return -ENOMEM;
4145
4146                 chg = to - from;
4147
4148                 set_vma_resv_map(vma, resv_map);
4149                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4150         }
4151
4152         if (chg < 0) {
4153                 ret = chg;
4154                 goto out_err;
4155         }
4156
4157         /*
4158          * There must be enough pages in the subpool for the mapping. If
4159          * the subpool has a minimum size, there may be some global
4160          * reservations already in place (gbl_reserve).
4161          */
4162         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4163         if (gbl_reserve < 0) {
4164                 ret = -ENOSPC;
4165                 goto out_err;
4166         }
4167
4168         /*
4169          * Check enough hugepages are available for the reservation.
4170          * Hand the pages back to the subpool if there are not
4171          */
4172         ret = hugetlb_acct_memory(h, gbl_reserve);
4173         if (ret < 0) {
4174                 /* put back original number of pages, chg */
4175                 (void)hugepage_subpool_put_pages(spool, chg);
4176                 goto out_err;
4177         }
4178
4179         /*
4180          * Account for the reservations made. Shared mappings record regions
4181          * that have reservations as they are shared by multiple VMAs.
4182          * When the last VMA disappears, the region map says how much
4183          * the reservation was and the page cache tells how much of
4184          * the reservation was consumed. Private mappings are per-VMA and
4185          * only the consumed reservations are tracked. When the VMA
4186          * disappears, the original reservation is the VMA size and the
4187          * consumed reservations are stored in the map. Hence, nothing
4188          * else has to be done for private mappings here
4189          */
4190         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4191                 long add = region_add(resv_map, from, to);
4192
4193                 if (unlikely(chg > add)) {
4194                         /*
4195                          * pages in this range were added to the reserve
4196                          * map between region_chg and region_add.  This
4197                          * indicates a race with alloc_huge_page.  Adjust
4198                          * the subpool and reserve counts modified above
4199                          * based on the difference.
4200                          */
4201                         long rsv_adjust;
4202
4203                         rsv_adjust = hugepage_subpool_put_pages(spool,
4204                                                                 chg - add);
4205                         hugetlb_acct_memory(h, -rsv_adjust);
4206                 }
4207         }
4208         return 0;
4209 out_err:
4210         if (!vma || vma->vm_flags & VM_MAYSHARE)
4211                 region_abort(resv_map, from, to);
4212         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4213                 kref_put(&resv_map->refs, resv_map_release);
4214         return ret;
4215 }
4216
4217 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4218                                                                 long freed)
4219 {
4220         struct hstate *h = hstate_inode(inode);
4221         struct resv_map *resv_map = inode_resv_map(inode);
4222         long chg = 0;
4223         struct hugepage_subpool *spool = subpool_inode(inode);
4224         long gbl_reserve;
4225
4226         if (resv_map) {
4227                 chg = region_del(resv_map, start, end);
4228                 /*
4229                  * region_del() can fail in the rare case where a region
4230                  * must be split and another region descriptor can not be
4231                  * allocated.  If end == LONG_MAX, it will not fail.
4232                  */
4233                 if (chg < 0)
4234                         return chg;
4235         }
4236
4237         spin_lock(&inode->i_lock);
4238         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4239         spin_unlock(&inode->i_lock);
4240
4241         /*
4242          * If the subpool has a minimum size, the number of global
4243          * reservations to be released may be adjusted.
4244          */
4245         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4246         hugetlb_acct_memory(h, -gbl_reserve);
4247
4248         return 0;
4249 }
4250
4251 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4252 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4253                                 struct vm_area_struct *vma,
4254                                 unsigned long addr, pgoff_t idx)
4255 {
4256         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4257                                 svma->vm_start;
4258         unsigned long sbase = saddr & PUD_MASK;
4259         unsigned long s_end = sbase + PUD_SIZE;
4260
4261         /* Allow segments to share if only one is marked locked */
4262         unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4263         unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4264
4265         /*
4266          * match the virtual addresses, permission and the alignment of the
4267          * page table page.
4268          */
4269         if (pmd_index(addr) != pmd_index(saddr) ||
4270             vm_flags != svm_flags ||
4271             sbase < svma->vm_start || svma->vm_end < s_end)
4272                 return 0;
4273
4274         return saddr;
4275 }
4276
4277 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4278 {
4279         unsigned long base = addr & PUD_MASK;
4280         unsigned long end = base + PUD_SIZE;
4281
4282         /*
4283          * check on proper vm_flags and page table alignment
4284          */
4285         if (vma->vm_flags & VM_MAYSHARE &&
4286             vma->vm_start <= base && end <= vma->vm_end)
4287                 return true;
4288         return false;
4289 }
4290
4291 /*
4292  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4293  * and returns the corresponding pte. While this is not necessary for the
4294  * !shared pmd case because we can allocate the pmd later as well, it makes the
4295  * code much cleaner. pmd allocation is essential for the shared case because
4296  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4297  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4298  * bad pmd for sharing.
4299  */
4300 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4301 {
4302         struct vm_area_struct *vma = find_vma(mm, addr);
4303         struct address_space *mapping = vma->vm_file->f_mapping;
4304         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4305                         vma->vm_pgoff;
4306         struct vm_area_struct *svma;
4307         unsigned long saddr;
4308         pte_t *spte = NULL;
4309         pte_t *pte;
4310         spinlock_t *ptl;
4311
4312         if (!vma_shareable(vma, addr))
4313                 return (pte_t *)pmd_alloc(mm, pud, addr);
4314
4315         i_mmap_lock_write(mapping);
4316         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4317                 if (svma == vma)
4318                         continue;
4319
4320                 saddr = page_table_shareable(svma, vma, addr, idx);
4321                 if (saddr) {
4322                         spte = huge_pte_offset(svma->vm_mm, saddr);
4323                         if (spte) {
4324                                 get_page(virt_to_page(spte));
4325                                 break;
4326                         }
4327                 }
4328         }
4329
4330         if (!spte)
4331                 goto out;
4332
4333         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
4334         spin_lock(ptl);
4335         if (pud_none(*pud)) {
4336                 pud_populate(mm, pud,
4337                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4338                 mm_inc_nr_pmds(mm);
4339         } else {
4340                 put_page(virt_to_page(spte));
4341         }
4342         spin_unlock(ptl);
4343 out:
4344         pte = (pte_t *)pmd_alloc(mm, pud, addr);
4345         i_mmap_unlock_write(mapping);
4346         return pte;
4347 }
4348
4349 /*
4350  * unmap huge page backed by shared pte.
4351  *
4352  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
4353  * indicated by page_count > 1, unmap is achieved by clearing pud and
4354  * decrementing the ref count. If count == 1, the pte page is not shared.
4355  *
4356  * called with page table lock held.
4357  *
4358  * returns: 1 successfully unmapped a shared pte page
4359  *          0 the underlying pte page is not shared, or it is the last user
4360  */
4361 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4362 {
4363         pgd_t *pgd = pgd_offset(mm, *addr);
4364         pud_t *pud = pud_offset(pgd, *addr);
4365
4366         BUG_ON(page_count(virt_to_page(ptep)) == 0);
4367         if (page_count(virt_to_page(ptep)) == 1)
4368                 return 0;
4369
4370         pud_clear(pud);
4371         put_page(virt_to_page(ptep));
4372         mm_dec_nr_pmds(mm);
4373         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4374         return 1;
4375 }
4376 #define want_pmd_share()        (1)
4377 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4378 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4379 {
4380         return NULL;
4381 }
4382
4383 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4384 {
4385         return 0;
4386 }
4387 #define want_pmd_share()        (0)
4388 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4389
4390 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4391 pte_t *huge_pte_alloc(struct mm_struct *mm,
4392                         unsigned long addr, unsigned long sz)
4393 {
4394         pgd_t *pgd;
4395         pud_t *pud;
4396         pte_t *pte = NULL;
4397
4398         pgd = pgd_offset(mm, addr);
4399         pud = pud_alloc(mm, pgd, addr);
4400         if (pud) {
4401                 if (sz == PUD_SIZE) {
4402                         pte = (pte_t *)pud;
4403                 } else {
4404                         BUG_ON(sz != PMD_SIZE);
4405                         if (want_pmd_share() && pud_none(*pud))
4406                                 pte = huge_pmd_share(mm, addr, pud);
4407                         else
4408                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4409                 }
4410         }
4411         BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
4412
4413         return pte;
4414 }
4415
4416 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4417 {
4418         pgd_t *pgd;
4419         pud_t *pud;
4420         pmd_t *pmd = NULL;
4421
4422         pgd = pgd_offset(mm, addr);
4423         if (pgd_present(*pgd)) {
4424                 pud = pud_offset(pgd, addr);
4425                 if (pud_present(*pud)) {
4426                         if (pud_huge(*pud))
4427                                 return (pte_t *)pud;
4428                         pmd = pmd_offset(pud, addr);
4429                 }
4430         }
4431         return (pte_t *) pmd;
4432 }
4433
4434 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4435
4436 /*
4437  * These functions are overwritable if your architecture needs its own
4438  * behavior.
4439  */
4440 struct page * __weak
4441 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4442                               int write)
4443 {
4444         return ERR_PTR(-EINVAL);
4445 }
4446
4447 struct page * __weak
4448 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4449                 pmd_t *pmd, int flags)
4450 {
4451         struct page *page = NULL;
4452         spinlock_t *ptl;
4453 retry:
4454         ptl = pmd_lockptr(mm, pmd);
4455         spin_lock(ptl);
4456         /*
4457          * make sure that the address range covered by this pmd is not
4458          * unmapped from other threads.
4459          */
4460         if (!pmd_huge(*pmd))
4461                 goto out;
4462         if (pmd_present(*pmd)) {
4463                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4464                 if (flags & FOLL_GET)
4465                         get_page(page);
4466         } else {
4467                 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
4468                         spin_unlock(ptl);
4469                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
4470                         goto retry;
4471                 }
4472                 /*
4473                  * hwpoisoned entry is treated as no_page_table in
4474                  * follow_page_mask().
4475                  */
4476         }
4477 out:
4478         spin_unlock(ptl);
4479         return page;
4480 }
4481
4482 struct page * __weak
4483 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4484                 pud_t *pud, int flags)
4485 {
4486         if (flags & FOLL_GET)
4487                 return NULL;
4488
4489         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4490 }
4491
4492 #ifdef CONFIG_MEMORY_FAILURE
4493
4494 /*
4495  * This function is called from memory failure code.
4496  */
4497 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4498 {
4499         struct hstate *h = page_hstate(hpage);
4500         int nid = page_to_nid(hpage);
4501         int ret = -EBUSY;
4502
4503         spin_lock(&hugetlb_lock);
4504         /*
4505          * Just checking !page_huge_active is not enough, because that could be
4506          * an isolated/hwpoisoned hugepage (which have >0 refcount).
4507          */
4508         if (!page_huge_active(hpage) && !page_count(hpage)) {
4509                 /*
4510                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
4511                  * but dangling hpage->lru can trigger list-debug warnings
4512                  * (this happens when we call unpoison_memory() on it),
4513                  * so let it point to itself with list_del_init().
4514                  */
4515                 list_del_init(&hpage->lru);
4516                 set_page_refcounted(hpage);
4517                 h->free_huge_pages--;
4518                 h->free_huge_pages_node[nid]--;
4519                 ret = 0;
4520         }
4521         spin_unlock(&hugetlb_lock);
4522         return ret;
4523 }
4524 #endif
4525
4526 bool isolate_huge_page(struct page *page, struct list_head *list)
4527 {
4528         bool ret = true;
4529
4530         VM_BUG_ON_PAGE(!PageHead(page), page);
4531         spin_lock(&hugetlb_lock);
4532         if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4533                 ret = false;
4534                 goto unlock;
4535         }
4536         clear_page_huge_active(page);
4537         list_move_tail(&page->lru, list);
4538 unlock:
4539         spin_unlock(&hugetlb_lock);
4540         return ret;
4541 }
4542
4543 void putback_active_hugepage(struct page *page)
4544 {
4545         VM_BUG_ON_PAGE(!PageHead(page), page);
4546         spin_lock(&hugetlb_lock);
4547         set_page_huge_active(page);
4548         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4549         spin_unlock(&hugetlb_lock);
4550         put_page(page);
4551 }