]> git.karo-electronics.de Git - karo-tx-linux.git/blob - mm/hugetlb.c
Merge tag 'armsoc-fixes-nc' of git://git.kernel.org/pub/scm/linux/kernel/git/arm...
[karo-tx-linux.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/seq_file.h>
9 #include <linux/sysctl.h>
10 #include <linux/highmem.h>
11 #include <linux/mmu_notifier.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/compiler.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24 #include <linux/page-isolation.h>
25 #include <linux/jhash.h>
26
27 #include <asm/page.h>
28 #include <asm/pgtable.h>
29 #include <asm/tlb.h>
30
31 #include <linux/io.h>
32 #include <linux/hugetlb.h>
33 #include <linux/hugetlb_cgroup.h>
34 #include <linux/node.h>
35 #include "internal.h"
36
37 int hugepages_treat_as_movable;
38
39 int hugetlb_max_hstate __read_mostly;
40 unsigned int default_hstate_idx;
41 struct hstate hstates[HUGE_MAX_HSTATE];
42 /*
43  * Minimum page order among possible hugepage sizes, set to a proper value
44  * at boot time.
45  */
46 static unsigned int minimum_order __read_mostly = UINT_MAX;
47
48 __initdata LIST_HEAD(huge_boot_pages);
49
50 /* for command line parsing */
51 static struct hstate * __initdata parsed_hstate;
52 static unsigned long __initdata default_hstate_max_huge_pages;
53 static unsigned long __initdata default_hstate_size;
54 static bool __initdata parsed_valid_hugepagesz = true;
55
56 /*
57  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
58  * free_huge_pages, and surplus_huge_pages.
59  */
60 DEFINE_SPINLOCK(hugetlb_lock);
61
62 /*
63  * Serializes faults on the same logical page.  This is used to
64  * prevent spurious OOMs when the hugepage pool is fully utilized.
65  */
66 static int num_fault_mutexes;
67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
68
69 /* Forward declaration */
70 static int hugetlb_acct_memory(struct hstate *h, long delta);
71
72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
73 {
74         bool free = (spool->count == 0) && (spool->used_hpages == 0);
75
76         spin_unlock(&spool->lock);
77
78         /* If no pages are used, and no other handles to the subpool
79          * remain, give up any reservations mased on minimum size and
80          * free the subpool */
81         if (free) {
82                 if (spool->min_hpages != -1)
83                         hugetlb_acct_memory(spool->hstate,
84                                                 -spool->min_hpages);
85                 kfree(spool);
86         }
87 }
88
89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
90                                                 long min_hpages)
91 {
92         struct hugepage_subpool *spool;
93
94         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
95         if (!spool)
96                 return NULL;
97
98         spin_lock_init(&spool->lock);
99         spool->count = 1;
100         spool->max_hpages = max_hpages;
101         spool->hstate = h;
102         spool->min_hpages = min_hpages;
103
104         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105                 kfree(spool);
106                 return NULL;
107         }
108         spool->rsv_hpages = min_hpages;
109
110         return spool;
111 }
112
113 void hugepage_put_subpool(struct hugepage_subpool *spool)
114 {
115         spin_lock(&spool->lock);
116         BUG_ON(!spool->count);
117         spool->count--;
118         unlock_or_release_subpool(spool);
119 }
120
121 /*
122  * Subpool accounting for allocating and reserving pages.
123  * Return -ENOMEM if there are not enough resources to satisfy the
124  * the request.  Otherwise, return the number of pages by which the
125  * global pools must be adjusted (upward).  The returned value may
126  * only be different than the passed value (delta) in the case where
127  * a subpool minimum size must be manitained.
128  */
129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130                                       long delta)
131 {
132         long ret = delta;
133
134         if (!spool)
135                 return ret;
136
137         spin_lock(&spool->lock);
138
139         if (spool->max_hpages != -1) {          /* maximum size accounting */
140                 if ((spool->used_hpages + delta) <= spool->max_hpages)
141                         spool->used_hpages += delta;
142                 else {
143                         ret = -ENOMEM;
144                         goto unlock_ret;
145                 }
146         }
147
148         /* minimum size accounting */
149         if (spool->min_hpages != -1 && spool->rsv_hpages) {
150                 if (delta > spool->rsv_hpages) {
151                         /*
152                          * Asking for more reserves than those already taken on
153                          * behalf of subpool.  Return difference.
154                          */
155                         ret = delta - spool->rsv_hpages;
156                         spool->rsv_hpages = 0;
157                 } else {
158                         ret = 0;        /* reserves already accounted for */
159                         spool->rsv_hpages -= delta;
160                 }
161         }
162
163 unlock_ret:
164         spin_unlock(&spool->lock);
165         return ret;
166 }
167
168 /*
169  * Subpool accounting for freeing and unreserving pages.
170  * Return the number of global page reservations that must be dropped.
171  * The return value may only be different than the passed value (delta)
172  * in the case where a subpool minimum size must be maintained.
173  */
174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
175                                        long delta)
176 {
177         long ret = delta;
178
179         if (!spool)
180                 return delta;
181
182         spin_lock(&spool->lock);
183
184         if (spool->max_hpages != -1)            /* maximum size accounting */
185                 spool->used_hpages -= delta;
186
187          /* minimum size accounting */
188         if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
189                 if (spool->rsv_hpages + delta <= spool->min_hpages)
190                         ret = 0;
191                 else
192                         ret = spool->rsv_hpages + delta - spool->min_hpages;
193
194                 spool->rsv_hpages += delta;
195                 if (spool->rsv_hpages > spool->min_hpages)
196                         spool->rsv_hpages = spool->min_hpages;
197         }
198
199         /*
200          * If hugetlbfs_put_super couldn't free spool due to an outstanding
201          * quota reference, free it now.
202          */
203         unlock_or_release_subpool(spool);
204
205         return ret;
206 }
207
208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
209 {
210         return HUGETLBFS_SB(inode->i_sb)->spool;
211 }
212
213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
214 {
215         return subpool_inode(file_inode(vma->vm_file));
216 }
217
218 /*
219  * Region tracking -- allows tracking of reservations and instantiated pages
220  *                    across the pages in a mapping.
221  *
222  * The region data structures are embedded into a resv_map and protected
223  * by a resv_map's lock.  The set of regions within the resv_map represent
224  * reservations for huge pages, or huge pages that have already been
225  * instantiated within the map.  The from and to elements are huge page
226  * indicies into the associated mapping.  from indicates the starting index
227  * of the region.  to represents the first index past the end of  the region.
228  *
229  * For example, a file region structure with from == 0 and to == 4 represents
230  * four huge pages in a mapping.  It is important to note that the to element
231  * represents the first element past the end of the region. This is used in
232  * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
233  *
234  * Interval notation of the form [from, to) will be used to indicate that
235  * the endpoint from is inclusive and to is exclusive.
236  */
237 struct file_region {
238         struct list_head link;
239         long from;
240         long to;
241 };
242
243 /*
244  * Add the huge page range represented by [f, t) to the reserve
245  * map.  In the normal case, existing regions will be expanded
246  * to accommodate the specified range.  Sufficient regions should
247  * exist for expansion due to the previous call to region_chg
248  * with the same range.  However, it is possible that region_del
249  * could have been called after region_chg and modifed the map
250  * in such a way that no region exists to be expanded.  In this
251  * case, pull a region descriptor from the cache associated with
252  * the map and use that for the new range.
253  *
254  * Return the number of new huge pages added to the map.  This
255  * number is greater than or equal to zero.
256  */
257 static long region_add(struct resv_map *resv, long f, long t)
258 {
259         struct list_head *head = &resv->regions;
260         struct file_region *rg, *nrg, *trg;
261         long add = 0;
262
263         spin_lock(&resv->lock);
264         /* Locate the region we are either in or before. */
265         list_for_each_entry(rg, head, link)
266                 if (f <= rg->to)
267                         break;
268
269         /*
270          * If no region exists which can be expanded to include the
271          * specified range, the list must have been modified by an
272          * interleving call to region_del().  Pull a region descriptor
273          * from the cache and use it for this range.
274          */
275         if (&rg->link == head || t < rg->from) {
276                 VM_BUG_ON(resv->region_cache_count <= 0);
277
278                 resv->region_cache_count--;
279                 nrg = list_first_entry(&resv->region_cache, struct file_region,
280                                         link);
281                 list_del(&nrg->link);
282
283                 nrg->from = f;
284                 nrg->to = t;
285                 list_add(&nrg->link, rg->link.prev);
286
287                 add += t - f;
288                 goto out_locked;
289         }
290
291         /* Round our left edge to the current segment if it encloses us. */
292         if (f > rg->from)
293                 f = rg->from;
294
295         /* Check for and consume any regions we now overlap with. */
296         nrg = rg;
297         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
298                 if (&rg->link == head)
299                         break;
300                 if (rg->from > t)
301                         break;
302
303                 /* If this area reaches higher then extend our area to
304                  * include it completely.  If this is not the first area
305                  * which we intend to reuse, free it. */
306                 if (rg->to > t)
307                         t = rg->to;
308                 if (rg != nrg) {
309                         /* Decrement return value by the deleted range.
310                          * Another range will span this area so that by
311                          * end of routine add will be >= zero
312                          */
313                         add -= (rg->to - rg->from);
314                         list_del(&rg->link);
315                         kfree(rg);
316                 }
317         }
318
319         add += (nrg->from - f);         /* Added to beginning of region */
320         nrg->from = f;
321         add += t - nrg->to;             /* Added to end of region */
322         nrg->to = t;
323
324 out_locked:
325         resv->adds_in_progress--;
326         spin_unlock(&resv->lock);
327         VM_BUG_ON(add < 0);
328         return add;
329 }
330
331 /*
332  * Examine the existing reserve map and determine how many
333  * huge pages in the specified range [f, t) are NOT currently
334  * represented.  This routine is called before a subsequent
335  * call to region_add that will actually modify the reserve
336  * map to add the specified range [f, t).  region_chg does
337  * not change the number of huge pages represented by the
338  * map.  However, if the existing regions in the map can not
339  * be expanded to represent the new range, a new file_region
340  * structure is added to the map as a placeholder.  This is
341  * so that the subsequent region_add call will have all the
342  * regions it needs and will not fail.
343  *
344  * Upon entry, region_chg will also examine the cache of region descriptors
345  * associated with the map.  If there are not enough descriptors cached, one
346  * will be allocated for the in progress add operation.
347  *
348  * Returns the number of huge pages that need to be added to the existing
349  * reservation map for the range [f, t).  This number is greater or equal to
350  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
351  * is needed and can not be allocated.
352  */
353 static long region_chg(struct resv_map *resv, long f, long t)
354 {
355         struct list_head *head = &resv->regions;
356         struct file_region *rg, *nrg = NULL;
357         long chg = 0;
358
359 retry:
360         spin_lock(&resv->lock);
361 retry_locked:
362         resv->adds_in_progress++;
363
364         /*
365          * Check for sufficient descriptors in the cache to accommodate
366          * the number of in progress add operations.
367          */
368         if (resv->adds_in_progress > resv->region_cache_count) {
369                 struct file_region *trg;
370
371                 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
372                 /* Must drop lock to allocate a new descriptor. */
373                 resv->adds_in_progress--;
374                 spin_unlock(&resv->lock);
375
376                 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
377                 if (!trg) {
378                         kfree(nrg);
379                         return -ENOMEM;
380                 }
381
382                 spin_lock(&resv->lock);
383                 list_add(&trg->link, &resv->region_cache);
384                 resv->region_cache_count++;
385                 goto retry_locked;
386         }
387
388         /* Locate the region we are before or in. */
389         list_for_each_entry(rg, head, link)
390                 if (f <= rg->to)
391                         break;
392
393         /* If we are below the current region then a new region is required.
394          * Subtle, allocate a new region at the position but make it zero
395          * size such that we can guarantee to record the reservation. */
396         if (&rg->link == head || t < rg->from) {
397                 if (!nrg) {
398                         resv->adds_in_progress--;
399                         spin_unlock(&resv->lock);
400                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
401                         if (!nrg)
402                                 return -ENOMEM;
403
404                         nrg->from = f;
405                         nrg->to   = f;
406                         INIT_LIST_HEAD(&nrg->link);
407                         goto retry;
408                 }
409
410                 list_add(&nrg->link, rg->link.prev);
411                 chg = t - f;
412                 goto out_nrg;
413         }
414
415         /* Round our left edge to the current segment if it encloses us. */
416         if (f > rg->from)
417                 f = rg->from;
418         chg = t - f;
419
420         /* Check for and consume any regions we now overlap with. */
421         list_for_each_entry(rg, rg->link.prev, link) {
422                 if (&rg->link == head)
423                         break;
424                 if (rg->from > t)
425                         goto out;
426
427                 /* We overlap with this area, if it extends further than
428                  * us then we must extend ourselves.  Account for its
429                  * existing reservation. */
430                 if (rg->to > t) {
431                         chg += rg->to - t;
432                         t = rg->to;
433                 }
434                 chg -= rg->to - rg->from;
435         }
436
437 out:
438         spin_unlock(&resv->lock);
439         /*  We already know we raced and no longer need the new region */
440         kfree(nrg);
441         return chg;
442 out_nrg:
443         spin_unlock(&resv->lock);
444         return chg;
445 }
446
447 /*
448  * Abort the in progress add operation.  The adds_in_progress field
449  * of the resv_map keeps track of the operations in progress between
450  * calls to region_chg and region_add.  Operations are sometimes
451  * aborted after the call to region_chg.  In such cases, region_abort
452  * is called to decrement the adds_in_progress counter.
453  *
454  * NOTE: The range arguments [f, t) are not needed or used in this
455  * routine.  They are kept to make reading the calling code easier as
456  * arguments will match the associated region_chg call.
457  */
458 static void region_abort(struct resv_map *resv, long f, long t)
459 {
460         spin_lock(&resv->lock);
461         VM_BUG_ON(!resv->region_cache_count);
462         resv->adds_in_progress--;
463         spin_unlock(&resv->lock);
464 }
465
466 /*
467  * Delete the specified range [f, t) from the reserve map.  If the
468  * t parameter is LONG_MAX, this indicates that ALL regions after f
469  * should be deleted.  Locate the regions which intersect [f, t)
470  * and either trim, delete or split the existing regions.
471  *
472  * Returns the number of huge pages deleted from the reserve map.
473  * In the normal case, the return value is zero or more.  In the
474  * case where a region must be split, a new region descriptor must
475  * be allocated.  If the allocation fails, -ENOMEM will be returned.
476  * NOTE: If the parameter t == LONG_MAX, then we will never split
477  * a region and possibly return -ENOMEM.  Callers specifying
478  * t == LONG_MAX do not need to check for -ENOMEM error.
479  */
480 static long region_del(struct resv_map *resv, long f, long t)
481 {
482         struct list_head *head = &resv->regions;
483         struct file_region *rg, *trg;
484         struct file_region *nrg = NULL;
485         long del = 0;
486
487 retry:
488         spin_lock(&resv->lock);
489         list_for_each_entry_safe(rg, trg, head, link) {
490                 /*
491                  * Skip regions before the range to be deleted.  file_region
492                  * ranges are normally of the form [from, to).  However, there
493                  * may be a "placeholder" entry in the map which is of the form
494                  * (from, to) with from == to.  Check for placeholder entries
495                  * at the beginning of the range to be deleted.
496                  */
497                 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
498                         continue;
499
500                 if (rg->from >= t)
501                         break;
502
503                 if (f > rg->from && t < rg->to) { /* Must split region */
504                         /*
505                          * Check for an entry in the cache before dropping
506                          * lock and attempting allocation.
507                          */
508                         if (!nrg &&
509                             resv->region_cache_count > resv->adds_in_progress) {
510                                 nrg = list_first_entry(&resv->region_cache,
511                                                         struct file_region,
512                                                         link);
513                                 list_del(&nrg->link);
514                                 resv->region_cache_count--;
515                         }
516
517                         if (!nrg) {
518                                 spin_unlock(&resv->lock);
519                                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
520                                 if (!nrg)
521                                         return -ENOMEM;
522                                 goto retry;
523                         }
524
525                         del += t - f;
526
527                         /* New entry for end of split region */
528                         nrg->from = t;
529                         nrg->to = rg->to;
530                         INIT_LIST_HEAD(&nrg->link);
531
532                         /* Original entry is trimmed */
533                         rg->to = f;
534
535                         list_add(&nrg->link, &rg->link);
536                         nrg = NULL;
537                         break;
538                 }
539
540                 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
541                         del += rg->to - rg->from;
542                         list_del(&rg->link);
543                         kfree(rg);
544                         continue;
545                 }
546
547                 if (f <= rg->from) {    /* Trim beginning of region */
548                         del += t - rg->from;
549                         rg->from = t;
550                 } else {                /* Trim end of region */
551                         del += rg->to - f;
552                         rg->to = f;
553                 }
554         }
555
556         spin_unlock(&resv->lock);
557         kfree(nrg);
558         return del;
559 }
560
561 /*
562  * A rare out of memory error was encountered which prevented removal of
563  * the reserve map region for a page.  The huge page itself was free'ed
564  * and removed from the page cache.  This routine will adjust the subpool
565  * usage count, and the global reserve count if needed.  By incrementing
566  * these counts, the reserve map entry which could not be deleted will
567  * appear as a "reserved" entry instead of simply dangling with incorrect
568  * counts.
569  */
570 void hugetlb_fix_reserve_counts(struct inode *inode)
571 {
572         struct hugepage_subpool *spool = subpool_inode(inode);
573         long rsv_adjust;
574
575         rsv_adjust = hugepage_subpool_get_pages(spool, 1);
576         if (rsv_adjust) {
577                 struct hstate *h = hstate_inode(inode);
578
579                 hugetlb_acct_memory(h, 1);
580         }
581 }
582
583 /*
584  * Count and return the number of huge pages in the reserve map
585  * that intersect with the range [f, t).
586  */
587 static long region_count(struct resv_map *resv, long f, long t)
588 {
589         struct list_head *head = &resv->regions;
590         struct file_region *rg;
591         long chg = 0;
592
593         spin_lock(&resv->lock);
594         /* Locate each segment we overlap with, and count that overlap. */
595         list_for_each_entry(rg, head, link) {
596                 long seg_from;
597                 long seg_to;
598
599                 if (rg->to <= f)
600                         continue;
601                 if (rg->from >= t)
602                         break;
603
604                 seg_from = max(rg->from, f);
605                 seg_to = min(rg->to, t);
606
607                 chg += seg_to - seg_from;
608         }
609         spin_unlock(&resv->lock);
610
611         return chg;
612 }
613
614 /*
615  * Convert the address within this vma to the page offset within
616  * the mapping, in pagecache page units; huge pages here.
617  */
618 static pgoff_t vma_hugecache_offset(struct hstate *h,
619                         struct vm_area_struct *vma, unsigned long address)
620 {
621         return ((address - vma->vm_start) >> huge_page_shift(h)) +
622                         (vma->vm_pgoff >> huge_page_order(h));
623 }
624
625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
626                                      unsigned long address)
627 {
628         return vma_hugecache_offset(hstate_vma(vma), vma, address);
629 }
630 EXPORT_SYMBOL_GPL(linear_hugepage_index);
631
632 /*
633  * Return the size of the pages allocated when backing a VMA. In the majority
634  * cases this will be same size as used by the page table entries.
635  */
636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
637 {
638         struct hstate *hstate;
639
640         if (!is_vm_hugetlb_page(vma))
641                 return PAGE_SIZE;
642
643         hstate = hstate_vma(vma);
644
645         return 1UL << huge_page_shift(hstate);
646 }
647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
648
649 /*
650  * Return the page size being used by the MMU to back a VMA. In the majority
651  * of cases, the page size used by the kernel matches the MMU size. On
652  * architectures where it differs, an architecture-specific version of this
653  * function is required.
654  */
655 #ifndef vma_mmu_pagesize
656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
657 {
658         return vma_kernel_pagesize(vma);
659 }
660 #endif
661
662 /*
663  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
664  * bits of the reservation map pointer, which are always clear due to
665  * alignment.
666  */
667 #define HPAGE_RESV_OWNER    (1UL << 0)
668 #define HPAGE_RESV_UNMAPPED (1UL << 1)
669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
670
671 /*
672  * These helpers are used to track how many pages are reserved for
673  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
674  * is guaranteed to have their future faults succeed.
675  *
676  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
677  * the reserve counters are updated with the hugetlb_lock held. It is safe
678  * to reset the VMA at fork() time as it is not in use yet and there is no
679  * chance of the global counters getting corrupted as a result of the values.
680  *
681  * The private mapping reservation is represented in a subtly different
682  * manner to a shared mapping.  A shared mapping has a region map associated
683  * with the underlying file, this region map represents the backing file
684  * pages which have ever had a reservation assigned which this persists even
685  * after the page is instantiated.  A private mapping has a region map
686  * associated with the original mmap which is attached to all VMAs which
687  * reference it, this region map represents those offsets which have consumed
688  * reservation ie. where pages have been instantiated.
689  */
690 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
691 {
692         return (unsigned long)vma->vm_private_data;
693 }
694
695 static void set_vma_private_data(struct vm_area_struct *vma,
696                                                         unsigned long value)
697 {
698         vma->vm_private_data = (void *)value;
699 }
700
701 struct resv_map *resv_map_alloc(void)
702 {
703         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
704         struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
705
706         if (!resv_map || !rg) {
707                 kfree(resv_map);
708                 kfree(rg);
709                 return NULL;
710         }
711
712         kref_init(&resv_map->refs);
713         spin_lock_init(&resv_map->lock);
714         INIT_LIST_HEAD(&resv_map->regions);
715
716         resv_map->adds_in_progress = 0;
717
718         INIT_LIST_HEAD(&resv_map->region_cache);
719         list_add(&rg->link, &resv_map->region_cache);
720         resv_map->region_cache_count = 1;
721
722         return resv_map;
723 }
724
725 void resv_map_release(struct kref *ref)
726 {
727         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
728         struct list_head *head = &resv_map->region_cache;
729         struct file_region *rg, *trg;
730
731         /* Clear out any active regions before we release the map. */
732         region_del(resv_map, 0, LONG_MAX);
733
734         /* ... and any entries left in the cache */
735         list_for_each_entry_safe(rg, trg, head, link) {
736                 list_del(&rg->link);
737                 kfree(rg);
738         }
739
740         VM_BUG_ON(resv_map->adds_in_progress);
741
742         kfree(resv_map);
743 }
744
745 static inline struct resv_map *inode_resv_map(struct inode *inode)
746 {
747         return inode->i_mapping->private_data;
748 }
749
750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
751 {
752         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
753         if (vma->vm_flags & VM_MAYSHARE) {
754                 struct address_space *mapping = vma->vm_file->f_mapping;
755                 struct inode *inode = mapping->host;
756
757                 return inode_resv_map(inode);
758
759         } else {
760                 return (struct resv_map *)(get_vma_private_data(vma) &
761                                                         ~HPAGE_RESV_MASK);
762         }
763 }
764
765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
766 {
767         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
768         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
769
770         set_vma_private_data(vma, (get_vma_private_data(vma) &
771                                 HPAGE_RESV_MASK) | (unsigned long)map);
772 }
773
774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
775 {
776         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
777         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
778
779         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
780 }
781
782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
783 {
784         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
785
786         return (get_vma_private_data(vma) & flag) != 0;
787 }
788
789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
791 {
792         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
793         if (!(vma->vm_flags & VM_MAYSHARE))
794                 vma->vm_private_data = (void *)0;
795 }
796
797 /* Returns true if the VMA has associated reserve pages */
798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
799 {
800         if (vma->vm_flags & VM_NORESERVE) {
801                 /*
802                  * This address is already reserved by other process(chg == 0),
803                  * so, we should decrement reserved count. Without decrementing,
804                  * reserve count remains after releasing inode, because this
805                  * allocated page will go into page cache and is regarded as
806                  * coming from reserved pool in releasing step.  Currently, we
807                  * don't have any other solution to deal with this situation
808                  * properly, so add work-around here.
809                  */
810                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
811                         return true;
812                 else
813                         return false;
814         }
815
816         /* Shared mappings always use reserves */
817         if (vma->vm_flags & VM_MAYSHARE) {
818                 /*
819                  * We know VM_NORESERVE is not set.  Therefore, there SHOULD
820                  * be a region map for all pages.  The only situation where
821                  * there is no region map is if a hole was punched via
822                  * fallocate.  In this case, there really are no reverves to
823                  * use.  This situation is indicated if chg != 0.
824                  */
825                 if (chg)
826                         return false;
827                 else
828                         return true;
829         }
830
831         /*
832          * Only the process that called mmap() has reserves for
833          * private mappings.
834          */
835         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
836                 /*
837                  * Like the shared case above, a hole punch or truncate
838                  * could have been performed on the private mapping.
839                  * Examine the value of chg to determine if reserves
840                  * actually exist or were previously consumed.
841                  * Very Subtle - The value of chg comes from a previous
842                  * call to vma_needs_reserves().  The reserve map for
843                  * private mappings has different (opposite) semantics
844                  * than that of shared mappings.  vma_needs_reserves()
845                  * has already taken this difference in semantics into
846                  * account.  Therefore, the meaning of chg is the same
847                  * as in the shared case above.  Code could easily be
848                  * combined, but keeping it separate draws attention to
849                  * subtle differences.
850                  */
851                 if (chg)
852                         return false;
853                 else
854                         return true;
855         }
856
857         return false;
858 }
859
860 static void enqueue_huge_page(struct hstate *h, struct page *page)
861 {
862         int nid = page_to_nid(page);
863         list_move(&page->lru, &h->hugepage_freelists[nid]);
864         h->free_huge_pages++;
865         h->free_huge_pages_node[nid]++;
866 }
867
868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
869 {
870         struct page *page;
871
872         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
873                 if (!is_migrate_isolate_page(page))
874                         break;
875         /*
876          * if 'non-isolated free hugepage' not found on the list,
877          * the allocation fails.
878          */
879         if (&h->hugepage_freelists[nid] == &page->lru)
880                 return NULL;
881         list_move(&page->lru, &h->hugepage_activelist);
882         set_page_refcounted(page);
883         h->free_huge_pages--;
884         h->free_huge_pages_node[nid]--;
885         return page;
886 }
887
888 /* Movability of hugepages depends on migration support. */
889 static inline gfp_t htlb_alloc_mask(struct hstate *h)
890 {
891         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
892                 return GFP_HIGHUSER_MOVABLE;
893         else
894                 return GFP_HIGHUSER;
895 }
896
897 static struct page *dequeue_huge_page_vma(struct hstate *h,
898                                 struct vm_area_struct *vma,
899                                 unsigned long address, int avoid_reserve,
900                                 long chg)
901 {
902         struct page *page = NULL;
903         struct mempolicy *mpol;
904         nodemask_t *nodemask;
905         struct zonelist *zonelist;
906         struct zone *zone;
907         struct zoneref *z;
908         unsigned int cpuset_mems_cookie;
909
910         /*
911          * A child process with MAP_PRIVATE mappings created by their parent
912          * have no page reserves. This check ensures that reservations are
913          * not "stolen". The child may still get SIGKILLed
914          */
915         if (!vma_has_reserves(vma, chg) &&
916                         h->free_huge_pages - h->resv_huge_pages == 0)
917                 goto err;
918
919         /* If reserves cannot be used, ensure enough pages are in the pool */
920         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
921                 goto err;
922
923 retry_cpuset:
924         cpuset_mems_cookie = read_mems_allowed_begin();
925         zonelist = huge_zonelist(vma, address,
926                                         htlb_alloc_mask(h), &mpol, &nodemask);
927
928         for_each_zone_zonelist_nodemask(zone, z, zonelist,
929                                                 MAX_NR_ZONES - 1, nodemask) {
930                 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
931                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
932                         if (page) {
933                                 if (avoid_reserve)
934                                         break;
935                                 if (!vma_has_reserves(vma, chg))
936                                         break;
937
938                                 SetPagePrivate(page);
939                                 h->resv_huge_pages--;
940                                 break;
941                         }
942                 }
943         }
944
945         mpol_cond_put(mpol);
946         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
947                 goto retry_cpuset;
948         return page;
949
950 err:
951         return NULL;
952 }
953
954 /*
955  * common helper functions for hstate_next_node_to_{alloc|free}.
956  * We may have allocated or freed a huge page based on a different
957  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
958  * be outside of *nodes_allowed.  Ensure that we use an allowed
959  * node for alloc or free.
960  */
961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
962 {
963         nid = next_node_in(nid, *nodes_allowed);
964         VM_BUG_ON(nid >= MAX_NUMNODES);
965
966         return nid;
967 }
968
969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
970 {
971         if (!node_isset(nid, *nodes_allowed))
972                 nid = next_node_allowed(nid, nodes_allowed);
973         return nid;
974 }
975
976 /*
977  * returns the previously saved node ["this node"] from which to
978  * allocate a persistent huge page for the pool and advance the
979  * next node from which to allocate, handling wrap at end of node
980  * mask.
981  */
982 static int hstate_next_node_to_alloc(struct hstate *h,
983                                         nodemask_t *nodes_allowed)
984 {
985         int nid;
986
987         VM_BUG_ON(!nodes_allowed);
988
989         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
990         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
991
992         return nid;
993 }
994
995 /*
996  * helper for free_pool_huge_page() - return the previously saved
997  * node ["this node"] from which to free a huge page.  Advance the
998  * next node id whether or not we find a free huge page to free so
999  * that the next attempt to free addresses the next node.
1000  */
1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1002 {
1003         int nid;
1004
1005         VM_BUG_ON(!nodes_allowed);
1006
1007         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1008         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1009
1010         return nid;
1011 }
1012
1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
1014         for (nr_nodes = nodes_weight(*mask);                            \
1015                 nr_nodes > 0 &&                                         \
1016                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
1017                 nr_nodes--)
1018
1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
1020         for (nr_nodes = nodes_weight(*mask);                            \
1021                 nr_nodes > 0 &&                                         \
1022                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
1023                 nr_nodes--)
1024
1025 #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \
1026         ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \
1027         defined(CONFIG_CMA))
1028 static void destroy_compound_gigantic_page(struct page *page,
1029                                         unsigned int order)
1030 {
1031         int i;
1032         int nr_pages = 1 << order;
1033         struct page *p = page + 1;
1034
1035         atomic_set(compound_mapcount_ptr(page), 0);
1036         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1037                 clear_compound_head(p);
1038                 set_page_refcounted(p);
1039         }
1040
1041         set_compound_order(page, 0);
1042         __ClearPageHead(page);
1043 }
1044
1045 static void free_gigantic_page(struct page *page, unsigned int order)
1046 {
1047         free_contig_range(page_to_pfn(page), 1 << order);
1048 }
1049
1050 static int __alloc_gigantic_page(unsigned long start_pfn,
1051                                 unsigned long nr_pages)
1052 {
1053         unsigned long end_pfn = start_pfn + nr_pages;
1054         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1055 }
1056
1057 static bool pfn_range_valid_gigantic(struct zone *z,
1058                         unsigned long start_pfn, unsigned long nr_pages)
1059 {
1060         unsigned long i, end_pfn = start_pfn + nr_pages;
1061         struct page *page;
1062
1063         for (i = start_pfn; i < end_pfn; i++) {
1064                 if (!pfn_valid(i))
1065                         return false;
1066
1067                 page = pfn_to_page(i);
1068
1069                 if (page_zone(page) != z)
1070                         return false;
1071
1072                 if (PageReserved(page))
1073                         return false;
1074
1075                 if (page_count(page) > 0)
1076                         return false;
1077
1078                 if (PageHuge(page))
1079                         return false;
1080         }
1081
1082         return true;
1083 }
1084
1085 static bool zone_spans_last_pfn(const struct zone *zone,
1086                         unsigned long start_pfn, unsigned long nr_pages)
1087 {
1088         unsigned long last_pfn = start_pfn + nr_pages - 1;
1089         return zone_spans_pfn(zone, last_pfn);
1090 }
1091
1092 static struct page *alloc_gigantic_page(int nid, unsigned int order)
1093 {
1094         unsigned long nr_pages = 1 << order;
1095         unsigned long ret, pfn, flags;
1096         struct zone *z;
1097
1098         z = NODE_DATA(nid)->node_zones;
1099         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1100                 spin_lock_irqsave(&z->lock, flags);
1101
1102                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
1103                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1104                         if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1105                                 /*
1106                                  * We release the zone lock here because
1107                                  * alloc_contig_range() will also lock the zone
1108                                  * at some point. If there's an allocation
1109                                  * spinning on this lock, it may win the race
1110                                  * and cause alloc_contig_range() to fail...
1111                                  */
1112                                 spin_unlock_irqrestore(&z->lock, flags);
1113                                 ret = __alloc_gigantic_page(pfn, nr_pages);
1114                                 if (!ret)
1115                                         return pfn_to_page(pfn);
1116                                 spin_lock_irqsave(&z->lock, flags);
1117                         }
1118                         pfn += nr_pages;
1119                 }
1120
1121                 spin_unlock_irqrestore(&z->lock, flags);
1122         }
1123
1124         return NULL;
1125 }
1126
1127 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1128 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1129
1130 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1131 {
1132         struct page *page;
1133
1134         page = alloc_gigantic_page(nid, huge_page_order(h));
1135         if (page) {
1136                 prep_compound_gigantic_page(page, huge_page_order(h));
1137                 prep_new_huge_page(h, page, nid);
1138         }
1139
1140         return page;
1141 }
1142
1143 static int alloc_fresh_gigantic_page(struct hstate *h,
1144                                 nodemask_t *nodes_allowed)
1145 {
1146         struct page *page = NULL;
1147         int nr_nodes, node;
1148
1149         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1150                 page = alloc_fresh_gigantic_page_node(h, node);
1151                 if (page)
1152                         return 1;
1153         }
1154
1155         return 0;
1156 }
1157
1158 static inline bool gigantic_page_supported(void) { return true; }
1159 #else
1160 static inline bool gigantic_page_supported(void) { return false; }
1161 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1162 static inline void destroy_compound_gigantic_page(struct page *page,
1163                                                 unsigned int order) { }
1164 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1165                                         nodemask_t *nodes_allowed) { return 0; }
1166 #endif
1167
1168 static void update_and_free_page(struct hstate *h, struct page *page)
1169 {
1170         int i;
1171
1172         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1173                 return;
1174
1175         h->nr_huge_pages--;
1176         h->nr_huge_pages_node[page_to_nid(page)]--;
1177         for (i = 0; i < pages_per_huge_page(h); i++) {
1178                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1179                                 1 << PG_referenced | 1 << PG_dirty |
1180                                 1 << PG_active | 1 << PG_private |
1181                                 1 << PG_writeback);
1182         }
1183         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1184         set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1185         set_page_refcounted(page);
1186         if (hstate_is_gigantic(h)) {
1187                 destroy_compound_gigantic_page(page, huge_page_order(h));
1188                 free_gigantic_page(page, huge_page_order(h));
1189         } else {
1190                 __free_pages(page, huge_page_order(h));
1191         }
1192 }
1193
1194 struct hstate *size_to_hstate(unsigned long size)
1195 {
1196         struct hstate *h;
1197
1198         for_each_hstate(h) {
1199                 if (huge_page_size(h) == size)
1200                         return h;
1201         }
1202         return NULL;
1203 }
1204
1205 /*
1206  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1207  * to hstate->hugepage_activelist.)
1208  *
1209  * This function can be called for tail pages, but never returns true for them.
1210  */
1211 bool page_huge_active(struct page *page)
1212 {
1213         VM_BUG_ON_PAGE(!PageHuge(page), page);
1214         return PageHead(page) && PagePrivate(&page[1]);
1215 }
1216
1217 /* never called for tail page */
1218 static void set_page_huge_active(struct page *page)
1219 {
1220         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1221         SetPagePrivate(&page[1]);
1222 }
1223
1224 static void clear_page_huge_active(struct page *page)
1225 {
1226         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1227         ClearPagePrivate(&page[1]);
1228 }
1229
1230 void free_huge_page(struct page *page)
1231 {
1232         /*
1233          * Can't pass hstate in here because it is called from the
1234          * compound page destructor.
1235          */
1236         struct hstate *h = page_hstate(page);
1237         int nid = page_to_nid(page);
1238         struct hugepage_subpool *spool =
1239                 (struct hugepage_subpool *)page_private(page);
1240         bool restore_reserve;
1241
1242         set_page_private(page, 0);
1243         page->mapping = NULL;
1244         VM_BUG_ON_PAGE(page_count(page), page);
1245         VM_BUG_ON_PAGE(page_mapcount(page), page);
1246         restore_reserve = PagePrivate(page);
1247         ClearPagePrivate(page);
1248
1249         /*
1250          * A return code of zero implies that the subpool will be under its
1251          * minimum size if the reservation is not restored after page is free.
1252          * Therefore, force restore_reserve operation.
1253          */
1254         if (hugepage_subpool_put_pages(spool, 1) == 0)
1255                 restore_reserve = true;
1256
1257         spin_lock(&hugetlb_lock);
1258         clear_page_huge_active(page);
1259         hugetlb_cgroup_uncharge_page(hstate_index(h),
1260                                      pages_per_huge_page(h), page);
1261         if (restore_reserve)
1262                 h->resv_huge_pages++;
1263
1264         if (h->surplus_huge_pages_node[nid]) {
1265                 /* remove the page from active list */
1266                 list_del(&page->lru);
1267                 update_and_free_page(h, page);
1268                 h->surplus_huge_pages--;
1269                 h->surplus_huge_pages_node[nid]--;
1270         } else {
1271                 arch_clear_hugepage_flags(page);
1272                 enqueue_huge_page(h, page);
1273         }
1274         spin_unlock(&hugetlb_lock);
1275 }
1276
1277 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1278 {
1279         INIT_LIST_HEAD(&page->lru);
1280         set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1281         spin_lock(&hugetlb_lock);
1282         set_hugetlb_cgroup(page, NULL);
1283         h->nr_huge_pages++;
1284         h->nr_huge_pages_node[nid]++;
1285         spin_unlock(&hugetlb_lock);
1286         put_page(page); /* free it into the hugepage allocator */
1287 }
1288
1289 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1290 {
1291         int i;
1292         int nr_pages = 1 << order;
1293         struct page *p = page + 1;
1294
1295         /* we rely on prep_new_huge_page to set the destructor */
1296         set_compound_order(page, order);
1297         __ClearPageReserved(page);
1298         __SetPageHead(page);
1299         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1300                 /*
1301                  * For gigantic hugepages allocated through bootmem at
1302                  * boot, it's safer to be consistent with the not-gigantic
1303                  * hugepages and clear the PG_reserved bit from all tail pages
1304                  * too.  Otherwse drivers using get_user_pages() to access tail
1305                  * pages may get the reference counting wrong if they see
1306                  * PG_reserved set on a tail page (despite the head page not
1307                  * having PG_reserved set).  Enforcing this consistency between
1308                  * head and tail pages allows drivers to optimize away a check
1309                  * on the head page when they need know if put_page() is needed
1310                  * after get_user_pages().
1311                  */
1312                 __ClearPageReserved(p);
1313                 set_page_count(p, 0);
1314                 set_compound_head(p, page);
1315         }
1316         atomic_set(compound_mapcount_ptr(page), -1);
1317 }
1318
1319 /*
1320  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1321  * transparent huge pages.  See the PageTransHuge() documentation for more
1322  * details.
1323  */
1324 int PageHuge(struct page *page)
1325 {
1326         if (!PageCompound(page))
1327                 return 0;
1328
1329         page = compound_head(page);
1330         return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1331 }
1332 EXPORT_SYMBOL_GPL(PageHuge);
1333
1334 /*
1335  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1336  * normal or transparent huge pages.
1337  */
1338 int PageHeadHuge(struct page *page_head)
1339 {
1340         if (!PageHead(page_head))
1341                 return 0;
1342
1343         return get_compound_page_dtor(page_head) == free_huge_page;
1344 }
1345
1346 pgoff_t __basepage_index(struct page *page)
1347 {
1348         struct page *page_head = compound_head(page);
1349         pgoff_t index = page_index(page_head);
1350         unsigned long compound_idx;
1351
1352         if (!PageHuge(page_head))
1353                 return page_index(page);
1354
1355         if (compound_order(page_head) >= MAX_ORDER)
1356                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1357         else
1358                 compound_idx = page - page_head;
1359
1360         return (index << compound_order(page_head)) + compound_idx;
1361 }
1362
1363 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1364 {
1365         struct page *page;
1366
1367         page = __alloc_pages_node(nid,
1368                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1369                                                 __GFP_REPEAT|__GFP_NOWARN,
1370                 huge_page_order(h));
1371         if (page) {
1372                 prep_new_huge_page(h, page, nid);
1373         }
1374
1375         return page;
1376 }
1377
1378 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1379 {
1380         struct page *page;
1381         int nr_nodes, node;
1382         int ret = 0;
1383
1384         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1385                 page = alloc_fresh_huge_page_node(h, node);
1386                 if (page) {
1387                         ret = 1;
1388                         break;
1389                 }
1390         }
1391
1392         if (ret)
1393                 count_vm_event(HTLB_BUDDY_PGALLOC);
1394         else
1395                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1396
1397         return ret;
1398 }
1399
1400 /*
1401  * Free huge page from pool from next node to free.
1402  * Attempt to keep persistent huge pages more or less
1403  * balanced over allowed nodes.
1404  * Called with hugetlb_lock locked.
1405  */
1406 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1407                                                          bool acct_surplus)
1408 {
1409         int nr_nodes, node;
1410         int ret = 0;
1411
1412         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1413                 /*
1414                  * If we're returning unused surplus pages, only examine
1415                  * nodes with surplus pages.
1416                  */
1417                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1418                     !list_empty(&h->hugepage_freelists[node])) {
1419                         struct page *page =
1420                                 list_entry(h->hugepage_freelists[node].next,
1421                                           struct page, lru);
1422                         list_del(&page->lru);
1423                         h->free_huge_pages--;
1424                         h->free_huge_pages_node[node]--;
1425                         if (acct_surplus) {
1426                                 h->surplus_huge_pages--;
1427                                 h->surplus_huge_pages_node[node]--;
1428                         }
1429                         update_and_free_page(h, page);
1430                         ret = 1;
1431                         break;
1432                 }
1433         }
1434
1435         return ret;
1436 }
1437
1438 /*
1439  * Dissolve a given free hugepage into free buddy pages. This function does
1440  * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the
1441  * number of free hugepages would be reduced below the number of reserved
1442  * hugepages.
1443  */
1444 static int dissolve_free_huge_page(struct page *page)
1445 {
1446         int rc = 0;
1447
1448         spin_lock(&hugetlb_lock);
1449         if (PageHuge(page) && !page_count(page)) {
1450                 struct page *head = compound_head(page);
1451                 struct hstate *h = page_hstate(head);
1452                 int nid = page_to_nid(head);
1453                 if (h->free_huge_pages - h->resv_huge_pages == 0) {
1454                         rc = -EBUSY;
1455                         goto out;
1456                 }
1457                 list_del(&head->lru);
1458                 h->free_huge_pages--;
1459                 h->free_huge_pages_node[nid]--;
1460                 h->max_huge_pages--;
1461                 update_and_free_page(h, head);
1462         }
1463 out:
1464         spin_unlock(&hugetlb_lock);
1465         return rc;
1466 }
1467
1468 /*
1469  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1470  * make specified memory blocks removable from the system.
1471  * Note that this will dissolve a free gigantic hugepage completely, if any
1472  * part of it lies within the given range.
1473  * Also note that if dissolve_free_huge_page() returns with an error, all
1474  * free hugepages that were dissolved before that error are lost.
1475  */
1476 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1477 {
1478         unsigned long pfn;
1479         struct page *page;
1480         int rc = 0;
1481
1482         if (!hugepages_supported())
1483                 return rc;
1484
1485         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1486                 page = pfn_to_page(pfn);
1487                 if (PageHuge(page) && !page_count(page)) {
1488                         rc = dissolve_free_huge_page(page);
1489                         if (rc)
1490                                 break;
1491                 }
1492         }
1493
1494         return rc;
1495 }
1496
1497 /*
1498  * There are 3 ways this can get called:
1499  * 1. With vma+addr: we use the VMA's memory policy
1500  * 2. With !vma, but nid=NUMA_NO_NODE:  We try to allocate a huge
1501  *    page from any node, and let the buddy allocator itself figure
1502  *    it out.
1503  * 3. With !vma, but nid!=NUMA_NO_NODE.  We allocate a huge page
1504  *    strictly from 'nid'
1505  */
1506 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1507                 struct vm_area_struct *vma, unsigned long addr, int nid)
1508 {
1509         int order = huge_page_order(h);
1510         gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1511         unsigned int cpuset_mems_cookie;
1512
1513         /*
1514          * We need a VMA to get a memory policy.  If we do not
1515          * have one, we use the 'nid' argument.
1516          *
1517          * The mempolicy stuff below has some non-inlined bits
1518          * and calls ->vm_ops.  That makes it hard to optimize at
1519          * compile-time, even when NUMA is off and it does
1520          * nothing.  This helps the compiler optimize it out.
1521          */
1522         if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1523                 /*
1524                  * If a specific node is requested, make sure to
1525                  * get memory from there, but only when a node
1526                  * is explicitly specified.
1527                  */
1528                 if (nid != NUMA_NO_NODE)
1529                         gfp |= __GFP_THISNODE;
1530                 /*
1531                  * Make sure to call something that can handle
1532                  * nid=NUMA_NO_NODE
1533                  */
1534                 return alloc_pages_node(nid, gfp, order);
1535         }
1536
1537         /*
1538          * OK, so we have a VMA.  Fetch the mempolicy and try to
1539          * allocate a huge page with it.  We will only reach this
1540          * when CONFIG_NUMA=y.
1541          */
1542         do {
1543                 struct page *page;
1544                 struct mempolicy *mpol;
1545                 struct zonelist *zl;
1546                 nodemask_t *nodemask;
1547
1548                 cpuset_mems_cookie = read_mems_allowed_begin();
1549                 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
1550                 mpol_cond_put(mpol);
1551                 page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
1552                 if (page)
1553                         return page;
1554         } while (read_mems_allowed_retry(cpuset_mems_cookie));
1555
1556         return NULL;
1557 }
1558
1559 /*
1560  * There are two ways to allocate a huge page:
1561  * 1. When you have a VMA and an address (like a fault)
1562  * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1563  *
1564  * 'vma' and 'addr' are only for (1).  'nid' is always NUMA_NO_NODE in
1565  * this case which signifies that the allocation should be done with
1566  * respect for the VMA's memory policy.
1567  *
1568  * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1569  * implies that memory policies will not be taken in to account.
1570  */
1571 static struct page *__alloc_buddy_huge_page(struct hstate *h,
1572                 struct vm_area_struct *vma, unsigned long addr, int nid)
1573 {
1574         struct page *page;
1575         unsigned int r_nid;
1576
1577         if (hstate_is_gigantic(h))
1578                 return NULL;
1579
1580         /*
1581          * Make sure that anyone specifying 'nid' is not also specifying a VMA.
1582          * This makes sure the caller is picking _one_ of the modes with which
1583          * we can call this function, not both.
1584          */
1585         if (vma || (addr != -1)) {
1586                 VM_WARN_ON_ONCE(addr == -1);
1587                 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1588         }
1589         /*
1590          * Assume we will successfully allocate the surplus page to
1591          * prevent racing processes from causing the surplus to exceed
1592          * overcommit
1593          *
1594          * This however introduces a different race, where a process B
1595          * tries to grow the static hugepage pool while alloc_pages() is
1596          * called by process A. B will only examine the per-node
1597          * counters in determining if surplus huge pages can be
1598          * converted to normal huge pages in adjust_pool_surplus(). A
1599          * won't be able to increment the per-node counter, until the
1600          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1601          * no more huge pages can be converted from surplus to normal
1602          * state (and doesn't try to convert again). Thus, we have a
1603          * case where a surplus huge page exists, the pool is grown, and
1604          * the surplus huge page still exists after, even though it
1605          * should just have been converted to a normal huge page. This
1606          * does not leak memory, though, as the hugepage will be freed
1607          * once it is out of use. It also does not allow the counters to
1608          * go out of whack in adjust_pool_surplus() as we don't modify
1609          * the node values until we've gotten the hugepage and only the
1610          * per-node value is checked there.
1611          */
1612         spin_lock(&hugetlb_lock);
1613         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1614                 spin_unlock(&hugetlb_lock);
1615                 return NULL;
1616         } else {
1617                 h->nr_huge_pages++;
1618                 h->surplus_huge_pages++;
1619         }
1620         spin_unlock(&hugetlb_lock);
1621
1622         page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1623
1624         spin_lock(&hugetlb_lock);
1625         if (page) {
1626                 INIT_LIST_HEAD(&page->lru);
1627                 r_nid = page_to_nid(page);
1628                 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1629                 set_hugetlb_cgroup(page, NULL);
1630                 /*
1631                  * We incremented the global counters already
1632                  */
1633                 h->nr_huge_pages_node[r_nid]++;
1634                 h->surplus_huge_pages_node[r_nid]++;
1635                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1636         } else {
1637                 h->nr_huge_pages--;
1638                 h->surplus_huge_pages--;
1639                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1640         }
1641         spin_unlock(&hugetlb_lock);
1642
1643         return page;
1644 }
1645
1646 /*
1647  * Allocate a huge page from 'nid'.  Note, 'nid' may be
1648  * NUMA_NO_NODE, which means that it may be allocated
1649  * anywhere.
1650  */
1651 static
1652 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1653 {
1654         unsigned long addr = -1;
1655
1656         return __alloc_buddy_huge_page(h, NULL, addr, nid);
1657 }
1658
1659 /*
1660  * Use the VMA's mpolicy to allocate a huge page from the buddy.
1661  */
1662 static
1663 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1664                 struct vm_area_struct *vma, unsigned long addr)
1665 {
1666         return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1667 }
1668
1669 /*
1670  * This allocation function is useful in the context where vma is irrelevant.
1671  * E.g. soft-offlining uses this function because it only cares physical
1672  * address of error page.
1673  */
1674 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1675 {
1676         struct page *page = NULL;
1677
1678         spin_lock(&hugetlb_lock);
1679         if (h->free_huge_pages - h->resv_huge_pages > 0)
1680                 page = dequeue_huge_page_node(h, nid);
1681         spin_unlock(&hugetlb_lock);
1682
1683         if (!page)
1684                 page = __alloc_buddy_huge_page_no_mpol(h, nid);
1685
1686         return page;
1687 }
1688
1689 /*
1690  * Increase the hugetlb pool such that it can accommodate a reservation
1691  * of size 'delta'.
1692  */
1693 static int gather_surplus_pages(struct hstate *h, int delta)
1694 {
1695         struct list_head surplus_list;
1696         struct page *page, *tmp;
1697         int ret, i;
1698         int needed, allocated;
1699         bool alloc_ok = true;
1700
1701         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1702         if (needed <= 0) {
1703                 h->resv_huge_pages += delta;
1704                 return 0;
1705         }
1706
1707         allocated = 0;
1708         INIT_LIST_HEAD(&surplus_list);
1709
1710         ret = -ENOMEM;
1711 retry:
1712         spin_unlock(&hugetlb_lock);
1713         for (i = 0; i < needed; i++) {
1714                 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1715                 if (!page) {
1716                         alloc_ok = false;
1717                         break;
1718                 }
1719                 list_add(&page->lru, &surplus_list);
1720         }
1721         allocated += i;
1722
1723         /*
1724          * After retaking hugetlb_lock, we need to recalculate 'needed'
1725          * because either resv_huge_pages or free_huge_pages may have changed.
1726          */
1727         spin_lock(&hugetlb_lock);
1728         needed = (h->resv_huge_pages + delta) -
1729                         (h->free_huge_pages + allocated);
1730         if (needed > 0) {
1731                 if (alloc_ok)
1732                         goto retry;
1733                 /*
1734                  * We were not able to allocate enough pages to
1735                  * satisfy the entire reservation so we free what
1736                  * we've allocated so far.
1737                  */
1738                 goto free;
1739         }
1740         /*
1741          * The surplus_list now contains _at_least_ the number of extra pages
1742          * needed to accommodate the reservation.  Add the appropriate number
1743          * of pages to the hugetlb pool and free the extras back to the buddy
1744          * allocator.  Commit the entire reservation here to prevent another
1745          * process from stealing the pages as they are added to the pool but
1746          * before they are reserved.
1747          */
1748         needed += allocated;
1749         h->resv_huge_pages += delta;
1750         ret = 0;
1751
1752         /* Free the needed pages to the hugetlb pool */
1753         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1754                 if ((--needed) < 0)
1755                         break;
1756                 /*
1757                  * This page is now managed by the hugetlb allocator and has
1758                  * no users -- drop the buddy allocator's reference.
1759                  */
1760                 put_page_testzero(page);
1761                 VM_BUG_ON_PAGE(page_count(page), page);
1762                 enqueue_huge_page(h, page);
1763         }
1764 free:
1765         spin_unlock(&hugetlb_lock);
1766
1767         /* Free unnecessary surplus pages to the buddy allocator */
1768         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1769                 put_page(page);
1770         spin_lock(&hugetlb_lock);
1771
1772         return ret;
1773 }
1774
1775 /*
1776  * When releasing a hugetlb pool reservation, any surplus pages that were
1777  * allocated to satisfy the reservation must be explicitly freed if they were
1778  * never used.
1779  * Called with hugetlb_lock held.
1780  */
1781 static void return_unused_surplus_pages(struct hstate *h,
1782                                         unsigned long unused_resv_pages)
1783 {
1784         unsigned long nr_pages;
1785
1786         /* Uncommit the reservation */
1787         h->resv_huge_pages -= unused_resv_pages;
1788
1789         /* Cannot return gigantic pages currently */
1790         if (hstate_is_gigantic(h))
1791                 return;
1792
1793         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1794
1795         /*
1796          * We want to release as many surplus pages as possible, spread
1797          * evenly across all nodes with memory. Iterate across these nodes
1798          * until we can no longer free unreserved surplus pages. This occurs
1799          * when the nodes with surplus pages have no free pages.
1800          * free_pool_huge_page() will balance the the freed pages across the
1801          * on-line nodes with memory and will handle the hstate accounting.
1802          */
1803         while (nr_pages--) {
1804                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1805                         break;
1806                 cond_resched_lock(&hugetlb_lock);
1807         }
1808 }
1809
1810
1811 /*
1812  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1813  * are used by the huge page allocation routines to manage reservations.
1814  *
1815  * vma_needs_reservation is called to determine if the huge page at addr
1816  * within the vma has an associated reservation.  If a reservation is
1817  * needed, the value 1 is returned.  The caller is then responsible for
1818  * managing the global reservation and subpool usage counts.  After
1819  * the huge page has been allocated, vma_commit_reservation is called
1820  * to add the page to the reservation map.  If the page allocation fails,
1821  * the reservation must be ended instead of committed.  vma_end_reservation
1822  * is called in such cases.
1823  *
1824  * In the normal case, vma_commit_reservation returns the same value
1825  * as the preceding vma_needs_reservation call.  The only time this
1826  * is not the case is if a reserve map was changed between calls.  It
1827  * is the responsibility of the caller to notice the difference and
1828  * take appropriate action.
1829  *
1830  * vma_add_reservation is used in error paths where a reservation must
1831  * be restored when a newly allocated huge page must be freed.  It is
1832  * to be called after calling vma_needs_reservation to determine if a
1833  * reservation exists.
1834  */
1835 enum vma_resv_mode {
1836         VMA_NEEDS_RESV,
1837         VMA_COMMIT_RESV,
1838         VMA_END_RESV,
1839         VMA_ADD_RESV,
1840 };
1841 static long __vma_reservation_common(struct hstate *h,
1842                                 struct vm_area_struct *vma, unsigned long addr,
1843                                 enum vma_resv_mode mode)
1844 {
1845         struct resv_map *resv;
1846         pgoff_t idx;
1847         long ret;
1848
1849         resv = vma_resv_map(vma);
1850         if (!resv)
1851                 return 1;
1852
1853         idx = vma_hugecache_offset(h, vma, addr);
1854         switch (mode) {
1855         case VMA_NEEDS_RESV:
1856                 ret = region_chg(resv, idx, idx + 1);
1857                 break;
1858         case VMA_COMMIT_RESV:
1859                 ret = region_add(resv, idx, idx + 1);
1860                 break;
1861         case VMA_END_RESV:
1862                 region_abort(resv, idx, idx + 1);
1863                 ret = 0;
1864                 break;
1865         case VMA_ADD_RESV:
1866                 if (vma->vm_flags & VM_MAYSHARE)
1867                         ret = region_add(resv, idx, idx + 1);
1868                 else {
1869                         region_abort(resv, idx, idx + 1);
1870                         ret = region_del(resv, idx, idx + 1);
1871                 }
1872                 break;
1873         default:
1874                 BUG();
1875         }
1876
1877         if (vma->vm_flags & VM_MAYSHARE)
1878                 return ret;
1879         else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1880                 /*
1881                  * In most cases, reserves always exist for private mappings.
1882                  * However, a file associated with mapping could have been
1883                  * hole punched or truncated after reserves were consumed.
1884                  * As subsequent fault on such a range will not use reserves.
1885                  * Subtle - The reserve map for private mappings has the
1886                  * opposite meaning than that of shared mappings.  If NO
1887                  * entry is in the reserve map, it means a reservation exists.
1888                  * If an entry exists in the reserve map, it means the
1889                  * reservation has already been consumed.  As a result, the
1890                  * return value of this routine is the opposite of the
1891                  * value returned from reserve map manipulation routines above.
1892                  */
1893                 if (ret)
1894                         return 0;
1895                 else
1896                         return 1;
1897         }
1898         else
1899                 return ret < 0 ? ret : 0;
1900 }
1901
1902 static long vma_needs_reservation(struct hstate *h,
1903                         struct vm_area_struct *vma, unsigned long addr)
1904 {
1905         return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1906 }
1907
1908 static long vma_commit_reservation(struct hstate *h,
1909                         struct vm_area_struct *vma, unsigned long addr)
1910 {
1911         return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1912 }
1913
1914 static void vma_end_reservation(struct hstate *h,
1915                         struct vm_area_struct *vma, unsigned long addr)
1916 {
1917         (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1918 }
1919
1920 static long vma_add_reservation(struct hstate *h,
1921                         struct vm_area_struct *vma, unsigned long addr)
1922 {
1923         return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
1924 }
1925
1926 /*
1927  * This routine is called to restore a reservation on error paths.  In the
1928  * specific error paths, a huge page was allocated (via alloc_huge_page)
1929  * and is about to be freed.  If a reservation for the page existed,
1930  * alloc_huge_page would have consumed the reservation and set PagePrivate
1931  * in the newly allocated page.  When the page is freed via free_huge_page,
1932  * the global reservation count will be incremented if PagePrivate is set.
1933  * However, free_huge_page can not adjust the reserve map.  Adjust the
1934  * reserve map here to be consistent with global reserve count adjustments
1935  * to be made by free_huge_page.
1936  */
1937 static void restore_reserve_on_error(struct hstate *h,
1938                         struct vm_area_struct *vma, unsigned long address,
1939                         struct page *page)
1940 {
1941         if (unlikely(PagePrivate(page))) {
1942                 long rc = vma_needs_reservation(h, vma, address);
1943
1944                 if (unlikely(rc < 0)) {
1945                         /*
1946                          * Rare out of memory condition in reserve map
1947                          * manipulation.  Clear PagePrivate so that
1948                          * global reserve count will not be incremented
1949                          * by free_huge_page.  This will make it appear
1950                          * as though the reservation for this page was
1951                          * consumed.  This may prevent the task from
1952                          * faulting in the page at a later time.  This
1953                          * is better than inconsistent global huge page
1954                          * accounting of reserve counts.
1955                          */
1956                         ClearPagePrivate(page);
1957                 } else if (rc) {
1958                         rc = vma_add_reservation(h, vma, address);
1959                         if (unlikely(rc < 0))
1960                                 /*
1961                                  * See above comment about rare out of
1962                                  * memory condition.
1963                                  */
1964                                 ClearPagePrivate(page);
1965                 } else
1966                         vma_end_reservation(h, vma, address);
1967         }
1968 }
1969
1970 struct page *alloc_huge_page(struct vm_area_struct *vma,
1971                                     unsigned long addr, int avoid_reserve)
1972 {
1973         struct hugepage_subpool *spool = subpool_vma(vma);
1974         struct hstate *h = hstate_vma(vma);
1975         struct page *page;
1976         long map_chg, map_commit;
1977         long gbl_chg;
1978         int ret, idx;
1979         struct hugetlb_cgroup *h_cg;
1980
1981         idx = hstate_index(h);
1982         /*
1983          * Examine the region/reserve map to determine if the process
1984          * has a reservation for the page to be allocated.  A return
1985          * code of zero indicates a reservation exists (no change).
1986          */
1987         map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
1988         if (map_chg < 0)
1989                 return ERR_PTR(-ENOMEM);
1990
1991         /*
1992          * Processes that did not create the mapping will have no
1993          * reserves as indicated by the region/reserve map. Check
1994          * that the allocation will not exceed the subpool limit.
1995          * Allocations for MAP_NORESERVE mappings also need to be
1996          * checked against any subpool limit.
1997          */
1998         if (map_chg || avoid_reserve) {
1999                 gbl_chg = hugepage_subpool_get_pages(spool, 1);
2000                 if (gbl_chg < 0) {
2001                         vma_end_reservation(h, vma, addr);
2002                         return ERR_PTR(-ENOSPC);
2003                 }
2004
2005                 /*
2006                  * Even though there was no reservation in the region/reserve
2007                  * map, there could be reservations associated with the
2008                  * subpool that can be used.  This would be indicated if the
2009                  * return value of hugepage_subpool_get_pages() is zero.
2010                  * However, if avoid_reserve is specified we still avoid even
2011                  * the subpool reservations.
2012                  */
2013                 if (avoid_reserve)
2014                         gbl_chg = 1;
2015         }
2016
2017         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2018         if (ret)
2019                 goto out_subpool_put;
2020
2021         spin_lock(&hugetlb_lock);
2022         /*
2023          * glb_chg is passed to indicate whether or not a page must be taken
2024          * from the global free pool (global change).  gbl_chg == 0 indicates
2025          * a reservation exists for the allocation.
2026          */
2027         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2028         if (!page) {
2029                 spin_unlock(&hugetlb_lock);
2030                 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
2031                 if (!page)
2032                         goto out_uncharge_cgroup;
2033                 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2034                         SetPagePrivate(page);
2035                         h->resv_huge_pages--;
2036                 }
2037                 spin_lock(&hugetlb_lock);
2038                 list_move(&page->lru, &h->hugepage_activelist);
2039                 /* Fall through */
2040         }
2041         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2042         spin_unlock(&hugetlb_lock);
2043
2044         set_page_private(page, (unsigned long)spool);
2045
2046         map_commit = vma_commit_reservation(h, vma, addr);
2047         if (unlikely(map_chg > map_commit)) {
2048                 /*
2049                  * The page was added to the reservation map between
2050                  * vma_needs_reservation and vma_commit_reservation.
2051                  * This indicates a race with hugetlb_reserve_pages.
2052                  * Adjust for the subpool count incremented above AND
2053                  * in hugetlb_reserve_pages for the same page.  Also,
2054                  * the reservation count added in hugetlb_reserve_pages
2055                  * no longer applies.
2056                  */
2057                 long rsv_adjust;
2058
2059                 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2060                 hugetlb_acct_memory(h, -rsv_adjust);
2061         }
2062         return page;
2063
2064 out_uncharge_cgroup:
2065         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2066 out_subpool_put:
2067         if (map_chg || avoid_reserve)
2068                 hugepage_subpool_put_pages(spool, 1);
2069         vma_end_reservation(h, vma, addr);
2070         return ERR_PTR(-ENOSPC);
2071 }
2072
2073 /*
2074  * alloc_huge_page()'s wrapper which simply returns the page if allocation
2075  * succeeds, otherwise NULL. This function is called from new_vma_page(),
2076  * where no ERR_VALUE is expected to be returned.
2077  */
2078 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
2079                                 unsigned long addr, int avoid_reserve)
2080 {
2081         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
2082         if (IS_ERR(page))
2083                 page = NULL;
2084         return page;
2085 }
2086
2087 int __weak alloc_bootmem_huge_page(struct hstate *h)
2088 {
2089         struct huge_bootmem_page *m;
2090         int nr_nodes, node;
2091
2092         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2093                 void *addr;
2094
2095                 addr = memblock_virt_alloc_try_nid_nopanic(
2096                                 huge_page_size(h), huge_page_size(h),
2097                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2098                 if (addr) {
2099                         /*
2100                          * Use the beginning of the huge page to store the
2101                          * huge_bootmem_page struct (until gather_bootmem
2102                          * puts them into the mem_map).
2103                          */
2104                         m = addr;
2105                         goto found;
2106                 }
2107         }
2108         return 0;
2109
2110 found:
2111         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2112         /* Put them into a private list first because mem_map is not up yet */
2113         list_add(&m->list, &huge_boot_pages);
2114         m->hstate = h;
2115         return 1;
2116 }
2117
2118 static void __init prep_compound_huge_page(struct page *page,
2119                 unsigned int order)
2120 {
2121         if (unlikely(order > (MAX_ORDER - 1)))
2122                 prep_compound_gigantic_page(page, order);
2123         else
2124                 prep_compound_page(page, order);
2125 }
2126
2127 /* Put bootmem huge pages into the standard lists after mem_map is up */
2128 static void __init gather_bootmem_prealloc(void)
2129 {
2130         struct huge_bootmem_page *m;
2131
2132         list_for_each_entry(m, &huge_boot_pages, list) {
2133                 struct hstate *h = m->hstate;
2134                 struct page *page;
2135
2136 #ifdef CONFIG_HIGHMEM
2137                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
2138                 memblock_free_late(__pa(m),
2139                                    sizeof(struct huge_bootmem_page));
2140 #else
2141                 page = virt_to_page(m);
2142 #endif
2143                 WARN_ON(page_count(page) != 1);
2144                 prep_compound_huge_page(page, h->order);
2145                 WARN_ON(PageReserved(page));
2146                 prep_new_huge_page(h, page, page_to_nid(page));
2147                 /*
2148                  * If we had gigantic hugepages allocated at boot time, we need
2149                  * to restore the 'stolen' pages to totalram_pages in order to
2150                  * fix confusing memory reports from free(1) and another
2151                  * side-effects, like CommitLimit going negative.
2152                  */
2153                 if (hstate_is_gigantic(h))
2154                         adjust_managed_page_count(page, 1 << h->order);
2155         }
2156 }
2157
2158 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2159 {
2160         unsigned long i;
2161
2162         for (i = 0; i < h->max_huge_pages; ++i) {
2163                 if (hstate_is_gigantic(h)) {
2164                         if (!alloc_bootmem_huge_page(h))
2165                                 break;
2166                 } else if (!alloc_fresh_huge_page(h,
2167                                          &node_states[N_MEMORY]))
2168                         break;
2169         }
2170         h->max_huge_pages = i;
2171 }
2172
2173 static void __init hugetlb_init_hstates(void)
2174 {
2175         struct hstate *h;
2176
2177         for_each_hstate(h) {
2178                 if (minimum_order > huge_page_order(h))
2179                         minimum_order = huge_page_order(h);
2180
2181                 /* oversize hugepages were init'ed in early boot */
2182                 if (!hstate_is_gigantic(h))
2183                         hugetlb_hstate_alloc_pages(h);
2184         }
2185         VM_BUG_ON(minimum_order == UINT_MAX);
2186 }
2187
2188 static char * __init memfmt(char *buf, unsigned long n)
2189 {
2190         if (n >= (1UL << 30))
2191                 sprintf(buf, "%lu GB", n >> 30);
2192         else if (n >= (1UL << 20))
2193                 sprintf(buf, "%lu MB", n >> 20);
2194         else
2195                 sprintf(buf, "%lu KB", n >> 10);
2196         return buf;
2197 }
2198
2199 static void __init report_hugepages(void)
2200 {
2201         struct hstate *h;
2202
2203         for_each_hstate(h) {
2204                 char buf[32];
2205                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2206                         memfmt(buf, huge_page_size(h)),
2207                         h->free_huge_pages);
2208         }
2209 }
2210
2211 #ifdef CONFIG_HIGHMEM
2212 static void try_to_free_low(struct hstate *h, unsigned long count,
2213                                                 nodemask_t *nodes_allowed)
2214 {
2215         int i;
2216
2217         if (hstate_is_gigantic(h))
2218                 return;
2219
2220         for_each_node_mask(i, *nodes_allowed) {
2221                 struct page *page, *next;
2222                 struct list_head *freel = &h->hugepage_freelists[i];
2223                 list_for_each_entry_safe(page, next, freel, lru) {
2224                         if (count >= h->nr_huge_pages)
2225                                 return;
2226                         if (PageHighMem(page))
2227                                 continue;
2228                         list_del(&page->lru);
2229                         update_and_free_page(h, page);
2230                         h->free_huge_pages--;
2231                         h->free_huge_pages_node[page_to_nid(page)]--;
2232                 }
2233         }
2234 }
2235 #else
2236 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2237                                                 nodemask_t *nodes_allowed)
2238 {
2239 }
2240 #endif
2241
2242 /*
2243  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2244  * balanced by operating on them in a round-robin fashion.
2245  * Returns 1 if an adjustment was made.
2246  */
2247 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2248                                 int delta)
2249 {
2250         int nr_nodes, node;
2251
2252         VM_BUG_ON(delta != -1 && delta != 1);
2253
2254         if (delta < 0) {
2255                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2256                         if (h->surplus_huge_pages_node[node])
2257                                 goto found;
2258                 }
2259         } else {
2260                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2261                         if (h->surplus_huge_pages_node[node] <
2262                                         h->nr_huge_pages_node[node])
2263                                 goto found;
2264                 }
2265         }
2266         return 0;
2267
2268 found:
2269         h->surplus_huge_pages += delta;
2270         h->surplus_huge_pages_node[node] += delta;
2271         return 1;
2272 }
2273
2274 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2275 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2276                                                 nodemask_t *nodes_allowed)
2277 {
2278         unsigned long min_count, ret;
2279
2280         if (hstate_is_gigantic(h) && !gigantic_page_supported())
2281                 return h->max_huge_pages;
2282
2283         /*
2284          * Increase the pool size
2285          * First take pages out of surplus state.  Then make up the
2286          * remaining difference by allocating fresh huge pages.
2287          *
2288          * We might race with __alloc_buddy_huge_page() here and be unable
2289          * to convert a surplus huge page to a normal huge page. That is
2290          * not critical, though, it just means the overall size of the
2291          * pool might be one hugepage larger than it needs to be, but
2292          * within all the constraints specified by the sysctls.
2293          */
2294         spin_lock(&hugetlb_lock);
2295         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2296                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2297                         break;
2298         }
2299
2300         while (count > persistent_huge_pages(h)) {
2301                 /*
2302                  * If this allocation races such that we no longer need the
2303                  * page, free_huge_page will handle it by freeing the page
2304                  * and reducing the surplus.
2305                  */
2306                 spin_unlock(&hugetlb_lock);
2307
2308                 /* yield cpu to avoid soft lockup */
2309                 cond_resched();
2310
2311                 if (hstate_is_gigantic(h))
2312                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2313                 else
2314                         ret = alloc_fresh_huge_page(h, nodes_allowed);
2315                 spin_lock(&hugetlb_lock);
2316                 if (!ret)
2317                         goto out;
2318
2319                 /* Bail for signals. Probably ctrl-c from user */
2320                 if (signal_pending(current))
2321                         goto out;
2322         }
2323
2324         /*
2325          * Decrease the pool size
2326          * First return free pages to the buddy allocator (being careful
2327          * to keep enough around to satisfy reservations).  Then place
2328          * pages into surplus state as needed so the pool will shrink
2329          * to the desired size as pages become free.
2330          *
2331          * By placing pages into the surplus state independent of the
2332          * overcommit value, we are allowing the surplus pool size to
2333          * exceed overcommit. There are few sane options here. Since
2334          * __alloc_buddy_huge_page() is checking the global counter,
2335          * though, we'll note that we're not allowed to exceed surplus
2336          * and won't grow the pool anywhere else. Not until one of the
2337          * sysctls are changed, or the surplus pages go out of use.
2338          */
2339         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2340         min_count = max(count, min_count);
2341         try_to_free_low(h, min_count, nodes_allowed);
2342         while (min_count < persistent_huge_pages(h)) {
2343                 if (!free_pool_huge_page(h, nodes_allowed, 0))
2344                         break;
2345                 cond_resched_lock(&hugetlb_lock);
2346         }
2347         while (count < persistent_huge_pages(h)) {
2348                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2349                         break;
2350         }
2351 out:
2352         ret = persistent_huge_pages(h);
2353         spin_unlock(&hugetlb_lock);
2354         return ret;
2355 }
2356
2357 #define HSTATE_ATTR_RO(_name) \
2358         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2359
2360 #define HSTATE_ATTR(_name) \
2361         static struct kobj_attribute _name##_attr = \
2362                 __ATTR(_name, 0644, _name##_show, _name##_store)
2363
2364 static struct kobject *hugepages_kobj;
2365 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2366
2367 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2368
2369 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2370 {
2371         int i;
2372
2373         for (i = 0; i < HUGE_MAX_HSTATE; i++)
2374                 if (hstate_kobjs[i] == kobj) {
2375                         if (nidp)
2376                                 *nidp = NUMA_NO_NODE;
2377                         return &hstates[i];
2378                 }
2379
2380         return kobj_to_node_hstate(kobj, nidp);
2381 }
2382
2383 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2384                                         struct kobj_attribute *attr, char *buf)
2385 {
2386         struct hstate *h;
2387         unsigned long nr_huge_pages;
2388         int nid;
2389
2390         h = kobj_to_hstate(kobj, &nid);
2391         if (nid == NUMA_NO_NODE)
2392                 nr_huge_pages = h->nr_huge_pages;
2393         else
2394                 nr_huge_pages = h->nr_huge_pages_node[nid];
2395
2396         return sprintf(buf, "%lu\n", nr_huge_pages);
2397 }
2398
2399 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2400                                            struct hstate *h, int nid,
2401                                            unsigned long count, size_t len)
2402 {
2403         int err;
2404         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2405
2406         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2407                 err = -EINVAL;
2408                 goto out;
2409         }
2410
2411         if (nid == NUMA_NO_NODE) {
2412                 /*
2413                  * global hstate attribute
2414                  */
2415                 if (!(obey_mempolicy &&
2416                                 init_nodemask_of_mempolicy(nodes_allowed))) {
2417                         NODEMASK_FREE(nodes_allowed);
2418                         nodes_allowed = &node_states[N_MEMORY];
2419                 }
2420         } else if (nodes_allowed) {
2421                 /*
2422                  * per node hstate attribute: adjust count to global,
2423                  * but restrict alloc/free to the specified node.
2424                  */
2425                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2426                 init_nodemask_of_node(nodes_allowed, nid);
2427         } else
2428                 nodes_allowed = &node_states[N_MEMORY];
2429
2430         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2431
2432         if (nodes_allowed != &node_states[N_MEMORY])
2433                 NODEMASK_FREE(nodes_allowed);
2434
2435         return len;
2436 out:
2437         NODEMASK_FREE(nodes_allowed);
2438         return err;
2439 }
2440
2441 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2442                                          struct kobject *kobj, const char *buf,
2443                                          size_t len)
2444 {
2445         struct hstate *h;
2446         unsigned long count;
2447         int nid;
2448         int err;
2449
2450         err = kstrtoul(buf, 10, &count);
2451         if (err)
2452                 return err;
2453
2454         h = kobj_to_hstate(kobj, &nid);
2455         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2456 }
2457
2458 static ssize_t nr_hugepages_show(struct kobject *kobj,
2459                                        struct kobj_attribute *attr, char *buf)
2460 {
2461         return nr_hugepages_show_common(kobj, attr, buf);
2462 }
2463
2464 static ssize_t nr_hugepages_store(struct kobject *kobj,
2465                struct kobj_attribute *attr, const char *buf, size_t len)
2466 {
2467         return nr_hugepages_store_common(false, kobj, buf, len);
2468 }
2469 HSTATE_ATTR(nr_hugepages);
2470
2471 #ifdef CONFIG_NUMA
2472
2473 /*
2474  * hstate attribute for optionally mempolicy-based constraint on persistent
2475  * huge page alloc/free.
2476  */
2477 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2478                                        struct kobj_attribute *attr, char *buf)
2479 {
2480         return nr_hugepages_show_common(kobj, attr, buf);
2481 }
2482
2483 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2484                struct kobj_attribute *attr, const char *buf, size_t len)
2485 {
2486         return nr_hugepages_store_common(true, kobj, buf, len);
2487 }
2488 HSTATE_ATTR(nr_hugepages_mempolicy);
2489 #endif
2490
2491
2492 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2493                                         struct kobj_attribute *attr, char *buf)
2494 {
2495         struct hstate *h = kobj_to_hstate(kobj, NULL);
2496         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2497 }
2498
2499 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2500                 struct kobj_attribute *attr, const char *buf, size_t count)
2501 {
2502         int err;
2503         unsigned long input;
2504         struct hstate *h = kobj_to_hstate(kobj, NULL);
2505
2506         if (hstate_is_gigantic(h))
2507                 return -EINVAL;
2508
2509         err = kstrtoul(buf, 10, &input);
2510         if (err)
2511                 return err;
2512
2513         spin_lock(&hugetlb_lock);
2514         h->nr_overcommit_huge_pages = input;
2515         spin_unlock(&hugetlb_lock);
2516
2517         return count;
2518 }
2519 HSTATE_ATTR(nr_overcommit_hugepages);
2520
2521 static ssize_t free_hugepages_show(struct kobject *kobj,
2522                                         struct kobj_attribute *attr, char *buf)
2523 {
2524         struct hstate *h;
2525         unsigned long free_huge_pages;
2526         int nid;
2527
2528         h = kobj_to_hstate(kobj, &nid);
2529         if (nid == NUMA_NO_NODE)
2530                 free_huge_pages = h->free_huge_pages;
2531         else
2532                 free_huge_pages = h->free_huge_pages_node[nid];
2533
2534         return sprintf(buf, "%lu\n", free_huge_pages);
2535 }
2536 HSTATE_ATTR_RO(free_hugepages);
2537
2538 static ssize_t resv_hugepages_show(struct kobject *kobj,
2539                                         struct kobj_attribute *attr, char *buf)
2540 {
2541         struct hstate *h = kobj_to_hstate(kobj, NULL);
2542         return sprintf(buf, "%lu\n", h->resv_huge_pages);
2543 }
2544 HSTATE_ATTR_RO(resv_hugepages);
2545
2546 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2547                                         struct kobj_attribute *attr, char *buf)
2548 {
2549         struct hstate *h;
2550         unsigned long surplus_huge_pages;
2551         int nid;
2552
2553         h = kobj_to_hstate(kobj, &nid);
2554         if (nid == NUMA_NO_NODE)
2555                 surplus_huge_pages = h->surplus_huge_pages;
2556         else
2557                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2558
2559         return sprintf(buf, "%lu\n", surplus_huge_pages);
2560 }
2561 HSTATE_ATTR_RO(surplus_hugepages);
2562
2563 static struct attribute *hstate_attrs[] = {
2564         &nr_hugepages_attr.attr,
2565         &nr_overcommit_hugepages_attr.attr,
2566         &free_hugepages_attr.attr,
2567         &resv_hugepages_attr.attr,
2568         &surplus_hugepages_attr.attr,
2569 #ifdef CONFIG_NUMA
2570         &nr_hugepages_mempolicy_attr.attr,
2571 #endif
2572         NULL,
2573 };
2574
2575 static struct attribute_group hstate_attr_group = {
2576         .attrs = hstate_attrs,
2577 };
2578
2579 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2580                                     struct kobject **hstate_kobjs,
2581                                     struct attribute_group *hstate_attr_group)
2582 {
2583         int retval;
2584         int hi = hstate_index(h);
2585
2586         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2587         if (!hstate_kobjs[hi])
2588                 return -ENOMEM;
2589
2590         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2591         if (retval)
2592                 kobject_put(hstate_kobjs[hi]);
2593
2594         return retval;
2595 }
2596
2597 static void __init hugetlb_sysfs_init(void)
2598 {
2599         struct hstate *h;
2600         int err;
2601
2602         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2603         if (!hugepages_kobj)
2604                 return;
2605
2606         for_each_hstate(h) {
2607                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2608                                          hstate_kobjs, &hstate_attr_group);
2609                 if (err)
2610                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
2611         }
2612 }
2613
2614 #ifdef CONFIG_NUMA
2615
2616 /*
2617  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2618  * with node devices in node_devices[] using a parallel array.  The array
2619  * index of a node device or _hstate == node id.
2620  * This is here to avoid any static dependency of the node device driver, in
2621  * the base kernel, on the hugetlb module.
2622  */
2623 struct node_hstate {
2624         struct kobject          *hugepages_kobj;
2625         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
2626 };
2627 static struct node_hstate node_hstates[MAX_NUMNODES];
2628
2629 /*
2630  * A subset of global hstate attributes for node devices
2631  */
2632 static struct attribute *per_node_hstate_attrs[] = {
2633         &nr_hugepages_attr.attr,
2634         &free_hugepages_attr.attr,
2635         &surplus_hugepages_attr.attr,
2636         NULL,
2637 };
2638
2639 static struct attribute_group per_node_hstate_attr_group = {
2640         .attrs = per_node_hstate_attrs,
2641 };
2642
2643 /*
2644  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2645  * Returns node id via non-NULL nidp.
2646  */
2647 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2648 {
2649         int nid;
2650
2651         for (nid = 0; nid < nr_node_ids; nid++) {
2652                 struct node_hstate *nhs = &node_hstates[nid];
2653                 int i;
2654                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2655                         if (nhs->hstate_kobjs[i] == kobj) {
2656                                 if (nidp)
2657                                         *nidp = nid;
2658                                 return &hstates[i];
2659                         }
2660         }
2661
2662         BUG();
2663         return NULL;
2664 }
2665
2666 /*
2667  * Unregister hstate attributes from a single node device.
2668  * No-op if no hstate attributes attached.
2669  */
2670 static void hugetlb_unregister_node(struct node *node)
2671 {
2672         struct hstate *h;
2673         struct node_hstate *nhs = &node_hstates[node->dev.id];
2674
2675         if (!nhs->hugepages_kobj)
2676                 return;         /* no hstate attributes */
2677
2678         for_each_hstate(h) {
2679                 int idx = hstate_index(h);
2680                 if (nhs->hstate_kobjs[idx]) {
2681                         kobject_put(nhs->hstate_kobjs[idx]);
2682                         nhs->hstate_kobjs[idx] = NULL;
2683                 }
2684         }
2685
2686         kobject_put(nhs->hugepages_kobj);
2687         nhs->hugepages_kobj = NULL;
2688 }
2689
2690
2691 /*
2692  * Register hstate attributes for a single node device.
2693  * No-op if attributes already registered.
2694  */
2695 static void hugetlb_register_node(struct node *node)
2696 {
2697         struct hstate *h;
2698         struct node_hstate *nhs = &node_hstates[node->dev.id];
2699         int err;
2700
2701         if (nhs->hugepages_kobj)
2702                 return;         /* already allocated */
2703
2704         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2705                                                         &node->dev.kobj);
2706         if (!nhs->hugepages_kobj)
2707                 return;
2708
2709         for_each_hstate(h) {
2710                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2711                                                 nhs->hstate_kobjs,
2712                                                 &per_node_hstate_attr_group);
2713                 if (err) {
2714                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2715                                 h->name, node->dev.id);
2716                         hugetlb_unregister_node(node);
2717                         break;
2718                 }
2719         }
2720 }
2721
2722 /*
2723  * hugetlb init time:  register hstate attributes for all registered node
2724  * devices of nodes that have memory.  All on-line nodes should have
2725  * registered their associated device by this time.
2726  */
2727 static void __init hugetlb_register_all_nodes(void)
2728 {
2729         int nid;
2730
2731         for_each_node_state(nid, N_MEMORY) {
2732                 struct node *node = node_devices[nid];
2733                 if (node->dev.id == nid)
2734                         hugetlb_register_node(node);
2735         }
2736
2737         /*
2738          * Let the node device driver know we're here so it can
2739          * [un]register hstate attributes on node hotplug.
2740          */
2741         register_hugetlbfs_with_node(hugetlb_register_node,
2742                                      hugetlb_unregister_node);
2743 }
2744 #else   /* !CONFIG_NUMA */
2745
2746 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2747 {
2748         BUG();
2749         if (nidp)
2750                 *nidp = -1;
2751         return NULL;
2752 }
2753
2754 static void hugetlb_register_all_nodes(void) { }
2755
2756 #endif
2757
2758 static int __init hugetlb_init(void)
2759 {
2760         int i;
2761
2762         if (!hugepages_supported())
2763                 return 0;
2764
2765         if (!size_to_hstate(default_hstate_size)) {
2766                 default_hstate_size = HPAGE_SIZE;
2767                 if (!size_to_hstate(default_hstate_size))
2768                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2769         }
2770         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2771         if (default_hstate_max_huge_pages) {
2772                 if (!default_hstate.max_huge_pages)
2773                         default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2774         }
2775
2776         hugetlb_init_hstates();
2777         gather_bootmem_prealloc();
2778         report_hugepages();
2779
2780         hugetlb_sysfs_init();
2781         hugetlb_register_all_nodes();
2782         hugetlb_cgroup_file_init();
2783
2784 #ifdef CONFIG_SMP
2785         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2786 #else
2787         num_fault_mutexes = 1;
2788 #endif
2789         hugetlb_fault_mutex_table =
2790                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2791         BUG_ON(!hugetlb_fault_mutex_table);
2792
2793         for (i = 0; i < num_fault_mutexes; i++)
2794                 mutex_init(&hugetlb_fault_mutex_table[i]);
2795         return 0;
2796 }
2797 subsys_initcall(hugetlb_init);
2798
2799 /* Should be called on processing a hugepagesz=... option */
2800 void __init hugetlb_bad_size(void)
2801 {
2802         parsed_valid_hugepagesz = false;
2803 }
2804
2805 void __init hugetlb_add_hstate(unsigned int order)
2806 {
2807         struct hstate *h;
2808         unsigned long i;
2809
2810         if (size_to_hstate(PAGE_SIZE << order)) {
2811                 pr_warn("hugepagesz= specified twice, ignoring\n");
2812                 return;
2813         }
2814         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2815         BUG_ON(order == 0);
2816         h = &hstates[hugetlb_max_hstate++];
2817         h->order = order;
2818         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2819         h->nr_huge_pages = 0;
2820         h->free_huge_pages = 0;
2821         for (i = 0; i < MAX_NUMNODES; ++i)
2822                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2823         INIT_LIST_HEAD(&h->hugepage_activelist);
2824         h->next_nid_to_alloc = first_memory_node;
2825         h->next_nid_to_free = first_memory_node;
2826         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2827                                         huge_page_size(h)/1024);
2828
2829         parsed_hstate = h;
2830 }
2831
2832 static int __init hugetlb_nrpages_setup(char *s)
2833 {
2834         unsigned long *mhp;
2835         static unsigned long *last_mhp;
2836
2837         if (!parsed_valid_hugepagesz) {
2838                 pr_warn("hugepages = %s preceded by "
2839                         "an unsupported hugepagesz, ignoring\n", s);
2840                 parsed_valid_hugepagesz = true;
2841                 return 1;
2842         }
2843         /*
2844          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2845          * so this hugepages= parameter goes to the "default hstate".
2846          */
2847         else if (!hugetlb_max_hstate)
2848                 mhp = &default_hstate_max_huge_pages;
2849         else
2850                 mhp = &parsed_hstate->max_huge_pages;
2851
2852         if (mhp == last_mhp) {
2853                 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2854                 return 1;
2855         }
2856
2857         if (sscanf(s, "%lu", mhp) <= 0)
2858                 *mhp = 0;
2859
2860         /*
2861          * Global state is always initialized later in hugetlb_init.
2862          * But we need to allocate >= MAX_ORDER hstates here early to still
2863          * use the bootmem allocator.
2864          */
2865         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2866                 hugetlb_hstate_alloc_pages(parsed_hstate);
2867
2868         last_mhp = mhp;
2869
2870         return 1;
2871 }
2872 __setup("hugepages=", hugetlb_nrpages_setup);
2873
2874 static int __init hugetlb_default_setup(char *s)
2875 {
2876         default_hstate_size = memparse(s, &s);
2877         return 1;
2878 }
2879 __setup("default_hugepagesz=", hugetlb_default_setup);
2880
2881 static unsigned int cpuset_mems_nr(unsigned int *array)
2882 {
2883         int node;
2884         unsigned int nr = 0;
2885
2886         for_each_node_mask(node, cpuset_current_mems_allowed)
2887                 nr += array[node];
2888
2889         return nr;
2890 }
2891
2892 #ifdef CONFIG_SYSCTL
2893 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2894                          struct ctl_table *table, int write,
2895                          void __user *buffer, size_t *length, loff_t *ppos)
2896 {
2897         struct hstate *h = &default_hstate;
2898         unsigned long tmp = h->max_huge_pages;
2899         int ret;
2900
2901         if (!hugepages_supported())
2902                 return -EOPNOTSUPP;
2903
2904         table->data = &tmp;
2905         table->maxlen = sizeof(unsigned long);
2906         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2907         if (ret)
2908                 goto out;
2909
2910         if (write)
2911                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2912                                                   NUMA_NO_NODE, tmp, *length);
2913 out:
2914         return ret;
2915 }
2916
2917 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2918                           void __user *buffer, size_t *length, loff_t *ppos)
2919 {
2920
2921         return hugetlb_sysctl_handler_common(false, table, write,
2922                                                         buffer, length, ppos);
2923 }
2924
2925 #ifdef CONFIG_NUMA
2926 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2927                           void __user *buffer, size_t *length, loff_t *ppos)
2928 {
2929         return hugetlb_sysctl_handler_common(true, table, write,
2930                                                         buffer, length, ppos);
2931 }
2932 #endif /* CONFIG_NUMA */
2933
2934 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2935                         void __user *buffer,
2936                         size_t *length, loff_t *ppos)
2937 {
2938         struct hstate *h = &default_hstate;
2939         unsigned long tmp;
2940         int ret;
2941
2942         if (!hugepages_supported())
2943                 return -EOPNOTSUPP;
2944
2945         tmp = h->nr_overcommit_huge_pages;
2946
2947         if (write && hstate_is_gigantic(h))
2948                 return -EINVAL;
2949
2950         table->data = &tmp;
2951         table->maxlen = sizeof(unsigned long);
2952         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2953         if (ret)
2954                 goto out;
2955
2956         if (write) {
2957                 spin_lock(&hugetlb_lock);
2958                 h->nr_overcommit_huge_pages = tmp;
2959                 spin_unlock(&hugetlb_lock);
2960         }
2961 out:
2962         return ret;
2963 }
2964
2965 #endif /* CONFIG_SYSCTL */
2966
2967 void hugetlb_report_meminfo(struct seq_file *m)
2968 {
2969         struct hstate *h = &default_hstate;
2970         if (!hugepages_supported())
2971                 return;
2972         seq_printf(m,
2973                         "HugePages_Total:   %5lu\n"
2974                         "HugePages_Free:    %5lu\n"
2975                         "HugePages_Rsvd:    %5lu\n"
2976                         "HugePages_Surp:    %5lu\n"
2977                         "Hugepagesize:   %8lu kB\n",
2978                         h->nr_huge_pages,
2979                         h->free_huge_pages,
2980                         h->resv_huge_pages,
2981                         h->surplus_huge_pages,
2982                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2983 }
2984
2985 int hugetlb_report_node_meminfo(int nid, char *buf)
2986 {
2987         struct hstate *h = &default_hstate;
2988         if (!hugepages_supported())
2989                 return 0;
2990         return sprintf(buf,
2991                 "Node %d HugePages_Total: %5u\n"
2992                 "Node %d HugePages_Free:  %5u\n"
2993                 "Node %d HugePages_Surp:  %5u\n",
2994                 nid, h->nr_huge_pages_node[nid],
2995                 nid, h->free_huge_pages_node[nid],
2996                 nid, h->surplus_huge_pages_node[nid]);
2997 }
2998
2999 void hugetlb_show_meminfo(void)
3000 {
3001         struct hstate *h;
3002         int nid;
3003
3004         if (!hugepages_supported())
3005                 return;
3006
3007         for_each_node_state(nid, N_MEMORY)
3008                 for_each_hstate(h)
3009                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3010                                 nid,
3011                                 h->nr_huge_pages_node[nid],
3012                                 h->free_huge_pages_node[nid],
3013                                 h->surplus_huge_pages_node[nid],
3014                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3015 }
3016
3017 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3018 {
3019         seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3020                    atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3021 }
3022
3023 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3024 unsigned long hugetlb_total_pages(void)
3025 {
3026         struct hstate *h;
3027         unsigned long nr_total_pages = 0;
3028
3029         for_each_hstate(h)
3030                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3031         return nr_total_pages;
3032 }
3033
3034 static int hugetlb_acct_memory(struct hstate *h, long delta)
3035 {
3036         int ret = -ENOMEM;
3037
3038         spin_lock(&hugetlb_lock);
3039         /*
3040          * When cpuset is configured, it breaks the strict hugetlb page
3041          * reservation as the accounting is done on a global variable. Such
3042          * reservation is completely rubbish in the presence of cpuset because
3043          * the reservation is not checked against page availability for the
3044          * current cpuset. Application can still potentially OOM'ed by kernel
3045          * with lack of free htlb page in cpuset that the task is in.
3046          * Attempt to enforce strict accounting with cpuset is almost
3047          * impossible (or too ugly) because cpuset is too fluid that
3048          * task or memory node can be dynamically moved between cpusets.
3049          *
3050          * The change of semantics for shared hugetlb mapping with cpuset is
3051          * undesirable. However, in order to preserve some of the semantics,
3052          * we fall back to check against current free page availability as
3053          * a best attempt and hopefully to minimize the impact of changing
3054          * semantics that cpuset has.
3055          */
3056         if (delta > 0) {
3057                 if (gather_surplus_pages(h, delta) < 0)
3058                         goto out;
3059
3060                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
3061                         return_unused_surplus_pages(h, delta);
3062                         goto out;
3063                 }
3064         }
3065
3066         ret = 0;
3067         if (delta < 0)
3068                 return_unused_surplus_pages(h, (unsigned long) -delta);
3069
3070 out:
3071         spin_unlock(&hugetlb_lock);
3072         return ret;
3073 }
3074
3075 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3076 {
3077         struct resv_map *resv = vma_resv_map(vma);
3078
3079         /*
3080          * This new VMA should share its siblings reservation map if present.
3081          * The VMA will only ever have a valid reservation map pointer where
3082          * it is being copied for another still existing VMA.  As that VMA
3083          * has a reference to the reservation map it cannot disappear until
3084          * after this open call completes.  It is therefore safe to take a
3085          * new reference here without additional locking.
3086          */
3087         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3088                 kref_get(&resv->refs);
3089 }
3090
3091 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3092 {
3093         struct hstate *h = hstate_vma(vma);
3094         struct resv_map *resv = vma_resv_map(vma);
3095         struct hugepage_subpool *spool = subpool_vma(vma);
3096         unsigned long reserve, start, end;
3097         long gbl_reserve;
3098
3099         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3100                 return;
3101
3102         start = vma_hugecache_offset(h, vma, vma->vm_start);
3103         end = vma_hugecache_offset(h, vma, vma->vm_end);
3104
3105         reserve = (end - start) - region_count(resv, start, end);
3106
3107         kref_put(&resv->refs, resv_map_release);
3108
3109         if (reserve) {
3110                 /*
3111                  * Decrement reserve counts.  The global reserve count may be
3112                  * adjusted if the subpool has a minimum size.
3113                  */
3114                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3115                 hugetlb_acct_memory(h, -gbl_reserve);
3116         }
3117 }
3118
3119 /*
3120  * We cannot handle pagefaults against hugetlb pages at all.  They cause
3121  * handle_mm_fault() to try to instantiate regular-sized pages in the
3122  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
3123  * this far.
3124  */
3125 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3126 {
3127         BUG();
3128         return 0;
3129 }
3130
3131 const struct vm_operations_struct hugetlb_vm_ops = {
3132         .fault = hugetlb_vm_op_fault,
3133         .open = hugetlb_vm_op_open,
3134         .close = hugetlb_vm_op_close,
3135 };
3136
3137 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3138                                 int writable)
3139 {
3140         pte_t entry;
3141
3142         if (writable) {
3143                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3144                                          vma->vm_page_prot)));
3145         } else {
3146                 entry = huge_pte_wrprotect(mk_huge_pte(page,
3147                                            vma->vm_page_prot));
3148         }
3149         entry = pte_mkyoung(entry);
3150         entry = pte_mkhuge(entry);
3151         entry = arch_make_huge_pte(entry, vma, page, writable);
3152
3153         return entry;
3154 }
3155
3156 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3157                                    unsigned long address, pte_t *ptep)
3158 {
3159         pte_t entry;
3160
3161         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3162         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3163                 update_mmu_cache(vma, address, ptep);
3164 }
3165
3166 static int is_hugetlb_entry_migration(pte_t pte)
3167 {
3168         swp_entry_t swp;
3169
3170         if (huge_pte_none(pte) || pte_present(pte))
3171                 return 0;
3172         swp = pte_to_swp_entry(pte);
3173         if (non_swap_entry(swp) && is_migration_entry(swp))
3174                 return 1;
3175         else
3176                 return 0;
3177 }
3178
3179 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3180 {
3181         swp_entry_t swp;
3182
3183         if (huge_pte_none(pte) || pte_present(pte))
3184                 return 0;
3185         swp = pte_to_swp_entry(pte);
3186         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3187                 return 1;
3188         else
3189                 return 0;
3190 }
3191
3192 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3193                             struct vm_area_struct *vma)
3194 {
3195         pte_t *src_pte, *dst_pte, entry;
3196         struct page *ptepage;
3197         unsigned long addr;
3198         int cow;
3199         struct hstate *h = hstate_vma(vma);
3200         unsigned long sz = huge_page_size(h);
3201         unsigned long mmun_start;       /* For mmu_notifiers */
3202         unsigned long mmun_end;         /* For mmu_notifiers */
3203         int ret = 0;
3204
3205         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3206
3207         mmun_start = vma->vm_start;
3208         mmun_end = vma->vm_end;
3209         if (cow)
3210                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3211
3212         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3213                 spinlock_t *src_ptl, *dst_ptl;
3214                 src_pte = huge_pte_offset(src, addr);
3215                 if (!src_pte)
3216                         continue;
3217                 dst_pte = huge_pte_alloc(dst, addr, sz);
3218                 if (!dst_pte) {
3219                         ret = -ENOMEM;
3220                         break;
3221                 }
3222
3223                 /* If the pagetables are shared don't copy or take references */
3224                 if (dst_pte == src_pte)
3225                         continue;
3226
3227                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3228                 src_ptl = huge_pte_lockptr(h, src, src_pte);
3229                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3230                 entry = huge_ptep_get(src_pte);
3231                 if (huge_pte_none(entry)) { /* skip none entry */
3232                         ;
3233                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3234                                     is_hugetlb_entry_hwpoisoned(entry))) {
3235                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
3236
3237                         if (is_write_migration_entry(swp_entry) && cow) {
3238                                 /*
3239                                  * COW mappings require pages in both
3240                                  * parent and child to be set to read.
3241                                  */
3242                                 make_migration_entry_read(&swp_entry);
3243                                 entry = swp_entry_to_pte(swp_entry);
3244                                 set_huge_pte_at(src, addr, src_pte, entry);
3245                         }
3246                         set_huge_pte_at(dst, addr, dst_pte, entry);
3247                 } else {
3248                         if (cow) {
3249                                 huge_ptep_set_wrprotect(src, addr, src_pte);
3250                                 mmu_notifier_invalidate_range(src, mmun_start,
3251                                                                    mmun_end);
3252                         }
3253                         entry = huge_ptep_get(src_pte);
3254                         ptepage = pte_page(entry);
3255                         get_page(ptepage);
3256                         page_dup_rmap(ptepage, true);
3257                         set_huge_pte_at(dst, addr, dst_pte, entry);
3258                         hugetlb_count_add(pages_per_huge_page(h), dst);
3259                 }
3260                 spin_unlock(src_ptl);
3261                 spin_unlock(dst_ptl);
3262         }
3263
3264         if (cow)
3265                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3266
3267         return ret;
3268 }
3269
3270 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3271                             unsigned long start, unsigned long end,
3272                             struct page *ref_page)
3273 {
3274         struct mm_struct *mm = vma->vm_mm;
3275         unsigned long address;
3276         pte_t *ptep;
3277         pte_t pte;
3278         spinlock_t *ptl;
3279         struct page *page;
3280         struct hstate *h = hstate_vma(vma);
3281         unsigned long sz = huge_page_size(h);
3282         const unsigned long mmun_start = start; /* For mmu_notifiers */
3283         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
3284
3285         WARN_ON(!is_vm_hugetlb_page(vma));
3286         BUG_ON(start & ~huge_page_mask(h));
3287         BUG_ON(end & ~huge_page_mask(h));
3288
3289         /*
3290          * This is a hugetlb vma, all the pte entries should point
3291          * to huge page.
3292          */
3293         tlb_remove_check_page_size_change(tlb, sz);
3294         tlb_start_vma(tlb, vma);
3295         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3296         address = start;
3297         for (; address < end; address += sz) {
3298                 ptep = huge_pte_offset(mm, address);
3299                 if (!ptep)
3300                         continue;
3301
3302                 ptl = huge_pte_lock(h, mm, ptep);
3303                 if (huge_pmd_unshare(mm, &address, ptep)) {
3304                         spin_unlock(ptl);
3305                         continue;
3306                 }
3307
3308                 pte = huge_ptep_get(ptep);
3309                 if (huge_pte_none(pte)) {
3310                         spin_unlock(ptl);
3311                         continue;
3312                 }
3313
3314                 /*
3315                  * Migrating hugepage or HWPoisoned hugepage is already
3316                  * unmapped and its refcount is dropped, so just clear pte here.
3317                  */
3318                 if (unlikely(!pte_present(pte))) {
3319                         huge_pte_clear(mm, address, ptep);
3320                         spin_unlock(ptl);
3321                         continue;
3322                 }
3323
3324                 page = pte_page(pte);
3325                 /*
3326                  * If a reference page is supplied, it is because a specific
3327                  * page is being unmapped, not a range. Ensure the page we
3328                  * are about to unmap is the actual page of interest.
3329                  */
3330                 if (ref_page) {
3331                         if (page != ref_page) {
3332                                 spin_unlock(ptl);
3333                                 continue;
3334                         }
3335                         /*
3336                          * Mark the VMA as having unmapped its page so that
3337                          * future faults in this VMA will fail rather than
3338                          * looking like data was lost
3339                          */
3340                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3341                 }
3342
3343                 pte = huge_ptep_get_and_clear(mm, address, ptep);
3344                 tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
3345                 if (huge_pte_dirty(pte))
3346                         set_page_dirty(page);
3347
3348                 hugetlb_count_sub(pages_per_huge_page(h), mm);
3349                 page_remove_rmap(page, true);
3350
3351                 spin_unlock(ptl);
3352                 tlb_remove_page_size(tlb, page, huge_page_size(h));
3353                 /*
3354                  * Bail out after unmapping reference page if supplied
3355                  */
3356                 if (ref_page)
3357                         break;
3358         }
3359         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3360         tlb_end_vma(tlb, vma);
3361 }
3362
3363 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3364                           struct vm_area_struct *vma, unsigned long start,
3365                           unsigned long end, struct page *ref_page)
3366 {
3367         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3368
3369         /*
3370          * Clear this flag so that x86's huge_pmd_share page_table_shareable
3371          * test will fail on a vma being torn down, and not grab a page table
3372          * on its way out.  We're lucky that the flag has such an appropriate
3373          * name, and can in fact be safely cleared here. We could clear it
3374          * before the __unmap_hugepage_range above, but all that's necessary
3375          * is to clear it before releasing the i_mmap_rwsem. This works
3376          * because in the context this is called, the VMA is about to be
3377          * destroyed and the i_mmap_rwsem is held.
3378          */
3379         vma->vm_flags &= ~VM_MAYSHARE;
3380 }
3381
3382 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3383                           unsigned long end, struct page *ref_page)
3384 {
3385         struct mm_struct *mm;
3386         struct mmu_gather tlb;
3387
3388         mm = vma->vm_mm;
3389
3390         tlb_gather_mmu(&tlb, mm, start, end);
3391         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3392         tlb_finish_mmu(&tlb, start, end);
3393 }
3394
3395 /*
3396  * This is called when the original mapper is failing to COW a MAP_PRIVATE
3397  * mappping it owns the reserve page for. The intention is to unmap the page
3398  * from other VMAs and let the children be SIGKILLed if they are faulting the
3399  * same region.
3400  */
3401 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3402                               struct page *page, unsigned long address)
3403 {
3404         struct hstate *h = hstate_vma(vma);
3405         struct vm_area_struct *iter_vma;
3406         struct address_space *mapping;
3407         pgoff_t pgoff;
3408
3409         /*
3410          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3411          * from page cache lookup which is in HPAGE_SIZE units.
3412          */
3413         address = address & huge_page_mask(h);
3414         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3415                         vma->vm_pgoff;
3416         mapping = vma->vm_file->f_mapping;
3417
3418         /*
3419          * Take the mapping lock for the duration of the table walk. As
3420          * this mapping should be shared between all the VMAs,
3421          * __unmap_hugepage_range() is called as the lock is already held
3422          */
3423         i_mmap_lock_write(mapping);
3424         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3425                 /* Do not unmap the current VMA */
3426                 if (iter_vma == vma)
3427                         continue;
3428
3429                 /*
3430                  * Shared VMAs have their own reserves and do not affect
3431                  * MAP_PRIVATE accounting but it is possible that a shared
3432                  * VMA is using the same page so check and skip such VMAs.
3433                  */
3434                 if (iter_vma->vm_flags & VM_MAYSHARE)
3435                         continue;
3436
3437                 /*
3438                  * Unmap the page from other VMAs without their own reserves.
3439                  * They get marked to be SIGKILLed if they fault in these
3440                  * areas. This is because a future no-page fault on this VMA
3441                  * could insert a zeroed page instead of the data existing
3442                  * from the time of fork. This would look like data corruption
3443                  */
3444                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3445                         unmap_hugepage_range(iter_vma, address,
3446                                              address + huge_page_size(h), page);
3447         }
3448         i_mmap_unlock_write(mapping);
3449 }
3450
3451 /*
3452  * Hugetlb_cow() should be called with page lock of the original hugepage held.
3453  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3454  * cannot race with other handlers or page migration.
3455  * Keep the pte_same checks anyway to make transition from the mutex easier.
3456  */
3457 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3458                        unsigned long address, pte_t *ptep,
3459                        struct page *pagecache_page, spinlock_t *ptl)
3460 {
3461         pte_t pte;
3462         struct hstate *h = hstate_vma(vma);
3463         struct page *old_page, *new_page;
3464         int ret = 0, outside_reserve = 0;
3465         unsigned long mmun_start;       /* For mmu_notifiers */
3466         unsigned long mmun_end;         /* For mmu_notifiers */
3467
3468         pte = huge_ptep_get(ptep);
3469         old_page = pte_page(pte);
3470
3471 retry_avoidcopy:
3472         /* If no-one else is actually using this page, avoid the copy
3473          * and just make the page writable */
3474         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3475                 page_move_anon_rmap(old_page, vma);
3476                 set_huge_ptep_writable(vma, address, ptep);
3477                 return 0;
3478         }
3479
3480         /*
3481          * If the process that created a MAP_PRIVATE mapping is about to
3482          * perform a COW due to a shared page count, attempt to satisfy
3483          * the allocation without using the existing reserves. The pagecache
3484          * page is used to determine if the reserve at this address was
3485          * consumed or not. If reserves were used, a partial faulted mapping
3486          * at the time of fork() could consume its reserves on COW instead
3487          * of the full address range.
3488          */
3489         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3490                         old_page != pagecache_page)
3491                 outside_reserve = 1;
3492
3493         get_page(old_page);
3494
3495         /*
3496          * Drop page table lock as buddy allocator may be called. It will
3497          * be acquired again before returning to the caller, as expected.
3498          */
3499         spin_unlock(ptl);
3500         new_page = alloc_huge_page(vma, address, outside_reserve);
3501
3502         if (IS_ERR(new_page)) {
3503                 /*
3504                  * If a process owning a MAP_PRIVATE mapping fails to COW,
3505                  * it is due to references held by a child and an insufficient
3506                  * huge page pool. To guarantee the original mappers
3507                  * reliability, unmap the page from child processes. The child
3508                  * may get SIGKILLed if it later faults.
3509                  */
3510                 if (outside_reserve) {
3511                         put_page(old_page);
3512                         BUG_ON(huge_pte_none(pte));
3513                         unmap_ref_private(mm, vma, old_page, address);
3514                         BUG_ON(huge_pte_none(pte));
3515                         spin_lock(ptl);
3516                         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3517                         if (likely(ptep &&
3518                                    pte_same(huge_ptep_get(ptep), pte)))
3519                                 goto retry_avoidcopy;
3520                         /*
3521                          * race occurs while re-acquiring page table
3522                          * lock, and our job is done.
3523                          */
3524                         return 0;
3525                 }
3526
3527                 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3528                         VM_FAULT_OOM : VM_FAULT_SIGBUS;
3529                 goto out_release_old;
3530         }
3531
3532         /*
3533          * When the original hugepage is shared one, it does not have
3534          * anon_vma prepared.
3535          */
3536         if (unlikely(anon_vma_prepare(vma))) {
3537                 ret = VM_FAULT_OOM;
3538                 goto out_release_all;
3539         }
3540
3541         copy_user_huge_page(new_page, old_page, address, vma,
3542                             pages_per_huge_page(h));
3543         __SetPageUptodate(new_page);
3544         set_page_huge_active(new_page);
3545
3546         mmun_start = address & huge_page_mask(h);
3547         mmun_end = mmun_start + huge_page_size(h);
3548         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3549
3550         /*
3551          * Retake the page table lock to check for racing updates
3552          * before the page tables are altered
3553          */
3554         spin_lock(ptl);
3555         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3556         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3557                 ClearPagePrivate(new_page);
3558
3559                 /* Break COW */
3560                 huge_ptep_clear_flush(vma, address, ptep);
3561                 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3562                 set_huge_pte_at(mm, address, ptep,
3563                                 make_huge_pte(vma, new_page, 1));
3564                 page_remove_rmap(old_page, true);
3565                 hugepage_add_new_anon_rmap(new_page, vma, address);
3566                 /* Make the old page be freed below */
3567                 new_page = old_page;
3568         }
3569         spin_unlock(ptl);
3570         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3571 out_release_all:
3572         restore_reserve_on_error(h, vma, address, new_page);
3573         put_page(new_page);
3574 out_release_old:
3575         put_page(old_page);
3576
3577         spin_lock(ptl); /* Caller expects lock to be held */
3578         return ret;
3579 }
3580
3581 /* Return the pagecache page at a given address within a VMA */
3582 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3583                         struct vm_area_struct *vma, unsigned long address)
3584 {
3585         struct address_space *mapping;
3586         pgoff_t idx;
3587
3588         mapping = vma->vm_file->f_mapping;
3589         idx = vma_hugecache_offset(h, vma, address);
3590
3591         return find_lock_page(mapping, idx);
3592 }
3593
3594 /*
3595  * Return whether there is a pagecache page to back given address within VMA.
3596  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3597  */
3598 static bool hugetlbfs_pagecache_present(struct hstate *h,
3599                         struct vm_area_struct *vma, unsigned long address)
3600 {
3601         struct address_space *mapping;
3602         pgoff_t idx;
3603         struct page *page;
3604
3605         mapping = vma->vm_file->f_mapping;
3606         idx = vma_hugecache_offset(h, vma, address);
3607
3608         page = find_get_page(mapping, idx);
3609         if (page)
3610                 put_page(page);
3611         return page != NULL;
3612 }
3613
3614 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3615                            pgoff_t idx)
3616 {
3617         struct inode *inode = mapping->host;
3618         struct hstate *h = hstate_inode(inode);
3619         int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3620
3621         if (err)
3622                 return err;
3623         ClearPagePrivate(page);
3624
3625         spin_lock(&inode->i_lock);
3626         inode->i_blocks += blocks_per_huge_page(h);
3627         spin_unlock(&inode->i_lock);
3628         return 0;
3629 }
3630
3631 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3632                            struct address_space *mapping, pgoff_t idx,
3633                            unsigned long address, pte_t *ptep, unsigned int flags)
3634 {
3635         struct hstate *h = hstate_vma(vma);
3636         int ret = VM_FAULT_SIGBUS;
3637         int anon_rmap = 0;
3638         unsigned long size;
3639         struct page *page;
3640         pte_t new_pte;
3641         spinlock_t *ptl;
3642
3643         /*
3644          * Currently, we are forced to kill the process in the event the
3645          * original mapper has unmapped pages from the child due to a failed
3646          * COW. Warn that such a situation has occurred as it may not be obvious
3647          */
3648         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3649                 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3650                            current->pid);
3651                 return ret;
3652         }
3653
3654         /*
3655          * Use page lock to guard against racing truncation
3656          * before we get page_table_lock.
3657          */
3658 retry:
3659         page = find_lock_page(mapping, idx);
3660         if (!page) {
3661                 size = i_size_read(mapping->host) >> huge_page_shift(h);
3662                 if (idx >= size)
3663                         goto out;
3664                 page = alloc_huge_page(vma, address, 0);
3665                 if (IS_ERR(page)) {
3666                         ret = PTR_ERR(page);
3667                         if (ret == -ENOMEM)
3668                                 ret = VM_FAULT_OOM;
3669                         else
3670                                 ret = VM_FAULT_SIGBUS;
3671                         goto out;
3672                 }
3673                 clear_huge_page(page, address, pages_per_huge_page(h));
3674                 __SetPageUptodate(page);
3675                 set_page_huge_active(page);
3676
3677                 if (vma->vm_flags & VM_MAYSHARE) {
3678                         int err = huge_add_to_page_cache(page, mapping, idx);
3679                         if (err) {
3680                                 put_page(page);
3681                                 if (err == -EEXIST)
3682                                         goto retry;
3683                                 goto out;
3684                         }
3685                 } else {
3686                         lock_page(page);
3687                         if (unlikely(anon_vma_prepare(vma))) {
3688                                 ret = VM_FAULT_OOM;
3689                                 goto backout_unlocked;
3690                         }
3691                         anon_rmap = 1;
3692                 }
3693         } else {
3694                 /*
3695                  * If memory error occurs between mmap() and fault, some process
3696                  * don't have hwpoisoned swap entry for errored virtual address.
3697                  * So we need to block hugepage fault by PG_hwpoison bit check.
3698                  */
3699                 if (unlikely(PageHWPoison(page))) {
3700                         ret = VM_FAULT_HWPOISON |
3701                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3702                         goto backout_unlocked;
3703                 }
3704         }
3705
3706         /*
3707          * If we are going to COW a private mapping later, we examine the
3708          * pending reservations for this page now. This will ensure that
3709          * any allocations necessary to record that reservation occur outside
3710          * the spinlock.
3711          */
3712         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3713                 if (vma_needs_reservation(h, vma, address) < 0) {
3714                         ret = VM_FAULT_OOM;
3715                         goto backout_unlocked;
3716                 }
3717                 /* Just decrements count, does not deallocate */
3718                 vma_end_reservation(h, vma, address);
3719         }
3720
3721         ptl = huge_pte_lock(h, mm, ptep);
3722         size = i_size_read(mapping->host) >> huge_page_shift(h);
3723         if (idx >= size)
3724                 goto backout;
3725
3726         ret = 0;
3727         if (!huge_pte_none(huge_ptep_get(ptep)))
3728                 goto backout;
3729
3730         if (anon_rmap) {
3731                 ClearPagePrivate(page);
3732                 hugepage_add_new_anon_rmap(page, vma, address);
3733         } else
3734                 page_dup_rmap(page, true);
3735         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3736                                 && (vma->vm_flags & VM_SHARED)));
3737         set_huge_pte_at(mm, address, ptep, new_pte);
3738
3739         hugetlb_count_add(pages_per_huge_page(h), mm);
3740         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3741                 /* Optimization, do the COW without a second fault */
3742                 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
3743         }
3744
3745         spin_unlock(ptl);
3746         unlock_page(page);
3747 out:
3748         return ret;
3749
3750 backout:
3751         spin_unlock(ptl);
3752 backout_unlocked:
3753         unlock_page(page);
3754         restore_reserve_on_error(h, vma, address, page);
3755         put_page(page);
3756         goto out;
3757 }
3758
3759 #ifdef CONFIG_SMP
3760 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3761                             struct vm_area_struct *vma,
3762                             struct address_space *mapping,
3763                             pgoff_t idx, unsigned long address)
3764 {
3765         unsigned long key[2];
3766         u32 hash;
3767
3768         if (vma->vm_flags & VM_SHARED) {
3769                 key[0] = (unsigned long) mapping;
3770                 key[1] = idx;
3771         } else {
3772                 key[0] = (unsigned long) mm;
3773                 key[1] = address >> huge_page_shift(h);
3774         }
3775
3776         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3777
3778         return hash & (num_fault_mutexes - 1);
3779 }
3780 #else
3781 /*
3782  * For uniprocesor systems we always use a single mutex, so just
3783  * return 0 and avoid the hashing overhead.
3784  */
3785 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3786                             struct vm_area_struct *vma,
3787                             struct address_space *mapping,
3788                             pgoff_t idx, unsigned long address)
3789 {
3790         return 0;
3791 }
3792 #endif
3793
3794 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3795                         unsigned long address, unsigned int flags)
3796 {
3797         pte_t *ptep, entry;
3798         spinlock_t *ptl;
3799         int ret;
3800         u32 hash;
3801         pgoff_t idx;
3802         struct page *page = NULL;
3803         struct page *pagecache_page = NULL;
3804         struct hstate *h = hstate_vma(vma);
3805         struct address_space *mapping;
3806         int need_wait_lock = 0;
3807
3808         address &= huge_page_mask(h);
3809
3810         ptep = huge_pte_offset(mm, address);
3811         if (ptep) {
3812                 entry = huge_ptep_get(ptep);
3813                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3814                         migration_entry_wait_huge(vma, mm, ptep);
3815                         return 0;
3816                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3817                         return VM_FAULT_HWPOISON_LARGE |
3818                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3819         } else {
3820                 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3821                 if (!ptep)
3822                         return VM_FAULT_OOM;
3823         }
3824
3825         mapping = vma->vm_file->f_mapping;
3826         idx = vma_hugecache_offset(h, vma, address);
3827
3828         /*
3829          * Serialize hugepage allocation and instantiation, so that we don't
3830          * get spurious allocation failures if two CPUs race to instantiate
3831          * the same page in the page cache.
3832          */
3833         hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
3834         mutex_lock(&hugetlb_fault_mutex_table[hash]);
3835
3836         entry = huge_ptep_get(ptep);
3837         if (huge_pte_none(entry)) {
3838                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3839                 goto out_mutex;
3840         }
3841
3842         ret = 0;
3843
3844         /*
3845          * entry could be a migration/hwpoison entry at this point, so this
3846          * check prevents the kernel from going below assuming that we have
3847          * a active hugepage in pagecache. This goto expects the 2nd page fault,
3848          * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3849          * handle it.
3850          */
3851         if (!pte_present(entry))
3852                 goto out_mutex;
3853
3854         /*
3855          * If we are going to COW the mapping later, we examine the pending
3856          * reservations for this page now. This will ensure that any
3857          * allocations necessary to record that reservation occur outside the
3858          * spinlock. For private mappings, we also lookup the pagecache
3859          * page now as it is used to determine if a reservation has been
3860          * consumed.
3861          */
3862         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3863                 if (vma_needs_reservation(h, vma, address) < 0) {
3864                         ret = VM_FAULT_OOM;
3865                         goto out_mutex;
3866                 }
3867                 /* Just decrements count, does not deallocate */
3868                 vma_end_reservation(h, vma, address);
3869
3870                 if (!(vma->vm_flags & VM_MAYSHARE))
3871                         pagecache_page = hugetlbfs_pagecache_page(h,
3872                                                                 vma, address);
3873         }
3874
3875         ptl = huge_pte_lock(h, mm, ptep);
3876
3877         /* Check for a racing update before calling hugetlb_cow */
3878         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3879                 goto out_ptl;
3880
3881         /*
3882          * hugetlb_cow() requires page locks of pte_page(entry) and
3883          * pagecache_page, so here we need take the former one
3884          * when page != pagecache_page or !pagecache_page.
3885          */
3886         page = pte_page(entry);
3887         if (page != pagecache_page)
3888                 if (!trylock_page(page)) {
3889                         need_wait_lock = 1;
3890                         goto out_ptl;
3891                 }
3892
3893         get_page(page);
3894
3895         if (flags & FAULT_FLAG_WRITE) {
3896                 if (!huge_pte_write(entry)) {
3897                         ret = hugetlb_cow(mm, vma, address, ptep,
3898                                           pagecache_page, ptl);
3899                         goto out_put_page;
3900                 }
3901                 entry = huge_pte_mkdirty(entry);
3902         }
3903         entry = pte_mkyoung(entry);
3904         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3905                                                 flags & FAULT_FLAG_WRITE))
3906                 update_mmu_cache(vma, address, ptep);
3907 out_put_page:
3908         if (page != pagecache_page)
3909                 unlock_page(page);
3910         put_page(page);
3911 out_ptl:
3912         spin_unlock(ptl);
3913
3914         if (pagecache_page) {
3915                 unlock_page(pagecache_page);
3916                 put_page(pagecache_page);
3917         }
3918 out_mutex:
3919         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3920         /*
3921          * Generally it's safe to hold refcount during waiting page lock. But
3922          * here we just wait to defer the next page fault to avoid busy loop and
3923          * the page is not used after unlocked before returning from the current
3924          * page fault. So we are safe from accessing freed page, even if we wait
3925          * here without taking refcount.
3926          */
3927         if (need_wait_lock)
3928                 wait_on_page_locked(page);
3929         return ret;
3930 }
3931
3932 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3933                          struct page **pages, struct vm_area_struct **vmas,
3934                          unsigned long *position, unsigned long *nr_pages,
3935                          long i, unsigned int flags)
3936 {
3937         unsigned long pfn_offset;
3938         unsigned long vaddr = *position;
3939         unsigned long remainder = *nr_pages;
3940         struct hstate *h = hstate_vma(vma);
3941
3942         while (vaddr < vma->vm_end && remainder) {
3943                 pte_t *pte;
3944                 spinlock_t *ptl = NULL;
3945                 int absent;
3946                 struct page *page;
3947
3948                 /*
3949                  * If we have a pending SIGKILL, don't keep faulting pages and
3950                  * potentially allocating memory.
3951                  */
3952                 if (unlikely(fatal_signal_pending(current))) {
3953                         remainder = 0;
3954                         break;
3955                 }
3956
3957                 /*
3958                  * Some archs (sparc64, sh*) have multiple pte_ts to
3959                  * each hugepage.  We have to make sure we get the
3960                  * first, for the page indexing below to work.
3961                  *
3962                  * Note that page table lock is not held when pte is null.
3963                  */
3964                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3965                 if (pte)
3966                         ptl = huge_pte_lock(h, mm, pte);
3967                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3968
3969                 /*
3970                  * When coredumping, it suits get_dump_page if we just return
3971                  * an error where there's an empty slot with no huge pagecache
3972                  * to back it.  This way, we avoid allocating a hugepage, and
3973                  * the sparse dumpfile avoids allocating disk blocks, but its
3974                  * huge holes still show up with zeroes where they need to be.
3975                  */
3976                 if (absent && (flags & FOLL_DUMP) &&
3977                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3978                         if (pte)
3979                                 spin_unlock(ptl);
3980                         remainder = 0;
3981                         break;
3982                 }
3983
3984                 /*
3985                  * We need call hugetlb_fault for both hugepages under migration
3986                  * (in which case hugetlb_fault waits for the migration,) and
3987                  * hwpoisoned hugepages (in which case we need to prevent the
3988                  * caller from accessing to them.) In order to do this, we use
3989                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3990                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3991                  * both cases, and because we can't follow correct pages
3992                  * directly from any kind of swap entries.
3993                  */
3994                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3995                     ((flags & FOLL_WRITE) &&
3996                       !huge_pte_write(huge_ptep_get(pte)))) {
3997                         int ret;
3998
3999                         if (pte)
4000                                 spin_unlock(ptl);
4001                         ret = hugetlb_fault(mm, vma, vaddr,
4002                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
4003                         if (!(ret & VM_FAULT_ERROR))
4004                                 continue;
4005
4006                         remainder = 0;
4007                         break;
4008                 }
4009
4010                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
4011                 page = pte_page(huge_ptep_get(pte));
4012 same_page:
4013                 if (pages) {
4014                         pages[i] = mem_map_offset(page, pfn_offset);
4015                         get_page(pages[i]);
4016                 }
4017
4018                 if (vmas)
4019                         vmas[i] = vma;
4020
4021                 vaddr += PAGE_SIZE;
4022                 ++pfn_offset;
4023                 --remainder;
4024                 ++i;
4025                 if (vaddr < vma->vm_end && remainder &&
4026                                 pfn_offset < pages_per_huge_page(h)) {
4027                         /*
4028                          * We use pfn_offset to avoid touching the pageframes
4029                          * of this compound page.
4030                          */
4031                         goto same_page;
4032                 }
4033                 spin_unlock(ptl);
4034         }
4035         *nr_pages = remainder;
4036         *position = vaddr;
4037
4038         return i ? i : -EFAULT;
4039 }
4040
4041 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
4042 /*
4043  * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
4044  * implement this.
4045  */
4046 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
4047 #endif
4048
4049 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
4050                 unsigned long address, unsigned long end, pgprot_t newprot)
4051 {
4052         struct mm_struct *mm = vma->vm_mm;
4053         unsigned long start = address;
4054         pte_t *ptep;
4055         pte_t pte;
4056         struct hstate *h = hstate_vma(vma);
4057         unsigned long pages = 0;
4058
4059         BUG_ON(address >= end);
4060         flush_cache_range(vma, address, end);
4061
4062         mmu_notifier_invalidate_range_start(mm, start, end);
4063         i_mmap_lock_write(vma->vm_file->f_mapping);
4064         for (; address < end; address += huge_page_size(h)) {
4065                 spinlock_t *ptl;
4066                 ptep = huge_pte_offset(mm, address);
4067                 if (!ptep)
4068                         continue;
4069                 ptl = huge_pte_lock(h, mm, ptep);
4070                 if (huge_pmd_unshare(mm, &address, ptep)) {
4071                         pages++;
4072                         spin_unlock(ptl);
4073                         continue;
4074                 }
4075                 pte = huge_ptep_get(ptep);
4076                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
4077                         spin_unlock(ptl);
4078                         continue;
4079                 }
4080                 if (unlikely(is_hugetlb_entry_migration(pte))) {
4081                         swp_entry_t entry = pte_to_swp_entry(pte);
4082
4083                         if (is_write_migration_entry(entry)) {
4084                                 pte_t newpte;
4085
4086                                 make_migration_entry_read(&entry);
4087                                 newpte = swp_entry_to_pte(entry);
4088                                 set_huge_pte_at(mm, address, ptep, newpte);
4089                                 pages++;
4090                         }
4091                         spin_unlock(ptl);
4092                         continue;
4093                 }
4094                 if (!huge_pte_none(pte)) {
4095                         pte = huge_ptep_get_and_clear(mm, address, ptep);
4096                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4097                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
4098                         set_huge_pte_at(mm, address, ptep, pte);
4099                         pages++;
4100                 }
4101                 spin_unlock(ptl);
4102         }
4103         /*
4104          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4105          * may have cleared our pud entry and done put_page on the page table:
4106          * once we release i_mmap_rwsem, another task can do the final put_page
4107          * and that page table be reused and filled with junk.
4108          */
4109         flush_hugetlb_tlb_range(vma, start, end);
4110         mmu_notifier_invalidate_range(mm, start, end);
4111         i_mmap_unlock_write(vma->vm_file->f_mapping);
4112         mmu_notifier_invalidate_range_end(mm, start, end);
4113
4114         return pages << h->order;
4115 }
4116
4117 int hugetlb_reserve_pages(struct inode *inode,
4118                                         long from, long to,
4119                                         struct vm_area_struct *vma,
4120                                         vm_flags_t vm_flags)
4121 {
4122         long ret, chg;
4123         struct hstate *h = hstate_inode(inode);
4124         struct hugepage_subpool *spool = subpool_inode(inode);
4125         struct resv_map *resv_map;
4126         long gbl_reserve;
4127
4128         /*
4129          * Only apply hugepage reservation if asked. At fault time, an
4130          * attempt will be made for VM_NORESERVE to allocate a page
4131          * without using reserves
4132          */
4133         if (vm_flags & VM_NORESERVE)
4134                 return 0;
4135
4136         /*
4137          * Shared mappings base their reservation on the number of pages that
4138          * are already allocated on behalf of the file. Private mappings need
4139          * to reserve the full area even if read-only as mprotect() may be
4140          * called to make the mapping read-write. Assume !vma is a shm mapping
4141          */
4142         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4143                 resv_map = inode_resv_map(inode);
4144
4145                 chg = region_chg(resv_map, from, to);
4146
4147         } else {
4148                 resv_map = resv_map_alloc();
4149                 if (!resv_map)
4150                         return -ENOMEM;
4151
4152                 chg = to - from;
4153
4154                 set_vma_resv_map(vma, resv_map);
4155                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4156         }
4157
4158         if (chg < 0) {
4159                 ret = chg;
4160                 goto out_err;
4161         }
4162
4163         /*
4164          * There must be enough pages in the subpool for the mapping. If
4165          * the subpool has a minimum size, there may be some global
4166          * reservations already in place (gbl_reserve).
4167          */
4168         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4169         if (gbl_reserve < 0) {
4170                 ret = -ENOSPC;
4171                 goto out_err;
4172         }
4173
4174         /*
4175          * Check enough hugepages are available for the reservation.
4176          * Hand the pages back to the subpool if there are not
4177          */
4178         ret = hugetlb_acct_memory(h, gbl_reserve);
4179         if (ret < 0) {
4180                 /* put back original number of pages, chg */
4181                 (void)hugepage_subpool_put_pages(spool, chg);
4182                 goto out_err;
4183         }
4184
4185         /*
4186          * Account for the reservations made. Shared mappings record regions
4187          * that have reservations as they are shared by multiple VMAs.
4188          * When the last VMA disappears, the region map says how much
4189          * the reservation was and the page cache tells how much of
4190          * the reservation was consumed. Private mappings are per-VMA and
4191          * only the consumed reservations are tracked. When the VMA
4192          * disappears, the original reservation is the VMA size and the
4193          * consumed reservations are stored in the map. Hence, nothing
4194          * else has to be done for private mappings here
4195          */
4196         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4197                 long add = region_add(resv_map, from, to);
4198
4199                 if (unlikely(chg > add)) {
4200                         /*
4201                          * pages in this range were added to the reserve
4202                          * map between region_chg and region_add.  This
4203                          * indicates a race with alloc_huge_page.  Adjust
4204                          * the subpool and reserve counts modified above
4205                          * based on the difference.
4206                          */
4207                         long rsv_adjust;
4208
4209                         rsv_adjust = hugepage_subpool_put_pages(spool,
4210                                                                 chg - add);
4211                         hugetlb_acct_memory(h, -rsv_adjust);
4212                 }
4213         }
4214         return 0;
4215 out_err:
4216         if (!vma || vma->vm_flags & VM_MAYSHARE)
4217                 region_abort(resv_map, from, to);
4218         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4219                 kref_put(&resv_map->refs, resv_map_release);
4220         return ret;
4221 }
4222
4223 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4224                                                                 long freed)
4225 {
4226         struct hstate *h = hstate_inode(inode);
4227         struct resv_map *resv_map = inode_resv_map(inode);
4228         long chg = 0;
4229         struct hugepage_subpool *spool = subpool_inode(inode);
4230         long gbl_reserve;
4231
4232         if (resv_map) {
4233                 chg = region_del(resv_map, start, end);
4234                 /*
4235                  * region_del() can fail in the rare case where a region
4236                  * must be split and another region descriptor can not be
4237                  * allocated.  If end == LONG_MAX, it will not fail.
4238                  */
4239                 if (chg < 0)
4240                         return chg;
4241         }
4242
4243         spin_lock(&inode->i_lock);
4244         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4245         spin_unlock(&inode->i_lock);
4246
4247         /*
4248          * If the subpool has a minimum size, the number of global
4249          * reservations to be released may be adjusted.
4250          */
4251         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4252         hugetlb_acct_memory(h, -gbl_reserve);
4253
4254         return 0;
4255 }
4256
4257 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4258 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4259                                 struct vm_area_struct *vma,
4260                                 unsigned long addr, pgoff_t idx)
4261 {
4262         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4263                                 svma->vm_start;
4264         unsigned long sbase = saddr & PUD_MASK;
4265         unsigned long s_end = sbase + PUD_SIZE;
4266
4267         /* Allow segments to share if only one is marked locked */
4268         unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4269         unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4270
4271         /*
4272          * match the virtual addresses, permission and the alignment of the
4273          * page table page.
4274          */
4275         if (pmd_index(addr) != pmd_index(saddr) ||
4276             vm_flags != svm_flags ||
4277             sbase < svma->vm_start || svma->vm_end < s_end)
4278                 return 0;
4279
4280         return saddr;
4281 }
4282
4283 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4284 {
4285         unsigned long base = addr & PUD_MASK;
4286         unsigned long end = base + PUD_SIZE;
4287
4288         /*
4289          * check on proper vm_flags and page table alignment
4290          */
4291         if (vma->vm_flags & VM_MAYSHARE &&
4292             vma->vm_start <= base && end <= vma->vm_end)
4293                 return true;
4294         return false;
4295 }
4296
4297 /*
4298  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4299  * and returns the corresponding pte. While this is not necessary for the
4300  * !shared pmd case because we can allocate the pmd later as well, it makes the
4301  * code much cleaner. pmd allocation is essential for the shared case because
4302  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4303  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4304  * bad pmd for sharing.
4305  */
4306 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4307 {
4308         struct vm_area_struct *vma = find_vma(mm, addr);
4309         struct address_space *mapping = vma->vm_file->f_mapping;
4310         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4311                         vma->vm_pgoff;
4312         struct vm_area_struct *svma;
4313         unsigned long saddr;
4314         pte_t *spte = NULL;
4315         pte_t *pte;
4316         spinlock_t *ptl;
4317
4318         if (!vma_shareable(vma, addr))
4319                 return (pte_t *)pmd_alloc(mm, pud, addr);
4320
4321         i_mmap_lock_write(mapping);
4322         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4323                 if (svma == vma)
4324                         continue;
4325
4326                 saddr = page_table_shareable(svma, vma, addr, idx);
4327                 if (saddr) {
4328                         spte = huge_pte_offset(svma->vm_mm, saddr);
4329                         if (spte) {
4330                                 get_page(virt_to_page(spte));
4331                                 break;
4332                         }
4333                 }
4334         }
4335
4336         if (!spte)
4337                 goto out;
4338
4339         ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
4340         if (pud_none(*pud)) {
4341                 pud_populate(mm, pud,
4342                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4343                 mm_inc_nr_pmds(mm);
4344         } else {
4345                 put_page(virt_to_page(spte));
4346         }
4347         spin_unlock(ptl);
4348 out:
4349         pte = (pte_t *)pmd_alloc(mm, pud, addr);
4350         i_mmap_unlock_write(mapping);
4351         return pte;
4352 }
4353
4354 /*
4355  * unmap huge page backed by shared pte.
4356  *
4357  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
4358  * indicated by page_count > 1, unmap is achieved by clearing pud and
4359  * decrementing the ref count. If count == 1, the pte page is not shared.
4360  *
4361  * called with page table lock held.
4362  *
4363  * returns: 1 successfully unmapped a shared pte page
4364  *          0 the underlying pte page is not shared, or it is the last user
4365  */
4366 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4367 {
4368         pgd_t *pgd = pgd_offset(mm, *addr);
4369         pud_t *pud = pud_offset(pgd, *addr);
4370
4371         BUG_ON(page_count(virt_to_page(ptep)) == 0);
4372         if (page_count(virt_to_page(ptep)) == 1)
4373                 return 0;
4374
4375         pud_clear(pud);
4376         put_page(virt_to_page(ptep));
4377         mm_dec_nr_pmds(mm);
4378         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4379         return 1;
4380 }
4381 #define want_pmd_share()        (1)
4382 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4383 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4384 {
4385         return NULL;
4386 }
4387
4388 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4389 {
4390         return 0;
4391 }
4392 #define want_pmd_share()        (0)
4393 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4394
4395 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4396 pte_t *huge_pte_alloc(struct mm_struct *mm,
4397                         unsigned long addr, unsigned long sz)
4398 {
4399         pgd_t *pgd;
4400         pud_t *pud;
4401         pte_t *pte = NULL;
4402
4403         pgd = pgd_offset(mm, addr);
4404         pud = pud_alloc(mm, pgd, addr);
4405         if (pud) {
4406                 if (sz == PUD_SIZE) {
4407                         pte = (pte_t *)pud;
4408                 } else {
4409                         BUG_ON(sz != PMD_SIZE);
4410                         if (want_pmd_share() && pud_none(*pud))
4411                                 pte = huge_pmd_share(mm, addr, pud);
4412                         else
4413                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4414                 }
4415         }
4416         BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
4417
4418         return pte;
4419 }
4420
4421 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4422 {
4423         pgd_t *pgd;
4424         pud_t *pud;
4425         pmd_t *pmd = NULL;
4426
4427         pgd = pgd_offset(mm, addr);
4428         if (pgd_present(*pgd)) {
4429                 pud = pud_offset(pgd, addr);
4430                 if (pud_present(*pud)) {
4431                         if (pud_huge(*pud))
4432                                 return (pte_t *)pud;
4433                         pmd = pmd_offset(pud, addr);
4434                 }
4435         }
4436         return (pte_t *) pmd;
4437 }
4438
4439 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4440
4441 /*
4442  * These functions are overwritable if your architecture needs its own
4443  * behavior.
4444  */
4445 struct page * __weak
4446 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4447                               int write)
4448 {
4449         return ERR_PTR(-EINVAL);
4450 }
4451
4452 struct page * __weak
4453 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4454                 pmd_t *pmd, int flags)
4455 {
4456         struct page *page = NULL;
4457         spinlock_t *ptl;
4458 retry:
4459         ptl = pmd_lockptr(mm, pmd);
4460         spin_lock(ptl);
4461         /*
4462          * make sure that the address range covered by this pmd is not
4463          * unmapped from other threads.
4464          */
4465         if (!pmd_huge(*pmd))
4466                 goto out;
4467         if (pmd_present(*pmd)) {
4468                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4469                 if (flags & FOLL_GET)
4470                         get_page(page);
4471         } else {
4472                 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
4473                         spin_unlock(ptl);
4474                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
4475                         goto retry;
4476                 }
4477                 /*
4478                  * hwpoisoned entry is treated as no_page_table in
4479                  * follow_page_mask().
4480                  */
4481         }
4482 out:
4483         spin_unlock(ptl);
4484         return page;
4485 }
4486
4487 struct page * __weak
4488 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4489                 pud_t *pud, int flags)
4490 {
4491         if (flags & FOLL_GET)
4492                 return NULL;
4493
4494         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4495 }
4496
4497 #ifdef CONFIG_MEMORY_FAILURE
4498
4499 /*
4500  * This function is called from memory failure code.
4501  */
4502 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4503 {
4504         struct hstate *h = page_hstate(hpage);
4505         int nid = page_to_nid(hpage);
4506         int ret = -EBUSY;
4507
4508         spin_lock(&hugetlb_lock);
4509         /*
4510          * Just checking !page_huge_active is not enough, because that could be
4511          * an isolated/hwpoisoned hugepage (which have >0 refcount).
4512          */
4513         if (!page_huge_active(hpage) && !page_count(hpage)) {
4514                 /*
4515                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
4516                  * but dangling hpage->lru can trigger list-debug warnings
4517                  * (this happens when we call unpoison_memory() on it),
4518                  * so let it point to itself with list_del_init().
4519                  */
4520                 list_del_init(&hpage->lru);
4521                 set_page_refcounted(hpage);
4522                 h->free_huge_pages--;
4523                 h->free_huge_pages_node[nid]--;
4524                 ret = 0;
4525         }
4526         spin_unlock(&hugetlb_lock);
4527         return ret;
4528 }
4529 #endif
4530
4531 bool isolate_huge_page(struct page *page, struct list_head *list)
4532 {
4533         bool ret = true;
4534
4535         VM_BUG_ON_PAGE(!PageHead(page), page);
4536         spin_lock(&hugetlb_lock);
4537         if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4538                 ret = false;
4539                 goto unlock;
4540         }
4541         clear_page_huge_active(page);
4542         list_move_tail(&page->lru, list);
4543 unlock:
4544         spin_unlock(&hugetlb_lock);
4545         return ret;
4546 }
4547
4548 void putback_active_hugepage(struct page *page)
4549 {
4550         VM_BUG_ON_PAGE(!PageHead(page), page);
4551         spin_lock(&hugetlb_lock);
4552         set_page_huge_active(page);
4553         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4554         spin_unlock(&hugetlb_lock);
4555         put_page(page);
4556 }