]> git.karo-electronics.de Git - linux-beck.git/blob - mm/hugetlb.c
Merge tag 'iio-fixes-for-3.16c' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-beck.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/bootmem.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/rmap.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/page-isolation.h>
26 #include <linux/jhash.h>
27
28 #include <asm/page.h>
29 #include <asm/pgtable.h>
30 #include <asm/tlb.h>
31
32 #include <linux/io.h>
33 #include <linux/hugetlb.h>
34 #include <linux/hugetlb_cgroup.h>
35 #include <linux/node.h>
36 #include "internal.h"
37
38 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
39 unsigned long hugepages_treat_as_movable;
40
41 int hugetlb_max_hstate __read_mostly;
42 unsigned int default_hstate_idx;
43 struct hstate hstates[HUGE_MAX_HSTATE];
44
45 __initdata LIST_HEAD(huge_boot_pages);
46
47 /* for command line parsing */
48 static struct hstate * __initdata parsed_hstate;
49 static unsigned long __initdata default_hstate_max_huge_pages;
50 static unsigned long __initdata default_hstate_size;
51
52 /*
53  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
54  * free_huge_pages, and surplus_huge_pages.
55  */
56 DEFINE_SPINLOCK(hugetlb_lock);
57
58 /*
59  * Serializes faults on the same logical page.  This is used to
60  * prevent spurious OOMs when the hugepage pool is fully utilized.
61  */
62 static int num_fault_mutexes;
63 static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
64
65 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
66 {
67         bool free = (spool->count == 0) && (spool->used_hpages == 0);
68
69         spin_unlock(&spool->lock);
70
71         /* If no pages are used, and no other handles to the subpool
72          * remain, free the subpool the subpool remain */
73         if (free)
74                 kfree(spool);
75 }
76
77 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
78 {
79         struct hugepage_subpool *spool;
80
81         spool = kmalloc(sizeof(*spool), GFP_KERNEL);
82         if (!spool)
83                 return NULL;
84
85         spin_lock_init(&spool->lock);
86         spool->count = 1;
87         spool->max_hpages = nr_blocks;
88         spool->used_hpages = 0;
89
90         return spool;
91 }
92
93 void hugepage_put_subpool(struct hugepage_subpool *spool)
94 {
95         spin_lock(&spool->lock);
96         BUG_ON(!spool->count);
97         spool->count--;
98         unlock_or_release_subpool(spool);
99 }
100
101 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
102                                       long delta)
103 {
104         int ret = 0;
105
106         if (!spool)
107                 return 0;
108
109         spin_lock(&spool->lock);
110         if ((spool->used_hpages + delta) <= spool->max_hpages) {
111                 spool->used_hpages += delta;
112         } else {
113                 ret = -ENOMEM;
114         }
115         spin_unlock(&spool->lock);
116
117         return ret;
118 }
119
120 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
121                                        long delta)
122 {
123         if (!spool)
124                 return;
125
126         spin_lock(&spool->lock);
127         spool->used_hpages -= delta;
128         /* If hugetlbfs_put_super couldn't free spool due to
129         * an outstanding quota reference, free it now. */
130         unlock_or_release_subpool(spool);
131 }
132
133 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
134 {
135         return HUGETLBFS_SB(inode->i_sb)->spool;
136 }
137
138 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
139 {
140         return subpool_inode(file_inode(vma->vm_file));
141 }
142
143 /*
144  * Region tracking -- allows tracking of reservations and instantiated pages
145  *                    across the pages in a mapping.
146  *
147  * The region data structures are embedded into a resv_map and
148  * protected by a resv_map's lock
149  */
150 struct file_region {
151         struct list_head link;
152         long from;
153         long to;
154 };
155
156 static long region_add(struct resv_map *resv, long f, long t)
157 {
158         struct list_head *head = &resv->regions;
159         struct file_region *rg, *nrg, *trg;
160
161         spin_lock(&resv->lock);
162         /* Locate the region we are either in or before. */
163         list_for_each_entry(rg, head, link)
164                 if (f <= rg->to)
165                         break;
166
167         /* Round our left edge to the current segment if it encloses us. */
168         if (f > rg->from)
169                 f = rg->from;
170
171         /* Check for and consume any regions we now overlap with. */
172         nrg = rg;
173         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
174                 if (&rg->link == head)
175                         break;
176                 if (rg->from > t)
177                         break;
178
179                 /* If this area reaches higher then extend our area to
180                  * include it completely.  If this is not the first area
181                  * which we intend to reuse, free it. */
182                 if (rg->to > t)
183                         t = rg->to;
184                 if (rg != nrg) {
185                         list_del(&rg->link);
186                         kfree(rg);
187                 }
188         }
189         nrg->from = f;
190         nrg->to = t;
191         spin_unlock(&resv->lock);
192         return 0;
193 }
194
195 static long region_chg(struct resv_map *resv, long f, long t)
196 {
197         struct list_head *head = &resv->regions;
198         struct file_region *rg, *nrg = NULL;
199         long chg = 0;
200
201 retry:
202         spin_lock(&resv->lock);
203         /* Locate the region we are before or in. */
204         list_for_each_entry(rg, head, link)
205                 if (f <= rg->to)
206                         break;
207
208         /* If we are below the current region then a new region is required.
209          * Subtle, allocate a new region at the position but make it zero
210          * size such that we can guarantee to record the reservation. */
211         if (&rg->link == head || t < rg->from) {
212                 if (!nrg) {
213                         spin_unlock(&resv->lock);
214                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
215                         if (!nrg)
216                                 return -ENOMEM;
217
218                         nrg->from = f;
219                         nrg->to   = f;
220                         INIT_LIST_HEAD(&nrg->link);
221                         goto retry;
222                 }
223
224                 list_add(&nrg->link, rg->link.prev);
225                 chg = t - f;
226                 goto out_nrg;
227         }
228
229         /* Round our left edge to the current segment if it encloses us. */
230         if (f > rg->from)
231                 f = rg->from;
232         chg = t - f;
233
234         /* Check for and consume any regions we now overlap with. */
235         list_for_each_entry(rg, rg->link.prev, link) {
236                 if (&rg->link == head)
237                         break;
238                 if (rg->from > t)
239                         goto out;
240
241                 /* We overlap with this area, if it extends further than
242                  * us then we must extend ourselves.  Account for its
243                  * existing reservation. */
244                 if (rg->to > t) {
245                         chg += rg->to - t;
246                         t = rg->to;
247                 }
248                 chg -= rg->to - rg->from;
249         }
250
251 out:
252         spin_unlock(&resv->lock);
253         /*  We already know we raced and no longer need the new region */
254         kfree(nrg);
255         return chg;
256 out_nrg:
257         spin_unlock(&resv->lock);
258         return chg;
259 }
260
261 static long region_truncate(struct resv_map *resv, long end)
262 {
263         struct list_head *head = &resv->regions;
264         struct file_region *rg, *trg;
265         long chg = 0;
266
267         spin_lock(&resv->lock);
268         /* Locate the region we are either in or before. */
269         list_for_each_entry(rg, head, link)
270                 if (end <= rg->to)
271                         break;
272         if (&rg->link == head)
273                 goto out;
274
275         /* If we are in the middle of a region then adjust it. */
276         if (end > rg->from) {
277                 chg = rg->to - end;
278                 rg->to = end;
279                 rg = list_entry(rg->link.next, typeof(*rg), link);
280         }
281
282         /* Drop any remaining regions. */
283         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
284                 if (&rg->link == head)
285                         break;
286                 chg += rg->to - rg->from;
287                 list_del(&rg->link);
288                 kfree(rg);
289         }
290
291 out:
292         spin_unlock(&resv->lock);
293         return chg;
294 }
295
296 static long region_count(struct resv_map *resv, long f, long t)
297 {
298         struct list_head *head = &resv->regions;
299         struct file_region *rg;
300         long chg = 0;
301
302         spin_lock(&resv->lock);
303         /* Locate each segment we overlap with, and count that overlap. */
304         list_for_each_entry(rg, head, link) {
305                 long seg_from;
306                 long seg_to;
307
308                 if (rg->to <= f)
309                         continue;
310                 if (rg->from >= t)
311                         break;
312
313                 seg_from = max(rg->from, f);
314                 seg_to = min(rg->to, t);
315
316                 chg += seg_to - seg_from;
317         }
318         spin_unlock(&resv->lock);
319
320         return chg;
321 }
322
323 /*
324  * Convert the address within this vma to the page offset within
325  * the mapping, in pagecache page units; huge pages here.
326  */
327 static pgoff_t vma_hugecache_offset(struct hstate *h,
328                         struct vm_area_struct *vma, unsigned long address)
329 {
330         return ((address - vma->vm_start) >> huge_page_shift(h)) +
331                         (vma->vm_pgoff >> huge_page_order(h));
332 }
333
334 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
335                                      unsigned long address)
336 {
337         return vma_hugecache_offset(hstate_vma(vma), vma, address);
338 }
339
340 /*
341  * Return the size of the pages allocated when backing a VMA. In the majority
342  * cases this will be same size as used by the page table entries.
343  */
344 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
345 {
346         struct hstate *hstate;
347
348         if (!is_vm_hugetlb_page(vma))
349                 return PAGE_SIZE;
350
351         hstate = hstate_vma(vma);
352
353         return 1UL << huge_page_shift(hstate);
354 }
355 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
356
357 /*
358  * Return the page size being used by the MMU to back a VMA. In the majority
359  * of cases, the page size used by the kernel matches the MMU size. On
360  * architectures where it differs, an architecture-specific version of this
361  * function is required.
362  */
363 #ifndef vma_mmu_pagesize
364 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
365 {
366         return vma_kernel_pagesize(vma);
367 }
368 #endif
369
370 /*
371  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
372  * bits of the reservation map pointer, which are always clear due to
373  * alignment.
374  */
375 #define HPAGE_RESV_OWNER    (1UL << 0)
376 #define HPAGE_RESV_UNMAPPED (1UL << 1)
377 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
378
379 /*
380  * These helpers are used to track how many pages are reserved for
381  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
382  * is guaranteed to have their future faults succeed.
383  *
384  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
385  * the reserve counters are updated with the hugetlb_lock held. It is safe
386  * to reset the VMA at fork() time as it is not in use yet and there is no
387  * chance of the global counters getting corrupted as a result of the values.
388  *
389  * The private mapping reservation is represented in a subtly different
390  * manner to a shared mapping.  A shared mapping has a region map associated
391  * with the underlying file, this region map represents the backing file
392  * pages which have ever had a reservation assigned which this persists even
393  * after the page is instantiated.  A private mapping has a region map
394  * associated with the original mmap which is attached to all VMAs which
395  * reference it, this region map represents those offsets which have consumed
396  * reservation ie. where pages have been instantiated.
397  */
398 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
399 {
400         return (unsigned long)vma->vm_private_data;
401 }
402
403 static void set_vma_private_data(struct vm_area_struct *vma,
404                                                         unsigned long value)
405 {
406         vma->vm_private_data = (void *)value;
407 }
408
409 struct resv_map *resv_map_alloc(void)
410 {
411         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
412         if (!resv_map)
413                 return NULL;
414
415         kref_init(&resv_map->refs);
416         spin_lock_init(&resv_map->lock);
417         INIT_LIST_HEAD(&resv_map->regions);
418
419         return resv_map;
420 }
421
422 void resv_map_release(struct kref *ref)
423 {
424         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
425
426         /* Clear out any active regions before we release the map. */
427         region_truncate(resv_map, 0);
428         kfree(resv_map);
429 }
430
431 static inline struct resv_map *inode_resv_map(struct inode *inode)
432 {
433         return inode->i_mapping->private_data;
434 }
435
436 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
437 {
438         VM_BUG_ON(!is_vm_hugetlb_page(vma));
439         if (vma->vm_flags & VM_MAYSHARE) {
440                 struct address_space *mapping = vma->vm_file->f_mapping;
441                 struct inode *inode = mapping->host;
442
443                 return inode_resv_map(inode);
444
445         } else {
446                 return (struct resv_map *)(get_vma_private_data(vma) &
447                                                         ~HPAGE_RESV_MASK);
448         }
449 }
450
451 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
452 {
453         VM_BUG_ON(!is_vm_hugetlb_page(vma));
454         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
455
456         set_vma_private_data(vma, (get_vma_private_data(vma) &
457                                 HPAGE_RESV_MASK) | (unsigned long)map);
458 }
459
460 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
461 {
462         VM_BUG_ON(!is_vm_hugetlb_page(vma));
463         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
464
465         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
466 }
467
468 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
469 {
470         VM_BUG_ON(!is_vm_hugetlb_page(vma));
471
472         return (get_vma_private_data(vma) & flag) != 0;
473 }
474
475 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
476 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
477 {
478         VM_BUG_ON(!is_vm_hugetlb_page(vma));
479         if (!(vma->vm_flags & VM_MAYSHARE))
480                 vma->vm_private_data = (void *)0;
481 }
482
483 /* Returns true if the VMA has associated reserve pages */
484 static int vma_has_reserves(struct vm_area_struct *vma, long chg)
485 {
486         if (vma->vm_flags & VM_NORESERVE) {
487                 /*
488                  * This address is already reserved by other process(chg == 0),
489                  * so, we should decrement reserved count. Without decrementing,
490                  * reserve count remains after releasing inode, because this
491                  * allocated page will go into page cache and is regarded as
492                  * coming from reserved pool in releasing step.  Currently, we
493                  * don't have any other solution to deal with this situation
494                  * properly, so add work-around here.
495                  */
496                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
497                         return 1;
498                 else
499                         return 0;
500         }
501
502         /* Shared mappings always use reserves */
503         if (vma->vm_flags & VM_MAYSHARE)
504                 return 1;
505
506         /*
507          * Only the process that called mmap() has reserves for
508          * private mappings.
509          */
510         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
511                 return 1;
512
513         return 0;
514 }
515
516 static void enqueue_huge_page(struct hstate *h, struct page *page)
517 {
518         int nid = page_to_nid(page);
519         list_move(&page->lru, &h->hugepage_freelists[nid]);
520         h->free_huge_pages++;
521         h->free_huge_pages_node[nid]++;
522 }
523
524 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
525 {
526         struct page *page;
527
528         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
529                 if (!is_migrate_isolate_page(page))
530                         break;
531         /*
532          * if 'non-isolated free hugepage' not found on the list,
533          * the allocation fails.
534          */
535         if (&h->hugepage_freelists[nid] == &page->lru)
536                 return NULL;
537         list_move(&page->lru, &h->hugepage_activelist);
538         set_page_refcounted(page);
539         h->free_huge_pages--;
540         h->free_huge_pages_node[nid]--;
541         return page;
542 }
543
544 /* Movability of hugepages depends on migration support. */
545 static inline gfp_t htlb_alloc_mask(struct hstate *h)
546 {
547         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
548                 return GFP_HIGHUSER_MOVABLE;
549         else
550                 return GFP_HIGHUSER;
551 }
552
553 static struct page *dequeue_huge_page_vma(struct hstate *h,
554                                 struct vm_area_struct *vma,
555                                 unsigned long address, int avoid_reserve,
556                                 long chg)
557 {
558         struct page *page = NULL;
559         struct mempolicy *mpol;
560         nodemask_t *nodemask;
561         struct zonelist *zonelist;
562         struct zone *zone;
563         struct zoneref *z;
564         unsigned int cpuset_mems_cookie;
565
566         /*
567          * A child process with MAP_PRIVATE mappings created by their parent
568          * have no page reserves. This check ensures that reservations are
569          * not "stolen". The child may still get SIGKILLed
570          */
571         if (!vma_has_reserves(vma, chg) &&
572                         h->free_huge_pages - h->resv_huge_pages == 0)
573                 goto err;
574
575         /* If reserves cannot be used, ensure enough pages are in the pool */
576         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
577                 goto err;
578
579 retry_cpuset:
580         cpuset_mems_cookie = read_mems_allowed_begin();
581         zonelist = huge_zonelist(vma, address,
582                                         htlb_alloc_mask(h), &mpol, &nodemask);
583
584         for_each_zone_zonelist_nodemask(zone, z, zonelist,
585                                                 MAX_NR_ZONES - 1, nodemask) {
586                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
587                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
588                         if (page) {
589                                 if (avoid_reserve)
590                                         break;
591                                 if (!vma_has_reserves(vma, chg))
592                                         break;
593
594                                 SetPagePrivate(page);
595                                 h->resv_huge_pages--;
596                                 break;
597                         }
598                 }
599         }
600
601         mpol_cond_put(mpol);
602         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
603                 goto retry_cpuset;
604         return page;
605
606 err:
607         return NULL;
608 }
609
610 /*
611  * common helper functions for hstate_next_node_to_{alloc|free}.
612  * We may have allocated or freed a huge page based on a different
613  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
614  * be outside of *nodes_allowed.  Ensure that we use an allowed
615  * node for alloc or free.
616  */
617 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
618 {
619         nid = next_node(nid, *nodes_allowed);
620         if (nid == MAX_NUMNODES)
621                 nid = first_node(*nodes_allowed);
622         VM_BUG_ON(nid >= MAX_NUMNODES);
623
624         return nid;
625 }
626
627 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
628 {
629         if (!node_isset(nid, *nodes_allowed))
630                 nid = next_node_allowed(nid, nodes_allowed);
631         return nid;
632 }
633
634 /*
635  * returns the previously saved node ["this node"] from which to
636  * allocate a persistent huge page for the pool and advance the
637  * next node from which to allocate, handling wrap at end of node
638  * mask.
639  */
640 static int hstate_next_node_to_alloc(struct hstate *h,
641                                         nodemask_t *nodes_allowed)
642 {
643         int nid;
644
645         VM_BUG_ON(!nodes_allowed);
646
647         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
648         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
649
650         return nid;
651 }
652
653 /*
654  * helper for free_pool_huge_page() - return the previously saved
655  * node ["this node"] from which to free a huge page.  Advance the
656  * next node id whether or not we find a free huge page to free so
657  * that the next attempt to free addresses the next node.
658  */
659 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
660 {
661         int nid;
662
663         VM_BUG_ON(!nodes_allowed);
664
665         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
666         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
667
668         return nid;
669 }
670
671 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
672         for (nr_nodes = nodes_weight(*mask);                            \
673                 nr_nodes > 0 &&                                         \
674                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
675                 nr_nodes--)
676
677 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
678         for (nr_nodes = nodes_weight(*mask);                            \
679                 nr_nodes > 0 &&                                         \
680                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
681                 nr_nodes--)
682
683 #if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
684 static void destroy_compound_gigantic_page(struct page *page,
685                                         unsigned long order)
686 {
687         int i;
688         int nr_pages = 1 << order;
689         struct page *p = page + 1;
690
691         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
692                 __ClearPageTail(p);
693                 set_page_refcounted(p);
694                 p->first_page = NULL;
695         }
696
697         set_compound_order(page, 0);
698         __ClearPageHead(page);
699 }
700
701 static void free_gigantic_page(struct page *page, unsigned order)
702 {
703         free_contig_range(page_to_pfn(page), 1 << order);
704 }
705
706 static int __alloc_gigantic_page(unsigned long start_pfn,
707                                 unsigned long nr_pages)
708 {
709         unsigned long end_pfn = start_pfn + nr_pages;
710         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
711 }
712
713 static bool pfn_range_valid_gigantic(unsigned long start_pfn,
714                                 unsigned long nr_pages)
715 {
716         unsigned long i, end_pfn = start_pfn + nr_pages;
717         struct page *page;
718
719         for (i = start_pfn; i < end_pfn; i++) {
720                 if (!pfn_valid(i))
721                         return false;
722
723                 page = pfn_to_page(i);
724
725                 if (PageReserved(page))
726                         return false;
727
728                 if (page_count(page) > 0)
729                         return false;
730
731                 if (PageHuge(page))
732                         return false;
733         }
734
735         return true;
736 }
737
738 static bool zone_spans_last_pfn(const struct zone *zone,
739                         unsigned long start_pfn, unsigned long nr_pages)
740 {
741         unsigned long last_pfn = start_pfn + nr_pages - 1;
742         return zone_spans_pfn(zone, last_pfn);
743 }
744
745 static struct page *alloc_gigantic_page(int nid, unsigned order)
746 {
747         unsigned long nr_pages = 1 << order;
748         unsigned long ret, pfn, flags;
749         struct zone *z;
750
751         z = NODE_DATA(nid)->node_zones;
752         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
753                 spin_lock_irqsave(&z->lock, flags);
754
755                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
756                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
757                         if (pfn_range_valid_gigantic(pfn, nr_pages)) {
758                                 /*
759                                  * We release the zone lock here because
760                                  * alloc_contig_range() will also lock the zone
761                                  * at some point. If there's an allocation
762                                  * spinning on this lock, it may win the race
763                                  * and cause alloc_contig_range() to fail...
764                                  */
765                                 spin_unlock_irqrestore(&z->lock, flags);
766                                 ret = __alloc_gigantic_page(pfn, nr_pages);
767                                 if (!ret)
768                                         return pfn_to_page(pfn);
769                                 spin_lock_irqsave(&z->lock, flags);
770                         }
771                         pfn += nr_pages;
772                 }
773
774                 spin_unlock_irqrestore(&z->lock, flags);
775         }
776
777         return NULL;
778 }
779
780 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
781 static void prep_compound_gigantic_page(struct page *page, unsigned long order);
782
783 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
784 {
785         struct page *page;
786
787         page = alloc_gigantic_page(nid, huge_page_order(h));
788         if (page) {
789                 prep_compound_gigantic_page(page, huge_page_order(h));
790                 prep_new_huge_page(h, page, nid);
791         }
792
793         return page;
794 }
795
796 static int alloc_fresh_gigantic_page(struct hstate *h,
797                                 nodemask_t *nodes_allowed)
798 {
799         struct page *page = NULL;
800         int nr_nodes, node;
801
802         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
803                 page = alloc_fresh_gigantic_page_node(h, node);
804                 if (page)
805                         return 1;
806         }
807
808         return 0;
809 }
810
811 static inline bool gigantic_page_supported(void) { return true; }
812 #else
813 static inline bool gigantic_page_supported(void) { return false; }
814 static inline void free_gigantic_page(struct page *page, unsigned order) { }
815 static inline void destroy_compound_gigantic_page(struct page *page,
816                                                 unsigned long order) { }
817 static inline int alloc_fresh_gigantic_page(struct hstate *h,
818                                         nodemask_t *nodes_allowed) { return 0; }
819 #endif
820
821 static void update_and_free_page(struct hstate *h, struct page *page)
822 {
823         int i;
824
825         if (hstate_is_gigantic(h) && !gigantic_page_supported())
826                 return;
827
828         h->nr_huge_pages--;
829         h->nr_huge_pages_node[page_to_nid(page)]--;
830         for (i = 0; i < pages_per_huge_page(h); i++) {
831                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
832                                 1 << PG_referenced | 1 << PG_dirty |
833                                 1 << PG_active | 1 << PG_private |
834                                 1 << PG_writeback);
835         }
836         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
837         set_compound_page_dtor(page, NULL);
838         set_page_refcounted(page);
839         if (hstate_is_gigantic(h)) {
840                 destroy_compound_gigantic_page(page, huge_page_order(h));
841                 free_gigantic_page(page, huge_page_order(h));
842         } else {
843                 arch_release_hugepage(page);
844                 __free_pages(page, huge_page_order(h));
845         }
846 }
847
848 struct hstate *size_to_hstate(unsigned long size)
849 {
850         struct hstate *h;
851
852         for_each_hstate(h) {
853                 if (huge_page_size(h) == size)
854                         return h;
855         }
856         return NULL;
857 }
858
859 static void free_huge_page(struct page *page)
860 {
861         /*
862          * Can't pass hstate in here because it is called from the
863          * compound page destructor.
864          */
865         struct hstate *h = page_hstate(page);
866         int nid = page_to_nid(page);
867         struct hugepage_subpool *spool =
868                 (struct hugepage_subpool *)page_private(page);
869         bool restore_reserve;
870
871         set_page_private(page, 0);
872         page->mapping = NULL;
873         BUG_ON(page_count(page));
874         BUG_ON(page_mapcount(page));
875         restore_reserve = PagePrivate(page);
876         ClearPagePrivate(page);
877
878         spin_lock(&hugetlb_lock);
879         hugetlb_cgroup_uncharge_page(hstate_index(h),
880                                      pages_per_huge_page(h), page);
881         if (restore_reserve)
882                 h->resv_huge_pages++;
883
884         if (h->surplus_huge_pages_node[nid]) {
885                 /* remove the page from active list */
886                 list_del(&page->lru);
887                 update_and_free_page(h, page);
888                 h->surplus_huge_pages--;
889                 h->surplus_huge_pages_node[nid]--;
890         } else {
891                 arch_clear_hugepage_flags(page);
892                 enqueue_huge_page(h, page);
893         }
894         spin_unlock(&hugetlb_lock);
895         hugepage_subpool_put_pages(spool, 1);
896 }
897
898 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
899 {
900         INIT_LIST_HEAD(&page->lru);
901         set_compound_page_dtor(page, free_huge_page);
902         spin_lock(&hugetlb_lock);
903         set_hugetlb_cgroup(page, NULL);
904         h->nr_huge_pages++;
905         h->nr_huge_pages_node[nid]++;
906         spin_unlock(&hugetlb_lock);
907         put_page(page); /* free it into the hugepage allocator */
908 }
909
910 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
911 {
912         int i;
913         int nr_pages = 1 << order;
914         struct page *p = page + 1;
915
916         /* we rely on prep_new_huge_page to set the destructor */
917         set_compound_order(page, order);
918         __SetPageHead(page);
919         __ClearPageReserved(page);
920         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
921                 __SetPageTail(p);
922                 /*
923                  * For gigantic hugepages allocated through bootmem at
924                  * boot, it's safer to be consistent with the not-gigantic
925                  * hugepages and clear the PG_reserved bit from all tail pages
926                  * too.  Otherwse drivers using get_user_pages() to access tail
927                  * pages may get the reference counting wrong if they see
928                  * PG_reserved set on a tail page (despite the head page not
929                  * having PG_reserved set).  Enforcing this consistency between
930                  * head and tail pages allows drivers to optimize away a check
931                  * on the head page when they need know if put_page() is needed
932                  * after get_user_pages().
933                  */
934                 __ClearPageReserved(p);
935                 set_page_count(p, 0);
936                 p->first_page = page;
937         }
938 }
939
940 /*
941  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
942  * transparent huge pages.  See the PageTransHuge() documentation for more
943  * details.
944  */
945 int PageHuge(struct page *page)
946 {
947         if (!PageCompound(page))
948                 return 0;
949
950         page = compound_head(page);
951         return get_compound_page_dtor(page) == free_huge_page;
952 }
953 EXPORT_SYMBOL_GPL(PageHuge);
954
955 /*
956  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
957  * normal or transparent huge pages.
958  */
959 int PageHeadHuge(struct page *page_head)
960 {
961         if (!PageHead(page_head))
962                 return 0;
963
964         return get_compound_page_dtor(page_head) == free_huge_page;
965 }
966
967 pgoff_t __basepage_index(struct page *page)
968 {
969         struct page *page_head = compound_head(page);
970         pgoff_t index = page_index(page_head);
971         unsigned long compound_idx;
972
973         if (!PageHuge(page_head))
974                 return page_index(page);
975
976         if (compound_order(page_head) >= MAX_ORDER)
977                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
978         else
979                 compound_idx = page - page_head;
980
981         return (index << compound_order(page_head)) + compound_idx;
982 }
983
984 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
985 {
986         struct page *page;
987
988         page = alloc_pages_exact_node(nid,
989                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
990                                                 __GFP_REPEAT|__GFP_NOWARN,
991                 huge_page_order(h));
992         if (page) {
993                 if (arch_prepare_hugepage(page)) {
994                         __free_pages(page, huge_page_order(h));
995                         return NULL;
996                 }
997                 prep_new_huge_page(h, page, nid);
998         }
999
1000         return page;
1001 }
1002
1003 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1004 {
1005         struct page *page;
1006         int nr_nodes, node;
1007         int ret = 0;
1008
1009         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1010                 page = alloc_fresh_huge_page_node(h, node);
1011                 if (page) {
1012                         ret = 1;
1013                         break;
1014                 }
1015         }
1016
1017         if (ret)
1018                 count_vm_event(HTLB_BUDDY_PGALLOC);
1019         else
1020                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1021
1022         return ret;
1023 }
1024
1025 /*
1026  * Free huge page from pool from next node to free.
1027  * Attempt to keep persistent huge pages more or less
1028  * balanced over allowed nodes.
1029  * Called with hugetlb_lock locked.
1030  */
1031 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1032                                                          bool acct_surplus)
1033 {
1034         int nr_nodes, node;
1035         int ret = 0;
1036
1037         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1038                 /*
1039                  * If we're returning unused surplus pages, only examine
1040                  * nodes with surplus pages.
1041                  */
1042                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1043                     !list_empty(&h->hugepage_freelists[node])) {
1044                         struct page *page =
1045                                 list_entry(h->hugepage_freelists[node].next,
1046                                           struct page, lru);
1047                         list_del(&page->lru);
1048                         h->free_huge_pages--;
1049                         h->free_huge_pages_node[node]--;
1050                         if (acct_surplus) {
1051                                 h->surplus_huge_pages--;
1052                                 h->surplus_huge_pages_node[node]--;
1053                         }
1054                         update_and_free_page(h, page);
1055                         ret = 1;
1056                         break;
1057                 }
1058         }
1059
1060         return ret;
1061 }
1062
1063 /*
1064  * Dissolve a given free hugepage into free buddy pages. This function does
1065  * nothing for in-use (including surplus) hugepages.
1066  */
1067 static void dissolve_free_huge_page(struct page *page)
1068 {
1069         spin_lock(&hugetlb_lock);
1070         if (PageHuge(page) && !page_count(page)) {
1071                 struct hstate *h = page_hstate(page);
1072                 int nid = page_to_nid(page);
1073                 list_del(&page->lru);
1074                 h->free_huge_pages--;
1075                 h->free_huge_pages_node[nid]--;
1076                 update_and_free_page(h, page);
1077         }
1078         spin_unlock(&hugetlb_lock);
1079 }
1080
1081 /*
1082  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1083  * make specified memory blocks removable from the system.
1084  * Note that start_pfn should aligned with (minimum) hugepage size.
1085  */
1086 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1087 {
1088         unsigned int order = 8 * sizeof(void *);
1089         unsigned long pfn;
1090         struct hstate *h;
1091
1092         /* Set scan step to minimum hugepage size */
1093         for_each_hstate(h)
1094                 if (order > huge_page_order(h))
1095                         order = huge_page_order(h);
1096         VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
1097         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
1098                 dissolve_free_huge_page(pfn_to_page(pfn));
1099 }
1100
1101 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
1102 {
1103         struct page *page;
1104         unsigned int r_nid;
1105
1106         if (hstate_is_gigantic(h))
1107                 return NULL;
1108
1109         /*
1110          * Assume we will successfully allocate the surplus page to
1111          * prevent racing processes from causing the surplus to exceed
1112          * overcommit
1113          *
1114          * This however introduces a different race, where a process B
1115          * tries to grow the static hugepage pool while alloc_pages() is
1116          * called by process A. B will only examine the per-node
1117          * counters in determining if surplus huge pages can be
1118          * converted to normal huge pages in adjust_pool_surplus(). A
1119          * won't be able to increment the per-node counter, until the
1120          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1121          * no more huge pages can be converted from surplus to normal
1122          * state (and doesn't try to convert again). Thus, we have a
1123          * case where a surplus huge page exists, the pool is grown, and
1124          * the surplus huge page still exists after, even though it
1125          * should just have been converted to a normal huge page. This
1126          * does not leak memory, though, as the hugepage will be freed
1127          * once it is out of use. It also does not allow the counters to
1128          * go out of whack in adjust_pool_surplus() as we don't modify
1129          * the node values until we've gotten the hugepage and only the
1130          * per-node value is checked there.
1131          */
1132         spin_lock(&hugetlb_lock);
1133         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1134                 spin_unlock(&hugetlb_lock);
1135                 return NULL;
1136         } else {
1137                 h->nr_huge_pages++;
1138                 h->surplus_huge_pages++;
1139         }
1140         spin_unlock(&hugetlb_lock);
1141
1142         if (nid == NUMA_NO_NODE)
1143                 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
1144                                    __GFP_REPEAT|__GFP_NOWARN,
1145                                    huge_page_order(h));
1146         else
1147                 page = alloc_pages_exact_node(nid,
1148                         htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1149                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
1150
1151         if (page && arch_prepare_hugepage(page)) {
1152                 __free_pages(page, huge_page_order(h));
1153                 page = NULL;
1154         }
1155
1156         spin_lock(&hugetlb_lock);
1157         if (page) {
1158                 INIT_LIST_HEAD(&page->lru);
1159                 r_nid = page_to_nid(page);
1160                 set_compound_page_dtor(page, free_huge_page);
1161                 set_hugetlb_cgroup(page, NULL);
1162                 /*
1163                  * We incremented the global counters already
1164                  */
1165                 h->nr_huge_pages_node[r_nid]++;
1166                 h->surplus_huge_pages_node[r_nid]++;
1167                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1168         } else {
1169                 h->nr_huge_pages--;
1170                 h->surplus_huge_pages--;
1171                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1172         }
1173         spin_unlock(&hugetlb_lock);
1174
1175         return page;
1176 }
1177
1178 /*
1179  * This allocation function is useful in the context where vma is irrelevant.
1180  * E.g. soft-offlining uses this function because it only cares physical
1181  * address of error page.
1182  */
1183 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1184 {
1185         struct page *page = NULL;
1186
1187         spin_lock(&hugetlb_lock);
1188         if (h->free_huge_pages - h->resv_huge_pages > 0)
1189                 page = dequeue_huge_page_node(h, nid);
1190         spin_unlock(&hugetlb_lock);
1191
1192         if (!page)
1193                 page = alloc_buddy_huge_page(h, nid);
1194
1195         return page;
1196 }
1197
1198 /*
1199  * Increase the hugetlb pool such that it can accommodate a reservation
1200  * of size 'delta'.
1201  */
1202 static int gather_surplus_pages(struct hstate *h, int delta)
1203 {
1204         struct list_head surplus_list;
1205         struct page *page, *tmp;
1206         int ret, i;
1207         int needed, allocated;
1208         bool alloc_ok = true;
1209
1210         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1211         if (needed <= 0) {
1212                 h->resv_huge_pages += delta;
1213                 return 0;
1214         }
1215
1216         allocated = 0;
1217         INIT_LIST_HEAD(&surplus_list);
1218
1219         ret = -ENOMEM;
1220 retry:
1221         spin_unlock(&hugetlb_lock);
1222         for (i = 0; i < needed; i++) {
1223                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1224                 if (!page) {
1225                         alloc_ok = false;
1226                         break;
1227                 }
1228                 list_add(&page->lru, &surplus_list);
1229         }
1230         allocated += i;
1231
1232         /*
1233          * After retaking hugetlb_lock, we need to recalculate 'needed'
1234          * because either resv_huge_pages or free_huge_pages may have changed.
1235          */
1236         spin_lock(&hugetlb_lock);
1237         needed = (h->resv_huge_pages + delta) -
1238                         (h->free_huge_pages + allocated);
1239         if (needed > 0) {
1240                 if (alloc_ok)
1241                         goto retry;
1242                 /*
1243                  * We were not able to allocate enough pages to
1244                  * satisfy the entire reservation so we free what
1245                  * we've allocated so far.
1246                  */
1247                 goto free;
1248         }
1249         /*
1250          * The surplus_list now contains _at_least_ the number of extra pages
1251          * needed to accommodate the reservation.  Add the appropriate number
1252          * of pages to the hugetlb pool and free the extras back to the buddy
1253          * allocator.  Commit the entire reservation here to prevent another
1254          * process from stealing the pages as they are added to the pool but
1255          * before they are reserved.
1256          */
1257         needed += allocated;
1258         h->resv_huge_pages += delta;
1259         ret = 0;
1260
1261         /* Free the needed pages to the hugetlb pool */
1262         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1263                 if ((--needed) < 0)
1264                         break;
1265                 /*
1266                  * This page is now managed by the hugetlb allocator and has
1267                  * no users -- drop the buddy allocator's reference.
1268                  */
1269                 put_page_testzero(page);
1270                 VM_BUG_ON_PAGE(page_count(page), page);
1271                 enqueue_huge_page(h, page);
1272         }
1273 free:
1274         spin_unlock(&hugetlb_lock);
1275
1276         /* Free unnecessary surplus pages to the buddy allocator */
1277         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1278                 put_page(page);
1279         spin_lock(&hugetlb_lock);
1280
1281         return ret;
1282 }
1283
1284 /*
1285  * When releasing a hugetlb pool reservation, any surplus pages that were
1286  * allocated to satisfy the reservation must be explicitly freed if they were
1287  * never used.
1288  * Called with hugetlb_lock held.
1289  */
1290 static void return_unused_surplus_pages(struct hstate *h,
1291                                         unsigned long unused_resv_pages)
1292 {
1293         unsigned long nr_pages;
1294
1295         /* Uncommit the reservation */
1296         h->resv_huge_pages -= unused_resv_pages;
1297
1298         /* Cannot return gigantic pages currently */
1299         if (hstate_is_gigantic(h))
1300                 return;
1301
1302         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1303
1304         /*
1305          * We want to release as many surplus pages as possible, spread
1306          * evenly across all nodes with memory. Iterate across these nodes
1307          * until we can no longer free unreserved surplus pages. This occurs
1308          * when the nodes with surplus pages have no free pages.
1309          * free_pool_huge_page() will balance the the freed pages across the
1310          * on-line nodes with memory and will handle the hstate accounting.
1311          */
1312         while (nr_pages--) {
1313                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1314                         break;
1315                 cond_resched_lock(&hugetlb_lock);
1316         }
1317 }
1318
1319 /*
1320  * Determine if the huge page at addr within the vma has an associated
1321  * reservation.  Where it does not we will need to logically increase
1322  * reservation and actually increase subpool usage before an allocation
1323  * can occur.  Where any new reservation would be required the
1324  * reservation change is prepared, but not committed.  Once the page
1325  * has been allocated from the subpool and instantiated the change should
1326  * be committed via vma_commit_reservation.  No action is required on
1327  * failure.
1328  */
1329 static long vma_needs_reservation(struct hstate *h,
1330                         struct vm_area_struct *vma, unsigned long addr)
1331 {
1332         struct resv_map *resv;
1333         pgoff_t idx;
1334         long chg;
1335
1336         resv = vma_resv_map(vma);
1337         if (!resv)
1338                 return 1;
1339
1340         idx = vma_hugecache_offset(h, vma, addr);
1341         chg = region_chg(resv, idx, idx + 1);
1342
1343         if (vma->vm_flags & VM_MAYSHARE)
1344                 return chg;
1345         else
1346                 return chg < 0 ? chg : 0;
1347 }
1348 static void vma_commit_reservation(struct hstate *h,
1349                         struct vm_area_struct *vma, unsigned long addr)
1350 {
1351         struct resv_map *resv;
1352         pgoff_t idx;
1353
1354         resv = vma_resv_map(vma);
1355         if (!resv)
1356                 return;
1357
1358         idx = vma_hugecache_offset(h, vma, addr);
1359         region_add(resv, idx, idx + 1);
1360 }
1361
1362 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1363                                     unsigned long addr, int avoid_reserve)
1364 {
1365         struct hugepage_subpool *spool = subpool_vma(vma);
1366         struct hstate *h = hstate_vma(vma);
1367         struct page *page;
1368         long chg;
1369         int ret, idx;
1370         struct hugetlb_cgroup *h_cg;
1371
1372         idx = hstate_index(h);
1373         /*
1374          * Processes that did not create the mapping will have no
1375          * reserves and will not have accounted against subpool
1376          * limit. Check that the subpool limit can be made before
1377          * satisfying the allocation MAP_NORESERVE mappings may also
1378          * need pages and subpool limit allocated allocated if no reserve
1379          * mapping overlaps.
1380          */
1381         chg = vma_needs_reservation(h, vma, addr);
1382         if (chg < 0)
1383                 return ERR_PTR(-ENOMEM);
1384         if (chg || avoid_reserve)
1385                 if (hugepage_subpool_get_pages(spool, 1))
1386                         return ERR_PTR(-ENOSPC);
1387
1388         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1389         if (ret)
1390                 goto out_subpool_put;
1391
1392         spin_lock(&hugetlb_lock);
1393         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1394         if (!page) {
1395                 spin_unlock(&hugetlb_lock);
1396                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1397                 if (!page)
1398                         goto out_uncharge_cgroup;
1399
1400                 spin_lock(&hugetlb_lock);
1401                 list_move(&page->lru, &h->hugepage_activelist);
1402                 /* Fall through */
1403         }
1404         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1405         spin_unlock(&hugetlb_lock);
1406
1407         set_page_private(page, (unsigned long)spool);
1408
1409         vma_commit_reservation(h, vma, addr);
1410         return page;
1411
1412 out_uncharge_cgroup:
1413         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1414 out_subpool_put:
1415         if (chg || avoid_reserve)
1416                 hugepage_subpool_put_pages(spool, 1);
1417         return ERR_PTR(-ENOSPC);
1418 }
1419
1420 /*
1421  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1422  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1423  * where no ERR_VALUE is expected to be returned.
1424  */
1425 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1426                                 unsigned long addr, int avoid_reserve)
1427 {
1428         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1429         if (IS_ERR(page))
1430                 page = NULL;
1431         return page;
1432 }
1433
1434 int __weak alloc_bootmem_huge_page(struct hstate *h)
1435 {
1436         struct huge_bootmem_page *m;
1437         int nr_nodes, node;
1438
1439         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1440                 void *addr;
1441
1442                 addr = memblock_virt_alloc_try_nid_nopanic(
1443                                 huge_page_size(h), huge_page_size(h),
1444                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
1445                 if (addr) {
1446                         /*
1447                          * Use the beginning of the huge page to store the
1448                          * huge_bootmem_page struct (until gather_bootmem
1449                          * puts them into the mem_map).
1450                          */
1451                         m = addr;
1452                         goto found;
1453                 }
1454         }
1455         return 0;
1456
1457 found:
1458         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1459         /* Put them into a private list first because mem_map is not up yet */
1460         list_add(&m->list, &huge_boot_pages);
1461         m->hstate = h;
1462         return 1;
1463 }
1464
1465 static void __init prep_compound_huge_page(struct page *page, int order)
1466 {
1467         if (unlikely(order > (MAX_ORDER - 1)))
1468                 prep_compound_gigantic_page(page, order);
1469         else
1470                 prep_compound_page(page, order);
1471 }
1472
1473 /* Put bootmem huge pages into the standard lists after mem_map is up */
1474 static void __init gather_bootmem_prealloc(void)
1475 {
1476         struct huge_bootmem_page *m;
1477
1478         list_for_each_entry(m, &huge_boot_pages, list) {
1479                 struct hstate *h = m->hstate;
1480                 struct page *page;
1481
1482 #ifdef CONFIG_HIGHMEM
1483                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1484                 memblock_free_late(__pa(m),
1485                                    sizeof(struct huge_bootmem_page));
1486 #else
1487                 page = virt_to_page(m);
1488 #endif
1489                 WARN_ON(page_count(page) != 1);
1490                 prep_compound_huge_page(page, h->order);
1491                 WARN_ON(PageReserved(page));
1492                 prep_new_huge_page(h, page, page_to_nid(page));
1493                 /*
1494                  * If we had gigantic hugepages allocated at boot time, we need
1495                  * to restore the 'stolen' pages to totalram_pages in order to
1496                  * fix confusing memory reports from free(1) and another
1497                  * side-effects, like CommitLimit going negative.
1498                  */
1499                 if (hstate_is_gigantic(h))
1500                         adjust_managed_page_count(page, 1 << h->order);
1501         }
1502 }
1503
1504 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1505 {
1506         unsigned long i;
1507
1508         for (i = 0; i < h->max_huge_pages; ++i) {
1509                 if (hstate_is_gigantic(h)) {
1510                         if (!alloc_bootmem_huge_page(h))
1511                                 break;
1512                 } else if (!alloc_fresh_huge_page(h,
1513                                          &node_states[N_MEMORY]))
1514                         break;
1515         }
1516         h->max_huge_pages = i;
1517 }
1518
1519 static void __init hugetlb_init_hstates(void)
1520 {
1521         struct hstate *h;
1522
1523         for_each_hstate(h) {
1524                 /* oversize hugepages were init'ed in early boot */
1525                 if (!hstate_is_gigantic(h))
1526                         hugetlb_hstate_alloc_pages(h);
1527         }
1528 }
1529
1530 static char * __init memfmt(char *buf, unsigned long n)
1531 {
1532         if (n >= (1UL << 30))
1533                 sprintf(buf, "%lu GB", n >> 30);
1534         else if (n >= (1UL << 20))
1535                 sprintf(buf, "%lu MB", n >> 20);
1536         else
1537                 sprintf(buf, "%lu KB", n >> 10);
1538         return buf;
1539 }
1540
1541 static void __init report_hugepages(void)
1542 {
1543         struct hstate *h;
1544
1545         for_each_hstate(h) {
1546                 char buf[32];
1547                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1548                         memfmt(buf, huge_page_size(h)),
1549                         h->free_huge_pages);
1550         }
1551 }
1552
1553 #ifdef CONFIG_HIGHMEM
1554 static void try_to_free_low(struct hstate *h, unsigned long count,
1555                                                 nodemask_t *nodes_allowed)
1556 {
1557         int i;
1558
1559         if (hstate_is_gigantic(h))
1560                 return;
1561
1562         for_each_node_mask(i, *nodes_allowed) {
1563                 struct page *page, *next;
1564                 struct list_head *freel = &h->hugepage_freelists[i];
1565                 list_for_each_entry_safe(page, next, freel, lru) {
1566                         if (count >= h->nr_huge_pages)
1567                                 return;
1568                         if (PageHighMem(page))
1569                                 continue;
1570                         list_del(&page->lru);
1571                         update_and_free_page(h, page);
1572                         h->free_huge_pages--;
1573                         h->free_huge_pages_node[page_to_nid(page)]--;
1574                 }
1575         }
1576 }
1577 #else
1578 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1579                                                 nodemask_t *nodes_allowed)
1580 {
1581 }
1582 #endif
1583
1584 /*
1585  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1586  * balanced by operating on them in a round-robin fashion.
1587  * Returns 1 if an adjustment was made.
1588  */
1589 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1590                                 int delta)
1591 {
1592         int nr_nodes, node;
1593
1594         VM_BUG_ON(delta != -1 && delta != 1);
1595
1596         if (delta < 0) {
1597                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1598                         if (h->surplus_huge_pages_node[node])
1599                                 goto found;
1600                 }
1601         } else {
1602                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1603                         if (h->surplus_huge_pages_node[node] <
1604                                         h->nr_huge_pages_node[node])
1605                                 goto found;
1606                 }
1607         }
1608         return 0;
1609
1610 found:
1611         h->surplus_huge_pages += delta;
1612         h->surplus_huge_pages_node[node] += delta;
1613         return 1;
1614 }
1615
1616 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1617 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1618                                                 nodemask_t *nodes_allowed)
1619 {
1620         unsigned long min_count, ret;
1621
1622         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1623                 return h->max_huge_pages;
1624
1625         /*
1626          * Increase the pool size
1627          * First take pages out of surplus state.  Then make up the
1628          * remaining difference by allocating fresh huge pages.
1629          *
1630          * We might race with alloc_buddy_huge_page() here and be unable
1631          * to convert a surplus huge page to a normal huge page. That is
1632          * not critical, though, it just means the overall size of the
1633          * pool might be one hugepage larger than it needs to be, but
1634          * within all the constraints specified by the sysctls.
1635          */
1636         spin_lock(&hugetlb_lock);
1637         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1638                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1639                         break;
1640         }
1641
1642         while (count > persistent_huge_pages(h)) {
1643                 /*
1644                  * If this allocation races such that we no longer need the
1645                  * page, free_huge_page will handle it by freeing the page
1646                  * and reducing the surplus.
1647                  */
1648                 spin_unlock(&hugetlb_lock);
1649                 if (hstate_is_gigantic(h))
1650                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
1651                 else
1652                         ret = alloc_fresh_huge_page(h, nodes_allowed);
1653                 spin_lock(&hugetlb_lock);
1654                 if (!ret)
1655                         goto out;
1656
1657                 /* Bail for signals. Probably ctrl-c from user */
1658                 if (signal_pending(current))
1659                         goto out;
1660         }
1661
1662         /*
1663          * Decrease the pool size
1664          * First return free pages to the buddy allocator (being careful
1665          * to keep enough around to satisfy reservations).  Then place
1666          * pages into surplus state as needed so the pool will shrink
1667          * to the desired size as pages become free.
1668          *
1669          * By placing pages into the surplus state independent of the
1670          * overcommit value, we are allowing the surplus pool size to
1671          * exceed overcommit. There are few sane options here. Since
1672          * alloc_buddy_huge_page() is checking the global counter,
1673          * though, we'll note that we're not allowed to exceed surplus
1674          * and won't grow the pool anywhere else. Not until one of the
1675          * sysctls are changed, or the surplus pages go out of use.
1676          */
1677         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1678         min_count = max(count, min_count);
1679         try_to_free_low(h, min_count, nodes_allowed);
1680         while (min_count < persistent_huge_pages(h)) {
1681                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1682                         break;
1683                 cond_resched_lock(&hugetlb_lock);
1684         }
1685         while (count < persistent_huge_pages(h)) {
1686                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1687                         break;
1688         }
1689 out:
1690         ret = persistent_huge_pages(h);
1691         spin_unlock(&hugetlb_lock);
1692         return ret;
1693 }
1694
1695 #define HSTATE_ATTR_RO(_name) \
1696         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1697
1698 #define HSTATE_ATTR(_name) \
1699         static struct kobj_attribute _name##_attr = \
1700                 __ATTR(_name, 0644, _name##_show, _name##_store)
1701
1702 static struct kobject *hugepages_kobj;
1703 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1704
1705 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1706
1707 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1708 {
1709         int i;
1710
1711         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1712                 if (hstate_kobjs[i] == kobj) {
1713                         if (nidp)
1714                                 *nidp = NUMA_NO_NODE;
1715                         return &hstates[i];
1716                 }
1717
1718         return kobj_to_node_hstate(kobj, nidp);
1719 }
1720
1721 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1722                                         struct kobj_attribute *attr, char *buf)
1723 {
1724         struct hstate *h;
1725         unsigned long nr_huge_pages;
1726         int nid;
1727
1728         h = kobj_to_hstate(kobj, &nid);
1729         if (nid == NUMA_NO_NODE)
1730                 nr_huge_pages = h->nr_huge_pages;
1731         else
1732                 nr_huge_pages = h->nr_huge_pages_node[nid];
1733
1734         return sprintf(buf, "%lu\n", nr_huge_pages);
1735 }
1736
1737 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1738                         struct kobject *kobj, struct kobj_attribute *attr,
1739                         const char *buf, size_t len)
1740 {
1741         int err;
1742         int nid;
1743         unsigned long count;
1744         struct hstate *h;
1745         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1746
1747         err = kstrtoul(buf, 10, &count);
1748         if (err)
1749                 goto out;
1750
1751         h = kobj_to_hstate(kobj, &nid);
1752         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
1753                 err = -EINVAL;
1754                 goto out;
1755         }
1756
1757         if (nid == NUMA_NO_NODE) {
1758                 /*
1759                  * global hstate attribute
1760                  */
1761                 if (!(obey_mempolicy &&
1762                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1763                         NODEMASK_FREE(nodes_allowed);
1764                         nodes_allowed = &node_states[N_MEMORY];
1765                 }
1766         } else if (nodes_allowed) {
1767                 /*
1768                  * per node hstate attribute: adjust count to global,
1769                  * but restrict alloc/free to the specified node.
1770                  */
1771                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1772                 init_nodemask_of_node(nodes_allowed, nid);
1773         } else
1774                 nodes_allowed = &node_states[N_MEMORY];
1775
1776         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1777
1778         if (nodes_allowed != &node_states[N_MEMORY])
1779                 NODEMASK_FREE(nodes_allowed);
1780
1781         return len;
1782 out:
1783         NODEMASK_FREE(nodes_allowed);
1784         return err;
1785 }
1786
1787 static ssize_t nr_hugepages_show(struct kobject *kobj,
1788                                        struct kobj_attribute *attr, char *buf)
1789 {
1790         return nr_hugepages_show_common(kobj, attr, buf);
1791 }
1792
1793 static ssize_t nr_hugepages_store(struct kobject *kobj,
1794                struct kobj_attribute *attr, const char *buf, size_t len)
1795 {
1796         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1797 }
1798 HSTATE_ATTR(nr_hugepages);
1799
1800 #ifdef CONFIG_NUMA
1801
1802 /*
1803  * hstate attribute for optionally mempolicy-based constraint on persistent
1804  * huge page alloc/free.
1805  */
1806 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1807                                        struct kobj_attribute *attr, char *buf)
1808 {
1809         return nr_hugepages_show_common(kobj, attr, buf);
1810 }
1811
1812 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1813                struct kobj_attribute *attr, const char *buf, size_t len)
1814 {
1815         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1816 }
1817 HSTATE_ATTR(nr_hugepages_mempolicy);
1818 #endif
1819
1820
1821 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1822                                         struct kobj_attribute *attr, char *buf)
1823 {
1824         struct hstate *h = kobj_to_hstate(kobj, NULL);
1825         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1826 }
1827
1828 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1829                 struct kobj_attribute *attr, const char *buf, size_t count)
1830 {
1831         int err;
1832         unsigned long input;
1833         struct hstate *h = kobj_to_hstate(kobj, NULL);
1834
1835         if (hstate_is_gigantic(h))
1836                 return -EINVAL;
1837
1838         err = kstrtoul(buf, 10, &input);
1839         if (err)
1840                 return err;
1841
1842         spin_lock(&hugetlb_lock);
1843         h->nr_overcommit_huge_pages = input;
1844         spin_unlock(&hugetlb_lock);
1845
1846         return count;
1847 }
1848 HSTATE_ATTR(nr_overcommit_hugepages);
1849
1850 static ssize_t free_hugepages_show(struct kobject *kobj,
1851                                         struct kobj_attribute *attr, char *buf)
1852 {
1853         struct hstate *h;
1854         unsigned long free_huge_pages;
1855         int nid;
1856
1857         h = kobj_to_hstate(kobj, &nid);
1858         if (nid == NUMA_NO_NODE)
1859                 free_huge_pages = h->free_huge_pages;
1860         else
1861                 free_huge_pages = h->free_huge_pages_node[nid];
1862
1863         return sprintf(buf, "%lu\n", free_huge_pages);
1864 }
1865 HSTATE_ATTR_RO(free_hugepages);
1866
1867 static ssize_t resv_hugepages_show(struct kobject *kobj,
1868                                         struct kobj_attribute *attr, char *buf)
1869 {
1870         struct hstate *h = kobj_to_hstate(kobj, NULL);
1871         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1872 }
1873 HSTATE_ATTR_RO(resv_hugepages);
1874
1875 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1876                                         struct kobj_attribute *attr, char *buf)
1877 {
1878         struct hstate *h;
1879         unsigned long surplus_huge_pages;
1880         int nid;
1881
1882         h = kobj_to_hstate(kobj, &nid);
1883         if (nid == NUMA_NO_NODE)
1884                 surplus_huge_pages = h->surplus_huge_pages;
1885         else
1886                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1887
1888         return sprintf(buf, "%lu\n", surplus_huge_pages);
1889 }
1890 HSTATE_ATTR_RO(surplus_hugepages);
1891
1892 static struct attribute *hstate_attrs[] = {
1893         &nr_hugepages_attr.attr,
1894         &nr_overcommit_hugepages_attr.attr,
1895         &free_hugepages_attr.attr,
1896         &resv_hugepages_attr.attr,
1897         &surplus_hugepages_attr.attr,
1898 #ifdef CONFIG_NUMA
1899         &nr_hugepages_mempolicy_attr.attr,
1900 #endif
1901         NULL,
1902 };
1903
1904 static struct attribute_group hstate_attr_group = {
1905         .attrs = hstate_attrs,
1906 };
1907
1908 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1909                                     struct kobject **hstate_kobjs,
1910                                     struct attribute_group *hstate_attr_group)
1911 {
1912         int retval;
1913         int hi = hstate_index(h);
1914
1915         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1916         if (!hstate_kobjs[hi])
1917                 return -ENOMEM;
1918
1919         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1920         if (retval)
1921                 kobject_put(hstate_kobjs[hi]);
1922
1923         return retval;
1924 }
1925
1926 static void __init hugetlb_sysfs_init(void)
1927 {
1928         struct hstate *h;
1929         int err;
1930
1931         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1932         if (!hugepages_kobj)
1933                 return;
1934
1935         for_each_hstate(h) {
1936                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1937                                          hstate_kobjs, &hstate_attr_group);
1938                 if (err)
1939                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
1940         }
1941 }
1942
1943 #ifdef CONFIG_NUMA
1944
1945 /*
1946  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1947  * with node devices in node_devices[] using a parallel array.  The array
1948  * index of a node device or _hstate == node id.
1949  * This is here to avoid any static dependency of the node device driver, in
1950  * the base kernel, on the hugetlb module.
1951  */
1952 struct node_hstate {
1953         struct kobject          *hugepages_kobj;
1954         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1955 };
1956 struct node_hstate node_hstates[MAX_NUMNODES];
1957
1958 /*
1959  * A subset of global hstate attributes for node devices
1960  */
1961 static struct attribute *per_node_hstate_attrs[] = {
1962         &nr_hugepages_attr.attr,
1963         &free_hugepages_attr.attr,
1964         &surplus_hugepages_attr.attr,
1965         NULL,
1966 };
1967
1968 static struct attribute_group per_node_hstate_attr_group = {
1969         .attrs = per_node_hstate_attrs,
1970 };
1971
1972 /*
1973  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1974  * Returns node id via non-NULL nidp.
1975  */
1976 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1977 {
1978         int nid;
1979
1980         for (nid = 0; nid < nr_node_ids; nid++) {
1981                 struct node_hstate *nhs = &node_hstates[nid];
1982                 int i;
1983                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1984                         if (nhs->hstate_kobjs[i] == kobj) {
1985                                 if (nidp)
1986                                         *nidp = nid;
1987                                 return &hstates[i];
1988                         }
1989         }
1990
1991         BUG();
1992         return NULL;
1993 }
1994
1995 /*
1996  * Unregister hstate attributes from a single node device.
1997  * No-op if no hstate attributes attached.
1998  */
1999 static void hugetlb_unregister_node(struct node *node)
2000 {
2001         struct hstate *h;
2002         struct node_hstate *nhs = &node_hstates[node->dev.id];
2003
2004         if (!nhs->hugepages_kobj)
2005                 return;         /* no hstate attributes */
2006
2007         for_each_hstate(h) {
2008                 int idx = hstate_index(h);
2009                 if (nhs->hstate_kobjs[idx]) {
2010                         kobject_put(nhs->hstate_kobjs[idx]);
2011                         nhs->hstate_kobjs[idx] = NULL;
2012                 }
2013         }
2014
2015         kobject_put(nhs->hugepages_kobj);
2016         nhs->hugepages_kobj = NULL;
2017 }
2018
2019 /*
2020  * hugetlb module exit:  unregister hstate attributes from node devices
2021  * that have them.
2022  */
2023 static void hugetlb_unregister_all_nodes(void)
2024 {
2025         int nid;
2026
2027         /*
2028          * disable node device registrations.
2029          */
2030         register_hugetlbfs_with_node(NULL, NULL);
2031
2032         /*
2033          * remove hstate attributes from any nodes that have them.
2034          */
2035         for (nid = 0; nid < nr_node_ids; nid++)
2036                 hugetlb_unregister_node(node_devices[nid]);
2037 }
2038
2039 /*
2040  * Register hstate attributes for a single node device.
2041  * No-op if attributes already registered.
2042  */
2043 static void hugetlb_register_node(struct node *node)
2044 {
2045         struct hstate *h;
2046         struct node_hstate *nhs = &node_hstates[node->dev.id];
2047         int err;
2048
2049         if (nhs->hugepages_kobj)
2050                 return;         /* already allocated */
2051
2052         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2053                                                         &node->dev.kobj);
2054         if (!nhs->hugepages_kobj)
2055                 return;
2056
2057         for_each_hstate(h) {
2058                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2059                                                 nhs->hstate_kobjs,
2060                                                 &per_node_hstate_attr_group);
2061                 if (err) {
2062                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2063                                 h->name, node->dev.id);
2064                         hugetlb_unregister_node(node);
2065                         break;
2066                 }
2067         }
2068 }
2069
2070 /*
2071  * hugetlb init time:  register hstate attributes for all registered node
2072  * devices of nodes that have memory.  All on-line nodes should have
2073  * registered their associated device by this time.
2074  */
2075 static void hugetlb_register_all_nodes(void)
2076 {
2077         int nid;
2078
2079         for_each_node_state(nid, N_MEMORY) {
2080                 struct node *node = node_devices[nid];
2081                 if (node->dev.id == nid)
2082                         hugetlb_register_node(node);
2083         }
2084
2085         /*
2086          * Let the node device driver know we're here so it can
2087          * [un]register hstate attributes on node hotplug.
2088          */
2089         register_hugetlbfs_with_node(hugetlb_register_node,
2090                                      hugetlb_unregister_node);
2091 }
2092 #else   /* !CONFIG_NUMA */
2093
2094 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2095 {
2096         BUG();
2097         if (nidp)
2098                 *nidp = -1;
2099         return NULL;
2100 }
2101
2102 static void hugetlb_unregister_all_nodes(void) { }
2103
2104 static void hugetlb_register_all_nodes(void) { }
2105
2106 #endif
2107
2108 static void __exit hugetlb_exit(void)
2109 {
2110         struct hstate *h;
2111
2112         hugetlb_unregister_all_nodes();
2113
2114         for_each_hstate(h) {
2115                 kobject_put(hstate_kobjs[hstate_index(h)]);
2116         }
2117
2118         kobject_put(hugepages_kobj);
2119         kfree(htlb_fault_mutex_table);
2120 }
2121 module_exit(hugetlb_exit);
2122
2123 static int __init hugetlb_init(void)
2124 {
2125         int i;
2126
2127         if (!hugepages_supported())
2128                 return 0;
2129
2130         if (!size_to_hstate(default_hstate_size)) {
2131                 default_hstate_size = HPAGE_SIZE;
2132                 if (!size_to_hstate(default_hstate_size))
2133                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2134         }
2135         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2136         if (default_hstate_max_huge_pages)
2137                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2138
2139         hugetlb_init_hstates();
2140         gather_bootmem_prealloc();
2141         report_hugepages();
2142
2143         hugetlb_sysfs_init();
2144         hugetlb_register_all_nodes();
2145         hugetlb_cgroup_file_init();
2146
2147 #ifdef CONFIG_SMP
2148         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2149 #else
2150         num_fault_mutexes = 1;
2151 #endif
2152         htlb_fault_mutex_table =
2153                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2154         BUG_ON(!htlb_fault_mutex_table);
2155
2156         for (i = 0; i < num_fault_mutexes; i++)
2157                 mutex_init(&htlb_fault_mutex_table[i]);
2158         return 0;
2159 }
2160 module_init(hugetlb_init);
2161
2162 /* Should be called on processing a hugepagesz=... option */
2163 void __init hugetlb_add_hstate(unsigned order)
2164 {
2165         struct hstate *h;
2166         unsigned long i;
2167
2168         if (size_to_hstate(PAGE_SIZE << order)) {
2169                 pr_warning("hugepagesz= specified twice, ignoring\n");
2170                 return;
2171         }
2172         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2173         BUG_ON(order == 0);
2174         h = &hstates[hugetlb_max_hstate++];
2175         h->order = order;
2176         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2177         h->nr_huge_pages = 0;
2178         h->free_huge_pages = 0;
2179         for (i = 0; i < MAX_NUMNODES; ++i)
2180                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2181         INIT_LIST_HEAD(&h->hugepage_activelist);
2182         h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2183         h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2184         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2185                                         huge_page_size(h)/1024);
2186
2187         parsed_hstate = h;
2188 }
2189
2190 static int __init hugetlb_nrpages_setup(char *s)
2191 {
2192         unsigned long *mhp;
2193         static unsigned long *last_mhp;
2194
2195         /*
2196          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2197          * so this hugepages= parameter goes to the "default hstate".
2198          */
2199         if (!hugetlb_max_hstate)
2200                 mhp = &default_hstate_max_huge_pages;
2201         else
2202                 mhp = &parsed_hstate->max_huge_pages;
2203
2204         if (mhp == last_mhp) {
2205                 pr_warning("hugepages= specified twice without "
2206                            "interleaving hugepagesz=, ignoring\n");
2207                 return 1;
2208         }
2209
2210         if (sscanf(s, "%lu", mhp) <= 0)
2211                 *mhp = 0;
2212
2213         /*
2214          * Global state is always initialized later in hugetlb_init.
2215          * But we need to allocate >= MAX_ORDER hstates here early to still
2216          * use the bootmem allocator.
2217          */
2218         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2219                 hugetlb_hstate_alloc_pages(parsed_hstate);
2220
2221         last_mhp = mhp;
2222
2223         return 1;
2224 }
2225 __setup("hugepages=", hugetlb_nrpages_setup);
2226
2227 static int __init hugetlb_default_setup(char *s)
2228 {
2229         default_hstate_size = memparse(s, &s);
2230         return 1;
2231 }
2232 __setup("default_hugepagesz=", hugetlb_default_setup);
2233
2234 static unsigned int cpuset_mems_nr(unsigned int *array)
2235 {
2236         int node;
2237         unsigned int nr = 0;
2238
2239         for_each_node_mask(node, cpuset_current_mems_allowed)
2240                 nr += array[node];
2241
2242         return nr;
2243 }
2244
2245 #ifdef CONFIG_SYSCTL
2246 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2247                          struct ctl_table *table, int write,
2248                          void __user *buffer, size_t *length, loff_t *ppos)
2249 {
2250         struct hstate *h = &default_hstate;
2251         unsigned long tmp;
2252         int ret;
2253
2254         if (!hugepages_supported())
2255                 return -ENOTSUPP;
2256
2257         tmp = h->max_huge_pages;
2258
2259         if (write && hstate_is_gigantic(h) && !gigantic_page_supported())
2260                 return -EINVAL;
2261
2262         table->data = &tmp;
2263         table->maxlen = sizeof(unsigned long);
2264         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2265         if (ret)
2266                 goto out;
2267
2268         if (write) {
2269                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
2270                                                 GFP_KERNEL | __GFP_NORETRY);
2271                 if (!(obey_mempolicy &&
2272                                init_nodemask_of_mempolicy(nodes_allowed))) {
2273                         NODEMASK_FREE(nodes_allowed);
2274                         nodes_allowed = &node_states[N_MEMORY];
2275                 }
2276                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2277
2278                 if (nodes_allowed != &node_states[N_MEMORY])
2279                         NODEMASK_FREE(nodes_allowed);
2280         }
2281 out:
2282         return ret;
2283 }
2284
2285 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2286                           void __user *buffer, size_t *length, loff_t *ppos)
2287 {
2288
2289         return hugetlb_sysctl_handler_common(false, table, write,
2290                                                         buffer, length, ppos);
2291 }
2292
2293 #ifdef CONFIG_NUMA
2294 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2295                           void __user *buffer, size_t *length, loff_t *ppos)
2296 {
2297         return hugetlb_sysctl_handler_common(true, table, write,
2298                                                         buffer, length, ppos);
2299 }
2300 #endif /* CONFIG_NUMA */
2301
2302 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2303                         void __user *buffer,
2304                         size_t *length, loff_t *ppos)
2305 {
2306         struct hstate *h = &default_hstate;
2307         unsigned long tmp;
2308         int ret;
2309
2310         if (!hugepages_supported())
2311                 return -ENOTSUPP;
2312
2313         tmp = h->nr_overcommit_huge_pages;
2314
2315         if (write && hstate_is_gigantic(h))
2316                 return -EINVAL;
2317
2318         table->data = &tmp;
2319         table->maxlen = sizeof(unsigned long);
2320         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2321         if (ret)
2322                 goto out;
2323
2324         if (write) {
2325                 spin_lock(&hugetlb_lock);
2326                 h->nr_overcommit_huge_pages = tmp;
2327                 spin_unlock(&hugetlb_lock);
2328         }
2329 out:
2330         return ret;
2331 }
2332
2333 #endif /* CONFIG_SYSCTL */
2334
2335 void hugetlb_report_meminfo(struct seq_file *m)
2336 {
2337         struct hstate *h = &default_hstate;
2338         if (!hugepages_supported())
2339                 return;
2340         seq_printf(m,
2341                         "HugePages_Total:   %5lu\n"
2342                         "HugePages_Free:    %5lu\n"
2343                         "HugePages_Rsvd:    %5lu\n"
2344                         "HugePages_Surp:    %5lu\n"
2345                         "Hugepagesize:   %8lu kB\n",
2346                         h->nr_huge_pages,
2347                         h->free_huge_pages,
2348                         h->resv_huge_pages,
2349                         h->surplus_huge_pages,
2350                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2351 }
2352
2353 int hugetlb_report_node_meminfo(int nid, char *buf)
2354 {
2355         struct hstate *h = &default_hstate;
2356         if (!hugepages_supported())
2357                 return 0;
2358         return sprintf(buf,
2359                 "Node %d HugePages_Total: %5u\n"
2360                 "Node %d HugePages_Free:  %5u\n"
2361                 "Node %d HugePages_Surp:  %5u\n",
2362                 nid, h->nr_huge_pages_node[nid],
2363                 nid, h->free_huge_pages_node[nid],
2364                 nid, h->surplus_huge_pages_node[nid]);
2365 }
2366
2367 void hugetlb_show_meminfo(void)
2368 {
2369         struct hstate *h;
2370         int nid;
2371
2372         if (!hugepages_supported())
2373                 return;
2374
2375         for_each_node_state(nid, N_MEMORY)
2376                 for_each_hstate(h)
2377                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2378                                 nid,
2379                                 h->nr_huge_pages_node[nid],
2380                                 h->free_huge_pages_node[nid],
2381                                 h->surplus_huge_pages_node[nid],
2382                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2383 }
2384
2385 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2386 unsigned long hugetlb_total_pages(void)
2387 {
2388         struct hstate *h;
2389         unsigned long nr_total_pages = 0;
2390
2391         for_each_hstate(h)
2392                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2393         return nr_total_pages;
2394 }
2395
2396 static int hugetlb_acct_memory(struct hstate *h, long delta)
2397 {
2398         int ret = -ENOMEM;
2399
2400         spin_lock(&hugetlb_lock);
2401         /*
2402          * When cpuset is configured, it breaks the strict hugetlb page
2403          * reservation as the accounting is done on a global variable. Such
2404          * reservation is completely rubbish in the presence of cpuset because
2405          * the reservation is not checked against page availability for the
2406          * current cpuset. Application can still potentially OOM'ed by kernel
2407          * with lack of free htlb page in cpuset that the task is in.
2408          * Attempt to enforce strict accounting with cpuset is almost
2409          * impossible (or too ugly) because cpuset is too fluid that
2410          * task or memory node can be dynamically moved between cpusets.
2411          *
2412          * The change of semantics for shared hugetlb mapping with cpuset is
2413          * undesirable. However, in order to preserve some of the semantics,
2414          * we fall back to check against current free page availability as
2415          * a best attempt and hopefully to minimize the impact of changing
2416          * semantics that cpuset has.
2417          */
2418         if (delta > 0) {
2419                 if (gather_surplus_pages(h, delta) < 0)
2420                         goto out;
2421
2422                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2423                         return_unused_surplus_pages(h, delta);
2424                         goto out;
2425                 }
2426         }
2427
2428         ret = 0;
2429         if (delta < 0)
2430                 return_unused_surplus_pages(h, (unsigned long) -delta);
2431
2432 out:
2433         spin_unlock(&hugetlb_lock);
2434         return ret;
2435 }
2436
2437 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2438 {
2439         struct resv_map *resv = vma_resv_map(vma);
2440
2441         /*
2442          * This new VMA should share its siblings reservation map if present.
2443          * The VMA will only ever have a valid reservation map pointer where
2444          * it is being copied for another still existing VMA.  As that VMA
2445          * has a reference to the reservation map it cannot disappear until
2446          * after this open call completes.  It is therefore safe to take a
2447          * new reference here without additional locking.
2448          */
2449         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2450                 kref_get(&resv->refs);
2451 }
2452
2453 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2454 {
2455         struct hstate *h = hstate_vma(vma);
2456         struct resv_map *resv = vma_resv_map(vma);
2457         struct hugepage_subpool *spool = subpool_vma(vma);
2458         unsigned long reserve, start, end;
2459
2460         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2461                 return;
2462
2463         start = vma_hugecache_offset(h, vma, vma->vm_start);
2464         end = vma_hugecache_offset(h, vma, vma->vm_end);
2465
2466         reserve = (end - start) - region_count(resv, start, end);
2467
2468         kref_put(&resv->refs, resv_map_release);
2469
2470         if (reserve) {
2471                 hugetlb_acct_memory(h, -reserve);
2472                 hugepage_subpool_put_pages(spool, reserve);
2473         }
2474 }
2475
2476 /*
2477  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2478  * handle_mm_fault() to try to instantiate regular-sized pages in the
2479  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2480  * this far.
2481  */
2482 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2483 {
2484         BUG();
2485         return 0;
2486 }
2487
2488 const struct vm_operations_struct hugetlb_vm_ops = {
2489         .fault = hugetlb_vm_op_fault,
2490         .open = hugetlb_vm_op_open,
2491         .close = hugetlb_vm_op_close,
2492 };
2493
2494 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2495                                 int writable)
2496 {
2497         pte_t entry;
2498
2499         if (writable) {
2500                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2501                                          vma->vm_page_prot)));
2502         } else {
2503                 entry = huge_pte_wrprotect(mk_huge_pte(page,
2504                                            vma->vm_page_prot));
2505         }
2506         entry = pte_mkyoung(entry);
2507         entry = pte_mkhuge(entry);
2508         entry = arch_make_huge_pte(entry, vma, page, writable);
2509
2510         return entry;
2511 }
2512
2513 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2514                                    unsigned long address, pte_t *ptep)
2515 {
2516         pte_t entry;
2517
2518         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2519         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2520                 update_mmu_cache(vma, address, ptep);
2521 }
2522
2523 static int is_hugetlb_entry_migration(pte_t pte)
2524 {
2525         swp_entry_t swp;
2526
2527         if (huge_pte_none(pte) || pte_present(pte))
2528                 return 0;
2529         swp = pte_to_swp_entry(pte);
2530         if (non_swap_entry(swp) && is_migration_entry(swp))
2531                 return 1;
2532         else
2533                 return 0;
2534 }
2535
2536 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2537 {
2538         swp_entry_t swp;
2539
2540         if (huge_pte_none(pte) || pte_present(pte))
2541                 return 0;
2542         swp = pte_to_swp_entry(pte);
2543         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2544                 return 1;
2545         else
2546                 return 0;
2547 }
2548
2549 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2550                             struct vm_area_struct *vma)
2551 {
2552         pte_t *src_pte, *dst_pte, entry;
2553         struct page *ptepage;
2554         unsigned long addr;
2555         int cow;
2556         struct hstate *h = hstate_vma(vma);
2557         unsigned long sz = huge_page_size(h);
2558         unsigned long mmun_start;       /* For mmu_notifiers */
2559         unsigned long mmun_end;         /* For mmu_notifiers */
2560         int ret = 0;
2561
2562         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2563
2564         mmun_start = vma->vm_start;
2565         mmun_end = vma->vm_end;
2566         if (cow)
2567                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2568
2569         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2570                 spinlock_t *src_ptl, *dst_ptl;
2571                 src_pte = huge_pte_offset(src, addr);
2572                 if (!src_pte)
2573                         continue;
2574                 dst_pte = huge_pte_alloc(dst, addr, sz);
2575                 if (!dst_pte) {
2576                         ret = -ENOMEM;
2577                         break;
2578                 }
2579
2580                 /* If the pagetables are shared don't copy or take references */
2581                 if (dst_pte == src_pte)
2582                         continue;
2583
2584                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2585                 src_ptl = huge_pte_lockptr(h, src, src_pte);
2586                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
2587                 entry = huge_ptep_get(src_pte);
2588                 if (huge_pte_none(entry)) { /* skip none entry */
2589                         ;
2590                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
2591                                     is_hugetlb_entry_hwpoisoned(entry))) {
2592                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
2593
2594                         if (is_write_migration_entry(swp_entry) && cow) {
2595                                 /*
2596                                  * COW mappings require pages in both
2597                                  * parent and child to be set to read.
2598                                  */
2599                                 make_migration_entry_read(&swp_entry);
2600                                 entry = swp_entry_to_pte(swp_entry);
2601                                 set_huge_pte_at(src, addr, src_pte, entry);
2602                         }
2603                         set_huge_pte_at(dst, addr, dst_pte, entry);
2604                 } else {
2605                         if (cow)
2606                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2607                         ptepage = pte_page(entry);
2608                         get_page(ptepage);
2609                         page_dup_rmap(ptepage);
2610                         set_huge_pte_at(dst, addr, dst_pte, entry);
2611                 }
2612                 spin_unlock(src_ptl);
2613                 spin_unlock(dst_ptl);
2614         }
2615
2616         if (cow)
2617                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
2618
2619         return ret;
2620 }
2621
2622 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2623                             unsigned long start, unsigned long end,
2624                             struct page *ref_page)
2625 {
2626         int force_flush = 0;
2627         struct mm_struct *mm = vma->vm_mm;
2628         unsigned long address;
2629         pte_t *ptep;
2630         pte_t pte;
2631         spinlock_t *ptl;
2632         struct page *page;
2633         struct hstate *h = hstate_vma(vma);
2634         unsigned long sz = huge_page_size(h);
2635         const unsigned long mmun_start = start; /* For mmu_notifiers */
2636         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
2637
2638         WARN_ON(!is_vm_hugetlb_page(vma));
2639         BUG_ON(start & ~huge_page_mask(h));
2640         BUG_ON(end & ~huge_page_mask(h));
2641
2642         tlb_start_vma(tlb, vma);
2643         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2644 again:
2645         for (address = start; address < end; address += sz) {
2646                 ptep = huge_pte_offset(mm, address);
2647                 if (!ptep)
2648                         continue;
2649
2650                 ptl = huge_pte_lock(h, mm, ptep);
2651                 if (huge_pmd_unshare(mm, &address, ptep))
2652                         goto unlock;
2653
2654                 pte = huge_ptep_get(ptep);
2655                 if (huge_pte_none(pte))
2656                         goto unlock;
2657
2658                 /*
2659                  * HWPoisoned hugepage is already unmapped and dropped reference
2660                  */
2661                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2662                         huge_pte_clear(mm, address, ptep);
2663                         goto unlock;
2664                 }
2665
2666                 page = pte_page(pte);
2667                 /*
2668                  * If a reference page is supplied, it is because a specific
2669                  * page is being unmapped, not a range. Ensure the page we
2670                  * are about to unmap is the actual page of interest.
2671                  */
2672                 if (ref_page) {
2673                         if (page != ref_page)
2674                                 goto unlock;
2675
2676                         /*
2677                          * Mark the VMA as having unmapped its page so that
2678                          * future faults in this VMA will fail rather than
2679                          * looking like data was lost
2680                          */
2681                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2682                 }
2683
2684                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2685                 tlb_remove_tlb_entry(tlb, ptep, address);
2686                 if (huge_pte_dirty(pte))
2687                         set_page_dirty(page);
2688
2689                 page_remove_rmap(page);
2690                 force_flush = !__tlb_remove_page(tlb, page);
2691                 if (force_flush) {
2692                         spin_unlock(ptl);
2693                         break;
2694                 }
2695                 /* Bail out after unmapping reference page if supplied */
2696                 if (ref_page) {
2697                         spin_unlock(ptl);
2698                         break;
2699                 }
2700 unlock:
2701                 spin_unlock(ptl);
2702         }
2703         /*
2704          * mmu_gather ran out of room to batch pages, we break out of
2705          * the PTE lock to avoid doing the potential expensive TLB invalidate
2706          * and page-free while holding it.
2707          */
2708         if (force_flush) {
2709                 force_flush = 0;
2710                 tlb_flush_mmu(tlb);
2711                 if (address < end && !ref_page)
2712                         goto again;
2713         }
2714         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2715         tlb_end_vma(tlb, vma);
2716 }
2717
2718 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2719                           struct vm_area_struct *vma, unsigned long start,
2720                           unsigned long end, struct page *ref_page)
2721 {
2722         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
2723
2724         /*
2725          * Clear this flag so that x86's huge_pmd_share page_table_shareable
2726          * test will fail on a vma being torn down, and not grab a page table
2727          * on its way out.  We're lucky that the flag has such an appropriate
2728          * name, and can in fact be safely cleared here. We could clear it
2729          * before the __unmap_hugepage_range above, but all that's necessary
2730          * is to clear it before releasing the i_mmap_mutex. This works
2731          * because in the context this is called, the VMA is about to be
2732          * destroyed and the i_mmap_mutex is held.
2733          */
2734         vma->vm_flags &= ~VM_MAYSHARE;
2735 }
2736
2737 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2738                           unsigned long end, struct page *ref_page)
2739 {
2740         struct mm_struct *mm;
2741         struct mmu_gather tlb;
2742
2743         mm = vma->vm_mm;
2744
2745         tlb_gather_mmu(&tlb, mm, start, end);
2746         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2747         tlb_finish_mmu(&tlb, start, end);
2748 }
2749
2750 /*
2751  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2752  * mappping it owns the reserve page for. The intention is to unmap the page
2753  * from other VMAs and let the children be SIGKILLed if they are faulting the
2754  * same region.
2755  */
2756 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2757                                 struct page *page, unsigned long address)
2758 {
2759         struct hstate *h = hstate_vma(vma);
2760         struct vm_area_struct *iter_vma;
2761         struct address_space *mapping;
2762         pgoff_t pgoff;
2763
2764         /*
2765          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2766          * from page cache lookup which is in HPAGE_SIZE units.
2767          */
2768         address = address & huge_page_mask(h);
2769         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2770                         vma->vm_pgoff;
2771         mapping = file_inode(vma->vm_file)->i_mapping;
2772
2773         /*
2774          * Take the mapping lock for the duration of the table walk. As
2775          * this mapping should be shared between all the VMAs,
2776          * __unmap_hugepage_range() is called as the lock is already held
2777          */
2778         mutex_lock(&mapping->i_mmap_mutex);
2779         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2780                 /* Do not unmap the current VMA */
2781                 if (iter_vma == vma)
2782                         continue;
2783
2784                 /*
2785                  * Unmap the page from other VMAs without their own reserves.
2786                  * They get marked to be SIGKILLed if they fault in these
2787                  * areas. This is because a future no-page fault on this VMA
2788                  * could insert a zeroed page instead of the data existing
2789                  * from the time of fork. This would look like data corruption
2790                  */
2791                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2792                         unmap_hugepage_range(iter_vma, address,
2793                                              address + huge_page_size(h), page);
2794         }
2795         mutex_unlock(&mapping->i_mmap_mutex);
2796
2797         return 1;
2798 }
2799
2800 /*
2801  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2802  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2803  * cannot race with other handlers or page migration.
2804  * Keep the pte_same checks anyway to make transition from the mutex easier.
2805  */
2806 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2807                         unsigned long address, pte_t *ptep, pte_t pte,
2808                         struct page *pagecache_page, spinlock_t *ptl)
2809 {
2810         struct hstate *h = hstate_vma(vma);
2811         struct page *old_page, *new_page;
2812         int outside_reserve = 0;
2813         unsigned long mmun_start;       /* For mmu_notifiers */
2814         unsigned long mmun_end;         /* For mmu_notifiers */
2815
2816         old_page = pte_page(pte);
2817
2818 retry_avoidcopy:
2819         /* If no-one else is actually using this page, avoid the copy
2820          * and just make the page writable */
2821         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2822                 page_move_anon_rmap(old_page, vma, address);
2823                 set_huge_ptep_writable(vma, address, ptep);
2824                 return 0;
2825         }
2826
2827         /*
2828          * If the process that created a MAP_PRIVATE mapping is about to
2829          * perform a COW due to a shared page count, attempt to satisfy
2830          * the allocation without using the existing reserves. The pagecache
2831          * page is used to determine if the reserve at this address was
2832          * consumed or not. If reserves were used, a partial faulted mapping
2833          * at the time of fork() could consume its reserves on COW instead
2834          * of the full address range.
2835          */
2836         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2837                         old_page != pagecache_page)
2838                 outside_reserve = 1;
2839
2840         page_cache_get(old_page);
2841
2842         /* Drop page table lock as buddy allocator may be called */
2843         spin_unlock(ptl);
2844         new_page = alloc_huge_page(vma, address, outside_reserve);
2845
2846         if (IS_ERR(new_page)) {
2847                 long err = PTR_ERR(new_page);
2848                 page_cache_release(old_page);
2849
2850                 /*
2851                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2852                  * it is due to references held by a child and an insufficient
2853                  * huge page pool. To guarantee the original mappers
2854                  * reliability, unmap the page from child processes. The child
2855                  * may get SIGKILLed if it later faults.
2856                  */
2857                 if (outside_reserve) {
2858                         BUG_ON(huge_pte_none(pte));
2859                         if (unmap_ref_private(mm, vma, old_page, address)) {
2860                                 BUG_ON(huge_pte_none(pte));
2861                                 spin_lock(ptl);
2862                                 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2863                                 if (likely(ptep &&
2864                                            pte_same(huge_ptep_get(ptep), pte)))
2865                                         goto retry_avoidcopy;
2866                                 /*
2867                                  * race occurs while re-acquiring page table
2868                                  * lock, and our job is done.
2869                                  */
2870                                 return 0;
2871                         }
2872                         WARN_ON_ONCE(1);
2873                 }
2874
2875                 /* Caller expects lock to be held */
2876                 spin_lock(ptl);
2877                 if (err == -ENOMEM)
2878                         return VM_FAULT_OOM;
2879                 else
2880                         return VM_FAULT_SIGBUS;
2881         }
2882
2883         /*
2884          * When the original hugepage is shared one, it does not have
2885          * anon_vma prepared.
2886          */
2887         if (unlikely(anon_vma_prepare(vma))) {
2888                 page_cache_release(new_page);
2889                 page_cache_release(old_page);
2890                 /* Caller expects lock to be held */
2891                 spin_lock(ptl);
2892                 return VM_FAULT_OOM;
2893         }
2894
2895         copy_user_huge_page(new_page, old_page, address, vma,
2896                             pages_per_huge_page(h));
2897         __SetPageUptodate(new_page);
2898
2899         mmun_start = address & huge_page_mask(h);
2900         mmun_end = mmun_start + huge_page_size(h);
2901         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2902         /*
2903          * Retake the page table lock to check for racing updates
2904          * before the page tables are altered
2905          */
2906         spin_lock(ptl);
2907         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2908         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
2909                 ClearPagePrivate(new_page);
2910
2911                 /* Break COW */
2912                 huge_ptep_clear_flush(vma, address, ptep);
2913                 set_huge_pte_at(mm, address, ptep,
2914                                 make_huge_pte(vma, new_page, 1));
2915                 page_remove_rmap(old_page);
2916                 hugepage_add_new_anon_rmap(new_page, vma, address);
2917                 /* Make the old page be freed below */
2918                 new_page = old_page;
2919         }
2920         spin_unlock(ptl);
2921         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2922         page_cache_release(new_page);
2923         page_cache_release(old_page);
2924
2925         /* Caller expects lock to be held */
2926         spin_lock(ptl);
2927         return 0;
2928 }
2929
2930 /* Return the pagecache page at a given address within a VMA */
2931 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2932                         struct vm_area_struct *vma, unsigned long address)
2933 {
2934         struct address_space *mapping;
2935         pgoff_t idx;
2936
2937         mapping = vma->vm_file->f_mapping;
2938         idx = vma_hugecache_offset(h, vma, address);
2939
2940         return find_lock_page(mapping, idx);
2941 }
2942
2943 /*
2944  * Return whether there is a pagecache page to back given address within VMA.
2945  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2946  */
2947 static bool hugetlbfs_pagecache_present(struct hstate *h,
2948                         struct vm_area_struct *vma, unsigned long address)
2949 {
2950         struct address_space *mapping;
2951         pgoff_t idx;
2952         struct page *page;
2953
2954         mapping = vma->vm_file->f_mapping;
2955         idx = vma_hugecache_offset(h, vma, address);
2956
2957         page = find_get_page(mapping, idx);
2958         if (page)
2959                 put_page(page);
2960         return page != NULL;
2961 }
2962
2963 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2964                            struct address_space *mapping, pgoff_t idx,
2965                            unsigned long address, pte_t *ptep, unsigned int flags)
2966 {
2967         struct hstate *h = hstate_vma(vma);
2968         int ret = VM_FAULT_SIGBUS;
2969         int anon_rmap = 0;
2970         unsigned long size;
2971         struct page *page;
2972         pte_t new_pte;
2973         spinlock_t *ptl;
2974
2975         /*
2976          * Currently, we are forced to kill the process in the event the
2977          * original mapper has unmapped pages from the child due to a failed
2978          * COW. Warn that such a situation has occurred as it may not be obvious
2979          */
2980         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2981                 pr_warning("PID %d killed due to inadequate hugepage pool\n",
2982                            current->pid);
2983                 return ret;
2984         }
2985
2986         /*
2987          * Use page lock to guard against racing truncation
2988          * before we get page_table_lock.
2989          */
2990 retry:
2991         page = find_lock_page(mapping, idx);
2992         if (!page) {
2993                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2994                 if (idx >= size)
2995                         goto out;
2996                 page = alloc_huge_page(vma, address, 0);
2997                 if (IS_ERR(page)) {
2998                         ret = PTR_ERR(page);
2999                         if (ret == -ENOMEM)
3000                                 ret = VM_FAULT_OOM;
3001                         else
3002                                 ret = VM_FAULT_SIGBUS;
3003                         goto out;
3004                 }
3005                 clear_huge_page(page, address, pages_per_huge_page(h));
3006                 __SetPageUptodate(page);
3007
3008                 if (vma->vm_flags & VM_MAYSHARE) {
3009                         int err;
3010                         struct inode *inode = mapping->host;
3011
3012                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3013                         if (err) {
3014                                 put_page(page);
3015                                 if (err == -EEXIST)
3016                                         goto retry;
3017                                 goto out;
3018                         }
3019                         ClearPagePrivate(page);
3020
3021                         spin_lock(&inode->i_lock);
3022                         inode->i_blocks += blocks_per_huge_page(h);
3023                         spin_unlock(&inode->i_lock);
3024                 } else {
3025                         lock_page(page);
3026                         if (unlikely(anon_vma_prepare(vma))) {
3027                                 ret = VM_FAULT_OOM;
3028                                 goto backout_unlocked;
3029                         }
3030                         anon_rmap = 1;
3031                 }
3032         } else {
3033                 /*
3034                  * If memory error occurs between mmap() and fault, some process
3035                  * don't have hwpoisoned swap entry for errored virtual address.
3036                  * So we need to block hugepage fault by PG_hwpoison bit check.
3037                  */
3038                 if (unlikely(PageHWPoison(page))) {
3039                         ret = VM_FAULT_HWPOISON |
3040                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3041                         goto backout_unlocked;
3042                 }
3043         }
3044
3045         /*
3046          * If we are going to COW a private mapping later, we examine the
3047          * pending reservations for this page now. This will ensure that
3048          * any allocations necessary to record that reservation occur outside
3049          * the spinlock.
3050          */
3051         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
3052                 if (vma_needs_reservation(h, vma, address) < 0) {
3053                         ret = VM_FAULT_OOM;
3054                         goto backout_unlocked;
3055                 }
3056
3057         ptl = huge_pte_lockptr(h, mm, ptep);
3058         spin_lock(ptl);
3059         size = i_size_read(mapping->host) >> huge_page_shift(h);
3060         if (idx >= size)
3061                 goto backout;
3062
3063         ret = 0;
3064         if (!huge_pte_none(huge_ptep_get(ptep)))
3065                 goto backout;
3066
3067         if (anon_rmap) {
3068                 ClearPagePrivate(page);
3069                 hugepage_add_new_anon_rmap(page, vma, address);
3070         } else
3071                 page_dup_rmap(page);
3072         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3073                                 && (vma->vm_flags & VM_SHARED)));
3074         set_huge_pte_at(mm, address, ptep, new_pte);
3075
3076         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3077                 /* Optimization, do the COW without a second fault */
3078                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3079         }
3080
3081         spin_unlock(ptl);
3082         unlock_page(page);
3083 out:
3084         return ret;
3085
3086 backout:
3087         spin_unlock(ptl);
3088 backout_unlocked:
3089         unlock_page(page);
3090         put_page(page);
3091         goto out;
3092 }
3093
3094 #ifdef CONFIG_SMP
3095 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3096                             struct vm_area_struct *vma,
3097                             struct address_space *mapping,
3098                             pgoff_t idx, unsigned long address)
3099 {
3100         unsigned long key[2];
3101         u32 hash;
3102
3103         if (vma->vm_flags & VM_SHARED) {
3104                 key[0] = (unsigned long) mapping;
3105                 key[1] = idx;
3106         } else {
3107                 key[0] = (unsigned long) mm;
3108                 key[1] = address >> huge_page_shift(h);
3109         }
3110
3111         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3112
3113         return hash & (num_fault_mutexes - 1);
3114 }
3115 #else
3116 /*
3117  * For uniprocesor systems we always use a single mutex, so just
3118  * return 0 and avoid the hashing overhead.
3119  */
3120 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3121                             struct vm_area_struct *vma,
3122                             struct address_space *mapping,
3123                             pgoff_t idx, unsigned long address)
3124 {
3125         return 0;
3126 }
3127 #endif
3128
3129 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3130                         unsigned long address, unsigned int flags)
3131 {
3132         pte_t *ptep, entry;
3133         spinlock_t *ptl;
3134         int ret;
3135         u32 hash;
3136         pgoff_t idx;
3137         struct page *page = NULL;
3138         struct page *pagecache_page = NULL;
3139         struct hstate *h = hstate_vma(vma);
3140         struct address_space *mapping;
3141
3142         address &= huge_page_mask(h);
3143
3144         ptep = huge_pte_offset(mm, address);
3145         if (ptep) {
3146                 entry = huge_ptep_get(ptep);
3147                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3148                         migration_entry_wait_huge(vma, mm, ptep);
3149                         return 0;
3150                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3151                         return VM_FAULT_HWPOISON_LARGE |
3152                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3153         }
3154
3155         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3156         if (!ptep)
3157                 return VM_FAULT_OOM;
3158
3159         mapping = vma->vm_file->f_mapping;
3160         idx = vma_hugecache_offset(h, vma, address);
3161
3162         /*
3163          * Serialize hugepage allocation and instantiation, so that we don't
3164          * get spurious allocation failures if two CPUs race to instantiate
3165          * the same page in the page cache.
3166          */
3167         hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
3168         mutex_lock(&htlb_fault_mutex_table[hash]);
3169
3170         entry = huge_ptep_get(ptep);
3171         if (huge_pte_none(entry)) {
3172                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3173                 goto out_mutex;
3174         }
3175
3176         ret = 0;
3177
3178         /*
3179          * If we are going to COW the mapping later, we examine the pending
3180          * reservations for this page now. This will ensure that any
3181          * allocations necessary to record that reservation occur outside the
3182          * spinlock. For private mappings, we also lookup the pagecache
3183          * page now as it is used to determine if a reservation has been
3184          * consumed.
3185          */
3186         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3187                 if (vma_needs_reservation(h, vma, address) < 0) {
3188                         ret = VM_FAULT_OOM;
3189                         goto out_mutex;
3190                 }
3191
3192                 if (!(vma->vm_flags & VM_MAYSHARE))
3193                         pagecache_page = hugetlbfs_pagecache_page(h,
3194                                                                 vma, address);
3195         }
3196
3197         /*
3198          * hugetlb_cow() requires page locks of pte_page(entry) and
3199          * pagecache_page, so here we need take the former one
3200          * when page != pagecache_page or !pagecache_page.
3201          * Note that locking order is always pagecache_page -> page,
3202          * so no worry about deadlock.
3203          */
3204         page = pte_page(entry);
3205         get_page(page);
3206         if (page != pagecache_page)
3207                 lock_page(page);
3208
3209         ptl = huge_pte_lockptr(h, mm, ptep);
3210         spin_lock(ptl);
3211         /* Check for a racing update before calling hugetlb_cow */
3212         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3213                 goto out_ptl;
3214
3215
3216         if (flags & FAULT_FLAG_WRITE) {
3217                 if (!huge_pte_write(entry)) {
3218                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
3219                                         pagecache_page, ptl);
3220                         goto out_ptl;
3221                 }
3222                 entry = huge_pte_mkdirty(entry);
3223         }
3224         entry = pte_mkyoung(entry);
3225         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3226                                                 flags & FAULT_FLAG_WRITE))
3227                 update_mmu_cache(vma, address, ptep);
3228
3229 out_ptl:
3230         spin_unlock(ptl);
3231
3232         if (pagecache_page) {
3233                 unlock_page(pagecache_page);
3234                 put_page(pagecache_page);
3235         }
3236         if (page != pagecache_page)
3237                 unlock_page(page);
3238         put_page(page);
3239
3240 out_mutex:
3241         mutex_unlock(&htlb_fault_mutex_table[hash]);
3242         return ret;
3243 }
3244
3245 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3246                          struct page **pages, struct vm_area_struct **vmas,
3247                          unsigned long *position, unsigned long *nr_pages,
3248                          long i, unsigned int flags)
3249 {
3250         unsigned long pfn_offset;
3251         unsigned long vaddr = *position;
3252         unsigned long remainder = *nr_pages;
3253         struct hstate *h = hstate_vma(vma);
3254
3255         while (vaddr < vma->vm_end && remainder) {
3256                 pte_t *pte;
3257                 spinlock_t *ptl = NULL;
3258                 int absent;
3259                 struct page *page;
3260
3261                 /*
3262                  * Some archs (sparc64, sh*) have multiple pte_ts to
3263                  * each hugepage.  We have to make sure we get the
3264                  * first, for the page indexing below to work.
3265                  *
3266                  * Note that page table lock is not held when pte is null.
3267                  */
3268                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3269                 if (pte)
3270                         ptl = huge_pte_lock(h, mm, pte);
3271                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3272
3273                 /*
3274                  * When coredumping, it suits get_dump_page if we just return
3275                  * an error where there's an empty slot with no huge pagecache
3276                  * to back it.  This way, we avoid allocating a hugepage, and
3277                  * the sparse dumpfile avoids allocating disk blocks, but its
3278                  * huge holes still show up with zeroes where they need to be.
3279                  */
3280                 if (absent && (flags & FOLL_DUMP) &&
3281                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3282                         if (pte)
3283                                 spin_unlock(ptl);
3284                         remainder = 0;
3285                         break;
3286                 }
3287
3288                 /*
3289                  * We need call hugetlb_fault for both hugepages under migration
3290                  * (in which case hugetlb_fault waits for the migration,) and
3291                  * hwpoisoned hugepages (in which case we need to prevent the
3292                  * caller from accessing to them.) In order to do this, we use
3293                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3294                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3295                  * both cases, and because we can't follow correct pages
3296                  * directly from any kind of swap entries.
3297                  */
3298                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3299                     ((flags & FOLL_WRITE) &&
3300                       !huge_pte_write(huge_ptep_get(pte)))) {
3301                         int ret;
3302
3303                         if (pte)
3304                                 spin_unlock(ptl);
3305                         ret = hugetlb_fault(mm, vma, vaddr,
3306                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3307                         if (!(ret & VM_FAULT_ERROR))
3308                                 continue;
3309
3310                         remainder = 0;
3311                         break;
3312                 }
3313
3314                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3315                 page = pte_page(huge_ptep_get(pte));
3316 same_page:
3317                 if (pages) {
3318                         pages[i] = mem_map_offset(page, pfn_offset);
3319                         get_page_foll(pages[i]);
3320                 }
3321
3322                 if (vmas)
3323                         vmas[i] = vma;
3324
3325                 vaddr += PAGE_SIZE;
3326                 ++pfn_offset;
3327                 --remainder;
3328                 ++i;
3329                 if (vaddr < vma->vm_end && remainder &&
3330                                 pfn_offset < pages_per_huge_page(h)) {
3331                         /*
3332                          * We use pfn_offset to avoid touching the pageframes
3333                          * of this compound page.
3334                          */
3335                         goto same_page;
3336                 }
3337                 spin_unlock(ptl);
3338         }
3339         *nr_pages = remainder;
3340         *position = vaddr;
3341
3342         return i ? i : -EFAULT;
3343 }
3344
3345 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3346                 unsigned long address, unsigned long end, pgprot_t newprot)
3347 {
3348         struct mm_struct *mm = vma->vm_mm;
3349         unsigned long start = address;
3350         pte_t *ptep;
3351         pte_t pte;
3352         struct hstate *h = hstate_vma(vma);
3353         unsigned long pages = 0;
3354
3355         BUG_ON(address >= end);
3356         flush_cache_range(vma, address, end);
3357
3358         mmu_notifier_invalidate_range_start(mm, start, end);
3359         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
3360         for (; address < end; address += huge_page_size(h)) {
3361                 spinlock_t *ptl;
3362                 ptep = huge_pte_offset(mm, address);
3363                 if (!ptep)
3364                         continue;
3365                 ptl = huge_pte_lock(h, mm, ptep);
3366                 if (huge_pmd_unshare(mm, &address, ptep)) {
3367                         pages++;
3368                         spin_unlock(ptl);
3369                         continue;
3370                 }
3371                 if (!huge_pte_none(huge_ptep_get(ptep))) {
3372                         pte = huge_ptep_get_and_clear(mm, address, ptep);
3373                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3374                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
3375                         set_huge_pte_at(mm, address, ptep, pte);
3376                         pages++;
3377                 }
3378                 spin_unlock(ptl);
3379         }
3380         /*
3381          * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
3382          * may have cleared our pud entry and done put_page on the page table:
3383          * once we release i_mmap_mutex, another task can do the final put_page
3384          * and that page table be reused and filled with junk.
3385          */
3386         flush_tlb_range(vma, start, end);
3387         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
3388         mmu_notifier_invalidate_range_end(mm, start, end);
3389
3390         return pages << h->order;
3391 }
3392
3393 int hugetlb_reserve_pages(struct inode *inode,
3394                                         long from, long to,
3395                                         struct vm_area_struct *vma,
3396                                         vm_flags_t vm_flags)
3397 {
3398         long ret, chg;
3399         struct hstate *h = hstate_inode(inode);
3400         struct hugepage_subpool *spool = subpool_inode(inode);
3401         struct resv_map *resv_map;
3402
3403         /*
3404          * Only apply hugepage reservation if asked. At fault time, an
3405          * attempt will be made for VM_NORESERVE to allocate a page
3406          * without using reserves
3407          */
3408         if (vm_flags & VM_NORESERVE)
3409                 return 0;
3410
3411         /*
3412          * Shared mappings base their reservation on the number of pages that
3413          * are already allocated on behalf of the file. Private mappings need
3414          * to reserve the full area even if read-only as mprotect() may be
3415          * called to make the mapping read-write. Assume !vma is a shm mapping
3416          */
3417         if (!vma || vma->vm_flags & VM_MAYSHARE) {
3418                 resv_map = inode_resv_map(inode);
3419
3420                 chg = region_chg(resv_map, from, to);
3421
3422         } else {
3423                 resv_map = resv_map_alloc();
3424                 if (!resv_map)
3425                         return -ENOMEM;
3426
3427                 chg = to - from;
3428
3429                 set_vma_resv_map(vma, resv_map);
3430                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3431         }
3432
3433         if (chg < 0) {
3434                 ret = chg;
3435                 goto out_err;
3436         }
3437
3438         /* There must be enough pages in the subpool for the mapping */
3439         if (hugepage_subpool_get_pages(spool, chg)) {
3440                 ret = -ENOSPC;
3441                 goto out_err;
3442         }
3443
3444         /*
3445          * Check enough hugepages are available for the reservation.
3446          * Hand the pages back to the subpool if there are not
3447          */
3448         ret = hugetlb_acct_memory(h, chg);
3449         if (ret < 0) {
3450                 hugepage_subpool_put_pages(spool, chg);
3451                 goto out_err;
3452         }
3453
3454         /*
3455          * Account for the reservations made. Shared mappings record regions
3456          * that have reservations as they are shared by multiple VMAs.
3457          * When the last VMA disappears, the region map says how much
3458          * the reservation was and the page cache tells how much of
3459          * the reservation was consumed. Private mappings are per-VMA and
3460          * only the consumed reservations are tracked. When the VMA
3461          * disappears, the original reservation is the VMA size and the
3462          * consumed reservations are stored in the map. Hence, nothing
3463          * else has to be done for private mappings here
3464          */
3465         if (!vma || vma->vm_flags & VM_MAYSHARE)
3466                 region_add(resv_map, from, to);
3467         return 0;
3468 out_err:
3469         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3470                 kref_put(&resv_map->refs, resv_map_release);
3471         return ret;
3472 }
3473
3474 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3475 {
3476         struct hstate *h = hstate_inode(inode);
3477         struct resv_map *resv_map = inode_resv_map(inode);
3478         long chg = 0;
3479         struct hugepage_subpool *spool = subpool_inode(inode);
3480
3481         if (resv_map)
3482                 chg = region_truncate(resv_map, offset);
3483         spin_lock(&inode->i_lock);
3484         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3485         spin_unlock(&inode->i_lock);
3486
3487         hugepage_subpool_put_pages(spool, (chg - freed));
3488         hugetlb_acct_memory(h, -(chg - freed));
3489 }
3490
3491 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3492 static unsigned long page_table_shareable(struct vm_area_struct *svma,
3493                                 struct vm_area_struct *vma,
3494                                 unsigned long addr, pgoff_t idx)
3495 {
3496         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3497                                 svma->vm_start;
3498         unsigned long sbase = saddr & PUD_MASK;
3499         unsigned long s_end = sbase + PUD_SIZE;
3500
3501         /* Allow segments to share if only one is marked locked */
3502         unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3503         unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3504
3505         /*
3506          * match the virtual addresses, permission and the alignment of the
3507          * page table page.
3508          */
3509         if (pmd_index(addr) != pmd_index(saddr) ||
3510             vm_flags != svm_flags ||
3511             sbase < svma->vm_start || svma->vm_end < s_end)
3512                 return 0;
3513
3514         return saddr;
3515 }
3516
3517 static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3518 {
3519         unsigned long base = addr & PUD_MASK;
3520         unsigned long end = base + PUD_SIZE;
3521
3522         /*
3523          * check on proper vm_flags and page table alignment
3524          */
3525         if (vma->vm_flags & VM_MAYSHARE &&
3526             vma->vm_start <= base && end <= vma->vm_end)
3527                 return 1;
3528         return 0;
3529 }
3530
3531 /*
3532  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3533  * and returns the corresponding pte. While this is not necessary for the
3534  * !shared pmd case because we can allocate the pmd later as well, it makes the
3535  * code much cleaner. pmd allocation is essential for the shared case because
3536  * pud has to be populated inside the same i_mmap_mutex section - otherwise
3537  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3538  * bad pmd for sharing.
3539  */
3540 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3541 {
3542         struct vm_area_struct *vma = find_vma(mm, addr);
3543         struct address_space *mapping = vma->vm_file->f_mapping;
3544         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3545                         vma->vm_pgoff;
3546         struct vm_area_struct *svma;
3547         unsigned long saddr;
3548         pte_t *spte = NULL;
3549         pte_t *pte;
3550         spinlock_t *ptl;
3551
3552         if (!vma_shareable(vma, addr))
3553                 return (pte_t *)pmd_alloc(mm, pud, addr);
3554
3555         mutex_lock(&mapping->i_mmap_mutex);
3556         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3557                 if (svma == vma)
3558                         continue;
3559
3560                 saddr = page_table_shareable(svma, vma, addr, idx);
3561                 if (saddr) {
3562                         spte = huge_pte_offset(svma->vm_mm, saddr);
3563                         if (spte) {
3564                                 get_page(virt_to_page(spte));
3565                                 break;
3566                         }
3567                 }
3568         }
3569
3570         if (!spte)
3571                 goto out;
3572
3573         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3574         spin_lock(ptl);
3575         if (pud_none(*pud))
3576                 pud_populate(mm, pud,
3577                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
3578         else
3579                 put_page(virt_to_page(spte));
3580         spin_unlock(ptl);
3581 out:
3582         pte = (pte_t *)pmd_alloc(mm, pud, addr);
3583         mutex_unlock(&mapping->i_mmap_mutex);
3584         return pte;
3585 }
3586
3587 /*
3588  * unmap huge page backed by shared pte.
3589  *
3590  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
3591  * indicated by page_count > 1, unmap is achieved by clearing pud and
3592  * decrementing the ref count. If count == 1, the pte page is not shared.
3593  *
3594  * called with page table lock held.
3595  *
3596  * returns: 1 successfully unmapped a shared pte page
3597  *          0 the underlying pte page is not shared, or it is the last user
3598  */
3599 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3600 {
3601         pgd_t *pgd = pgd_offset(mm, *addr);
3602         pud_t *pud = pud_offset(pgd, *addr);
3603
3604         BUG_ON(page_count(virt_to_page(ptep)) == 0);
3605         if (page_count(virt_to_page(ptep)) == 1)
3606                 return 0;
3607
3608         pud_clear(pud);
3609         put_page(virt_to_page(ptep));
3610         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3611         return 1;
3612 }
3613 #define want_pmd_share()        (1)
3614 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3615 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3616 {
3617         return NULL;
3618 }
3619 #define want_pmd_share()        (0)
3620 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3621
3622 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3623 pte_t *huge_pte_alloc(struct mm_struct *mm,
3624                         unsigned long addr, unsigned long sz)
3625 {
3626         pgd_t *pgd;
3627         pud_t *pud;
3628         pte_t *pte = NULL;
3629
3630         pgd = pgd_offset(mm, addr);
3631         pud = pud_alloc(mm, pgd, addr);
3632         if (pud) {
3633                 if (sz == PUD_SIZE) {
3634                         pte = (pte_t *)pud;
3635                 } else {
3636                         BUG_ON(sz != PMD_SIZE);
3637                         if (want_pmd_share() && pud_none(*pud))
3638                                 pte = huge_pmd_share(mm, addr, pud);
3639                         else
3640                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3641                 }
3642         }
3643         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3644
3645         return pte;
3646 }
3647
3648 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3649 {
3650         pgd_t *pgd;
3651         pud_t *pud;
3652         pmd_t *pmd = NULL;
3653
3654         pgd = pgd_offset(mm, addr);
3655         if (pgd_present(*pgd)) {
3656                 pud = pud_offset(pgd, addr);
3657                 if (pud_present(*pud)) {
3658                         if (pud_huge(*pud))
3659                                 return (pte_t *)pud;
3660                         pmd = pmd_offset(pud, addr);
3661                 }
3662         }
3663         return (pte_t *) pmd;
3664 }
3665
3666 struct page *
3667 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3668                 pmd_t *pmd, int write)
3669 {
3670         struct page *page;
3671
3672         page = pte_page(*(pte_t *)pmd);
3673         if (page)
3674                 page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
3675         return page;
3676 }
3677
3678 struct page *
3679 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3680                 pud_t *pud, int write)
3681 {
3682         struct page *page;
3683
3684         page = pte_page(*(pte_t *)pud);
3685         if (page)
3686                 page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
3687         return page;
3688 }
3689
3690 #else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3691
3692 /* Can be overriden by architectures */
3693 struct page * __weak
3694 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3695                pud_t *pud, int write)
3696 {
3697         BUG();
3698         return NULL;
3699 }
3700
3701 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3702
3703 #ifdef CONFIG_MEMORY_FAILURE
3704
3705 /* Should be called in hugetlb_lock */
3706 static int is_hugepage_on_freelist(struct page *hpage)
3707 {
3708         struct page *page;
3709         struct page *tmp;
3710         struct hstate *h = page_hstate(hpage);
3711         int nid = page_to_nid(hpage);
3712
3713         list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3714                 if (page == hpage)
3715                         return 1;
3716         return 0;
3717 }
3718
3719 /*
3720  * This function is called from memory failure code.
3721  * Assume the caller holds page lock of the head page.
3722  */
3723 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3724 {
3725         struct hstate *h = page_hstate(hpage);
3726         int nid = page_to_nid(hpage);
3727         int ret = -EBUSY;
3728
3729         spin_lock(&hugetlb_lock);
3730         if (is_hugepage_on_freelist(hpage)) {
3731                 /*
3732                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
3733                  * but dangling hpage->lru can trigger list-debug warnings
3734                  * (this happens when we call unpoison_memory() on it),
3735                  * so let it point to itself with list_del_init().
3736                  */
3737                 list_del_init(&hpage->lru);
3738                 set_page_refcounted(hpage);
3739                 h->free_huge_pages--;
3740                 h->free_huge_pages_node[nid]--;
3741                 ret = 0;
3742         }
3743         spin_unlock(&hugetlb_lock);
3744         return ret;
3745 }
3746 #endif
3747
3748 bool isolate_huge_page(struct page *page, struct list_head *list)
3749 {
3750         VM_BUG_ON_PAGE(!PageHead(page), page);
3751         if (!get_page_unless_zero(page))
3752                 return false;
3753         spin_lock(&hugetlb_lock);
3754         list_move_tail(&page->lru, list);
3755         spin_unlock(&hugetlb_lock);
3756         return true;
3757 }
3758
3759 void putback_active_hugepage(struct page *page)
3760 {
3761         VM_BUG_ON_PAGE(!PageHead(page), page);
3762         spin_lock(&hugetlb_lock);
3763         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3764         spin_unlock(&hugetlb_lock);
3765         put_page(page);
3766 }
3767
3768 bool is_hugepage_active(struct page *page)
3769 {
3770         VM_BUG_ON_PAGE(!PageHuge(page), page);
3771         /*
3772          * This function can be called for a tail page because the caller,
3773          * scan_movable_pages, scans through a given pfn-range which typically
3774          * covers one memory block. In systems using gigantic hugepage (1GB
3775          * for x86_64,) a hugepage is larger than a memory block, and we don't
3776          * support migrating such large hugepages for now, so return false
3777          * when called for tail pages.
3778          */
3779         if (PageTail(page))
3780                 return false;
3781         /*
3782          * Refcount of a hwpoisoned hugepages is 1, but they are not active,
3783          * so we should return false for them.
3784          */
3785         if (unlikely(PageHWPoison(page)))
3786                 return false;
3787         return page_count(page) > 0;
3788 }