]> git.karo-electronics.de Git - linux-beck.git/blob - mm/hugetlb.c
Merge branch 'drm-fixes' of git://people.freedesktop.org/~airlied/linux
[linux-beck.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/bootmem.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/rmap.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/page-isolation.h>
26 #include <linux/jhash.h>
27
28 #include <asm/page.h>
29 #include <asm/pgtable.h>
30 #include <asm/tlb.h>
31
32 #include <linux/io.h>
33 #include <linux/hugetlb.h>
34 #include <linux/hugetlb_cgroup.h>
35 #include <linux/node.h>
36 #include "internal.h"
37
38 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
39 unsigned long hugepages_treat_as_movable;
40
41 int hugetlb_max_hstate __read_mostly;
42 unsigned int default_hstate_idx;
43 struct hstate hstates[HUGE_MAX_HSTATE];
44
45 __initdata LIST_HEAD(huge_boot_pages);
46
47 /* for command line parsing */
48 static struct hstate * __initdata parsed_hstate;
49 static unsigned long __initdata default_hstate_max_huge_pages;
50 static unsigned long __initdata default_hstate_size;
51
52 /*
53  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
54  * free_huge_pages, and surplus_huge_pages.
55  */
56 DEFINE_SPINLOCK(hugetlb_lock);
57
58 /*
59  * Serializes faults on the same logical page.  This is used to
60  * prevent spurious OOMs when the hugepage pool is fully utilized.
61  */
62 static int num_fault_mutexes;
63 static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
64
65 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
66 {
67         bool free = (spool->count == 0) && (spool->used_hpages == 0);
68
69         spin_unlock(&spool->lock);
70
71         /* If no pages are used, and no other handles to the subpool
72          * remain, free the subpool the subpool remain */
73         if (free)
74                 kfree(spool);
75 }
76
77 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
78 {
79         struct hugepage_subpool *spool;
80
81         spool = kmalloc(sizeof(*spool), GFP_KERNEL);
82         if (!spool)
83                 return NULL;
84
85         spin_lock_init(&spool->lock);
86         spool->count = 1;
87         spool->max_hpages = nr_blocks;
88         spool->used_hpages = 0;
89
90         return spool;
91 }
92
93 void hugepage_put_subpool(struct hugepage_subpool *spool)
94 {
95         spin_lock(&spool->lock);
96         BUG_ON(!spool->count);
97         spool->count--;
98         unlock_or_release_subpool(spool);
99 }
100
101 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
102                                       long delta)
103 {
104         int ret = 0;
105
106         if (!spool)
107                 return 0;
108
109         spin_lock(&spool->lock);
110         if ((spool->used_hpages + delta) <= spool->max_hpages) {
111                 spool->used_hpages += delta;
112         } else {
113                 ret = -ENOMEM;
114         }
115         spin_unlock(&spool->lock);
116
117         return ret;
118 }
119
120 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
121                                        long delta)
122 {
123         if (!spool)
124                 return;
125
126         spin_lock(&spool->lock);
127         spool->used_hpages -= delta;
128         /* If hugetlbfs_put_super couldn't free spool due to
129         * an outstanding quota reference, free it now. */
130         unlock_or_release_subpool(spool);
131 }
132
133 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
134 {
135         return HUGETLBFS_SB(inode->i_sb)->spool;
136 }
137
138 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
139 {
140         return subpool_inode(file_inode(vma->vm_file));
141 }
142
143 /*
144  * Region tracking -- allows tracking of reservations and instantiated pages
145  *                    across the pages in a mapping.
146  *
147  * The region data structures are embedded into a resv_map and
148  * protected by a resv_map's lock
149  */
150 struct file_region {
151         struct list_head link;
152         long from;
153         long to;
154 };
155
156 static long region_add(struct resv_map *resv, long f, long t)
157 {
158         struct list_head *head = &resv->regions;
159         struct file_region *rg, *nrg, *trg;
160
161         spin_lock(&resv->lock);
162         /* Locate the region we are either in or before. */
163         list_for_each_entry(rg, head, link)
164                 if (f <= rg->to)
165                         break;
166
167         /* Round our left edge to the current segment if it encloses us. */
168         if (f > rg->from)
169                 f = rg->from;
170
171         /* Check for and consume any regions we now overlap with. */
172         nrg = rg;
173         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
174                 if (&rg->link == head)
175                         break;
176                 if (rg->from > t)
177                         break;
178
179                 /* If this area reaches higher then extend our area to
180                  * include it completely.  If this is not the first area
181                  * which we intend to reuse, free it. */
182                 if (rg->to > t)
183                         t = rg->to;
184                 if (rg != nrg) {
185                         list_del(&rg->link);
186                         kfree(rg);
187                 }
188         }
189         nrg->from = f;
190         nrg->to = t;
191         spin_unlock(&resv->lock);
192         return 0;
193 }
194
195 static long region_chg(struct resv_map *resv, long f, long t)
196 {
197         struct list_head *head = &resv->regions;
198         struct file_region *rg, *nrg = NULL;
199         long chg = 0;
200
201 retry:
202         spin_lock(&resv->lock);
203         /* Locate the region we are before or in. */
204         list_for_each_entry(rg, head, link)
205                 if (f <= rg->to)
206                         break;
207
208         /* If we are below the current region then a new region is required.
209          * Subtle, allocate a new region at the position but make it zero
210          * size such that we can guarantee to record the reservation. */
211         if (&rg->link == head || t < rg->from) {
212                 if (!nrg) {
213                         spin_unlock(&resv->lock);
214                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
215                         if (!nrg)
216                                 return -ENOMEM;
217
218                         nrg->from = f;
219                         nrg->to   = f;
220                         INIT_LIST_HEAD(&nrg->link);
221                         goto retry;
222                 }
223
224                 list_add(&nrg->link, rg->link.prev);
225                 chg = t - f;
226                 goto out_nrg;
227         }
228
229         /* Round our left edge to the current segment if it encloses us. */
230         if (f > rg->from)
231                 f = rg->from;
232         chg = t - f;
233
234         /* Check for and consume any regions we now overlap with. */
235         list_for_each_entry(rg, rg->link.prev, link) {
236                 if (&rg->link == head)
237                         break;
238                 if (rg->from > t)
239                         goto out;
240
241                 /* We overlap with this area, if it extends further than
242                  * us then we must extend ourselves.  Account for its
243                  * existing reservation. */
244                 if (rg->to > t) {
245                         chg += rg->to - t;
246                         t = rg->to;
247                 }
248                 chg -= rg->to - rg->from;
249         }
250
251 out:
252         spin_unlock(&resv->lock);
253         /*  We already know we raced and no longer need the new region */
254         kfree(nrg);
255         return chg;
256 out_nrg:
257         spin_unlock(&resv->lock);
258         return chg;
259 }
260
261 static long region_truncate(struct resv_map *resv, long end)
262 {
263         struct list_head *head = &resv->regions;
264         struct file_region *rg, *trg;
265         long chg = 0;
266
267         spin_lock(&resv->lock);
268         /* Locate the region we are either in or before. */
269         list_for_each_entry(rg, head, link)
270                 if (end <= rg->to)
271                         break;
272         if (&rg->link == head)
273                 goto out;
274
275         /* If we are in the middle of a region then adjust it. */
276         if (end > rg->from) {
277                 chg = rg->to - end;
278                 rg->to = end;
279                 rg = list_entry(rg->link.next, typeof(*rg), link);
280         }
281
282         /* Drop any remaining regions. */
283         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
284                 if (&rg->link == head)
285                         break;
286                 chg += rg->to - rg->from;
287                 list_del(&rg->link);
288                 kfree(rg);
289         }
290
291 out:
292         spin_unlock(&resv->lock);
293         return chg;
294 }
295
296 static long region_count(struct resv_map *resv, long f, long t)
297 {
298         struct list_head *head = &resv->regions;
299         struct file_region *rg;
300         long chg = 0;
301
302         spin_lock(&resv->lock);
303         /* Locate each segment we overlap with, and count that overlap. */
304         list_for_each_entry(rg, head, link) {
305                 long seg_from;
306                 long seg_to;
307
308                 if (rg->to <= f)
309                         continue;
310                 if (rg->from >= t)
311                         break;
312
313                 seg_from = max(rg->from, f);
314                 seg_to = min(rg->to, t);
315
316                 chg += seg_to - seg_from;
317         }
318         spin_unlock(&resv->lock);
319
320         return chg;
321 }
322
323 /*
324  * Convert the address within this vma to the page offset within
325  * the mapping, in pagecache page units; huge pages here.
326  */
327 static pgoff_t vma_hugecache_offset(struct hstate *h,
328                         struct vm_area_struct *vma, unsigned long address)
329 {
330         return ((address - vma->vm_start) >> huge_page_shift(h)) +
331                         (vma->vm_pgoff >> huge_page_order(h));
332 }
333
334 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
335                                      unsigned long address)
336 {
337         return vma_hugecache_offset(hstate_vma(vma), vma, address);
338 }
339
340 /*
341  * Return the size of the pages allocated when backing a VMA. In the majority
342  * cases this will be same size as used by the page table entries.
343  */
344 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
345 {
346         struct hstate *hstate;
347
348         if (!is_vm_hugetlb_page(vma))
349                 return PAGE_SIZE;
350
351         hstate = hstate_vma(vma);
352
353         return 1UL << huge_page_shift(hstate);
354 }
355 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
356
357 /*
358  * Return the page size being used by the MMU to back a VMA. In the majority
359  * of cases, the page size used by the kernel matches the MMU size. On
360  * architectures where it differs, an architecture-specific version of this
361  * function is required.
362  */
363 #ifndef vma_mmu_pagesize
364 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
365 {
366         return vma_kernel_pagesize(vma);
367 }
368 #endif
369
370 /*
371  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
372  * bits of the reservation map pointer, which are always clear due to
373  * alignment.
374  */
375 #define HPAGE_RESV_OWNER    (1UL << 0)
376 #define HPAGE_RESV_UNMAPPED (1UL << 1)
377 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
378
379 /*
380  * These helpers are used to track how many pages are reserved for
381  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
382  * is guaranteed to have their future faults succeed.
383  *
384  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
385  * the reserve counters are updated with the hugetlb_lock held. It is safe
386  * to reset the VMA at fork() time as it is not in use yet and there is no
387  * chance of the global counters getting corrupted as a result of the values.
388  *
389  * The private mapping reservation is represented in a subtly different
390  * manner to a shared mapping.  A shared mapping has a region map associated
391  * with the underlying file, this region map represents the backing file
392  * pages which have ever had a reservation assigned which this persists even
393  * after the page is instantiated.  A private mapping has a region map
394  * associated with the original mmap which is attached to all VMAs which
395  * reference it, this region map represents those offsets which have consumed
396  * reservation ie. where pages have been instantiated.
397  */
398 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
399 {
400         return (unsigned long)vma->vm_private_data;
401 }
402
403 static void set_vma_private_data(struct vm_area_struct *vma,
404                                                         unsigned long value)
405 {
406         vma->vm_private_data = (void *)value;
407 }
408
409 struct resv_map *resv_map_alloc(void)
410 {
411         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
412         if (!resv_map)
413                 return NULL;
414
415         kref_init(&resv_map->refs);
416         spin_lock_init(&resv_map->lock);
417         INIT_LIST_HEAD(&resv_map->regions);
418
419         return resv_map;
420 }
421
422 void resv_map_release(struct kref *ref)
423 {
424         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
425
426         /* Clear out any active regions before we release the map. */
427         region_truncate(resv_map, 0);
428         kfree(resv_map);
429 }
430
431 static inline struct resv_map *inode_resv_map(struct inode *inode)
432 {
433         return inode->i_mapping->private_data;
434 }
435
436 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
437 {
438         VM_BUG_ON(!is_vm_hugetlb_page(vma));
439         if (vma->vm_flags & VM_MAYSHARE) {
440                 struct address_space *mapping = vma->vm_file->f_mapping;
441                 struct inode *inode = mapping->host;
442
443                 return inode_resv_map(inode);
444
445         } else {
446                 return (struct resv_map *)(get_vma_private_data(vma) &
447                                                         ~HPAGE_RESV_MASK);
448         }
449 }
450
451 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
452 {
453         VM_BUG_ON(!is_vm_hugetlb_page(vma));
454         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
455
456         set_vma_private_data(vma, (get_vma_private_data(vma) &
457                                 HPAGE_RESV_MASK) | (unsigned long)map);
458 }
459
460 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
461 {
462         VM_BUG_ON(!is_vm_hugetlb_page(vma));
463         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
464
465         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
466 }
467
468 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
469 {
470         VM_BUG_ON(!is_vm_hugetlb_page(vma));
471
472         return (get_vma_private_data(vma) & flag) != 0;
473 }
474
475 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
476 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
477 {
478         VM_BUG_ON(!is_vm_hugetlb_page(vma));
479         if (!(vma->vm_flags & VM_MAYSHARE))
480                 vma->vm_private_data = (void *)0;
481 }
482
483 /* Returns true if the VMA has associated reserve pages */
484 static int vma_has_reserves(struct vm_area_struct *vma, long chg)
485 {
486         if (vma->vm_flags & VM_NORESERVE) {
487                 /*
488                  * This address is already reserved by other process(chg == 0),
489                  * so, we should decrement reserved count. Without decrementing,
490                  * reserve count remains after releasing inode, because this
491                  * allocated page will go into page cache and is regarded as
492                  * coming from reserved pool in releasing step.  Currently, we
493                  * don't have any other solution to deal with this situation
494                  * properly, so add work-around here.
495                  */
496                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
497                         return 1;
498                 else
499                         return 0;
500         }
501
502         /* Shared mappings always use reserves */
503         if (vma->vm_flags & VM_MAYSHARE)
504                 return 1;
505
506         /*
507          * Only the process that called mmap() has reserves for
508          * private mappings.
509          */
510         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
511                 return 1;
512
513         return 0;
514 }
515
516 static void enqueue_huge_page(struct hstate *h, struct page *page)
517 {
518         int nid = page_to_nid(page);
519         list_move(&page->lru, &h->hugepage_freelists[nid]);
520         h->free_huge_pages++;
521         h->free_huge_pages_node[nid]++;
522 }
523
524 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
525 {
526         struct page *page;
527
528         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
529                 if (!is_migrate_isolate_page(page))
530                         break;
531         /*
532          * if 'non-isolated free hugepage' not found on the list,
533          * the allocation fails.
534          */
535         if (&h->hugepage_freelists[nid] == &page->lru)
536                 return NULL;
537         list_move(&page->lru, &h->hugepage_activelist);
538         set_page_refcounted(page);
539         h->free_huge_pages--;
540         h->free_huge_pages_node[nid]--;
541         return page;
542 }
543
544 /* Movability of hugepages depends on migration support. */
545 static inline gfp_t htlb_alloc_mask(struct hstate *h)
546 {
547         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
548                 return GFP_HIGHUSER_MOVABLE;
549         else
550                 return GFP_HIGHUSER;
551 }
552
553 static struct page *dequeue_huge_page_vma(struct hstate *h,
554                                 struct vm_area_struct *vma,
555                                 unsigned long address, int avoid_reserve,
556                                 long chg)
557 {
558         struct page *page = NULL;
559         struct mempolicy *mpol;
560         nodemask_t *nodemask;
561         struct zonelist *zonelist;
562         struct zone *zone;
563         struct zoneref *z;
564         unsigned int cpuset_mems_cookie;
565
566         /*
567          * A child process with MAP_PRIVATE mappings created by their parent
568          * have no page reserves. This check ensures that reservations are
569          * not "stolen". The child may still get SIGKILLed
570          */
571         if (!vma_has_reserves(vma, chg) &&
572                         h->free_huge_pages - h->resv_huge_pages == 0)
573                 goto err;
574
575         /* If reserves cannot be used, ensure enough pages are in the pool */
576         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
577                 goto err;
578
579 retry_cpuset:
580         cpuset_mems_cookie = read_mems_allowed_begin();
581         zonelist = huge_zonelist(vma, address,
582                                         htlb_alloc_mask(h), &mpol, &nodemask);
583
584         for_each_zone_zonelist_nodemask(zone, z, zonelist,
585                                                 MAX_NR_ZONES - 1, nodemask) {
586                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
587                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
588                         if (page) {
589                                 if (avoid_reserve)
590                                         break;
591                                 if (!vma_has_reserves(vma, chg))
592                                         break;
593
594                                 SetPagePrivate(page);
595                                 h->resv_huge_pages--;
596                                 break;
597                         }
598                 }
599         }
600
601         mpol_cond_put(mpol);
602         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
603                 goto retry_cpuset;
604         return page;
605
606 err:
607         return NULL;
608 }
609
610 /*
611  * common helper functions for hstate_next_node_to_{alloc|free}.
612  * We may have allocated or freed a huge page based on a different
613  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
614  * be outside of *nodes_allowed.  Ensure that we use an allowed
615  * node for alloc or free.
616  */
617 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
618 {
619         nid = next_node(nid, *nodes_allowed);
620         if (nid == MAX_NUMNODES)
621                 nid = first_node(*nodes_allowed);
622         VM_BUG_ON(nid >= MAX_NUMNODES);
623
624         return nid;
625 }
626
627 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
628 {
629         if (!node_isset(nid, *nodes_allowed))
630                 nid = next_node_allowed(nid, nodes_allowed);
631         return nid;
632 }
633
634 /*
635  * returns the previously saved node ["this node"] from which to
636  * allocate a persistent huge page for the pool and advance the
637  * next node from which to allocate, handling wrap at end of node
638  * mask.
639  */
640 static int hstate_next_node_to_alloc(struct hstate *h,
641                                         nodemask_t *nodes_allowed)
642 {
643         int nid;
644
645         VM_BUG_ON(!nodes_allowed);
646
647         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
648         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
649
650         return nid;
651 }
652
653 /*
654  * helper for free_pool_huge_page() - return the previously saved
655  * node ["this node"] from which to free a huge page.  Advance the
656  * next node id whether or not we find a free huge page to free so
657  * that the next attempt to free addresses the next node.
658  */
659 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
660 {
661         int nid;
662
663         VM_BUG_ON(!nodes_allowed);
664
665         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
666         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
667
668         return nid;
669 }
670
671 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
672         for (nr_nodes = nodes_weight(*mask);                            \
673                 nr_nodes > 0 &&                                         \
674                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
675                 nr_nodes--)
676
677 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
678         for (nr_nodes = nodes_weight(*mask);                            \
679                 nr_nodes > 0 &&                                         \
680                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
681                 nr_nodes--)
682
683 #if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
684 static void destroy_compound_gigantic_page(struct page *page,
685                                         unsigned long order)
686 {
687         int i;
688         int nr_pages = 1 << order;
689         struct page *p = page + 1;
690
691         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
692                 __ClearPageTail(p);
693                 set_page_refcounted(p);
694                 p->first_page = NULL;
695         }
696
697         set_compound_order(page, 0);
698         __ClearPageHead(page);
699 }
700
701 static void free_gigantic_page(struct page *page, unsigned order)
702 {
703         free_contig_range(page_to_pfn(page), 1 << order);
704 }
705
706 static int __alloc_gigantic_page(unsigned long start_pfn,
707                                 unsigned long nr_pages)
708 {
709         unsigned long end_pfn = start_pfn + nr_pages;
710         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
711 }
712
713 static bool pfn_range_valid_gigantic(unsigned long start_pfn,
714                                 unsigned long nr_pages)
715 {
716         unsigned long i, end_pfn = start_pfn + nr_pages;
717         struct page *page;
718
719         for (i = start_pfn; i < end_pfn; i++) {
720                 if (!pfn_valid(i))
721                         return false;
722
723                 page = pfn_to_page(i);
724
725                 if (PageReserved(page))
726                         return false;
727
728                 if (page_count(page) > 0)
729                         return false;
730
731                 if (PageHuge(page))
732                         return false;
733         }
734
735         return true;
736 }
737
738 static bool zone_spans_last_pfn(const struct zone *zone,
739                         unsigned long start_pfn, unsigned long nr_pages)
740 {
741         unsigned long last_pfn = start_pfn + nr_pages - 1;
742         return zone_spans_pfn(zone, last_pfn);
743 }
744
745 static struct page *alloc_gigantic_page(int nid, unsigned order)
746 {
747         unsigned long nr_pages = 1 << order;
748         unsigned long ret, pfn, flags;
749         struct zone *z;
750
751         z = NODE_DATA(nid)->node_zones;
752         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
753                 spin_lock_irqsave(&z->lock, flags);
754
755                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
756                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
757                         if (pfn_range_valid_gigantic(pfn, nr_pages)) {
758                                 /*
759                                  * We release the zone lock here because
760                                  * alloc_contig_range() will also lock the zone
761                                  * at some point. If there's an allocation
762                                  * spinning on this lock, it may win the race
763                                  * and cause alloc_contig_range() to fail...
764                                  */
765                                 spin_unlock_irqrestore(&z->lock, flags);
766                                 ret = __alloc_gigantic_page(pfn, nr_pages);
767                                 if (!ret)
768                                         return pfn_to_page(pfn);
769                                 spin_lock_irqsave(&z->lock, flags);
770                         }
771                         pfn += nr_pages;
772                 }
773
774                 spin_unlock_irqrestore(&z->lock, flags);
775         }
776
777         return NULL;
778 }
779
780 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
781 static void prep_compound_gigantic_page(struct page *page, unsigned long order);
782
783 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
784 {
785         struct page *page;
786
787         page = alloc_gigantic_page(nid, huge_page_order(h));
788         if (page) {
789                 prep_compound_gigantic_page(page, huge_page_order(h));
790                 prep_new_huge_page(h, page, nid);
791         }
792
793         return page;
794 }
795
796 static int alloc_fresh_gigantic_page(struct hstate *h,
797                                 nodemask_t *nodes_allowed)
798 {
799         struct page *page = NULL;
800         int nr_nodes, node;
801
802         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
803                 page = alloc_fresh_gigantic_page_node(h, node);
804                 if (page)
805                         return 1;
806         }
807
808         return 0;
809 }
810
811 static inline bool gigantic_page_supported(void) { return true; }
812 #else
813 static inline bool gigantic_page_supported(void) { return false; }
814 static inline void free_gigantic_page(struct page *page, unsigned order) { }
815 static inline void destroy_compound_gigantic_page(struct page *page,
816                                                 unsigned long order) { }
817 static inline int alloc_fresh_gigantic_page(struct hstate *h,
818                                         nodemask_t *nodes_allowed) { return 0; }
819 #endif
820
821 static void update_and_free_page(struct hstate *h, struct page *page)
822 {
823         int i;
824
825         if (hstate_is_gigantic(h) && !gigantic_page_supported())
826                 return;
827
828         h->nr_huge_pages--;
829         h->nr_huge_pages_node[page_to_nid(page)]--;
830         for (i = 0; i < pages_per_huge_page(h); i++) {
831                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
832                                 1 << PG_referenced | 1 << PG_dirty |
833                                 1 << PG_active | 1 << PG_private |
834                                 1 << PG_writeback);
835         }
836         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
837         set_compound_page_dtor(page, NULL);
838         set_page_refcounted(page);
839         if (hstate_is_gigantic(h)) {
840                 destroy_compound_gigantic_page(page, huge_page_order(h));
841                 free_gigantic_page(page, huge_page_order(h));
842         } else {
843                 arch_release_hugepage(page);
844                 __free_pages(page, huge_page_order(h));
845         }
846 }
847
848 struct hstate *size_to_hstate(unsigned long size)
849 {
850         struct hstate *h;
851
852         for_each_hstate(h) {
853                 if (huge_page_size(h) == size)
854                         return h;
855         }
856         return NULL;
857 }
858
859 static void free_huge_page(struct page *page)
860 {
861         /*
862          * Can't pass hstate in here because it is called from the
863          * compound page destructor.
864          */
865         struct hstate *h = page_hstate(page);
866         int nid = page_to_nid(page);
867         struct hugepage_subpool *spool =
868                 (struct hugepage_subpool *)page_private(page);
869         bool restore_reserve;
870
871         set_page_private(page, 0);
872         page->mapping = NULL;
873         BUG_ON(page_count(page));
874         BUG_ON(page_mapcount(page));
875         restore_reserve = PagePrivate(page);
876         ClearPagePrivate(page);
877
878         spin_lock(&hugetlb_lock);
879         hugetlb_cgroup_uncharge_page(hstate_index(h),
880                                      pages_per_huge_page(h), page);
881         if (restore_reserve)
882                 h->resv_huge_pages++;
883
884         if (h->surplus_huge_pages_node[nid]) {
885                 /* remove the page from active list */
886                 list_del(&page->lru);
887                 update_and_free_page(h, page);
888                 h->surplus_huge_pages--;
889                 h->surplus_huge_pages_node[nid]--;
890         } else {
891                 arch_clear_hugepage_flags(page);
892                 enqueue_huge_page(h, page);
893         }
894         spin_unlock(&hugetlb_lock);
895         hugepage_subpool_put_pages(spool, 1);
896 }
897
898 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
899 {
900         INIT_LIST_HEAD(&page->lru);
901         set_compound_page_dtor(page, free_huge_page);
902         spin_lock(&hugetlb_lock);
903         set_hugetlb_cgroup(page, NULL);
904         h->nr_huge_pages++;
905         h->nr_huge_pages_node[nid]++;
906         spin_unlock(&hugetlb_lock);
907         put_page(page); /* free it into the hugepage allocator */
908 }
909
910 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
911 {
912         int i;
913         int nr_pages = 1 << order;
914         struct page *p = page + 1;
915
916         /* we rely on prep_new_huge_page to set the destructor */
917         set_compound_order(page, order);
918         __SetPageHead(page);
919         __ClearPageReserved(page);
920         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
921                 __SetPageTail(p);
922                 /*
923                  * For gigantic hugepages allocated through bootmem at
924                  * boot, it's safer to be consistent with the not-gigantic
925                  * hugepages and clear the PG_reserved bit from all tail pages
926                  * too.  Otherwse drivers using get_user_pages() to access tail
927                  * pages may get the reference counting wrong if they see
928                  * PG_reserved set on a tail page (despite the head page not
929                  * having PG_reserved set).  Enforcing this consistency between
930                  * head and tail pages allows drivers to optimize away a check
931                  * on the head page when they need know if put_page() is needed
932                  * after get_user_pages().
933                  */
934                 __ClearPageReserved(p);
935                 set_page_count(p, 0);
936                 p->first_page = page;
937         }
938 }
939
940 /*
941  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
942  * transparent huge pages.  See the PageTransHuge() documentation for more
943  * details.
944  */
945 int PageHuge(struct page *page)
946 {
947         if (!PageCompound(page))
948                 return 0;
949
950         page = compound_head(page);
951         return get_compound_page_dtor(page) == free_huge_page;
952 }
953 EXPORT_SYMBOL_GPL(PageHuge);
954
955 /*
956  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
957  * normal or transparent huge pages.
958  */
959 int PageHeadHuge(struct page *page_head)
960 {
961         if (!PageHead(page_head))
962                 return 0;
963
964         return get_compound_page_dtor(page_head) == free_huge_page;
965 }
966
967 pgoff_t __basepage_index(struct page *page)
968 {
969         struct page *page_head = compound_head(page);
970         pgoff_t index = page_index(page_head);
971         unsigned long compound_idx;
972
973         if (!PageHuge(page_head))
974                 return page_index(page);
975
976         if (compound_order(page_head) >= MAX_ORDER)
977                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
978         else
979                 compound_idx = page - page_head;
980
981         return (index << compound_order(page_head)) + compound_idx;
982 }
983
984 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
985 {
986         struct page *page;
987
988         page = alloc_pages_exact_node(nid,
989                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
990                                                 __GFP_REPEAT|__GFP_NOWARN,
991                 huge_page_order(h));
992         if (page) {
993                 if (arch_prepare_hugepage(page)) {
994                         __free_pages(page, huge_page_order(h));
995                         return NULL;
996                 }
997                 prep_new_huge_page(h, page, nid);
998         }
999
1000         return page;
1001 }
1002
1003 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1004 {
1005         struct page *page;
1006         int nr_nodes, node;
1007         int ret = 0;
1008
1009         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1010                 page = alloc_fresh_huge_page_node(h, node);
1011                 if (page) {
1012                         ret = 1;
1013                         break;
1014                 }
1015         }
1016
1017         if (ret)
1018                 count_vm_event(HTLB_BUDDY_PGALLOC);
1019         else
1020                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1021
1022         return ret;
1023 }
1024
1025 /*
1026  * Free huge page from pool from next node to free.
1027  * Attempt to keep persistent huge pages more or less
1028  * balanced over allowed nodes.
1029  * Called with hugetlb_lock locked.
1030  */
1031 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1032                                                          bool acct_surplus)
1033 {
1034         int nr_nodes, node;
1035         int ret = 0;
1036
1037         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1038                 /*
1039                  * If we're returning unused surplus pages, only examine
1040                  * nodes with surplus pages.
1041                  */
1042                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1043                     !list_empty(&h->hugepage_freelists[node])) {
1044                         struct page *page =
1045                                 list_entry(h->hugepage_freelists[node].next,
1046                                           struct page, lru);
1047                         list_del(&page->lru);
1048                         h->free_huge_pages--;
1049                         h->free_huge_pages_node[node]--;
1050                         if (acct_surplus) {
1051                                 h->surplus_huge_pages--;
1052                                 h->surplus_huge_pages_node[node]--;
1053                         }
1054                         update_and_free_page(h, page);
1055                         ret = 1;
1056                         break;
1057                 }
1058         }
1059
1060         return ret;
1061 }
1062
1063 /*
1064  * Dissolve a given free hugepage into free buddy pages. This function does
1065  * nothing for in-use (including surplus) hugepages.
1066  */
1067 static void dissolve_free_huge_page(struct page *page)
1068 {
1069         spin_lock(&hugetlb_lock);
1070         if (PageHuge(page) && !page_count(page)) {
1071                 struct hstate *h = page_hstate(page);
1072                 int nid = page_to_nid(page);
1073                 list_del(&page->lru);
1074                 h->free_huge_pages--;
1075                 h->free_huge_pages_node[nid]--;
1076                 update_and_free_page(h, page);
1077         }
1078         spin_unlock(&hugetlb_lock);
1079 }
1080
1081 /*
1082  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1083  * make specified memory blocks removable from the system.
1084  * Note that start_pfn should aligned with (minimum) hugepage size.
1085  */
1086 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1087 {
1088         unsigned int order = 8 * sizeof(void *);
1089         unsigned long pfn;
1090         struct hstate *h;
1091
1092         /* Set scan step to minimum hugepage size */
1093         for_each_hstate(h)
1094                 if (order > huge_page_order(h))
1095                         order = huge_page_order(h);
1096         VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
1097         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
1098                 dissolve_free_huge_page(pfn_to_page(pfn));
1099 }
1100
1101 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
1102 {
1103         struct page *page;
1104         unsigned int r_nid;
1105
1106         if (hstate_is_gigantic(h))
1107                 return NULL;
1108
1109         /*
1110          * Assume we will successfully allocate the surplus page to
1111          * prevent racing processes from causing the surplus to exceed
1112          * overcommit
1113          *
1114          * This however introduces a different race, where a process B
1115          * tries to grow the static hugepage pool while alloc_pages() is
1116          * called by process A. B will only examine the per-node
1117          * counters in determining if surplus huge pages can be
1118          * converted to normal huge pages in adjust_pool_surplus(). A
1119          * won't be able to increment the per-node counter, until the
1120          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1121          * no more huge pages can be converted from surplus to normal
1122          * state (and doesn't try to convert again). Thus, we have a
1123          * case where a surplus huge page exists, the pool is grown, and
1124          * the surplus huge page still exists after, even though it
1125          * should just have been converted to a normal huge page. This
1126          * does not leak memory, though, as the hugepage will be freed
1127          * once it is out of use. It also does not allow the counters to
1128          * go out of whack in adjust_pool_surplus() as we don't modify
1129          * the node values until we've gotten the hugepage and only the
1130          * per-node value is checked there.
1131          */
1132         spin_lock(&hugetlb_lock);
1133         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1134                 spin_unlock(&hugetlb_lock);
1135                 return NULL;
1136         } else {
1137                 h->nr_huge_pages++;
1138                 h->surplus_huge_pages++;
1139         }
1140         spin_unlock(&hugetlb_lock);
1141
1142         if (nid == NUMA_NO_NODE)
1143                 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
1144                                    __GFP_REPEAT|__GFP_NOWARN,
1145                                    huge_page_order(h));
1146         else
1147                 page = alloc_pages_exact_node(nid,
1148                         htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1149                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
1150
1151         if (page && arch_prepare_hugepage(page)) {
1152                 __free_pages(page, huge_page_order(h));
1153                 page = NULL;
1154         }
1155
1156         spin_lock(&hugetlb_lock);
1157         if (page) {
1158                 INIT_LIST_HEAD(&page->lru);
1159                 r_nid = page_to_nid(page);
1160                 set_compound_page_dtor(page, free_huge_page);
1161                 set_hugetlb_cgroup(page, NULL);
1162                 /*
1163                  * We incremented the global counters already
1164                  */
1165                 h->nr_huge_pages_node[r_nid]++;
1166                 h->surplus_huge_pages_node[r_nid]++;
1167                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1168         } else {
1169                 h->nr_huge_pages--;
1170                 h->surplus_huge_pages--;
1171                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1172         }
1173         spin_unlock(&hugetlb_lock);
1174
1175         return page;
1176 }
1177
1178 /*
1179  * This allocation function is useful in the context where vma is irrelevant.
1180  * E.g. soft-offlining uses this function because it only cares physical
1181  * address of error page.
1182  */
1183 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1184 {
1185         struct page *page = NULL;
1186
1187         spin_lock(&hugetlb_lock);
1188         if (h->free_huge_pages - h->resv_huge_pages > 0)
1189                 page = dequeue_huge_page_node(h, nid);
1190         spin_unlock(&hugetlb_lock);
1191
1192         if (!page)
1193                 page = alloc_buddy_huge_page(h, nid);
1194
1195         return page;
1196 }
1197
1198 /*
1199  * Increase the hugetlb pool such that it can accommodate a reservation
1200  * of size 'delta'.
1201  */
1202 static int gather_surplus_pages(struct hstate *h, int delta)
1203 {
1204         struct list_head surplus_list;
1205         struct page *page, *tmp;
1206         int ret, i;
1207         int needed, allocated;
1208         bool alloc_ok = true;
1209
1210         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1211         if (needed <= 0) {
1212                 h->resv_huge_pages += delta;
1213                 return 0;
1214         }
1215
1216         allocated = 0;
1217         INIT_LIST_HEAD(&surplus_list);
1218
1219         ret = -ENOMEM;
1220 retry:
1221         spin_unlock(&hugetlb_lock);
1222         for (i = 0; i < needed; i++) {
1223                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1224                 if (!page) {
1225                         alloc_ok = false;
1226                         break;
1227                 }
1228                 list_add(&page->lru, &surplus_list);
1229         }
1230         allocated += i;
1231
1232         /*
1233          * After retaking hugetlb_lock, we need to recalculate 'needed'
1234          * because either resv_huge_pages or free_huge_pages may have changed.
1235          */
1236         spin_lock(&hugetlb_lock);
1237         needed = (h->resv_huge_pages + delta) -
1238                         (h->free_huge_pages + allocated);
1239         if (needed > 0) {
1240                 if (alloc_ok)
1241                         goto retry;
1242                 /*
1243                  * We were not able to allocate enough pages to
1244                  * satisfy the entire reservation so we free what
1245                  * we've allocated so far.
1246                  */
1247                 goto free;
1248         }
1249         /*
1250          * The surplus_list now contains _at_least_ the number of extra pages
1251          * needed to accommodate the reservation.  Add the appropriate number
1252          * of pages to the hugetlb pool and free the extras back to the buddy
1253          * allocator.  Commit the entire reservation here to prevent another
1254          * process from stealing the pages as they are added to the pool but
1255          * before they are reserved.
1256          */
1257         needed += allocated;
1258         h->resv_huge_pages += delta;
1259         ret = 0;
1260
1261         /* Free the needed pages to the hugetlb pool */
1262         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1263                 if ((--needed) < 0)
1264                         break;
1265                 /*
1266                  * This page is now managed by the hugetlb allocator and has
1267                  * no users -- drop the buddy allocator's reference.
1268                  */
1269                 put_page_testzero(page);
1270                 VM_BUG_ON_PAGE(page_count(page), page);
1271                 enqueue_huge_page(h, page);
1272         }
1273 free:
1274         spin_unlock(&hugetlb_lock);
1275
1276         /* Free unnecessary surplus pages to the buddy allocator */
1277         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1278                 put_page(page);
1279         spin_lock(&hugetlb_lock);
1280
1281         return ret;
1282 }
1283
1284 /*
1285  * When releasing a hugetlb pool reservation, any surplus pages that were
1286  * allocated to satisfy the reservation must be explicitly freed if they were
1287  * never used.
1288  * Called with hugetlb_lock held.
1289  */
1290 static void return_unused_surplus_pages(struct hstate *h,
1291                                         unsigned long unused_resv_pages)
1292 {
1293         unsigned long nr_pages;
1294
1295         /* Uncommit the reservation */
1296         h->resv_huge_pages -= unused_resv_pages;
1297
1298         /* Cannot return gigantic pages currently */
1299         if (hstate_is_gigantic(h))
1300                 return;
1301
1302         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1303
1304         /*
1305          * We want to release as many surplus pages as possible, spread
1306          * evenly across all nodes with memory. Iterate across these nodes
1307          * until we can no longer free unreserved surplus pages. This occurs
1308          * when the nodes with surplus pages have no free pages.
1309          * free_pool_huge_page() will balance the the freed pages across the
1310          * on-line nodes with memory and will handle the hstate accounting.
1311          */
1312         while (nr_pages--) {
1313                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1314                         break;
1315                 cond_resched_lock(&hugetlb_lock);
1316         }
1317 }
1318
1319 /*
1320  * Determine if the huge page at addr within the vma has an associated
1321  * reservation.  Where it does not we will need to logically increase
1322  * reservation and actually increase subpool usage before an allocation
1323  * can occur.  Where any new reservation would be required the
1324  * reservation change is prepared, but not committed.  Once the page
1325  * has been allocated from the subpool and instantiated the change should
1326  * be committed via vma_commit_reservation.  No action is required on
1327  * failure.
1328  */
1329 static long vma_needs_reservation(struct hstate *h,
1330                         struct vm_area_struct *vma, unsigned long addr)
1331 {
1332         struct resv_map *resv;
1333         pgoff_t idx;
1334         long chg;
1335
1336         resv = vma_resv_map(vma);
1337         if (!resv)
1338                 return 1;
1339
1340         idx = vma_hugecache_offset(h, vma, addr);
1341         chg = region_chg(resv, idx, idx + 1);
1342
1343         if (vma->vm_flags & VM_MAYSHARE)
1344                 return chg;
1345         else
1346                 return chg < 0 ? chg : 0;
1347 }
1348 static void vma_commit_reservation(struct hstate *h,
1349                         struct vm_area_struct *vma, unsigned long addr)
1350 {
1351         struct resv_map *resv;
1352         pgoff_t idx;
1353
1354         resv = vma_resv_map(vma);
1355         if (!resv)
1356                 return;
1357
1358         idx = vma_hugecache_offset(h, vma, addr);
1359         region_add(resv, idx, idx + 1);
1360 }
1361
1362 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1363                                     unsigned long addr, int avoid_reserve)
1364 {
1365         struct hugepage_subpool *spool = subpool_vma(vma);
1366         struct hstate *h = hstate_vma(vma);
1367         struct page *page;
1368         long chg;
1369         int ret, idx;
1370         struct hugetlb_cgroup *h_cg;
1371
1372         idx = hstate_index(h);
1373         /*
1374          * Processes that did not create the mapping will have no
1375          * reserves and will not have accounted against subpool
1376          * limit. Check that the subpool limit can be made before
1377          * satisfying the allocation MAP_NORESERVE mappings may also
1378          * need pages and subpool limit allocated allocated if no reserve
1379          * mapping overlaps.
1380          */
1381         chg = vma_needs_reservation(h, vma, addr);
1382         if (chg < 0)
1383                 return ERR_PTR(-ENOMEM);
1384         if (chg || avoid_reserve)
1385                 if (hugepage_subpool_get_pages(spool, 1))
1386                         return ERR_PTR(-ENOSPC);
1387
1388         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1389         if (ret)
1390                 goto out_subpool_put;
1391
1392         spin_lock(&hugetlb_lock);
1393         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1394         if (!page) {
1395                 spin_unlock(&hugetlb_lock);
1396                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1397                 if (!page)
1398                         goto out_uncharge_cgroup;
1399
1400                 spin_lock(&hugetlb_lock);
1401                 list_move(&page->lru, &h->hugepage_activelist);
1402                 /* Fall through */
1403         }
1404         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1405         spin_unlock(&hugetlb_lock);
1406
1407         set_page_private(page, (unsigned long)spool);
1408
1409         vma_commit_reservation(h, vma, addr);
1410         return page;
1411
1412 out_uncharge_cgroup:
1413         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1414 out_subpool_put:
1415         if (chg || avoid_reserve)
1416                 hugepage_subpool_put_pages(spool, 1);
1417         return ERR_PTR(-ENOSPC);
1418 }
1419
1420 /*
1421  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1422  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1423  * where no ERR_VALUE is expected to be returned.
1424  */
1425 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1426                                 unsigned long addr, int avoid_reserve)
1427 {
1428         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1429         if (IS_ERR(page))
1430                 page = NULL;
1431         return page;
1432 }
1433
1434 int __weak alloc_bootmem_huge_page(struct hstate *h)
1435 {
1436         struct huge_bootmem_page *m;
1437         int nr_nodes, node;
1438
1439         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1440                 void *addr;
1441
1442                 addr = memblock_virt_alloc_try_nid_nopanic(
1443                                 huge_page_size(h), huge_page_size(h),
1444                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
1445                 if (addr) {
1446                         /*
1447                          * Use the beginning of the huge page to store the
1448                          * huge_bootmem_page struct (until gather_bootmem
1449                          * puts them into the mem_map).
1450                          */
1451                         m = addr;
1452                         goto found;
1453                 }
1454         }
1455         return 0;
1456
1457 found:
1458         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1459         /* Put them into a private list first because mem_map is not up yet */
1460         list_add(&m->list, &huge_boot_pages);
1461         m->hstate = h;
1462         return 1;
1463 }
1464
1465 static void __init prep_compound_huge_page(struct page *page, int order)
1466 {
1467         if (unlikely(order > (MAX_ORDER - 1)))
1468                 prep_compound_gigantic_page(page, order);
1469         else
1470                 prep_compound_page(page, order);
1471 }
1472
1473 /* Put bootmem huge pages into the standard lists after mem_map is up */
1474 static void __init gather_bootmem_prealloc(void)
1475 {
1476         struct huge_bootmem_page *m;
1477
1478         list_for_each_entry(m, &huge_boot_pages, list) {
1479                 struct hstate *h = m->hstate;
1480                 struct page *page;
1481
1482 #ifdef CONFIG_HIGHMEM
1483                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1484                 memblock_free_late(__pa(m),
1485                                    sizeof(struct huge_bootmem_page));
1486 #else
1487                 page = virt_to_page(m);
1488 #endif
1489                 WARN_ON(page_count(page) != 1);
1490                 prep_compound_huge_page(page, h->order);
1491                 WARN_ON(PageReserved(page));
1492                 prep_new_huge_page(h, page, page_to_nid(page));
1493                 /*
1494                  * If we had gigantic hugepages allocated at boot time, we need
1495                  * to restore the 'stolen' pages to totalram_pages in order to
1496                  * fix confusing memory reports from free(1) and another
1497                  * side-effects, like CommitLimit going negative.
1498                  */
1499                 if (hstate_is_gigantic(h))
1500                         adjust_managed_page_count(page, 1 << h->order);
1501         }
1502 }
1503
1504 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1505 {
1506         unsigned long i;
1507
1508         for (i = 0; i < h->max_huge_pages; ++i) {
1509                 if (hstate_is_gigantic(h)) {
1510                         if (!alloc_bootmem_huge_page(h))
1511                                 break;
1512                 } else if (!alloc_fresh_huge_page(h,
1513                                          &node_states[N_MEMORY]))
1514                         break;
1515         }
1516         h->max_huge_pages = i;
1517 }
1518
1519 static void __init hugetlb_init_hstates(void)
1520 {
1521         struct hstate *h;
1522
1523         for_each_hstate(h) {
1524                 /* oversize hugepages were init'ed in early boot */
1525                 if (!hstate_is_gigantic(h))
1526                         hugetlb_hstate_alloc_pages(h);
1527         }
1528 }
1529
1530 static char * __init memfmt(char *buf, unsigned long n)
1531 {
1532         if (n >= (1UL << 30))
1533                 sprintf(buf, "%lu GB", n >> 30);
1534         else if (n >= (1UL << 20))
1535                 sprintf(buf, "%lu MB", n >> 20);
1536         else
1537                 sprintf(buf, "%lu KB", n >> 10);
1538         return buf;
1539 }
1540
1541 static void __init report_hugepages(void)
1542 {
1543         struct hstate *h;
1544
1545         for_each_hstate(h) {
1546                 char buf[32];
1547                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1548                         memfmt(buf, huge_page_size(h)),
1549                         h->free_huge_pages);
1550         }
1551 }
1552
1553 #ifdef CONFIG_HIGHMEM
1554 static void try_to_free_low(struct hstate *h, unsigned long count,
1555                                                 nodemask_t *nodes_allowed)
1556 {
1557         int i;
1558
1559         if (hstate_is_gigantic(h))
1560                 return;
1561
1562         for_each_node_mask(i, *nodes_allowed) {
1563                 struct page *page, *next;
1564                 struct list_head *freel = &h->hugepage_freelists[i];
1565                 list_for_each_entry_safe(page, next, freel, lru) {
1566                         if (count >= h->nr_huge_pages)
1567                                 return;
1568                         if (PageHighMem(page))
1569                                 continue;
1570                         list_del(&page->lru);
1571                         update_and_free_page(h, page);
1572                         h->free_huge_pages--;
1573                         h->free_huge_pages_node[page_to_nid(page)]--;
1574                 }
1575         }
1576 }
1577 #else
1578 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1579                                                 nodemask_t *nodes_allowed)
1580 {
1581 }
1582 #endif
1583
1584 /*
1585  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1586  * balanced by operating on them in a round-robin fashion.
1587  * Returns 1 if an adjustment was made.
1588  */
1589 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1590                                 int delta)
1591 {
1592         int nr_nodes, node;
1593
1594         VM_BUG_ON(delta != -1 && delta != 1);
1595
1596         if (delta < 0) {
1597                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1598                         if (h->surplus_huge_pages_node[node])
1599                                 goto found;
1600                 }
1601         } else {
1602                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1603                         if (h->surplus_huge_pages_node[node] <
1604                                         h->nr_huge_pages_node[node])
1605                                 goto found;
1606                 }
1607         }
1608         return 0;
1609
1610 found:
1611         h->surplus_huge_pages += delta;
1612         h->surplus_huge_pages_node[node] += delta;
1613         return 1;
1614 }
1615
1616 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1617 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1618                                                 nodemask_t *nodes_allowed)
1619 {
1620         unsigned long min_count, ret;
1621
1622         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1623                 return h->max_huge_pages;
1624
1625         /*
1626          * Increase the pool size
1627          * First take pages out of surplus state.  Then make up the
1628          * remaining difference by allocating fresh huge pages.
1629          *
1630          * We might race with alloc_buddy_huge_page() here and be unable
1631          * to convert a surplus huge page to a normal huge page. That is
1632          * not critical, though, it just means the overall size of the
1633          * pool might be one hugepage larger than it needs to be, but
1634          * within all the constraints specified by the sysctls.
1635          */
1636         spin_lock(&hugetlb_lock);
1637         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1638                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1639                         break;
1640         }
1641
1642         while (count > persistent_huge_pages(h)) {
1643                 /*
1644                  * If this allocation races such that we no longer need the
1645                  * page, free_huge_page will handle it by freeing the page
1646                  * and reducing the surplus.
1647                  */
1648                 spin_unlock(&hugetlb_lock);
1649                 if (hstate_is_gigantic(h))
1650                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
1651                 else
1652                         ret = alloc_fresh_huge_page(h, nodes_allowed);
1653                 spin_lock(&hugetlb_lock);
1654                 if (!ret)
1655                         goto out;
1656
1657                 /* Bail for signals. Probably ctrl-c from user */
1658                 if (signal_pending(current))
1659                         goto out;
1660         }
1661
1662         /*
1663          * Decrease the pool size
1664          * First return free pages to the buddy allocator (being careful
1665          * to keep enough around to satisfy reservations).  Then place
1666          * pages into surplus state as needed so the pool will shrink
1667          * to the desired size as pages become free.
1668          *
1669          * By placing pages into the surplus state independent of the
1670          * overcommit value, we are allowing the surplus pool size to
1671          * exceed overcommit. There are few sane options here. Since
1672          * alloc_buddy_huge_page() is checking the global counter,
1673          * though, we'll note that we're not allowed to exceed surplus
1674          * and won't grow the pool anywhere else. Not until one of the
1675          * sysctls are changed, or the surplus pages go out of use.
1676          */
1677         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1678         min_count = max(count, min_count);
1679         try_to_free_low(h, min_count, nodes_allowed);
1680         while (min_count < persistent_huge_pages(h)) {
1681                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1682                         break;
1683                 cond_resched_lock(&hugetlb_lock);
1684         }
1685         while (count < persistent_huge_pages(h)) {
1686                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1687                         break;
1688         }
1689 out:
1690         ret = persistent_huge_pages(h);
1691         spin_unlock(&hugetlb_lock);
1692         return ret;
1693 }
1694
1695 #define HSTATE_ATTR_RO(_name) \
1696         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1697
1698 #define HSTATE_ATTR(_name) \
1699         static struct kobj_attribute _name##_attr = \
1700                 __ATTR(_name, 0644, _name##_show, _name##_store)
1701
1702 static struct kobject *hugepages_kobj;
1703 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1704
1705 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1706
1707 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1708 {
1709         int i;
1710
1711         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1712                 if (hstate_kobjs[i] == kobj) {
1713                         if (nidp)
1714                                 *nidp = NUMA_NO_NODE;
1715                         return &hstates[i];
1716                 }
1717
1718         return kobj_to_node_hstate(kobj, nidp);
1719 }
1720
1721 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1722                                         struct kobj_attribute *attr, char *buf)
1723 {
1724         struct hstate *h;
1725         unsigned long nr_huge_pages;
1726         int nid;
1727
1728         h = kobj_to_hstate(kobj, &nid);
1729         if (nid == NUMA_NO_NODE)
1730                 nr_huge_pages = h->nr_huge_pages;
1731         else
1732                 nr_huge_pages = h->nr_huge_pages_node[nid];
1733
1734         return sprintf(buf, "%lu\n", nr_huge_pages);
1735 }
1736
1737 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1738                         struct kobject *kobj, struct kobj_attribute *attr,
1739                         const char *buf, size_t len)
1740 {
1741         int err;
1742         int nid;
1743         unsigned long count;
1744         struct hstate *h;
1745         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1746
1747         err = kstrtoul(buf, 10, &count);
1748         if (err)
1749                 goto out;
1750
1751         h = kobj_to_hstate(kobj, &nid);
1752         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
1753                 err = -EINVAL;
1754                 goto out;
1755         }
1756
1757         if (nid == NUMA_NO_NODE) {
1758                 /*
1759                  * global hstate attribute
1760                  */
1761                 if (!(obey_mempolicy &&
1762                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1763                         NODEMASK_FREE(nodes_allowed);
1764                         nodes_allowed = &node_states[N_MEMORY];
1765                 }
1766         } else if (nodes_allowed) {
1767                 /*
1768                  * per node hstate attribute: adjust count to global,
1769                  * but restrict alloc/free to the specified node.
1770                  */
1771                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1772                 init_nodemask_of_node(nodes_allowed, nid);
1773         } else
1774                 nodes_allowed = &node_states[N_MEMORY];
1775
1776         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1777
1778         if (nodes_allowed != &node_states[N_MEMORY])
1779                 NODEMASK_FREE(nodes_allowed);
1780
1781         return len;
1782 out:
1783         NODEMASK_FREE(nodes_allowed);
1784         return err;
1785 }
1786
1787 static ssize_t nr_hugepages_show(struct kobject *kobj,
1788                                        struct kobj_attribute *attr, char *buf)
1789 {
1790         return nr_hugepages_show_common(kobj, attr, buf);
1791 }
1792
1793 static ssize_t nr_hugepages_store(struct kobject *kobj,
1794                struct kobj_attribute *attr, const char *buf, size_t len)
1795 {
1796         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1797 }
1798 HSTATE_ATTR(nr_hugepages);
1799
1800 #ifdef CONFIG_NUMA
1801
1802 /*
1803  * hstate attribute for optionally mempolicy-based constraint on persistent
1804  * huge page alloc/free.
1805  */
1806 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1807                                        struct kobj_attribute *attr, char *buf)
1808 {
1809         return nr_hugepages_show_common(kobj, attr, buf);
1810 }
1811
1812 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1813                struct kobj_attribute *attr, const char *buf, size_t len)
1814 {
1815         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1816 }
1817 HSTATE_ATTR(nr_hugepages_mempolicy);
1818 #endif
1819
1820
1821 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1822                                         struct kobj_attribute *attr, char *buf)
1823 {
1824         struct hstate *h = kobj_to_hstate(kobj, NULL);
1825         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1826 }
1827
1828 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1829                 struct kobj_attribute *attr, const char *buf, size_t count)
1830 {
1831         int err;
1832         unsigned long input;
1833         struct hstate *h = kobj_to_hstate(kobj, NULL);
1834
1835         if (hstate_is_gigantic(h))
1836                 return -EINVAL;
1837
1838         err = kstrtoul(buf, 10, &input);
1839         if (err)
1840                 return err;
1841
1842         spin_lock(&hugetlb_lock);
1843         h->nr_overcommit_huge_pages = input;
1844         spin_unlock(&hugetlb_lock);
1845
1846         return count;
1847 }
1848 HSTATE_ATTR(nr_overcommit_hugepages);
1849
1850 static ssize_t free_hugepages_show(struct kobject *kobj,
1851                                         struct kobj_attribute *attr, char *buf)
1852 {
1853         struct hstate *h;
1854         unsigned long free_huge_pages;
1855         int nid;
1856
1857         h = kobj_to_hstate(kobj, &nid);
1858         if (nid == NUMA_NO_NODE)
1859                 free_huge_pages = h->free_huge_pages;
1860         else
1861                 free_huge_pages = h->free_huge_pages_node[nid];
1862
1863         return sprintf(buf, "%lu\n", free_huge_pages);
1864 }
1865 HSTATE_ATTR_RO(free_hugepages);
1866
1867 static ssize_t resv_hugepages_show(struct kobject *kobj,
1868                                         struct kobj_attribute *attr, char *buf)
1869 {
1870         struct hstate *h = kobj_to_hstate(kobj, NULL);
1871         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1872 }
1873 HSTATE_ATTR_RO(resv_hugepages);
1874
1875 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1876                                         struct kobj_attribute *attr, char *buf)
1877 {
1878         struct hstate *h;
1879         unsigned long surplus_huge_pages;
1880         int nid;
1881
1882         h = kobj_to_hstate(kobj, &nid);
1883         if (nid == NUMA_NO_NODE)
1884                 surplus_huge_pages = h->surplus_huge_pages;
1885         else
1886                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1887
1888         return sprintf(buf, "%lu\n", surplus_huge_pages);
1889 }
1890 HSTATE_ATTR_RO(surplus_hugepages);
1891
1892 static struct attribute *hstate_attrs[] = {
1893         &nr_hugepages_attr.attr,
1894         &nr_overcommit_hugepages_attr.attr,
1895         &free_hugepages_attr.attr,
1896         &resv_hugepages_attr.attr,
1897         &surplus_hugepages_attr.attr,
1898 #ifdef CONFIG_NUMA
1899         &nr_hugepages_mempolicy_attr.attr,
1900 #endif
1901         NULL,
1902 };
1903
1904 static struct attribute_group hstate_attr_group = {
1905         .attrs = hstate_attrs,
1906 };
1907
1908 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1909                                     struct kobject **hstate_kobjs,
1910                                     struct attribute_group *hstate_attr_group)
1911 {
1912         int retval;
1913         int hi = hstate_index(h);
1914
1915         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1916         if (!hstate_kobjs[hi])
1917                 return -ENOMEM;
1918
1919         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1920         if (retval)
1921                 kobject_put(hstate_kobjs[hi]);
1922
1923         return retval;
1924 }
1925
1926 static void __init hugetlb_sysfs_init(void)
1927 {
1928         struct hstate *h;
1929         int err;
1930
1931         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1932         if (!hugepages_kobj)
1933                 return;
1934
1935         for_each_hstate(h) {
1936                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1937                                          hstate_kobjs, &hstate_attr_group);
1938                 if (err)
1939                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
1940         }
1941 }
1942
1943 #ifdef CONFIG_NUMA
1944
1945 /*
1946  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1947  * with node devices in node_devices[] using a parallel array.  The array
1948  * index of a node device or _hstate == node id.
1949  * This is here to avoid any static dependency of the node device driver, in
1950  * the base kernel, on the hugetlb module.
1951  */
1952 struct node_hstate {
1953         struct kobject          *hugepages_kobj;
1954         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1955 };
1956 struct node_hstate node_hstates[MAX_NUMNODES];
1957
1958 /*
1959  * A subset of global hstate attributes for node devices
1960  */
1961 static struct attribute *per_node_hstate_attrs[] = {
1962         &nr_hugepages_attr.attr,
1963         &free_hugepages_attr.attr,
1964         &surplus_hugepages_attr.attr,
1965         NULL,
1966 };
1967
1968 static struct attribute_group per_node_hstate_attr_group = {
1969         .attrs = per_node_hstate_attrs,
1970 };
1971
1972 /*
1973  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1974  * Returns node id via non-NULL nidp.
1975  */
1976 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1977 {
1978         int nid;
1979
1980         for (nid = 0; nid < nr_node_ids; nid++) {
1981                 struct node_hstate *nhs = &node_hstates[nid];
1982                 int i;
1983                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1984                         if (nhs->hstate_kobjs[i] == kobj) {
1985                                 if (nidp)
1986                                         *nidp = nid;
1987                                 return &hstates[i];
1988                         }
1989         }
1990
1991         BUG();
1992         return NULL;
1993 }
1994
1995 /*
1996  * Unregister hstate attributes from a single node device.
1997  * No-op if no hstate attributes attached.
1998  */
1999 static void hugetlb_unregister_node(struct node *node)
2000 {
2001         struct hstate *h;
2002         struct node_hstate *nhs = &node_hstates[node->dev.id];
2003
2004         if (!nhs->hugepages_kobj)
2005                 return;         /* no hstate attributes */
2006
2007         for_each_hstate(h) {
2008                 int idx = hstate_index(h);
2009                 if (nhs->hstate_kobjs[idx]) {
2010                         kobject_put(nhs->hstate_kobjs[idx]);
2011                         nhs->hstate_kobjs[idx] = NULL;
2012                 }
2013         }
2014
2015         kobject_put(nhs->hugepages_kobj);
2016         nhs->hugepages_kobj = NULL;
2017 }
2018
2019 /*
2020  * hugetlb module exit:  unregister hstate attributes from node devices
2021  * that have them.
2022  */
2023 static void hugetlb_unregister_all_nodes(void)
2024 {
2025         int nid;
2026
2027         /*
2028          * disable node device registrations.
2029          */
2030         register_hugetlbfs_with_node(NULL, NULL);
2031
2032         /*
2033          * remove hstate attributes from any nodes that have them.
2034          */
2035         for (nid = 0; nid < nr_node_ids; nid++)
2036                 hugetlb_unregister_node(node_devices[nid]);
2037 }
2038
2039 /*
2040  * Register hstate attributes for a single node device.
2041  * No-op if attributes already registered.
2042  */
2043 static void hugetlb_register_node(struct node *node)
2044 {
2045         struct hstate *h;
2046         struct node_hstate *nhs = &node_hstates[node->dev.id];
2047         int err;
2048
2049         if (nhs->hugepages_kobj)
2050                 return;         /* already allocated */
2051
2052         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2053                                                         &node->dev.kobj);
2054         if (!nhs->hugepages_kobj)
2055                 return;
2056
2057         for_each_hstate(h) {
2058                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2059                                                 nhs->hstate_kobjs,
2060                                                 &per_node_hstate_attr_group);
2061                 if (err) {
2062                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2063                                 h->name, node->dev.id);
2064                         hugetlb_unregister_node(node);
2065                         break;
2066                 }
2067         }
2068 }
2069
2070 /*
2071  * hugetlb init time:  register hstate attributes for all registered node
2072  * devices of nodes that have memory.  All on-line nodes should have
2073  * registered their associated device by this time.
2074  */
2075 static void hugetlb_register_all_nodes(void)
2076 {
2077         int nid;
2078
2079         for_each_node_state(nid, N_MEMORY) {
2080                 struct node *node = node_devices[nid];
2081                 if (node->dev.id == nid)
2082                         hugetlb_register_node(node);
2083         }
2084
2085         /*
2086          * Let the node device driver know we're here so it can
2087          * [un]register hstate attributes on node hotplug.
2088          */
2089         register_hugetlbfs_with_node(hugetlb_register_node,
2090                                      hugetlb_unregister_node);
2091 }
2092 #else   /* !CONFIG_NUMA */
2093
2094 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2095 {
2096         BUG();
2097         if (nidp)
2098                 *nidp = -1;
2099         return NULL;
2100 }
2101
2102 static void hugetlb_unregister_all_nodes(void) { }
2103
2104 static void hugetlb_register_all_nodes(void) { }
2105
2106 #endif
2107
2108 static void __exit hugetlb_exit(void)
2109 {
2110         struct hstate *h;
2111
2112         hugetlb_unregister_all_nodes();
2113
2114         for_each_hstate(h) {
2115                 kobject_put(hstate_kobjs[hstate_index(h)]);
2116         }
2117
2118         kobject_put(hugepages_kobj);
2119         kfree(htlb_fault_mutex_table);
2120 }
2121 module_exit(hugetlb_exit);
2122
2123 static int __init hugetlb_init(void)
2124 {
2125         int i;
2126
2127         if (!hugepages_supported())
2128                 return 0;
2129
2130         if (!size_to_hstate(default_hstate_size)) {
2131                 default_hstate_size = HPAGE_SIZE;
2132                 if (!size_to_hstate(default_hstate_size))
2133                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2134         }
2135         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2136         if (default_hstate_max_huge_pages)
2137                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2138
2139         hugetlb_init_hstates();
2140         gather_bootmem_prealloc();
2141         report_hugepages();
2142
2143         hugetlb_sysfs_init();
2144         hugetlb_register_all_nodes();
2145         hugetlb_cgroup_file_init();
2146
2147 #ifdef CONFIG_SMP
2148         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2149 #else
2150         num_fault_mutexes = 1;
2151 #endif
2152         htlb_fault_mutex_table =
2153                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2154         BUG_ON(!htlb_fault_mutex_table);
2155
2156         for (i = 0; i < num_fault_mutexes; i++)
2157                 mutex_init(&htlb_fault_mutex_table[i]);
2158         return 0;
2159 }
2160 module_init(hugetlb_init);
2161
2162 /* Should be called on processing a hugepagesz=... option */
2163 void __init hugetlb_add_hstate(unsigned order)
2164 {
2165         struct hstate *h;
2166         unsigned long i;
2167
2168         if (size_to_hstate(PAGE_SIZE << order)) {
2169                 pr_warning("hugepagesz= specified twice, ignoring\n");
2170                 return;
2171         }
2172         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2173         BUG_ON(order == 0);
2174         h = &hstates[hugetlb_max_hstate++];
2175         h->order = order;
2176         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2177         h->nr_huge_pages = 0;
2178         h->free_huge_pages = 0;
2179         for (i = 0; i < MAX_NUMNODES; ++i)
2180                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2181         INIT_LIST_HEAD(&h->hugepage_activelist);
2182         h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2183         h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2184         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2185                                         huge_page_size(h)/1024);
2186
2187         parsed_hstate = h;
2188 }
2189
2190 static int __init hugetlb_nrpages_setup(char *s)
2191 {
2192         unsigned long *mhp;
2193         static unsigned long *last_mhp;
2194
2195         /*
2196          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2197          * so this hugepages= parameter goes to the "default hstate".
2198          */
2199         if (!hugetlb_max_hstate)
2200                 mhp = &default_hstate_max_huge_pages;
2201         else
2202                 mhp = &parsed_hstate->max_huge_pages;
2203
2204         if (mhp == last_mhp) {
2205                 pr_warning("hugepages= specified twice without "
2206                            "interleaving hugepagesz=, ignoring\n");
2207                 return 1;
2208         }
2209
2210         if (sscanf(s, "%lu", mhp) <= 0)
2211                 *mhp = 0;
2212
2213         /*
2214          * Global state is always initialized later in hugetlb_init.
2215          * But we need to allocate >= MAX_ORDER hstates here early to still
2216          * use the bootmem allocator.
2217          */
2218         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2219                 hugetlb_hstate_alloc_pages(parsed_hstate);
2220
2221         last_mhp = mhp;
2222
2223         return 1;
2224 }
2225 __setup("hugepages=", hugetlb_nrpages_setup);
2226
2227 static int __init hugetlb_default_setup(char *s)
2228 {
2229         default_hstate_size = memparse(s, &s);
2230         return 1;
2231 }
2232 __setup("default_hugepagesz=", hugetlb_default_setup);
2233
2234 static unsigned int cpuset_mems_nr(unsigned int *array)
2235 {
2236         int node;
2237         unsigned int nr = 0;
2238
2239         for_each_node_mask(node, cpuset_current_mems_allowed)
2240                 nr += array[node];
2241
2242         return nr;
2243 }
2244
2245 #ifdef CONFIG_SYSCTL
2246 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2247                          struct ctl_table *table, int write,
2248                          void __user *buffer, size_t *length, loff_t *ppos)
2249 {
2250         struct hstate *h = &default_hstate;
2251         unsigned long tmp;
2252         int ret;
2253
2254         if (!hugepages_supported())
2255                 return -ENOTSUPP;
2256
2257         tmp = h->max_huge_pages;
2258
2259         if (write && hstate_is_gigantic(h) && !gigantic_page_supported())
2260                 return -EINVAL;
2261
2262         table->data = &tmp;
2263         table->maxlen = sizeof(unsigned long);
2264         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2265         if (ret)
2266                 goto out;
2267
2268         if (write) {
2269                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
2270                                                 GFP_KERNEL | __GFP_NORETRY);
2271                 if (!(obey_mempolicy &&
2272                                init_nodemask_of_mempolicy(nodes_allowed))) {
2273                         NODEMASK_FREE(nodes_allowed);
2274                         nodes_allowed = &node_states[N_MEMORY];
2275                 }
2276                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2277
2278                 if (nodes_allowed != &node_states[N_MEMORY])
2279                         NODEMASK_FREE(nodes_allowed);
2280         }
2281 out:
2282         return ret;
2283 }
2284
2285 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2286                           void __user *buffer, size_t *length, loff_t *ppos)
2287 {
2288
2289         return hugetlb_sysctl_handler_common(false, table, write,
2290                                                         buffer, length, ppos);
2291 }
2292
2293 #ifdef CONFIG_NUMA
2294 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2295                           void __user *buffer, size_t *length, loff_t *ppos)
2296 {
2297         return hugetlb_sysctl_handler_common(true, table, write,
2298                                                         buffer, length, ppos);
2299 }
2300 #endif /* CONFIG_NUMA */
2301
2302 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2303                         void __user *buffer,
2304                         size_t *length, loff_t *ppos)
2305 {
2306         struct hstate *h = &default_hstate;
2307         unsigned long tmp;
2308         int ret;
2309
2310         if (!hugepages_supported())
2311                 return -ENOTSUPP;
2312
2313         tmp = h->nr_overcommit_huge_pages;
2314
2315         if (write && hstate_is_gigantic(h))
2316                 return -EINVAL;
2317
2318         table->data = &tmp;
2319         table->maxlen = sizeof(unsigned long);
2320         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2321         if (ret)
2322                 goto out;
2323
2324         if (write) {
2325                 spin_lock(&hugetlb_lock);
2326                 h->nr_overcommit_huge_pages = tmp;
2327                 spin_unlock(&hugetlb_lock);
2328         }
2329 out:
2330         return ret;
2331 }
2332
2333 #endif /* CONFIG_SYSCTL */
2334
2335 void hugetlb_report_meminfo(struct seq_file *m)
2336 {
2337         struct hstate *h = &default_hstate;
2338         if (!hugepages_supported())
2339                 return;
2340         seq_printf(m,
2341                         "HugePages_Total:   %5lu\n"
2342                         "HugePages_Free:    %5lu\n"
2343                         "HugePages_Rsvd:    %5lu\n"
2344                         "HugePages_Surp:    %5lu\n"
2345                         "Hugepagesize:   %8lu kB\n",
2346                         h->nr_huge_pages,
2347                         h->free_huge_pages,
2348                         h->resv_huge_pages,
2349                         h->surplus_huge_pages,
2350                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2351 }
2352
2353 int hugetlb_report_node_meminfo(int nid, char *buf)
2354 {
2355         struct hstate *h = &default_hstate;
2356         if (!hugepages_supported())
2357                 return 0;
2358         return sprintf(buf,
2359                 "Node %d HugePages_Total: %5u\n"
2360                 "Node %d HugePages_Free:  %5u\n"
2361                 "Node %d HugePages_Surp:  %5u\n",
2362                 nid, h->nr_huge_pages_node[nid],
2363                 nid, h->free_huge_pages_node[nid],
2364                 nid, h->surplus_huge_pages_node[nid]);
2365 }
2366
2367 void hugetlb_show_meminfo(void)
2368 {
2369         struct hstate *h;
2370         int nid;
2371
2372         if (!hugepages_supported())
2373                 return;
2374
2375         for_each_node_state(nid, N_MEMORY)
2376                 for_each_hstate(h)
2377                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2378                                 nid,
2379                                 h->nr_huge_pages_node[nid],
2380                                 h->free_huge_pages_node[nid],
2381                                 h->surplus_huge_pages_node[nid],
2382                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2383 }
2384
2385 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2386 unsigned long hugetlb_total_pages(void)
2387 {
2388         struct hstate *h;
2389         unsigned long nr_total_pages = 0;
2390
2391         for_each_hstate(h)
2392                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2393         return nr_total_pages;
2394 }
2395
2396 static int hugetlb_acct_memory(struct hstate *h, long delta)
2397 {
2398         int ret = -ENOMEM;
2399
2400         spin_lock(&hugetlb_lock);
2401         /*
2402          * When cpuset is configured, it breaks the strict hugetlb page
2403          * reservation as the accounting is done on a global variable. Such
2404          * reservation is completely rubbish in the presence of cpuset because
2405          * the reservation is not checked against page availability for the
2406          * current cpuset. Application can still potentially OOM'ed by kernel
2407          * with lack of free htlb page in cpuset that the task is in.
2408          * Attempt to enforce strict accounting with cpuset is almost
2409          * impossible (or too ugly) because cpuset is too fluid that
2410          * task or memory node can be dynamically moved between cpusets.
2411          *
2412          * The change of semantics for shared hugetlb mapping with cpuset is
2413          * undesirable. However, in order to preserve some of the semantics,
2414          * we fall back to check against current free page availability as
2415          * a best attempt and hopefully to minimize the impact of changing
2416          * semantics that cpuset has.
2417          */
2418         if (delta > 0) {
2419                 if (gather_surplus_pages(h, delta) < 0)
2420                         goto out;
2421
2422                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2423                         return_unused_surplus_pages(h, delta);
2424                         goto out;
2425                 }
2426         }
2427
2428         ret = 0;
2429         if (delta < 0)
2430                 return_unused_surplus_pages(h, (unsigned long) -delta);
2431
2432 out:
2433         spin_unlock(&hugetlb_lock);
2434         return ret;
2435 }
2436
2437 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2438 {
2439         struct resv_map *resv = vma_resv_map(vma);
2440
2441         /*
2442          * This new VMA should share its siblings reservation map if present.
2443          * The VMA will only ever have a valid reservation map pointer where
2444          * it is being copied for another still existing VMA.  As that VMA
2445          * has a reference to the reservation map it cannot disappear until
2446          * after this open call completes.  It is therefore safe to take a
2447          * new reference here without additional locking.
2448          */
2449         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2450                 kref_get(&resv->refs);
2451 }
2452
2453 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2454 {
2455         struct hstate *h = hstate_vma(vma);
2456         struct resv_map *resv = vma_resv_map(vma);
2457         struct hugepage_subpool *spool = subpool_vma(vma);
2458         unsigned long reserve, start, end;
2459
2460         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2461                 return;
2462
2463         start = vma_hugecache_offset(h, vma, vma->vm_start);
2464         end = vma_hugecache_offset(h, vma, vma->vm_end);
2465
2466         reserve = (end - start) - region_count(resv, start, end);
2467
2468         kref_put(&resv->refs, resv_map_release);
2469
2470         if (reserve) {
2471                 hugetlb_acct_memory(h, -reserve);
2472                 hugepage_subpool_put_pages(spool, reserve);
2473         }
2474 }
2475
2476 /*
2477  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2478  * handle_mm_fault() to try to instantiate regular-sized pages in the
2479  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2480  * this far.
2481  */
2482 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2483 {
2484         BUG();
2485         return 0;
2486 }
2487
2488 const struct vm_operations_struct hugetlb_vm_ops = {
2489         .fault = hugetlb_vm_op_fault,
2490         .open = hugetlb_vm_op_open,
2491         .close = hugetlb_vm_op_close,
2492 };
2493
2494 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2495                                 int writable)
2496 {
2497         pte_t entry;
2498
2499         if (writable) {
2500                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2501                                          vma->vm_page_prot)));
2502         } else {
2503                 entry = huge_pte_wrprotect(mk_huge_pte(page,
2504                                            vma->vm_page_prot));
2505         }
2506         entry = pte_mkyoung(entry);
2507         entry = pte_mkhuge(entry);
2508         entry = arch_make_huge_pte(entry, vma, page, writable);
2509
2510         return entry;
2511 }
2512
2513 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2514                                    unsigned long address, pte_t *ptep)
2515 {
2516         pte_t entry;
2517
2518         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2519         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2520                 update_mmu_cache(vma, address, ptep);
2521 }
2522
2523 static int is_hugetlb_entry_migration(pte_t pte)
2524 {
2525         swp_entry_t swp;
2526
2527         if (huge_pte_none(pte) || pte_present(pte))
2528                 return 0;
2529         swp = pte_to_swp_entry(pte);
2530         if (non_swap_entry(swp) && is_migration_entry(swp))
2531                 return 1;
2532         else
2533                 return 0;
2534 }
2535
2536 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2537 {
2538         swp_entry_t swp;
2539
2540         if (huge_pte_none(pte) || pte_present(pte))
2541                 return 0;
2542         swp = pte_to_swp_entry(pte);
2543         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2544                 return 1;
2545         else
2546                 return 0;
2547 }
2548
2549 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2550                             struct vm_area_struct *vma)
2551 {
2552         pte_t *src_pte, *dst_pte, entry;
2553         struct page *ptepage;
2554         unsigned long addr;
2555         int cow;
2556         struct hstate *h = hstate_vma(vma);
2557         unsigned long sz = huge_page_size(h);
2558         unsigned long mmun_start;       /* For mmu_notifiers */
2559         unsigned long mmun_end;         /* For mmu_notifiers */
2560         int ret = 0;
2561
2562         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2563
2564         mmun_start = vma->vm_start;
2565         mmun_end = vma->vm_end;
2566         if (cow)
2567                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2568
2569         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2570                 spinlock_t *src_ptl, *dst_ptl;
2571                 src_pte = huge_pte_offset(src, addr);
2572                 if (!src_pte)
2573                         continue;
2574                 dst_pte = huge_pte_alloc(dst, addr, sz);
2575                 if (!dst_pte) {
2576                         ret = -ENOMEM;
2577                         break;
2578                 }
2579
2580                 /* If the pagetables are shared don't copy or take references */
2581                 if (dst_pte == src_pte)
2582                         continue;
2583
2584                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2585                 src_ptl = huge_pte_lockptr(h, src, src_pte);
2586                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
2587                 entry = huge_ptep_get(src_pte);
2588                 if (huge_pte_none(entry)) { /* skip none entry */
2589                         ;
2590                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
2591                                     is_hugetlb_entry_hwpoisoned(entry))) {
2592                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
2593
2594                         if (is_write_migration_entry(swp_entry) && cow) {
2595                                 /*
2596                                  * COW mappings require pages in both
2597                                  * parent and child to be set to read.
2598                                  */
2599                                 make_migration_entry_read(&swp_entry);
2600                                 entry = swp_entry_to_pte(swp_entry);
2601                                 set_huge_pte_at(src, addr, src_pte, entry);
2602                         }
2603                         set_huge_pte_at(dst, addr, dst_pte, entry);
2604                 } else {
2605                         if (cow)
2606                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2607                         entry = huge_ptep_get(src_pte);
2608                         ptepage = pte_page(entry);
2609                         get_page(ptepage);
2610                         page_dup_rmap(ptepage);
2611                         set_huge_pte_at(dst, addr, dst_pte, entry);
2612                 }
2613                 spin_unlock(src_ptl);
2614                 spin_unlock(dst_ptl);
2615         }
2616
2617         if (cow)
2618                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
2619
2620         return ret;
2621 }
2622
2623 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2624                             unsigned long start, unsigned long end,
2625                             struct page *ref_page)
2626 {
2627         int force_flush = 0;
2628         struct mm_struct *mm = vma->vm_mm;
2629         unsigned long address;
2630         pte_t *ptep;
2631         pte_t pte;
2632         spinlock_t *ptl;
2633         struct page *page;
2634         struct hstate *h = hstate_vma(vma);
2635         unsigned long sz = huge_page_size(h);
2636         const unsigned long mmun_start = start; /* For mmu_notifiers */
2637         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
2638
2639         WARN_ON(!is_vm_hugetlb_page(vma));
2640         BUG_ON(start & ~huge_page_mask(h));
2641         BUG_ON(end & ~huge_page_mask(h));
2642
2643         tlb_start_vma(tlb, vma);
2644         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2645 again:
2646         for (address = start; address < end; address += sz) {
2647                 ptep = huge_pte_offset(mm, address);
2648                 if (!ptep)
2649                         continue;
2650
2651                 ptl = huge_pte_lock(h, mm, ptep);
2652                 if (huge_pmd_unshare(mm, &address, ptep))
2653                         goto unlock;
2654
2655                 pte = huge_ptep_get(ptep);
2656                 if (huge_pte_none(pte))
2657                         goto unlock;
2658
2659                 /*
2660                  * HWPoisoned hugepage is already unmapped and dropped reference
2661                  */
2662                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2663                         huge_pte_clear(mm, address, ptep);
2664                         goto unlock;
2665                 }
2666
2667                 page = pte_page(pte);
2668                 /*
2669                  * If a reference page is supplied, it is because a specific
2670                  * page is being unmapped, not a range. Ensure the page we
2671                  * are about to unmap is the actual page of interest.
2672                  */
2673                 if (ref_page) {
2674                         if (page != ref_page)
2675                                 goto unlock;
2676
2677                         /*
2678                          * Mark the VMA as having unmapped its page so that
2679                          * future faults in this VMA will fail rather than
2680                          * looking like data was lost
2681                          */
2682                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2683                 }
2684
2685                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2686                 tlb_remove_tlb_entry(tlb, ptep, address);
2687                 if (huge_pte_dirty(pte))
2688                         set_page_dirty(page);
2689
2690                 page_remove_rmap(page);
2691                 force_flush = !__tlb_remove_page(tlb, page);
2692                 if (force_flush) {
2693                         spin_unlock(ptl);
2694                         break;
2695                 }
2696                 /* Bail out after unmapping reference page if supplied */
2697                 if (ref_page) {
2698                         spin_unlock(ptl);
2699                         break;
2700                 }
2701 unlock:
2702                 spin_unlock(ptl);
2703         }
2704         /*
2705          * mmu_gather ran out of room to batch pages, we break out of
2706          * the PTE lock to avoid doing the potential expensive TLB invalidate
2707          * and page-free while holding it.
2708          */
2709         if (force_flush) {
2710                 force_flush = 0;
2711                 tlb_flush_mmu(tlb);
2712                 if (address < end && !ref_page)
2713                         goto again;
2714         }
2715         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2716         tlb_end_vma(tlb, vma);
2717 }
2718
2719 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2720                           struct vm_area_struct *vma, unsigned long start,
2721                           unsigned long end, struct page *ref_page)
2722 {
2723         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
2724
2725         /*
2726          * Clear this flag so that x86's huge_pmd_share page_table_shareable
2727          * test will fail on a vma being torn down, and not grab a page table
2728          * on its way out.  We're lucky that the flag has such an appropriate
2729          * name, and can in fact be safely cleared here. We could clear it
2730          * before the __unmap_hugepage_range above, but all that's necessary
2731          * is to clear it before releasing the i_mmap_mutex. This works
2732          * because in the context this is called, the VMA is about to be
2733          * destroyed and the i_mmap_mutex is held.
2734          */
2735         vma->vm_flags &= ~VM_MAYSHARE;
2736 }
2737
2738 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2739                           unsigned long end, struct page *ref_page)
2740 {
2741         struct mm_struct *mm;
2742         struct mmu_gather tlb;
2743
2744         mm = vma->vm_mm;
2745
2746         tlb_gather_mmu(&tlb, mm, start, end);
2747         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2748         tlb_finish_mmu(&tlb, start, end);
2749 }
2750
2751 /*
2752  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2753  * mappping it owns the reserve page for. The intention is to unmap the page
2754  * from other VMAs and let the children be SIGKILLed if they are faulting the
2755  * same region.
2756  */
2757 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2758                                 struct page *page, unsigned long address)
2759 {
2760         struct hstate *h = hstate_vma(vma);
2761         struct vm_area_struct *iter_vma;
2762         struct address_space *mapping;
2763         pgoff_t pgoff;
2764
2765         /*
2766          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2767          * from page cache lookup which is in HPAGE_SIZE units.
2768          */
2769         address = address & huge_page_mask(h);
2770         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2771                         vma->vm_pgoff;
2772         mapping = file_inode(vma->vm_file)->i_mapping;
2773
2774         /*
2775          * Take the mapping lock for the duration of the table walk. As
2776          * this mapping should be shared between all the VMAs,
2777          * __unmap_hugepage_range() is called as the lock is already held
2778          */
2779         mutex_lock(&mapping->i_mmap_mutex);
2780         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2781                 /* Do not unmap the current VMA */
2782                 if (iter_vma == vma)
2783                         continue;
2784
2785                 /*
2786                  * Unmap the page from other VMAs without their own reserves.
2787                  * They get marked to be SIGKILLed if they fault in these
2788                  * areas. This is because a future no-page fault on this VMA
2789                  * could insert a zeroed page instead of the data existing
2790                  * from the time of fork. This would look like data corruption
2791                  */
2792                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2793                         unmap_hugepage_range(iter_vma, address,
2794                                              address + huge_page_size(h), page);
2795         }
2796         mutex_unlock(&mapping->i_mmap_mutex);
2797
2798         return 1;
2799 }
2800
2801 /*
2802  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2803  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2804  * cannot race with other handlers or page migration.
2805  * Keep the pte_same checks anyway to make transition from the mutex easier.
2806  */
2807 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2808                         unsigned long address, pte_t *ptep, pte_t pte,
2809                         struct page *pagecache_page, spinlock_t *ptl)
2810 {
2811         struct hstate *h = hstate_vma(vma);
2812         struct page *old_page, *new_page;
2813         int outside_reserve = 0;
2814         unsigned long mmun_start;       /* For mmu_notifiers */
2815         unsigned long mmun_end;         /* For mmu_notifiers */
2816
2817         old_page = pte_page(pte);
2818
2819 retry_avoidcopy:
2820         /* If no-one else is actually using this page, avoid the copy
2821          * and just make the page writable */
2822         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2823                 page_move_anon_rmap(old_page, vma, address);
2824                 set_huge_ptep_writable(vma, address, ptep);
2825                 return 0;
2826         }
2827
2828         /*
2829          * If the process that created a MAP_PRIVATE mapping is about to
2830          * perform a COW due to a shared page count, attempt to satisfy
2831          * the allocation without using the existing reserves. The pagecache
2832          * page is used to determine if the reserve at this address was
2833          * consumed or not. If reserves were used, a partial faulted mapping
2834          * at the time of fork() could consume its reserves on COW instead
2835          * of the full address range.
2836          */
2837         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2838                         old_page != pagecache_page)
2839                 outside_reserve = 1;
2840
2841         page_cache_get(old_page);
2842
2843         /* Drop page table lock as buddy allocator may be called */
2844         spin_unlock(ptl);
2845         new_page = alloc_huge_page(vma, address, outside_reserve);
2846
2847         if (IS_ERR(new_page)) {
2848                 long err = PTR_ERR(new_page);
2849                 page_cache_release(old_page);
2850
2851                 /*
2852                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2853                  * it is due to references held by a child and an insufficient
2854                  * huge page pool. To guarantee the original mappers
2855                  * reliability, unmap the page from child processes. The child
2856                  * may get SIGKILLed if it later faults.
2857                  */
2858                 if (outside_reserve) {
2859                         BUG_ON(huge_pte_none(pte));
2860                         if (unmap_ref_private(mm, vma, old_page, address)) {
2861                                 BUG_ON(huge_pte_none(pte));
2862                                 spin_lock(ptl);
2863                                 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2864                                 if (likely(ptep &&
2865                                            pte_same(huge_ptep_get(ptep), pte)))
2866                                         goto retry_avoidcopy;
2867                                 /*
2868                                  * race occurs while re-acquiring page table
2869                                  * lock, and our job is done.
2870                                  */
2871                                 return 0;
2872                         }
2873                         WARN_ON_ONCE(1);
2874                 }
2875
2876                 /* Caller expects lock to be held */
2877                 spin_lock(ptl);
2878                 if (err == -ENOMEM)
2879                         return VM_FAULT_OOM;
2880                 else
2881                         return VM_FAULT_SIGBUS;
2882         }
2883
2884         /*
2885          * When the original hugepage is shared one, it does not have
2886          * anon_vma prepared.
2887          */
2888         if (unlikely(anon_vma_prepare(vma))) {
2889                 page_cache_release(new_page);
2890                 page_cache_release(old_page);
2891                 /* Caller expects lock to be held */
2892                 spin_lock(ptl);
2893                 return VM_FAULT_OOM;
2894         }
2895
2896         copy_user_huge_page(new_page, old_page, address, vma,
2897                             pages_per_huge_page(h));
2898         __SetPageUptodate(new_page);
2899
2900         mmun_start = address & huge_page_mask(h);
2901         mmun_end = mmun_start + huge_page_size(h);
2902         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2903         /*
2904          * Retake the page table lock to check for racing updates
2905          * before the page tables are altered
2906          */
2907         spin_lock(ptl);
2908         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2909         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
2910                 ClearPagePrivate(new_page);
2911
2912                 /* Break COW */
2913                 huge_ptep_clear_flush(vma, address, ptep);
2914                 set_huge_pte_at(mm, address, ptep,
2915                                 make_huge_pte(vma, new_page, 1));
2916                 page_remove_rmap(old_page);
2917                 hugepage_add_new_anon_rmap(new_page, vma, address);
2918                 /* Make the old page be freed below */
2919                 new_page = old_page;
2920         }
2921         spin_unlock(ptl);
2922         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2923         page_cache_release(new_page);
2924         page_cache_release(old_page);
2925
2926         /* Caller expects lock to be held */
2927         spin_lock(ptl);
2928         return 0;
2929 }
2930
2931 /* Return the pagecache page at a given address within a VMA */
2932 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2933                         struct vm_area_struct *vma, unsigned long address)
2934 {
2935         struct address_space *mapping;
2936         pgoff_t idx;
2937
2938         mapping = vma->vm_file->f_mapping;
2939         idx = vma_hugecache_offset(h, vma, address);
2940
2941         return find_lock_page(mapping, idx);
2942 }
2943
2944 /*
2945  * Return whether there is a pagecache page to back given address within VMA.
2946  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2947  */
2948 static bool hugetlbfs_pagecache_present(struct hstate *h,
2949                         struct vm_area_struct *vma, unsigned long address)
2950 {
2951         struct address_space *mapping;
2952         pgoff_t idx;
2953         struct page *page;
2954
2955         mapping = vma->vm_file->f_mapping;
2956         idx = vma_hugecache_offset(h, vma, address);
2957
2958         page = find_get_page(mapping, idx);
2959         if (page)
2960                 put_page(page);
2961         return page != NULL;
2962 }
2963
2964 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2965                            struct address_space *mapping, pgoff_t idx,
2966                            unsigned long address, pte_t *ptep, unsigned int flags)
2967 {
2968         struct hstate *h = hstate_vma(vma);
2969         int ret = VM_FAULT_SIGBUS;
2970         int anon_rmap = 0;
2971         unsigned long size;
2972         struct page *page;
2973         pte_t new_pte;
2974         spinlock_t *ptl;
2975
2976         /*
2977          * Currently, we are forced to kill the process in the event the
2978          * original mapper has unmapped pages from the child due to a failed
2979          * COW. Warn that such a situation has occurred as it may not be obvious
2980          */
2981         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2982                 pr_warning("PID %d killed due to inadequate hugepage pool\n",
2983                            current->pid);
2984                 return ret;
2985         }
2986
2987         /*
2988          * Use page lock to guard against racing truncation
2989          * before we get page_table_lock.
2990          */
2991 retry:
2992         page = find_lock_page(mapping, idx);
2993         if (!page) {
2994                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2995                 if (idx >= size)
2996                         goto out;
2997                 page = alloc_huge_page(vma, address, 0);
2998                 if (IS_ERR(page)) {
2999                         ret = PTR_ERR(page);
3000                         if (ret == -ENOMEM)
3001                                 ret = VM_FAULT_OOM;
3002                         else
3003                                 ret = VM_FAULT_SIGBUS;
3004                         goto out;
3005                 }
3006                 clear_huge_page(page, address, pages_per_huge_page(h));
3007                 __SetPageUptodate(page);
3008
3009                 if (vma->vm_flags & VM_MAYSHARE) {
3010                         int err;
3011                         struct inode *inode = mapping->host;
3012
3013                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3014                         if (err) {
3015                                 put_page(page);
3016                                 if (err == -EEXIST)
3017                                         goto retry;
3018                                 goto out;
3019                         }
3020                         ClearPagePrivate(page);
3021
3022                         spin_lock(&inode->i_lock);
3023                         inode->i_blocks += blocks_per_huge_page(h);
3024                         spin_unlock(&inode->i_lock);
3025                 } else {
3026                         lock_page(page);
3027                         if (unlikely(anon_vma_prepare(vma))) {
3028                                 ret = VM_FAULT_OOM;
3029                                 goto backout_unlocked;
3030                         }
3031                         anon_rmap = 1;
3032                 }
3033         } else {
3034                 /*
3035                  * If memory error occurs between mmap() and fault, some process
3036                  * don't have hwpoisoned swap entry for errored virtual address.
3037                  * So we need to block hugepage fault by PG_hwpoison bit check.
3038                  */
3039                 if (unlikely(PageHWPoison(page))) {
3040                         ret = VM_FAULT_HWPOISON |
3041                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3042                         goto backout_unlocked;
3043                 }
3044         }
3045
3046         /*
3047          * If we are going to COW a private mapping later, we examine the
3048          * pending reservations for this page now. This will ensure that
3049          * any allocations necessary to record that reservation occur outside
3050          * the spinlock.
3051          */
3052         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
3053                 if (vma_needs_reservation(h, vma, address) < 0) {
3054                         ret = VM_FAULT_OOM;
3055                         goto backout_unlocked;
3056                 }
3057
3058         ptl = huge_pte_lockptr(h, mm, ptep);
3059         spin_lock(ptl);
3060         size = i_size_read(mapping->host) >> huge_page_shift(h);
3061         if (idx >= size)
3062                 goto backout;
3063
3064         ret = 0;
3065         if (!huge_pte_none(huge_ptep_get(ptep)))
3066                 goto backout;
3067
3068         if (anon_rmap) {
3069                 ClearPagePrivate(page);
3070                 hugepage_add_new_anon_rmap(page, vma, address);
3071         } else
3072                 page_dup_rmap(page);
3073         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3074                                 && (vma->vm_flags & VM_SHARED)));
3075         set_huge_pte_at(mm, address, ptep, new_pte);
3076
3077         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3078                 /* Optimization, do the COW without a second fault */
3079                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3080         }
3081
3082         spin_unlock(ptl);
3083         unlock_page(page);
3084 out:
3085         return ret;
3086
3087 backout:
3088         spin_unlock(ptl);
3089 backout_unlocked:
3090         unlock_page(page);
3091         put_page(page);
3092         goto out;
3093 }
3094
3095 #ifdef CONFIG_SMP
3096 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3097                             struct vm_area_struct *vma,
3098                             struct address_space *mapping,
3099                             pgoff_t idx, unsigned long address)
3100 {
3101         unsigned long key[2];
3102         u32 hash;
3103
3104         if (vma->vm_flags & VM_SHARED) {
3105                 key[0] = (unsigned long) mapping;
3106                 key[1] = idx;
3107         } else {
3108                 key[0] = (unsigned long) mm;
3109                 key[1] = address >> huge_page_shift(h);
3110         }
3111
3112         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3113
3114         return hash & (num_fault_mutexes - 1);
3115 }
3116 #else
3117 /*
3118  * For uniprocesor systems we always use a single mutex, so just
3119  * return 0 and avoid the hashing overhead.
3120  */
3121 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3122                             struct vm_area_struct *vma,
3123                             struct address_space *mapping,
3124                             pgoff_t idx, unsigned long address)
3125 {
3126         return 0;
3127 }
3128 #endif
3129
3130 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3131                         unsigned long address, unsigned int flags)
3132 {
3133         pte_t *ptep, entry;
3134         spinlock_t *ptl;
3135         int ret;
3136         u32 hash;
3137         pgoff_t idx;
3138         struct page *page = NULL;
3139         struct page *pagecache_page = NULL;
3140         struct hstate *h = hstate_vma(vma);
3141         struct address_space *mapping;
3142
3143         address &= huge_page_mask(h);
3144
3145         ptep = huge_pte_offset(mm, address);
3146         if (ptep) {
3147                 entry = huge_ptep_get(ptep);
3148                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3149                         migration_entry_wait_huge(vma, mm, ptep);
3150                         return 0;
3151                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3152                         return VM_FAULT_HWPOISON_LARGE |
3153                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3154         }
3155
3156         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3157         if (!ptep)
3158                 return VM_FAULT_OOM;
3159
3160         mapping = vma->vm_file->f_mapping;
3161         idx = vma_hugecache_offset(h, vma, address);
3162
3163         /*
3164          * Serialize hugepage allocation and instantiation, so that we don't
3165          * get spurious allocation failures if two CPUs race to instantiate
3166          * the same page in the page cache.
3167          */
3168         hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
3169         mutex_lock(&htlb_fault_mutex_table[hash]);
3170
3171         entry = huge_ptep_get(ptep);
3172         if (huge_pte_none(entry)) {
3173                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3174                 goto out_mutex;
3175         }
3176
3177         ret = 0;
3178
3179         /*
3180          * If we are going to COW the mapping later, we examine the pending
3181          * reservations for this page now. This will ensure that any
3182          * allocations necessary to record that reservation occur outside the
3183          * spinlock. For private mappings, we also lookup the pagecache
3184          * page now as it is used to determine if a reservation has been
3185          * consumed.
3186          */
3187         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3188                 if (vma_needs_reservation(h, vma, address) < 0) {
3189                         ret = VM_FAULT_OOM;
3190                         goto out_mutex;
3191                 }
3192
3193                 if (!(vma->vm_flags & VM_MAYSHARE))
3194                         pagecache_page = hugetlbfs_pagecache_page(h,
3195                                                                 vma, address);
3196         }
3197
3198         /*
3199          * hugetlb_cow() requires page locks of pte_page(entry) and
3200          * pagecache_page, so here we need take the former one
3201          * when page != pagecache_page or !pagecache_page.
3202          * Note that locking order is always pagecache_page -> page,
3203          * so no worry about deadlock.
3204          */
3205         page = pte_page(entry);
3206         get_page(page);
3207         if (page != pagecache_page)
3208                 lock_page(page);
3209
3210         ptl = huge_pte_lockptr(h, mm, ptep);
3211         spin_lock(ptl);
3212         /* Check for a racing update before calling hugetlb_cow */
3213         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3214                 goto out_ptl;
3215
3216
3217         if (flags & FAULT_FLAG_WRITE) {
3218                 if (!huge_pte_write(entry)) {
3219                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
3220                                         pagecache_page, ptl);
3221                         goto out_ptl;
3222                 }
3223                 entry = huge_pte_mkdirty(entry);
3224         }
3225         entry = pte_mkyoung(entry);
3226         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3227                                                 flags & FAULT_FLAG_WRITE))
3228                 update_mmu_cache(vma, address, ptep);
3229
3230 out_ptl:
3231         spin_unlock(ptl);
3232
3233         if (pagecache_page) {
3234                 unlock_page(pagecache_page);
3235                 put_page(pagecache_page);
3236         }
3237         if (page != pagecache_page)
3238                 unlock_page(page);
3239         put_page(page);
3240
3241 out_mutex:
3242         mutex_unlock(&htlb_fault_mutex_table[hash]);
3243         return ret;
3244 }
3245
3246 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3247                          struct page **pages, struct vm_area_struct **vmas,
3248                          unsigned long *position, unsigned long *nr_pages,
3249                          long i, unsigned int flags)
3250 {
3251         unsigned long pfn_offset;
3252         unsigned long vaddr = *position;
3253         unsigned long remainder = *nr_pages;
3254         struct hstate *h = hstate_vma(vma);
3255
3256         while (vaddr < vma->vm_end && remainder) {
3257                 pte_t *pte;
3258                 spinlock_t *ptl = NULL;
3259                 int absent;
3260                 struct page *page;
3261
3262                 /*
3263                  * Some archs (sparc64, sh*) have multiple pte_ts to
3264                  * each hugepage.  We have to make sure we get the
3265                  * first, for the page indexing below to work.
3266                  *
3267                  * Note that page table lock is not held when pte is null.
3268                  */
3269                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3270                 if (pte)
3271                         ptl = huge_pte_lock(h, mm, pte);
3272                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3273
3274                 /*
3275                  * When coredumping, it suits get_dump_page if we just return
3276                  * an error where there's an empty slot with no huge pagecache
3277                  * to back it.  This way, we avoid allocating a hugepage, and
3278                  * the sparse dumpfile avoids allocating disk blocks, but its
3279                  * huge holes still show up with zeroes where they need to be.
3280                  */
3281                 if (absent && (flags & FOLL_DUMP) &&
3282                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3283                         if (pte)
3284                                 spin_unlock(ptl);
3285                         remainder = 0;
3286                         break;
3287                 }
3288
3289                 /*
3290                  * We need call hugetlb_fault for both hugepages under migration
3291                  * (in which case hugetlb_fault waits for the migration,) and
3292                  * hwpoisoned hugepages (in which case we need to prevent the
3293                  * caller from accessing to them.) In order to do this, we use
3294                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3295                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3296                  * both cases, and because we can't follow correct pages
3297                  * directly from any kind of swap entries.
3298                  */
3299                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3300                     ((flags & FOLL_WRITE) &&
3301                       !huge_pte_write(huge_ptep_get(pte)))) {
3302                         int ret;
3303
3304                         if (pte)
3305                                 spin_unlock(ptl);
3306                         ret = hugetlb_fault(mm, vma, vaddr,
3307                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3308                         if (!(ret & VM_FAULT_ERROR))
3309                                 continue;
3310
3311                         remainder = 0;
3312                         break;
3313                 }
3314
3315                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3316                 page = pte_page(huge_ptep_get(pte));
3317 same_page:
3318                 if (pages) {
3319                         pages[i] = mem_map_offset(page, pfn_offset);
3320                         get_page_foll(pages[i]);
3321                 }
3322
3323                 if (vmas)
3324                         vmas[i] = vma;
3325
3326                 vaddr += PAGE_SIZE;
3327                 ++pfn_offset;
3328                 --remainder;
3329                 ++i;
3330                 if (vaddr < vma->vm_end && remainder &&
3331                                 pfn_offset < pages_per_huge_page(h)) {
3332                         /*
3333                          * We use pfn_offset to avoid touching the pageframes
3334                          * of this compound page.
3335                          */
3336                         goto same_page;
3337                 }
3338                 spin_unlock(ptl);
3339         }
3340         *nr_pages = remainder;
3341         *position = vaddr;
3342
3343         return i ? i : -EFAULT;
3344 }
3345
3346 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3347                 unsigned long address, unsigned long end, pgprot_t newprot)
3348 {
3349         struct mm_struct *mm = vma->vm_mm;
3350         unsigned long start = address;
3351         pte_t *ptep;
3352         pte_t pte;
3353         struct hstate *h = hstate_vma(vma);
3354         unsigned long pages = 0;
3355
3356         BUG_ON(address >= end);
3357         flush_cache_range(vma, address, end);
3358
3359         mmu_notifier_invalidate_range_start(mm, start, end);
3360         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
3361         for (; address < end; address += huge_page_size(h)) {
3362                 spinlock_t *ptl;
3363                 ptep = huge_pte_offset(mm, address);
3364                 if (!ptep)
3365                         continue;
3366                 ptl = huge_pte_lock(h, mm, ptep);
3367                 if (huge_pmd_unshare(mm, &address, ptep)) {
3368                         pages++;
3369                         spin_unlock(ptl);
3370                         continue;
3371                 }
3372                 if (!huge_pte_none(huge_ptep_get(ptep))) {
3373                         pte = huge_ptep_get_and_clear(mm, address, ptep);
3374                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3375                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
3376                         set_huge_pte_at(mm, address, ptep, pte);
3377                         pages++;
3378                 }
3379                 spin_unlock(ptl);
3380         }
3381         /*
3382          * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
3383          * may have cleared our pud entry and done put_page on the page table:
3384          * once we release i_mmap_mutex, another task can do the final put_page
3385          * and that page table be reused and filled with junk.
3386          */
3387         flush_tlb_range(vma, start, end);
3388         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
3389         mmu_notifier_invalidate_range_end(mm, start, end);
3390
3391         return pages << h->order;
3392 }
3393
3394 int hugetlb_reserve_pages(struct inode *inode,
3395                                         long from, long to,
3396                                         struct vm_area_struct *vma,
3397                                         vm_flags_t vm_flags)
3398 {
3399         long ret, chg;
3400         struct hstate *h = hstate_inode(inode);
3401         struct hugepage_subpool *spool = subpool_inode(inode);
3402         struct resv_map *resv_map;
3403
3404         /*
3405          * Only apply hugepage reservation if asked. At fault time, an
3406          * attempt will be made for VM_NORESERVE to allocate a page
3407          * without using reserves
3408          */
3409         if (vm_flags & VM_NORESERVE)
3410                 return 0;
3411
3412         /*
3413          * Shared mappings base their reservation on the number of pages that
3414          * are already allocated on behalf of the file. Private mappings need
3415          * to reserve the full area even if read-only as mprotect() may be
3416          * called to make the mapping read-write. Assume !vma is a shm mapping
3417          */
3418         if (!vma || vma->vm_flags & VM_MAYSHARE) {
3419                 resv_map = inode_resv_map(inode);
3420
3421                 chg = region_chg(resv_map, from, to);
3422
3423         } else {
3424                 resv_map = resv_map_alloc();
3425                 if (!resv_map)
3426                         return -ENOMEM;
3427
3428                 chg = to - from;
3429
3430                 set_vma_resv_map(vma, resv_map);
3431                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3432         }
3433
3434         if (chg < 0) {
3435                 ret = chg;
3436                 goto out_err;
3437         }
3438
3439         /* There must be enough pages in the subpool for the mapping */
3440         if (hugepage_subpool_get_pages(spool, chg)) {
3441                 ret = -ENOSPC;
3442                 goto out_err;
3443         }
3444
3445         /*
3446          * Check enough hugepages are available for the reservation.
3447          * Hand the pages back to the subpool if there are not
3448          */
3449         ret = hugetlb_acct_memory(h, chg);
3450         if (ret < 0) {
3451                 hugepage_subpool_put_pages(spool, chg);
3452                 goto out_err;
3453         }
3454
3455         /*
3456          * Account for the reservations made. Shared mappings record regions
3457          * that have reservations as they are shared by multiple VMAs.
3458          * When the last VMA disappears, the region map says how much
3459          * the reservation was and the page cache tells how much of
3460          * the reservation was consumed. Private mappings are per-VMA and
3461          * only the consumed reservations are tracked. When the VMA
3462          * disappears, the original reservation is the VMA size and the
3463          * consumed reservations are stored in the map. Hence, nothing
3464          * else has to be done for private mappings here
3465          */
3466         if (!vma || vma->vm_flags & VM_MAYSHARE)
3467                 region_add(resv_map, from, to);
3468         return 0;
3469 out_err:
3470         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3471                 kref_put(&resv_map->refs, resv_map_release);
3472         return ret;
3473 }
3474
3475 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3476 {
3477         struct hstate *h = hstate_inode(inode);
3478         struct resv_map *resv_map = inode_resv_map(inode);
3479         long chg = 0;
3480         struct hugepage_subpool *spool = subpool_inode(inode);
3481
3482         if (resv_map)
3483                 chg = region_truncate(resv_map, offset);
3484         spin_lock(&inode->i_lock);
3485         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3486         spin_unlock(&inode->i_lock);
3487
3488         hugepage_subpool_put_pages(spool, (chg - freed));
3489         hugetlb_acct_memory(h, -(chg - freed));
3490 }
3491
3492 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3493 static unsigned long page_table_shareable(struct vm_area_struct *svma,
3494                                 struct vm_area_struct *vma,
3495                                 unsigned long addr, pgoff_t idx)
3496 {
3497         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3498                                 svma->vm_start;
3499         unsigned long sbase = saddr & PUD_MASK;
3500         unsigned long s_end = sbase + PUD_SIZE;
3501
3502         /* Allow segments to share if only one is marked locked */
3503         unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3504         unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3505
3506         /*
3507          * match the virtual addresses, permission and the alignment of the
3508          * page table page.
3509          */
3510         if (pmd_index(addr) != pmd_index(saddr) ||
3511             vm_flags != svm_flags ||
3512             sbase < svma->vm_start || svma->vm_end < s_end)
3513                 return 0;
3514
3515         return saddr;
3516 }
3517
3518 static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3519 {
3520         unsigned long base = addr & PUD_MASK;
3521         unsigned long end = base + PUD_SIZE;
3522
3523         /*
3524          * check on proper vm_flags and page table alignment
3525          */
3526         if (vma->vm_flags & VM_MAYSHARE &&
3527             vma->vm_start <= base && end <= vma->vm_end)
3528                 return 1;
3529         return 0;
3530 }
3531
3532 /*
3533  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3534  * and returns the corresponding pte. While this is not necessary for the
3535  * !shared pmd case because we can allocate the pmd later as well, it makes the
3536  * code much cleaner. pmd allocation is essential for the shared case because
3537  * pud has to be populated inside the same i_mmap_mutex section - otherwise
3538  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3539  * bad pmd for sharing.
3540  */
3541 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3542 {
3543         struct vm_area_struct *vma = find_vma(mm, addr);
3544         struct address_space *mapping = vma->vm_file->f_mapping;
3545         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3546                         vma->vm_pgoff;
3547         struct vm_area_struct *svma;
3548         unsigned long saddr;
3549         pte_t *spte = NULL;
3550         pte_t *pte;
3551         spinlock_t *ptl;
3552
3553         if (!vma_shareable(vma, addr))
3554                 return (pte_t *)pmd_alloc(mm, pud, addr);
3555
3556         mutex_lock(&mapping->i_mmap_mutex);
3557         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3558                 if (svma == vma)
3559                         continue;
3560
3561                 saddr = page_table_shareable(svma, vma, addr, idx);
3562                 if (saddr) {
3563                         spte = huge_pte_offset(svma->vm_mm, saddr);
3564                         if (spte) {
3565                                 get_page(virt_to_page(spte));
3566                                 break;
3567                         }
3568                 }
3569         }
3570
3571         if (!spte)
3572                 goto out;
3573
3574         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3575         spin_lock(ptl);
3576         if (pud_none(*pud))
3577                 pud_populate(mm, pud,
3578                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
3579         else
3580                 put_page(virt_to_page(spte));
3581         spin_unlock(ptl);
3582 out:
3583         pte = (pte_t *)pmd_alloc(mm, pud, addr);
3584         mutex_unlock(&mapping->i_mmap_mutex);
3585         return pte;
3586 }
3587
3588 /*
3589  * unmap huge page backed by shared pte.
3590  *
3591  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
3592  * indicated by page_count > 1, unmap is achieved by clearing pud and
3593  * decrementing the ref count. If count == 1, the pte page is not shared.
3594  *
3595  * called with page table lock held.
3596  *
3597  * returns: 1 successfully unmapped a shared pte page
3598  *          0 the underlying pte page is not shared, or it is the last user
3599  */
3600 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3601 {
3602         pgd_t *pgd = pgd_offset(mm, *addr);
3603         pud_t *pud = pud_offset(pgd, *addr);
3604
3605         BUG_ON(page_count(virt_to_page(ptep)) == 0);
3606         if (page_count(virt_to_page(ptep)) == 1)
3607                 return 0;
3608
3609         pud_clear(pud);
3610         put_page(virt_to_page(ptep));
3611         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3612         return 1;
3613 }
3614 #define want_pmd_share()        (1)
3615 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3616 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3617 {
3618         return NULL;
3619 }
3620 #define want_pmd_share()        (0)
3621 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3622
3623 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3624 pte_t *huge_pte_alloc(struct mm_struct *mm,
3625                         unsigned long addr, unsigned long sz)
3626 {
3627         pgd_t *pgd;
3628         pud_t *pud;
3629         pte_t *pte = NULL;
3630
3631         pgd = pgd_offset(mm, addr);
3632         pud = pud_alloc(mm, pgd, addr);
3633         if (pud) {
3634                 if (sz == PUD_SIZE) {
3635                         pte = (pte_t *)pud;
3636                 } else {
3637                         BUG_ON(sz != PMD_SIZE);
3638                         if (want_pmd_share() && pud_none(*pud))
3639                                 pte = huge_pmd_share(mm, addr, pud);
3640                         else
3641                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3642                 }
3643         }
3644         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3645
3646         return pte;
3647 }
3648
3649 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3650 {
3651         pgd_t *pgd;
3652         pud_t *pud;
3653         pmd_t *pmd = NULL;
3654
3655         pgd = pgd_offset(mm, addr);
3656         if (pgd_present(*pgd)) {
3657                 pud = pud_offset(pgd, addr);
3658                 if (pud_present(*pud)) {
3659                         if (pud_huge(*pud))
3660                                 return (pte_t *)pud;
3661                         pmd = pmd_offset(pud, addr);
3662                 }
3663         }
3664         return (pte_t *) pmd;
3665 }
3666
3667 struct page *
3668 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3669                 pmd_t *pmd, int write)
3670 {
3671         struct page *page;
3672
3673         page = pte_page(*(pte_t *)pmd);
3674         if (page)
3675                 page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
3676         return page;
3677 }
3678
3679 struct page *
3680 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3681                 pud_t *pud, int write)
3682 {
3683         struct page *page;
3684
3685         page = pte_page(*(pte_t *)pud);
3686         if (page)
3687                 page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
3688         return page;
3689 }
3690
3691 #else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3692
3693 /* Can be overriden by architectures */
3694 struct page * __weak
3695 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3696                pud_t *pud, int write)
3697 {
3698         BUG();
3699         return NULL;
3700 }
3701
3702 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3703
3704 #ifdef CONFIG_MEMORY_FAILURE
3705
3706 /* Should be called in hugetlb_lock */
3707 static int is_hugepage_on_freelist(struct page *hpage)
3708 {
3709         struct page *page;
3710         struct page *tmp;
3711         struct hstate *h = page_hstate(hpage);
3712         int nid = page_to_nid(hpage);
3713
3714         list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3715                 if (page == hpage)
3716                         return 1;
3717         return 0;
3718 }
3719
3720 /*
3721  * This function is called from memory failure code.
3722  * Assume the caller holds page lock of the head page.
3723  */
3724 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3725 {
3726         struct hstate *h = page_hstate(hpage);
3727         int nid = page_to_nid(hpage);
3728         int ret = -EBUSY;
3729
3730         spin_lock(&hugetlb_lock);
3731         if (is_hugepage_on_freelist(hpage)) {
3732                 /*
3733                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
3734                  * but dangling hpage->lru can trigger list-debug warnings
3735                  * (this happens when we call unpoison_memory() on it),
3736                  * so let it point to itself with list_del_init().
3737                  */
3738                 list_del_init(&hpage->lru);
3739                 set_page_refcounted(hpage);
3740                 h->free_huge_pages--;
3741                 h->free_huge_pages_node[nid]--;
3742                 ret = 0;
3743         }
3744         spin_unlock(&hugetlb_lock);
3745         return ret;
3746 }
3747 #endif
3748
3749 bool isolate_huge_page(struct page *page, struct list_head *list)
3750 {
3751         VM_BUG_ON_PAGE(!PageHead(page), page);
3752         if (!get_page_unless_zero(page))
3753                 return false;
3754         spin_lock(&hugetlb_lock);
3755         list_move_tail(&page->lru, list);
3756         spin_unlock(&hugetlb_lock);
3757         return true;
3758 }
3759
3760 void putback_active_hugepage(struct page *page)
3761 {
3762         VM_BUG_ON_PAGE(!PageHead(page), page);
3763         spin_lock(&hugetlb_lock);
3764         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3765         spin_unlock(&hugetlb_lock);
3766         put_page(page);
3767 }
3768
3769 bool is_hugepage_active(struct page *page)
3770 {
3771         VM_BUG_ON_PAGE(!PageHuge(page), page);
3772         /*
3773          * This function can be called for a tail page because the caller,
3774          * scan_movable_pages, scans through a given pfn-range which typically
3775          * covers one memory block. In systems using gigantic hugepage (1GB
3776          * for x86_64,) a hugepage is larger than a memory block, and we don't
3777          * support migrating such large hugepages for now, so return false
3778          * when called for tail pages.
3779          */
3780         if (PageTail(page))
3781                 return false;
3782         /*
3783          * Refcount of a hwpoisoned hugepages is 1, but they are not active,
3784          * so we should return false for them.
3785          */
3786         if (unlikely(PageHWPoison(page)))
3787                 return false;
3788         return page_count(page) > 0;
3789 }