2 * SPARC64 Huge TLB page support.
4 * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
7 #include <linux/init.h>
10 #include <linux/hugetlb.h>
11 #include <linux/pagemap.h>
12 #include <linux/sysctl.h>
15 #include <asm/pgalloc.h>
17 #include <asm/tlbflush.h>
18 #include <asm/cacheflush.h>
19 #include <asm/mmu_context.h>
21 /* Slightly simplified from the non-hugepage variant because by
22 * definition we don't have to worry about any page coloring stuff
25 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
31 unsigned long task_size = TASK_SIZE;
32 struct vm_unmapped_area_info info;
34 if (test_thread_flag(TIF_32BIT))
35 task_size = STACK_TOP32;
39 info.low_limit = TASK_UNMAPPED_BASE;
40 info.high_limit = min(task_size, VA_EXCLUDE_START);
41 info.align_mask = PAGE_MASK & ~HPAGE_MASK;
42 info.align_offset = 0;
43 addr = vm_unmapped_area(&info);
45 if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
46 VM_BUG_ON(addr != -ENOMEM);
47 info.low_limit = VA_EXCLUDE_END;
48 info.high_limit = task_size;
49 addr = vm_unmapped_area(&info);
56 hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
57 const unsigned long len,
58 const unsigned long pgoff,
59 const unsigned long flags)
61 struct mm_struct *mm = current->mm;
62 unsigned long addr = addr0;
63 struct vm_unmapped_area_info info;
65 /* This should only ever run for 32-bit processes. */
66 BUG_ON(!test_thread_flag(TIF_32BIT));
68 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
70 info.low_limit = PAGE_SIZE;
71 info.high_limit = mm->mmap_base;
72 info.align_mask = PAGE_MASK & ~HPAGE_MASK;
73 info.align_offset = 0;
74 addr = vm_unmapped_area(&info);
77 * A failed mmap() very likely causes application failure,
78 * so fall back to the bottom-up function here. This scenario
79 * can happen with large stack limits and large mmap()
82 if (addr & ~PAGE_MASK) {
83 VM_BUG_ON(addr != -ENOMEM);
85 info.low_limit = TASK_UNMAPPED_BASE;
86 info.high_limit = STACK_TOP32;
87 addr = vm_unmapped_area(&info);
94 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
95 unsigned long len, unsigned long pgoff, unsigned long flags)
97 struct mm_struct *mm = current->mm;
98 struct vm_area_struct *vma;
99 unsigned long task_size = TASK_SIZE;
101 if (test_thread_flag(TIF_32BIT))
102 task_size = STACK_TOP32;
104 if (len & ~HPAGE_MASK)
109 if (flags & MAP_FIXED) {
110 if (prepare_hugepage_range(file, addr, len))
116 addr = ALIGN(addr, HPAGE_SIZE);
117 vma = find_vma(mm, addr);
118 if (task_size - len >= addr &&
119 (!vma || addr + len <= vma->vm_start))
122 if (mm->get_unmapped_area == arch_get_unmapped_area)
123 return hugetlb_get_unmapped_area_bottomup(file, addr, len,
126 return hugetlb_get_unmapped_area_topdown(file, addr, len,
130 pte_t *huge_pte_alloc(struct mm_struct *mm,
131 unsigned long addr, unsigned long sz)
138 /* We must align the address, because our caller will run
139 * set_huge_pte_at() on whatever we return, which writes out
140 * all of the sub-ptes for the hugepage range. So we have
141 * to give it the first such sub-pte.
145 pgd = pgd_offset(mm, addr);
146 pud = pud_alloc(mm, pgd, addr);
148 pmd = pmd_alloc(mm, pud, addr);
150 pte = pte_alloc_map(mm, NULL, pmd, addr);
155 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
164 pgd = pgd_offset(mm, addr);
165 if (!pgd_none(*pgd)) {
166 pud = pud_offset(pgd, addr);
167 if (!pud_none(*pud)) {
168 pmd = pmd_offset(pud, addr);
170 pte = pte_offset_map(pmd, addr);
176 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
181 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
182 pte_t *ptep, pte_t entry)
186 if (!pte_present(*ptep) && pte_present(entry))
187 mm->context.huge_pte_count++;
190 for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
191 set_pte_at(mm, addr, ptep, entry);
194 pte_val(entry) += PAGE_SIZE;
198 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
205 if (pte_present(entry))
206 mm->context.huge_pte_count--;
210 for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
211 pte_clear(mm, addr, ptep);
219 struct page *follow_huge_addr(struct mm_struct *mm,
220 unsigned long address, int write)
222 return ERR_PTR(-EINVAL);
225 int pmd_huge(pmd_t pmd)
230 int pud_huge(pud_t pud)
235 int pmd_huge_support(void)
240 struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
241 pmd_t *pmd, int write)