2 * IBM System z Huge TLB Page Support for Kernel.
4 * Copyright IBM Corp. 2007
5 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
9 #include <linux/hugetlb.h>
11 static inline pmd_t __pte_to_pmd(pte_t pte)
16 * Convert encoding pte bits pmd bits
17 * .IR...wrdytp dy..R...I...wr
18 * empty .10...000000 -> 00..0...1...00
19 * prot-none, clean, old .11...000001 -> 00..1...1...00
20 * prot-none, clean, young .11...000101 -> 01..1...1...00
21 * prot-none, dirty, old .10...001001 -> 10..1...1...00
22 * prot-none, dirty, young .10...001101 -> 11..1...1...00
23 * read-only, clean, old .11...010001 -> 00..1...1...01
24 * read-only, clean, young .01...010101 -> 01..1...0...01
25 * read-only, dirty, old .11...011001 -> 10..1...1...01
26 * read-only, dirty, young .01...011101 -> 11..1...0...01
27 * read-write, clean, old .11...110001 -> 00..0...1...11
28 * read-write, clean, young .01...110101 -> 01..0...0...11
29 * read-write, dirty, old .10...111001 -> 10..0...1...11
30 * read-write, dirty, young .00...111101 -> 11..0...0...11
32 if (pte_present(pte)) {
33 pmd_val(pmd) = pte_val(pte) & PAGE_MASK;
34 pmd_val(pmd) |= (pte_val(pte) & _PAGE_READ) >> 4;
35 pmd_val(pmd) |= (pte_val(pte) & _PAGE_WRITE) >> 4;
36 pmd_val(pmd) |= (pte_val(pte) & _PAGE_INVALID) >> 5;
37 pmd_val(pmd) |= (pte_val(pte) & _PAGE_PROTECT);
38 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10;
39 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10;
41 pmd_val(pmd) = _SEGMENT_ENTRY_INVALID;
45 static inline pte_t __pmd_to_pte(pmd_t pmd)
50 * Convert encoding pmd bits pte bits
51 * dy..R...I...wr .IR...wrdytp
52 * empty 00..0...1...00 -> .10...001100
53 * prot-none, clean, old 00..0...1...00 -> .10...000001
54 * prot-none, clean, young 01..0...1...00 -> .10...000101
55 * prot-none, dirty, old 10..0...1...00 -> .10...001001
56 * prot-none, dirty, young 11..0...1...00 -> .10...001101
57 * read-only, clean, old 00..1...1...01 -> .11...010001
58 * read-only, clean, young 01..1...1...01 -> .11...010101
59 * read-only, dirty, old 10..1...1...01 -> .11...011001
60 * read-only, dirty, young 11..1...1...01 -> .11...011101
61 * read-write, clean, old 00..0...1...11 -> .10...110001
62 * read-write, clean, young 01..0...1...11 -> .10...110101
63 * read-write, dirty, old 10..0...1...11 -> .10...111001
64 * read-write, dirty, young 11..0...1...11 -> .10...111101
66 if (pmd_present(pmd)) {
67 pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE;
68 pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
69 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_READ) << 4;
70 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4;
71 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5;
72 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT);
73 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10;
74 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10;
76 pte_val(pte) = _PAGE_INVALID;
80 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
81 pte_t *ptep, pte_t pte)
85 pmd = __pte_to_pmd(pte);
86 if (!MACHINE_HAS_HPAGE) {
87 /* Emulated huge ptes loose the dirty and young bit */
88 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
89 pmd_val(pmd) |= pte_page(pte)[1].index;
91 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
92 *(pmd_t *) ptep = pmd;
95 pte_t huge_ptep_get(pte_t *ptep)
100 pmd = *(pmd_t *) ptep;
101 if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) {
102 origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN;
103 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
104 pmd_val(pmd) |= *(unsigned long *) origin;
105 /* Emulated huge ptes are young and dirty by definition */
106 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG | _SEGMENT_ENTRY_DIRTY;
108 return __pmd_to_pte(pmd);
111 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
112 unsigned long addr, pte_t *ptep)
114 pmd_t *pmdp = (pmd_t *) ptep;
115 pte_t pte = huge_ptep_get(ptep);
117 pmdp_flush_direct(mm, addr, pmdp);
118 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
122 int arch_prepare_hugepage(struct page *page)
124 unsigned long addr = page_to_phys(page);
129 if (MACHINE_HAS_HPAGE)
132 ptep = (pte_t *) pte_alloc_one(&init_mm, addr);
137 for (i = 0; i < PTRS_PER_PTE; i++) {
138 set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte);
139 pte_val(pte) += PAGE_SIZE;
141 page[1].index = (unsigned long) ptep;
145 void arch_release_hugepage(struct page *page)
149 if (MACHINE_HAS_HPAGE)
152 ptep = (pte_t *) page[1].index;
155 clear_table((unsigned long *) ptep, _PAGE_INVALID,
156 PTRS_PER_PTE * sizeof(pte_t));
157 page_table_free(&init_mm, (unsigned long *) ptep);
161 pte_t *huge_pte_alloc(struct mm_struct *mm,
162 unsigned long addr, unsigned long sz)
168 pgdp = pgd_offset(mm, addr);
169 pudp = pud_alloc(mm, pgdp, addr);
171 pmdp = pmd_alloc(mm, pudp, addr);
172 return (pte_t *) pmdp;
175 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
181 pgdp = pgd_offset(mm, addr);
182 if (pgd_present(*pgdp)) {
183 pudp = pud_offset(pgdp, addr);
184 if (pud_present(*pudp))
185 pmdp = pmd_offset(pudp, addr);
187 return (pte_t *) pmdp;
190 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
195 int pmd_huge(pmd_t pmd)
197 if (!MACHINE_HAS_HPAGE)
200 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
203 int pud_huge(pud_t pud)