2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * Copyright 2016 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 #include <linux/types.h>
10 #include <linux/string.h>
11 #include <linux/kvm.h>
12 #include <linux/kvm_host.h>
14 #include <asm/kvm_ppc.h>
15 #include <asm/kvm_book3s.h>
18 #include <asm/pgtable.h>
19 #include <asm/pgalloc.h>
22 * Supported radix tree geometry.
23 * Like p9, we support either 5 or 9 bits at the first (lowest) level,
24 * for a page size of 64k or 4k.
26 static int p9_supported_radix_bits[4] = { 5, 9, 9, 13 };
28 int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
29 struct kvmppc_pte *gpte, bool data, bool iswrite)
31 struct kvm *kvm = vcpu->kvm;
35 unsigned long root, pte, index;
36 unsigned long rts, bits, offset;
38 unsigned long proc_tbl_size;
40 /* Work out effective PID */
41 switch (eaddr >> 62) {
51 proc_tbl_size = 1 << ((kvm->arch.process_table & PRTS_MASK) + 12);
52 if (pid * 16 >= proc_tbl_size)
55 /* Read partition table to find root of tree for effective PID */
56 ret = kvm_read_guest(kvm, kvm->arch.process_table + pid * 16,
61 root = be64_to_cpu(prte);
62 rts = ((root & RTS1_MASK) >> (RTS1_SHIFT - 3)) |
63 ((root & RTS2_MASK) >> RTS2_SHIFT);
64 bits = root & RPDS_MASK;
65 root = root & RPDB_MASK;
67 /* P9 DD1 interprets RTS (radix tree size) differently */
69 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
72 /* current implementations only support 52-bit space */
76 for (level = 3; level >= 0; --level) {
77 if (level && bits != p9_supported_radix_bits[level])
79 if (level == 0 && !(bits == 5 || bits == 9))
82 index = (eaddr >> offset) & ((1UL << bits) - 1);
83 /* check that low bits of page table base are zero */
84 if (root & ((1UL << (bits + 3)) - 1))
86 ret = kvm_read_guest(kvm, root + index * 8,
90 pte = __be64_to_cpu(rpte);
91 if (!(pte & _PAGE_PRESENT))
96 root = pte & 0x0fffffffffffff00ul;
98 /* need a leaf at lowest level; 512GB pages not supported */
99 if (level < 0 || level == 3)
102 /* offset is now log base 2 of the page size */
103 gpa = pte & 0x01fffffffffff000ul;
104 if (gpa & ((1ul << offset) - 1))
106 gpa += eaddr & ((1ul << offset) - 1);
107 for (ps = MMU_PAGE_4K; ps < MMU_PAGE_COUNT; ++ps)
108 if (offset == mmu_psize_defs[ps].shift)
110 gpte->page_size = ps;
115 /* Work out permissions */
116 gpte->may_read = !!(pte & _PAGE_READ);
117 gpte->may_write = !!(pte & _PAGE_WRITE);
118 gpte->may_execute = !!(pte & _PAGE_EXEC);
119 if (kvmppc_get_msr(vcpu) & MSR_PR) {
120 if (pte & _PAGE_PRIVILEGED) {
123 gpte->may_execute = 0;
126 if (!(pte & _PAGE_PRIVILEGED)) {
127 /* Check AMR/IAMR to see if strict mode is in force */
128 if (vcpu->arch.amr & (1ul << 62))
130 if (vcpu->arch.amr & (1ul << 63))
132 if (vcpu->arch.iamr & (1ul << 62))
133 gpte->may_execute = 0;
140 #ifdef CONFIG_PPC_64K_PAGES
141 #define MMU_BASE_PSIZE MMU_PAGE_64K
143 #define MMU_BASE_PSIZE MMU_PAGE_4K
146 static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
149 int psize = MMU_BASE_PSIZE;
151 if (pshift >= PMD_SHIFT)
154 addr |= mmu_psize_defs[psize].ap << 5;
155 asm volatile("ptesync": : :"memory");
156 asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
157 : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
158 asm volatile("ptesync": : :"memory");
161 unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
162 unsigned long clr, unsigned long set,
163 unsigned long addr, unsigned int shift)
165 unsigned long old = 0;
167 if (!(clr & _PAGE_PRESENT) && cpu_has_feature(CPU_FTR_POWER9_DD1) &&
168 pte_present(*ptep)) {
169 /* have to invalidate it first */
170 old = __radix_pte_update(ptep, _PAGE_PRESENT, 0);
171 kvmppc_radix_tlbie_page(kvm, addr, shift);
172 set |= _PAGE_PRESENT;
173 old &= _PAGE_PRESENT;
175 return __radix_pte_update(ptep, clr, set) | old;
178 void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
179 pte_t *ptep, pte_t pte)
181 radix__set_pte_at(kvm->mm, addr, ptep, pte, 0);
184 static struct kmem_cache *kvm_pte_cache;
186 static pte_t *kvmppc_pte_alloc(void)
188 return kmem_cache_alloc(kvm_pte_cache, GFP_KERNEL);
191 static void kvmppc_pte_free(pte_t *ptep)
193 kmem_cache_free(kvm_pte_cache, ptep);
196 static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
197 unsigned int level, unsigned long mmu_seq)
200 pud_t *pud, *new_pud = NULL;
201 pmd_t *pmd, *new_pmd = NULL;
202 pte_t *ptep, *new_ptep = NULL;
206 /* Traverse the guest's 2nd-level tree, allocate new levels needed */
207 pgd = kvm->arch.pgtable + pgd_index(gpa);
209 if (pgd_present(*pgd))
210 pud = pud_offset(pgd, gpa);
212 new_pud = pud_alloc_one(kvm->mm, gpa);
215 if (pud && pud_present(*pud))
216 pmd = pmd_offset(pud, gpa);
218 new_pmd = pmd_alloc_one(kvm->mm, gpa);
220 if (level == 0 && !(pmd && pmd_present(*pmd)))
221 new_ptep = kvmppc_pte_alloc();
223 /* Check if we might have been invalidated; let the guest retry if so */
224 spin_lock(&kvm->mmu_lock);
226 if (mmu_notifier_retry(kvm, mmu_seq))
229 /* Now traverse again under the lock and change the tree */
231 if (pgd_none(*pgd)) {
234 pgd_populate(kvm->mm, pgd, new_pud);
237 pud = pud_offset(pgd, gpa);
238 if (pud_none(*pud)) {
241 pud_populate(kvm->mm, pud, new_pmd);
244 pmd = pmd_offset(pud, gpa);
245 if (pmd_large(*pmd)) {
246 /* Someone else has instantiated a large page here; retry */
250 if (level == 1 && !pmd_none(*pmd)) {
252 * There's a page table page here, but we wanted
253 * to install a large page. Tell the caller and let
254 * it try installing a normal page if it wants.
260 if (pmd_none(*pmd)) {
263 pmd_populate(kvm->mm, pmd, new_ptep);
266 ptep = pte_offset_kernel(pmd, gpa);
267 if (pte_present(*ptep)) {
268 /* PTE was previously valid, so invalidate it */
269 old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT,
271 kvmppc_radix_tlbie_page(kvm, gpa, 0);
272 if (old & _PAGE_DIRTY)
273 mark_page_dirty(kvm, gpa >> PAGE_SHIFT);
275 kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
277 kvmppc_radix_set_pte_at(kvm, gpa, pmdp_ptep(pmd), pte);
282 spin_unlock(&kvm->mmu_lock);
284 pud_free(kvm->mm, new_pud);
286 pmd_free(kvm->mm, new_pmd);
288 kvmppc_pte_free(new_ptep);
292 int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
293 unsigned long ea, unsigned long dsisr)
295 struct kvm *kvm = vcpu->kvm;
296 unsigned long mmu_seq, pte_size;
297 unsigned long gpa, gfn, hva, pfn;
298 struct kvm_memory_slot *memslot;
299 struct page *page = NULL, *pages[1];
300 long ret, npages, ok;
301 unsigned int writing;
302 struct vm_area_struct *vma;
305 unsigned long pgflags;
306 unsigned int shift, level;
308 /* Check for unusual errors */
309 if (dsisr & DSISR_UNSUPP_MMU) {
310 pr_err("KVM: Got unsupported MMU fault\n");
313 if (dsisr & DSISR_BADACCESS) {
314 /* Reflect to the guest as DSI */
315 pr_err("KVM: Got radix HV page fault with DSISR=%lx\n", dsisr);
316 kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
320 /* Translate the logical address and get the page */
321 gpa = vcpu->arch.fault_gpa & ~0xfffUL;
322 gpa &= ~0xF000000000000000ul;
323 gfn = gpa >> PAGE_SHIFT;
324 if (!(dsisr & DSISR_PGDIRFAULT))
326 memslot = gfn_to_memslot(kvm, gfn);
328 /* No memslot means it's an emulated MMIO region */
329 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) {
330 if (dsisr & (DSISR_PGDIRFAULT | DSISR_BADACCESS |
333 * Bad address in guest page table tree, or other
334 * unusual error - reflect it to the guest as DSI.
336 kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
339 return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
340 dsisr & DSISR_ISSTORE);
343 /* used to check for invalidations in progress */
344 mmu_seq = kvm->mmu_notifier_seq;
347 writing = (dsisr & DSISR_ISSTORE) != 0;
348 hva = gfn_to_hva_memslot(memslot, gfn);
349 if (dsisr & DSISR_SET_RC) {
351 * Need to set an R or C bit in the 2nd-level tables;
352 * if the relevant bits aren't already set in the linux
353 * page tables, fall through to do the gup_fast to
354 * set them in the linux page tables too.
357 pgflags = _PAGE_ACCESSED;
359 pgflags |= _PAGE_DIRTY;
360 local_irq_save(flags);
361 ptep = __find_linux_pte_or_hugepte(current->mm->pgd, hva,
364 pte = READ_ONCE(*ptep);
365 if (pte_present(pte) &&
366 (pte_val(pte) & pgflags) == pgflags)
369 local_irq_restore(flags);
371 spin_lock(&kvm->mmu_lock);
372 if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {
373 spin_unlock(&kvm->mmu_lock);
376 ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable,
378 if (ptep && pte_present(*ptep)) {
379 kvmppc_radix_update_pte(kvm, ptep, 0, pgflags,
381 spin_unlock(&kvm->mmu_lock);
384 spin_unlock(&kvm->mmu_lock);
390 pte_size = PAGE_SIZE;
391 pgflags = _PAGE_READ | _PAGE_EXEC;
393 npages = get_user_pages_fast(hva, 1, writing, pages);
395 /* Check if it's an I/O mapping */
396 down_read(¤t->mm->mmap_sem);
397 vma = find_vma(current->mm, hva);
398 if (vma && vma->vm_start <= hva && hva < vma->vm_end &&
399 (vma->vm_flags & VM_PFNMAP)) {
400 pfn = vma->vm_pgoff +
401 ((hva - vma->vm_start) >> PAGE_SHIFT);
402 pgflags = pgprot_val(vma->vm_page_prot);
404 up_read(¤t->mm->mmap_sem);
409 pfn = page_to_pfn(page);
410 if (PageHuge(page)) {
411 page = compound_head(page);
412 pte_size <<= compound_order(page);
413 /* See if we can insert a 2MB large-page PTE here */
414 if (pte_size >= PMD_SIZE &&
415 (gpa & PMD_MASK & PAGE_MASK) ==
416 (hva & PMD_MASK & PAGE_MASK)) {
418 pfn &= ~((PMD_SIZE >> PAGE_SHIFT) - 1);
421 /* See if we can provide write access */
424 * We assume gup_fast has set dirty on the host PTE.
426 pgflags |= _PAGE_WRITE;
428 local_irq_save(flags);
429 ptep = __find_linux_pte_or_hugepte(current->mm->pgd,
431 if (ptep && pte_write(*ptep) && pte_dirty(*ptep))
432 pgflags |= _PAGE_WRITE;
433 local_irq_restore(flags);
438 * Compute the PTE value that we need to insert.
440 pgflags |= _PAGE_PRESENT | _PAGE_PTE | _PAGE_ACCESSED;
441 if (pgflags & _PAGE_WRITE)
442 pgflags |= _PAGE_DIRTY;
443 pte = pfn_pte(pfn, __pgprot(pgflags));
445 /* Allocate space in the tree and write the PTE */
446 ret = kvmppc_create_pte(kvm, pte, gpa, level, mmu_seq);
449 * There's already a PMD where wanted to install a large page;
450 * for now, fall back to installing a small page.
453 pfn |= gfn & ((PMD_SIZE >> PAGE_SHIFT) - 1);
454 pte = pfn_pte(pfn, __pgprot(pgflags));
455 ret = kvmppc_create_pte(kvm, pte, gpa, level, mmu_seq);
457 if (ret == 0 || ret == -EAGAIN)
462 * We drop pages[0] here, not page because page might
463 * have been set to the head page of a compound, but
464 * we have to drop the reference on the correct tail
465 * page to match the get inside gup()
472 static void mark_pages_dirty(struct kvm *kvm, struct kvm_memory_slot *memslot,
473 unsigned long gfn, unsigned int order)
475 unsigned long i, limit;
478 if (!memslot->dirty_bitmap)
480 limit = 1ul << order;
481 if (limit < BITS_PER_LONG) {
482 for (i = 0; i < limit; ++i)
483 mark_page_dirty(kvm, gfn + i);
486 dp = memslot->dirty_bitmap + (gfn - memslot->base_gfn);
487 limit /= BITS_PER_LONG;
488 for (i = 0; i < limit; ++i)
492 /* Called with kvm->lock held */
493 int kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
497 unsigned long gpa = gfn << PAGE_SHIFT;
501 ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable, gpa,
503 if (ptep && pte_present(*ptep)) {
504 old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT, 0,
506 kvmppc_radix_tlbie_page(kvm, gpa, shift);
507 if (old & _PAGE_DIRTY) {
509 mark_page_dirty(kvm, gfn);
511 mark_pages_dirty(kvm, memslot,
512 gfn, shift - PAGE_SHIFT);
518 /* Called with kvm->lock held */
519 int kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
523 unsigned long gpa = gfn << PAGE_SHIFT;
527 ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable, gpa,
529 if (ptep && pte_present(*ptep) && pte_young(*ptep)) {
530 kvmppc_radix_update_pte(kvm, ptep, _PAGE_ACCESSED, 0,
532 /* XXX need to flush tlb here? */
538 /* Called with kvm->lock held */
539 int kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
543 unsigned long gpa = gfn << PAGE_SHIFT;
547 ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable, gpa,
549 if (ptep && pte_present(*ptep) && pte_young(*ptep))
554 /* Returns the number of PAGE_SIZE pages that are dirty */
555 static int kvm_radix_test_clear_dirty(struct kvm *kvm,
556 struct kvm_memory_slot *memslot, int pagenum)
558 unsigned long gfn = memslot->base_gfn + pagenum;
559 unsigned long gpa = gfn << PAGE_SHIFT;
564 ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable, gpa,
566 if (ptep && pte_present(*ptep) && pte_dirty(*ptep)) {
569 ret = 1 << (shift - PAGE_SHIFT);
570 kvmppc_radix_update_pte(kvm, ptep, _PAGE_DIRTY, 0,
572 kvmppc_radix_tlbie_page(kvm, gpa, shift);
577 long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
578 struct kvm_memory_slot *memslot, unsigned long *map)
585 * Radix accumulates dirty bits in the first half of the
586 * memslot's dirty_bitmap area, for when pages are paged
587 * out or modified by the host directly. Pick up these
588 * bits and add them to the map.
590 n = kvm_dirty_bitmap_bytes(memslot) / sizeof(long);
591 p = memslot->dirty_bitmap;
592 for (i = 0; i < n; ++i)
593 map[i] |= xchg(&p[i], 0);
595 for (i = 0; i < memslot->npages; i = j) {
596 npages = kvm_radix_test_clear_dirty(kvm, memslot, i);
599 * Note that if npages > 0 then i must be a multiple of npages,
600 * since huge pages are only used to back the guest at guest
601 * real addresses that are a multiple of their size.
602 * Since we have at most one PTE covering any given guest
603 * real address, if npages > 1 we can skip to i + npages.
607 for (j = i; npages; ++j, --npages)
608 __set_bit_le(j, map);
613 static void add_rmmu_ap_encoding(struct kvm_ppc_rmmu_info *info,
614 int psize, int *indexp)
616 if (!mmu_psize_defs[psize].shift)
618 info->ap_encodings[*indexp] = mmu_psize_defs[psize].shift |
619 (mmu_psize_defs[psize].ap << 29);
623 int kvmhv_get_rmmu_info(struct kvm *kvm, struct kvm_ppc_rmmu_info *info)
627 if (!radix_enabled())
629 memset(info, 0, sizeof(*info));
632 info->geometries[0].page_shift = 12;
633 info->geometries[0].level_bits[0] = 9;
634 for (i = 1; i < 4; ++i)
635 info->geometries[0].level_bits[i] = p9_supported_radix_bits[i];
637 info->geometries[1].page_shift = 16;
638 for (i = 0; i < 4; ++i)
639 info->geometries[1].level_bits[i] = p9_supported_radix_bits[i];
642 add_rmmu_ap_encoding(info, MMU_PAGE_4K, &i);
643 add_rmmu_ap_encoding(info, MMU_PAGE_64K, &i);
644 add_rmmu_ap_encoding(info, MMU_PAGE_2M, &i);
645 add_rmmu_ap_encoding(info, MMU_PAGE_1G, &i);
650 int kvmppc_init_vm_radix(struct kvm *kvm)
652 kvm->arch.pgtable = pgd_alloc(kvm->mm);
653 if (!kvm->arch.pgtable)
658 void kvmppc_free_radix(struct kvm *kvm)
660 unsigned long ig, iu, im;
666 if (!kvm->arch.pgtable)
668 pgd = kvm->arch.pgtable;
669 for (ig = 0; ig < PTRS_PER_PGD; ++ig, ++pgd) {
670 if (!pgd_present(*pgd))
672 pud = pud_offset(pgd, 0);
673 for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++pud) {
674 if (!pud_present(*pud))
676 pmd = pmd_offset(pud, 0);
677 for (im = 0; im < PTRS_PER_PMD; ++im, ++pmd) {
678 if (pmd_huge(*pmd)) {
682 if (!pmd_present(*pmd))
684 pte = pte_offset_map(pmd, 0);
685 memset(pte, 0, sizeof(long) << PTE_INDEX_SIZE);
686 kvmppc_pte_free(pte);
689 pmd_free(kvm->mm, pmd_offset(pud, 0));
692 pud_free(kvm->mm, pud_offset(pgd, 0));
695 pgd_free(kvm->mm, kvm->arch.pgtable);
698 static void pte_ctor(void *addr)
700 memset(addr, 0, PTE_TABLE_SIZE);
703 int kvmppc_radix_init(void)
705 unsigned long size = sizeof(void *) << PTE_INDEX_SIZE;
707 kvm_pte_cache = kmem_cache_create("kvm-pte", size, size, 0, pte_ctor);
713 void kvmppc_radix_exit(void)
715 kmem_cache_destroy(kvm_pte_cache);