(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
-#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
+#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
#define PT64_DIR_BASE_ADDR_MASK \
(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
{
struct kvm *kvm = vcpu->kvm;
struct page *page;
- struct kvm_memory_slot *slot;
struct kvm_rmap_desc *desc;
u64 *spte;
- slot = gfn_to_memslot(kvm, gfn);
- BUG_ON(!slot);
- page = gfn_to_page(slot, gfn);
+ page = gfn_to_page(kvm, gfn);
+ BUG_ON(!page);
while (page_private(page)) {
if (!(page_private(page) & 1))
spte = desc->shadow_ptes[0];
}
BUG_ON(!spte);
- BUG_ON((*spte & PT64_BASE_ADDR_MASK) !=
- page_to_pfn(page) << PAGE_SHIFT);
+ BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
+ != page_to_pfn(page));
BUG_ON(!(*spte & PT_PRESENT_MASK));
BUG_ON(!(*spte & PT_WRITABLE_MASK));
rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
struct kvm_mmu_page *page_head = page_header(page_hpa);
ASSERT(is_empty_shadow_page(page_hpa));
- list_del(&page_head->link);
page_head->page_hpa = page_hpa;
- list_add(&page_head->link, &vcpu->free_pages);
+ list_move(&page_head->link, &vcpu->free_pages);
++vcpu->kvm->n_free_mmu_pages;
}
return NULL;
page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
- list_del(&page->link);
- list_add(&page->link, &vcpu->kvm->active_mmu_pages);
+ list_move(&page->link, &vcpu->kvm->active_mmu_pages);
ASSERT(is_empty_shadow_page(page->page_hpa));
page->slot_bitmap = 0;
- page->global = 1;
page->multimapped = 0;
page->parent_pte = parent_pte;
--vcpu->kvm->n_free_mmu_pages;
gva_t gaddr,
unsigned level,
int metaphysical,
+ unsigned hugepage_access,
u64 *parent_pte)
{
union kvm_mmu_page_role role;
role.glevels = vcpu->mmu.root_level;
role.level = level;
role.metaphysical = metaphysical;
+ role.hugepage_access = hugepage_access;
if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
if (!page->root_count) {
hlist_del(&page->hash_link);
kvm_mmu_free_page(vcpu, page->page_hpa);
- } else {
- list_del(&page->link);
- list_add(&page->link, &vcpu->kvm->active_mmu_pages);
- }
+ } else
+ list_move(&page->link, &vcpu->kvm->active_mmu_pages);
}
static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
- struct kvm_memory_slot *slot;
struct page *page;
ASSERT((gpa & HPA_ERR_MASK) == 0);
- slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
- if (!slot)
+ page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+ if (!page)
return gpa | HPA_ERR_MASK;
- page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
| (gpa & (PAGE_SIZE-1));
}
return gpa_to_hpa(vcpu, gpa);
}
+struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
+{
+ gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
+
+ if (gpa == UNMAPPED_GVA)
+ return NULL;
+ return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
+}
+
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
}
>> PAGE_SHIFT;
new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
v, level - 1,
- 1, &table[index]);
+ 1, 0, &table[index]);
if (!new_table) {
pgprintk("nonpaging_map: ENOMEM\n");
return -ENOMEM;
ASSERT(!VALID_PAGE(root));
page = kvm_mmu_get_page(vcpu, root_gfn, 0,
- PT64_ROOT_LEVEL, 0, NULL);
+ PT64_ROOT_LEVEL, 0, 0, NULL);
root = page->page_hpa;
++page->root_count;
vcpu->mmu.root_hpa = root;
root_gfn = 0;
page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
PT32_ROOT_LEVEL, !is_paging(vcpu),
- NULL);
+ 0, NULL);
root = page->page_hpa;
++page->root_count;
vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
}
-static void mark_pagetable_nonglobal(void *shadow_pte)
-{
- page_header(__pa(shadow_pte))->global = 0;
-}
-
static inline void set_pte_common(struct kvm_vcpu *vcpu,
u64 *shadow_pte,
gpa_t gaddr,
*shadow_pte |= access_bits;
- if (!(*shadow_pte & PT_GLOBAL_MASK))
- mark_pagetable_nonglobal(shadow_pte);
-
if (is_error_hpa(paddr)) {
*shadow_pte |= gaddr;
*shadow_pte |= PT_SHADOW_IO_MARK;
return r;
}
+static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
+ struct kvm_mmu_page *page,
+ u64 *spte)
+{
+ u64 pte;
+ struct kvm_mmu_page *child;
+
+ pte = *spte;
+ if (is_present_pte(pte)) {
+ if (page->role.level == PT_PAGE_TABLE_LEVEL)
+ rmap_remove(vcpu, spte);
+ else {
+ child = page_header(pte & PT64_BASE_ADDR_MASK);
+ mmu_page_remove_parent_pte(vcpu, child, spte);
+ }
+ }
+ *spte = 0;
+}
+
void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
{
gfn_t gfn = gpa >> PAGE_SHIFT;
struct kvm_mmu_page *page;
- struct kvm_mmu_page *child;
struct hlist_node *node, *n;
struct hlist_head *bucket;
unsigned index;
u64 *spte;
- u64 pte;
unsigned offset = offset_in_page(gpa);
unsigned pte_size;
unsigned page_offset;
unsigned misaligned;
int level;
int flooded = 0;
+ int npte;
pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
if (gfn == vcpu->last_pt_write_gfn) {
}
page_offset = offset;
level = page->role.level;
+ npte = 1;
if (page->role.glevels == PT32_ROOT_LEVEL) {
- page_offset <<= 1; /* 32->64 */
+ page_offset <<= 1; /* 32->64 */
+ /*
+ * A 32-bit pde maps 4MB while the shadow pdes map
+ * only 2MB. So we need to double the offset again
+ * and zap two pdes instead of one.
+ */
+ if (level == PT32_ROOT_LEVEL) {
+ page_offset &= ~7; /* kill rounding error */
+ page_offset <<= 1;
+ npte = 2;
+ }
page_offset &= ~PAGE_MASK;
}
spte = __va(page->page_hpa);
spte += page_offset / sizeof(*spte);
- pte = *spte;
- if (is_present_pte(pte)) {
- if (level == PT_PAGE_TABLE_LEVEL)
- rmap_remove(vcpu, spte);
- else {
- child = page_header(pte & PT64_BASE_ADDR_MASK);
- mmu_page_remove_parent_pte(vcpu, child, spte);
- }
+ while (npte--) {
+ mmu_pre_write_zap_pte(vcpu, page, spte);
+ ++spte;
}
- *spte = 0;
}
}
}
}
+void kvm_mmu_zap_all(struct kvm_vcpu *vcpu)
+{
+ destroy_kvm_mmu(vcpu);
+
+ while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
+ struct kvm_mmu_page *page;
+
+ page = container_of(vcpu->kvm->active_mmu_pages.next,
+ struct kvm_mmu_page, link);
+ kvm_mmu_zap_page(vcpu, page);
+ }
+
+ mmu_free_memory_caches(vcpu);
+ kvm_arch_ops->tlb_flush(vcpu);
+ init_kvm_mmu(vcpu);
+}
+
#ifdef AUDIT
static const char *audit_msg;
static void audit_mappings(struct kvm_vcpu *vcpu)
{
- int i;
+ unsigned i;
if (vcpu->mmu.root_level == 4)
audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);