KVM: x86: reserve bit 8 of non-leaf PDPEs and PML4Es in 64-bit mode on AMD

[karo-tx-linux.git] / arch / x86 / kvm / mmu.c

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 323c3f5f5c84968eb35561978392286405323afe..5b93a597e0c81536e7851f483e6f91754076224d 100644 (file)
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -199,16 +199,20 @@ void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
 EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
 
 /*
- * spte bits of bit 3 ~ bit 11 are used as low 9 bits of generation number,
- * the bits of bits 52 ~ bit 61 are used as high 10 bits of generation
- * number.
+ * the low bit of the generation number is always presumed to be zero.
+ * This disables mmio caching during memslot updates.  The concept is
+ * similar to a seqcount but instead of retrying the access we just punt
+ * and ignore the cache.
+ *
+ * spte bits 3-11 are used as bits 1-9 of the generation number,
+ * the bits 52-61 are used as bits 10-19 of the generation number.
  */
-#define MMIO_SPTE_GEN_LOW_SHIFT                3
+#define MMIO_SPTE_GEN_LOW_SHIFT                2
 #define MMIO_SPTE_GEN_HIGH_SHIFT       52
 
-#define MMIO_GEN_SHIFT                 19
-#define MMIO_GEN_LOW_SHIFT             9
-#define MMIO_GEN_LOW_MASK              ((1 << MMIO_GEN_LOW_SHIFT) - 1)
+#define MMIO_GEN_SHIFT                 20
+#define MMIO_GEN_LOW_SHIFT             10
+#define MMIO_GEN_LOW_MASK              ((1 << MMIO_GEN_LOW_SHIFT) - 2)
 #define MMIO_GEN_MASK                  ((1 << MMIO_GEN_SHIFT) - 1)
 #define MMIO_MAX_GEN                   ((1 << MMIO_GEN_SHIFT) - 1)
 
@@ -291,11 +295,6 @@ static bool check_mmio_spte(struct kvm *kvm, u64 spte)
        return likely(kvm_gen == spte_gen);
 }
 
-static inline u64 rsvd_bits(int s, int e)
-{
-       return ((1ULL << (e - s + 1)) - 1) << s;
-}
-
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                u64 dirty_mask, u64 nx_mask, u64 x_mask)
 {
@@ -3158,7 +3157,7 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu)
        if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
                return;
 
-       vcpu_clear_mmio_info(vcpu, ~0ul);
+       vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
        kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
        if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
                hpa_t root = vcpu->arch.mmu.root_hpa;
@@ -3513,6 +3512,7 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
        int maxphyaddr = cpuid_maxphyaddr(vcpu);
        u64 exb_bit_rsvd = 0;
        u64 gbpages_bit_rsvd = 0;
+       u64 nonleaf_bit8_rsvd = 0;
 
        context->bad_mt_xwr = 0;
 
@@ -3520,6 +3520,14 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
                exb_bit_rsvd = rsvd_bits(63, 63);
        if (!guest_cpuid_has_gbpages(vcpu))
                gbpages_bit_rsvd = rsvd_bits(7, 7);
+
+       /*
+        * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
+        * leaf entries) on AMD CPUs only.
+        */
+       if (guest_cpuid_is_amd(vcpu))
+               nonleaf_bit8_rsvd = rsvd_bits(8, 8);
+
        switch (context->root_level) {
        case PT32_ROOT_LEVEL:
                /* no rsvd bits for 2 level 4K page table entries */
@@ -3554,9 +3562,9 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
                break;
        case PT64_ROOT_LEVEL:
                context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
-                       rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 7);
+                       nonleaf_bit8_rsvd | rsvd_bits(7, 7) | rsvd_bits(maxphyaddr, 51);
                context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
-                       gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51);
+                       nonleaf_bit8_rsvd | gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51);
                context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
                        rsvd_bits(maxphyaddr, 51);
                context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
@@ -4428,7 +4436,7 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm)
         * The very rare case: if the generation-number is round,
         * zap all shadow pages.
         */
-       if (unlikely(kvm_current_mmio_generation(kvm) >= MMIO_MAX_GEN)) {
+       if (unlikely(kvm_current_mmio_generation(kvm) == 0)) {
                printk_ratelimited(KERN_INFO "kvm: zapping shadow pages for mmio generation wraparound\n");
                kvm_mmu_invalidate_zap_all_pages(kvm);
        }