kvm: x86: avoid atomic operations on APICv vmentry

On some benchmarks (e.g. netperf with ioeventfd disabled), APICv
posted interrupts turn out to be slower than interrupt injection via
KVM_REQ_EVENT.

This patch optimizes a bit the IRR update, avoiding expensive atomic
operations in the common case where PI.ON=0 at vmentry or the PIR vector
is mostly zero.  This saves at least 20 cycles (1%) per vmexit, as
measured by kvm-unit-tests' inl_from_qemu test (20 runs):

              | enable_apicv=1  |  enable_apicv=0
              | mean     stdev  |  mean     stdev
    ----------|-----------------|------------------
    before    | 5826     32.65  |  5765     47.09
    after     | 5809     43.42  |  5777     77.02

Of course, any change in the right column is just placebo effect. :)
The savings are bigger if interrupts are frequent.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 6acd76b7e71d553ac72ce32d467f966f0d49368c..890f218ddd7a26f28b7692440c981f3da6d7cbfc 100644 (file)
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -342,9 +342,11 @@ void __kvm_apic_update_irr(u32 *pir, void *regs)
        u32 i, pir_val;
 
        for (i = 0; i <= 7; i++) {
-               pir_val = xchg(&pir[i], 0);
-               if (pir_val)
+               pir_val = READ_ONCE(pir[i]);
+               if (pir_val) {
+                       pir_val = xchg(&pir[i], 0);
                        *((u32 *)(regs + APIC_IRR + i * 0x10)) |= pir_val;
+               }
        }
 }
 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index dcb840e8bfe3bdf84b83a4352969ae9824f9e07d..a91a5b01f38ef5e31b84c689912d374bc42133d5 100644 (file)
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -520,6 +520,12 @@ static inline void pi_set_sn(struct pi_desc *pi_desc)
                        (unsigned long *)&pi_desc->control);
 }
 
+static inline void pi_clear_on(struct pi_desc *pi_desc)
+{
+       clear_bit(POSTED_INTR_ON,
+                 (unsigned long *)&pi_desc->control);
+}
+
 static inline int pi_test_on(struct pi_desc *pi_desc)
 {
        return test_bit(POSTED_INTR_ON,
@@ -4780,9 +4786,15 @@ static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-       if (!pi_test_and_clear_on(&vmx->pi_desc))
+       if (!pi_test_on(&vmx->pi_desc))
                return;
 
+       pi_clear_on(&vmx->pi_desc);
+       /*
+        * IOMMU can write to PIR.ON, so the barrier matters even on UP.
+        * But on x86 this is just a compiler barrier anyway.
+        */
+       smp_mb__after_atomic();
        kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
 }