Merge branch 'for-linus-4.7' of git://git.kernel.org/pub/scm/linux/kernel/git/mason...

[karo-tx-linux.git] / virt / kvm / kvm_main.c

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 7ba1d10ffed2d5a416701153caea619c7ab22f8d..dd4ac9d9e8f5c71abd930e1ed079b4c52834cd51 100644 (file)
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -170,8 +170,8 @@ bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
                kvm_make_request(req, vcpu);
                cpu = vcpu->cpu;
 
-               /* Set ->requests bit before we read ->mode */
-               smp_mb();
+               /* Set ->requests bit before we read ->mode. */
+               smp_mb__after_atomic();
 
                if (cpus != NULL && cpu != -1 && cpu != me &&
                      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
@@ -191,9 +191,23 @@ bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
 void kvm_flush_remote_tlbs(struct kvm *kvm)
 {
-       long dirty_count = kvm->tlbs_dirty;
+       /*
+        * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
+        * kvm_make_all_cpus_request.
+        */
+       long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
 
-       smp_mb();
+       /*
+        * We want to publish modifications to the page tables before reading
+        * mode. Pairs with a memory barrier in arch-specific code.
+        * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
+        * and smp_mb in walk_shadow_page_lockless_begin/end.
+        * - powerpc: smp_mb in kvmppc_prepare_to_enter.
+        *
+        * There is already an smp_mb__after_atomic() before
+        * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
+        * barrier here.
+        */
        if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
                ++kvm->stat.remote_tlb_flush;
        cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
@@ -536,6 +550,16 @@ static struct kvm *kvm_create_vm(unsigned long type)
        if (!kvm)
                return ERR_PTR(-ENOMEM);
 
+       spin_lock_init(&kvm->mmu_lock);
+       atomic_inc(&current->mm->mm_count);
+       kvm->mm = current->mm;
+       kvm_eventfd_init(kvm);
+       mutex_init(&kvm->lock);
+       mutex_init(&kvm->irq_lock);
+       mutex_init(&kvm->slots_lock);
+       atomic_set(&kvm->users_count, 1);
+       INIT_LIST_HEAD(&kvm->devices);
+
        r = kvm_arch_init_vm(kvm, type);
        if (r)
                goto out_err_no_disable;
@@ -568,16 +592,6 @@ static struct kvm *kvm_create_vm(unsigned long type)
                        goto out_err;
        }
 
-       spin_lock_init(&kvm->mmu_lock);
-       kvm->mm = current->mm;
-       atomic_inc(&kvm->mm->mm_count);
-       kvm_eventfd_init(kvm);
-       mutex_init(&kvm->lock);
-       mutex_init(&kvm->irq_lock);
-       mutex_init(&kvm->slots_lock);
-       atomic_set(&kvm->users_count, 1);
-       INIT_LIST_HEAD(&kvm->devices);
-
        r = kvm_init_mmu_notifier(kvm);
        if (r)
                goto out_err;
@@ -602,6 +616,7 @@ out_err_no_disable:
        for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
                kvm_free_memslots(kvm, kvm->memslots[i]);
        kvm_arch_free_vm(kvm);
+       mmdrop(current->mm);
        return ERR_PTR(r);
 }
 
@@ -1260,15 +1275,16 @@ unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *w
        return gfn_to_hva_memslot_prot(slot, gfn, writable);
 }
 
-static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
-       unsigned long start, int write, struct page **page)
+static int get_user_page_nowait(unsigned long start, int write,
+               struct page **page)
 {
        int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
 
        if (write)
                flags |= FOLL_WRITE;
 
-       return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
+       return __get_user_pages(current, current->mm, start, 1, flags, page,
+                       NULL, NULL);
 }
 
 static inline int check_user_page_hwpoison(unsigned long addr)
@@ -1330,8 +1346,7 @@ static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
 
        if (async) {
                down_read(&current->mm->mmap_sem);
-               npages = get_user_page_nowait(current, current->mm,
-                                             addr, write_fault, page);
+               npages = get_user_page_nowait(addr, write_fault, page);
                up_read(&current->mm->mmap_sem);
        } else
                npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
@@ -2013,6 +2028,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
                         */
                        if (kvm_vcpu_check_block(vcpu) < 0) {
                                ++vcpu->stat.halt_successful_poll;
+                               if (!vcpu_valid_wakeup(vcpu))
+                                       ++vcpu->stat.halt_poll_invalid;
                                goto out;
                        }
                        cur = ktime_get();
@@ -2038,7 +2055,9 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 out:
        block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
 
-       if (halt_poll_ns) {
+       if (!vcpu_valid_wakeup(vcpu))
+               shrink_halt_poll_ns(vcpu);
+       else if (halt_poll_ns) {
                if (block_ns <= vcpu->halt_poll_ns)
                        ;
                /* we had a long block, shrink polling */
@@ -2051,18 +2070,14 @@ out:
        } else
                vcpu->halt_poll_ns = 0;
 
-       trace_kvm_vcpu_wakeup(block_ns, waited);
+       trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
+       kvm_arch_vcpu_block_finish(vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
 
 #ifndef CONFIG_S390
-/*
- * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
- */
-void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
+void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
 {
-       int me;
-       int cpu = vcpu->cpu;
        struct swait_queue_head *wqp;
 
        wqp = kvm_arch_vcpu_wq(vcpu);
@@ -2071,6 +2086,18 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
                ++vcpu->stat.halt_wakeup;
        }
 
+}
+EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
+
+/*
+ * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
+ */
+void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
+{
+       int me;
+       int cpu = vcpu->cpu;
+
+       kvm_vcpu_wake_up(vcpu);
        me = get_cpu();
        if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
                if (kvm_arch_vcpu_should_kick(vcpu))
@@ -2257,7 +2284,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
        int r;
        struct kvm_vcpu *vcpu;
 
-       if (id >= KVM_MAX_VCPUS)
+       if (id >= KVM_MAX_VCPU_ID)
                return -EINVAL;
 
        vcpu = kvm_arch_vcpu_create(kvm, id);
@@ -2731,6 +2758,8 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
        case KVM_CAP_MULTI_ADDRESS_SPACE:
                return KVM_ADDRESS_SPACE_NUM;
 #endif
+       case KVM_CAP_MAX_VCPU_ID:
+               return KVM_MAX_VCPU_ID;
        default:
                break;
        }