2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/slab.h>
26 #include <linux/timer.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/lowcore.h>
29 #include <asm/pgtable.h>
31 #include <asm/switch_to.h>
32 #include <asm/facility.h>
37 #define CREATE_TRACE_POINTS
39 #include "trace-s390.h"
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
43 struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
45 { "exit_null", VCPU_STAT(exit_null) },
46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
53 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
54 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
55 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
56 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
57 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
58 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
59 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
60 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
61 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
62 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
63 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
64 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
65 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
66 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
67 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
68 { "instruction_spx", VCPU_STAT(instruction_spx) },
69 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
70 { "instruction_stap", VCPU_STAT(instruction_stap) },
71 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
72 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
73 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
74 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
75 { "instruction_essa", VCPU_STAT(instruction_essa) },
76 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
77 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
78 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
79 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
80 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
81 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
82 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
83 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
84 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
85 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
86 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
87 { "diagnose_10", VCPU_STAT(diagnose_10) },
88 { "diagnose_44", VCPU_STAT(diagnose_44) },
89 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
93 unsigned long *vfacilities;
94 static struct gmap_notifier gmap_notifier;
96 /* test availability of vfacility */
97 int test_vfacility(unsigned long nr)
99 return __test_facility(nr, (void *) vfacilities);
102 /* Section: not file related */
103 int kvm_arch_hardware_enable(void *garbage)
105 /* every s390 is virtualization enabled ;-) */
109 void kvm_arch_hardware_disable(void *garbage)
113 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
115 int kvm_arch_hardware_setup(void)
117 gmap_notifier.notifier_call = kvm_gmap_notifier;
118 gmap_register_ipte_notifier(&gmap_notifier);
122 void kvm_arch_hardware_unsetup(void)
124 gmap_unregister_ipte_notifier(&gmap_notifier);
127 void kvm_arch_check_processor_compat(void *rtn)
131 int kvm_arch_init(void *opaque)
136 void kvm_arch_exit(void)
140 /* Section: device related */
141 long kvm_arch_dev_ioctl(struct file *filp,
142 unsigned int ioctl, unsigned long arg)
144 if (ioctl == KVM_S390_ENABLE_SIE)
145 return s390_enable_sie();
149 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
154 case KVM_CAP_S390_PSW:
155 case KVM_CAP_S390_GMAP:
156 case KVM_CAP_SYNC_MMU:
157 #ifdef CONFIG_KVM_S390_UCONTROL
158 case KVM_CAP_S390_UCONTROL:
160 case KVM_CAP_ASYNC_PF:
161 case KVM_CAP_SYNC_REGS:
162 case KVM_CAP_ONE_REG:
163 case KVM_CAP_ENABLE_CAP:
164 case KVM_CAP_S390_CSS_SUPPORT:
166 case KVM_CAP_IOEVENTFD:
167 case KVM_CAP_DEVICE_CTRL:
168 case KVM_CAP_ENABLE_CAP_VM:
169 case KVM_CAP_S390_IRQCHIP:
170 case KVM_CAP_VM_ATTRIBUTES:
171 case KVM_CAP_MP_STATE:
174 case KVM_CAP_NR_VCPUS:
175 case KVM_CAP_MAX_VCPUS:
178 case KVM_CAP_NR_MEMSLOTS:
179 r = KVM_USER_MEM_SLOTS;
181 case KVM_CAP_S390_COW:
182 r = MACHINE_HAS_ESOP;
190 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
191 struct kvm_memory_slot *memslot)
193 gfn_t cur_gfn, last_gfn;
194 unsigned long address;
195 struct gmap *gmap = kvm->arch.gmap;
197 down_read(&gmap->mm->mmap_sem);
198 /* Loop over all guest pages */
199 last_gfn = memslot->base_gfn + memslot->npages;
200 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
201 address = gfn_to_hva_memslot(memslot, cur_gfn);
203 if (gmap_test_and_clear_dirty(address, gmap))
204 mark_page_dirty(kvm, cur_gfn);
206 up_read(&gmap->mm->mmap_sem);
209 /* Section: vm related */
211 * Get (and clear) the dirty memory log for a memory slot.
213 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
214 struct kvm_dirty_log *log)
218 struct kvm_memory_slot *memslot;
221 mutex_lock(&kvm->slots_lock);
224 if (log->slot >= KVM_USER_MEM_SLOTS)
227 memslot = id_to_memslot(kvm->memslots, log->slot);
229 if (!memslot->dirty_bitmap)
232 kvm_s390_sync_dirty_log(kvm, memslot);
233 r = kvm_get_dirty_log(kvm, log, &is_dirty);
237 /* Clear the dirty log */
239 n = kvm_dirty_bitmap_bytes(memslot);
240 memset(memslot->dirty_bitmap, 0, n);
244 mutex_unlock(&kvm->slots_lock);
248 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
256 case KVM_CAP_S390_IRQCHIP:
257 kvm->arch.use_irqchip = 1;
267 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
271 switch (attr->attr) {
272 case KVM_S390_VM_MEM_ENABLE_CMMA:
274 mutex_lock(&kvm->lock);
275 if (atomic_read(&kvm->online_vcpus) == 0) {
276 kvm->arch.use_cmma = 1;
279 mutex_unlock(&kvm->lock);
281 case KVM_S390_VM_MEM_CLR_CMMA:
282 mutex_lock(&kvm->lock);
283 idx = srcu_read_lock(&kvm->srcu);
284 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
285 srcu_read_unlock(&kvm->srcu, idx);
286 mutex_unlock(&kvm->lock);
296 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
300 switch (attr->group) {
301 case KVM_S390_VM_MEM_CTRL:
302 ret = kvm_s390_mem_control(kvm, attr);
312 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
317 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
321 switch (attr->group) {
322 case KVM_S390_VM_MEM_CTRL:
323 switch (attr->attr) {
324 case KVM_S390_VM_MEM_ENABLE_CMMA:
325 case KVM_S390_VM_MEM_CLR_CMMA:
341 long kvm_arch_vm_ioctl(struct file *filp,
342 unsigned int ioctl, unsigned long arg)
344 struct kvm *kvm = filp->private_data;
345 void __user *argp = (void __user *)arg;
346 struct kvm_device_attr attr;
350 case KVM_S390_INTERRUPT: {
351 struct kvm_s390_interrupt s390int;
354 if (copy_from_user(&s390int, argp, sizeof(s390int)))
356 r = kvm_s390_inject_vm(kvm, &s390int);
359 case KVM_ENABLE_CAP: {
360 struct kvm_enable_cap cap;
362 if (copy_from_user(&cap, argp, sizeof(cap)))
364 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
367 case KVM_CREATE_IRQCHIP: {
368 struct kvm_irq_routing_entry routing;
371 if (kvm->arch.use_irqchip) {
372 /* Set up dummy routing. */
373 memset(&routing, 0, sizeof(routing));
374 kvm_set_irq_routing(kvm, &routing, 0, 0);
379 case KVM_SET_DEVICE_ATTR: {
381 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
383 r = kvm_s390_vm_set_attr(kvm, &attr);
386 case KVM_GET_DEVICE_ATTR: {
388 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
390 r = kvm_s390_vm_get_attr(kvm, &attr);
393 case KVM_HAS_DEVICE_ATTR: {
395 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
397 r = kvm_s390_vm_has_attr(kvm, &attr);
407 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
411 static unsigned long sca_offset;
414 #ifdef CONFIG_KVM_S390_UCONTROL
415 if (type & ~KVM_VM_S390_UCONTROL)
417 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
424 rc = s390_enable_sie();
430 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
433 spin_lock(&kvm_lock);
434 sca_offset = (sca_offset + 16) & 0x7f0;
435 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
436 spin_unlock(&kvm_lock);
438 sprintf(debug_name, "kvm-%u", current->pid);
440 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
444 spin_lock_init(&kvm->arch.float_int.lock);
445 INIT_LIST_HEAD(&kvm->arch.float_int.list);
446 init_waitqueue_head(&kvm->arch.ipte_wq);
448 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
449 VM_EVENT(kvm, 3, "%s", "vm created");
451 if (type & KVM_VM_S390_UCONTROL) {
452 kvm->arch.gmap = NULL;
454 kvm->arch.gmap = gmap_alloc(current->mm);
457 kvm->arch.gmap->private = kvm;
458 kvm->arch.gmap->pfault_enabled = 0;
461 kvm->arch.css_support = 0;
462 kvm->arch.use_irqchip = 0;
464 spin_lock_init(&kvm->arch.start_stop_lock);
468 debug_unregister(kvm->arch.dbf);
470 free_page((unsigned long)(kvm->arch.sca));
475 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
477 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
478 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
479 kvm_s390_clear_local_irqs(vcpu);
480 kvm_clear_async_pf_completion_queue(vcpu);
481 if (!kvm_is_ucontrol(vcpu->kvm)) {
482 clear_bit(63 - vcpu->vcpu_id,
483 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
484 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
485 (__u64) vcpu->arch.sie_block)
486 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
490 if (kvm_is_ucontrol(vcpu->kvm))
491 gmap_free(vcpu->arch.gmap);
493 if (kvm_s390_cmma_enabled(vcpu->kvm))
494 kvm_s390_vcpu_unsetup_cmma(vcpu);
495 free_page((unsigned long)(vcpu->arch.sie_block));
497 kvm_vcpu_uninit(vcpu);
498 kmem_cache_free(kvm_vcpu_cache, vcpu);
501 static void kvm_free_vcpus(struct kvm *kvm)
504 struct kvm_vcpu *vcpu;
506 kvm_for_each_vcpu(i, vcpu, kvm)
507 kvm_arch_vcpu_destroy(vcpu);
509 mutex_lock(&kvm->lock);
510 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
511 kvm->vcpus[i] = NULL;
513 atomic_set(&kvm->online_vcpus, 0);
514 mutex_unlock(&kvm->lock);
517 void kvm_arch_sync_events(struct kvm *kvm)
521 void kvm_arch_destroy_vm(struct kvm *kvm)
524 free_page((unsigned long)(kvm->arch.sca));
525 debug_unregister(kvm->arch.dbf);
526 if (!kvm_is_ucontrol(kvm))
527 gmap_free(kvm->arch.gmap);
528 kvm_s390_destroy_adapters(kvm);
529 kvm_s390_clear_float_irqs(kvm);
532 /* Section: vcpu related */
533 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
535 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
536 kvm_clear_async_pf_completion_queue(vcpu);
537 if (kvm_is_ucontrol(vcpu->kvm)) {
538 vcpu->arch.gmap = gmap_alloc(current->mm);
539 if (!vcpu->arch.gmap)
541 vcpu->arch.gmap->private = vcpu->kvm;
545 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
546 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
553 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
558 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
560 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
561 save_fp_regs(vcpu->arch.host_fpregs.fprs);
562 save_access_regs(vcpu->arch.host_acrs);
563 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
564 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
565 restore_access_regs(vcpu->run->s.regs.acrs);
566 gmap_enable(vcpu->arch.gmap);
567 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
570 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
572 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
573 gmap_disable(vcpu->arch.gmap);
574 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
575 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
576 save_access_regs(vcpu->run->s.regs.acrs);
577 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
578 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
579 restore_access_regs(vcpu->arch.host_acrs);
582 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
584 /* this equals initial cpu reset in pop, but we don't switch to ESA */
585 vcpu->arch.sie_block->gpsw.mask = 0UL;
586 vcpu->arch.sie_block->gpsw.addr = 0UL;
587 kvm_s390_set_prefix(vcpu, 0);
588 vcpu->arch.sie_block->cputm = 0UL;
589 vcpu->arch.sie_block->ckc = 0UL;
590 vcpu->arch.sie_block->todpr = 0;
591 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
592 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
593 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
594 vcpu->arch.guest_fpregs.fpc = 0;
595 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
596 vcpu->arch.sie_block->gbea = 1;
597 vcpu->arch.sie_block->pp = 0;
598 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
599 kvm_clear_async_pf_completion_queue(vcpu);
600 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
601 kvm_s390_vcpu_stop(vcpu);
602 kvm_s390_clear_local_irqs(vcpu);
605 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
610 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
612 free_page(vcpu->arch.sie_block->cbrlo);
613 vcpu->arch.sie_block->cbrlo = 0;
616 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
618 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
619 if (!vcpu->arch.sie_block->cbrlo)
622 vcpu->arch.sie_block->ecb2 |= 0x80;
623 vcpu->arch.sie_block->ecb2 &= ~0x08;
627 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
631 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
635 vcpu->arch.sie_block->ecb = 6;
636 if (test_vfacility(50) && test_vfacility(73))
637 vcpu->arch.sie_block->ecb |= 0x10;
639 vcpu->arch.sie_block->ecb2 = 8;
640 vcpu->arch.sie_block->eca = 0xD1002000U;
642 vcpu->arch.sie_block->eca |= 1;
643 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
644 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
647 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
648 rc = kvm_s390_vcpu_setup_cmma(vcpu);
652 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
653 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
654 get_cpu_id(&vcpu->arch.cpu_id);
655 vcpu->arch.cpu_id.version = 0xff;
659 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
662 struct kvm_vcpu *vcpu;
663 struct sie_page *sie_page;
666 if (id >= KVM_MAX_VCPUS)
671 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
675 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
679 vcpu->arch.sie_block = &sie_page->sie_block;
680 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
682 vcpu->arch.sie_block->icpua = id;
683 if (!kvm_is_ucontrol(kvm)) {
684 if (!kvm->arch.sca) {
688 if (!kvm->arch.sca->cpu[id].sda)
689 kvm->arch.sca->cpu[id].sda =
690 (__u64) vcpu->arch.sie_block;
691 vcpu->arch.sie_block->scaoh =
692 (__u32)(((__u64)kvm->arch.sca) >> 32);
693 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
694 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
697 spin_lock_init(&vcpu->arch.local_int.lock);
698 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
699 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
700 vcpu->arch.local_int.wq = &vcpu->wq;
701 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
703 rc = kvm_vcpu_init(vcpu, kvm, id);
705 goto out_free_sie_block;
706 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
707 vcpu->arch.sie_block);
708 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
712 free_page((unsigned long)(vcpu->arch.sie_block));
714 kmem_cache_free(kvm_vcpu_cache, vcpu);
719 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
721 return kvm_cpu_has_interrupt(vcpu);
724 void s390_vcpu_block(struct kvm_vcpu *vcpu)
726 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
729 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
731 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
735 * Kick a guest cpu out of SIE and wait until SIE is not running.
736 * If the CPU is not running (e.g. waiting as idle) the function will
737 * return immediately. */
738 void exit_sie(struct kvm_vcpu *vcpu)
740 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
741 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
745 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
746 void exit_sie_sync(struct kvm_vcpu *vcpu)
748 s390_vcpu_block(vcpu);
752 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
755 struct kvm *kvm = gmap->private;
756 struct kvm_vcpu *vcpu;
758 kvm_for_each_vcpu(i, vcpu, kvm) {
759 /* match against both prefix pages */
760 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
761 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
762 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
768 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
770 /* kvm common code refers to this, but never calls it */
775 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
776 struct kvm_one_reg *reg)
781 case KVM_REG_S390_TODPR:
782 r = put_user(vcpu->arch.sie_block->todpr,
783 (u32 __user *)reg->addr);
785 case KVM_REG_S390_EPOCHDIFF:
786 r = put_user(vcpu->arch.sie_block->epoch,
787 (u64 __user *)reg->addr);
789 case KVM_REG_S390_CPU_TIMER:
790 r = put_user(vcpu->arch.sie_block->cputm,
791 (u64 __user *)reg->addr);
793 case KVM_REG_S390_CLOCK_COMP:
794 r = put_user(vcpu->arch.sie_block->ckc,
795 (u64 __user *)reg->addr);
797 case KVM_REG_S390_PFTOKEN:
798 r = put_user(vcpu->arch.pfault_token,
799 (u64 __user *)reg->addr);
801 case KVM_REG_S390_PFCOMPARE:
802 r = put_user(vcpu->arch.pfault_compare,
803 (u64 __user *)reg->addr);
805 case KVM_REG_S390_PFSELECT:
806 r = put_user(vcpu->arch.pfault_select,
807 (u64 __user *)reg->addr);
809 case KVM_REG_S390_PP:
810 r = put_user(vcpu->arch.sie_block->pp,
811 (u64 __user *)reg->addr);
813 case KVM_REG_S390_GBEA:
814 r = put_user(vcpu->arch.sie_block->gbea,
815 (u64 __user *)reg->addr);
824 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
825 struct kvm_one_reg *reg)
830 case KVM_REG_S390_TODPR:
831 r = get_user(vcpu->arch.sie_block->todpr,
832 (u32 __user *)reg->addr);
834 case KVM_REG_S390_EPOCHDIFF:
835 r = get_user(vcpu->arch.sie_block->epoch,
836 (u64 __user *)reg->addr);
838 case KVM_REG_S390_CPU_TIMER:
839 r = get_user(vcpu->arch.sie_block->cputm,
840 (u64 __user *)reg->addr);
842 case KVM_REG_S390_CLOCK_COMP:
843 r = get_user(vcpu->arch.sie_block->ckc,
844 (u64 __user *)reg->addr);
846 case KVM_REG_S390_PFTOKEN:
847 r = get_user(vcpu->arch.pfault_token,
848 (u64 __user *)reg->addr);
850 case KVM_REG_S390_PFCOMPARE:
851 r = get_user(vcpu->arch.pfault_compare,
852 (u64 __user *)reg->addr);
854 case KVM_REG_S390_PFSELECT:
855 r = get_user(vcpu->arch.pfault_select,
856 (u64 __user *)reg->addr);
858 case KVM_REG_S390_PP:
859 r = get_user(vcpu->arch.sie_block->pp,
860 (u64 __user *)reg->addr);
862 case KVM_REG_S390_GBEA:
863 r = get_user(vcpu->arch.sie_block->gbea,
864 (u64 __user *)reg->addr);
873 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
875 kvm_s390_vcpu_initial_reset(vcpu);
879 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
881 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
885 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
887 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
891 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
892 struct kvm_sregs *sregs)
894 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
895 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
896 restore_access_regs(vcpu->run->s.regs.acrs);
900 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
901 struct kvm_sregs *sregs)
903 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
904 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
908 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
910 if (test_fp_ctl(fpu->fpc))
912 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
913 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
914 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
915 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
919 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
921 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
922 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
926 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
930 if (!is_vcpu_stopped(vcpu))
933 vcpu->run->psw_mask = psw.mask;
934 vcpu->run->psw_addr = psw.addr;
939 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
940 struct kvm_translation *tr)
942 return -EINVAL; /* not implemented yet */
945 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
946 KVM_GUESTDBG_USE_HW_BP | \
949 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
950 struct kvm_guest_debug *dbg)
954 vcpu->guest_debug = 0;
955 kvm_s390_clear_bp_data(vcpu);
957 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
960 if (dbg->control & KVM_GUESTDBG_ENABLE) {
961 vcpu->guest_debug = dbg->control;
962 /* enforce guest PER */
963 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
965 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
966 rc = kvm_s390_import_bp_data(vcpu, dbg);
968 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
969 vcpu->arch.guestdbg.last_bp = 0;
973 vcpu->guest_debug = 0;
974 kvm_s390_clear_bp_data(vcpu);
975 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
981 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
982 struct kvm_mp_state *mp_state)
984 /* CHECK_STOP and LOAD are not supported yet */
985 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
986 KVM_MP_STATE_OPERATING;
989 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
990 struct kvm_mp_state *mp_state)
994 /* user space knows about this interface - let it control the state */
995 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
997 switch (mp_state->mp_state) {
998 case KVM_MP_STATE_STOPPED:
999 kvm_s390_vcpu_stop(vcpu);
1001 case KVM_MP_STATE_OPERATING:
1002 kvm_s390_vcpu_start(vcpu);
1004 case KVM_MP_STATE_LOAD:
1005 case KVM_MP_STATE_CHECK_STOP:
1006 /* fall through - CHECK_STOP and LOAD are not supported yet */
1014 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1016 if (!MACHINE_IS_LPAR)
1018 /* only enable for z10 and later */
1019 if (!MACHINE_HAS_EDAT1)
1021 if (!kvm->arch.use_cmma)
1026 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1028 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1031 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1034 s390_vcpu_unblock(vcpu);
1036 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1037 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1038 * This ensures that the ipte instruction for this request has
1039 * already finished. We might race against a second unmapper that
1040 * wants to set the blocking bit. Lets just retry the request loop.
1042 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1044 rc = gmap_ipte_notify(vcpu->arch.gmap,
1045 kvm_s390_get_prefix(vcpu),
1052 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1053 if (!ibs_enabled(vcpu)) {
1054 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1055 atomic_set_mask(CPUSTAT_IBS,
1056 &vcpu->arch.sie_block->cpuflags);
1061 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1062 if (ibs_enabled(vcpu)) {
1063 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1064 atomic_clear_mask(CPUSTAT_IBS,
1065 &vcpu->arch.sie_block->cpuflags);
1070 /* nothing to do, just clear the request */
1071 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1077 * kvm_arch_fault_in_page - fault-in guest page if necessary
1078 * @vcpu: The corresponding virtual cpu
1079 * @gpa: Guest physical address
1080 * @writable: Whether the page should be writable or not
1082 * Make sure that a guest page has been faulted-in on the host.
1084 * Return: Zero on success, negative error code otherwise.
1086 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1088 struct mm_struct *mm = current->mm;
1092 hva = gmap_fault(gpa, vcpu->arch.gmap);
1093 if (IS_ERR_VALUE(hva))
1095 down_read(&mm->mmap_sem);
1096 rc = get_user_pages(current, mm, hva, 1, writable, 0, NULL, NULL);
1097 up_read(&mm->mmap_sem);
1099 return rc < 0 ? rc : 0;
1102 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1103 unsigned long token)
1105 struct kvm_s390_interrupt inti;
1106 inti.parm64 = token;
1109 inti.type = KVM_S390_INT_PFAULT_INIT;
1110 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1112 inti.type = KVM_S390_INT_PFAULT_DONE;
1113 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1117 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1118 struct kvm_async_pf *work)
1120 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1121 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1124 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1125 struct kvm_async_pf *work)
1127 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1128 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1131 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1132 struct kvm_async_pf *work)
1134 /* s390 will always inject the page directly */
1137 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1140 * s390 will always inject the page directly,
1141 * but we still want check_async_completion to cleanup
1146 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1149 struct kvm_arch_async_pf arch;
1152 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1154 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1155 vcpu->arch.pfault_compare)
1157 if (psw_extint_disabled(vcpu))
1159 if (kvm_cpu_has_interrupt(vcpu))
1161 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1163 if (!vcpu->arch.gmap->pfault_enabled)
1166 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1167 hva += current->thread.gmap_addr & ~PAGE_MASK;
1168 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1171 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1175 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1180 * On s390 notifications for arriving pages will be delivered directly
1181 * to the guest but the house keeping for completed pfaults is
1182 * handled outside the worker.
1184 kvm_check_async_pf_completion(vcpu);
1186 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1191 if (test_cpu_flag(CIF_MCCK_PENDING))
1194 if (!kvm_is_ucontrol(vcpu->kvm))
1195 kvm_s390_deliver_pending_interrupts(vcpu);
1197 rc = kvm_s390_handle_requests(vcpu);
1201 if (guestdbg_enabled(vcpu)) {
1202 kvm_s390_backup_guest_per_regs(vcpu);
1203 kvm_s390_patch_guest_per_regs(vcpu);
1206 vcpu->arch.sie_block->icptcode = 0;
1207 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1208 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1209 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1214 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1218 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1219 vcpu->arch.sie_block->icptcode);
1220 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1222 if (guestdbg_enabled(vcpu))
1223 kvm_s390_restore_guest_per_regs(vcpu);
1225 if (exit_reason >= 0) {
1227 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1228 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1229 vcpu->run->s390_ucontrol.trans_exc_code =
1230 current->thread.gmap_addr;
1231 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1234 } else if (current->thread.gmap_pfault) {
1235 trace_kvm_s390_major_guest_pfault(vcpu);
1236 current->thread.gmap_pfault = 0;
1237 if (kvm_arch_setup_async_pf(vcpu)) {
1240 gpa_t gpa = current->thread.gmap_addr;
1241 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1246 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1247 trace_kvm_s390_sie_fault(vcpu);
1248 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1251 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1254 if (kvm_is_ucontrol(vcpu->kvm))
1255 /* Don't exit for host interrupts. */
1256 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1258 rc = kvm_handle_sie_intercept(vcpu);
1264 static int __vcpu_run(struct kvm_vcpu *vcpu)
1266 int rc, exit_reason;
1269 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1270 * ning the guest), so that memslots (and other stuff) are protected
1272 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1275 rc = vcpu_pre_run(vcpu);
1279 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1281 * As PF_VCPU will be used in fault handler, between
1282 * guest_enter and guest_exit should be no uaccess.
1287 exit_reason = sie64a(vcpu->arch.sie_block,
1288 vcpu->run->s.regs.gprs);
1290 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1292 rc = vcpu_post_run(vcpu, exit_reason);
1293 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1295 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1299 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1304 if (guestdbg_exit_pending(vcpu)) {
1305 kvm_s390_prepare_debug_exit(vcpu);
1309 if (vcpu->sigset_active)
1310 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1312 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1313 kvm_s390_vcpu_start(vcpu);
1314 } else if (is_vcpu_stopped(vcpu)) {
1315 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1320 switch (kvm_run->exit_reason) {
1321 case KVM_EXIT_S390_SIEIC:
1322 case KVM_EXIT_UNKNOWN:
1324 case KVM_EXIT_S390_RESET:
1325 case KVM_EXIT_S390_UCONTROL:
1326 case KVM_EXIT_S390_TSCH:
1327 case KVM_EXIT_DEBUG:
1333 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1334 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1335 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1336 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1337 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1339 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1340 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1341 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1342 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1346 rc = __vcpu_run(vcpu);
1348 if (signal_pending(current) && !rc) {
1349 kvm_run->exit_reason = KVM_EXIT_INTR;
1353 if (guestdbg_exit_pending(vcpu) && !rc) {
1354 kvm_s390_prepare_debug_exit(vcpu);
1358 if (rc == -EOPNOTSUPP) {
1359 /* intercept cannot be handled in-kernel, prepare kvm-run */
1360 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1361 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1362 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1363 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1367 if (rc == -EREMOTE) {
1368 /* intercept was handled, but userspace support is needed
1369 * kvm_run has been prepared by the handler */
1373 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1374 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1375 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1376 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1378 if (vcpu->sigset_active)
1379 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1381 vcpu->stat.exit_userspace++;
1386 * store status at address
1387 * we use have two special cases:
1388 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1389 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1391 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1393 unsigned char archmode = 1;
1398 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1399 if (write_guest_abs(vcpu, 163, &archmode, 1))
1401 gpa = SAVE_AREA_BASE;
1402 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1403 if (write_guest_real(vcpu, 163, &archmode, 1))
1405 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1407 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1408 vcpu->arch.guest_fpregs.fprs, 128);
1409 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1410 vcpu->run->s.regs.gprs, 128);
1411 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1412 &vcpu->arch.sie_block->gpsw, 16);
1413 px = kvm_s390_get_prefix(vcpu);
1414 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1416 rc |= write_guest_abs(vcpu,
1417 gpa + offsetof(struct save_area, fp_ctrl_reg),
1418 &vcpu->arch.guest_fpregs.fpc, 4);
1419 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1420 &vcpu->arch.sie_block->todpr, 4);
1421 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1422 &vcpu->arch.sie_block->cputm, 8);
1423 clkcomp = vcpu->arch.sie_block->ckc >> 8;
1424 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1426 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1427 &vcpu->run->s.regs.acrs, 64);
1428 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1429 &vcpu->arch.sie_block->gcr, 128);
1430 return rc ? -EFAULT : 0;
1433 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1436 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1437 * copying in vcpu load/put. Lets update our copies before we save
1438 * it into the save area
1440 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1441 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1442 save_access_regs(vcpu->run->s.regs.acrs);
1444 return kvm_s390_store_status_unloaded(vcpu, addr);
1447 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1449 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1450 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1451 exit_sie_sync(vcpu);
1454 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1457 struct kvm_vcpu *vcpu;
1459 kvm_for_each_vcpu(i, vcpu, kvm) {
1460 __disable_ibs_on_vcpu(vcpu);
1464 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1466 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1467 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1468 exit_sie_sync(vcpu);
1471 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1473 int i, online_vcpus, started_vcpus = 0;
1475 if (!is_vcpu_stopped(vcpu))
1478 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1479 /* Only one cpu at a time may enter/leave the STOPPED state. */
1480 spin_lock(&vcpu->kvm->arch.start_stop_lock);
1481 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1483 for (i = 0; i < online_vcpus; i++) {
1484 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1488 if (started_vcpus == 0) {
1489 /* we're the only active VCPU -> speed it up */
1490 __enable_ibs_on_vcpu(vcpu);
1491 } else if (started_vcpus == 1) {
1493 * As we are starting a second VCPU, we have to disable
1494 * the IBS facility on all VCPUs to remove potentially
1495 * oustanding ENABLE requests.
1497 __disable_ibs_on_all_vcpus(vcpu->kvm);
1500 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1502 * Another VCPU might have used IBS while we were offline.
1503 * Let's play safe and flush the VCPU at startup.
1505 vcpu->arch.sie_block->ihcpu = 0xffff;
1506 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1510 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1512 int i, online_vcpus, started_vcpus = 0;
1513 struct kvm_vcpu *started_vcpu = NULL;
1515 if (is_vcpu_stopped(vcpu))
1518 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1519 /* Only one cpu at a time may enter/leave the STOPPED state. */
1520 spin_lock(&vcpu->kvm->arch.start_stop_lock);
1521 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1523 /* Need to lock access to action_bits to avoid a SIGP race condition */
1524 spin_lock(&vcpu->arch.local_int.lock);
1525 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1527 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1528 vcpu->arch.local_int.action_bits &=
1529 ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
1530 spin_unlock(&vcpu->arch.local_int.lock);
1532 __disable_ibs_on_vcpu(vcpu);
1534 for (i = 0; i < online_vcpus; i++) {
1535 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1537 started_vcpu = vcpu->kvm->vcpus[i];
1541 if (started_vcpus == 1) {
1543 * As we only have one VCPU left, we want to enable the
1544 * IBS facility for that VCPU to speed it up.
1546 __enable_ibs_on_vcpu(started_vcpu);
1549 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1553 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1554 struct kvm_enable_cap *cap)
1562 case KVM_CAP_S390_CSS_SUPPORT:
1563 if (!vcpu->kvm->arch.css_support) {
1564 vcpu->kvm->arch.css_support = 1;
1565 trace_kvm_s390_enable_css(vcpu->kvm);
1576 long kvm_arch_vcpu_ioctl(struct file *filp,
1577 unsigned int ioctl, unsigned long arg)
1579 struct kvm_vcpu *vcpu = filp->private_data;
1580 void __user *argp = (void __user *)arg;
1585 case KVM_S390_INTERRUPT: {
1586 struct kvm_s390_interrupt s390int;
1589 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1591 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1594 case KVM_S390_STORE_STATUS:
1595 idx = srcu_read_lock(&vcpu->kvm->srcu);
1596 r = kvm_s390_vcpu_store_status(vcpu, arg);
1597 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1599 case KVM_S390_SET_INITIAL_PSW: {
1603 if (copy_from_user(&psw, argp, sizeof(psw)))
1605 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1608 case KVM_S390_INITIAL_RESET:
1609 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1611 case KVM_SET_ONE_REG:
1612 case KVM_GET_ONE_REG: {
1613 struct kvm_one_reg reg;
1615 if (copy_from_user(®, argp, sizeof(reg)))
1617 if (ioctl == KVM_SET_ONE_REG)
1618 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
1620 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
1623 #ifdef CONFIG_KVM_S390_UCONTROL
1624 case KVM_S390_UCAS_MAP: {
1625 struct kvm_s390_ucas_mapping ucasmap;
1627 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1632 if (!kvm_is_ucontrol(vcpu->kvm)) {
1637 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1638 ucasmap.vcpu_addr, ucasmap.length);
1641 case KVM_S390_UCAS_UNMAP: {
1642 struct kvm_s390_ucas_mapping ucasmap;
1644 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1649 if (!kvm_is_ucontrol(vcpu->kvm)) {
1654 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1659 case KVM_S390_VCPU_FAULT: {
1660 r = gmap_fault(arg, vcpu->arch.gmap);
1661 if (!IS_ERR_VALUE(r))
1665 case KVM_ENABLE_CAP:
1667 struct kvm_enable_cap cap;
1669 if (copy_from_user(&cap, argp, sizeof(cap)))
1671 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1680 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1682 #ifdef CONFIG_KVM_S390_UCONTROL
1683 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1684 && (kvm_is_ucontrol(vcpu->kvm))) {
1685 vmf->page = virt_to_page(vcpu->arch.sie_block);
1686 get_page(vmf->page);
1690 return VM_FAULT_SIGBUS;
1693 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
1694 struct kvm_memory_slot *dont)
1698 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1699 unsigned long npages)
1704 void kvm_arch_memslots_updated(struct kvm *kvm)
1708 /* Section: memory related */
1709 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1710 struct kvm_memory_slot *memslot,
1711 struct kvm_userspace_memory_region *mem,
1712 enum kvm_mr_change change)
1714 /* A few sanity checks. We can have memory slots which have to be
1715 located/ended at a segment boundary (1MB). The memory in userland is
1716 ok to be fragmented into various different vmas. It is okay to mmap()
1717 and munmap() stuff in this slot after doing this call at any time */
1719 if (mem->userspace_addr & 0xffffful)
1722 if (mem->memory_size & 0xffffful)
1728 void kvm_arch_commit_memory_region(struct kvm *kvm,
1729 struct kvm_userspace_memory_region *mem,
1730 const struct kvm_memory_slot *old,
1731 enum kvm_mr_change change)
1735 /* If the basics of the memslot do not change, we do not want
1736 * to update the gmap. Every update causes several unnecessary
1737 * segment translation exceptions. This is usually handled just
1738 * fine by the normal fault handler + gmap, but it will also
1739 * cause faults on the prefix page of running guest CPUs.
1741 if (old->userspace_addr == mem->userspace_addr &&
1742 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1743 old->npages * PAGE_SIZE == mem->memory_size)
1746 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1747 mem->guest_phys_addr, mem->memory_size);
1749 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1753 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1757 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1758 struct kvm_memory_slot *slot)
1762 static int __init kvm_s390_init(void)
1765 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1770 * guests can ask for up to 255+1 double words, we need a full page
1771 * to hold the maximum amount of facilities. On the other hand, we
1772 * only set facilities that are known to work in KVM.
1774 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1779 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1780 vfacilities[0] &= 0xff82fff3f4fc2000UL;
1781 vfacilities[1] &= 0x005c000000000000UL;
1785 static void __exit kvm_s390_exit(void)
1787 free_page((unsigned long) vfacilities);
1791 module_init(kvm_s390_init);
1792 module_exit(kvm_s390_exit);
1795 * Enable autoloading of the kvm module.
1796 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1797 * since x86 takes a different approach.
1799 #include <linux/miscdevice.h>
1800 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1801 MODULE_ALIAS("devname:kvm");