2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
32 #include <asm/pgtable.h>
34 #include <asm/switch_to.h>
40 #define KMSG_COMPONENT "kvm-s390"
42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
44 #define CREATE_TRACE_POINTS
46 #include "trace-s390.h"
48 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
55 struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
57 { "exit_null", VCPU_STAT(exit_null) },
58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
90 { "instruction_essa", VCPU_STAT(instruction_essa) },
91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
110 { "diagnose_10", VCPU_STAT(diagnose_10) },
111 { "diagnose_44", VCPU_STAT(diagnose_44) },
112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
119 /* upper facilities limit for kvm */
120 unsigned long kvm_s390_fac_list_mask[] = {
121 0xffe6fffbfcfdfc40UL,
122 0x005e800000000000UL,
125 unsigned long kvm_s390_fac_list_mask_size(void)
127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
131 static struct gmap_notifier gmap_notifier;
132 debug_info_t *kvm_s390_dbf;
134 /* Section: not file related */
135 int kvm_arch_hardware_enable(void)
137 /* every s390 is virtualization enabled ;-) */
141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
153 struct kvm_vcpu *vcpu;
155 unsigned long long *delta = v;
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
166 static struct notifier_block kvm_clock_notifier = {
167 .notifier_call = kvm_clock_sync,
170 int kvm_arch_hardware_setup(void)
172 gmap_notifier.notifier_call = kvm_gmap_notifier;
173 gmap_register_ipte_notifier(&gmap_notifier);
174 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 &kvm_clock_notifier);
179 void kvm_arch_hardware_unsetup(void)
181 gmap_unregister_ipte_notifier(&gmap_notifier);
182 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 &kvm_clock_notifier);
186 int kvm_arch_init(void *opaque)
188 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
192 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 debug_unregister(kvm_s390_dbf);
197 /* Register floating interrupt controller interface. */
198 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
201 void kvm_arch_exit(void)
203 debug_unregister(kvm_s390_dbf);
206 /* Section: device related */
207 long kvm_arch_dev_ioctl(struct file *filp,
208 unsigned int ioctl, unsigned long arg)
210 if (ioctl == KVM_S390_ENABLE_SIE)
211 return s390_enable_sie();
215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
220 case KVM_CAP_S390_PSW:
221 case KVM_CAP_S390_GMAP:
222 case KVM_CAP_SYNC_MMU:
223 #ifdef CONFIG_KVM_S390_UCONTROL
224 case KVM_CAP_S390_UCONTROL:
226 case KVM_CAP_ASYNC_PF:
227 case KVM_CAP_SYNC_REGS:
228 case KVM_CAP_ONE_REG:
229 case KVM_CAP_ENABLE_CAP:
230 case KVM_CAP_S390_CSS_SUPPORT:
231 case KVM_CAP_IOEVENTFD:
232 case KVM_CAP_DEVICE_CTRL:
233 case KVM_CAP_ENABLE_CAP_VM:
234 case KVM_CAP_S390_IRQCHIP:
235 case KVM_CAP_VM_ATTRIBUTES:
236 case KVM_CAP_MP_STATE:
237 case KVM_CAP_S390_INJECT_IRQ:
238 case KVM_CAP_S390_USER_SIGP:
239 case KVM_CAP_S390_USER_STSI:
240 case KVM_CAP_S390_SKEYS:
241 case KVM_CAP_S390_IRQ_STATE:
244 case KVM_CAP_S390_MEM_OP:
247 case KVM_CAP_NR_VCPUS:
248 case KVM_CAP_MAX_VCPUS:
251 case KVM_CAP_NR_MEMSLOTS:
252 r = KVM_USER_MEM_SLOTS;
254 case KVM_CAP_S390_COW:
255 r = MACHINE_HAS_ESOP;
257 case KVM_CAP_S390_VECTOR_REGISTERS:
266 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
267 struct kvm_memory_slot *memslot)
269 gfn_t cur_gfn, last_gfn;
270 unsigned long address;
271 struct gmap *gmap = kvm->arch.gmap;
273 down_read(&gmap->mm->mmap_sem);
274 /* Loop over all guest pages */
275 last_gfn = memslot->base_gfn + memslot->npages;
276 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
277 address = gfn_to_hva_memslot(memslot, cur_gfn);
279 if (gmap_test_and_clear_dirty(address, gmap))
280 mark_page_dirty(kvm, cur_gfn);
282 up_read(&gmap->mm->mmap_sem);
285 /* Section: vm related */
286 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
289 * Get (and clear) the dirty memory log for a memory slot.
291 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
292 struct kvm_dirty_log *log)
296 struct kvm_memslots *slots;
297 struct kvm_memory_slot *memslot;
300 mutex_lock(&kvm->slots_lock);
303 if (log->slot >= KVM_USER_MEM_SLOTS)
306 slots = kvm_memslots(kvm);
307 memslot = id_to_memslot(slots, log->slot);
309 if (!memslot->dirty_bitmap)
312 kvm_s390_sync_dirty_log(kvm, memslot);
313 r = kvm_get_dirty_log(kvm, log, &is_dirty);
317 /* Clear the dirty log */
319 n = kvm_dirty_bitmap_bytes(memslot);
320 memset(memslot->dirty_bitmap, 0, n);
324 mutex_unlock(&kvm->slots_lock);
328 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
336 case KVM_CAP_S390_IRQCHIP:
337 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
338 kvm->arch.use_irqchip = 1;
341 case KVM_CAP_S390_USER_SIGP:
342 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
343 kvm->arch.user_sigp = 1;
346 case KVM_CAP_S390_VECTOR_REGISTERS:
347 mutex_lock(&kvm->lock);
348 if (atomic_read(&kvm->online_vcpus)) {
350 } else if (MACHINE_HAS_VX) {
351 set_kvm_facility(kvm->arch.model.fac->mask, 129);
352 set_kvm_facility(kvm->arch.model.fac->list, 129);
356 mutex_unlock(&kvm->lock);
357 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
358 r ? "(not available)" : "(success)");
360 case KVM_CAP_S390_USER_STSI:
361 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
362 kvm->arch.user_stsi = 1;
372 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
376 switch (attr->attr) {
377 case KVM_S390_VM_MEM_LIMIT_SIZE:
379 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
380 kvm->arch.gmap->asce_end);
381 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
391 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
395 switch (attr->attr) {
396 case KVM_S390_VM_MEM_ENABLE_CMMA:
397 /* enable CMMA only for z10 and later (EDAT_1) */
399 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
403 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
404 mutex_lock(&kvm->lock);
405 if (atomic_read(&kvm->online_vcpus) == 0) {
406 kvm->arch.use_cmma = 1;
409 mutex_unlock(&kvm->lock);
411 case KVM_S390_VM_MEM_CLR_CMMA:
413 if (!kvm->arch.use_cmma)
416 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
417 mutex_lock(&kvm->lock);
418 idx = srcu_read_lock(&kvm->srcu);
419 s390_reset_cmma(kvm->arch.gmap->mm);
420 srcu_read_unlock(&kvm->srcu, idx);
421 mutex_unlock(&kvm->lock);
424 case KVM_S390_VM_MEM_LIMIT_SIZE: {
425 unsigned long new_limit;
427 if (kvm_is_ucontrol(kvm))
430 if (get_user(new_limit, (u64 __user *)attr->addr))
433 if (new_limit > kvm->arch.gmap->asce_end)
437 mutex_lock(&kvm->lock);
438 if (atomic_read(&kvm->online_vcpus) == 0) {
439 /* gmap_alloc will round the limit up */
440 struct gmap *new = gmap_alloc(current->mm, new_limit);
445 gmap_free(kvm->arch.gmap);
447 kvm->arch.gmap = new;
451 mutex_unlock(&kvm->lock);
452 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
462 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
464 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
466 struct kvm_vcpu *vcpu;
469 if (!test_kvm_facility(kvm, 76))
472 mutex_lock(&kvm->lock);
473 switch (attr->attr) {
474 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
476 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
477 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
478 kvm->arch.crypto.aes_kw = 1;
479 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
481 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
483 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
484 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
485 kvm->arch.crypto.dea_kw = 1;
486 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
488 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
489 kvm->arch.crypto.aes_kw = 0;
490 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
491 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
492 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
494 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
495 kvm->arch.crypto.dea_kw = 0;
496 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
497 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
498 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
501 mutex_unlock(&kvm->lock);
505 kvm_for_each_vcpu(i, vcpu, kvm) {
506 kvm_s390_vcpu_crypto_setup(vcpu);
509 mutex_unlock(&kvm->lock);
513 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
517 if (copy_from_user(>od_high, (void __user *)attr->addr,
523 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
528 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
532 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
535 kvm_s390_set_tod_clock(kvm, gtod);
536 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
540 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
547 switch (attr->attr) {
548 case KVM_S390_VM_TOD_HIGH:
549 ret = kvm_s390_set_tod_high(kvm, attr);
551 case KVM_S390_VM_TOD_LOW:
552 ret = kvm_s390_set_tod_low(kvm, attr);
561 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
565 if (copy_to_user((void __user *)attr->addr, >od_high,
568 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
573 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
577 gtod = kvm_s390_get_tod_clock_fast(kvm);
578 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
580 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
585 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
592 switch (attr->attr) {
593 case KVM_S390_VM_TOD_HIGH:
594 ret = kvm_s390_get_tod_high(kvm, attr);
596 case KVM_S390_VM_TOD_LOW:
597 ret = kvm_s390_get_tod_low(kvm, attr);
606 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
608 struct kvm_s390_vm_cpu_processor *proc;
611 mutex_lock(&kvm->lock);
612 if (atomic_read(&kvm->online_vcpus)) {
616 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
621 if (!copy_from_user(proc, (void __user *)attr->addr,
623 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
624 sizeof(struct cpuid));
625 kvm->arch.model.ibc = proc->ibc;
626 memcpy(kvm->arch.model.fac->list, proc->fac_list,
627 S390_ARCH_FAC_LIST_SIZE_BYTE);
632 mutex_unlock(&kvm->lock);
636 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
640 switch (attr->attr) {
641 case KVM_S390_VM_CPU_PROCESSOR:
642 ret = kvm_s390_set_processor(kvm, attr);
648 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
650 struct kvm_s390_vm_cpu_processor *proc;
653 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
658 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
659 proc->ibc = kvm->arch.model.ibc;
660 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
661 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
668 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
670 struct kvm_s390_vm_cpu_machine *mach;
673 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
678 get_cpu_id((struct cpuid *) &mach->cpuid);
679 mach->ibc = sclp.ibc;
680 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
681 S390_ARCH_FAC_LIST_SIZE_BYTE);
682 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
683 S390_ARCH_FAC_LIST_SIZE_BYTE);
684 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
691 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
695 switch (attr->attr) {
696 case KVM_S390_VM_CPU_PROCESSOR:
697 ret = kvm_s390_get_processor(kvm, attr);
699 case KVM_S390_VM_CPU_MACHINE:
700 ret = kvm_s390_get_machine(kvm, attr);
706 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
710 switch (attr->group) {
711 case KVM_S390_VM_MEM_CTRL:
712 ret = kvm_s390_set_mem_control(kvm, attr);
714 case KVM_S390_VM_TOD:
715 ret = kvm_s390_set_tod(kvm, attr);
717 case KVM_S390_VM_CPU_MODEL:
718 ret = kvm_s390_set_cpu_model(kvm, attr);
720 case KVM_S390_VM_CRYPTO:
721 ret = kvm_s390_vm_set_crypto(kvm, attr);
731 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
735 switch (attr->group) {
736 case KVM_S390_VM_MEM_CTRL:
737 ret = kvm_s390_get_mem_control(kvm, attr);
739 case KVM_S390_VM_TOD:
740 ret = kvm_s390_get_tod(kvm, attr);
742 case KVM_S390_VM_CPU_MODEL:
743 ret = kvm_s390_get_cpu_model(kvm, attr);
753 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
757 switch (attr->group) {
758 case KVM_S390_VM_MEM_CTRL:
759 switch (attr->attr) {
760 case KVM_S390_VM_MEM_ENABLE_CMMA:
761 case KVM_S390_VM_MEM_CLR_CMMA:
762 case KVM_S390_VM_MEM_LIMIT_SIZE:
770 case KVM_S390_VM_TOD:
771 switch (attr->attr) {
772 case KVM_S390_VM_TOD_LOW:
773 case KVM_S390_VM_TOD_HIGH:
781 case KVM_S390_VM_CPU_MODEL:
782 switch (attr->attr) {
783 case KVM_S390_VM_CPU_PROCESSOR:
784 case KVM_S390_VM_CPU_MACHINE:
792 case KVM_S390_VM_CRYPTO:
793 switch (attr->attr) {
794 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
795 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
796 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
797 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
813 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
817 unsigned long curkey;
820 if (args->flags != 0)
823 /* Is this guest using storage keys? */
824 if (!mm_use_skey(current->mm))
825 return KVM_S390_GET_SKEYS_NONE;
827 /* Enforce sane limit on memory allocation */
828 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
831 keys = kmalloc_array(args->count, sizeof(uint8_t),
832 GFP_KERNEL | __GFP_NOWARN);
834 keys = vmalloc(sizeof(uint8_t) * args->count);
838 for (i = 0; i < args->count; i++) {
839 hva = gfn_to_hva(kvm, args->start_gfn + i);
840 if (kvm_is_error_hva(hva)) {
845 curkey = get_guest_storage_key(current->mm, hva);
846 if (IS_ERR_VALUE(curkey)) {
853 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
854 sizeof(uint8_t) * args->count);
862 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
868 if (args->flags != 0)
871 /* Enforce sane limit on memory allocation */
872 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
875 keys = kmalloc_array(args->count, sizeof(uint8_t),
876 GFP_KERNEL | __GFP_NOWARN);
878 keys = vmalloc(sizeof(uint8_t) * args->count);
882 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
883 sizeof(uint8_t) * args->count);
889 /* Enable storage key handling for the guest */
890 r = s390_enable_skey();
894 for (i = 0; i < args->count; i++) {
895 hva = gfn_to_hva(kvm, args->start_gfn + i);
896 if (kvm_is_error_hva(hva)) {
901 /* Lowest order bit is reserved */
902 if (keys[i] & 0x01) {
907 r = set_guest_storage_key(current->mm, hva,
908 (unsigned long)keys[i], 0);
917 long kvm_arch_vm_ioctl(struct file *filp,
918 unsigned int ioctl, unsigned long arg)
920 struct kvm *kvm = filp->private_data;
921 void __user *argp = (void __user *)arg;
922 struct kvm_device_attr attr;
926 case KVM_S390_INTERRUPT: {
927 struct kvm_s390_interrupt s390int;
930 if (copy_from_user(&s390int, argp, sizeof(s390int)))
932 r = kvm_s390_inject_vm(kvm, &s390int);
935 case KVM_ENABLE_CAP: {
936 struct kvm_enable_cap cap;
938 if (copy_from_user(&cap, argp, sizeof(cap)))
940 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
943 case KVM_CREATE_IRQCHIP: {
944 struct kvm_irq_routing_entry routing;
947 if (kvm->arch.use_irqchip) {
948 /* Set up dummy routing. */
949 memset(&routing, 0, sizeof(routing));
950 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
954 case KVM_SET_DEVICE_ATTR: {
956 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
958 r = kvm_s390_vm_set_attr(kvm, &attr);
961 case KVM_GET_DEVICE_ATTR: {
963 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
965 r = kvm_s390_vm_get_attr(kvm, &attr);
968 case KVM_HAS_DEVICE_ATTR: {
970 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
972 r = kvm_s390_vm_has_attr(kvm, &attr);
975 case KVM_S390_GET_SKEYS: {
976 struct kvm_s390_skeys args;
979 if (copy_from_user(&args, argp,
980 sizeof(struct kvm_s390_skeys)))
982 r = kvm_s390_get_skeys(kvm, &args);
985 case KVM_S390_SET_SKEYS: {
986 struct kvm_s390_skeys args;
989 if (copy_from_user(&args, argp,
990 sizeof(struct kvm_s390_skeys)))
992 r = kvm_s390_set_skeys(kvm, &args);
1002 static int kvm_s390_query_ap_config(u8 *config)
1004 u32 fcn_code = 0x04000000UL;
1007 memset(config, 0, 128);
1011 ".long 0xb2af0000\n" /* PQAP(QCI) */
1017 : "r" (fcn_code), "r" (config)
1018 : "cc", "0", "2", "memory"
1024 static int kvm_s390_apxa_installed(void)
1029 if (test_facility(2) && test_facility(12)) {
1030 cc = kvm_s390_query_ap_config(config);
1033 pr_err("PQAP(QCI) failed with cc=%d", cc);
1035 return config[0] & 0x40;
1041 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1043 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1045 if (kvm_s390_apxa_installed())
1046 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1048 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1051 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1054 cpu_id->version = 0xff;
1057 static int kvm_s390_crypto_init(struct kvm *kvm)
1059 if (!test_kvm_facility(kvm, 76))
1062 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1063 GFP_KERNEL | GFP_DMA);
1064 if (!kvm->arch.crypto.crycb)
1067 kvm_s390_set_crycb_format(kvm);
1069 /* Enable AES/DEA protected key functions by default */
1070 kvm->arch.crypto.aes_kw = 1;
1071 kvm->arch.crypto.dea_kw = 1;
1072 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1073 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1074 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1075 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1080 static void sca_dispose(struct kvm *kvm)
1082 if (kvm->arch.use_esca)
1083 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1085 free_page((unsigned long)(kvm->arch.sca));
1086 kvm->arch.sca = NULL;
1089 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1092 char debug_name[16];
1093 static unsigned long sca_offset;
1096 #ifdef CONFIG_KVM_S390_UCONTROL
1097 if (type & ~KVM_VM_S390_UCONTROL)
1099 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1106 rc = s390_enable_sie();
1112 kvm->arch.use_esca = 0; /* start with basic SCA */
1113 rwlock_init(&kvm->arch.sca_lock);
1114 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
1117 spin_lock(&kvm_lock);
1119 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1121 kvm->arch.sca = (struct bsca_block *)
1122 ((char *) kvm->arch.sca + sca_offset);
1123 spin_unlock(&kvm_lock);
1125 sprintf(debug_name, "kvm-%u", current->pid);
1127 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1132 * The architectural maximum amount of facilities is 16 kbit. To store
1133 * this amount, 2 kbyte of memory is required. Thus we need a full
1134 * page to hold the guest facility list (arch.model.fac->list) and the
1135 * facility mask (arch.model.fac->mask). Its address size has to be
1136 * 31 bits and word aligned.
1138 kvm->arch.model.fac =
1139 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1140 if (!kvm->arch.model.fac)
1143 /* Populate the facility mask initially. */
1144 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1145 S390_ARCH_FAC_LIST_SIZE_BYTE);
1146 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1147 if (i < kvm_s390_fac_list_mask_size())
1148 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1150 kvm->arch.model.fac->mask[i] = 0UL;
1153 /* Populate the facility list initially. */
1154 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1155 S390_ARCH_FAC_LIST_SIZE_BYTE);
1157 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1158 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1160 if (kvm_s390_crypto_init(kvm) < 0)
1163 spin_lock_init(&kvm->arch.float_int.lock);
1164 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1165 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1166 init_waitqueue_head(&kvm->arch.ipte_wq);
1167 mutex_init(&kvm->arch.ipte_mutex);
1169 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1170 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1172 if (type & KVM_VM_S390_UCONTROL) {
1173 kvm->arch.gmap = NULL;
1175 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1176 if (!kvm->arch.gmap)
1178 kvm->arch.gmap->private = kvm;
1179 kvm->arch.gmap->pfault_enabled = 0;
1182 kvm->arch.css_support = 0;
1183 kvm->arch.use_irqchip = 0;
1184 kvm->arch.epoch = 0;
1186 spin_lock_init(&kvm->arch.start_stop_lock);
1187 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1191 kfree(kvm->arch.crypto.crycb);
1192 free_page((unsigned long)kvm->arch.model.fac);
1193 debug_unregister(kvm->arch.dbf);
1195 KVM_EVENT(3, "creation of vm failed: %d", rc);
1199 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1201 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1202 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1203 kvm_s390_clear_local_irqs(vcpu);
1204 kvm_clear_async_pf_completion_queue(vcpu);
1205 if (!kvm_is_ucontrol(vcpu->kvm))
1209 if (kvm_is_ucontrol(vcpu->kvm))
1210 gmap_free(vcpu->arch.gmap);
1212 if (vcpu->kvm->arch.use_cmma)
1213 kvm_s390_vcpu_unsetup_cmma(vcpu);
1214 free_page((unsigned long)(vcpu->arch.sie_block));
1216 kvm_vcpu_uninit(vcpu);
1217 kmem_cache_free(kvm_vcpu_cache, vcpu);
1220 static void kvm_free_vcpus(struct kvm *kvm)
1223 struct kvm_vcpu *vcpu;
1225 kvm_for_each_vcpu(i, vcpu, kvm)
1226 kvm_arch_vcpu_destroy(vcpu);
1228 mutex_lock(&kvm->lock);
1229 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1230 kvm->vcpus[i] = NULL;
1232 atomic_set(&kvm->online_vcpus, 0);
1233 mutex_unlock(&kvm->lock);
1236 void kvm_arch_destroy_vm(struct kvm *kvm)
1238 kvm_free_vcpus(kvm);
1239 free_page((unsigned long)kvm->arch.model.fac);
1241 debug_unregister(kvm->arch.dbf);
1242 kfree(kvm->arch.crypto.crycb);
1243 if (!kvm_is_ucontrol(kvm))
1244 gmap_free(kvm->arch.gmap);
1245 kvm_s390_destroy_adapters(kvm);
1246 kvm_s390_clear_float_irqs(kvm);
1247 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1250 /* Section: vcpu related */
1251 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1253 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1254 if (!vcpu->arch.gmap)
1256 vcpu->arch.gmap->private = vcpu->kvm;
1261 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1263 read_lock(&vcpu->kvm->arch.sca_lock);
1264 if (vcpu->kvm->arch.use_esca) {
1265 struct esca_block *sca = vcpu->kvm->arch.sca;
1267 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1268 if (sca->cpu[vcpu->vcpu_id].sda == (__u64) vcpu->arch.sie_block)
1269 sca->cpu[vcpu->vcpu_id].sda = 0;
1271 struct bsca_block *sca = vcpu->kvm->arch.sca;
1273 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1274 if (sca->cpu[vcpu->vcpu_id].sda == (__u64) vcpu->arch.sie_block)
1275 sca->cpu[vcpu->vcpu_id].sda = 0;
1277 read_unlock(&vcpu->kvm->arch.sca_lock);
1280 static void sca_add_vcpu(struct kvm_vcpu *vcpu, struct kvm *kvm,
1283 read_lock(&kvm->arch.sca_lock);
1284 if (kvm->arch.use_esca) {
1285 struct esca_block *sca = kvm->arch.sca;
1287 if (!sca->cpu[id].sda)
1288 sca->cpu[id].sda = (__u64) vcpu->arch.sie_block;
1289 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1290 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1291 set_bit_inv(id, (unsigned long *) sca->mcn);
1293 struct bsca_block *sca = kvm->arch.sca;
1295 if (!sca->cpu[id].sda)
1296 sca->cpu[id].sda = (__u64) vcpu->arch.sie_block;
1297 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1298 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1299 set_bit_inv(id, (unsigned long *) &sca->mcn);
1301 read_unlock(&kvm->arch.sca_lock);
1304 /* Basic SCA to Extended SCA data copy routines */
1305 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1308 d->sigp_ctrl.c = s->sigp_ctrl.c;
1309 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1312 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1316 d->ipte_control = s->ipte_control;
1318 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1319 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1322 static int sca_switch_to_extended(struct kvm *kvm)
1324 struct bsca_block *old_sca = kvm->arch.sca;
1325 struct esca_block *new_sca;
1326 struct kvm_vcpu *vcpu;
1327 unsigned int vcpu_idx;
1330 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1334 scaoh = (u32)((u64)(new_sca) >> 32);
1335 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1337 kvm_s390_vcpu_block_all(kvm);
1338 write_lock(&kvm->arch.sca_lock);
1340 sca_copy_b_to_e(new_sca, old_sca);
1342 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1343 vcpu->arch.sie_block->scaoh = scaoh;
1344 vcpu->arch.sie_block->scaol = scaol;
1345 vcpu->arch.sie_block->ecb2 |= 0x04U;
1347 kvm->arch.sca = new_sca;
1348 kvm->arch.use_esca = 1;
1350 write_unlock(&kvm->arch.sca_lock);
1351 kvm_s390_vcpu_unblock_all(kvm);
1353 free_page((unsigned long)old_sca);
1355 VM_EVENT(kvm, 2, "Switched to ESCA (%p -> %p)", old_sca, kvm->arch.sca);
1359 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1363 if (id < KVM_S390_BSCA_CPU_SLOTS)
1368 mutex_lock(&kvm->lock);
1369 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1370 mutex_unlock(&kvm->lock);
1372 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1375 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1377 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1378 kvm_clear_async_pf_completion_queue(vcpu);
1379 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1385 if (test_kvm_facility(vcpu->kvm, 129))
1386 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1388 if (kvm_is_ucontrol(vcpu->kvm))
1389 return __kvm_ucontrol_vcpu_init(vcpu);
1395 * Backs up the current FP/VX register save area on a particular
1396 * destination. Used to switch between different register save
1399 static inline void save_fpu_to(struct fpu *dst)
1401 dst->fpc = current->thread.fpu.fpc;
1402 dst->regs = current->thread.fpu.regs;
1406 * Switches the FP/VX register save area from which to lazy
1407 * restore register contents.
1409 static inline void load_fpu_from(struct fpu *from)
1411 current->thread.fpu.fpc = from->fpc;
1412 current->thread.fpu.regs = from->regs;
1415 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1417 /* Save host register state */
1419 save_fpu_to(&vcpu->arch.host_fpregs);
1421 if (test_kvm_facility(vcpu->kvm, 129)) {
1422 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1424 * Use the register save area in the SIE-control block
1425 * for register restore and save in kvm_arch_vcpu_put()
1427 current->thread.fpu.vxrs =
1428 (__vector128 *)&vcpu->run->s.regs.vrs;
1430 load_fpu_from(&vcpu->arch.guest_fpregs);
1432 if (test_fp_ctl(current->thread.fpu.fpc))
1433 /* User space provided an invalid FPC, let's clear it */
1434 current->thread.fpu.fpc = 0;
1436 save_access_regs(vcpu->arch.host_acrs);
1437 restore_access_regs(vcpu->run->s.regs.acrs);
1438 gmap_enable(vcpu->arch.gmap);
1439 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1442 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1444 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1445 gmap_disable(vcpu->arch.gmap);
1449 if (test_kvm_facility(vcpu->kvm, 129))
1451 * kvm_arch_vcpu_load() set up the register save area to
1452 * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1453 * are already saved. Only the floating-point control must be
1456 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1458 save_fpu_to(&vcpu->arch.guest_fpregs);
1459 load_fpu_from(&vcpu->arch.host_fpregs);
1461 save_access_regs(vcpu->run->s.regs.acrs);
1462 restore_access_regs(vcpu->arch.host_acrs);
1465 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1467 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1468 vcpu->arch.sie_block->gpsw.mask = 0UL;
1469 vcpu->arch.sie_block->gpsw.addr = 0UL;
1470 kvm_s390_set_prefix(vcpu, 0);
1471 vcpu->arch.sie_block->cputm = 0UL;
1472 vcpu->arch.sie_block->ckc = 0UL;
1473 vcpu->arch.sie_block->todpr = 0;
1474 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1475 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1476 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1477 vcpu->arch.guest_fpregs.fpc = 0;
1478 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1479 vcpu->arch.sie_block->gbea = 1;
1480 vcpu->arch.sie_block->pp = 0;
1481 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1482 kvm_clear_async_pf_completion_queue(vcpu);
1483 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1484 kvm_s390_vcpu_stop(vcpu);
1485 kvm_s390_clear_local_irqs(vcpu);
1488 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1490 mutex_lock(&vcpu->kvm->lock);
1492 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1494 mutex_unlock(&vcpu->kvm->lock);
1495 if (!kvm_is_ucontrol(vcpu->kvm))
1496 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1499 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1501 if (!test_kvm_facility(vcpu->kvm, 76))
1504 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1506 if (vcpu->kvm->arch.crypto.aes_kw)
1507 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1508 if (vcpu->kvm->arch.crypto.dea_kw)
1509 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1511 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1514 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1516 free_page(vcpu->arch.sie_block->cbrlo);
1517 vcpu->arch.sie_block->cbrlo = 0;
1520 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1522 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1523 if (!vcpu->arch.sie_block->cbrlo)
1526 vcpu->arch.sie_block->ecb2 |= 0x80;
1527 vcpu->arch.sie_block->ecb2 &= ~0x08;
1531 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1533 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1535 vcpu->arch.cpu_id = model->cpu_id;
1536 vcpu->arch.sie_block->ibc = model->ibc;
1537 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1540 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1544 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1548 if (test_kvm_facility(vcpu->kvm, 78))
1549 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1550 else if (test_kvm_facility(vcpu->kvm, 8))
1551 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1553 kvm_s390_vcpu_setup_model(vcpu);
1555 vcpu->arch.sie_block->ecb = 6;
1556 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1557 vcpu->arch.sie_block->ecb |= 0x10;
1559 vcpu->arch.sie_block->ecb2 = 8;
1560 if (vcpu->kvm->arch.use_esca)
1561 vcpu->arch.sie_block->ecb2 |= 4;
1562 vcpu->arch.sie_block->eca = 0xC1002000U;
1564 vcpu->arch.sie_block->eca |= 1;
1565 if (sclp.has_sigpif)
1566 vcpu->arch.sie_block->eca |= 0x10000000U;
1567 if (test_kvm_facility(vcpu->kvm, 129)) {
1568 vcpu->arch.sie_block->eca |= 0x00020000;
1569 vcpu->arch.sie_block->ecd |= 0x20000000;
1571 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1573 if (vcpu->kvm->arch.use_cmma) {
1574 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1578 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1579 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1581 kvm_s390_vcpu_crypto_setup(vcpu);
1586 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1589 struct kvm_vcpu *vcpu;
1590 struct sie_page *sie_page;
1593 if (!sca_can_add_vcpu(kvm, id))
1598 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1602 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1606 vcpu->arch.sie_block = &sie_page->sie_block;
1607 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1609 vcpu->arch.sie_block->icpua = id;
1610 if (!kvm_is_ucontrol(kvm)) {
1611 if (!kvm->arch.sca) {
1615 sca_add_vcpu(vcpu, kvm, id);
1618 spin_lock_init(&vcpu->arch.local_int.lock);
1619 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1620 vcpu->arch.local_int.wq = &vcpu->wq;
1621 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1624 * Allocate a save area for floating-point registers. If the vector
1625 * extension is available, register contents are saved in the SIE
1626 * control block. The allocated save area is still required in
1627 * particular places, for example, in kvm_s390_vcpu_store_status().
1629 vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1631 if (!vcpu->arch.guest_fpregs.fprs) {
1633 goto out_free_sie_block;
1636 rc = kvm_vcpu_init(vcpu, kvm, id);
1638 goto out_free_sie_block;
1639 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1640 vcpu->arch.sie_block);
1641 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1645 free_page((unsigned long)(vcpu->arch.sie_block));
1647 kmem_cache_free(kvm_vcpu_cache, vcpu);
1652 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1654 return kvm_s390_vcpu_has_irq(vcpu, 0);
1657 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1659 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1663 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1665 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1668 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1670 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1674 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1676 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1680 * Kick a guest cpu out of SIE and wait until SIE is not running.
1681 * If the CPU is not running (e.g. waiting as idle) the function will
1682 * return immediately. */
1683 void exit_sie(struct kvm_vcpu *vcpu)
1685 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1686 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1690 /* Kick a guest cpu out of SIE to process a request synchronously */
1691 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1693 kvm_make_request(req, vcpu);
1694 kvm_s390_vcpu_request(vcpu);
1697 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1700 struct kvm *kvm = gmap->private;
1701 struct kvm_vcpu *vcpu;
1703 kvm_for_each_vcpu(i, vcpu, kvm) {
1704 /* match against both prefix pages */
1705 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1706 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1707 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1712 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1714 /* kvm common code refers to this, but never calls it */
1719 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1720 struct kvm_one_reg *reg)
1725 case KVM_REG_S390_TODPR:
1726 r = put_user(vcpu->arch.sie_block->todpr,
1727 (u32 __user *)reg->addr);
1729 case KVM_REG_S390_EPOCHDIFF:
1730 r = put_user(vcpu->arch.sie_block->epoch,
1731 (u64 __user *)reg->addr);
1733 case KVM_REG_S390_CPU_TIMER:
1734 r = put_user(vcpu->arch.sie_block->cputm,
1735 (u64 __user *)reg->addr);
1737 case KVM_REG_S390_CLOCK_COMP:
1738 r = put_user(vcpu->arch.sie_block->ckc,
1739 (u64 __user *)reg->addr);
1741 case KVM_REG_S390_PFTOKEN:
1742 r = put_user(vcpu->arch.pfault_token,
1743 (u64 __user *)reg->addr);
1745 case KVM_REG_S390_PFCOMPARE:
1746 r = put_user(vcpu->arch.pfault_compare,
1747 (u64 __user *)reg->addr);
1749 case KVM_REG_S390_PFSELECT:
1750 r = put_user(vcpu->arch.pfault_select,
1751 (u64 __user *)reg->addr);
1753 case KVM_REG_S390_PP:
1754 r = put_user(vcpu->arch.sie_block->pp,
1755 (u64 __user *)reg->addr);
1757 case KVM_REG_S390_GBEA:
1758 r = put_user(vcpu->arch.sie_block->gbea,
1759 (u64 __user *)reg->addr);
1768 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1769 struct kvm_one_reg *reg)
1774 case KVM_REG_S390_TODPR:
1775 r = get_user(vcpu->arch.sie_block->todpr,
1776 (u32 __user *)reg->addr);
1778 case KVM_REG_S390_EPOCHDIFF:
1779 r = get_user(vcpu->arch.sie_block->epoch,
1780 (u64 __user *)reg->addr);
1782 case KVM_REG_S390_CPU_TIMER:
1783 r = get_user(vcpu->arch.sie_block->cputm,
1784 (u64 __user *)reg->addr);
1786 case KVM_REG_S390_CLOCK_COMP:
1787 r = get_user(vcpu->arch.sie_block->ckc,
1788 (u64 __user *)reg->addr);
1790 case KVM_REG_S390_PFTOKEN:
1791 r = get_user(vcpu->arch.pfault_token,
1792 (u64 __user *)reg->addr);
1793 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1794 kvm_clear_async_pf_completion_queue(vcpu);
1796 case KVM_REG_S390_PFCOMPARE:
1797 r = get_user(vcpu->arch.pfault_compare,
1798 (u64 __user *)reg->addr);
1800 case KVM_REG_S390_PFSELECT:
1801 r = get_user(vcpu->arch.pfault_select,
1802 (u64 __user *)reg->addr);
1804 case KVM_REG_S390_PP:
1805 r = get_user(vcpu->arch.sie_block->pp,
1806 (u64 __user *)reg->addr);
1808 case KVM_REG_S390_GBEA:
1809 r = get_user(vcpu->arch.sie_block->gbea,
1810 (u64 __user *)reg->addr);
1819 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1821 kvm_s390_vcpu_initial_reset(vcpu);
1825 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1827 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
1831 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1833 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1837 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1838 struct kvm_sregs *sregs)
1840 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1841 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1842 restore_access_regs(vcpu->run->s.regs.acrs);
1846 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1847 struct kvm_sregs *sregs)
1849 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1850 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1854 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1856 if (test_fp_ctl(fpu->fpc))
1858 memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1859 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1861 load_fpu_from(&vcpu->arch.guest_fpregs);
1865 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1867 memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1868 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1872 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1876 if (!is_vcpu_stopped(vcpu))
1879 vcpu->run->psw_mask = psw.mask;
1880 vcpu->run->psw_addr = psw.addr;
1885 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1886 struct kvm_translation *tr)
1888 return -EINVAL; /* not implemented yet */
1891 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1892 KVM_GUESTDBG_USE_HW_BP | \
1893 KVM_GUESTDBG_ENABLE)
1895 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1896 struct kvm_guest_debug *dbg)
1900 vcpu->guest_debug = 0;
1901 kvm_s390_clear_bp_data(vcpu);
1903 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1906 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1907 vcpu->guest_debug = dbg->control;
1908 /* enforce guest PER */
1909 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1911 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1912 rc = kvm_s390_import_bp_data(vcpu, dbg);
1914 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1915 vcpu->arch.guestdbg.last_bp = 0;
1919 vcpu->guest_debug = 0;
1920 kvm_s390_clear_bp_data(vcpu);
1921 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1927 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1928 struct kvm_mp_state *mp_state)
1930 /* CHECK_STOP and LOAD are not supported yet */
1931 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1932 KVM_MP_STATE_OPERATING;
1935 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1936 struct kvm_mp_state *mp_state)
1940 /* user space knows about this interface - let it control the state */
1941 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1943 switch (mp_state->mp_state) {
1944 case KVM_MP_STATE_STOPPED:
1945 kvm_s390_vcpu_stop(vcpu);
1947 case KVM_MP_STATE_OPERATING:
1948 kvm_s390_vcpu_start(vcpu);
1950 case KVM_MP_STATE_LOAD:
1951 case KVM_MP_STATE_CHECK_STOP:
1952 /* fall through - CHECK_STOP and LOAD are not supported yet */
1960 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1962 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1965 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1968 kvm_s390_vcpu_request_handled(vcpu);
1969 if (!vcpu->requests)
1972 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1973 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1974 * This ensures that the ipte instruction for this request has
1975 * already finished. We might race against a second unmapper that
1976 * wants to set the blocking bit. Lets just retry the request loop.
1978 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1980 rc = gmap_ipte_notify(vcpu->arch.gmap,
1981 kvm_s390_get_prefix(vcpu),
1988 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1989 vcpu->arch.sie_block->ihcpu = 0xffff;
1993 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1994 if (!ibs_enabled(vcpu)) {
1995 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1996 atomic_or(CPUSTAT_IBS,
1997 &vcpu->arch.sie_block->cpuflags);
2002 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2003 if (ibs_enabled(vcpu)) {
2004 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2005 atomic_andnot(CPUSTAT_IBS,
2006 &vcpu->arch.sie_block->cpuflags);
2011 /* nothing to do, just clear the request */
2012 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2017 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2019 struct kvm_vcpu *vcpu;
2022 mutex_lock(&kvm->lock);
2024 kvm->arch.epoch = tod - get_tod_clock();
2025 kvm_s390_vcpu_block_all(kvm);
2026 kvm_for_each_vcpu(i, vcpu, kvm)
2027 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2028 kvm_s390_vcpu_unblock_all(kvm);
2030 mutex_unlock(&kvm->lock);
2034 * kvm_arch_fault_in_page - fault-in guest page if necessary
2035 * @vcpu: The corresponding virtual cpu
2036 * @gpa: Guest physical address
2037 * @writable: Whether the page should be writable or not
2039 * Make sure that a guest page has been faulted-in on the host.
2041 * Return: Zero on success, negative error code otherwise.
2043 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2045 return gmap_fault(vcpu->arch.gmap, gpa,
2046 writable ? FAULT_FLAG_WRITE : 0);
2049 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2050 unsigned long token)
2052 struct kvm_s390_interrupt inti;
2053 struct kvm_s390_irq irq;
2056 irq.u.ext.ext_params2 = token;
2057 irq.type = KVM_S390_INT_PFAULT_INIT;
2058 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2060 inti.type = KVM_S390_INT_PFAULT_DONE;
2061 inti.parm64 = token;
2062 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2066 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2067 struct kvm_async_pf *work)
2069 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2070 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2073 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2074 struct kvm_async_pf *work)
2076 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2077 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2080 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2081 struct kvm_async_pf *work)
2083 /* s390 will always inject the page directly */
2086 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2089 * s390 will always inject the page directly,
2090 * but we still want check_async_completion to cleanup
2095 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2098 struct kvm_arch_async_pf arch;
2101 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2103 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2104 vcpu->arch.pfault_compare)
2106 if (psw_extint_disabled(vcpu))
2108 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2110 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2112 if (!vcpu->arch.gmap->pfault_enabled)
2115 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2116 hva += current->thread.gmap_addr & ~PAGE_MASK;
2117 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2120 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2124 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2129 * On s390 notifications for arriving pages will be delivered directly
2130 * to the guest but the house keeping for completed pfaults is
2131 * handled outside the worker.
2133 kvm_check_async_pf_completion(vcpu);
2135 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
2140 if (test_cpu_flag(CIF_MCCK_PENDING))
2143 if (!kvm_is_ucontrol(vcpu->kvm)) {
2144 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2149 rc = kvm_s390_handle_requests(vcpu);
2153 if (guestdbg_enabled(vcpu)) {
2154 kvm_s390_backup_guest_per_regs(vcpu);
2155 kvm_s390_patch_guest_per_regs(vcpu);
2158 vcpu->arch.sie_block->icptcode = 0;
2159 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2160 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2161 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2166 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2168 psw_t *psw = &vcpu->arch.sie_block->gpsw;
2172 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2173 trace_kvm_s390_sie_fault(vcpu);
2176 * We want to inject an addressing exception, which is defined as a
2177 * suppressing or terminating exception. However, since we came here
2178 * by a DAT access exception, the PSW still points to the faulting
2179 * instruction since DAT exceptions are nullifying. So we've got
2180 * to look up the current opcode to get the length of the instruction
2181 * to be able to forward the PSW.
2183 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2185 return kvm_s390_inject_prog_cond(vcpu, rc);
2186 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2188 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2191 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2193 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2194 vcpu->arch.sie_block->icptcode);
2195 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2197 if (guestdbg_enabled(vcpu))
2198 kvm_s390_restore_guest_per_regs(vcpu);
2200 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2202 if (vcpu->arch.sie_block->icptcode > 0) {
2203 int rc = kvm_handle_sie_intercept(vcpu);
2205 if (rc != -EOPNOTSUPP)
2207 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2208 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2209 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2210 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2212 } else if (exit_reason != -EFAULT) {
2213 vcpu->stat.exit_null++;
2215 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2216 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2217 vcpu->run->s390_ucontrol.trans_exc_code =
2218 current->thread.gmap_addr;
2219 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2221 } else if (current->thread.gmap_pfault) {
2222 trace_kvm_s390_major_guest_pfault(vcpu);
2223 current->thread.gmap_pfault = 0;
2224 if (kvm_arch_setup_async_pf(vcpu))
2226 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2228 return vcpu_post_run_fault_in_sie(vcpu);
2231 static int __vcpu_run(struct kvm_vcpu *vcpu)
2233 int rc, exit_reason;
2236 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2237 * ning the guest), so that memslots (and other stuff) are protected
2239 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2242 rc = vcpu_pre_run(vcpu);
2246 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2248 * As PF_VCPU will be used in fault handler, between
2249 * guest_enter and guest_exit should be no uaccess.
2251 local_irq_disable();
2252 __kvm_guest_enter();
2254 exit_reason = sie64a(vcpu->arch.sie_block,
2255 vcpu->run->s.regs.gprs);
2256 local_irq_disable();
2259 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2261 rc = vcpu_post_run(vcpu, exit_reason);
2262 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2264 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2268 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2270 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2271 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2272 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2273 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2274 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2275 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2276 /* some control register changes require a tlb flush */
2277 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2279 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2280 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2281 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2282 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2283 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2284 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2286 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2287 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2288 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2289 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2290 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2291 kvm_clear_async_pf_completion_queue(vcpu);
2293 kvm_run->kvm_dirty_regs = 0;
2296 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2298 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2299 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2300 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2301 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2302 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2303 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2304 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2305 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2306 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2307 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2308 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2309 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2312 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2317 if (guestdbg_exit_pending(vcpu)) {
2318 kvm_s390_prepare_debug_exit(vcpu);
2322 if (vcpu->sigset_active)
2323 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2325 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2326 kvm_s390_vcpu_start(vcpu);
2327 } else if (is_vcpu_stopped(vcpu)) {
2328 pr_err_ratelimited("can't run stopped vcpu %d\n",
2333 sync_regs(vcpu, kvm_run);
2336 rc = __vcpu_run(vcpu);
2338 if (signal_pending(current) && !rc) {
2339 kvm_run->exit_reason = KVM_EXIT_INTR;
2343 if (guestdbg_exit_pending(vcpu) && !rc) {
2344 kvm_s390_prepare_debug_exit(vcpu);
2348 if (rc == -EREMOTE) {
2349 /* userspace support is needed, kvm_run has been prepared */
2353 store_regs(vcpu, kvm_run);
2355 if (vcpu->sigset_active)
2356 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2358 vcpu->stat.exit_userspace++;
2363 * store status at address
2364 * we use have two special cases:
2365 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2366 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2368 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2370 unsigned char archmode = 1;
2375 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2376 if (write_guest_abs(vcpu, 163, &archmode, 1))
2378 gpa = SAVE_AREA_BASE;
2379 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2380 if (write_guest_real(vcpu, 163, &archmode, 1))
2382 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2384 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2385 vcpu->arch.guest_fpregs.fprs, 128);
2386 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2387 vcpu->run->s.regs.gprs, 128);
2388 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2389 &vcpu->arch.sie_block->gpsw, 16);
2390 px = kvm_s390_get_prefix(vcpu);
2391 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2393 rc |= write_guest_abs(vcpu,
2394 gpa + offsetof(struct save_area, fp_ctrl_reg),
2395 &vcpu->arch.guest_fpregs.fpc, 4);
2396 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2397 &vcpu->arch.sie_block->todpr, 4);
2398 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2399 &vcpu->arch.sie_block->cputm, 8);
2400 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2401 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2403 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2404 &vcpu->run->s.regs.acrs, 64);
2405 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2406 &vcpu->arch.sie_block->gcr, 128);
2407 return rc ? -EFAULT : 0;
2410 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2413 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2414 * copying in vcpu load/put. Lets update our copies before we save
2415 * it into the save area
2418 if (test_kvm_facility(vcpu->kvm, 129)) {
2420 * If the vector extension is available, the vector registers
2421 * which overlaps with floating-point registers are saved in
2422 * the SIE-control block. Hence, extract the floating-point
2423 * registers and the FPC value and store them in the
2424 * guest_fpregs structure.
2426 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2427 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2428 current->thread.fpu.vxrs);
2430 save_fpu_to(&vcpu->arch.guest_fpregs);
2431 save_access_regs(vcpu->run->s.regs.acrs);
2433 return kvm_s390_store_status_unloaded(vcpu, addr);
2437 * store additional status at address
2439 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2442 /* Only bits 0-53 are used for address formation */
2443 if (!(gpa & ~0x3ff))
2446 return write_guest_abs(vcpu, gpa & ~0x3ff,
2447 (void *)&vcpu->run->s.regs.vrs, 512);
2450 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2452 if (!test_kvm_facility(vcpu->kvm, 129))
2456 * The guest VXRS are in the host VXRs due to the lazy
2457 * copying in vcpu load/put. We can simply call save_fpu_regs()
2458 * to save the current register state because we are in the
2459 * middle of a load/put cycle.
2461 * Let's update our copies before we save it into the save area.
2465 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2468 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2470 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2471 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2474 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2477 struct kvm_vcpu *vcpu;
2479 kvm_for_each_vcpu(i, vcpu, kvm) {
2480 __disable_ibs_on_vcpu(vcpu);
2484 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2486 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2487 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2490 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2492 int i, online_vcpus, started_vcpus = 0;
2494 if (!is_vcpu_stopped(vcpu))
2497 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2498 /* Only one cpu at a time may enter/leave the STOPPED state. */
2499 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2500 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2502 for (i = 0; i < online_vcpus; i++) {
2503 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2507 if (started_vcpus == 0) {
2508 /* we're the only active VCPU -> speed it up */
2509 __enable_ibs_on_vcpu(vcpu);
2510 } else if (started_vcpus == 1) {
2512 * As we are starting a second VCPU, we have to disable
2513 * the IBS facility on all VCPUs to remove potentially
2514 * oustanding ENABLE requests.
2516 __disable_ibs_on_all_vcpus(vcpu->kvm);
2519 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2521 * Another VCPU might have used IBS while we were offline.
2522 * Let's play safe and flush the VCPU at startup.
2524 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2525 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2529 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2531 int i, online_vcpus, started_vcpus = 0;
2532 struct kvm_vcpu *started_vcpu = NULL;
2534 if (is_vcpu_stopped(vcpu))
2537 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2538 /* Only one cpu at a time may enter/leave the STOPPED state. */
2539 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2540 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2542 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2543 kvm_s390_clear_stop_irq(vcpu);
2545 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2546 __disable_ibs_on_vcpu(vcpu);
2548 for (i = 0; i < online_vcpus; i++) {
2549 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2551 started_vcpu = vcpu->kvm->vcpus[i];
2555 if (started_vcpus == 1) {
2557 * As we only have one VCPU left, we want to enable the
2558 * IBS facility for that VCPU to speed it up.
2560 __enable_ibs_on_vcpu(started_vcpu);
2563 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2567 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2568 struct kvm_enable_cap *cap)
2576 case KVM_CAP_S390_CSS_SUPPORT:
2577 if (!vcpu->kvm->arch.css_support) {
2578 vcpu->kvm->arch.css_support = 1;
2579 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2580 trace_kvm_s390_enable_css(vcpu->kvm);
2591 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2592 struct kvm_s390_mem_op *mop)
2594 void __user *uaddr = (void __user *)mop->buf;
2595 void *tmpbuf = NULL;
2597 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2598 | KVM_S390_MEMOP_F_CHECK_ONLY;
2600 if (mop->flags & ~supported_flags)
2603 if (mop->size > MEM_OP_MAX_SIZE)
2606 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2607 tmpbuf = vmalloc(mop->size);
2612 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2615 case KVM_S390_MEMOP_LOGICAL_READ:
2616 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2617 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2620 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2622 if (copy_to_user(uaddr, tmpbuf, mop->size))
2626 case KVM_S390_MEMOP_LOGICAL_WRITE:
2627 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2628 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2631 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2635 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2641 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2643 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2644 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2650 long kvm_arch_vcpu_ioctl(struct file *filp,
2651 unsigned int ioctl, unsigned long arg)
2653 struct kvm_vcpu *vcpu = filp->private_data;
2654 void __user *argp = (void __user *)arg;
2659 case KVM_S390_IRQ: {
2660 struct kvm_s390_irq s390irq;
2663 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2665 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2668 case KVM_S390_INTERRUPT: {
2669 struct kvm_s390_interrupt s390int;
2670 struct kvm_s390_irq s390irq;
2673 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2675 if (s390int_to_s390irq(&s390int, &s390irq))
2677 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2680 case KVM_S390_STORE_STATUS:
2681 idx = srcu_read_lock(&vcpu->kvm->srcu);
2682 r = kvm_s390_vcpu_store_status(vcpu, arg);
2683 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2685 case KVM_S390_SET_INITIAL_PSW: {
2689 if (copy_from_user(&psw, argp, sizeof(psw)))
2691 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2694 case KVM_S390_INITIAL_RESET:
2695 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2697 case KVM_SET_ONE_REG:
2698 case KVM_GET_ONE_REG: {
2699 struct kvm_one_reg reg;
2701 if (copy_from_user(®, argp, sizeof(reg)))
2703 if (ioctl == KVM_SET_ONE_REG)
2704 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
2706 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
2709 #ifdef CONFIG_KVM_S390_UCONTROL
2710 case KVM_S390_UCAS_MAP: {
2711 struct kvm_s390_ucas_mapping ucasmap;
2713 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2718 if (!kvm_is_ucontrol(vcpu->kvm)) {
2723 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2724 ucasmap.vcpu_addr, ucasmap.length);
2727 case KVM_S390_UCAS_UNMAP: {
2728 struct kvm_s390_ucas_mapping ucasmap;
2730 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2735 if (!kvm_is_ucontrol(vcpu->kvm)) {
2740 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2745 case KVM_S390_VCPU_FAULT: {
2746 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2749 case KVM_ENABLE_CAP:
2751 struct kvm_enable_cap cap;
2753 if (copy_from_user(&cap, argp, sizeof(cap)))
2755 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2758 case KVM_S390_MEM_OP: {
2759 struct kvm_s390_mem_op mem_op;
2761 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2762 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2767 case KVM_S390_SET_IRQ_STATE: {
2768 struct kvm_s390_irq_state irq_state;
2771 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2773 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2774 irq_state.len == 0 ||
2775 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2779 r = kvm_s390_set_irq_state(vcpu,
2780 (void __user *) irq_state.buf,
2784 case KVM_S390_GET_IRQ_STATE: {
2785 struct kvm_s390_irq_state irq_state;
2788 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2790 if (irq_state.len == 0) {
2794 r = kvm_s390_get_irq_state(vcpu,
2795 (__u8 __user *) irq_state.buf,
2805 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2807 #ifdef CONFIG_KVM_S390_UCONTROL
2808 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2809 && (kvm_is_ucontrol(vcpu->kvm))) {
2810 vmf->page = virt_to_page(vcpu->arch.sie_block);
2811 get_page(vmf->page);
2815 return VM_FAULT_SIGBUS;
2818 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2819 unsigned long npages)
2824 /* Section: memory related */
2825 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2826 struct kvm_memory_slot *memslot,
2827 const struct kvm_userspace_memory_region *mem,
2828 enum kvm_mr_change change)
2830 /* A few sanity checks. We can have memory slots which have to be
2831 located/ended at a segment boundary (1MB). The memory in userland is
2832 ok to be fragmented into various different vmas. It is okay to mmap()
2833 and munmap() stuff in this slot after doing this call at any time */
2835 if (mem->userspace_addr & 0xffffful)
2838 if (mem->memory_size & 0xffffful)
2844 void kvm_arch_commit_memory_region(struct kvm *kvm,
2845 const struct kvm_userspace_memory_region *mem,
2846 const struct kvm_memory_slot *old,
2847 const struct kvm_memory_slot *new,
2848 enum kvm_mr_change change)
2852 /* If the basics of the memslot do not change, we do not want
2853 * to update the gmap. Every update causes several unnecessary
2854 * segment translation exceptions. This is usually handled just
2855 * fine by the normal fault handler + gmap, but it will also
2856 * cause faults on the prefix page of running guest CPUs.
2858 if (old->userspace_addr == mem->userspace_addr &&
2859 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2860 old->npages * PAGE_SIZE == mem->memory_size)
2863 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2864 mem->guest_phys_addr, mem->memory_size);
2866 pr_warn("failed to commit memory region\n");
2870 static int __init kvm_s390_init(void)
2872 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2875 static void __exit kvm_s390_exit(void)
2880 module_init(kvm_s390_init);
2881 module_exit(kvm_s390_exit);
2884 * Enable autoloading of the kvm module.
2885 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2886 * since x86 takes a different approach.
2888 #include <linux/miscdevice.h>
2889 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2890 MODULE_ALIAS("devname:kvm");