2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
32 #include <asm/pgtable.h>
34 #include <asm/switch_to.h>
40 #define KMSG_COMPONENT "kvm-s390"
42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
44 #define CREATE_TRACE_POINTS
46 #include "trace-s390.h"
48 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
55 struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
57 { "exit_null", VCPU_STAT(exit_null) },
58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
90 { "instruction_essa", VCPU_STAT(instruction_essa) },
91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
110 { "diagnose_10", VCPU_STAT(diagnose_10) },
111 { "diagnose_44", VCPU_STAT(diagnose_44) },
112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
119 /* upper facilities limit for kvm */
120 unsigned long kvm_s390_fac_list_mask[] = {
121 0xffe6fffbfcfdfc40UL,
122 0x005e800000000000UL,
125 unsigned long kvm_s390_fac_list_mask_size(void)
127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
131 static struct gmap_notifier gmap_notifier;
132 debug_info_t *kvm_s390_dbf;
134 /* Section: not file related */
135 int kvm_arch_hardware_enable(void)
137 /* every s390 is virtualization enabled ;-) */
141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
153 struct kvm_vcpu *vcpu;
155 unsigned long long *delta = v;
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
166 static struct notifier_block kvm_clock_notifier = {
167 .notifier_call = kvm_clock_sync,
170 int kvm_arch_hardware_setup(void)
172 gmap_notifier.notifier_call = kvm_gmap_notifier;
173 gmap_register_ipte_notifier(&gmap_notifier);
174 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 &kvm_clock_notifier);
179 void kvm_arch_hardware_unsetup(void)
181 gmap_unregister_ipte_notifier(&gmap_notifier);
182 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 &kvm_clock_notifier);
186 int kvm_arch_init(void *opaque)
188 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
192 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 debug_unregister(kvm_s390_dbf);
197 /* Register floating interrupt controller interface. */
198 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
201 void kvm_arch_exit(void)
203 debug_unregister(kvm_s390_dbf);
206 /* Section: device related */
207 long kvm_arch_dev_ioctl(struct file *filp,
208 unsigned int ioctl, unsigned long arg)
210 if (ioctl == KVM_S390_ENABLE_SIE)
211 return s390_enable_sie();
215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
220 case KVM_CAP_S390_PSW:
221 case KVM_CAP_S390_GMAP:
222 case KVM_CAP_SYNC_MMU:
223 #ifdef CONFIG_KVM_S390_UCONTROL
224 case KVM_CAP_S390_UCONTROL:
226 case KVM_CAP_ASYNC_PF:
227 case KVM_CAP_SYNC_REGS:
228 case KVM_CAP_ONE_REG:
229 case KVM_CAP_ENABLE_CAP:
230 case KVM_CAP_S390_CSS_SUPPORT:
231 case KVM_CAP_IOEVENTFD:
232 case KVM_CAP_DEVICE_CTRL:
233 case KVM_CAP_ENABLE_CAP_VM:
234 case KVM_CAP_S390_IRQCHIP:
235 case KVM_CAP_VM_ATTRIBUTES:
236 case KVM_CAP_MP_STATE:
237 case KVM_CAP_S390_INJECT_IRQ:
238 case KVM_CAP_S390_USER_SIGP:
239 case KVM_CAP_S390_USER_STSI:
240 case KVM_CAP_S390_SKEYS:
241 case KVM_CAP_S390_IRQ_STATE:
244 case KVM_CAP_S390_MEM_OP:
247 case KVM_CAP_NR_VCPUS:
248 case KVM_CAP_MAX_VCPUS:
249 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
250 : KVM_S390_BSCA_CPU_SLOTS;
252 case KVM_CAP_NR_MEMSLOTS:
253 r = KVM_USER_MEM_SLOTS;
255 case KVM_CAP_S390_COW:
256 r = MACHINE_HAS_ESOP;
258 case KVM_CAP_S390_VECTOR_REGISTERS:
267 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
268 struct kvm_memory_slot *memslot)
270 gfn_t cur_gfn, last_gfn;
271 unsigned long address;
272 struct gmap *gmap = kvm->arch.gmap;
274 down_read(&gmap->mm->mmap_sem);
275 /* Loop over all guest pages */
276 last_gfn = memslot->base_gfn + memslot->npages;
277 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
278 address = gfn_to_hva_memslot(memslot, cur_gfn);
280 if (gmap_test_and_clear_dirty(address, gmap))
281 mark_page_dirty(kvm, cur_gfn);
283 up_read(&gmap->mm->mmap_sem);
286 /* Section: vm related */
287 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
290 * Get (and clear) the dirty memory log for a memory slot.
292 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
293 struct kvm_dirty_log *log)
297 struct kvm_memslots *slots;
298 struct kvm_memory_slot *memslot;
301 mutex_lock(&kvm->slots_lock);
304 if (log->slot >= KVM_USER_MEM_SLOTS)
307 slots = kvm_memslots(kvm);
308 memslot = id_to_memslot(slots, log->slot);
310 if (!memslot->dirty_bitmap)
313 kvm_s390_sync_dirty_log(kvm, memslot);
314 r = kvm_get_dirty_log(kvm, log, &is_dirty);
318 /* Clear the dirty log */
320 n = kvm_dirty_bitmap_bytes(memslot);
321 memset(memslot->dirty_bitmap, 0, n);
325 mutex_unlock(&kvm->slots_lock);
329 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
337 case KVM_CAP_S390_IRQCHIP:
338 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
339 kvm->arch.use_irqchip = 1;
342 case KVM_CAP_S390_USER_SIGP:
343 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
344 kvm->arch.user_sigp = 1;
347 case KVM_CAP_S390_VECTOR_REGISTERS:
348 mutex_lock(&kvm->lock);
349 if (atomic_read(&kvm->online_vcpus)) {
351 } else if (MACHINE_HAS_VX) {
352 set_kvm_facility(kvm->arch.model.fac->mask, 129);
353 set_kvm_facility(kvm->arch.model.fac->list, 129);
357 mutex_unlock(&kvm->lock);
358 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
359 r ? "(not available)" : "(success)");
361 case KVM_CAP_S390_USER_STSI:
362 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
363 kvm->arch.user_stsi = 1;
373 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
377 switch (attr->attr) {
378 case KVM_S390_VM_MEM_LIMIT_SIZE:
380 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
381 kvm->arch.gmap->asce_end);
382 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
392 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
396 switch (attr->attr) {
397 case KVM_S390_VM_MEM_ENABLE_CMMA:
398 /* enable CMMA only for z10 and later (EDAT_1) */
400 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
404 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
405 mutex_lock(&kvm->lock);
406 if (atomic_read(&kvm->online_vcpus) == 0) {
407 kvm->arch.use_cmma = 1;
410 mutex_unlock(&kvm->lock);
412 case KVM_S390_VM_MEM_CLR_CMMA:
414 if (!kvm->arch.use_cmma)
417 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
418 mutex_lock(&kvm->lock);
419 idx = srcu_read_lock(&kvm->srcu);
420 s390_reset_cmma(kvm->arch.gmap->mm);
421 srcu_read_unlock(&kvm->srcu, idx);
422 mutex_unlock(&kvm->lock);
425 case KVM_S390_VM_MEM_LIMIT_SIZE: {
426 unsigned long new_limit;
428 if (kvm_is_ucontrol(kvm))
431 if (get_user(new_limit, (u64 __user *)attr->addr))
434 if (new_limit > kvm->arch.gmap->asce_end)
438 mutex_lock(&kvm->lock);
439 if (atomic_read(&kvm->online_vcpus) == 0) {
440 /* gmap_alloc will round the limit up */
441 struct gmap *new = gmap_alloc(current->mm, new_limit);
446 gmap_free(kvm->arch.gmap);
448 kvm->arch.gmap = new;
452 mutex_unlock(&kvm->lock);
453 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
463 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
465 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
467 struct kvm_vcpu *vcpu;
470 if (!test_kvm_facility(kvm, 76))
473 mutex_lock(&kvm->lock);
474 switch (attr->attr) {
475 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
477 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
478 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
479 kvm->arch.crypto.aes_kw = 1;
480 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
482 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
484 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
485 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
486 kvm->arch.crypto.dea_kw = 1;
487 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
489 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
490 kvm->arch.crypto.aes_kw = 0;
491 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
492 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
493 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
495 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
496 kvm->arch.crypto.dea_kw = 0;
497 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
498 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
499 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
502 mutex_unlock(&kvm->lock);
506 kvm_for_each_vcpu(i, vcpu, kvm) {
507 kvm_s390_vcpu_crypto_setup(vcpu);
510 mutex_unlock(&kvm->lock);
514 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
518 if (copy_from_user(>od_high, (void __user *)attr->addr,
524 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
529 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
533 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
536 kvm_s390_set_tod_clock(kvm, gtod);
537 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
541 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
548 switch (attr->attr) {
549 case KVM_S390_VM_TOD_HIGH:
550 ret = kvm_s390_set_tod_high(kvm, attr);
552 case KVM_S390_VM_TOD_LOW:
553 ret = kvm_s390_set_tod_low(kvm, attr);
562 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
566 if (copy_to_user((void __user *)attr->addr, >od_high,
569 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
574 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
578 gtod = kvm_s390_get_tod_clock_fast(kvm);
579 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
581 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
586 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
593 switch (attr->attr) {
594 case KVM_S390_VM_TOD_HIGH:
595 ret = kvm_s390_get_tod_high(kvm, attr);
597 case KVM_S390_VM_TOD_LOW:
598 ret = kvm_s390_get_tod_low(kvm, attr);
607 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
609 struct kvm_s390_vm_cpu_processor *proc;
612 mutex_lock(&kvm->lock);
613 if (atomic_read(&kvm->online_vcpus)) {
617 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
622 if (!copy_from_user(proc, (void __user *)attr->addr,
624 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
625 sizeof(struct cpuid));
626 kvm->arch.model.ibc = proc->ibc;
627 memcpy(kvm->arch.model.fac->list, proc->fac_list,
628 S390_ARCH_FAC_LIST_SIZE_BYTE);
633 mutex_unlock(&kvm->lock);
637 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
641 switch (attr->attr) {
642 case KVM_S390_VM_CPU_PROCESSOR:
643 ret = kvm_s390_set_processor(kvm, attr);
649 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
651 struct kvm_s390_vm_cpu_processor *proc;
654 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
659 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
660 proc->ibc = kvm->arch.model.ibc;
661 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
662 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
669 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
671 struct kvm_s390_vm_cpu_machine *mach;
674 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
679 get_cpu_id((struct cpuid *) &mach->cpuid);
680 mach->ibc = sclp.ibc;
681 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
682 S390_ARCH_FAC_LIST_SIZE_BYTE);
683 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
684 S390_ARCH_FAC_LIST_SIZE_BYTE);
685 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
692 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
696 switch (attr->attr) {
697 case KVM_S390_VM_CPU_PROCESSOR:
698 ret = kvm_s390_get_processor(kvm, attr);
700 case KVM_S390_VM_CPU_MACHINE:
701 ret = kvm_s390_get_machine(kvm, attr);
707 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
711 switch (attr->group) {
712 case KVM_S390_VM_MEM_CTRL:
713 ret = kvm_s390_set_mem_control(kvm, attr);
715 case KVM_S390_VM_TOD:
716 ret = kvm_s390_set_tod(kvm, attr);
718 case KVM_S390_VM_CPU_MODEL:
719 ret = kvm_s390_set_cpu_model(kvm, attr);
721 case KVM_S390_VM_CRYPTO:
722 ret = kvm_s390_vm_set_crypto(kvm, attr);
732 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
736 switch (attr->group) {
737 case KVM_S390_VM_MEM_CTRL:
738 ret = kvm_s390_get_mem_control(kvm, attr);
740 case KVM_S390_VM_TOD:
741 ret = kvm_s390_get_tod(kvm, attr);
743 case KVM_S390_VM_CPU_MODEL:
744 ret = kvm_s390_get_cpu_model(kvm, attr);
754 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
758 switch (attr->group) {
759 case KVM_S390_VM_MEM_CTRL:
760 switch (attr->attr) {
761 case KVM_S390_VM_MEM_ENABLE_CMMA:
762 case KVM_S390_VM_MEM_CLR_CMMA:
763 case KVM_S390_VM_MEM_LIMIT_SIZE:
771 case KVM_S390_VM_TOD:
772 switch (attr->attr) {
773 case KVM_S390_VM_TOD_LOW:
774 case KVM_S390_VM_TOD_HIGH:
782 case KVM_S390_VM_CPU_MODEL:
783 switch (attr->attr) {
784 case KVM_S390_VM_CPU_PROCESSOR:
785 case KVM_S390_VM_CPU_MACHINE:
793 case KVM_S390_VM_CRYPTO:
794 switch (attr->attr) {
795 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
796 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
797 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
798 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
814 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
818 unsigned long curkey;
821 if (args->flags != 0)
824 /* Is this guest using storage keys? */
825 if (!mm_use_skey(current->mm))
826 return KVM_S390_GET_SKEYS_NONE;
828 /* Enforce sane limit on memory allocation */
829 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
832 keys = kmalloc_array(args->count, sizeof(uint8_t),
833 GFP_KERNEL | __GFP_NOWARN);
835 keys = vmalloc(sizeof(uint8_t) * args->count);
839 for (i = 0; i < args->count; i++) {
840 hva = gfn_to_hva(kvm, args->start_gfn + i);
841 if (kvm_is_error_hva(hva)) {
846 curkey = get_guest_storage_key(current->mm, hva);
847 if (IS_ERR_VALUE(curkey)) {
854 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
855 sizeof(uint8_t) * args->count);
863 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
869 if (args->flags != 0)
872 /* Enforce sane limit on memory allocation */
873 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
876 keys = kmalloc_array(args->count, sizeof(uint8_t),
877 GFP_KERNEL | __GFP_NOWARN);
879 keys = vmalloc(sizeof(uint8_t) * args->count);
883 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
884 sizeof(uint8_t) * args->count);
890 /* Enable storage key handling for the guest */
891 r = s390_enable_skey();
895 for (i = 0; i < args->count; i++) {
896 hva = gfn_to_hva(kvm, args->start_gfn + i);
897 if (kvm_is_error_hva(hva)) {
902 /* Lowest order bit is reserved */
903 if (keys[i] & 0x01) {
908 r = set_guest_storage_key(current->mm, hva,
909 (unsigned long)keys[i], 0);
918 long kvm_arch_vm_ioctl(struct file *filp,
919 unsigned int ioctl, unsigned long arg)
921 struct kvm *kvm = filp->private_data;
922 void __user *argp = (void __user *)arg;
923 struct kvm_device_attr attr;
927 case KVM_S390_INTERRUPT: {
928 struct kvm_s390_interrupt s390int;
931 if (copy_from_user(&s390int, argp, sizeof(s390int)))
933 r = kvm_s390_inject_vm(kvm, &s390int);
936 case KVM_ENABLE_CAP: {
937 struct kvm_enable_cap cap;
939 if (copy_from_user(&cap, argp, sizeof(cap)))
941 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
944 case KVM_CREATE_IRQCHIP: {
945 struct kvm_irq_routing_entry routing;
948 if (kvm->arch.use_irqchip) {
949 /* Set up dummy routing. */
950 memset(&routing, 0, sizeof(routing));
951 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
955 case KVM_SET_DEVICE_ATTR: {
957 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
959 r = kvm_s390_vm_set_attr(kvm, &attr);
962 case KVM_GET_DEVICE_ATTR: {
964 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
966 r = kvm_s390_vm_get_attr(kvm, &attr);
969 case KVM_HAS_DEVICE_ATTR: {
971 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
973 r = kvm_s390_vm_has_attr(kvm, &attr);
976 case KVM_S390_GET_SKEYS: {
977 struct kvm_s390_skeys args;
980 if (copy_from_user(&args, argp,
981 sizeof(struct kvm_s390_skeys)))
983 r = kvm_s390_get_skeys(kvm, &args);
986 case KVM_S390_SET_SKEYS: {
987 struct kvm_s390_skeys args;
990 if (copy_from_user(&args, argp,
991 sizeof(struct kvm_s390_skeys)))
993 r = kvm_s390_set_skeys(kvm, &args);
1003 static int kvm_s390_query_ap_config(u8 *config)
1005 u32 fcn_code = 0x04000000UL;
1008 memset(config, 0, 128);
1012 ".long 0xb2af0000\n" /* PQAP(QCI) */
1018 : "r" (fcn_code), "r" (config)
1019 : "cc", "0", "2", "memory"
1025 static int kvm_s390_apxa_installed(void)
1030 if (test_facility(2) && test_facility(12)) {
1031 cc = kvm_s390_query_ap_config(config);
1034 pr_err("PQAP(QCI) failed with cc=%d", cc);
1036 return config[0] & 0x40;
1042 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1044 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1046 if (kvm_s390_apxa_installed())
1047 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1049 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1052 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1055 cpu_id->version = 0xff;
1058 static int kvm_s390_crypto_init(struct kvm *kvm)
1060 if (!test_kvm_facility(kvm, 76))
1063 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1064 GFP_KERNEL | GFP_DMA);
1065 if (!kvm->arch.crypto.crycb)
1068 kvm_s390_set_crycb_format(kvm);
1070 /* Enable AES/DEA protected key functions by default */
1071 kvm->arch.crypto.aes_kw = 1;
1072 kvm->arch.crypto.dea_kw = 1;
1073 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1074 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1075 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1076 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1081 static void sca_dispose(struct kvm *kvm)
1083 if (kvm->arch.use_esca)
1084 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1086 free_page((unsigned long)(kvm->arch.sca));
1087 kvm->arch.sca = NULL;
1090 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1093 char debug_name[16];
1094 static unsigned long sca_offset;
1097 #ifdef CONFIG_KVM_S390_UCONTROL
1098 if (type & ~KVM_VM_S390_UCONTROL)
1100 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1107 rc = s390_enable_sie();
1113 kvm->arch.use_esca = 0; /* start with basic SCA */
1114 rwlock_init(&kvm->arch.sca_lock);
1115 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
1118 spin_lock(&kvm_lock);
1120 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1122 kvm->arch.sca = (struct bsca_block *)
1123 ((char *) kvm->arch.sca + sca_offset);
1124 spin_unlock(&kvm_lock);
1126 sprintf(debug_name, "kvm-%u", current->pid);
1128 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1133 * The architectural maximum amount of facilities is 16 kbit. To store
1134 * this amount, 2 kbyte of memory is required. Thus we need a full
1135 * page to hold the guest facility list (arch.model.fac->list) and the
1136 * facility mask (arch.model.fac->mask). Its address size has to be
1137 * 31 bits and word aligned.
1139 kvm->arch.model.fac =
1140 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1141 if (!kvm->arch.model.fac)
1144 /* Populate the facility mask initially. */
1145 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1146 S390_ARCH_FAC_LIST_SIZE_BYTE);
1147 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1148 if (i < kvm_s390_fac_list_mask_size())
1149 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1151 kvm->arch.model.fac->mask[i] = 0UL;
1154 /* Populate the facility list initially. */
1155 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1156 S390_ARCH_FAC_LIST_SIZE_BYTE);
1158 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1159 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1161 if (kvm_s390_crypto_init(kvm) < 0)
1164 spin_lock_init(&kvm->arch.float_int.lock);
1165 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1166 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1167 init_waitqueue_head(&kvm->arch.ipte_wq);
1168 mutex_init(&kvm->arch.ipte_mutex);
1170 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1171 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1173 if (type & KVM_VM_S390_UCONTROL) {
1174 kvm->arch.gmap = NULL;
1176 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1177 if (!kvm->arch.gmap)
1179 kvm->arch.gmap->private = kvm;
1180 kvm->arch.gmap->pfault_enabled = 0;
1183 kvm->arch.css_support = 0;
1184 kvm->arch.use_irqchip = 0;
1185 kvm->arch.epoch = 0;
1187 spin_lock_init(&kvm->arch.start_stop_lock);
1188 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1192 kfree(kvm->arch.crypto.crycb);
1193 free_page((unsigned long)kvm->arch.model.fac);
1194 debug_unregister(kvm->arch.dbf);
1196 KVM_EVENT(3, "creation of vm failed: %d", rc);
1200 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1202 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1203 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1204 kvm_s390_clear_local_irqs(vcpu);
1205 kvm_clear_async_pf_completion_queue(vcpu);
1206 if (!kvm_is_ucontrol(vcpu->kvm))
1210 if (kvm_is_ucontrol(vcpu->kvm))
1211 gmap_free(vcpu->arch.gmap);
1213 if (vcpu->kvm->arch.use_cmma)
1214 kvm_s390_vcpu_unsetup_cmma(vcpu);
1215 free_page((unsigned long)(vcpu->arch.sie_block));
1217 kvm_vcpu_uninit(vcpu);
1218 kmem_cache_free(kvm_vcpu_cache, vcpu);
1221 static void kvm_free_vcpus(struct kvm *kvm)
1224 struct kvm_vcpu *vcpu;
1226 kvm_for_each_vcpu(i, vcpu, kvm)
1227 kvm_arch_vcpu_destroy(vcpu);
1229 mutex_lock(&kvm->lock);
1230 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1231 kvm->vcpus[i] = NULL;
1233 atomic_set(&kvm->online_vcpus, 0);
1234 mutex_unlock(&kvm->lock);
1237 void kvm_arch_destroy_vm(struct kvm *kvm)
1239 kvm_free_vcpus(kvm);
1240 free_page((unsigned long)kvm->arch.model.fac);
1242 debug_unregister(kvm->arch.dbf);
1243 kfree(kvm->arch.crypto.crycb);
1244 if (!kvm_is_ucontrol(kvm))
1245 gmap_free(kvm->arch.gmap);
1246 kvm_s390_destroy_adapters(kvm);
1247 kvm_s390_clear_float_irqs(kvm);
1248 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1251 /* Section: vcpu related */
1252 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1254 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1255 if (!vcpu->arch.gmap)
1257 vcpu->arch.gmap->private = vcpu->kvm;
1262 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1264 read_lock(&vcpu->kvm->arch.sca_lock);
1265 if (vcpu->kvm->arch.use_esca) {
1266 struct esca_block *sca = vcpu->kvm->arch.sca;
1268 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1269 sca->cpu[vcpu->vcpu_id].sda = 0;
1271 struct bsca_block *sca = vcpu->kvm->arch.sca;
1273 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1274 sca->cpu[vcpu->vcpu_id].sda = 0;
1276 read_unlock(&vcpu->kvm->arch.sca_lock);
1279 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1281 read_lock(&vcpu->kvm->arch.sca_lock);
1282 if (vcpu->kvm->arch.use_esca) {
1283 struct esca_block *sca = vcpu->kvm->arch.sca;
1285 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1286 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1287 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1288 vcpu->arch.sie_block->ecb2 |= 0x04U;
1289 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1291 struct bsca_block *sca = vcpu->kvm->arch.sca;
1293 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1294 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1295 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1296 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1298 read_unlock(&vcpu->kvm->arch.sca_lock);
1301 /* Basic SCA to Extended SCA data copy routines */
1302 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1305 d->sigp_ctrl.c = s->sigp_ctrl.c;
1306 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1309 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1313 d->ipte_control = s->ipte_control;
1315 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1316 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1319 static int sca_switch_to_extended(struct kvm *kvm)
1321 struct bsca_block *old_sca = kvm->arch.sca;
1322 struct esca_block *new_sca;
1323 struct kvm_vcpu *vcpu;
1324 unsigned int vcpu_idx;
1327 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1331 scaoh = (u32)((u64)(new_sca) >> 32);
1332 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1334 kvm_s390_vcpu_block_all(kvm);
1335 write_lock(&kvm->arch.sca_lock);
1337 sca_copy_b_to_e(new_sca, old_sca);
1339 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1340 vcpu->arch.sie_block->scaoh = scaoh;
1341 vcpu->arch.sie_block->scaol = scaol;
1342 vcpu->arch.sie_block->ecb2 |= 0x04U;
1344 kvm->arch.sca = new_sca;
1345 kvm->arch.use_esca = 1;
1347 write_unlock(&kvm->arch.sca_lock);
1348 kvm_s390_vcpu_unblock_all(kvm);
1350 free_page((unsigned long)old_sca);
1352 VM_EVENT(kvm, 2, "Switched to ESCA (%p -> %p)", old_sca, kvm->arch.sca);
1356 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1360 if (id < KVM_S390_BSCA_CPU_SLOTS)
1365 mutex_lock(&kvm->lock);
1366 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1367 mutex_unlock(&kvm->lock);
1369 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1372 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1374 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1375 kvm_clear_async_pf_completion_queue(vcpu);
1376 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1382 if (test_kvm_facility(vcpu->kvm, 129))
1383 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1385 if (kvm_is_ucontrol(vcpu->kvm))
1386 return __kvm_ucontrol_vcpu_init(vcpu);
1392 * Backs up the current FP/VX register save area on a particular
1393 * destination. Used to switch between different register save
1396 static inline void save_fpu_to(struct fpu *dst)
1398 dst->fpc = current->thread.fpu.fpc;
1399 dst->regs = current->thread.fpu.regs;
1403 * Switches the FP/VX register save area from which to lazy
1404 * restore register contents.
1406 static inline void load_fpu_from(struct fpu *from)
1408 current->thread.fpu.fpc = from->fpc;
1409 current->thread.fpu.regs = from->regs;
1412 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1414 /* Save host register state */
1416 save_fpu_to(&vcpu->arch.host_fpregs);
1418 if (test_kvm_facility(vcpu->kvm, 129)) {
1419 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1421 * Use the register save area in the SIE-control block
1422 * for register restore and save in kvm_arch_vcpu_put()
1424 current->thread.fpu.vxrs =
1425 (__vector128 *)&vcpu->run->s.regs.vrs;
1427 load_fpu_from(&vcpu->arch.guest_fpregs);
1429 if (test_fp_ctl(current->thread.fpu.fpc))
1430 /* User space provided an invalid FPC, let's clear it */
1431 current->thread.fpu.fpc = 0;
1433 save_access_regs(vcpu->arch.host_acrs);
1434 restore_access_regs(vcpu->run->s.regs.acrs);
1435 gmap_enable(vcpu->arch.gmap);
1436 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1439 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1441 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1442 gmap_disable(vcpu->arch.gmap);
1446 if (test_kvm_facility(vcpu->kvm, 129))
1448 * kvm_arch_vcpu_load() set up the register save area to
1449 * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1450 * are already saved. Only the floating-point control must be
1453 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1455 save_fpu_to(&vcpu->arch.guest_fpregs);
1456 load_fpu_from(&vcpu->arch.host_fpregs);
1458 save_access_regs(vcpu->run->s.regs.acrs);
1459 restore_access_regs(vcpu->arch.host_acrs);
1462 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1464 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1465 vcpu->arch.sie_block->gpsw.mask = 0UL;
1466 vcpu->arch.sie_block->gpsw.addr = 0UL;
1467 kvm_s390_set_prefix(vcpu, 0);
1468 vcpu->arch.sie_block->cputm = 0UL;
1469 vcpu->arch.sie_block->ckc = 0UL;
1470 vcpu->arch.sie_block->todpr = 0;
1471 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1472 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1473 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1474 vcpu->arch.guest_fpregs.fpc = 0;
1475 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1476 vcpu->arch.sie_block->gbea = 1;
1477 vcpu->arch.sie_block->pp = 0;
1478 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1479 kvm_clear_async_pf_completion_queue(vcpu);
1480 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1481 kvm_s390_vcpu_stop(vcpu);
1482 kvm_s390_clear_local_irqs(vcpu);
1485 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1487 mutex_lock(&vcpu->kvm->lock);
1489 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1491 mutex_unlock(&vcpu->kvm->lock);
1492 if (!kvm_is_ucontrol(vcpu->kvm)) {
1493 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1499 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1501 if (!test_kvm_facility(vcpu->kvm, 76))
1504 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1506 if (vcpu->kvm->arch.crypto.aes_kw)
1507 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1508 if (vcpu->kvm->arch.crypto.dea_kw)
1509 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1511 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1514 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1516 free_page(vcpu->arch.sie_block->cbrlo);
1517 vcpu->arch.sie_block->cbrlo = 0;
1520 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1522 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1523 if (!vcpu->arch.sie_block->cbrlo)
1526 vcpu->arch.sie_block->ecb2 |= 0x80;
1527 vcpu->arch.sie_block->ecb2 &= ~0x08;
1531 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1533 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1535 vcpu->arch.cpu_id = model->cpu_id;
1536 vcpu->arch.sie_block->ibc = model->ibc;
1537 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1540 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1544 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1548 if (test_kvm_facility(vcpu->kvm, 78))
1549 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1550 else if (test_kvm_facility(vcpu->kvm, 8))
1551 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1553 kvm_s390_vcpu_setup_model(vcpu);
1555 vcpu->arch.sie_block->ecb = 6;
1556 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1557 vcpu->arch.sie_block->ecb |= 0x10;
1559 vcpu->arch.sie_block->ecb2 = 8;
1560 vcpu->arch.sie_block->eca = 0xC1002000U;
1562 vcpu->arch.sie_block->eca |= 1;
1563 if (sclp.has_sigpif)
1564 vcpu->arch.sie_block->eca |= 0x10000000U;
1565 if (test_kvm_facility(vcpu->kvm, 129)) {
1566 vcpu->arch.sie_block->eca |= 0x00020000;
1567 vcpu->arch.sie_block->ecd |= 0x20000000;
1569 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1571 if (vcpu->kvm->arch.use_cmma) {
1572 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1576 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1577 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1579 kvm_s390_vcpu_crypto_setup(vcpu);
1584 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1587 struct kvm_vcpu *vcpu;
1588 struct sie_page *sie_page;
1591 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1596 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1600 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1604 vcpu->arch.sie_block = &sie_page->sie_block;
1605 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1607 vcpu->arch.sie_block->icpua = id;
1608 spin_lock_init(&vcpu->arch.local_int.lock);
1609 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1610 vcpu->arch.local_int.wq = &vcpu->wq;
1611 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1614 * Allocate a save area for floating-point registers. If the vector
1615 * extension is available, register contents are saved in the SIE
1616 * control block. The allocated save area is still required in
1617 * particular places, for example, in kvm_s390_vcpu_store_status().
1619 vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1621 if (!vcpu->arch.guest_fpregs.fprs) {
1623 goto out_free_sie_block;
1626 rc = kvm_vcpu_init(vcpu, kvm, id);
1628 goto out_free_sie_block;
1629 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1630 vcpu->arch.sie_block);
1631 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1635 free_page((unsigned long)(vcpu->arch.sie_block));
1637 kmem_cache_free(kvm_vcpu_cache, vcpu);
1642 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1644 return kvm_s390_vcpu_has_irq(vcpu, 0);
1647 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1649 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1653 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1655 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1658 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1660 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1664 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1666 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1670 * Kick a guest cpu out of SIE and wait until SIE is not running.
1671 * If the CPU is not running (e.g. waiting as idle) the function will
1672 * return immediately. */
1673 void exit_sie(struct kvm_vcpu *vcpu)
1675 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1676 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1680 /* Kick a guest cpu out of SIE to process a request synchronously */
1681 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1683 kvm_make_request(req, vcpu);
1684 kvm_s390_vcpu_request(vcpu);
1687 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1690 struct kvm *kvm = gmap->private;
1691 struct kvm_vcpu *vcpu;
1693 kvm_for_each_vcpu(i, vcpu, kvm) {
1694 /* match against both prefix pages */
1695 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1696 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1697 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1702 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1704 /* kvm common code refers to this, but never calls it */
1709 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1710 struct kvm_one_reg *reg)
1715 case KVM_REG_S390_TODPR:
1716 r = put_user(vcpu->arch.sie_block->todpr,
1717 (u32 __user *)reg->addr);
1719 case KVM_REG_S390_EPOCHDIFF:
1720 r = put_user(vcpu->arch.sie_block->epoch,
1721 (u64 __user *)reg->addr);
1723 case KVM_REG_S390_CPU_TIMER:
1724 r = put_user(vcpu->arch.sie_block->cputm,
1725 (u64 __user *)reg->addr);
1727 case KVM_REG_S390_CLOCK_COMP:
1728 r = put_user(vcpu->arch.sie_block->ckc,
1729 (u64 __user *)reg->addr);
1731 case KVM_REG_S390_PFTOKEN:
1732 r = put_user(vcpu->arch.pfault_token,
1733 (u64 __user *)reg->addr);
1735 case KVM_REG_S390_PFCOMPARE:
1736 r = put_user(vcpu->arch.pfault_compare,
1737 (u64 __user *)reg->addr);
1739 case KVM_REG_S390_PFSELECT:
1740 r = put_user(vcpu->arch.pfault_select,
1741 (u64 __user *)reg->addr);
1743 case KVM_REG_S390_PP:
1744 r = put_user(vcpu->arch.sie_block->pp,
1745 (u64 __user *)reg->addr);
1747 case KVM_REG_S390_GBEA:
1748 r = put_user(vcpu->arch.sie_block->gbea,
1749 (u64 __user *)reg->addr);
1758 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1759 struct kvm_one_reg *reg)
1764 case KVM_REG_S390_TODPR:
1765 r = get_user(vcpu->arch.sie_block->todpr,
1766 (u32 __user *)reg->addr);
1768 case KVM_REG_S390_EPOCHDIFF:
1769 r = get_user(vcpu->arch.sie_block->epoch,
1770 (u64 __user *)reg->addr);
1772 case KVM_REG_S390_CPU_TIMER:
1773 r = get_user(vcpu->arch.sie_block->cputm,
1774 (u64 __user *)reg->addr);
1776 case KVM_REG_S390_CLOCK_COMP:
1777 r = get_user(vcpu->arch.sie_block->ckc,
1778 (u64 __user *)reg->addr);
1780 case KVM_REG_S390_PFTOKEN:
1781 r = get_user(vcpu->arch.pfault_token,
1782 (u64 __user *)reg->addr);
1783 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1784 kvm_clear_async_pf_completion_queue(vcpu);
1786 case KVM_REG_S390_PFCOMPARE:
1787 r = get_user(vcpu->arch.pfault_compare,
1788 (u64 __user *)reg->addr);
1790 case KVM_REG_S390_PFSELECT:
1791 r = get_user(vcpu->arch.pfault_select,
1792 (u64 __user *)reg->addr);
1794 case KVM_REG_S390_PP:
1795 r = get_user(vcpu->arch.sie_block->pp,
1796 (u64 __user *)reg->addr);
1798 case KVM_REG_S390_GBEA:
1799 r = get_user(vcpu->arch.sie_block->gbea,
1800 (u64 __user *)reg->addr);
1809 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1811 kvm_s390_vcpu_initial_reset(vcpu);
1815 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1817 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
1821 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1823 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1827 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1828 struct kvm_sregs *sregs)
1830 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1831 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1832 restore_access_regs(vcpu->run->s.regs.acrs);
1836 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1837 struct kvm_sregs *sregs)
1839 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1840 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1844 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1846 if (test_fp_ctl(fpu->fpc))
1848 memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1849 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1851 load_fpu_from(&vcpu->arch.guest_fpregs);
1855 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1857 memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1858 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1862 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1866 if (!is_vcpu_stopped(vcpu))
1869 vcpu->run->psw_mask = psw.mask;
1870 vcpu->run->psw_addr = psw.addr;
1875 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1876 struct kvm_translation *tr)
1878 return -EINVAL; /* not implemented yet */
1881 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1882 KVM_GUESTDBG_USE_HW_BP | \
1883 KVM_GUESTDBG_ENABLE)
1885 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1886 struct kvm_guest_debug *dbg)
1890 vcpu->guest_debug = 0;
1891 kvm_s390_clear_bp_data(vcpu);
1893 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1896 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1897 vcpu->guest_debug = dbg->control;
1898 /* enforce guest PER */
1899 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1901 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1902 rc = kvm_s390_import_bp_data(vcpu, dbg);
1904 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1905 vcpu->arch.guestdbg.last_bp = 0;
1909 vcpu->guest_debug = 0;
1910 kvm_s390_clear_bp_data(vcpu);
1911 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1917 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1918 struct kvm_mp_state *mp_state)
1920 /* CHECK_STOP and LOAD are not supported yet */
1921 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1922 KVM_MP_STATE_OPERATING;
1925 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1926 struct kvm_mp_state *mp_state)
1930 /* user space knows about this interface - let it control the state */
1931 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1933 switch (mp_state->mp_state) {
1934 case KVM_MP_STATE_STOPPED:
1935 kvm_s390_vcpu_stop(vcpu);
1937 case KVM_MP_STATE_OPERATING:
1938 kvm_s390_vcpu_start(vcpu);
1940 case KVM_MP_STATE_LOAD:
1941 case KVM_MP_STATE_CHECK_STOP:
1942 /* fall through - CHECK_STOP and LOAD are not supported yet */
1950 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1952 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1955 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1958 kvm_s390_vcpu_request_handled(vcpu);
1959 if (!vcpu->requests)
1962 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1963 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1964 * This ensures that the ipte instruction for this request has
1965 * already finished. We might race against a second unmapper that
1966 * wants to set the blocking bit. Lets just retry the request loop.
1968 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1970 rc = gmap_ipte_notify(vcpu->arch.gmap,
1971 kvm_s390_get_prefix(vcpu),
1978 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1979 vcpu->arch.sie_block->ihcpu = 0xffff;
1983 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1984 if (!ibs_enabled(vcpu)) {
1985 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1986 atomic_or(CPUSTAT_IBS,
1987 &vcpu->arch.sie_block->cpuflags);
1992 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1993 if (ibs_enabled(vcpu)) {
1994 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1995 atomic_andnot(CPUSTAT_IBS,
1996 &vcpu->arch.sie_block->cpuflags);
2001 /* nothing to do, just clear the request */
2002 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2007 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2009 struct kvm_vcpu *vcpu;
2012 mutex_lock(&kvm->lock);
2014 kvm->arch.epoch = tod - get_tod_clock();
2015 kvm_s390_vcpu_block_all(kvm);
2016 kvm_for_each_vcpu(i, vcpu, kvm)
2017 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2018 kvm_s390_vcpu_unblock_all(kvm);
2020 mutex_unlock(&kvm->lock);
2024 * kvm_arch_fault_in_page - fault-in guest page if necessary
2025 * @vcpu: The corresponding virtual cpu
2026 * @gpa: Guest physical address
2027 * @writable: Whether the page should be writable or not
2029 * Make sure that a guest page has been faulted-in on the host.
2031 * Return: Zero on success, negative error code otherwise.
2033 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2035 return gmap_fault(vcpu->arch.gmap, gpa,
2036 writable ? FAULT_FLAG_WRITE : 0);
2039 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2040 unsigned long token)
2042 struct kvm_s390_interrupt inti;
2043 struct kvm_s390_irq irq;
2046 irq.u.ext.ext_params2 = token;
2047 irq.type = KVM_S390_INT_PFAULT_INIT;
2048 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2050 inti.type = KVM_S390_INT_PFAULT_DONE;
2051 inti.parm64 = token;
2052 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2056 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2057 struct kvm_async_pf *work)
2059 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2060 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2063 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2064 struct kvm_async_pf *work)
2066 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2067 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2070 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2071 struct kvm_async_pf *work)
2073 /* s390 will always inject the page directly */
2076 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2079 * s390 will always inject the page directly,
2080 * but we still want check_async_completion to cleanup
2085 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2088 struct kvm_arch_async_pf arch;
2091 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2093 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2094 vcpu->arch.pfault_compare)
2096 if (psw_extint_disabled(vcpu))
2098 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2100 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2102 if (!vcpu->arch.gmap->pfault_enabled)
2105 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2106 hva += current->thread.gmap_addr & ~PAGE_MASK;
2107 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2110 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2114 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2119 * On s390 notifications for arriving pages will be delivered directly
2120 * to the guest but the house keeping for completed pfaults is
2121 * handled outside the worker.
2123 kvm_check_async_pf_completion(vcpu);
2125 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
2130 if (test_cpu_flag(CIF_MCCK_PENDING))
2133 if (!kvm_is_ucontrol(vcpu->kvm)) {
2134 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2139 rc = kvm_s390_handle_requests(vcpu);
2143 if (guestdbg_enabled(vcpu)) {
2144 kvm_s390_backup_guest_per_regs(vcpu);
2145 kvm_s390_patch_guest_per_regs(vcpu);
2148 vcpu->arch.sie_block->icptcode = 0;
2149 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2150 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2151 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2156 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2158 psw_t *psw = &vcpu->arch.sie_block->gpsw;
2162 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2163 trace_kvm_s390_sie_fault(vcpu);
2166 * We want to inject an addressing exception, which is defined as a
2167 * suppressing or terminating exception. However, since we came here
2168 * by a DAT access exception, the PSW still points to the faulting
2169 * instruction since DAT exceptions are nullifying. So we've got
2170 * to look up the current opcode to get the length of the instruction
2171 * to be able to forward the PSW.
2173 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2175 return kvm_s390_inject_prog_cond(vcpu, rc);
2176 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2178 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2181 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2183 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2184 vcpu->arch.sie_block->icptcode);
2185 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2187 if (guestdbg_enabled(vcpu))
2188 kvm_s390_restore_guest_per_regs(vcpu);
2190 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2192 if (vcpu->arch.sie_block->icptcode > 0) {
2193 int rc = kvm_handle_sie_intercept(vcpu);
2195 if (rc != -EOPNOTSUPP)
2197 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2198 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2199 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2200 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2202 } else if (exit_reason != -EFAULT) {
2203 vcpu->stat.exit_null++;
2205 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2206 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2207 vcpu->run->s390_ucontrol.trans_exc_code =
2208 current->thread.gmap_addr;
2209 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2211 } else if (current->thread.gmap_pfault) {
2212 trace_kvm_s390_major_guest_pfault(vcpu);
2213 current->thread.gmap_pfault = 0;
2214 if (kvm_arch_setup_async_pf(vcpu))
2216 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2218 return vcpu_post_run_fault_in_sie(vcpu);
2221 static int __vcpu_run(struct kvm_vcpu *vcpu)
2223 int rc, exit_reason;
2226 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2227 * ning the guest), so that memslots (and other stuff) are protected
2229 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2232 rc = vcpu_pre_run(vcpu);
2236 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2238 * As PF_VCPU will be used in fault handler, between
2239 * guest_enter and guest_exit should be no uaccess.
2241 local_irq_disable();
2242 __kvm_guest_enter();
2244 exit_reason = sie64a(vcpu->arch.sie_block,
2245 vcpu->run->s.regs.gprs);
2246 local_irq_disable();
2249 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2251 rc = vcpu_post_run(vcpu, exit_reason);
2252 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2254 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2258 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2260 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2261 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2262 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2263 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2264 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2265 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2266 /* some control register changes require a tlb flush */
2267 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2269 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2270 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2271 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2272 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2273 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2274 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2276 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2277 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2278 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2279 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2280 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2281 kvm_clear_async_pf_completion_queue(vcpu);
2283 kvm_run->kvm_dirty_regs = 0;
2286 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2288 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2289 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2290 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2291 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2292 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2293 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2294 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2295 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2296 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2297 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2298 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2299 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2302 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2307 if (guestdbg_exit_pending(vcpu)) {
2308 kvm_s390_prepare_debug_exit(vcpu);
2312 if (vcpu->sigset_active)
2313 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2315 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2316 kvm_s390_vcpu_start(vcpu);
2317 } else if (is_vcpu_stopped(vcpu)) {
2318 pr_err_ratelimited("can't run stopped vcpu %d\n",
2323 sync_regs(vcpu, kvm_run);
2326 rc = __vcpu_run(vcpu);
2328 if (signal_pending(current) && !rc) {
2329 kvm_run->exit_reason = KVM_EXIT_INTR;
2333 if (guestdbg_exit_pending(vcpu) && !rc) {
2334 kvm_s390_prepare_debug_exit(vcpu);
2338 if (rc == -EREMOTE) {
2339 /* userspace support is needed, kvm_run has been prepared */
2343 store_regs(vcpu, kvm_run);
2345 if (vcpu->sigset_active)
2346 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2348 vcpu->stat.exit_userspace++;
2353 * store status at address
2354 * we use have two special cases:
2355 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2356 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2358 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2360 unsigned char archmode = 1;
2365 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2366 if (write_guest_abs(vcpu, 163, &archmode, 1))
2368 gpa = SAVE_AREA_BASE;
2369 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2370 if (write_guest_real(vcpu, 163, &archmode, 1))
2372 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2374 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2375 vcpu->arch.guest_fpregs.fprs, 128);
2376 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2377 vcpu->run->s.regs.gprs, 128);
2378 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2379 &vcpu->arch.sie_block->gpsw, 16);
2380 px = kvm_s390_get_prefix(vcpu);
2381 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2383 rc |= write_guest_abs(vcpu,
2384 gpa + offsetof(struct save_area, fp_ctrl_reg),
2385 &vcpu->arch.guest_fpregs.fpc, 4);
2386 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2387 &vcpu->arch.sie_block->todpr, 4);
2388 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2389 &vcpu->arch.sie_block->cputm, 8);
2390 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2391 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2393 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2394 &vcpu->run->s.regs.acrs, 64);
2395 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2396 &vcpu->arch.sie_block->gcr, 128);
2397 return rc ? -EFAULT : 0;
2400 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2403 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2404 * copying in vcpu load/put. Lets update our copies before we save
2405 * it into the save area
2408 if (test_kvm_facility(vcpu->kvm, 129)) {
2410 * If the vector extension is available, the vector registers
2411 * which overlaps with floating-point registers are saved in
2412 * the SIE-control block. Hence, extract the floating-point
2413 * registers and the FPC value and store them in the
2414 * guest_fpregs structure.
2416 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2417 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2418 current->thread.fpu.vxrs);
2420 save_fpu_to(&vcpu->arch.guest_fpregs);
2421 save_access_regs(vcpu->run->s.regs.acrs);
2423 return kvm_s390_store_status_unloaded(vcpu, addr);
2427 * store additional status at address
2429 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2432 /* Only bits 0-53 are used for address formation */
2433 if (!(gpa & ~0x3ff))
2436 return write_guest_abs(vcpu, gpa & ~0x3ff,
2437 (void *)&vcpu->run->s.regs.vrs, 512);
2440 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2442 if (!test_kvm_facility(vcpu->kvm, 129))
2446 * The guest VXRS are in the host VXRs due to the lazy
2447 * copying in vcpu load/put. We can simply call save_fpu_regs()
2448 * to save the current register state because we are in the
2449 * middle of a load/put cycle.
2451 * Let's update our copies before we save it into the save area.
2455 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2458 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2460 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2461 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2464 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2467 struct kvm_vcpu *vcpu;
2469 kvm_for_each_vcpu(i, vcpu, kvm) {
2470 __disable_ibs_on_vcpu(vcpu);
2474 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2476 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2477 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2480 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2482 int i, online_vcpus, started_vcpus = 0;
2484 if (!is_vcpu_stopped(vcpu))
2487 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2488 /* Only one cpu at a time may enter/leave the STOPPED state. */
2489 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2490 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2492 for (i = 0; i < online_vcpus; i++) {
2493 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2497 if (started_vcpus == 0) {
2498 /* we're the only active VCPU -> speed it up */
2499 __enable_ibs_on_vcpu(vcpu);
2500 } else if (started_vcpus == 1) {
2502 * As we are starting a second VCPU, we have to disable
2503 * the IBS facility on all VCPUs to remove potentially
2504 * oustanding ENABLE requests.
2506 __disable_ibs_on_all_vcpus(vcpu->kvm);
2509 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2511 * Another VCPU might have used IBS while we were offline.
2512 * Let's play safe and flush the VCPU at startup.
2514 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2515 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2519 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2521 int i, online_vcpus, started_vcpus = 0;
2522 struct kvm_vcpu *started_vcpu = NULL;
2524 if (is_vcpu_stopped(vcpu))
2527 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2528 /* Only one cpu at a time may enter/leave the STOPPED state. */
2529 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2530 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2532 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2533 kvm_s390_clear_stop_irq(vcpu);
2535 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2536 __disable_ibs_on_vcpu(vcpu);
2538 for (i = 0; i < online_vcpus; i++) {
2539 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2541 started_vcpu = vcpu->kvm->vcpus[i];
2545 if (started_vcpus == 1) {
2547 * As we only have one VCPU left, we want to enable the
2548 * IBS facility for that VCPU to speed it up.
2550 __enable_ibs_on_vcpu(started_vcpu);
2553 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2557 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2558 struct kvm_enable_cap *cap)
2566 case KVM_CAP_S390_CSS_SUPPORT:
2567 if (!vcpu->kvm->arch.css_support) {
2568 vcpu->kvm->arch.css_support = 1;
2569 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2570 trace_kvm_s390_enable_css(vcpu->kvm);
2581 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2582 struct kvm_s390_mem_op *mop)
2584 void __user *uaddr = (void __user *)mop->buf;
2585 void *tmpbuf = NULL;
2587 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2588 | KVM_S390_MEMOP_F_CHECK_ONLY;
2590 if (mop->flags & ~supported_flags)
2593 if (mop->size > MEM_OP_MAX_SIZE)
2596 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2597 tmpbuf = vmalloc(mop->size);
2602 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2605 case KVM_S390_MEMOP_LOGICAL_READ:
2606 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2607 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2610 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2612 if (copy_to_user(uaddr, tmpbuf, mop->size))
2616 case KVM_S390_MEMOP_LOGICAL_WRITE:
2617 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2618 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2621 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2625 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2631 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2633 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2634 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2640 long kvm_arch_vcpu_ioctl(struct file *filp,
2641 unsigned int ioctl, unsigned long arg)
2643 struct kvm_vcpu *vcpu = filp->private_data;
2644 void __user *argp = (void __user *)arg;
2649 case KVM_S390_IRQ: {
2650 struct kvm_s390_irq s390irq;
2653 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2655 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2658 case KVM_S390_INTERRUPT: {
2659 struct kvm_s390_interrupt s390int;
2660 struct kvm_s390_irq s390irq;
2663 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2665 if (s390int_to_s390irq(&s390int, &s390irq))
2667 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2670 case KVM_S390_STORE_STATUS:
2671 idx = srcu_read_lock(&vcpu->kvm->srcu);
2672 r = kvm_s390_vcpu_store_status(vcpu, arg);
2673 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2675 case KVM_S390_SET_INITIAL_PSW: {
2679 if (copy_from_user(&psw, argp, sizeof(psw)))
2681 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2684 case KVM_S390_INITIAL_RESET:
2685 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2687 case KVM_SET_ONE_REG:
2688 case KVM_GET_ONE_REG: {
2689 struct kvm_one_reg reg;
2691 if (copy_from_user(®, argp, sizeof(reg)))
2693 if (ioctl == KVM_SET_ONE_REG)
2694 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
2696 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
2699 #ifdef CONFIG_KVM_S390_UCONTROL
2700 case KVM_S390_UCAS_MAP: {
2701 struct kvm_s390_ucas_mapping ucasmap;
2703 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2708 if (!kvm_is_ucontrol(vcpu->kvm)) {
2713 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2714 ucasmap.vcpu_addr, ucasmap.length);
2717 case KVM_S390_UCAS_UNMAP: {
2718 struct kvm_s390_ucas_mapping ucasmap;
2720 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2725 if (!kvm_is_ucontrol(vcpu->kvm)) {
2730 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2735 case KVM_S390_VCPU_FAULT: {
2736 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2739 case KVM_ENABLE_CAP:
2741 struct kvm_enable_cap cap;
2743 if (copy_from_user(&cap, argp, sizeof(cap)))
2745 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2748 case KVM_S390_MEM_OP: {
2749 struct kvm_s390_mem_op mem_op;
2751 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2752 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2757 case KVM_S390_SET_IRQ_STATE: {
2758 struct kvm_s390_irq_state irq_state;
2761 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2763 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2764 irq_state.len == 0 ||
2765 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2769 r = kvm_s390_set_irq_state(vcpu,
2770 (void __user *) irq_state.buf,
2774 case KVM_S390_GET_IRQ_STATE: {
2775 struct kvm_s390_irq_state irq_state;
2778 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2780 if (irq_state.len == 0) {
2784 r = kvm_s390_get_irq_state(vcpu,
2785 (__u8 __user *) irq_state.buf,
2795 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2797 #ifdef CONFIG_KVM_S390_UCONTROL
2798 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2799 && (kvm_is_ucontrol(vcpu->kvm))) {
2800 vmf->page = virt_to_page(vcpu->arch.sie_block);
2801 get_page(vmf->page);
2805 return VM_FAULT_SIGBUS;
2808 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2809 unsigned long npages)
2814 /* Section: memory related */
2815 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2816 struct kvm_memory_slot *memslot,
2817 const struct kvm_userspace_memory_region *mem,
2818 enum kvm_mr_change change)
2820 /* A few sanity checks. We can have memory slots which have to be
2821 located/ended at a segment boundary (1MB). The memory in userland is
2822 ok to be fragmented into various different vmas. It is okay to mmap()
2823 and munmap() stuff in this slot after doing this call at any time */
2825 if (mem->userspace_addr & 0xffffful)
2828 if (mem->memory_size & 0xffffful)
2834 void kvm_arch_commit_memory_region(struct kvm *kvm,
2835 const struct kvm_userspace_memory_region *mem,
2836 const struct kvm_memory_slot *old,
2837 const struct kvm_memory_slot *new,
2838 enum kvm_mr_change change)
2842 /* If the basics of the memslot do not change, we do not want
2843 * to update the gmap. Every update causes several unnecessary
2844 * segment translation exceptions. This is usually handled just
2845 * fine by the normal fault handler + gmap, but it will also
2846 * cause faults on the prefix page of running guest CPUs.
2848 if (old->userspace_addr == mem->userspace_addr &&
2849 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2850 old->npages * PAGE_SIZE == mem->memory_size)
2853 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2854 mem->guest_phys_addr, mem->memory_size);
2856 pr_warn("failed to commit memory region\n");
2860 static int __init kvm_s390_init(void)
2862 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2865 static void __exit kvm_s390_exit(void)
2870 module_init(kvm_s390_init);
2871 module_exit(kvm_s390_exit);
2874 * Enable autoloading of the kvm module.
2875 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2876 * since x86 takes a different approach.
2878 #include <linux/miscdevice.h>
2879 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2880 MODULE_ALIAS("devname:kvm");