arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/random.h>
  28 #include <linux/slab.h>
  29 #include <linux/timer.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/bitmap.h>
  32 #include <linux/sched/signal.h>
  33
  34 #include <asm/asm-offsets.h>
  35 #include <asm/lowcore.h>
  36 #include <asm/stp.h>
  37 #include <asm/pgtable.h>
  38 #include <asm/gmap.h>
  39 #include <asm/nmi.h>
  40 #include <asm/switch_to.h>
  41 #include <asm/isc.h>
  42 #include <asm/sclp.h>
  43 #include <asm/cpacf.h>
  44 #include <asm/timex.h>
  45 #include "kvm-s390.h"
  46 #include "gaccess.h"
  47
  48 #define KMSG_COMPONENT "kvm-s390"
  49 #undef pr_fmt
  50 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  51
  52 #define CREATE_TRACE_POINTS
  53 #include "trace.h"
  54 #include "trace-s390.h"
  55
  56 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  57 #define LOCAL_IRQS 32
  58 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  59                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  60
  61 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  62
  63 struct kvm_stats_debugfs_item debugfs_entries[] = {
  64         { "userspace_handled", VCPU_STAT(exit_userspace) },
  65         { "exit_null", VCPU_STAT(exit_null) },
  66         { "exit_validity", VCPU_STAT(exit_validity) },
  67         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  68         { "exit_external_request", VCPU_STAT(exit_external_request) },
  69         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  70         { "exit_instruction", VCPU_STAT(exit_instruction) },
  71         { "exit_pei", VCPU_STAT(exit_pei) },
  72         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  73         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  74         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  75         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  76         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  77         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  78         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  79         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  80         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  81         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  82         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  83         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  84         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  85         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  86         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  87         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  88         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  89         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  90         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  91         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  92         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  93         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  94         { "instruction_spx", VCPU_STAT(instruction_spx) },
  95         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  96         { "instruction_stap", VCPU_STAT(instruction_stap) },
  97         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  98         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  99         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
 100         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
 101         { "instruction_essa", VCPU_STAT(instruction_essa) },
 102         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 103         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 104         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 105         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 106         { "instruction_sie", VCPU_STAT(instruction_sie) },
 107         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 108         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 109         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 110         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 111         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 112         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 113         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 114         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 115         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 116         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 117         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 118         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 119         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 120         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 121         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 122         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 123         { "diagnose_10", VCPU_STAT(diagnose_10) },
 124         { "diagnose_44", VCPU_STAT(diagnose_44) },
 125         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 126         { "diagnose_258", VCPU_STAT(diagnose_258) },
 127         { "diagnose_308", VCPU_STAT(diagnose_308) },
 128         { "diagnose_500", VCPU_STAT(diagnose_500) },
 129         { NULL }
 130 };
 131
 132 /* allow nested virtualization in KVM (if enabled by user space) */
 133 static int nested;
 134 module_param(nested, int, S_IRUGO);
 135 MODULE_PARM_DESC(nested, "Nested virtualization support");
 136
 137 /* upper facilities limit for kvm */
 138 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
 139
 140 unsigned long kvm_s390_fac_list_mask_size(void)
 141 {
 142         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 143         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 144 }
 145
 146 /* available cpu features supported by kvm */
 147 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 148 /* available subfunctions indicated via query / "test bit" */
 149 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 150
 151 static struct gmap_notifier gmap_notifier;
 152 static struct gmap_notifier vsie_gmap_notifier;
 153 debug_info_t *kvm_s390_dbf;
 154
 155 /* Section: not file related */
 156 int kvm_arch_hardware_enable(void)
 157 {
 158         /* every s390 is virtualization enabled ;-) */
 159         return 0;
 160 }
 161
 162 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 163                               unsigned long end);
 164
 165 /*
 166  * This callback is executed during stop_machine(). All CPUs are therefore
 167  * temporarily stopped. In order not to change guest behavior, we have to
 168  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 169  * so a CPU won't be stopped while calculating with the epoch.
 170  */
 171 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 172                           void *v)
 173 {
 174         struct kvm *kvm;
 175         struct kvm_vcpu *vcpu;
 176         int i;
 177         unsigned long long *delta = v;
 178
 179         list_for_each_entry(kvm, &vm_list, vm_list) {
 180                 kvm->arch.epoch -= *delta;
 181                 kvm_for_each_vcpu(i, vcpu, kvm) {
 182                         vcpu->arch.sie_block->epoch -= *delta;
 183                         if (vcpu->arch.cputm_enabled)
 184                                 vcpu->arch.cputm_start += *delta;
 185                         if (vcpu->arch.vsie_block)
 186                                 vcpu->arch.vsie_block->epoch -= *delta;
 187                 }
 188         }
 189         return NOTIFY_OK;
 190 }
 191
 192 static struct notifier_block kvm_clock_notifier = {
 193         .notifier_call = kvm_clock_sync,
 194 };
 195
 196 int kvm_arch_hardware_setup(void)
 197 {
 198         gmap_notifier.notifier_call = kvm_gmap_notifier;
 199         gmap_register_pte_notifier(&gmap_notifier);
 200         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 201         gmap_register_pte_notifier(&vsie_gmap_notifier);
 202         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 203                                        &kvm_clock_notifier);
 204         return 0;
 205 }
 206
 207 void kvm_arch_hardware_unsetup(void)
 208 {
 209         gmap_unregister_pte_notifier(&gmap_notifier);
 210         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 211         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 212                                          &kvm_clock_notifier);
 213 }
 214
 215 static void allow_cpu_feat(unsigned long nr)
 216 {
 217         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 218 }
 219
 220 static inline int plo_test_bit(unsigned char nr)
 221 {
 222         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 223         int cc;
 224
 225         asm volatile(
 226                 /* Parameter registers are ignored for "test bit" */
 227                 "       plo     0,0,0,0(0)\n"
 228                 "       ipm     %0\n"
 229                 "       srl     %0,28\n"
 230                 : "=d" (cc)
 231                 : "d" (r0)
 232                 : "cc");
 233         return cc == 0;
 234 }
 235
 236 static void kvm_s390_cpu_feat_init(void)
 237 {
 238         int i;
 239
 240         for (i = 0; i < 256; ++i) {
 241                 if (plo_test_bit(i))
 242                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 243         }
 244
 245         if (test_facility(28)) /* TOD-clock steering */
 246                 ptff(kvm_s390_available_subfunc.ptff,
 247                      sizeof(kvm_s390_available_subfunc.ptff),
 248                      PTFF_QAF);
 249
 250         if (test_facility(17)) { /* MSA */
 251                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 252                               kvm_s390_available_subfunc.kmac);
 253                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 254                               kvm_s390_available_subfunc.kmc);
 255                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 256                               kvm_s390_available_subfunc.km);
 257                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 258                               kvm_s390_available_subfunc.kimd);
 259                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 260                               kvm_s390_available_subfunc.klmd);
 261         }
 262         if (test_facility(76)) /* MSA3 */
 263                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 264                               kvm_s390_available_subfunc.pckmo);
 265         if (test_facility(77)) { /* MSA4 */
 266                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 267                               kvm_s390_available_subfunc.kmctr);
 268                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 269                               kvm_s390_available_subfunc.kmf);
 270                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 271                               kvm_s390_available_subfunc.kmo);
 272                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 273                               kvm_s390_available_subfunc.pcc);
 274         }
 275         if (test_facility(57)) /* MSA5 */
 276                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
 277                               kvm_s390_available_subfunc.ppno);
 278
 279         if (test_facility(146)) /* MSA8 */
 280                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
 281                               kvm_s390_available_subfunc.kma);
 282
 283         if (MACHINE_HAS_ESOP)
 284                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 285         /*
 286          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 287          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 288          */
 289         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 290             !test_facility(3) || !nested)
 291                 return;
 292         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 293         if (sclp.has_64bscao)
 294                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 295         if (sclp.has_siif)
 296                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 297         if (sclp.has_gpere)
 298                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 299         if (sclp.has_gsls)
 300                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 301         if (sclp.has_ib)
 302                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 303         if (sclp.has_cei)
 304                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 305         if (sclp.has_ibs)
 306                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 307         if (sclp.has_kss)
 308                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
 309         /*
 310          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 311          * all skey handling functions read/set the skey from the PGSTE
 312          * instead of the real storage key.
 313          *
 314          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 315          * pages being detected as preserved although they are resident.
 316          *
 317          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 318          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 319          *
 320          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 321          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 322          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 323          *
 324          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 325          * cannot easily shadow the SCA because of the ipte lock.
 326          */
 327 }
 328
 329 int kvm_arch_init(void *opaque)
 330 {
 331         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 332         if (!kvm_s390_dbf)
 333                 return -ENOMEM;
 334
 335         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 336                 debug_unregister(kvm_s390_dbf);
 337                 return -ENOMEM;
 338         }
 339
 340         kvm_s390_cpu_feat_init();
 341
 342         /* Register floating interrupt controller interface. */
 343         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 344 }
 345
 346 void kvm_arch_exit(void)
 347 {
 348         debug_unregister(kvm_s390_dbf);
 349 }
 350
 351 /* Section: device related */
 352 long kvm_arch_dev_ioctl(struct file *filp,
 353                         unsigned int ioctl, unsigned long arg)
 354 {
 355         if (ioctl == KVM_S390_ENABLE_SIE)
 356                 return s390_enable_sie();
 357         return -EINVAL;
 358 }
 359
 360 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 361 {
 362         int r;
 363
 364         switch (ext) {
 365         case KVM_CAP_S390_PSW:
 366         case KVM_CAP_S390_GMAP:
 367         case KVM_CAP_SYNC_MMU:
 368 #ifdef CONFIG_KVM_S390_UCONTROL
 369         case KVM_CAP_S390_UCONTROL:
 370 #endif
 371         case KVM_CAP_ASYNC_PF:
 372         case KVM_CAP_SYNC_REGS:
 373         case KVM_CAP_ONE_REG:
 374         case KVM_CAP_ENABLE_CAP:
 375         case KVM_CAP_S390_CSS_SUPPORT:
 376         case KVM_CAP_IOEVENTFD:
 377         case KVM_CAP_DEVICE_CTRL:
 378         case KVM_CAP_ENABLE_CAP_VM:
 379         case KVM_CAP_S390_IRQCHIP:
 380         case KVM_CAP_VM_ATTRIBUTES:
 381         case KVM_CAP_MP_STATE:
 382         case KVM_CAP_IMMEDIATE_EXIT:
 383         case KVM_CAP_S390_INJECT_IRQ:
 384         case KVM_CAP_S390_USER_SIGP:
 385         case KVM_CAP_S390_USER_STSI:
 386         case KVM_CAP_S390_SKEYS:
 387         case KVM_CAP_S390_IRQ_STATE:
 388         case KVM_CAP_S390_USER_INSTR0:
 389         case KVM_CAP_S390_AIS:
 390                 r = 1;
 391                 break;
 392         case KVM_CAP_S390_MEM_OP:
 393                 r = MEM_OP_MAX_SIZE;
 394                 break;
 395         case KVM_CAP_NR_VCPUS:
 396         case KVM_CAP_MAX_VCPUS:
 397                 r = KVM_S390_BSCA_CPU_SLOTS;
 398                 if (!kvm_s390_use_sca_entries())
 399                         r = KVM_MAX_VCPUS;
 400                 else if (sclp.has_esca && sclp.has_64bscao)
 401                         r = KVM_S390_ESCA_CPU_SLOTS;
 402                 break;
 403         case KVM_CAP_NR_MEMSLOTS:
 404                 r = KVM_USER_MEM_SLOTS;
 405                 break;
 406         case KVM_CAP_S390_COW:
 407                 r = MACHINE_HAS_ESOP;
 408                 break;
 409         case KVM_CAP_S390_VECTOR_REGISTERS:
 410                 r = MACHINE_HAS_VX;
 411                 break;
 412         case KVM_CAP_S390_RI:
 413                 r = test_facility(64);
 414                 break;
 415         case KVM_CAP_S390_GS:
 416                 r = test_facility(133);
 417                 break;
 418         default:
 419                 r = 0;
 420         }
 421         return r;
 422 }
 423
 424 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 425                                         struct kvm_memory_slot *memslot)
 426 {
 427         gfn_t cur_gfn, last_gfn;
 428         unsigned long address;
 429         struct gmap *gmap = kvm->arch.gmap;
 430
 431         /* Loop over all guest pages */
 432         last_gfn = memslot->base_gfn + memslot->npages;
 433         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 434                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 435
 436                 if (test_and_clear_guest_dirty(gmap->mm, address))
 437                         mark_page_dirty(kvm, cur_gfn);
 438                 if (fatal_signal_pending(current))
 439                         return;
 440                 cond_resched();
 441         }
 442 }
 443
 444 /* Section: vm related */
 445 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 446
 447 /*
 448  * Get (and clear) the dirty memory log for a memory slot.
 449  */
 450 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 451                                struct kvm_dirty_log *log)
 452 {
 453         int r;
 454         unsigned long n;
 455         struct kvm_memslots *slots;
 456         struct kvm_memory_slot *memslot;
 457         int is_dirty = 0;
 458
 459         if (kvm_is_ucontrol(kvm))
 460                 return -EINVAL;
 461
 462         mutex_lock(&kvm->slots_lock);
 463
 464         r = -EINVAL;
 465         if (log->slot >= KVM_USER_MEM_SLOTS)
 466                 goto out;
 467
 468         slots = kvm_memslots(kvm);
 469         memslot = id_to_memslot(slots, log->slot);
 470         r = -ENOENT;
 471         if (!memslot->dirty_bitmap)
 472                 goto out;
 473
 474         kvm_s390_sync_dirty_log(kvm, memslot);
 475         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 476         if (r)
 477                 goto out;
 478
 479         /* Clear the dirty log */
 480         if (is_dirty) {
 481                 n = kvm_dirty_bitmap_bytes(memslot);
 482                 memset(memslot->dirty_bitmap, 0, n);
 483         }
 484         r = 0;
 485 out:
 486         mutex_unlock(&kvm->slots_lock);
 487         return r;
 488 }
 489
 490 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 491 {
 492         unsigned int i;
 493         struct kvm_vcpu *vcpu;
 494
 495         kvm_for_each_vcpu(i, vcpu, kvm) {
 496                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 497         }
 498 }
 499
 500 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 501 {
 502         int r;
 503
 504         if (cap->flags)
 505                 return -EINVAL;
 506
 507         switch (cap->cap) {
 508         case KVM_CAP_S390_IRQCHIP:
 509                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 510                 kvm->arch.use_irqchip = 1;
 511                 r = 0;
 512                 break;
 513         case KVM_CAP_S390_USER_SIGP:
 514                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 515                 kvm->arch.user_sigp = 1;
 516                 r = 0;
 517                 break;
 518         case KVM_CAP_S390_VECTOR_REGISTERS:
 519                 mutex_lock(&kvm->lock);
 520                 if (kvm->created_vcpus) {
 521                         r = -EBUSY;
 522                 } else if (MACHINE_HAS_VX) {
 523                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 524                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 525                         if (test_facility(134)) {
 526                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 527                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 528                         }
 529                         if (test_facility(135)) {
 530                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 531                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 532                         }
 533                         r = 0;
 534                 } else
 535                         r = -EINVAL;
 536                 mutex_unlock(&kvm->lock);
 537                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 538                          r ? "(not available)" : "(success)");
 539                 break;
 540         case KVM_CAP_S390_RI:
 541                 r = -EINVAL;
 542                 mutex_lock(&kvm->lock);
 543                 if (kvm->created_vcpus) {
 544                         r = -EBUSY;
 545                 } else if (test_facility(64)) {
 546                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 547                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 548                         r = 0;
 549                 }
 550                 mutex_unlock(&kvm->lock);
 551                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 552                          r ? "(not available)" : "(success)");
 553                 break;
 554         case KVM_CAP_S390_AIS:
 555                 mutex_lock(&kvm->lock);
 556                 if (kvm->created_vcpus) {
 557                         r = -EBUSY;
 558                 } else {
 559                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
 560                         set_kvm_facility(kvm->arch.model.fac_list, 72);
 561                         kvm->arch.float_int.ais_enabled = 1;
 562                         r = 0;
 563                 }
 564                 mutex_unlock(&kvm->lock);
 565                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
 566                          r ? "(not available)" : "(success)");
 567                 break;
 568         case KVM_CAP_S390_GS:
 569                 r = -EINVAL;
 570                 mutex_lock(&kvm->lock);
 571                 if (atomic_read(&kvm->online_vcpus)) {
 572                         r = -EBUSY;
 573                 } else if (test_facility(133)) {
 574                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
 575                         set_kvm_facility(kvm->arch.model.fac_list, 133);
 576                         r = 0;
 577                 }
 578                 mutex_unlock(&kvm->lock);
 579                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
 580                          r ? "(not available)" : "(success)");
 581                 break;
 582         case KVM_CAP_S390_USER_STSI:
 583                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 584                 kvm->arch.user_stsi = 1;
 585                 r = 0;
 586                 break;
 587         case KVM_CAP_S390_USER_INSTR0:
 588                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 589                 kvm->arch.user_instr0 = 1;
 590                 icpt_operexc_on_all_vcpus(kvm);
 591                 r = 0;
 592                 break;
 593         default:
 594                 r = -EINVAL;
 595                 break;
 596         }
 597         return r;
 598 }
 599
 600 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 601 {
 602         int ret;
 603
 604         switch (attr->attr) {
 605         case KVM_S390_VM_MEM_LIMIT_SIZE:
 606                 ret = 0;
 607                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 608                          kvm->arch.mem_limit);
 609                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 610                         ret = -EFAULT;
 611                 break;
 612         default:
 613                 ret = -ENXIO;
 614                 break;
 615         }
 616         return ret;
 617 }
 618
 619 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 620 {
 621         int ret;
 622         unsigned int idx;
 623         switch (attr->attr) {
 624         case KVM_S390_VM_MEM_ENABLE_CMMA:
 625                 ret = -ENXIO;
 626                 if (!sclp.has_cmma)
 627                         break;
 628
 629                 ret = -EBUSY;
 630                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 631                 mutex_lock(&kvm->lock);
 632                 if (!kvm->created_vcpus) {
 633                         kvm->arch.use_cmma = 1;
 634                         ret = 0;
 635                 }
 636                 mutex_unlock(&kvm->lock);
 637                 break;
 638         case KVM_S390_VM_MEM_CLR_CMMA:
 639                 ret = -ENXIO;
 640                 if (!sclp.has_cmma)
 641                         break;
 642                 ret = -EINVAL;
 643                 if (!kvm->arch.use_cmma)
 644                         break;
 645
 646                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 647                 mutex_lock(&kvm->lock);
 648                 idx = srcu_read_lock(&kvm->srcu);
 649                 s390_reset_cmma(kvm->arch.gmap->mm);
 650                 srcu_read_unlock(&kvm->srcu, idx);
 651                 mutex_unlock(&kvm->lock);
 652                 ret = 0;
 653                 break;
 654         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 655                 unsigned long new_limit;
 656
 657                 if (kvm_is_ucontrol(kvm))
 658                         return -EINVAL;
 659
 660                 if (get_user(new_limit, (u64 __user *)attr->addr))
 661                         return -EFAULT;
 662
 663                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 664                     new_limit > kvm->arch.mem_limit)
 665                         return -E2BIG;
 666
 667                 if (!new_limit)
 668                         return -EINVAL;
 669
 670                 /* gmap_create takes last usable address */
 671                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 672                         new_limit -= 1;
 673
 674                 ret = -EBUSY;
 675                 mutex_lock(&kvm->lock);
 676                 if (!kvm->created_vcpus) {
 677                         /* gmap_create will round the limit up */
 678                         struct gmap *new = gmap_create(current->mm, new_limit);
 679
 680                         if (!new) {
 681                                 ret = -ENOMEM;
 682                         } else {
 683                                 gmap_remove(kvm->arch.gmap);
 684                                 new->private = kvm;
 685                                 kvm->arch.gmap = new;
 686                                 ret = 0;
 687                         }
 688                 }
 689                 mutex_unlock(&kvm->lock);
 690                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 691                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 692                          (void *) kvm->arch.gmap->asce);
 693                 break;
 694         }
 695         default:
 696                 ret = -ENXIO;
 697                 break;
 698         }
 699         return ret;
 700 }
 701
 702 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 703
 704 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 705 {
 706         struct kvm_vcpu *vcpu;
 707         int i;
 708
 709         if (!test_kvm_facility(kvm, 76))
 710                 return -EINVAL;
 711
 712         mutex_lock(&kvm->lock);
 713         switch (attr->attr) {
 714         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 715                 get_random_bytes(
 716                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 717                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 718                 kvm->arch.crypto.aes_kw = 1;
 719                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 720                 break;
 721         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 722                 get_random_bytes(
 723                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 724                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 725                 kvm->arch.crypto.dea_kw = 1;
 726                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 727                 break;
 728         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 729                 kvm->arch.crypto.aes_kw = 0;
 730                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 731                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 732                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 733                 break;
 734         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 735                 kvm->arch.crypto.dea_kw = 0;
 736                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 737                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 738                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 739                 break;
 740         default:
 741                 mutex_unlock(&kvm->lock);
 742                 return -ENXIO;
 743         }
 744
 745         kvm_for_each_vcpu(i, vcpu, kvm) {
 746                 kvm_s390_vcpu_crypto_setup(vcpu);
 747                 exit_sie(vcpu);
 748         }
 749         mutex_unlock(&kvm->lock);
 750         return 0;
 751 }
 752
 753 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 754 {
 755         u8 gtod_high;
 756
 757         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 758                                            sizeof(gtod_high)))
 759                 return -EFAULT;
 760
 761         if (gtod_high != 0)
 762                 return -EINVAL;
 763         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 764
 765         return 0;
 766 }
 767
 768 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 769 {
 770         u64 gtod;
 771
 772         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 773                 return -EFAULT;
 774
 775         kvm_s390_set_tod_clock(kvm, gtod);
 776         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 777         return 0;
 778 }
 779
 780 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 781 {
 782         int ret;
 783
 784         if (attr->flags)
 785                 return -EINVAL;
 786
 787         switch (attr->attr) {
 788         case KVM_S390_VM_TOD_HIGH:
 789                 ret = kvm_s390_set_tod_high(kvm, attr);
 790                 break;
 791         case KVM_S390_VM_TOD_LOW:
 792                 ret = kvm_s390_set_tod_low(kvm, attr);
 793                 break;
 794         default:
 795                 ret = -ENXIO;
 796                 break;
 797         }
 798         return ret;
 799 }
 800
 801 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 802 {
 803         u8 gtod_high = 0;
 804
 805         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 806                                          sizeof(gtod_high)))
 807                 return -EFAULT;
 808         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 809
 810         return 0;
 811 }
 812
 813 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 814 {
 815         u64 gtod;
 816
 817         gtod = kvm_s390_get_tod_clock_fast(kvm);
 818         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 819                 return -EFAULT;
 820         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 821
 822         return 0;
 823 }
 824
 825 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 826 {
 827         int ret;
 828
 829         if (attr->flags)
 830                 return -EINVAL;
 831
 832         switch (attr->attr) {
 833         case KVM_S390_VM_TOD_HIGH:
 834                 ret = kvm_s390_get_tod_high(kvm, attr);
 835                 break;
 836         case KVM_S390_VM_TOD_LOW:
 837                 ret = kvm_s390_get_tod_low(kvm, attr);
 838                 break;
 839         default:
 840                 ret = -ENXIO;
 841                 break;
 842         }
 843         return ret;
 844 }
 845
 846 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 847 {
 848         struct kvm_s390_vm_cpu_processor *proc;
 849         u16 lowest_ibc, unblocked_ibc;
 850         int ret = 0;
 851
 852         mutex_lock(&kvm->lock);
 853         if (kvm->created_vcpus) {
 854                 ret = -EBUSY;
 855                 goto out;
 856         }
 857         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 858         if (!proc) {
 859                 ret = -ENOMEM;
 860                 goto out;
 861         }
 862         if (!copy_from_user(proc, (void __user *)attr->addr,
 863                             sizeof(*proc))) {
 864                 kvm->arch.model.cpuid = proc->cpuid;
 865                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
 866                 unblocked_ibc = sclp.ibc & 0xfff;
 867                 if (lowest_ibc && proc->ibc) {
 868                         if (proc->ibc > unblocked_ibc)
 869                                 kvm->arch.model.ibc = unblocked_ibc;
 870                         else if (proc->ibc < lowest_ibc)
 871                                 kvm->arch.model.ibc = lowest_ibc;
 872                         else
 873                                 kvm->arch.model.ibc = proc->ibc;
 874                 }
 875                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
 876                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 877                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
 878                          kvm->arch.model.ibc,
 879                          kvm->arch.model.cpuid);
 880                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
 881                          kvm->arch.model.fac_list[0],
 882                          kvm->arch.model.fac_list[1],
 883                          kvm->arch.model.fac_list[2]);
 884         } else
 885                 ret = -EFAULT;
 886         kfree(proc);
 887 out:
 888         mutex_unlock(&kvm->lock);
 889         return ret;
 890 }
 891
 892 static int kvm_s390_set_processor_feat(struct kvm *kvm,
 893                                        struct kvm_device_attr *attr)
 894 {
 895         struct kvm_s390_vm_cpu_feat data;
 896         int ret = -EBUSY;
 897
 898         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
 899                 return -EFAULT;
 900         if (!bitmap_subset((unsigned long *) data.feat,
 901                            kvm_s390_available_cpu_feat,
 902                            KVM_S390_VM_CPU_FEAT_NR_BITS))
 903                 return -EINVAL;
 904
 905         mutex_lock(&kvm->lock);
 906         if (!atomic_read(&kvm->online_vcpus)) {
 907                 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
 908                             KVM_S390_VM_CPU_FEAT_NR_BITS);
 909                 ret = 0;
 910         }
 911         mutex_unlock(&kvm->lock);
 912         return ret;
 913 }
 914
 915 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
 916                                           struct kvm_device_attr *attr)
 917 {
 918         /*
 919          * Once supported by kernel + hw, we have to store the subfunctions
 920          * in kvm->arch and remember that user space configured them.
 921          */
 922         return -ENXIO;
 923 }
 924
 925 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 926 {
 927         int ret = -ENXIO;
 928
 929         switch (attr->attr) {
 930         case KVM_S390_VM_CPU_PROCESSOR:
 931                 ret = kvm_s390_set_processor(kvm, attr);
 932                 break;
 933         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
 934                 ret = kvm_s390_set_processor_feat(kvm, attr);
 935                 break;
 936         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
 937                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
 938                 break;
 939         }
 940         return ret;
 941 }
 942
 943 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 944 {
 945         struct kvm_s390_vm_cpu_processor *proc;
 946         int ret = 0;
 947
 948         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 949         if (!proc) {
 950                 ret = -ENOMEM;
 951                 goto out;
 952         }
 953         proc->cpuid = kvm->arch.model.cpuid;
 954         proc->ibc = kvm->arch.model.ibc;
 955         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
 956                S390_ARCH_FAC_LIST_SIZE_BYTE);
 957         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
 958                  kvm->arch.model.ibc,
 959                  kvm->arch.model.cpuid);
 960         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
 961                  kvm->arch.model.fac_list[0],
 962                  kvm->arch.model.fac_list[1],
 963                  kvm->arch.model.fac_list[2]);
 964         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 965                 ret = -EFAULT;
 966         kfree(proc);
 967 out:
 968         return ret;
 969 }
 970
 971 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 972 {
 973         struct kvm_s390_vm_cpu_machine *mach;
 974         int ret = 0;
 975
 976         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 977         if (!mach) {
 978                 ret = -ENOMEM;
 979                 goto out;
 980         }
 981         get_cpu_id((struct cpuid *) &mach->cpuid);
 982         mach->ibc = sclp.ibc;
 983         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
 984                S390_ARCH_FAC_LIST_SIZE_BYTE);
 985         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 986                sizeof(S390_lowcore.stfle_fac_list));
 987         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
 988                  kvm->arch.model.ibc,
 989                  kvm->arch.model.cpuid);
 990         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
 991                  mach->fac_mask[0],
 992                  mach->fac_mask[1],
 993                  mach->fac_mask[2]);
 994         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
 995                  mach->fac_list[0],
 996                  mach->fac_list[1],
 997                  mach->fac_list[2]);
 998         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 999                 ret = -EFAULT;
1000         kfree(mach);
1001 out:
1002         return ret;
1003 }
1004
1005 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1006                                        struct kvm_device_attr *attr)
1007 {
1008         struct kvm_s390_vm_cpu_feat data;
1009
1010         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1011                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1012         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1013                 return -EFAULT;
1014         return 0;
1015 }
1016
1017 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1018                                      struct kvm_device_attr *attr)
1019 {
1020         struct kvm_s390_vm_cpu_feat data;
1021
1022         bitmap_copy((unsigned long *) data.feat,
1023                     kvm_s390_available_cpu_feat,
1024                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1025         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1026                 return -EFAULT;
1027         return 0;
1028 }
1029
1030 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1031                                           struct kvm_device_attr *attr)
1032 {
1033         /*
1034          * Once we can actually configure subfunctions (kernel + hw support),
1035          * we have to check if they were already set by user space, if so copy
1036          * them from kvm->arch.
1037          */
1038         return -ENXIO;
1039 }
1040
1041 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1042                                         struct kvm_device_attr *attr)
1043 {
1044         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1045             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1046                 return -EFAULT;
1047         return 0;
1048 }
1049 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1050 {
1051         int ret = -ENXIO;
1052
1053         switch (attr->attr) {
1054         case KVM_S390_VM_CPU_PROCESSOR:
1055                 ret = kvm_s390_get_processor(kvm, attr);
1056                 break;
1057         case KVM_S390_VM_CPU_MACHINE:
1058                 ret = kvm_s390_get_machine(kvm, attr);
1059                 break;
1060         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1061                 ret = kvm_s390_get_processor_feat(kvm, attr);
1062                 break;
1063         case KVM_S390_VM_CPU_MACHINE_FEAT:
1064                 ret = kvm_s390_get_machine_feat(kvm, attr);
1065                 break;
1066         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1067                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1068                 break;
1069         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1070                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1071                 break;
1072         }
1073         return ret;
1074 }
1075
1076 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1077 {
1078         int ret;
1079
1080         switch (attr->group) {
1081         case KVM_S390_VM_MEM_CTRL:
1082                 ret = kvm_s390_set_mem_control(kvm, attr);
1083                 break;
1084         case KVM_S390_VM_TOD:
1085                 ret = kvm_s390_set_tod(kvm, attr);
1086                 break;
1087         case KVM_S390_VM_CPU_MODEL:
1088                 ret = kvm_s390_set_cpu_model(kvm, attr);
1089                 break;
1090         case KVM_S390_VM_CRYPTO:
1091                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1092                 break;
1093         default:
1094                 ret = -ENXIO;
1095                 break;
1096         }
1097
1098         return ret;
1099 }
1100
1101 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1102 {
1103         int ret;
1104
1105         switch (attr->group) {
1106         case KVM_S390_VM_MEM_CTRL:
1107                 ret = kvm_s390_get_mem_control(kvm, attr);
1108                 break;
1109         case KVM_S390_VM_TOD:
1110                 ret = kvm_s390_get_tod(kvm, attr);
1111                 break;
1112         case KVM_S390_VM_CPU_MODEL:
1113                 ret = kvm_s390_get_cpu_model(kvm, attr);
1114                 break;
1115         default:
1116                 ret = -ENXIO;
1117                 break;
1118         }
1119
1120         return ret;
1121 }
1122
1123 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1124 {
1125         int ret;
1126
1127         switch (attr->group) {
1128         case KVM_S390_VM_MEM_CTRL:
1129                 switch (attr->attr) {
1130                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1131                 case KVM_S390_VM_MEM_CLR_CMMA:
1132                         ret = sclp.has_cmma ? 0 : -ENXIO;
1133                         break;
1134                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1135                         ret = 0;
1136                         break;
1137                 default:
1138                         ret = -ENXIO;
1139                         break;
1140                 }
1141                 break;
1142         case KVM_S390_VM_TOD:
1143                 switch (attr->attr) {
1144                 case KVM_S390_VM_TOD_LOW:
1145                 case KVM_S390_VM_TOD_HIGH:
1146                         ret = 0;
1147                         break;
1148                 default:
1149                         ret = -ENXIO;
1150                         break;
1151                 }
1152                 break;
1153         case KVM_S390_VM_CPU_MODEL:
1154                 switch (attr->attr) {
1155                 case KVM_S390_VM_CPU_PROCESSOR:
1156                 case KVM_S390_VM_CPU_MACHINE:
1157                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1158                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1159                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1160                         ret = 0;
1161                         break;
1162                 /* configuring subfunctions is not supported yet */
1163                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1164                 default:
1165                         ret = -ENXIO;
1166                         break;
1167                 }
1168                 break;
1169         case KVM_S390_VM_CRYPTO:
1170                 switch (attr->attr) {
1171                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1172                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1173                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1174                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1175                         ret = 0;
1176                         break;
1177                 default:
1178                         ret = -ENXIO;
1179                         break;
1180                 }
1181                 break;
1182         default:
1183                 ret = -ENXIO;
1184                 break;
1185         }
1186
1187         return ret;
1188 }
1189
1190 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1191 {
1192         uint8_t *keys;
1193         uint64_t hva;
1194         int i, r = 0;
1195
1196         if (args->flags != 0)
1197                 return -EINVAL;
1198
1199         /* Is this guest using storage keys? */
1200         if (!mm_use_skey(current->mm))
1201                 return KVM_S390_GET_SKEYS_NONE;
1202
1203         /* Enforce sane limit on memory allocation */
1204         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1205                 return -EINVAL;
1206
1207         keys = kmalloc_array(args->count, sizeof(uint8_t),
1208                              GFP_KERNEL | __GFP_NOWARN);
1209         if (!keys)
1210                 keys = vmalloc(sizeof(uint8_t) * args->count);
1211         if (!keys)
1212                 return -ENOMEM;
1213
1214         down_read(&current->mm->mmap_sem);
1215         for (i = 0; i < args->count; i++) {
1216                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1217                 if (kvm_is_error_hva(hva)) {
1218                         r = -EFAULT;
1219                         break;
1220                 }
1221
1222                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1223                 if (r)
1224                         break;
1225         }
1226         up_read(&current->mm->mmap_sem);
1227
1228         if (!r) {
1229                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1230                                  sizeof(uint8_t) * args->count);
1231                 if (r)
1232                         r = -EFAULT;
1233         }
1234
1235         kvfree(keys);
1236         return r;
1237 }
1238
1239 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1240 {
1241         uint8_t *keys;
1242         uint64_t hva;
1243         int i, r = 0;
1244
1245         if (args->flags != 0)
1246                 return -EINVAL;
1247
1248         /* Enforce sane limit on memory allocation */
1249         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1250                 return -EINVAL;
1251
1252         keys = kmalloc_array(args->count, sizeof(uint8_t),
1253                              GFP_KERNEL | __GFP_NOWARN);
1254         if (!keys)
1255                 keys = vmalloc(sizeof(uint8_t) * args->count);
1256         if (!keys)
1257                 return -ENOMEM;
1258
1259         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1260                            sizeof(uint8_t) * args->count);
1261         if (r) {
1262                 r = -EFAULT;
1263                 goto out;
1264         }
1265
1266         /* Enable storage key handling for the guest */
1267         r = s390_enable_skey();
1268         if (r)
1269                 goto out;
1270
1271         down_read(&current->mm->mmap_sem);
1272         for (i = 0; i < args->count; i++) {
1273                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1274                 if (kvm_is_error_hva(hva)) {
1275                         r = -EFAULT;
1276                         break;
1277                 }
1278
1279                 /* Lowest order bit is reserved */
1280                 if (keys[i] & 0x01) {
1281                         r = -EINVAL;
1282                         break;
1283                 }
1284
1285                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1286                 if (r)
1287                         break;
1288         }
1289         up_read(&current->mm->mmap_sem);
1290 out:
1291         kvfree(keys);
1292         return r;
1293 }
1294
1295 long kvm_arch_vm_ioctl(struct file *filp,
1296                        unsigned int ioctl, unsigned long arg)
1297 {
1298         struct kvm *kvm = filp->private_data;
1299         void __user *argp = (void __user *)arg;
1300         struct kvm_device_attr attr;
1301         int r;
1302
1303         switch (ioctl) {
1304         case KVM_S390_INTERRUPT: {
1305                 struct kvm_s390_interrupt s390int;
1306
1307                 r = -EFAULT;
1308                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1309                         break;
1310                 r = kvm_s390_inject_vm(kvm, &s390int);
1311                 break;
1312         }
1313         case KVM_ENABLE_CAP: {
1314                 struct kvm_enable_cap cap;
1315                 r = -EFAULT;
1316                 if (copy_from_user(&cap, argp, sizeof(cap)))
1317                         break;
1318                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1319                 break;
1320         }
1321         case KVM_CREATE_IRQCHIP: {
1322                 struct kvm_irq_routing_entry routing;
1323
1324                 r = -EINVAL;
1325                 if (kvm->arch.use_irqchip) {
1326                         /* Set up dummy routing. */
1327                         memset(&routing, 0, sizeof(routing));
1328                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1329                 }
1330                 break;
1331         }
1332         case KVM_SET_DEVICE_ATTR: {
1333                 r = -EFAULT;
1334                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1335                         break;
1336                 r = kvm_s390_vm_set_attr(kvm, &attr);
1337                 break;
1338         }
1339         case KVM_GET_DEVICE_ATTR: {
1340                 r = -EFAULT;
1341                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1342                         break;
1343                 r = kvm_s390_vm_get_attr(kvm, &attr);
1344                 break;
1345         }
1346         case KVM_HAS_DEVICE_ATTR: {
1347                 r = -EFAULT;
1348                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1349                         break;
1350                 r = kvm_s390_vm_has_attr(kvm, &attr);
1351                 break;
1352         }
1353         case KVM_S390_GET_SKEYS: {
1354                 struct kvm_s390_skeys args;
1355
1356                 r = -EFAULT;
1357                 if (copy_from_user(&args, argp,
1358                                    sizeof(struct kvm_s390_skeys)))
1359                         break;
1360                 r = kvm_s390_get_skeys(kvm, &args);
1361                 break;
1362         }
1363         case KVM_S390_SET_SKEYS: {
1364                 struct kvm_s390_skeys args;
1365
1366                 r = -EFAULT;
1367                 if (copy_from_user(&args, argp,
1368                                    sizeof(struct kvm_s390_skeys)))
1369                         break;
1370                 r = kvm_s390_set_skeys(kvm, &args);
1371                 break;
1372         }
1373         default:
1374                 r = -ENOTTY;
1375         }
1376
1377         return r;
1378 }
1379
1380 static int kvm_s390_query_ap_config(u8 *config)
1381 {
1382         u32 fcn_code = 0x04000000UL;
1383         u32 cc = 0;
1384
1385         memset(config, 0, 128);
1386         asm volatile(
1387                 "lgr 0,%1\n"
1388                 "lgr 2,%2\n"
1389                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1390                 "0: ipm %0\n"
1391                 "srl %0,28\n"
1392                 "1:\n"
1393                 EX_TABLE(0b, 1b)
1394                 : "+r" (cc)
1395                 : "r" (fcn_code), "r" (config)
1396                 : "cc", "0", "2", "memory"
1397         );
1398
1399         return cc;
1400 }
1401
1402 static int kvm_s390_apxa_installed(void)
1403 {
1404         u8 config[128];
1405         int cc;
1406
1407         if (test_facility(12)) {
1408                 cc = kvm_s390_query_ap_config(config);
1409
1410                 if (cc)
1411                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1412                 else
1413                         return config[0] & 0x40;
1414         }
1415
1416         return 0;
1417 }
1418
1419 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1420 {
1421         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1422
1423         if (kvm_s390_apxa_installed())
1424                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1425         else
1426                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1427 }
1428
1429 static u64 kvm_s390_get_initial_cpuid(void)
1430 {
1431         struct cpuid cpuid;
1432
1433         get_cpu_id(&cpuid);
1434         cpuid.version = 0xff;
1435         return *((u64 *) &cpuid);
1436 }
1437
1438 static void kvm_s390_crypto_init(struct kvm *kvm)
1439 {
1440         if (!test_kvm_facility(kvm, 76))
1441                 return;
1442
1443         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1444         kvm_s390_set_crycb_format(kvm);
1445
1446         /* Enable AES/DEA protected key functions by default */
1447         kvm->arch.crypto.aes_kw = 1;
1448         kvm->arch.crypto.dea_kw = 1;
1449         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1450                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1451         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1452                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1453 }
1454
1455 static void sca_dispose(struct kvm *kvm)
1456 {
1457         if (kvm->arch.use_esca)
1458                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1459         else
1460                 free_page((unsigned long)(kvm->arch.sca));
1461         kvm->arch.sca = NULL;
1462 }
1463
1464 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1465 {
1466         gfp_t alloc_flags = GFP_KERNEL;
1467         int i, rc;
1468         char debug_name[16];
1469         static unsigned long sca_offset;
1470
1471         rc = -EINVAL;
1472 #ifdef CONFIG_KVM_S390_UCONTROL
1473         if (type & ~KVM_VM_S390_UCONTROL)
1474                 goto out_err;
1475         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1476                 goto out_err;
1477 #else
1478         if (type)
1479                 goto out_err;
1480 #endif
1481
1482         rc = s390_enable_sie();
1483         if (rc)
1484                 goto out_err;
1485
1486         rc = -ENOMEM;
1487
1488         ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1489
1490         kvm->arch.use_esca = 0; /* start with basic SCA */
1491         if (!sclp.has_64bscao)
1492                 alloc_flags |= GFP_DMA;
1493         rwlock_init(&kvm->arch.sca_lock);
1494         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1495         if (!kvm->arch.sca)
1496                 goto out_err;
1497         spin_lock(&kvm_lock);
1498         sca_offset += 16;
1499         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1500                 sca_offset = 0;
1501         kvm->arch.sca = (struct bsca_block *)
1502                         ((char *) kvm->arch.sca + sca_offset);
1503         spin_unlock(&kvm_lock);
1504
1505         sprintf(debug_name, "kvm-%u", current->pid);
1506
1507         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1508         if (!kvm->arch.dbf)
1509                 goto out_err;
1510
1511         kvm->arch.sie_page2 =
1512              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1513         if (!kvm->arch.sie_page2)
1514                 goto out_err;
1515
1516         /* Populate the facility mask initially. */
1517         memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1518                sizeof(S390_lowcore.stfle_fac_list));
1519         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1520                 if (i < kvm_s390_fac_list_mask_size())
1521                         kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1522                 else
1523                         kvm->arch.model.fac_mask[i] = 0UL;
1524         }
1525
1526         /* Populate the facility list initially. */
1527         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1528         memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1529                S390_ARCH_FAC_LIST_SIZE_BYTE);
1530
1531         set_kvm_facility(kvm->arch.model.fac_mask, 74);
1532         set_kvm_facility(kvm->arch.model.fac_list, 74);
1533
1534         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1535         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1536
1537         kvm_s390_crypto_init(kvm);
1538
1539         mutex_init(&kvm->arch.float_int.ais_lock);
1540         kvm->arch.float_int.simm = 0;
1541         kvm->arch.float_int.nimm = 0;
1542         kvm->arch.float_int.ais_enabled = 0;
1543         spin_lock_init(&kvm->arch.float_int.lock);
1544         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1545                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1546         init_waitqueue_head(&kvm->arch.ipte_wq);
1547         mutex_init(&kvm->arch.ipte_mutex);
1548
1549         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1550         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1551
1552         if (type & KVM_VM_S390_UCONTROL) {
1553                 kvm->arch.gmap = NULL;
1554                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1555         } else {
1556                 if (sclp.hamax == U64_MAX)
1557                         kvm->arch.mem_limit = TASK_SIZE_MAX;
1558                 else
1559                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
1560                                                     sclp.hamax + 1);
1561                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1562                 if (!kvm->arch.gmap)
1563                         goto out_err;
1564                 kvm->arch.gmap->private = kvm;
1565                 kvm->arch.gmap->pfault_enabled = 0;
1566         }
1567
1568         kvm->arch.css_support = 0;
1569         kvm->arch.use_irqchip = 0;
1570         kvm->arch.epoch = 0;
1571
1572         spin_lock_init(&kvm->arch.start_stop_lock);
1573         kvm_s390_vsie_init(kvm);
1574         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1575
1576         return 0;
1577 out_err:
1578         free_page((unsigned long)kvm->arch.sie_page2);
1579         debug_unregister(kvm->arch.dbf);
1580         sca_dispose(kvm);
1581         KVM_EVENT(3, "creation of vm failed: %d", rc);
1582         return rc;
1583 }
1584
1585 bool kvm_arch_has_vcpu_debugfs(void)
1586 {
1587         return false;
1588 }
1589
1590 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1591 {
1592         return 0;
1593 }
1594
1595 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1596 {
1597         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1598         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1599         kvm_s390_clear_local_irqs(vcpu);
1600         kvm_clear_async_pf_completion_queue(vcpu);
1601         if (!kvm_is_ucontrol(vcpu->kvm))
1602                 sca_del_vcpu(vcpu);
1603
1604         if (kvm_is_ucontrol(vcpu->kvm))
1605                 gmap_remove(vcpu->arch.gmap);
1606
1607         if (vcpu->kvm->arch.use_cmma)
1608                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1609         free_page((unsigned long)(vcpu->arch.sie_block));
1610
1611         kvm_vcpu_uninit(vcpu);
1612         kmem_cache_free(kvm_vcpu_cache, vcpu);
1613 }
1614
1615 static void kvm_free_vcpus(struct kvm *kvm)
1616 {
1617         unsigned int i;
1618         struct kvm_vcpu *vcpu;
1619
1620         kvm_for_each_vcpu(i, vcpu, kvm)
1621                 kvm_arch_vcpu_destroy(vcpu);
1622
1623         mutex_lock(&kvm->lock);
1624         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1625                 kvm->vcpus[i] = NULL;
1626
1627         atomic_set(&kvm->online_vcpus, 0);
1628         mutex_unlock(&kvm->lock);
1629 }
1630
1631 void kvm_arch_destroy_vm(struct kvm *kvm)
1632 {
1633         kvm_free_vcpus(kvm);
1634         sca_dispose(kvm);
1635         debug_unregister(kvm->arch.dbf);
1636         free_page((unsigned long)kvm->arch.sie_page2);
1637         if (!kvm_is_ucontrol(kvm))
1638                 gmap_remove(kvm->arch.gmap);
1639         kvm_s390_destroy_adapters(kvm);
1640         kvm_s390_clear_float_irqs(kvm);
1641         kvm_s390_vsie_destroy(kvm);
1642         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1643 }
1644
1645 /* Section: vcpu related */
1646 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1647 {
1648         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1649         if (!vcpu->arch.gmap)
1650                 return -ENOMEM;
1651         vcpu->arch.gmap->private = vcpu->kvm;
1652
1653         return 0;
1654 }
1655
1656 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1657 {
1658         if (!kvm_s390_use_sca_entries())
1659                 return;
1660         read_lock(&vcpu->kvm->arch.sca_lock);
1661         if (vcpu->kvm->arch.use_esca) {
1662                 struct esca_block *sca = vcpu->kvm->arch.sca;
1663
1664                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1665                 sca->cpu[vcpu->vcpu_id].sda = 0;
1666         } else {
1667                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1668
1669                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1670                 sca->cpu[vcpu->vcpu_id].sda = 0;
1671         }
1672         read_unlock(&vcpu->kvm->arch.sca_lock);
1673 }
1674
1675 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1676 {
1677         if (!kvm_s390_use_sca_entries()) {
1678                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1679
1680                 /* we still need the basic sca for the ipte control */
1681                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1682                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1683         }
1684         read_lock(&vcpu->kvm->arch.sca_lock);
1685         if (vcpu->kvm->arch.use_esca) {
1686                 struct esca_block *sca = vcpu->kvm->arch.sca;
1687
1688                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1689                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1690                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1691                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
1692                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1693         } else {
1694                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1695
1696                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1697                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1698                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1699                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1700         }
1701         read_unlock(&vcpu->kvm->arch.sca_lock);
1702 }
1703
1704 /* Basic SCA to Extended SCA data copy routines */
1705 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1706 {
1707         d->sda = s->sda;
1708         d->sigp_ctrl.c = s->sigp_ctrl.c;
1709         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1710 }
1711
1712 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1713 {
1714         int i;
1715
1716         d->ipte_control = s->ipte_control;
1717         d->mcn[0] = s->mcn;
1718         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1719                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1720 }
1721
1722 static int sca_switch_to_extended(struct kvm *kvm)
1723 {
1724         struct bsca_block *old_sca = kvm->arch.sca;
1725         struct esca_block *new_sca;
1726         struct kvm_vcpu *vcpu;
1727         unsigned int vcpu_idx;
1728         u32 scaol, scaoh;
1729
1730         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1731         if (!new_sca)
1732                 return -ENOMEM;
1733
1734         scaoh = (u32)((u64)(new_sca) >> 32);
1735         scaol = (u32)(u64)(new_sca) & ~0x3fU;
1736
1737         kvm_s390_vcpu_block_all(kvm);
1738         write_lock(&kvm->arch.sca_lock);
1739
1740         sca_copy_b_to_e(new_sca, old_sca);
1741
1742         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1743                 vcpu->arch.sie_block->scaoh = scaoh;
1744                 vcpu->arch.sie_block->scaol = scaol;
1745                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
1746         }
1747         kvm->arch.sca = new_sca;
1748         kvm->arch.use_esca = 1;
1749
1750         write_unlock(&kvm->arch.sca_lock);
1751         kvm_s390_vcpu_unblock_all(kvm);
1752
1753         free_page((unsigned long)old_sca);
1754
1755         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1756                  old_sca, kvm->arch.sca);
1757         return 0;
1758 }
1759
1760 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1761 {
1762         int rc;
1763
1764         if (!kvm_s390_use_sca_entries()) {
1765                 if (id < KVM_MAX_VCPUS)
1766                         return true;
1767                 return false;
1768         }
1769         if (id < KVM_S390_BSCA_CPU_SLOTS)
1770                 return true;
1771         if (!sclp.has_esca || !sclp.has_64bscao)
1772                 return false;
1773
1774         mutex_lock(&kvm->lock);
1775         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1776         mutex_unlock(&kvm->lock);
1777
1778         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1779 }
1780
1781 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1782 {
1783         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1784         kvm_clear_async_pf_completion_queue(vcpu);
1785         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1786                                     KVM_SYNC_GPRS |
1787                                     KVM_SYNC_ACRS |
1788                                     KVM_SYNC_CRS |
1789                                     KVM_SYNC_ARCH0 |
1790                                     KVM_SYNC_PFAULT;
1791         kvm_s390_set_prefix(vcpu, 0);
1792         if (test_kvm_facility(vcpu->kvm, 64))
1793                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1794         if (test_kvm_facility(vcpu->kvm, 133))
1795                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
1796         /* fprs can be synchronized via vrs, even if the guest has no vx. With
1797          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1798          */
1799         if (MACHINE_HAS_VX)
1800                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1801         else
1802                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1803
1804         if (kvm_is_ucontrol(vcpu->kvm))
1805                 return __kvm_ucontrol_vcpu_init(vcpu);
1806
1807         return 0;
1808 }
1809
1810 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1811 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1812 {
1813         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1814         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1815         vcpu->arch.cputm_start = get_tod_clock_fast();
1816         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1817 }
1818
1819 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1820 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1821 {
1822         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1823         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1824         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1825         vcpu->arch.cputm_start = 0;
1826         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1827 }
1828
1829 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1830 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1831 {
1832         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1833         vcpu->arch.cputm_enabled = true;
1834         __start_cpu_timer_accounting(vcpu);
1835 }
1836
1837 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1838 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1839 {
1840         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1841         __stop_cpu_timer_accounting(vcpu);
1842         vcpu->arch.cputm_enabled = false;
1843 }
1844
1845 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1846 {
1847         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1848         __enable_cpu_timer_accounting(vcpu);
1849         preempt_enable();
1850 }
1851
1852 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1853 {
1854         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1855         __disable_cpu_timer_accounting(vcpu);
1856         preempt_enable();
1857 }
1858
1859 /* set the cpu timer - may only be called from the VCPU thread itself */
1860 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1861 {
1862         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1863         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1864         if (vcpu->arch.cputm_enabled)
1865                 vcpu->arch.cputm_start = get_tod_clock_fast();
1866         vcpu->arch.sie_block->cputm = cputm;
1867         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1868         preempt_enable();
1869 }
1870
1871 /* update and get the cpu timer - can also be called from other VCPU threads */
1872 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1873 {
1874         unsigned int seq;
1875         __u64 value;
1876
1877         if (unlikely(!vcpu->arch.cputm_enabled))
1878                 return vcpu->arch.sie_block->cputm;
1879
1880         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1881         do {
1882                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1883                 /*
1884                  * If the writer would ever execute a read in the critical
1885                  * section, e.g. in irq context, we have a deadlock.
1886                  */
1887                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1888                 value = vcpu->arch.sie_block->cputm;
1889                 /* if cputm_start is 0, accounting is being started/stopped */
1890                 if (likely(vcpu->arch.cputm_start))
1891                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1892         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1893         preempt_enable();
1894         return value;
1895 }
1896
1897 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1898 {
1899
1900         gmap_enable(vcpu->arch.enabled_gmap);
1901         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1902         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1903                 __start_cpu_timer_accounting(vcpu);
1904         vcpu->cpu = cpu;
1905 }
1906
1907 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1908 {
1909         vcpu->cpu = -1;
1910         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1911                 __stop_cpu_timer_accounting(vcpu);
1912         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1913         vcpu->arch.enabled_gmap = gmap_get_enabled();
1914         gmap_disable(vcpu->arch.enabled_gmap);
1915
1916 }
1917
1918 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1919 {
1920         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1921         vcpu->arch.sie_block->gpsw.mask = 0UL;
1922         vcpu->arch.sie_block->gpsw.addr = 0UL;
1923         kvm_s390_set_prefix(vcpu, 0);
1924         kvm_s390_set_cpu_timer(vcpu, 0);
1925         vcpu->arch.sie_block->ckc       = 0UL;
1926         vcpu->arch.sie_block->todpr     = 0;
1927         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1928         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1929         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1930         /* make sure the new fpc will be lazily loaded */
1931         save_fpu_regs();
1932         current->thread.fpu.fpc = 0;
1933         vcpu->arch.sie_block->gbea = 1;
1934         vcpu->arch.sie_block->pp = 0;
1935         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1936         kvm_clear_async_pf_completion_queue(vcpu);
1937         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1938                 kvm_s390_vcpu_stop(vcpu);
1939         kvm_s390_clear_local_irqs(vcpu);
1940 }
1941
1942 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1943 {
1944         mutex_lock(&vcpu->kvm->lock);
1945         preempt_disable();
1946         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1947         preempt_enable();
1948         mutex_unlock(&vcpu->kvm->lock);
1949         if (!kvm_is_ucontrol(vcpu->kvm)) {
1950                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1951                 sca_add_vcpu(vcpu);
1952         }
1953         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1954                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1955         /* make vcpu_load load the right gmap on the first trigger */
1956         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
1957 }
1958
1959 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1960 {
1961         if (!test_kvm_facility(vcpu->kvm, 76))
1962                 return;
1963
1964         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1965
1966         if (vcpu->kvm->arch.crypto.aes_kw)
1967                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1968         if (vcpu->kvm->arch.crypto.dea_kw)
1969                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1970
1971         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1972 }
1973
1974 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1975 {
1976         free_page(vcpu->arch.sie_block->cbrlo);
1977         vcpu->arch.sie_block->cbrlo = 0;
1978 }
1979
1980 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1981 {
1982         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1983         if (!vcpu->arch.sie_block->cbrlo)
1984                 return -ENOMEM;
1985
1986         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
1987         vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
1988         return 0;
1989 }
1990
1991 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1992 {
1993         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1994
1995         vcpu->arch.sie_block->ibc = model->ibc;
1996         if (test_kvm_facility(vcpu->kvm, 7))
1997                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1998 }
1999
2000 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2001 {
2002         int rc = 0;
2003
2004         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2005                                                     CPUSTAT_SM |
2006                                                     CPUSTAT_STOPPED);
2007
2008         if (test_kvm_facility(vcpu->kvm, 78))
2009                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
2010         else if (test_kvm_facility(vcpu->kvm, 8))
2011                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
2012
2013         kvm_s390_vcpu_setup_model(vcpu);
2014
2015         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2016         if (MACHINE_HAS_ESOP)
2017                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2018         if (test_kvm_facility(vcpu->kvm, 9))
2019                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2020         if (test_kvm_facility(vcpu->kvm, 73))
2021                 vcpu->arch.sie_block->ecb |= ECB_TE;
2022
2023         if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
2024                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2025         if (test_kvm_facility(vcpu->kvm, 130))
2026                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2027         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2028         if (sclp.has_cei)
2029                 vcpu->arch.sie_block->eca |= ECA_CEI;
2030         if (sclp.has_ib)
2031                 vcpu->arch.sie_block->eca |= ECA_IB;
2032         if (sclp.has_siif)
2033                 vcpu->arch.sie_block->eca |= ECA_SII;
2034         if (sclp.has_sigpif)
2035                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2036         if (test_kvm_facility(vcpu->kvm, 129)) {
2037                 vcpu->arch.sie_block->eca |= ECA_VX;
2038                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2039         }
2040         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2041                                         | SDNXC;
2042         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2043
2044         if (sclp.has_kss)
2045                 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2046         else
2047                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2048
2049         if (vcpu->kvm->arch.use_cmma) {
2050                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2051                 if (rc)
2052                         return rc;
2053         }
2054         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2055         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2056
2057         kvm_s390_vcpu_crypto_setup(vcpu);
2058
2059         return rc;
2060 }
2061
2062 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2063                                       unsigned int id)
2064 {
2065         struct kvm_vcpu *vcpu;
2066         struct sie_page *sie_page;
2067         int rc = -EINVAL;
2068
2069         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2070                 goto out;
2071
2072         rc = -ENOMEM;
2073
2074         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2075         if (!vcpu)
2076                 goto out;
2077
2078         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2079         if (!sie_page)
2080                 goto out_free_cpu;
2081
2082         vcpu->arch.sie_block = &sie_page->sie_block;
2083         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2084
2085         /* the real guest size will always be smaller than msl */
2086         vcpu->arch.sie_block->mso = 0;
2087         vcpu->arch.sie_block->msl = sclp.hamax;
2088
2089         vcpu->arch.sie_block->icpua = id;
2090         spin_lock_init(&vcpu->arch.local_int.lock);
2091         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2092         vcpu->arch.local_int.wq = &vcpu->wq;
2093         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2094         seqcount_init(&vcpu->arch.cputm_seqcount);
2095
2096         rc = kvm_vcpu_init(vcpu, kvm, id);
2097         if (rc)
2098                 goto out_free_sie_block;
2099         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2100                  vcpu->arch.sie_block);
2101         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2102
2103         return vcpu;
2104 out_free_sie_block:
2105         free_page((unsigned long)(vcpu->arch.sie_block));
2106 out_free_cpu:
2107         kmem_cache_free(kvm_vcpu_cache, vcpu);
2108 out:
2109         return ERR_PTR(rc);
2110 }
2111
2112 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2113 {
2114         return kvm_s390_vcpu_has_irq(vcpu, 0);
2115 }
2116
2117 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2118 {
2119         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2120         exit_sie(vcpu);
2121 }
2122
2123 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2124 {
2125         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2126 }
2127
2128 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2129 {
2130         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2131         exit_sie(vcpu);
2132 }
2133
2134 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2135 {
2136         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2137 }
2138
2139 /*
2140  * Kick a guest cpu out of SIE and wait until SIE is not running.
2141  * If the CPU is not running (e.g. waiting as idle) the function will
2142  * return immediately. */
2143 void exit_sie(struct kvm_vcpu *vcpu)
2144 {
2145         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2146         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2147                 cpu_relax();
2148 }
2149
2150 /* Kick a guest cpu out of SIE to process a request synchronously */
2151 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2152 {
2153         kvm_make_request(req, vcpu);
2154         kvm_s390_vcpu_request(vcpu);
2155 }
2156
2157 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2158                               unsigned long end)
2159 {
2160         struct kvm *kvm = gmap->private;
2161         struct kvm_vcpu *vcpu;
2162         unsigned long prefix;
2163         int i;
2164
2165         if (gmap_is_shadow(gmap))
2166                 return;
2167         if (start >= 1UL << 31)
2168                 /* We are only interested in prefix pages */
2169                 return;
2170         kvm_for_each_vcpu(i, vcpu, kvm) {
2171                 /* match against both prefix pages */
2172                 prefix = kvm_s390_get_prefix(vcpu);
2173                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2174                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2175                                    start, end);
2176                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2177                 }
2178         }
2179 }
2180
2181 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2182 {
2183         /* kvm common code refers to this, but never calls it */
2184         BUG();
2185         return 0;
2186 }
2187
2188 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2189                                            struct kvm_one_reg *reg)
2190 {
2191         int r = -EINVAL;
2192
2193         switch (reg->id) {
2194         case KVM_REG_S390_TODPR:
2195                 r = put_user(vcpu->arch.sie_block->todpr,
2196                              (u32 __user *)reg->addr);
2197                 break;
2198         case KVM_REG_S390_EPOCHDIFF:
2199                 r = put_user(vcpu->arch.sie_block->epoch,
2200                              (u64 __user *)reg->addr);
2201                 break;
2202         case KVM_REG_S390_CPU_TIMER:
2203                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2204                              (u64 __user *)reg->addr);
2205                 break;
2206         case KVM_REG_S390_CLOCK_COMP:
2207                 r = put_user(vcpu->arch.sie_block->ckc,
2208                              (u64 __user *)reg->addr);
2209                 break;
2210         case KVM_REG_S390_PFTOKEN:
2211                 r = put_user(vcpu->arch.pfault_token,
2212                              (u64 __user *)reg->addr);
2213                 break;
2214         case KVM_REG_S390_PFCOMPARE:
2215                 r = put_user(vcpu->arch.pfault_compare,
2216                              (u64 __user *)reg->addr);
2217                 break;
2218         case KVM_REG_S390_PFSELECT:
2219                 r = put_user(vcpu->arch.pfault_select,
2220                              (u64 __user *)reg->addr);
2221                 break;
2222         case KVM_REG_S390_PP:
2223                 r = put_user(vcpu->arch.sie_block->pp,
2224                              (u64 __user *)reg->addr);
2225                 break;
2226         case KVM_REG_S390_GBEA:
2227                 r = put_user(vcpu->arch.sie_block->gbea,
2228                              (u64 __user *)reg->addr);
2229                 break;
2230         default:
2231                 break;
2232         }
2233
2234         return r;
2235 }
2236
2237 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2238                                            struct kvm_one_reg *reg)
2239 {
2240         int r = -EINVAL;
2241         __u64 val;
2242
2243         switch (reg->id) {
2244         case KVM_REG_S390_TODPR:
2245                 r = get_user(vcpu->arch.sie_block->todpr,
2246                              (u32 __user *)reg->addr);
2247                 break;
2248         case KVM_REG_S390_EPOCHDIFF:
2249                 r = get_user(vcpu->arch.sie_block->epoch,
2250                              (u64 __user *)reg->addr);
2251                 break;
2252         case KVM_REG_S390_CPU_TIMER:
2253                 r = get_user(val, (u64 __user *)reg->addr);
2254                 if (!r)
2255                         kvm_s390_set_cpu_timer(vcpu, val);
2256                 break;
2257         case KVM_REG_S390_CLOCK_COMP:
2258                 r = get_user(vcpu->arch.sie_block->ckc,
2259                              (u64 __user *)reg->addr);
2260                 break;
2261         case KVM_REG_S390_PFTOKEN:
2262                 r = get_user(vcpu->arch.pfault_token,
2263                              (u64 __user *)reg->addr);
2264                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2265                         kvm_clear_async_pf_completion_queue(vcpu);
2266                 break;
2267         case KVM_REG_S390_PFCOMPARE:
2268                 r = get_user(vcpu->arch.pfault_compare,
2269                              (u64 __user *)reg->addr);
2270                 break;
2271         case KVM_REG_S390_PFSELECT:
2272                 r = get_user(vcpu->arch.pfault_select,
2273                              (u64 __user *)reg->addr);
2274                 break;
2275         case KVM_REG_S390_PP:
2276                 r = get_user(vcpu->arch.sie_block->pp,
2277                              (u64 __user *)reg->addr);
2278                 break;
2279         case KVM_REG_S390_GBEA:
2280                 r = get_user(vcpu->arch.sie_block->gbea,
2281                              (u64 __user *)reg->addr);
2282                 break;
2283         default:
2284                 break;
2285         }
2286
2287         return r;
2288 }
2289
2290 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2291 {
2292         kvm_s390_vcpu_initial_reset(vcpu);
2293         return 0;
2294 }
2295
2296 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2297 {
2298         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2299         return 0;
2300 }
2301
2302 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2303 {
2304         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2305         return 0;
2306 }
2307
2308 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2309                                   struct kvm_sregs *sregs)
2310 {
2311         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2312         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2313         return 0;
2314 }
2315
2316 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2317                                   struct kvm_sregs *sregs)
2318 {
2319         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2320         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2321         return 0;
2322 }
2323
2324 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2325 {
2326         if (test_fp_ctl(fpu->fpc))
2327                 return -EINVAL;
2328         vcpu->run->s.regs.fpc = fpu->fpc;
2329         if (MACHINE_HAS_VX)
2330                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2331                                  (freg_t *) fpu->fprs);
2332         else
2333                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2334         return 0;
2335 }
2336
2337 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2338 {
2339         /* make sure we have the latest values */
2340         save_fpu_regs();
2341         if (MACHINE_HAS_VX)
2342                 convert_vx_to_fp((freg_t *) fpu->fprs,
2343                                  (__vector128 *) vcpu->run->s.regs.vrs);
2344         else
2345                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2346         fpu->fpc = vcpu->run->s.regs.fpc;
2347         return 0;
2348 }
2349
2350 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2351 {
2352         int rc = 0;
2353
2354         if (!is_vcpu_stopped(vcpu))
2355                 rc = -EBUSY;
2356         else {
2357                 vcpu->run->psw_mask = psw.mask;
2358                 vcpu->run->psw_addr = psw.addr;
2359         }
2360         return rc;
2361 }
2362
2363 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2364                                   struct kvm_translation *tr)
2365 {
2366         return -EINVAL; /* not implemented yet */
2367 }
2368
2369 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2370                               KVM_GUESTDBG_USE_HW_BP | \
2371                               KVM_GUESTDBG_ENABLE)
2372
2373 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2374                                         struct kvm_guest_debug *dbg)
2375 {
2376         int rc = 0;
2377
2378         vcpu->guest_debug = 0;
2379         kvm_s390_clear_bp_data(vcpu);
2380
2381         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2382                 return -EINVAL;
2383         if (!sclp.has_gpere)
2384                 return -EINVAL;
2385
2386         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2387                 vcpu->guest_debug = dbg->control;
2388                 /* enforce guest PER */
2389                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2390
2391                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2392                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2393         } else {
2394                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2395                 vcpu->arch.guestdbg.last_bp = 0;
2396         }
2397
2398         if (rc) {
2399                 vcpu->guest_debug = 0;
2400                 kvm_s390_clear_bp_data(vcpu);
2401                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2402         }
2403
2404         return rc;
2405 }
2406
2407 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2408                                     struct kvm_mp_state *mp_state)
2409 {
2410         /* CHECK_STOP and LOAD are not supported yet */
2411         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2412                                        KVM_MP_STATE_OPERATING;
2413 }
2414
2415 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2416                                     struct kvm_mp_state *mp_state)
2417 {
2418         int rc = 0;
2419
2420         /* user space knows about this interface - let it control the state */
2421         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2422
2423         switch (mp_state->mp_state) {
2424         case KVM_MP_STATE_STOPPED:
2425                 kvm_s390_vcpu_stop(vcpu);
2426                 break;
2427         case KVM_MP_STATE_OPERATING:
2428                 kvm_s390_vcpu_start(vcpu);
2429                 break;
2430         case KVM_MP_STATE_LOAD:
2431         case KVM_MP_STATE_CHECK_STOP:
2432                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2433         default:
2434                 rc = -ENXIO;
2435         }
2436
2437         return rc;
2438 }
2439
2440 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2441 {
2442         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2443 }
2444
2445 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2446 {
2447 retry:
2448         kvm_s390_vcpu_request_handled(vcpu);
2449         if (!vcpu->requests)
2450                 return 0;
2451         /*
2452          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2453          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2454          * This ensures that the ipte instruction for this request has
2455          * already finished. We might race against a second unmapper that
2456          * wants to set the blocking bit. Lets just retry the request loop.
2457          */
2458         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2459                 int rc;
2460                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2461                                           kvm_s390_get_prefix(vcpu),
2462                                           PAGE_SIZE * 2, PROT_WRITE);
2463                 if (rc) {
2464                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2465                         return rc;
2466                 }
2467                 goto retry;
2468         }
2469
2470         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2471                 vcpu->arch.sie_block->ihcpu = 0xffff;
2472                 goto retry;
2473         }
2474
2475         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2476                 if (!ibs_enabled(vcpu)) {
2477                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2478                         atomic_or(CPUSTAT_IBS,
2479                                         &vcpu->arch.sie_block->cpuflags);
2480                 }
2481                 goto retry;
2482         }
2483
2484         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2485                 if (ibs_enabled(vcpu)) {
2486                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2487                         atomic_andnot(CPUSTAT_IBS,
2488                                           &vcpu->arch.sie_block->cpuflags);
2489                 }
2490                 goto retry;
2491         }
2492
2493         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2494                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2495                 goto retry;
2496         }
2497
2498         /* nothing to do, just clear the request */
2499         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
2500
2501         return 0;
2502 }
2503
2504 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2505 {
2506         struct kvm_vcpu *vcpu;
2507         int i;
2508
2509         mutex_lock(&kvm->lock);
2510         preempt_disable();
2511         kvm->arch.epoch = tod - get_tod_clock();
2512         kvm_s390_vcpu_block_all(kvm);
2513         kvm_for_each_vcpu(i, vcpu, kvm)
2514                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2515         kvm_s390_vcpu_unblock_all(kvm);
2516         preempt_enable();
2517         mutex_unlock(&kvm->lock);
2518 }
2519
2520 /**
2521  * kvm_arch_fault_in_page - fault-in guest page if necessary
2522  * @vcpu: The corresponding virtual cpu
2523  * @gpa: Guest physical address
2524  * @writable: Whether the page should be writable or not
2525  *
2526  * Make sure that a guest page has been faulted-in on the host.
2527  *
2528  * Return: Zero on success, negative error code otherwise.
2529  */
2530 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2531 {
2532         return gmap_fault(vcpu->arch.gmap, gpa,
2533                           writable ? FAULT_FLAG_WRITE : 0);
2534 }
2535
2536 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2537                                       unsigned long token)
2538 {
2539         struct kvm_s390_interrupt inti;
2540         struct kvm_s390_irq irq;
2541
2542         if (start_token) {
2543                 irq.u.ext.ext_params2 = token;
2544                 irq.type = KVM_S390_INT_PFAULT_INIT;
2545                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2546         } else {
2547                 inti.type = KVM_S390_INT_PFAULT_DONE;
2548                 inti.parm64 = token;
2549                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2550         }
2551 }
2552
2553 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2554                                      struct kvm_async_pf *work)
2555 {
2556         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2557         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2558 }
2559
2560 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2561                                  struct kvm_async_pf *work)
2562 {
2563         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2564         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2565 }
2566
2567 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2568                                struct kvm_async_pf *work)
2569 {
2570         /* s390 will always inject the page directly */
2571 }
2572
2573 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2574 {
2575         /*
2576          * s390 will always inject the page directly,
2577          * but we still want check_async_completion to cleanup
2578          */
2579         return true;
2580 }
2581
2582 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2583 {
2584         hva_t hva;
2585         struct kvm_arch_async_pf arch;
2586         int rc;
2587
2588         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2589                 return 0;
2590         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2591             vcpu->arch.pfault_compare)
2592                 return 0;
2593         if (psw_extint_disabled(vcpu))
2594                 return 0;
2595         if (kvm_s390_vcpu_has_irq(vcpu, 0))
2596                 return 0;
2597         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2598                 return 0;
2599         if (!vcpu->arch.gmap->pfault_enabled)
2600                 return 0;
2601
2602         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2603         hva += current->thread.gmap_addr & ~PAGE_MASK;
2604         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2605                 return 0;
2606
2607         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2608         return rc;
2609 }
2610
2611 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2612 {
2613         int rc, cpuflags;
2614
2615         /*
2616          * On s390 notifications for arriving pages will be delivered directly
2617          * to the guest but the house keeping for completed pfaults is
2618          * handled outside the worker.
2619          */
2620         kvm_check_async_pf_completion(vcpu);
2621
2622         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2623         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2624
2625         if (need_resched())
2626                 schedule();
2627
2628         if (test_cpu_flag(CIF_MCCK_PENDING))
2629                 s390_handle_mcck();
2630
2631         if (!kvm_is_ucontrol(vcpu->kvm)) {
2632                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2633                 if (rc)
2634                         return rc;
2635         }
2636
2637         rc = kvm_s390_handle_requests(vcpu);
2638         if (rc)
2639                 return rc;
2640
2641         if (guestdbg_enabled(vcpu)) {
2642                 kvm_s390_backup_guest_per_regs(vcpu);
2643                 kvm_s390_patch_guest_per_regs(vcpu);
2644         }
2645
2646         vcpu->arch.sie_block->icptcode = 0;
2647         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2648         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2649         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2650
2651         return 0;
2652 }
2653
2654 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2655 {
2656         struct kvm_s390_pgm_info pgm_info = {
2657                 .code = PGM_ADDRESSING,
2658         };
2659         u8 opcode, ilen;
2660         int rc;
2661
2662         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2663         trace_kvm_s390_sie_fault(vcpu);
2664
2665         /*
2666          * We want to inject an addressing exception, which is defined as a
2667          * suppressing or terminating exception. However, since we came here
2668          * by a DAT access exception, the PSW still points to the faulting
2669          * instruction since DAT exceptions are nullifying. So we've got
2670          * to look up the current opcode to get the length of the instruction
2671          * to be able to forward the PSW.
2672          */
2673         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
2674         ilen = insn_length(opcode);
2675         if (rc < 0) {
2676                 return rc;
2677         } else if (rc) {
2678                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2679                  * Forward by arbitrary ilc, injection will take care of
2680                  * nullification if necessary.
2681                  */
2682                 pgm_info = vcpu->arch.pgm;
2683                 ilen = 4;
2684         }
2685         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2686         kvm_s390_forward_psw(vcpu, ilen);
2687         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2688 }
2689
2690 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2691 {
2692         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2693                    vcpu->arch.sie_block->icptcode);
2694         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2695
2696         if (guestdbg_enabled(vcpu))
2697                 kvm_s390_restore_guest_per_regs(vcpu);
2698
2699         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2700         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2701
2702         if (vcpu->arch.sie_block->icptcode > 0) {
2703                 int rc = kvm_handle_sie_intercept(vcpu);
2704
2705                 if (rc != -EOPNOTSUPP)
2706                         return rc;
2707                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2708                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2709                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2710                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2711                 return -EREMOTE;
2712         } else if (exit_reason != -EFAULT) {
2713                 vcpu->stat.exit_null++;
2714                 return 0;
2715         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2716                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2717                 vcpu->run->s390_ucontrol.trans_exc_code =
2718                                                 current->thread.gmap_addr;
2719                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2720                 return -EREMOTE;
2721         } else if (current->thread.gmap_pfault) {
2722                 trace_kvm_s390_major_guest_pfault(vcpu);
2723                 current->thread.gmap_pfault = 0;
2724                 if (kvm_arch_setup_async_pf(vcpu))
2725                         return 0;
2726                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2727         }
2728         return vcpu_post_run_fault_in_sie(vcpu);
2729 }
2730
2731 static int __vcpu_run(struct kvm_vcpu *vcpu)
2732 {
2733         int rc, exit_reason;
2734
2735         /*
2736          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2737          * ning the guest), so that memslots (and other stuff) are protected
2738          */
2739         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2740
2741         do {
2742                 rc = vcpu_pre_run(vcpu);
2743                 if (rc)
2744                         break;
2745
2746                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2747                 /*
2748                  * As PF_VCPU will be used in fault handler, between
2749                  * guest_enter and guest_exit should be no uaccess.
2750                  */
2751                 local_irq_disable();
2752                 guest_enter_irqoff();
2753                 __disable_cpu_timer_accounting(vcpu);
2754                 local_irq_enable();
2755                 exit_reason = sie64a(vcpu->arch.sie_block,
2756                                      vcpu->run->s.regs.gprs);
2757                 local_irq_disable();
2758                 __enable_cpu_timer_accounting(vcpu);
2759                 guest_exit_irqoff();
2760                 local_irq_enable();
2761                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2762
2763                 rc = vcpu_post_run(vcpu, exit_reason);
2764         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2765
2766         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2767         return rc;
2768 }
2769
2770 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2771 {
2772         struct runtime_instr_cb *riccb;
2773         struct gs_cb *gscb;
2774
2775         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
2776         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
2777         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2778         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2779         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2780                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2781         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2782                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2783                 /* some control register changes require a tlb flush */
2784                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2785         }
2786         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2787                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2788                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2789                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2790                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2791                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2792         }
2793         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2794                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2795                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2796                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2797                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2798                         kvm_clear_async_pf_completion_queue(vcpu);
2799         }
2800         /*
2801          * If userspace sets the riccb (e.g. after migration) to a valid state,
2802          * we should enable RI here instead of doing the lazy enablement.
2803          */
2804         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
2805             test_kvm_facility(vcpu->kvm, 64) &&
2806             riccb->valid &&
2807             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
2808                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
2809                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
2810         }
2811         /*
2812          * If userspace sets the gscb (e.g. after migration) to non-zero,
2813          * we should enable GS here instead of doing the lazy enablement.
2814          */
2815         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
2816             test_kvm_facility(vcpu->kvm, 133) &&
2817             gscb->gssm &&
2818             !vcpu->arch.gs_enabled) {
2819                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
2820                 vcpu->arch.sie_block->ecb |= ECB_GS;
2821                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2822                 vcpu->arch.gs_enabled = 1;
2823         }
2824         save_access_regs(vcpu->arch.host_acrs);
2825         restore_access_regs(vcpu->run->s.regs.acrs);
2826         /* save host (userspace) fprs/vrs */
2827         save_fpu_regs();
2828         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
2829         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
2830         if (MACHINE_HAS_VX)
2831                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
2832         else
2833                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
2834         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
2835         if (test_fp_ctl(current->thread.fpu.fpc))
2836                 /* User space provided an invalid FPC, let's clear it */
2837                 current->thread.fpu.fpc = 0;
2838         if (MACHINE_HAS_GS) {
2839                 preempt_disable();
2840                 __ctl_set_bit(2, 4);
2841                 if (current->thread.gs_cb) {
2842                         vcpu->arch.host_gscb = current->thread.gs_cb;
2843                         save_gs_cb(vcpu->arch.host_gscb);
2844                 }
2845                 if (vcpu->arch.gs_enabled) {
2846                         current->thread.gs_cb = (struct gs_cb *)
2847                                                 &vcpu->run->s.regs.gscb;
2848                         restore_gs_cb(current->thread.gs_cb);
2849                 }
2850                 preempt_enable();
2851         }
2852
2853         kvm_run->kvm_dirty_regs = 0;
2854 }
2855
2856 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2857 {
2858         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2859         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2860         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2861         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2862         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2863         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2864         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2865         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2866         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2867         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2868         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2869         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2870         save_access_regs(vcpu->run->s.regs.acrs);
2871         restore_access_regs(vcpu->arch.host_acrs);
2872         /* Save guest register state */
2873         save_fpu_regs();
2874         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2875         /* Restore will be done lazily at return */
2876         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
2877         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
2878         if (MACHINE_HAS_GS) {
2879                 __ctl_set_bit(2, 4);
2880                 if (vcpu->arch.gs_enabled)
2881                         save_gs_cb(current->thread.gs_cb);
2882                 preempt_disable();
2883                 current->thread.gs_cb = vcpu->arch.host_gscb;
2884                 restore_gs_cb(vcpu->arch.host_gscb);
2885                 preempt_enable();
2886                 if (!vcpu->arch.host_gscb)
2887                         __ctl_clear_bit(2, 4);
2888                 vcpu->arch.host_gscb = NULL;
2889         }
2890
2891 }
2892
2893 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2894 {
2895         int rc;
2896         sigset_t sigsaved;
2897
2898         if (kvm_run->immediate_exit)
2899                 return -EINTR;
2900
2901         if (guestdbg_exit_pending(vcpu)) {
2902                 kvm_s390_prepare_debug_exit(vcpu);
2903                 return 0;
2904         }
2905
2906         if (vcpu->sigset_active)
2907                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2908
2909         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2910                 kvm_s390_vcpu_start(vcpu);
2911         } else if (is_vcpu_stopped(vcpu)) {
2912                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2913                                    vcpu->vcpu_id);
2914                 return -EINVAL;
2915         }
2916
2917         sync_regs(vcpu, kvm_run);
2918         enable_cpu_timer_accounting(vcpu);
2919
2920         might_fault();
2921         rc = __vcpu_run(vcpu);
2922
2923         if (signal_pending(current) && !rc) {
2924                 kvm_run->exit_reason = KVM_EXIT_INTR;
2925                 rc = -EINTR;
2926         }
2927
2928         if (guestdbg_exit_pending(vcpu) && !rc)  {
2929                 kvm_s390_prepare_debug_exit(vcpu);
2930                 rc = 0;
2931         }
2932
2933         if (rc == -EREMOTE) {
2934                 /* userspace support is needed, kvm_run has been prepared */
2935                 rc = 0;
2936         }
2937
2938         disable_cpu_timer_accounting(vcpu);
2939         store_regs(vcpu, kvm_run);
2940
2941         if (vcpu->sigset_active)
2942                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2943
2944         vcpu->stat.exit_userspace++;
2945         return rc;
2946 }
2947
2948 /*
2949  * store status at address
2950  * we use have two special cases:
2951  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2952  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2953  */
2954 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2955 {
2956         unsigned char archmode = 1;
2957         freg_t fprs[NUM_FPRS];
2958         unsigned int px;
2959         u64 clkcomp, cputm;
2960         int rc;
2961
2962         px = kvm_s390_get_prefix(vcpu);
2963         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2964                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2965                         return -EFAULT;
2966                 gpa = 0;
2967         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2968                 if (write_guest_real(vcpu, 163, &archmode, 1))
2969                         return -EFAULT;
2970                 gpa = px;
2971         } else
2972                 gpa -= __LC_FPREGS_SAVE_AREA;
2973
2974         /* manually convert vector registers if necessary */
2975         if (MACHINE_HAS_VX) {
2976                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2977                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2978                                      fprs, 128);
2979         } else {
2980                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2981                                      vcpu->run->s.regs.fprs, 128);
2982         }
2983         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2984                               vcpu->run->s.regs.gprs, 128);
2985         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2986                               &vcpu->arch.sie_block->gpsw, 16);
2987         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2988                               &px, 4);
2989         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2990                               &vcpu->run->s.regs.fpc, 4);
2991         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2992                               &vcpu->arch.sie_block->todpr, 4);
2993         cputm = kvm_s390_get_cpu_timer(vcpu);
2994         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2995                               &cputm, 8);
2996         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2997         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2998                               &clkcomp, 8);
2999         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3000                               &vcpu->run->s.regs.acrs, 64);
3001         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3002                               &vcpu->arch.sie_block->gcr, 128);
3003         return rc ? -EFAULT : 0;
3004 }
3005
3006 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3007 {
3008         /*
3009          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3010          * switch in the run ioctl. Let's update our copies before we save
3011          * it into the save area
3012          */
3013         save_fpu_regs();
3014         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3015         save_access_regs(vcpu->run->s.regs.acrs);
3016
3017         return kvm_s390_store_status_unloaded(vcpu, addr);
3018 }
3019
3020 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3021 {
3022         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3023         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3024 }
3025
3026 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3027 {
3028         unsigned int i;
3029         struct kvm_vcpu *vcpu;
3030
3031         kvm_for_each_vcpu(i, vcpu, kvm) {
3032                 __disable_ibs_on_vcpu(vcpu);
3033         }
3034 }
3035
3036 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3037 {
3038         if (!sclp.has_ibs)
3039                 return;
3040         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3041         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3042 }
3043
3044 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3045 {
3046         int i, online_vcpus, started_vcpus = 0;
3047
3048         if (!is_vcpu_stopped(vcpu))
3049                 return;
3050
3051         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3052         /* Only one cpu at a time may enter/leave the STOPPED state. */
3053         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3054         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3055
3056         for (i = 0; i < online_vcpus; i++) {
3057                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3058                         started_vcpus++;
3059         }
3060
3061         if (started_vcpus == 0) {
3062                 /* we're the only active VCPU -> speed it up */
3063                 __enable_ibs_on_vcpu(vcpu);
3064         } else if (started_vcpus == 1) {
3065                 /*
3066                  * As we are starting a second VCPU, we have to disable
3067                  * the IBS facility on all VCPUs to remove potentially
3068                  * oustanding ENABLE requests.
3069                  */
3070                 __disable_ibs_on_all_vcpus(vcpu->kvm);
3071         }
3072
3073         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3074         /*
3075          * Another VCPU might have used IBS while we were offline.
3076          * Let's play safe and flush the VCPU at startup.
3077          */
3078         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3079         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3080         return;
3081 }
3082
3083 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3084 {
3085         int i, online_vcpus, started_vcpus = 0;
3086         struct kvm_vcpu *started_vcpu = NULL;
3087
3088         if (is_vcpu_stopped(vcpu))
3089                 return;
3090
3091         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3092         /* Only one cpu at a time may enter/leave the STOPPED state. */
3093         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3094         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3095
3096         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3097         kvm_s390_clear_stop_irq(vcpu);
3098
3099         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3100         __disable_ibs_on_vcpu(vcpu);
3101
3102         for (i = 0; i < online_vcpus; i++) {
3103                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3104                         started_vcpus++;
3105                         started_vcpu = vcpu->kvm->vcpus[i];
3106                 }
3107         }
3108
3109         if (started_vcpus == 1) {
3110                 /*
3111                  * As we only have one VCPU left, we want to enable the
3112                  * IBS facility for that VCPU to speed it up.
3113                  */
3114                 __enable_ibs_on_vcpu(started_vcpu);
3115         }
3116
3117         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3118         return;
3119 }
3120
3121 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3122                                      struct kvm_enable_cap *cap)
3123 {
3124         int r;
3125
3126         if (cap->flags)
3127                 return -EINVAL;
3128
3129         switch (cap->cap) {
3130         case KVM_CAP_S390_CSS_SUPPORT:
3131                 if (!vcpu->kvm->arch.css_support) {
3132                         vcpu->kvm->arch.css_support = 1;
3133                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3134                         trace_kvm_s390_enable_css(vcpu->kvm);
3135                 }
3136                 r = 0;
3137                 break;
3138         default:
3139                 r = -EINVAL;
3140                 break;
3141         }
3142         return r;
3143 }
3144
3145 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3146                                   struct kvm_s390_mem_op *mop)
3147 {
3148         void __user *uaddr = (void __user *)mop->buf;
3149         void *tmpbuf = NULL;
3150         int r, srcu_idx;
3151         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3152                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
3153
3154         if (mop->flags & ~supported_flags)
3155                 return -EINVAL;
3156
3157         if (mop->size > MEM_OP_MAX_SIZE)
3158                 return -E2BIG;
3159
3160         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3161                 tmpbuf = vmalloc(mop->size);
3162                 if (!tmpbuf)
3163                         return -ENOMEM;
3164         }
3165
3166         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3167
3168         switch (mop->op) {
3169         case KVM_S390_MEMOP_LOGICAL_READ:
3170                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3171                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3172                                             mop->size, GACC_FETCH);
3173                         break;
3174                 }
3175                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3176                 if (r == 0) {
3177                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3178                                 r = -EFAULT;
3179                 }
3180                 break;
3181         case KVM_S390_MEMOP_LOGICAL_WRITE:
3182                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3183                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3184                                             mop->size, GACC_STORE);
3185                         break;
3186                 }
3187                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3188                         r = -EFAULT;
3189                         break;
3190                 }
3191                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3192                 break;
3193         default:
3194                 r = -EINVAL;
3195         }
3196
3197         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3198
3199         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3200                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3201
3202         vfree(tmpbuf);
3203         return r;
3204 }
3205
3206 long kvm_arch_vcpu_ioctl(struct file *filp,
3207                          unsigned int ioctl, unsigned long arg)
3208 {
3209         struct kvm_vcpu *vcpu = filp->private_data;
3210         void __user *argp = (void __user *)arg;
3211         int idx;
3212         long r;
3213
3214         switch (ioctl) {
3215         case KVM_S390_IRQ: {
3216                 struct kvm_s390_irq s390irq;
3217
3218                 r = -EFAULT;
3219                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3220                         break;
3221                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3222                 break;
3223         }
3224         case KVM_S390_INTERRUPT: {
3225                 struct kvm_s390_interrupt s390int;
3226                 struct kvm_s390_irq s390irq;
3227
3228                 r = -EFAULT;
3229                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3230                         break;
3231                 if (s390int_to_s390irq(&s390int, &s390irq))
3232                         return -EINVAL;
3233                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3234                 break;
3235         }
3236         case KVM_S390_STORE_STATUS:
3237                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3238                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3239                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3240                 break;
3241         case KVM_S390_SET_INITIAL_PSW: {
3242                 psw_t psw;
3243
3244                 r = -EFAULT;
3245                 if (copy_from_user(&psw, argp, sizeof(psw)))
3246                         break;
3247                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3248                 break;
3249         }
3250         case KVM_S390_INITIAL_RESET:
3251                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3252                 break;
3253         case KVM_SET_ONE_REG:
3254         case KVM_GET_ONE_REG: {
3255                 struct kvm_one_reg reg;
3256                 r = -EFAULT;
3257                 if (copy_from_user(&reg, argp, sizeof(reg)))
3258                         break;
3259                 if (ioctl == KVM_SET_ONE_REG)
3260                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3261                 else
3262                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3263                 break;
3264         }
3265 #ifdef CONFIG_KVM_S390_UCONTROL
3266         case KVM_S390_UCAS_MAP: {
3267                 struct kvm_s390_ucas_mapping ucasmap;
3268
3269                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3270                         r = -EFAULT;
3271                         break;
3272                 }
3273
3274                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3275                         r = -EINVAL;
3276                         break;
3277                 }
3278
3279                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3280                                      ucasmap.vcpu_addr, ucasmap.length);
3281                 break;
3282         }
3283         case KVM_S390_UCAS_UNMAP: {
3284                 struct kvm_s390_ucas_mapping ucasmap;
3285
3286                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3287                         r = -EFAULT;
3288                         break;
3289                 }
3290
3291                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3292                         r = -EINVAL;
3293                         break;
3294                 }
3295
3296                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3297                         ucasmap.length);
3298                 break;
3299         }
3300 #endif
3301         case KVM_S390_VCPU_FAULT: {
3302                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3303                 break;
3304         }
3305         case KVM_ENABLE_CAP:
3306         {
3307                 struct kvm_enable_cap cap;
3308                 r = -EFAULT;
3309                 if (copy_from_user(&cap, argp, sizeof(cap)))
3310                         break;
3311                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3312                 break;
3313         }
3314         case KVM_S390_MEM_OP: {
3315                 struct kvm_s390_mem_op mem_op;
3316
3317                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3318                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3319                 else
3320                         r = -EFAULT;
3321                 break;
3322         }
3323         case KVM_S390_SET_IRQ_STATE: {
3324                 struct kvm_s390_irq_state irq_state;
3325
3326                 r = -EFAULT;
3327                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3328                         break;
3329                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3330                     irq_state.len == 0 ||
3331                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3332                         r = -EINVAL;
3333                         break;
3334                 }
3335                 r = kvm_s390_set_irq_state(vcpu,
3336                                            (void __user *) irq_state.buf,
3337                                            irq_state.len);
3338                 break;
3339         }
3340         case KVM_S390_GET_IRQ_STATE: {
3341                 struct kvm_s390_irq_state irq_state;
3342
3343                 r = -EFAULT;
3344                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3345                         break;
3346                 if (irq_state.len == 0) {
3347                         r = -EINVAL;
3348                         break;
3349                 }
3350                 r = kvm_s390_get_irq_state(vcpu,
3351                                            (__u8 __user *)  irq_state.buf,
3352                                            irq_state.len);
3353                 break;
3354         }
3355         default:
3356                 r = -ENOTTY;
3357         }
3358         return r;
3359 }
3360
3361 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3362 {
3363 #ifdef CONFIG_KVM_S390_UCONTROL
3364         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3365                  && (kvm_is_ucontrol(vcpu->kvm))) {
3366                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3367                 get_page(vmf->page);
3368                 return 0;
3369         }
3370 #endif
3371         return VM_FAULT_SIGBUS;
3372 }
3373
3374 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3375                             unsigned long npages)
3376 {
3377         return 0;
3378 }
3379
3380 /* Section: memory related */
3381 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3382                                    struct kvm_memory_slot *memslot,
3383                                    const struct kvm_userspace_memory_region *mem,
3384                                    enum kvm_mr_change change)
3385 {
3386         /* A few sanity checks. We can have memory slots which have to be
3387            located/ended at a segment boundary (1MB). The memory in userland is
3388            ok to be fragmented into various different vmas. It is okay to mmap()
3389            and munmap() stuff in this slot after doing this call at any time */
3390
3391         if (mem->userspace_addr & 0xffffful)
3392                 return -EINVAL;
3393
3394         if (mem->memory_size & 0xffffful)
3395                 return -EINVAL;
3396
3397         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3398                 return -EINVAL;
3399
3400         return 0;
3401 }
3402
3403 void kvm_arch_commit_memory_region(struct kvm *kvm,
3404                                 const struct kvm_userspace_memory_region *mem,
3405                                 const struct kvm_memory_slot *old,
3406                                 const struct kvm_memory_slot *new,
3407                                 enum kvm_mr_change change)
3408 {
3409         int rc;
3410
3411         /* If the basics of the memslot do not change, we do not want
3412          * to update the gmap. Every update causes several unnecessary
3413          * segment translation exceptions. This is usually handled just
3414          * fine by the normal fault handler + gmap, but it will also
3415          * cause faults on the prefix page of running guest CPUs.
3416          */
3417         if (old->userspace_addr == mem->userspace_addr &&
3418             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3419             old->npages * PAGE_SIZE == mem->memory_size)
3420                 return;
3421
3422         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3423                 mem->guest_phys_addr, mem->memory_size);
3424         if (rc)
3425                 pr_warn("failed to commit memory region\n");
3426         return;
3427 }
3428
3429 static inline unsigned long nonhyp_mask(int i)
3430 {
3431         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3432
3433         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3434 }
3435
3436 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3437 {
3438         vcpu->valid_wakeup = false;
3439 }
3440
3441 static int __init kvm_s390_init(void)
3442 {
3443         int i;
3444
3445         if (!sclp.has_sief2) {
3446                 pr_info("SIE not available\n");
3447                 return -ENODEV;
3448         }
3449
3450         for (i = 0; i < 16; i++)
3451                 kvm_s390_fac_list_mask[i] |=
3452                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3453
3454         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3455 }
3456
3457 static void __exit kvm_s390_exit(void)
3458 {
3459         kvm_exit();
3460 }
3461
3462 module_init(kvm_s390_init);
3463 module_exit(kvm_s390_exit);
3464
3465 /*
3466  * Enable autoloading of the kvm module.
3467  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3468  * since x86 takes a different approach.
3469  */
3470 #include <linux/miscdevice.h>
3471 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3472 MODULE_ALIAS("devname:kvm");