arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/random.h>
  26 #include <linux/slab.h>
  27 #include <linux/timer.h>
  28 #include <linux/vmalloc.h>
  29 #include <asm/asm-offsets.h>
  30 #include <asm/lowcore.h>
  31 #include <asm/etr.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/nmi.h>
  34 #include <asm/switch_to.h>
  35 #include <asm/isc.h>
  36 #include <asm/sclp.h>
  37 #include "kvm-s390.h"
  38 #include "gaccess.h"
  39
  40 #define KMSG_COMPONENT "kvm-s390"
  41 #undef pr_fmt
  42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  43
  44 #define CREATE_TRACE_POINTS
  45 #include "trace.h"
  46 #include "trace-s390.h"
  47
  48 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  49 #define LOCAL_IRQS 32
  50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  51                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  52
  53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  54
  55 struct kvm_stats_debugfs_item debugfs_entries[] = {
  56         { "userspace_handled", VCPU_STAT(exit_userspace) },
  57         { "exit_null", VCPU_STAT(exit_null) },
  58         { "exit_validity", VCPU_STAT(exit_validity) },
  59         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  60         { "exit_external_request", VCPU_STAT(exit_external_request) },
  61         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  62         { "exit_instruction", VCPU_STAT(exit_instruction) },
  63         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  64         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  65         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  66         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  67         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  68         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  69         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  70         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  71         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  72         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  73         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  74         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  75         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  76         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  77         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  78         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  79         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  80         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  81         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  82         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  83         { "instruction_spx", VCPU_STAT(instruction_spx) },
  84         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  85         { "instruction_stap", VCPU_STAT(instruction_stap) },
  86         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  87         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  88         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  89         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  90         { "instruction_essa", VCPU_STAT(instruction_essa) },
  91         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  92         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  93         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  94         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  95         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  96         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  97         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  98         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  99         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 100         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 101         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 102         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 103         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 104         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 105         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 106         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 107         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 108         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 109         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 110         { "diagnose_10", VCPU_STAT(diagnose_10) },
 111         { "diagnose_44", VCPU_STAT(diagnose_44) },
 112         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 113         { "diagnose_258", VCPU_STAT(diagnose_258) },
 114         { "diagnose_308", VCPU_STAT(diagnose_308) },
 115         { "diagnose_500", VCPU_STAT(diagnose_500) },
 116         { NULL }
 117 };
 118
 119 /* upper facilities limit for kvm */
 120 unsigned long kvm_s390_fac_list_mask[] = {
 121         0xffe6fffbfcfdfc40UL,
 122         0x005e800000000000UL,
 123 };
 124
 125 unsigned long kvm_s390_fac_list_mask_size(void)
 126 {
 127         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 128         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 129 }
 130
 131 static struct gmap_notifier gmap_notifier;
 132 debug_info_t *kvm_s390_dbf;
 133
 134 /* Section: not file related */
 135 int kvm_arch_hardware_enable(void)
 136 {
 137         /* every s390 is virtualization enabled ;-) */
 138         return 0;
 139 }
 140
 141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 142
 143 /*
 144  * This callback is executed during stop_machine(). All CPUs are therefore
 145  * temporarily stopped. In order not to change guest behavior, we have to
 146  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 147  * so a CPU won't be stopped while calculating with the epoch.
 148  */
 149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 150                           void *v)
 151 {
 152         struct kvm *kvm;
 153         struct kvm_vcpu *vcpu;
 154         int i;
 155         unsigned long long *delta = v;
 156
 157         list_for_each_entry(kvm, &vm_list, vm_list) {
 158                 kvm->arch.epoch -= *delta;
 159                 kvm_for_each_vcpu(i, vcpu, kvm) {
 160                         vcpu->arch.sie_block->epoch -= *delta;
 161                 }
 162         }
 163         return NOTIFY_OK;
 164 }
 165
 166 static struct notifier_block kvm_clock_notifier = {
 167         .notifier_call = kvm_clock_sync,
 168 };
 169
 170 int kvm_arch_hardware_setup(void)
 171 {
 172         gmap_notifier.notifier_call = kvm_gmap_notifier;
 173         gmap_register_ipte_notifier(&gmap_notifier);
 174         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 175                                        &kvm_clock_notifier);
 176         return 0;
 177 }
 178
 179 void kvm_arch_hardware_unsetup(void)
 180 {
 181         gmap_unregister_ipte_notifier(&gmap_notifier);
 182         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 183                                          &kvm_clock_notifier);
 184 }
 185
 186 int kvm_arch_init(void *opaque)
 187 {
 188         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 189         if (!kvm_s390_dbf)
 190                 return -ENOMEM;
 191
 192         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 193                 debug_unregister(kvm_s390_dbf);
 194                 return -ENOMEM;
 195         }
 196
 197         /* Register floating interrupt controller interface. */
 198         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 199 }
 200
 201 void kvm_arch_exit(void)
 202 {
 203         debug_unregister(kvm_s390_dbf);
 204 }
 205
 206 /* Section: device related */
 207 long kvm_arch_dev_ioctl(struct file *filp,
 208                         unsigned int ioctl, unsigned long arg)
 209 {
 210         if (ioctl == KVM_S390_ENABLE_SIE)
 211                 return s390_enable_sie();
 212         return -EINVAL;
 213 }
 214
 215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 216 {
 217         int r;
 218
 219         switch (ext) {
 220         case KVM_CAP_S390_PSW:
 221         case KVM_CAP_S390_GMAP:
 222         case KVM_CAP_SYNC_MMU:
 223 #ifdef CONFIG_KVM_S390_UCONTROL
 224         case KVM_CAP_S390_UCONTROL:
 225 #endif
 226         case KVM_CAP_ASYNC_PF:
 227         case KVM_CAP_SYNC_REGS:
 228         case KVM_CAP_ONE_REG:
 229         case KVM_CAP_ENABLE_CAP:
 230         case KVM_CAP_S390_CSS_SUPPORT:
 231         case KVM_CAP_IOEVENTFD:
 232         case KVM_CAP_DEVICE_CTRL:
 233         case KVM_CAP_ENABLE_CAP_VM:
 234         case KVM_CAP_S390_IRQCHIP:
 235         case KVM_CAP_VM_ATTRIBUTES:
 236         case KVM_CAP_MP_STATE:
 237         case KVM_CAP_S390_INJECT_IRQ:
 238         case KVM_CAP_S390_USER_SIGP:
 239         case KVM_CAP_S390_USER_STSI:
 240         case KVM_CAP_S390_SKEYS:
 241         case KVM_CAP_S390_IRQ_STATE:
 242                 r = 1;
 243                 break;
 244         case KVM_CAP_S390_MEM_OP:
 245                 r = MEM_OP_MAX_SIZE;
 246                 break;
 247         case KVM_CAP_NR_VCPUS:
 248         case KVM_CAP_MAX_VCPUS:
 249                 r = KVM_MAX_VCPUS;
 250                 break;
 251         case KVM_CAP_NR_MEMSLOTS:
 252                 r = KVM_USER_MEM_SLOTS;
 253                 break;
 254         case KVM_CAP_S390_COW:
 255                 r = MACHINE_HAS_ESOP;
 256                 break;
 257         case KVM_CAP_S390_VECTOR_REGISTERS:
 258                 r = MACHINE_HAS_VX;
 259                 break;
 260         default:
 261                 r = 0;
 262         }
 263         return r;
 264 }
 265
 266 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 267                                         struct kvm_memory_slot *memslot)
 268 {
 269         gfn_t cur_gfn, last_gfn;
 270         unsigned long address;
 271         struct gmap *gmap = kvm->arch.gmap;
 272
 273         down_read(&gmap->mm->mmap_sem);
 274         /* Loop over all guest pages */
 275         last_gfn = memslot->base_gfn + memslot->npages;
 276         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 277                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 278
 279                 if (gmap_test_and_clear_dirty(address, gmap))
 280                         mark_page_dirty(kvm, cur_gfn);
 281         }
 282         up_read(&gmap->mm->mmap_sem);
 283 }
 284
 285 /* Section: vm related */
 286 /*
 287  * Get (and clear) the dirty memory log for a memory slot.
 288  */
 289 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 290                                struct kvm_dirty_log *log)
 291 {
 292         int r;
 293         unsigned long n;
 294         struct kvm_memslots *slots;
 295         struct kvm_memory_slot *memslot;
 296         int is_dirty = 0;
 297
 298         mutex_lock(&kvm->slots_lock);
 299
 300         r = -EINVAL;
 301         if (log->slot >= KVM_USER_MEM_SLOTS)
 302                 goto out;
 303
 304         slots = kvm_memslots(kvm);
 305         memslot = id_to_memslot(slots, log->slot);
 306         r = -ENOENT;
 307         if (!memslot->dirty_bitmap)
 308                 goto out;
 309
 310         kvm_s390_sync_dirty_log(kvm, memslot);
 311         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 312         if (r)
 313                 goto out;
 314
 315         /* Clear the dirty log */
 316         if (is_dirty) {
 317                 n = kvm_dirty_bitmap_bytes(memslot);
 318                 memset(memslot->dirty_bitmap, 0, n);
 319         }
 320         r = 0;
 321 out:
 322         mutex_unlock(&kvm->slots_lock);
 323         return r;
 324 }
 325
 326 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 327 {
 328         int r;
 329
 330         if (cap->flags)
 331                 return -EINVAL;
 332
 333         switch (cap->cap) {
 334         case KVM_CAP_S390_IRQCHIP:
 335                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 336                 kvm->arch.use_irqchip = 1;
 337                 r = 0;
 338                 break;
 339         case KVM_CAP_S390_USER_SIGP:
 340                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 341                 kvm->arch.user_sigp = 1;
 342                 r = 0;
 343                 break;
 344         case KVM_CAP_S390_VECTOR_REGISTERS:
 345                 if (MACHINE_HAS_VX) {
 346                         set_kvm_facility(kvm->arch.model.fac->mask, 129);
 347                         set_kvm_facility(kvm->arch.model.fac->list, 129);
 348                         r = 0;
 349                 } else
 350                         r = -EINVAL;
 351                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 352                          r ? "(not available)" : "(success)");
 353                 break;
 354         case KVM_CAP_S390_USER_STSI:
 355                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 356                 kvm->arch.user_stsi = 1;
 357                 r = 0;
 358                 break;
 359         default:
 360                 r = -EINVAL;
 361                 break;
 362         }
 363         return r;
 364 }
 365
 366 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 367 {
 368         int ret;
 369
 370         switch (attr->attr) {
 371         case KVM_S390_VM_MEM_LIMIT_SIZE:
 372                 ret = 0;
 373                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 374                          kvm->arch.gmap->asce_end);
 375                 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
 376                         ret = -EFAULT;
 377                 break;
 378         default:
 379                 ret = -ENXIO;
 380                 break;
 381         }
 382         return ret;
 383 }
 384
 385 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 386 {
 387         int ret;
 388         unsigned int idx;
 389         switch (attr->attr) {
 390         case KVM_S390_VM_MEM_ENABLE_CMMA:
 391                 /* enable CMMA only for z10 and later (EDAT_1) */
 392                 ret = -EINVAL;
 393                 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
 394                         break;
 395
 396                 ret = -EBUSY;
 397                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 398                 mutex_lock(&kvm->lock);
 399                 if (atomic_read(&kvm->online_vcpus) == 0) {
 400                         kvm->arch.use_cmma = 1;
 401                         ret = 0;
 402                 }
 403                 mutex_unlock(&kvm->lock);
 404                 break;
 405         case KVM_S390_VM_MEM_CLR_CMMA:
 406                 ret = -EINVAL;
 407                 if (!kvm->arch.use_cmma)
 408                         break;
 409
 410                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 411                 mutex_lock(&kvm->lock);
 412                 idx = srcu_read_lock(&kvm->srcu);
 413                 s390_reset_cmma(kvm->arch.gmap->mm);
 414                 srcu_read_unlock(&kvm->srcu, idx);
 415                 mutex_unlock(&kvm->lock);
 416                 ret = 0;
 417                 break;
 418         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 419                 unsigned long new_limit;
 420
 421                 if (kvm_is_ucontrol(kvm))
 422                         return -EINVAL;
 423
 424                 if (get_user(new_limit, (u64 __user *)attr->addr))
 425                         return -EFAULT;
 426
 427                 if (new_limit > kvm->arch.gmap->asce_end)
 428                         return -E2BIG;
 429
 430                 ret = -EBUSY;
 431                 mutex_lock(&kvm->lock);
 432                 if (atomic_read(&kvm->online_vcpus) == 0) {
 433                         /* gmap_alloc will round the limit up */
 434                         struct gmap *new = gmap_alloc(current->mm, new_limit);
 435
 436                         if (!new) {
 437                                 ret = -ENOMEM;
 438                         } else {
 439                                 gmap_free(kvm->arch.gmap);
 440                                 new->private = kvm;
 441                                 kvm->arch.gmap = new;
 442                                 ret = 0;
 443                         }
 444                 }
 445                 mutex_unlock(&kvm->lock);
 446                 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
 447                 break;
 448         }
 449         default:
 450                 ret = -ENXIO;
 451                 break;
 452         }
 453         return ret;
 454 }
 455
 456 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 457
 458 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 459 {
 460         struct kvm_vcpu *vcpu;
 461         int i;
 462
 463         if (!test_kvm_facility(kvm, 76))
 464                 return -EINVAL;
 465
 466         mutex_lock(&kvm->lock);
 467         switch (attr->attr) {
 468         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 469                 get_random_bytes(
 470                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 471                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 472                 kvm->arch.crypto.aes_kw = 1;
 473                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 474                 break;
 475         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 476                 get_random_bytes(
 477                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 478                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 479                 kvm->arch.crypto.dea_kw = 1;
 480                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 481                 break;
 482         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 483                 kvm->arch.crypto.aes_kw = 0;
 484                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 485                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 486                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 487                 break;
 488         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 489                 kvm->arch.crypto.dea_kw = 0;
 490                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 491                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 492                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 493                 break;
 494         default:
 495                 mutex_unlock(&kvm->lock);
 496                 return -ENXIO;
 497         }
 498
 499         kvm_for_each_vcpu(i, vcpu, kvm) {
 500                 kvm_s390_vcpu_crypto_setup(vcpu);
 501                 exit_sie(vcpu);
 502         }
 503         mutex_unlock(&kvm->lock);
 504         return 0;
 505 }
 506
 507 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 508 {
 509         u8 gtod_high;
 510
 511         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 512                                            sizeof(gtod_high)))
 513                 return -EFAULT;
 514
 515         if (gtod_high != 0)
 516                 return -EINVAL;
 517         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x\n", gtod_high);
 518
 519         return 0;
 520 }
 521
 522 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 523 {
 524         struct kvm_vcpu *cur_vcpu;
 525         unsigned int vcpu_idx;
 526         u64 host_tod, gtod;
 527         int r;
 528
 529         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 530                 return -EFAULT;
 531
 532         r = store_tod_clock(&host_tod);
 533         if (r)
 534                 return r;
 535
 536         mutex_lock(&kvm->lock);
 537         preempt_disable();
 538         kvm->arch.epoch = gtod - host_tod;
 539         kvm_s390_vcpu_block_all(kvm);
 540         kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm)
 541                 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
 542         kvm_s390_vcpu_unblock_all(kvm);
 543         preempt_enable();
 544         mutex_unlock(&kvm->lock);
 545         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx\n", gtod);
 546         return 0;
 547 }
 548
 549 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 550 {
 551         int ret;
 552
 553         if (attr->flags)
 554                 return -EINVAL;
 555
 556         switch (attr->attr) {
 557         case KVM_S390_VM_TOD_HIGH:
 558                 ret = kvm_s390_set_tod_high(kvm, attr);
 559                 break;
 560         case KVM_S390_VM_TOD_LOW:
 561                 ret = kvm_s390_set_tod_low(kvm, attr);
 562                 break;
 563         default:
 564                 ret = -ENXIO;
 565                 break;
 566         }
 567         return ret;
 568 }
 569
 570 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 571 {
 572         u8 gtod_high = 0;
 573
 574         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 575                                          sizeof(gtod_high)))
 576                 return -EFAULT;
 577         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x\n", gtod_high);
 578
 579         return 0;
 580 }
 581
 582 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 583 {
 584         u64 host_tod, gtod;
 585         int r;
 586
 587         r = store_tod_clock(&host_tod);
 588         if (r)
 589                 return r;
 590
 591         preempt_disable();
 592         gtod = host_tod + kvm->arch.epoch;
 593         preempt_enable();
 594         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 595                 return -EFAULT;
 596         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx\n", gtod);
 597
 598         return 0;
 599 }
 600
 601 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 602 {
 603         int ret;
 604
 605         if (attr->flags)
 606                 return -EINVAL;
 607
 608         switch (attr->attr) {
 609         case KVM_S390_VM_TOD_HIGH:
 610                 ret = kvm_s390_get_tod_high(kvm, attr);
 611                 break;
 612         case KVM_S390_VM_TOD_LOW:
 613                 ret = kvm_s390_get_tod_low(kvm, attr);
 614                 break;
 615         default:
 616                 ret = -ENXIO;
 617                 break;
 618         }
 619         return ret;
 620 }
 621
 622 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 623 {
 624         struct kvm_s390_vm_cpu_processor *proc;
 625         int ret = 0;
 626
 627         mutex_lock(&kvm->lock);
 628         if (atomic_read(&kvm->online_vcpus)) {
 629                 ret = -EBUSY;
 630                 goto out;
 631         }
 632         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 633         if (!proc) {
 634                 ret = -ENOMEM;
 635                 goto out;
 636         }
 637         if (!copy_from_user(proc, (void __user *)attr->addr,
 638                             sizeof(*proc))) {
 639                 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
 640                        sizeof(struct cpuid));
 641                 kvm->arch.model.ibc = proc->ibc;
 642                 memcpy(kvm->arch.model.fac->list, proc->fac_list,
 643                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 644         } else
 645                 ret = -EFAULT;
 646         kfree(proc);
 647 out:
 648         mutex_unlock(&kvm->lock);
 649         return ret;
 650 }
 651
 652 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 653 {
 654         int ret = -ENXIO;
 655
 656         switch (attr->attr) {
 657         case KVM_S390_VM_CPU_PROCESSOR:
 658                 ret = kvm_s390_set_processor(kvm, attr);
 659                 break;
 660         }
 661         return ret;
 662 }
 663
 664 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 665 {
 666         struct kvm_s390_vm_cpu_processor *proc;
 667         int ret = 0;
 668
 669         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 670         if (!proc) {
 671                 ret = -ENOMEM;
 672                 goto out;
 673         }
 674         memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
 675         proc->ibc = kvm->arch.model.ibc;
 676         memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
 677         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 678                 ret = -EFAULT;
 679         kfree(proc);
 680 out:
 681         return ret;
 682 }
 683
 684 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 685 {
 686         struct kvm_s390_vm_cpu_machine *mach;
 687         int ret = 0;
 688
 689         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 690         if (!mach) {
 691                 ret = -ENOMEM;
 692                 goto out;
 693         }
 694         get_cpu_id((struct cpuid *) &mach->cpuid);
 695         mach->ibc = sclp.ibc;
 696         memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
 697                S390_ARCH_FAC_LIST_SIZE_BYTE);
 698         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 699                S390_ARCH_FAC_LIST_SIZE_BYTE);
 700         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 701                 ret = -EFAULT;
 702         kfree(mach);
 703 out:
 704         return ret;
 705 }
 706
 707 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 708 {
 709         int ret = -ENXIO;
 710
 711         switch (attr->attr) {
 712         case KVM_S390_VM_CPU_PROCESSOR:
 713                 ret = kvm_s390_get_processor(kvm, attr);
 714                 break;
 715         case KVM_S390_VM_CPU_MACHINE:
 716                 ret = kvm_s390_get_machine(kvm, attr);
 717                 break;
 718         }
 719         return ret;
 720 }
 721
 722 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 723 {
 724         int ret;
 725
 726         switch (attr->group) {
 727         case KVM_S390_VM_MEM_CTRL:
 728                 ret = kvm_s390_set_mem_control(kvm, attr);
 729                 break;
 730         case KVM_S390_VM_TOD:
 731                 ret = kvm_s390_set_tod(kvm, attr);
 732                 break;
 733         case KVM_S390_VM_CPU_MODEL:
 734                 ret = kvm_s390_set_cpu_model(kvm, attr);
 735                 break;
 736         case KVM_S390_VM_CRYPTO:
 737                 ret = kvm_s390_vm_set_crypto(kvm, attr);
 738                 break;
 739         default:
 740                 ret = -ENXIO;
 741                 break;
 742         }
 743
 744         return ret;
 745 }
 746
 747 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 748 {
 749         int ret;
 750
 751         switch (attr->group) {
 752         case KVM_S390_VM_MEM_CTRL:
 753                 ret = kvm_s390_get_mem_control(kvm, attr);
 754                 break;
 755         case KVM_S390_VM_TOD:
 756                 ret = kvm_s390_get_tod(kvm, attr);
 757                 break;
 758         case KVM_S390_VM_CPU_MODEL:
 759                 ret = kvm_s390_get_cpu_model(kvm, attr);
 760                 break;
 761         default:
 762                 ret = -ENXIO;
 763                 break;
 764         }
 765
 766         return ret;
 767 }
 768
 769 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 770 {
 771         int ret;
 772
 773         switch (attr->group) {
 774         case KVM_S390_VM_MEM_CTRL:
 775                 switch (attr->attr) {
 776                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 777                 case KVM_S390_VM_MEM_CLR_CMMA:
 778                 case KVM_S390_VM_MEM_LIMIT_SIZE:
 779                         ret = 0;
 780                         break;
 781                 default:
 782                         ret = -ENXIO;
 783                         break;
 784                 }
 785                 break;
 786         case KVM_S390_VM_TOD:
 787                 switch (attr->attr) {
 788                 case KVM_S390_VM_TOD_LOW:
 789                 case KVM_S390_VM_TOD_HIGH:
 790                         ret = 0;
 791                         break;
 792                 default:
 793                         ret = -ENXIO;
 794                         break;
 795                 }
 796                 break;
 797         case KVM_S390_VM_CPU_MODEL:
 798                 switch (attr->attr) {
 799                 case KVM_S390_VM_CPU_PROCESSOR:
 800                 case KVM_S390_VM_CPU_MACHINE:
 801                         ret = 0;
 802                         break;
 803                 default:
 804                         ret = -ENXIO;
 805                         break;
 806                 }
 807                 break;
 808         case KVM_S390_VM_CRYPTO:
 809                 switch (attr->attr) {
 810                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 811                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 812                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 813                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 814                         ret = 0;
 815                         break;
 816                 default:
 817                         ret = -ENXIO;
 818                         break;
 819                 }
 820                 break;
 821         default:
 822                 ret = -ENXIO;
 823                 break;
 824         }
 825
 826         return ret;
 827 }
 828
 829 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 830 {
 831         uint8_t *keys;
 832         uint64_t hva;
 833         unsigned long curkey;
 834         int i, r = 0;
 835
 836         if (args->flags != 0)
 837                 return -EINVAL;
 838
 839         /* Is this guest using storage keys? */
 840         if (!mm_use_skey(current->mm))
 841                 return KVM_S390_GET_SKEYS_NONE;
 842
 843         /* Enforce sane limit on memory allocation */
 844         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 845                 return -EINVAL;
 846
 847         keys = kmalloc_array(args->count, sizeof(uint8_t),
 848                              GFP_KERNEL | __GFP_NOWARN);
 849         if (!keys)
 850                 keys = vmalloc(sizeof(uint8_t) * args->count);
 851         if (!keys)
 852                 return -ENOMEM;
 853
 854         for (i = 0; i < args->count; i++) {
 855                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 856                 if (kvm_is_error_hva(hva)) {
 857                         r = -EFAULT;
 858                         goto out;
 859                 }
 860
 861                 curkey = get_guest_storage_key(current->mm, hva);
 862                 if (IS_ERR_VALUE(curkey)) {
 863                         r = curkey;
 864                         goto out;
 865                 }
 866                 keys[i] = curkey;
 867         }
 868
 869         r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
 870                          sizeof(uint8_t) * args->count);
 871         if (r)
 872                 r = -EFAULT;
 873 out:
 874         kvfree(keys);
 875         return r;
 876 }
 877
 878 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 879 {
 880         uint8_t *keys;
 881         uint64_t hva;
 882         int i, r = 0;
 883
 884         if (args->flags != 0)
 885                 return -EINVAL;
 886
 887         /* Enforce sane limit on memory allocation */
 888         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 889                 return -EINVAL;
 890
 891         keys = kmalloc_array(args->count, sizeof(uint8_t),
 892                              GFP_KERNEL | __GFP_NOWARN);
 893         if (!keys)
 894                 keys = vmalloc(sizeof(uint8_t) * args->count);
 895         if (!keys)
 896                 return -ENOMEM;
 897
 898         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
 899                            sizeof(uint8_t) * args->count);
 900         if (r) {
 901                 r = -EFAULT;
 902                 goto out;
 903         }
 904
 905         /* Enable storage key handling for the guest */
 906         r = s390_enable_skey();
 907         if (r)
 908                 goto out;
 909
 910         for (i = 0; i < args->count; i++) {
 911                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 912                 if (kvm_is_error_hva(hva)) {
 913                         r = -EFAULT;
 914                         goto out;
 915                 }
 916
 917                 /* Lowest order bit is reserved */
 918                 if (keys[i] & 0x01) {
 919                         r = -EINVAL;
 920                         goto out;
 921                 }
 922
 923                 r = set_guest_storage_key(current->mm, hva,
 924                                           (unsigned long)keys[i], 0);
 925                 if (r)
 926                         goto out;
 927         }
 928 out:
 929         kvfree(keys);
 930         return r;
 931 }
 932
 933 long kvm_arch_vm_ioctl(struct file *filp,
 934                        unsigned int ioctl, unsigned long arg)
 935 {
 936         struct kvm *kvm = filp->private_data;
 937         void __user *argp = (void __user *)arg;
 938         struct kvm_device_attr attr;
 939         int r;
 940
 941         switch (ioctl) {
 942         case KVM_S390_INTERRUPT: {
 943                 struct kvm_s390_interrupt s390int;
 944
 945                 r = -EFAULT;
 946                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 947                         break;
 948                 r = kvm_s390_inject_vm(kvm, &s390int);
 949                 break;
 950         }
 951         case KVM_ENABLE_CAP: {
 952                 struct kvm_enable_cap cap;
 953                 r = -EFAULT;
 954                 if (copy_from_user(&cap, argp, sizeof(cap)))
 955                         break;
 956                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 957                 break;
 958         }
 959         case KVM_CREATE_IRQCHIP: {
 960                 struct kvm_irq_routing_entry routing;
 961
 962                 r = -EINVAL;
 963                 if (kvm->arch.use_irqchip) {
 964                         /* Set up dummy routing. */
 965                         memset(&routing, 0, sizeof(routing));
 966                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
 967                 }
 968                 break;
 969         }
 970         case KVM_SET_DEVICE_ATTR: {
 971                 r = -EFAULT;
 972                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 973                         break;
 974                 r = kvm_s390_vm_set_attr(kvm, &attr);
 975                 break;
 976         }
 977         case KVM_GET_DEVICE_ATTR: {
 978                 r = -EFAULT;
 979                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 980                         break;
 981                 r = kvm_s390_vm_get_attr(kvm, &attr);
 982                 break;
 983         }
 984         case KVM_HAS_DEVICE_ATTR: {
 985                 r = -EFAULT;
 986                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 987                         break;
 988                 r = kvm_s390_vm_has_attr(kvm, &attr);
 989                 break;
 990         }
 991         case KVM_S390_GET_SKEYS: {
 992                 struct kvm_s390_skeys args;
 993
 994                 r = -EFAULT;
 995                 if (copy_from_user(&args, argp,
 996                                    sizeof(struct kvm_s390_skeys)))
 997                         break;
 998                 r = kvm_s390_get_skeys(kvm, &args);
 999                 break;
1000         }
1001         case KVM_S390_SET_SKEYS: {
1002                 struct kvm_s390_skeys args;
1003
1004                 r = -EFAULT;
1005                 if (copy_from_user(&args, argp,
1006                                    sizeof(struct kvm_s390_skeys)))
1007                         break;
1008                 r = kvm_s390_set_skeys(kvm, &args);
1009                 break;
1010         }
1011         default:
1012                 r = -ENOTTY;
1013         }
1014
1015         return r;
1016 }
1017
1018 static int kvm_s390_query_ap_config(u8 *config)
1019 {
1020         u32 fcn_code = 0x04000000UL;
1021         u32 cc = 0;
1022
1023         memset(config, 0, 128);
1024         asm volatile(
1025                 "lgr 0,%1\n"
1026                 "lgr 2,%2\n"
1027                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1028                 "0: ipm %0\n"
1029                 "srl %0,28\n"
1030                 "1:\n"
1031                 EX_TABLE(0b, 1b)
1032                 : "+r" (cc)
1033                 : "r" (fcn_code), "r" (config)
1034                 : "cc", "0", "2", "memory"
1035         );
1036
1037         return cc;
1038 }
1039
1040 static int kvm_s390_apxa_installed(void)
1041 {
1042         u8 config[128];
1043         int cc;
1044
1045         if (test_facility(2) && test_facility(12)) {
1046                 cc = kvm_s390_query_ap_config(config);
1047
1048                 if (cc)
1049                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1050                 else
1051                         return config[0] & 0x40;
1052         }
1053
1054         return 0;
1055 }
1056
1057 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1058 {
1059         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1060
1061         if (kvm_s390_apxa_installed())
1062                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1063         else
1064                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1065 }
1066
1067 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1068 {
1069         get_cpu_id(cpu_id);
1070         cpu_id->version = 0xff;
1071 }
1072
1073 static int kvm_s390_crypto_init(struct kvm *kvm)
1074 {
1075         if (!test_kvm_facility(kvm, 76))
1076                 return 0;
1077
1078         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1079                                          GFP_KERNEL | GFP_DMA);
1080         if (!kvm->arch.crypto.crycb)
1081                 return -ENOMEM;
1082
1083         kvm_s390_set_crycb_format(kvm);
1084
1085         /* Enable AES/DEA protected key functions by default */
1086         kvm->arch.crypto.aes_kw = 1;
1087         kvm->arch.crypto.dea_kw = 1;
1088         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1089                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1090         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1091                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1092
1093         return 0;
1094 }
1095
1096 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1097 {
1098         int i, rc;
1099         char debug_name[16];
1100         static unsigned long sca_offset;
1101
1102         rc = -EINVAL;
1103 #ifdef CONFIG_KVM_S390_UCONTROL
1104         if (type & ~KVM_VM_S390_UCONTROL)
1105                 goto out_err;
1106         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1107                 goto out_err;
1108 #else
1109         if (type)
1110                 goto out_err;
1111 #endif
1112
1113         rc = s390_enable_sie();
1114         if (rc)
1115                 goto out_err;
1116
1117         rc = -ENOMEM;
1118
1119         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1120         if (!kvm->arch.sca)
1121                 goto out_err;
1122         spin_lock(&kvm_lock);
1123         sca_offset = (sca_offset + 16) & 0x7f0;
1124         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1125         spin_unlock(&kvm_lock);
1126
1127         sprintf(debug_name, "kvm-%u", current->pid);
1128
1129         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1130         if (!kvm->arch.dbf)
1131                 goto out_err;
1132
1133         /*
1134          * The architectural maximum amount of facilities is 16 kbit. To store
1135          * this amount, 2 kbyte of memory is required. Thus we need a full
1136          * page to hold the guest facility list (arch.model.fac->list) and the
1137          * facility mask (arch.model.fac->mask). Its address size has to be
1138          * 31 bits and word aligned.
1139          */
1140         kvm->arch.model.fac =
1141                 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1142         if (!kvm->arch.model.fac)
1143                 goto out_err;
1144
1145         /* Populate the facility mask initially. */
1146         memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1147                S390_ARCH_FAC_LIST_SIZE_BYTE);
1148         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1149                 if (i < kvm_s390_fac_list_mask_size())
1150                         kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1151                 else
1152                         kvm->arch.model.fac->mask[i] = 0UL;
1153         }
1154
1155         /* Populate the facility list initially. */
1156         memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1157                S390_ARCH_FAC_LIST_SIZE_BYTE);
1158
1159         kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1160         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1161
1162         if (kvm_s390_crypto_init(kvm) < 0)
1163                 goto out_err;
1164
1165         spin_lock_init(&kvm->arch.float_int.lock);
1166         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1167                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1168         init_waitqueue_head(&kvm->arch.ipte_wq);
1169         mutex_init(&kvm->arch.ipte_mutex);
1170
1171         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1172         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1173
1174         if (type & KVM_VM_S390_UCONTROL) {
1175                 kvm->arch.gmap = NULL;
1176         } else {
1177                 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1178                 if (!kvm->arch.gmap)
1179                         goto out_err;
1180                 kvm->arch.gmap->private = kvm;
1181                 kvm->arch.gmap->pfault_enabled = 0;
1182         }
1183
1184         kvm->arch.css_support = 0;
1185         kvm->arch.use_irqchip = 0;
1186         kvm->arch.epoch = 0;
1187
1188         spin_lock_init(&kvm->arch.start_stop_lock);
1189         KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1190
1191         return 0;
1192 out_err:
1193         kfree(kvm->arch.crypto.crycb);
1194         free_page((unsigned long)kvm->arch.model.fac);
1195         debug_unregister(kvm->arch.dbf);
1196         free_page((unsigned long)(kvm->arch.sca));
1197         KVM_EVENT(3, "creation of vm failed: %d", rc);
1198         return rc;
1199 }
1200
1201 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1202 {
1203         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1204         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1205         kvm_s390_clear_local_irqs(vcpu);
1206         kvm_clear_async_pf_completion_queue(vcpu);
1207         if (!kvm_is_ucontrol(vcpu->kvm)) {
1208                 clear_bit(63 - vcpu->vcpu_id,
1209                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1210                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1211                     (__u64) vcpu->arch.sie_block)
1212                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1213         }
1214         smp_mb();
1215
1216         if (kvm_is_ucontrol(vcpu->kvm))
1217                 gmap_free(vcpu->arch.gmap);
1218
1219         if (vcpu->kvm->arch.use_cmma)
1220                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1221         free_page((unsigned long)(vcpu->arch.sie_block));
1222
1223         kvm_vcpu_uninit(vcpu);
1224         kmem_cache_free(kvm_vcpu_cache, vcpu);
1225 }
1226
1227 static void kvm_free_vcpus(struct kvm *kvm)
1228 {
1229         unsigned int i;
1230         struct kvm_vcpu *vcpu;
1231
1232         kvm_for_each_vcpu(i, vcpu, kvm)
1233                 kvm_arch_vcpu_destroy(vcpu);
1234
1235         mutex_lock(&kvm->lock);
1236         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1237                 kvm->vcpus[i] = NULL;
1238
1239         atomic_set(&kvm->online_vcpus, 0);
1240         mutex_unlock(&kvm->lock);
1241 }
1242
1243 void kvm_arch_destroy_vm(struct kvm *kvm)
1244 {
1245         kvm_free_vcpus(kvm);
1246         free_page((unsigned long)kvm->arch.model.fac);
1247         free_page((unsigned long)(kvm->arch.sca));
1248         debug_unregister(kvm->arch.dbf);
1249         kfree(kvm->arch.crypto.crycb);
1250         if (!kvm_is_ucontrol(kvm))
1251                 gmap_free(kvm->arch.gmap);
1252         kvm_s390_destroy_adapters(kvm);
1253         kvm_s390_clear_float_irqs(kvm);
1254         KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1255 }
1256
1257 /* Section: vcpu related */
1258 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1259 {
1260         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1261         if (!vcpu->arch.gmap)
1262                 return -ENOMEM;
1263         vcpu->arch.gmap->private = vcpu->kvm;
1264
1265         return 0;
1266 }
1267
1268 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1269 {
1270         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1271         kvm_clear_async_pf_completion_queue(vcpu);
1272         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1273                                     KVM_SYNC_GPRS |
1274                                     KVM_SYNC_ACRS |
1275                                     KVM_SYNC_CRS |
1276                                     KVM_SYNC_ARCH0 |
1277                                     KVM_SYNC_PFAULT;
1278         if (test_kvm_facility(vcpu->kvm, 129))
1279                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1280
1281         if (kvm_is_ucontrol(vcpu->kvm))
1282                 return __kvm_ucontrol_vcpu_init(vcpu);
1283
1284         return 0;
1285 }
1286
1287 /*
1288  * Backs up the current FP/VX register save area on a particular
1289  * destination.  Used to switch between different register save
1290  * areas.
1291  */
1292 static inline void save_fpu_to(struct fpu *dst)
1293 {
1294         dst->fpc = current->thread.fpu.fpc;
1295         dst->regs = current->thread.fpu.regs;
1296 }
1297
1298 /*
1299  * Switches the FP/VX register save area from which to lazy
1300  * restore register contents.
1301  */
1302 static inline void load_fpu_from(struct fpu *from)
1303 {
1304         current->thread.fpu.fpc = from->fpc;
1305         current->thread.fpu.regs = from->regs;
1306 }
1307
1308 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1309 {
1310         /* Save host register state */
1311         save_fpu_regs();
1312         save_fpu_to(&vcpu->arch.host_fpregs);
1313
1314         if (test_kvm_facility(vcpu->kvm, 129)) {
1315                 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1316                 /*
1317                  * Use the register save area in the SIE-control block
1318                  * for register restore and save in kvm_arch_vcpu_put()
1319                  */
1320                 current->thread.fpu.vxrs =
1321                         (__vector128 *)&vcpu->run->s.regs.vrs;
1322         } else
1323                 load_fpu_from(&vcpu->arch.guest_fpregs);
1324
1325         if (test_fp_ctl(current->thread.fpu.fpc))
1326                 /* User space provided an invalid FPC, let's clear it */
1327                 current->thread.fpu.fpc = 0;
1328
1329         save_access_regs(vcpu->arch.host_acrs);
1330         restore_access_regs(vcpu->run->s.regs.acrs);
1331         gmap_enable(vcpu->arch.gmap);
1332         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1333 }
1334
1335 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1336 {
1337         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1338         gmap_disable(vcpu->arch.gmap);
1339
1340         save_fpu_regs();
1341
1342         if (test_kvm_facility(vcpu->kvm, 129))
1343                 /*
1344                  * kvm_arch_vcpu_load() set up the register save area to
1345                  * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1346                  * are already saved.  Only the floating-point control must be
1347                  * copied.
1348                  */
1349                 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1350         else
1351                 save_fpu_to(&vcpu->arch.guest_fpregs);
1352         load_fpu_from(&vcpu->arch.host_fpregs);
1353
1354         save_access_regs(vcpu->run->s.regs.acrs);
1355         restore_access_regs(vcpu->arch.host_acrs);
1356 }
1357
1358 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1359 {
1360         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1361         vcpu->arch.sie_block->gpsw.mask = 0UL;
1362         vcpu->arch.sie_block->gpsw.addr = 0UL;
1363         kvm_s390_set_prefix(vcpu, 0);
1364         vcpu->arch.sie_block->cputm     = 0UL;
1365         vcpu->arch.sie_block->ckc       = 0UL;
1366         vcpu->arch.sie_block->todpr     = 0;
1367         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1368         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1369         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1370         vcpu->arch.guest_fpregs.fpc = 0;
1371         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1372         vcpu->arch.sie_block->gbea = 1;
1373         vcpu->arch.sie_block->pp = 0;
1374         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1375         kvm_clear_async_pf_completion_queue(vcpu);
1376         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1377                 kvm_s390_vcpu_stop(vcpu);
1378         kvm_s390_clear_local_irqs(vcpu);
1379 }
1380
1381 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1382 {
1383         mutex_lock(&vcpu->kvm->lock);
1384         preempt_disable();
1385         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1386         preempt_enable();
1387         mutex_unlock(&vcpu->kvm->lock);
1388         if (!kvm_is_ucontrol(vcpu->kvm))
1389                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1390 }
1391
1392 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1393 {
1394         if (!test_kvm_facility(vcpu->kvm, 76))
1395                 return;
1396
1397         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1398
1399         if (vcpu->kvm->arch.crypto.aes_kw)
1400                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1401         if (vcpu->kvm->arch.crypto.dea_kw)
1402                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1403
1404         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1405 }
1406
1407 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1408 {
1409         free_page(vcpu->arch.sie_block->cbrlo);
1410         vcpu->arch.sie_block->cbrlo = 0;
1411 }
1412
1413 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1414 {
1415         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1416         if (!vcpu->arch.sie_block->cbrlo)
1417                 return -ENOMEM;
1418
1419         vcpu->arch.sie_block->ecb2 |= 0x80;
1420         vcpu->arch.sie_block->ecb2 &= ~0x08;
1421         return 0;
1422 }
1423
1424 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1425 {
1426         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1427
1428         vcpu->arch.cpu_id = model->cpu_id;
1429         vcpu->arch.sie_block->ibc = model->ibc;
1430         vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1431 }
1432
1433 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1434 {
1435         int rc = 0;
1436
1437         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1438                                                     CPUSTAT_SM |
1439                                                     CPUSTAT_STOPPED);
1440
1441         if (test_kvm_facility(vcpu->kvm, 78))
1442                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1443         else if (test_kvm_facility(vcpu->kvm, 8))
1444                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1445
1446         kvm_s390_vcpu_setup_model(vcpu);
1447
1448         vcpu->arch.sie_block->ecb   = 6;
1449         if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1450                 vcpu->arch.sie_block->ecb |= 0x10;
1451
1452         vcpu->arch.sie_block->ecb2  = 8;
1453         vcpu->arch.sie_block->eca   = 0xC1002000U;
1454         if (sclp.has_siif)
1455                 vcpu->arch.sie_block->eca |= 1;
1456         if (sclp.has_sigpif)
1457                 vcpu->arch.sie_block->eca |= 0x10000000U;
1458         if (test_kvm_facility(vcpu->kvm, 129)) {
1459                 vcpu->arch.sie_block->eca |= 0x00020000;
1460                 vcpu->arch.sie_block->ecd |= 0x20000000;
1461         }
1462         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1463
1464         if (vcpu->kvm->arch.use_cmma) {
1465                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1466                 if (rc)
1467                         return rc;
1468         }
1469         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1470         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1471
1472         kvm_s390_vcpu_crypto_setup(vcpu);
1473
1474         return rc;
1475 }
1476
1477 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1478                                       unsigned int id)
1479 {
1480         struct kvm_vcpu *vcpu;
1481         struct sie_page *sie_page;
1482         int rc = -EINVAL;
1483
1484         if (id >= KVM_MAX_VCPUS)
1485                 goto out;
1486
1487         rc = -ENOMEM;
1488
1489         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1490         if (!vcpu)
1491                 goto out;
1492
1493         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1494         if (!sie_page)
1495                 goto out_free_cpu;
1496
1497         vcpu->arch.sie_block = &sie_page->sie_block;
1498         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1499
1500         vcpu->arch.sie_block->icpua = id;
1501         if (!kvm_is_ucontrol(kvm)) {
1502                 if (!kvm->arch.sca) {
1503                         WARN_ON_ONCE(1);
1504                         goto out_free_cpu;
1505                 }
1506                 if (!kvm->arch.sca->cpu[id].sda)
1507                         kvm->arch.sca->cpu[id].sda =
1508                                 (__u64) vcpu->arch.sie_block;
1509                 vcpu->arch.sie_block->scaoh =
1510                         (__u32)(((__u64)kvm->arch.sca) >> 32);
1511                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1512                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1513         }
1514
1515         spin_lock_init(&vcpu->arch.local_int.lock);
1516         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1517         vcpu->arch.local_int.wq = &vcpu->wq;
1518         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1519
1520         /*
1521          * Allocate a save area for floating-point registers.  If the vector
1522          * extension is available, register contents are saved in the SIE
1523          * control block.  The allocated save area is still required in
1524          * particular places, for example, in kvm_s390_vcpu_store_status().
1525          */
1526         vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1527                                                GFP_KERNEL);
1528         if (!vcpu->arch.guest_fpregs.fprs) {
1529                 rc = -ENOMEM;
1530                 goto out_free_sie_block;
1531         }
1532
1533         rc = kvm_vcpu_init(vcpu, kvm, id);
1534         if (rc)
1535                 goto out_free_sie_block;
1536         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1537                  vcpu->arch.sie_block);
1538         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1539
1540         return vcpu;
1541 out_free_sie_block:
1542         free_page((unsigned long)(vcpu->arch.sie_block));
1543 out_free_cpu:
1544         kmem_cache_free(kvm_vcpu_cache, vcpu);
1545 out:
1546         return ERR_PTR(rc);
1547 }
1548
1549 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1550 {
1551         return kvm_s390_vcpu_has_irq(vcpu, 0);
1552 }
1553
1554 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1555 {
1556         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1557         exit_sie(vcpu);
1558 }
1559
1560 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1561 {
1562         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1563 }
1564
1565 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1566 {
1567         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1568         exit_sie(vcpu);
1569 }
1570
1571 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1572 {
1573         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1574 }
1575
1576 /*
1577  * Kick a guest cpu out of SIE and wait until SIE is not running.
1578  * If the CPU is not running (e.g. waiting as idle) the function will
1579  * return immediately. */
1580 void exit_sie(struct kvm_vcpu *vcpu)
1581 {
1582         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1583         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1584                 cpu_relax();
1585 }
1586
1587 /* Kick a guest cpu out of SIE to process a request synchronously */
1588 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1589 {
1590         kvm_make_request(req, vcpu);
1591         kvm_s390_vcpu_request(vcpu);
1592 }
1593
1594 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1595 {
1596         int i;
1597         struct kvm *kvm = gmap->private;
1598         struct kvm_vcpu *vcpu;
1599
1600         kvm_for_each_vcpu(i, vcpu, kvm) {
1601                 /* match against both prefix pages */
1602                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1603                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1604                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1605                 }
1606         }
1607 }
1608
1609 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1610 {
1611         /* kvm common code refers to this, but never calls it */
1612         BUG();
1613         return 0;
1614 }
1615
1616 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1617                                            struct kvm_one_reg *reg)
1618 {
1619         int r = -EINVAL;
1620
1621         switch (reg->id) {
1622         case KVM_REG_S390_TODPR:
1623                 r = put_user(vcpu->arch.sie_block->todpr,
1624                              (u32 __user *)reg->addr);
1625                 break;
1626         case KVM_REG_S390_EPOCHDIFF:
1627                 r = put_user(vcpu->arch.sie_block->epoch,
1628                              (u64 __user *)reg->addr);
1629                 break;
1630         case KVM_REG_S390_CPU_TIMER:
1631                 r = put_user(vcpu->arch.sie_block->cputm,
1632                              (u64 __user *)reg->addr);
1633                 break;
1634         case KVM_REG_S390_CLOCK_COMP:
1635                 r = put_user(vcpu->arch.sie_block->ckc,
1636                              (u64 __user *)reg->addr);
1637                 break;
1638         case KVM_REG_S390_PFTOKEN:
1639                 r = put_user(vcpu->arch.pfault_token,
1640                              (u64 __user *)reg->addr);
1641                 break;
1642         case KVM_REG_S390_PFCOMPARE:
1643                 r = put_user(vcpu->arch.pfault_compare,
1644                              (u64 __user *)reg->addr);
1645                 break;
1646         case KVM_REG_S390_PFSELECT:
1647                 r = put_user(vcpu->arch.pfault_select,
1648                              (u64 __user *)reg->addr);
1649                 break;
1650         case KVM_REG_S390_PP:
1651                 r = put_user(vcpu->arch.sie_block->pp,
1652                              (u64 __user *)reg->addr);
1653                 break;
1654         case KVM_REG_S390_GBEA:
1655                 r = put_user(vcpu->arch.sie_block->gbea,
1656                              (u64 __user *)reg->addr);
1657                 break;
1658         default:
1659                 break;
1660         }
1661
1662         return r;
1663 }
1664
1665 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1666                                            struct kvm_one_reg *reg)
1667 {
1668         int r = -EINVAL;
1669
1670         switch (reg->id) {
1671         case KVM_REG_S390_TODPR:
1672                 r = get_user(vcpu->arch.sie_block->todpr,
1673                              (u32 __user *)reg->addr);
1674                 break;
1675         case KVM_REG_S390_EPOCHDIFF:
1676                 r = get_user(vcpu->arch.sie_block->epoch,
1677                              (u64 __user *)reg->addr);
1678                 break;
1679         case KVM_REG_S390_CPU_TIMER:
1680                 r = get_user(vcpu->arch.sie_block->cputm,
1681                              (u64 __user *)reg->addr);
1682                 break;
1683         case KVM_REG_S390_CLOCK_COMP:
1684                 r = get_user(vcpu->arch.sie_block->ckc,
1685                              (u64 __user *)reg->addr);
1686                 break;
1687         case KVM_REG_S390_PFTOKEN:
1688                 r = get_user(vcpu->arch.pfault_token,
1689                              (u64 __user *)reg->addr);
1690                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1691                         kvm_clear_async_pf_completion_queue(vcpu);
1692                 break;
1693         case KVM_REG_S390_PFCOMPARE:
1694                 r = get_user(vcpu->arch.pfault_compare,
1695                              (u64 __user *)reg->addr);
1696                 break;
1697         case KVM_REG_S390_PFSELECT:
1698                 r = get_user(vcpu->arch.pfault_select,
1699                              (u64 __user *)reg->addr);
1700                 break;
1701         case KVM_REG_S390_PP:
1702                 r = get_user(vcpu->arch.sie_block->pp,
1703                              (u64 __user *)reg->addr);
1704                 break;
1705         case KVM_REG_S390_GBEA:
1706                 r = get_user(vcpu->arch.sie_block->gbea,
1707                              (u64 __user *)reg->addr);
1708                 break;
1709         default:
1710                 break;
1711         }
1712
1713         return r;
1714 }
1715
1716 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1717 {
1718         kvm_s390_vcpu_initial_reset(vcpu);
1719         return 0;
1720 }
1721
1722 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1723 {
1724         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1725         return 0;
1726 }
1727
1728 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1729 {
1730         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1731         return 0;
1732 }
1733
1734 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1735                                   struct kvm_sregs *sregs)
1736 {
1737         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1738         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1739         restore_access_regs(vcpu->run->s.regs.acrs);
1740         return 0;
1741 }
1742
1743 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1744                                   struct kvm_sregs *sregs)
1745 {
1746         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1747         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1748         return 0;
1749 }
1750
1751 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1752 {
1753         if (test_fp_ctl(fpu->fpc))
1754                 return -EINVAL;
1755         memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1756         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1757         save_fpu_regs();
1758         load_fpu_from(&vcpu->arch.guest_fpregs);
1759         return 0;
1760 }
1761
1762 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1763 {
1764         memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1765         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1766         return 0;
1767 }
1768
1769 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1770 {
1771         int rc = 0;
1772
1773         if (!is_vcpu_stopped(vcpu))
1774                 rc = -EBUSY;
1775         else {
1776                 vcpu->run->psw_mask = psw.mask;
1777                 vcpu->run->psw_addr = psw.addr;
1778         }
1779         return rc;
1780 }
1781
1782 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1783                                   struct kvm_translation *tr)
1784 {
1785         return -EINVAL; /* not implemented yet */
1786 }
1787
1788 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1789                               KVM_GUESTDBG_USE_HW_BP | \
1790                               KVM_GUESTDBG_ENABLE)
1791
1792 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1793                                         struct kvm_guest_debug *dbg)
1794 {
1795         int rc = 0;
1796
1797         vcpu->guest_debug = 0;
1798         kvm_s390_clear_bp_data(vcpu);
1799
1800         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1801                 return -EINVAL;
1802
1803         if (dbg->control & KVM_GUESTDBG_ENABLE) {
1804                 vcpu->guest_debug = dbg->control;
1805                 /* enforce guest PER */
1806                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1807
1808                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1809                         rc = kvm_s390_import_bp_data(vcpu, dbg);
1810         } else {
1811                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1812                 vcpu->arch.guestdbg.last_bp = 0;
1813         }
1814
1815         if (rc) {
1816                 vcpu->guest_debug = 0;
1817                 kvm_s390_clear_bp_data(vcpu);
1818                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1819         }
1820
1821         return rc;
1822 }
1823
1824 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1825                                     struct kvm_mp_state *mp_state)
1826 {
1827         /* CHECK_STOP and LOAD are not supported yet */
1828         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1829                                        KVM_MP_STATE_OPERATING;
1830 }
1831
1832 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1833                                     struct kvm_mp_state *mp_state)
1834 {
1835         int rc = 0;
1836
1837         /* user space knows about this interface - let it control the state */
1838         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1839
1840         switch (mp_state->mp_state) {
1841         case KVM_MP_STATE_STOPPED:
1842                 kvm_s390_vcpu_stop(vcpu);
1843                 break;
1844         case KVM_MP_STATE_OPERATING:
1845                 kvm_s390_vcpu_start(vcpu);
1846                 break;
1847         case KVM_MP_STATE_LOAD:
1848         case KVM_MP_STATE_CHECK_STOP:
1849                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1850         default:
1851                 rc = -ENXIO;
1852         }
1853
1854         return rc;
1855 }
1856
1857 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1858 {
1859         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1860 }
1861
1862 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1863 {
1864 retry:
1865         kvm_s390_vcpu_request_handled(vcpu);
1866         if (!vcpu->requests)
1867                 return 0;
1868         /*
1869          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1870          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1871          * This ensures that the ipte instruction for this request has
1872          * already finished. We might race against a second unmapper that
1873          * wants to set the blocking bit. Lets just retry the request loop.
1874          */
1875         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1876                 int rc;
1877                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1878                                       kvm_s390_get_prefix(vcpu),
1879                                       PAGE_SIZE * 2);
1880                 if (rc)
1881                         return rc;
1882                 goto retry;
1883         }
1884
1885         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1886                 vcpu->arch.sie_block->ihcpu = 0xffff;
1887                 goto retry;
1888         }
1889
1890         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1891                 if (!ibs_enabled(vcpu)) {
1892                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1893                         atomic_or(CPUSTAT_IBS,
1894                                         &vcpu->arch.sie_block->cpuflags);
1895                 }
1896                 goto retry;
1897         }
1898
1899         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1900                 if (ibs_enabled(vcpu)) {
1901                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1902                         atomic_andnot(CPUSTAT_IBS,
1903                                           &vcpu->arch.sie_block->cpuflags);
1904                 }
1905                 goto retry;
1906         }
1907
1908         /* nothing to do, just clear the request */
1909         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1910
1911         return 0;
1912 }
1913
1914 /**
1915  * kvm_arch_fault_in_page - fault-in guest page if necessary
1916  * @vcpu: The corresponding virtual cpu
1917  * @gpa: Guest physical address
1918  * @writable: Whether the page should be writable or not
1919  *
1920  * Make sure that a guest page has been faulted-in on the host.
1921  *
1922  * Return: Zero on success, negative error code otherwise.
1923  */
1924 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1925 {
1926         return gmap_fault(vcpu->arch.gmap, gpa,
1927                           writable ? FAULT_FLAG_WRITE : 0);
1928 }
1929
1930 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1931                                       unsigned long token)
1932 {
1933         struct kvm_s390_interrupt inti;
1934         struct kvm_s390_irq irq;
1935
1936         if (start_token) {
1937                 irq.u.ext.ext_params2 = token;
1938                 irq.type = KVM_S390_INT_PFAULT_INIT;
1939                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1940         } else {
1941                 inti.type = KVM_S390_INT_PFAULT_DONE;
1942                 inti.parm64 = token;
1943                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1944         }
1945 }
1946
1947 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1948                                      struct kvm_async_pf *work)
1949 {
1950         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1951         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1952 }
1953
1954 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1955                                  struct kvm_async_pf *work)
1956 {
1957         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1958         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1959 }
1960
1961 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1962                                struct kvm_async_pf *work)
1963 {
1964         /* s390 will always inject the page directly */
1965 }
1966
1967 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1968 {
1969         /*
1970          * s390 will always inject the page directly,
1971          * but we still want check_async_completion to cleanup
1972          */
1973         return true;
1974 }
1975
1976 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1977 {
1978         hva_t hva;
1979         struct kvm_arch_async_pf arch;
1980         int rc;
1981
1982         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1983                 return 0;
1984         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1985             vcpu->arch.pfault_compare)
1986                 return 0;
1987         if (psw_extint_disabled(vcpu))
1988                 return 0;
1989         if (kvm_s390_vcpu_has_irq(vcpu, 0))
1990                 return 0;
1991         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1992                 return 0;
1993         if (!vcpu->arch.gmap->pfault_enabled)
1994                 return 0;
1995
1996         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1997         hva += current->thread.gmap_addr & ~PAGE_MASK;
1998         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1999                 return 0;
2000
2001         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2002         return rc;
2003 }
2004
2005 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2006 {
2007         int rc, cpuflags;
2008
2009         /*
2010          * On s390 notifications for arriving pages will be delivered directly
2011          * to the guest but the house keeping for completed pfaults is
2012          * handled outside the worker.
2013          */
2014         kvm_check_async_pf_completion(vcpu);
2015
2016         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
2017
2018         if (need_resched())
2019                 schedule();
2020
2021         if (test_cpu_flag(CIF_MCCK_PENDING))
2022                 s390_handle_mcck();
2023
2024         if (!kvm_is_ucontrol(vcpu->kvm)) {
2025                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2026                 if (rc)
2027                         return rc;
2028         }
2029
2030         rc = kvm_s390_handle_requests(vcpu);
2031         if (rc)
2032                 return rc;
2033
2034         if (guestdbg_enabled(vcpu)) {
2035                 kvm_s390_backup_guest_per_regs(vcpu);
2036                 kvm_s390_patch_guest_per_regs(vcpu);
2037         }
2038
2039         vcpu->arch.sie_block->icptcode = 0;
2040         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2041         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2042         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2043
2044         return 0;
2045 }
2046
2047 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2048 {
2049         psw_t *psw = &vcpu->arch.sie_block->gpsw;
2050         u8 opcode;
2051         int rc;
2052
2053         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2054         trace_kvm_s390_sie_fault(vcpu);
2055
2056         /*
2057          * We want to inject an addressing exception, which is defined as a
2058          * suppressing or terminating exception. However, since we came here
2059          * by a DAT access exception, the PSW still points to the faulting
2060          * instruction since DAT exceptions are nullifying. So we've got
2061          * to look up the current opcode to get the length of the instruction
2062          * to be able to forward the PSW.
2063          */
2064         rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2065         if (rc)
2066                 return kvm_s390_inject_prog_cond(vcpu, rc);
2067         psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2068
2069         return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2070 }
2071
2072 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2073 {
2074         int rc = -1;
2075
2076         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2077                    vcpu->arch.sie_block->icptcode);
2078         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2079
2080         if (guestdbg_enabled(vcpu))
2081                 kvm_s390_restore_guest_per_regs(vcpu);
2082
2083         if (exit_reason >= 0) {
2084                 rc = 0;
2085         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2086                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2087                 vcpu->run->s390_ucontrol.trans_exc_code =
2088                                                 current->thread.gmap_addr;
2089                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2090                 rc = -EREMOTE;
2091
2092         } else if (current->thread.gmap_pfault) {
2093                 trace_kvm_s390_major_guest_pfault(vcpu);
2094                 current->thread.gmap_pfault = 0;
2095                 if (kvm_arch_setup_async_pf(vcpu)) {
2096                         rc = 0;
2097                 } else {
2098                         gpa_t gpa = current->thread.gmap_addr;
2099                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
2100                 }
2101         }
2102
2103         if (rc == -1)
2104                 rc = vcpu_post_run_fault_in_sie(vcpu);
2105
2106         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2107
2108         if (rc == 0) {
2109                 if (kvm_is_ucontrol(vcpu->kvm))
2110                         /* Don't exit for host interrupts. */
2111                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
2112                 else
2113                         rc = kvm_handle_sie_intercept(vcpu);
2114         }
2115
2116         return rc;
2117 }
2118
2119 static int __vcpu_run(struct kvm_vcpu *vcpu)
2120 {
2121         int rc, exit_reason;
2122
2123         /*
2124          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2125          * ning the guest), so that memslots (and other stuff) are protected
2126          */
2127         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2128
2129         do {
2130                 rc = vcpu_pre_run(vcpu);
2131                 if (rc)
2132                         break;
2133
2134                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2135                 /*
2136                  * As PF_VCPU will be used in fault handler, between
2137                  * guest_enter and guest_exit should be no uaccess.
2138                  */
2139                 local_irq_disable();
2140                 __kvm_guest_enter();
2141                 local_irq_enable();
2142                 exit_reason = sie64a(vcpu->arch.sie_block,
2143                                      vcpu->run->s.regs.gprs);
2144                 local_irq_disable();
2145                 __kvm_guest_exit();
2146                 local_irq_enable();
2147                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2148
2149                 rc = vcpu_post_run(vcpu, exit_reason);
2150         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2151
2152         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2153         return rc;
2154 }
2155
2156 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2157 {
2158         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2159         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2160         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2161                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2162         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2163                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2164                 /* some control register changes require a tlb flush */
2165                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2166         }
2167         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2168                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2169                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2170                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2171                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2172                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2173         }
2174         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2175                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2176                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2177                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2178                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2179                         kvm_clear_async_pf_completion_queue(vcpu);
2180         }
2181         kvm_run->kvm_dirty_regs = 0;
2182 }
2183
2184 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2185 {
2186         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2187         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2188         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2189         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2190         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2191         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2192         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2193         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2194         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2195         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2196         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2197         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2198 }
2199
2200 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2201 {
2202         int rc;
2203         sigset_t sigsaved;
2204
2205         if (guestdbg_exit_pending(vcpu)) {
2206                 kvm_s390_prepare_debug_exit(vcpu);
2207                 return 0;
2208         }
2209
2210         if (vcpu->sigset_active)
2211                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2212
2213         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2214                 kvm_s390_vcpu_start(vcpu);
2215         } else if (is_vcpu_stopped(vcpu)) {
2216                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2217                                    vcpu->vcpu_id);
2218                 return -EINVAL;
2219         }
2220
2221         sync_regs(vcpu, kvm_run);
2222
2223         might_fault();
2224         rc = __vcpu_run(vcpu);
2225
2226         if (signal_pending(current) && !rc) {
2227                 kvm_run->exit_reason = KVM_EXIT_INTR;
2228                 rc = -EINTR;
2229         }
2230
2231         if (guestdbg_exit_pending(vcpu) && !rc)  {
2232                 kvm_s390_prepare_debug_exit(vcpu);
2233                 rc = 0;
2234         }
2235
2236         if (rc == -EOPNOTSUPP) {
2237                 /* intercept cannot be handled in-kernel, prepare kvm-run */
2238                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
2239                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2240                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
2241                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
2242                 rc = 0;
2243         }
2244
2245         if (rc == -EREMOTE) {
2246                 /* intercept was handled, but userspace support is needed
2247                  * kvm_run has been prepared by the handler */
2248                 rc = 0;
2249         }
2250
2251         store_regs(vcpu, kvm_run);
2252
2253         if (vcpu->sigset_active)
2254                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2255
2256         vcpu->stat.exit_userspace++;
2257         return rc;
2258 }
2259
2260 /*
2261  * store status at address
2262  * we use have two special cases:
2263  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2264  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2265  */
2266 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2267 {
2268         unsigned char archmode = 1;
2269         unsigned int px;
2270         u64 clkcomp;
2271         int rc;
2272
2273         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2274                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2275                         return -EFAULT;
2276                 gpa = SAVE_AREA_BASE;
2277         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2278                 if (write_guest_real(vcpu, 163, &archmode, 1))
2279                         return -EFAULT;
2280                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2281         }
2282         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2283                              vcpu->arch.guest_fpregs.fprs, 128);
2284         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2285                               vcpu->run->s.regs.gprs, 128);
2286         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2287                               &vcpu->arch.sie_block->gpsw, 16);
2288         px = kvm_s390_get_prefix(vcpu);
2289         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2290                               &px, 4);
2291         rc |= write_guest_abs(vcpu,
2292                               gpa + offsetof(struct save_area, fp_ctrl_reg),
2293                               &vcpu->arch.guest_fpregs.fpc, 4);
2294         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2295                               &vcpu->arch.sie_block->todpr, 4);
2296         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2297                               &vcpu->arch.sie_block->cputm, 8);
2298         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2299         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2300                               &clkcomp, 8);
2301         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2302                               &vcpu->run->s.regs.acrs, 64);
2303         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2304                               &vcpu->arch.sie_block->gcr, 128);
2305         return rc ? -EFAULT : 0;
2306 }
2307
2308 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2309 {
2310         /*
2311          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2312          * copying in vcpu load/put. Lets update our copies before we save
2313          * it into the save area
2314          */
2315         save_fpu_regs();
2316         if (test_kvm_facility(vcpu->kvm, 129)) {
2317                 /*
2318                  * If the vector extension is available, the vector registers
2319                  * which overlaps with floating-point registers are saved in
2320                  * the SIE-control block.  Hence, extract the floating-point
2321                  * registers and the FPC value and store them in the
2322                  * guest_fpregs structure.
2323                  */
2324                 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2325                 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2326                                  current->thread.fpu.vxrs);
2327         } else
2328                 save_fpu_to(&vcpu->arch.guest_fpregs);
2329         save_access_regs(vcpu->run->s.regs.acrs);
2330
2331         return kvm_s390_store_status_unloaded(vcpu, addr);
2332 }
2333
2334 /*
2335  * store additional status at address
2336  */
2337 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2338                                         unsigned long gpa)
2339 {
2340         /* Only bits 0-53 are used for address formation */
2341         if (!(gpa & ~0x3ff))
2342                 return 0;
2343
2344         return write_guest_abs(vcpu, gpa & ~0x3ff,
2345                                (void *)&vcpu->run->s.regs.vrs, 512);
2346 }
2347
2348 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2349 {
2350         if (!test_kvm_facility(vcpu->kvm, 129))
2351                 return 0;
2352
2353         /*
2354          * The guest VXRS are in the host VXRs due to the lazy
2355          * copying in vcpu load/put. We can simply call save_fpu_regs()
2356          * to save the current register state because we are in the
2357          * middle of a load/put cycle.
2358          *
2359          * Let's update our copies before we save it into the save area.
2360          */
2361         save_fpu_regs();
2362
2363         return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2364 }
2365
2366 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2367 {
2368         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2369         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2370 }
2371
2372 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2373 {
2374         unsigned int i;
2375         struct kvm_vcpu *vcpu;
2376
2377         kvm_for_each_vcpu(i, vcpu, kvm) {
2378                 __disable_ibs_on_vcpu(vcpu);
2379         }
2380 }
2381
2382 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2383 {
2384         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2385         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2386 }
2387
2388 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2389 {
2390         int i, online_vcpus, started_vcpus = 0;
2391
2392         if (!is_vcpu_stopped(vcpu))
2393                 return;
2394
2395         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2396         /* Only one cpu at a time may enter/leave the STOPPED state. */
2397         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2398         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2399
2400         for (i = 0; i < online_vcpus; i++) {
2401                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2402                         started_vcpus++;
2403         }
2404
2405         if (started_vcpus == 0) {
2406                 /* we're the only active VCPU -> speed it up */
2407                 __enable_ibs_on_vcpu(vcpu);
2408         } else if (started_vcpus == 1) {
2409                 /*
2410                  * As we are starting a second VCPU, we have to disable
2411                  * the IBS facility on all VCPUs to remove potentially
2412                  * oustanding ENABLE requests.
2413                  */
2414                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2415         }
2416
2417         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2418         /*
2419          * Another VCPU might have used IBS while we were offline.
2420          * Let's play safe and flush the VCPU at startup.
2421          */
2422         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2423         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2424         return;
2425 }
2426
2427 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2428 {
2429         int i, online_vcpus, started_vcpus = 0;
2430         struct kvm_vcpu *started_vcpu = NULL;
2431
2432         if (is_vcpu_stopped(vcpu))
2433                 return;
2434
2435         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2436         /* Only one cpu at a time may enter/leave the STOPPED state. */
2437         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2438         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2439
2440         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2441         kvm_s390_clear_stop_irq(vcpu);
2442
2443         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2444         __disable_ibs_on_vcpu(vcpu);
2445
2446         for (i = 0; i < online_vcpus; i++) {
2447                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2448                         started_vcpus++;
2449                         started_vcpu = vcpu->kvm->vcpus[i];
2450                 }
2451         }
2452
2453         if (started_vcpus == 1) {
2454                 /*
2455                  * As we only have one VCPU left, we want to enable the
2456                  * IBS facility for that VCPU to speed it up.
2457                  */
2458                 __enable_ibs_on_vcpu(started_vcpu);
2459         }
2460
2461         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2462         return;
2463 }
2464
2465 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2466                                      struct kvm_enable_cap *cap)
2467 {
2468         int r;
2469
2470         if (cap->flags)
2471                 return -EINVAL;
2472
2473         switch (cap->cap) {
2474         case KVM_CAP_S390_CSS_SUPPORT:
2475                 if (!vcpu->kvm->arch.css_support) {
2476                         vcpu->kvm->arch.css_support = 1;
2477                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2478                         trace_kvm_s390_enable_css(vcpu->kvm);
2479                 }
2480                 r = 0;
2481                 break;
2482         default:
2483                 r = -EINVAL;
2484                 break;
2485         }
2486         return r;
2487 }
2488
2489 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2490                                   struct kvm_s390_mem_op *mop)
2491 {
2492         void __user *uaddr = (void __user *)mop->buf;
2493         void *tmpbuf = NULL;
2494         int r, srcu_idx;
2495         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2496                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
2497
2498         if (mop->flags & ~supported_flags)
2499                 return -EINVAL;
2500
2501         if (mop->size > MEM_OP_MAX_SIZE)
2502                 return -E2BIG;
2503
2504         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2505                 tmpbuf = vmalloc(mop->size);
2506                 if (!tmpbuf)
2507                         return -ENOMEM;
2508         }
2509
2510         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2511
2512         switch (mop->op) {
2513         case KVM_S390_MEMOP_LOGICAL_READ:
2514                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2515                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2516                         break;
2517                 }
2518                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2519                 if (r == 0) {
2520                         if (copy_to_user(uaddr, tmpbuf, mop->size))
2521                                 r = -EFAULT;
2522                 }
2523                 break;
2524         case KVM_S390_MEMOP_LOGICAL_WRITE:
2525                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2526                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2527                         break;
2528                 }
2529                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2530                         r = -EFAULT;
2531                         break;
2532                 }
2533                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2534                 break;
2535         default:
2536                 r = -EINVAL;
2537         }
2538
2539         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2540
2541         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2542                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2543
2544         vfree(tmpbuf);
2545         return r;
2546 }
2547
2548 long kvm_arch_vcpu_ioctl(struct file *filp,
2549                          unsigned int ioctl, unsigned long arg)
2550 {
2551         struct kvm_vcpu *vcpu = filp->private_data;
2552         void __user *argp = (void __user *)arg;
2553         int idx;
2554         long r;
2555
2556         switch (ioctl) {
2557         case KVM_S390_IRQ: {
2558                 struct kvm_s390_irq s390irq;
2559
2560                 r = -EFAULT;
2561                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2562                         break;
2563                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2564                 break;
2565         }
2566         case KVM_S390_INTERRUPT: {
2567                 struct kvm_s390_interrupt s390int;
2568                 struct kvm_s390_irq s390irq;
2569
2570                 r = -EFAULT;
2571                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2572                         break;
2573                 if (s390int_to_s390irq(&s390int, &s390irq))
2574                         return -EINVAL;
2575                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2576                 break;
2577         }
2578         case KVM_S390_STORE_STATUS:
2579                 idx = srcu_read_lock(&vcpu->kvm->srcu);
2580                 r = kvm_s390_vcpu_store_status(vcpu, arg);
2581                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2582                 break;
2583         case KVM_S390_SET_INITIAL_PSW: {
2584                 psw_t psw;
2585
2586                 r = -EFAULT;
2587                 if (copy_from_user(&psw, argp, sizeof(psw)))
2588                         break;
2589                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2590                 break;
2591         }
2592         case KVM_S390_INITIAL_RESET:
2593                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2594                 break;
2595         case KVM_SET_ONE_REG:
2596         case KVM_GET_ONE_REG: {
2597                 struct kvm_one_reg reg;
2598                 r = -EFAULT;
2599                 if (copy_from_user(&reg, argp, sizeof(reg)))
2600                         break;
2601                 if (ioctl == KVM_SET_ONE_REG)
2602                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2603                 else
2604                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2605                 break;
2606         }
2607 #ifdef CONFIG_KVM_S390_UCONTROL
2608         case KVM_S390_UCAS_MAP: {
2609                 struct kvm_s390_ucas_mapping ucasmap;
2610
2611                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2612                         r = -EFAULT;
2613                         break;
2614                 }
2615
2616                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2617                         r = -EINVAL;
2618                         break;
2619                 }
2620
2621                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2622                                      ucasmap.vcpu_addr, ucasmap.length);
2623                 break;
2624         }
2625         case KVM_S390_UCAS_UNMAP: {
2626                 struct kvm_s390_ucas_mapping ucasmap;
2627
2628                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2629                         r = -EFAULT;
2630                         break;
2631                 }
2632
2633                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2634                         r = -EINVAL;
2635                         break;
2636                 }
2637
2638                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2639                         ucasmap.length);
2640                 break;
2641         }
2642 #endif
2643         case KVM_S390_VCPU_FAULT: {
2644                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2645                 break;
2646         }
2647         case KVM_ENABLE_CAP:
2648         {
2649                 struct kvm_enable_cap cap;
2650                 r = -EFAULT;
2651                 if (copy_from_user(&cap, argp, sizeof(cap)))
2652                         break;
2653                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2654                 break;
2655         }
2656         case KVM_S390_MEM_OP: {
2657                 struct kvm_s390_mem_op mem_op;
2658
2659                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2660                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2661                 else
2662                         r = -EFAULT;
2663                 break;
2664         }
2665         case KVM_S390_SET_IRQ_STATE: {
2666                 struct kvm_s390_irq_state irq_state;
2667
2668                 r = -EFAULT;
2669                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2670                         break;
2671                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2672                     irq_state.len == 0 ||
2673                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2674                         r = -EINVAL;
2675                         break;
2676                 }
2677                 r = kvm_s390_set_irq_state(vcpu,
2678                                            (void __user *) irq_state.buf,
2679                                            irq_state.len);
2680                 break;
2681         }
2682         case KVM_S390_GET_IRQ_STATE: {
2683                 struct kvm_s390_irq_state irq_state;
2684
2685                 r = -EFAULT;
2686                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2687                         break;
2688                 if (irq_state.len == 0) {
2689                         r = -EINVAL;
2690                         break;
2691                 }
2692                 r = kvm_s390_get_irq_state(vcpu,
2693                                            (__u8 __user *)  irq_state.buf,
2694                                            irq_state.len);
2695                 break;
2696         }
2697         default:
2698                 r = -ENOTTY;
2699         }
2700         return r;
2701 }
2702
2703 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2704 {
2705 #ifdef CONFIG_KVM_S390_UCONTROL
2706         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2707                  && (kvm_is_ucontrol(vcpu->kvm))) {
2708                 vmf->page = virt_to_page(vcpu->arch.sie_block);
2709                 get_page(vmf->page);
2710                 return 0;
2711         }
2712 #endif
2713         return VM_FAULT_SIGBUS;
2714 }
2715
2716 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2717                             unsigned long npages)
2718 {
2719         return 0;
2720 }
2721
2722 /* Section: memory related */
2723 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2724                                    struct kvm_memory_slot *memslot,
2725                                    const struct kvm_userspace_memory_region *mem,
2726                                    enum kvm_mr_change change)
2727 {
2728         /* A few sanity checks. We can have memory slots which have to be
2729            located/ended at a segment boundary (1MB). The memory in userland is
2730            ok to be fragmented into various different vmas. It is okay to mmap()
2731            and munmap() stuff in this slot after doing this call at any time */
2732
2733         if (mem->userspace_addr & 0xffffful)
2734                 return -EINVAL;
2735
2736         if (mem->memory_size & 0xffffful)
2737                 return -EINVAL;
2738
2739         return 0;
2740 }
2741
2742 void kvm_arch_commit_memory_region(struct kvm *kvm,
2743                                 const struct kvm_userspace_memory_region *mem,
2744                                 const struct kvm_memory_slot *old,
2745                                 const struct kvm_memory_slot *new,
2746                                 enum kvm_mr_change change)
2747 {
2748         int rc;
2749
2750         /* If the basics of the memslot do not change, we do not want
2751          * to update the gmap. Every update causes several unnecessary
2752          * segment translation exceptions. This is usually handled just
2753          * fine by the normal fault handler + gmap, but it will also
2754          * cause faults on the prefix page of running guest CPUs.
2755          */
2756         if (old->userspace_addr == mem->userspace_addr &&
2757             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2758             old->npages * PAGE_SIZE == mem->memory_size)
2759                 return;
2760
2761         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2762                 mem->guest_phys_addr, mem->memory_size);
2763         if (rc)
2764                 pr_warn("failed to commit memory region\n");
2765         return;
2766 }
2767
2768 static int __init kvm_s390_init(void)
2769 {
2770         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2771 }
2772
2773 static void __exit kvm_s390_exit(void)
2774 {
2775         kvm_exit();
2776 }
2777
2778 module_init(kvm_s390_init);
2779 module_exit(kvm_s390_exit);
2780
2781 /*
2782  * Enable autoloading of the kvm module.
2783  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2784  * since x86 takes a different approach.
2785  */
2786 #include <linux/miscdevice.h>
2787 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2788 MODULE_ALIAS("devname:kvm");