arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/random.h>
  26 #include <linux/slab.h>
  27 #include <linux/timer.h>
  28 #include <linux/vmalloc.h>
  29 #include <asm/asm-offsets.h>
  30 #include <asm/lowcore.h>
  31 #include <asm/etr.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/nmi.h>
  34 #include <asm/switch_to.h>
  35 #include <asm/isc.h>
  36 #include <asm/sclp.h>
  37 #include "kvm-s390.h"
  38 #include "gaccess.h"
  39
  40 #define KMSG_COMPONENT "kvm-s390"
  41 #undef pr_fmt
  42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  43
  44 #define CREATE_TRACE_POINTS
  45 #include "trace.h"
  46 #include "trace-s390.h"
  47
  48 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  49 #define LOCAL_IRQS 32
  50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  51                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  52
  53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  54
  55 struct kvm_stats_debugfs_item debugfs_entries[] = {
  56         { "userspace_handled", VCPU_STAT(exit_userspace) },
  57         { "exit_null", VCPU_STAT(exit_null) },
  58         { "exit_validity", VCPU_STAT(exit_validity) },
  59         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  60         { "exit_external_request", VCPU_STAT(exit_external_request) },
  61         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  62         { "exit_instruction", VCPU_STAT(exit_instruction) },
  63         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  64         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  65         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  66         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  67         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  68         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  69         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  70         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  71         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  72         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  73         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  74         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  75         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  76         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  77         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  78         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  79         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  80         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  81         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  82         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  83         { "instruction_spx", VCPU_STAT(instruction_spx) },
  84         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  85         { "instruction_stap", VCPU_STAT(instruction_stap) },
  86         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  87         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  88         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  89         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  90         { "instruction_essa", VCPU_STAT(instruction_essa) },
  91         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  92         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  93         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  94         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  95         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  96         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  97         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  98         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  99         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 100         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 101         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 102         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 103         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 104         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 105         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 106         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 107         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 108         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 109         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 110         { "diagnose_10", VCPU_STAT(diagnose_10) },
 111         { "diagnose_44", VCPU_STAT(diagnose_44) },
 112         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 113         { "diagnose_258", VCPU_STAT(diagnose_258) },
 114         { "diagnose_308", VCPU_STAT(diagnose_308) },
 115         { "diagnose_500", VCPU_STAT(diagnose_500) },
 116         { NULL }
 117 };
 118
 119 /* upper facilities limit for kvm */
 120 unsigned long kvm_s390_fac_list_mask[] = {
 121         0xffe6fffbfcfdfc40UL,
 122         0x005e800000000000UL,
 123 };
 124
 125 unsigned long kvm_s390_fac_list_mask_size(void)
 126 {
 127         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 128         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 129 }
 130
 131 static struct gmap_notifier gmap_notifier;
 132 debug_info_t *kvm_s390_dbf;
 133
 134 /* Section: not file related */
 135 int kvm_arch_hardware_enable(void)
 136 {
 137         /* every s390 is virtualization enabled ;-) */
 138         return 0;
 139 }
 140
 141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 142
 143 /*
 144  * This callback is executed during stop_machine(). All CPUs are therefore
 145  * temporarily stopped. In order not to change guest behavior, we have to
 146  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 147  * so a CPU won't be stopped while calculating with the epoch.
 148  */
 149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 150                           void *v)
 151 {
 152         struct kvm *kvm;
 153         struct kvm_vcpu *vcpu;
 154         int i;
 155         unsigned long long *delta = v;
 156
 157         list_for_each_entry(kvm, &vm_list, vm_list) {
 158                 kvm->arch.epoch -= *delta;
 159                 kvm_for_each_vcpu(i, vcpu, kvm) {
 160                         vcpu->arch.sie_block->epoch -= *delta;
 161                 }
 162         }
 163         return NOTIFY_OK;
 164 }
 165
 166 static struct notifier_block kvm_clock_notifier = {
 167         .notifier_call = kvm_clock_sync,
 168 };
 169
 170 int kvm_arch_hardware_setup(void)
 171 {
 172         gmap_notifier.notifier_call = kvm_gmap_notifier;
 173         gmap_register_ipte_notifier(&gmap_notifier);
 174         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 175                                        &kvm_clock_notifier);
 176         return 0;
 177 }
 178
 179 void kvm_arch_hardware_unsetup(void)
 180 {
 181         gmap_unregister_ipte_notifier(&gmap_notifier);
 182         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 183                                          &kvm_clock_notifier);
 184 }
 185
 186 int kvm_arch_init(void *opaque)
 187 {
 188         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 189         if (!kvm_s390_dbf)
 190                 return -ENOMEM;
 191
 192         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 193                 debug_unregister(kvm_s390_dbf);
 194                 return -ENOMEM;
 195         }
 196
 197         /* Register floating interrupt controller interface. */
 198         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 199 }
 200
 201 void kvm_arch_exit(void)
 202 {
 203         debug_unregister(kvm_s390_dbf);
 204 }
 205
 206 /* Section: device related */
 207 long kvm_arch_dev_ioctl(struct file *filp,
 208                         unsigned int ioctl, unsigned long arg)
 209 {
 210         if (ioctl == KVM_S390_ENABLE_SIE)
 211                 return s390_enable_sie();
 212         return -EINVAL;
 213 }
 214
 215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 216 {
 217         int r;
 218
 219         switch (ext) {
 220         case KVM_CAP_S390_PSW:
 221         case KVM_CAP_S390_GMAP:
 222         case KVM_CAP_SYNC_MMU:
 223 #ifdef CONFIG_KVM_S390_UCONTROL
 224         case KVM_CAP_S390_UCONTROL:
 225 #endif
 226         case KVM_CAP_ASYNC_PF:
 227         case KVM_CAP_SYNC_REGS:
 228         case KVM_CAP_ONE_REG:
 229         case KVM_CAP_ENABLE_CAP:
 230         case KVM_CAP_S390_CSS_SUPPORT:
 231         case KVM_CAP_IOEVENTFD:
 232         case KVM_CAP_DEVICE_CTRL:
 233         case KVM_CAP_ENABLE_CAP_VM:
 234         case KVM_CAP_S390_IRQCHIP:
 235         case KVM_CAP_VM_ATTRIBUTES:
 236         case KVM_CAP_MP_STATE:
 237         case KVM_CAP_S390_INJECT_IRQ:
 238         case KVM_CAP_S390_USER_SIGP:
 239         case KVM_CAP_S390_USER_STSI:
 240         case KVM_CAP_S390_SKEYS:
 241         case KVM_CAP_S390_IRQ_STATE:
 242                 r = 1;
 243                 break;
 244         case KVM_CAP_S390_MEM_OP:
 245                 r = MEM_OP_MAX_SIZE;
 246                 break;
 247         case KVM_CAP_NR_VCPUS:
 248         case KVM_CAP_MAX_VCPUS:
 249                 r = KVM_MAX_VCPUS;
 250                 break;
 251         case KVM_CAP_NR_MEMSLOTS:
 252                 r = KVM_USER_MEM_SLOTS;
 253                 break;
 254         case KVM_CAP_S390_COW:
 255                 r = MACHINE_HAS_ESOP;
 256                 break;
 257         case KVM_CAP_S390_VECTOR_REGISTERS:
 258                 r = MACHINE_HAS_VX;
 259                 break;
 260         default:
 261                 r = 0;
 262         }
 263         return r;
 264 }
 265
 266 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 267                                         struct kvm_memory_slot *memslot)
 268 {
 269         gfn_t cur_gfn, last_gfn;
 270         unsigned long address;
 271         struct gmap *gmap = kvm->arch.gmap;
 272
 273         down_read(&gmap->mm->mmap_sem);
 274         /* Loop over all guest pages */
 275         last_gfn = memslot->base_gfn + memslot->npages;
 276         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 277                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 278
 279                 if (gmap_test_and_clear_dirty(address, gmap))
 280                         mark_page_dirty(kvm, cur_gfn);
 281         }
 282         up_read(&gmap->mm->mmap_sem);
 283 }
 284
 285 /* Section: vm related */
 286 /*
 287  * Get (and clear) the dirty memory log for a memory slot.
 288  */
 289 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 290                                struct kvm_dirty_log *log)
 291 {
 292         int r;
 293         unsigned long n;
 294         struct kvm_memslots *slots;
 295         struct kvm_memory_slot *memslot;
 296         int is_dirty = 0;
 297
 298         mutex_lock(&kvm->slots_lock);
 299
 300         r = -EINVAL;
 301         if (log->slot >= KVM_USER_MEM_SLOTS)
 302                 goto out;
 303
 304         slots = kvm_memslots(kvm);
 305         memslot = id_to_memslot(slots, log->slot);
 306         r = -ENOENT;
 307         if (!memslot->dirty_bitmap)
 308                 goto out;
 309
 310         kvm_s390_sync_dirty_log(kvm, memslot);
 311         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 312         if (r)
 313                 goto out;
 314
 315         /* Clear the dirty log */
 316         if (is_dirty) {
 317                 n = kvm_dirty_bitmap_bytes(memslot);
 318                 memset(memslot->dirty_bitmap, 0, n);
 319         }
 320         r = 0;
 321 out:
 322         mutex_unlock(&kvm->slots_lock);
 323         return r;
 324 }
 325
 326 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 327 {
 328         int r;
 329
 330         if (cap->flags)
 331                 return -EINVAL;
 332
 333         switch (cap->cap) {
 334         case KVM_CAP_S390_IRQCHIP:
 335                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 336                 kvm->arch.use_irqchip = 1;
 337                 r = 0;
 338                 break;
 339         case KVM_CAP_S390_USER_SIGP:
 340                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 341                 kvm->arch.user_sigp = 1;
 342                 r = 0;
 343                 break;
 344         case KVM_CAP_S390_VECTOR_REGISTERS:
 345                 if (MACHINE_HAS_VX) {
 346                         set_kvm_facility(kvm->arch.model.fac->mask, 129);
 347                         set_kvm_facility(kvm->arch.model.fac->list, 129);
 348                         r = 0;
 349                 } else
 350                         r = -EINVAL;
 351                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 352                          r ? "(not available)" : "(success)");
 353                 break;
 354         case KVM_CAP_S390_USER_STSI:
 355                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 356                 kvm->arch.user_stsi = 1;
 357                 r = 0;
 358                 break;
 359         default:
 360                 r = -EINVAL;
 361                 break;
 362         }
 363         return r;
 364 }
 365
 366 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 367 {
 368         int ret;
 369
 370         switch (attr->attr) {
 371         case KVM_S390_VM_MEM_LIMIT_SIZE:
 372                 ret = 0;
 373                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 374                          kvm->arch.gmap->asce_end);
 375                 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
 376                         ret = -EFAULT;
 377                 break;
 378         default:
 379                 ret = -ENXIO;
 380                 break;
 381         }
 382         return ret;
 383 }
 384
 385 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 386 {
 387         int ret;
 388         unsigned int idx;
 389         switch (attr->attr) {
 390         case KVM_S390_VM_MEM_ENABLE_CMMA:
 391                 /* enable CMMA only for z10 and later (EDAT_1) */
 392                 ret = -EINVAL;
 393                 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
 394                         break;
 395
 396                 ret = -EBUSY;
 397                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 398                 mutex_lock(&kvm->lock);
 399                 if (atomic_read(&kvm->online_vcpus) == 0) {
 400                         kvm->arch.use_cmma = 1;
 401                         ret = 0;
 402                 }
 403                 mutex_unlock(&kvm->lock);
 404                 break;
 405         case KVM_S390_VM_MEM_CLR_CMMA:
 406                 ret = -EINVAL;
 407                 if (!kvm->arch.use_cmma)
 408                         break;
 409
 410                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 411                 mutex_lock(&kvm->lock);
 412                 idx = srcu_read_lock(&kvm->srcu);
 413                 s390_reset_cmma(kvm->arch.gmap->mm);
 414                 srcu_read_unlock(&kvm->srcu, idx);
 415                 mutex_unlock(&kvm->lock);
 416                 ret = 0;
 417                 break;
 418         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 419                 unsigned long new_limit;
 420
 421                 if (kvm_is_ucontrol(kvm))
 422                         return -EINVAL;
 423
 424                 if (get_user(new_limit, (u64 __user *)attr->addr))
 425                         return -EFAULT;
 426
 427                 if (new_limit > kvm->arch.gmap->asce_end)
 428                         return -E2BIG;
 429
 430                 ret = -EBUSY;
 431                 mutex_lock(&kvm->lock);
 432                 if (atomic_read(&kvm->online_vcpus) == 0) {
 433                         /* gmap_alloc will round the limit up */
 434                         struct gmap *new = gmap_alloc(current->mm, new_limit);
 435
 436                         if (!new) {
 437                                 ret = -ENOMEM;
 438                         } else {
 439                                 gmap_free(kvm->arch.gmap);
 440                                 new->private = kvm;
 441                                 kvm->arch.gmap = new;
 442                                 ret = 0;
 443                         }
 444                 }
 445                 mutex_unlock(&kvm->lock);
 446                 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
 447                 break;
 448         }
 449         default:
 450                 ret = -ENXIO;
 451                 break;
 452         }
 453         return ret;
 454 }
 455
 456 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 457
 458 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 459 {
 460         struct kvm_vcpu *vcpu;
 461         int i;
 462
 463         if (!test_kvm_facility(kvm, 76))
 464                 return -EINVAL;
 465
 466         mutex_lock(&kvm->lock);
 467         switch (attr->attr) {
 468         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 469                 get_random_bytes(
 470                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 471                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 472                 kvm->arch.crypto.aes_kw = 1;
 473                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 474                 break;
 475         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 476                 get_random_bytes(
 477                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 478                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 479                 kvm->arch.crypto.dea_kw = 1;
 480                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 481                 break;
 482         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 483                 kvm->arch.crypto.aes_kw = 0;
 484                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 485                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 486                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 487                 break;
 488         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 489                 kvm->arch.crypto.dea_kw = 0;
 490                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 491                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 492                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 493                 break;
 494         default:
 495                 mutex_unlock(&kvm->lock);
 496                 return -ENXIO;
 497         }
 498
 499         kvm_for_each_vcpu(i, vcpu, kvm) {
 500                 kvm_s390_vcpu_crypto_setup(vcpu);
 501                 exit_sie(vcpu);
 502         }
 503         mutex_unlock(&kvm->lock);
 504         return 0;
 505 }
 506
 507 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 508 {
 509         u8 gtod_high;
 510
 511         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 512                                            sizeof(gtod_high)))
 513                 return -EFAULT;
 514
 515         if (gtod_high != 0)
 516                 return -EINVAL;
 517         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 518
 519         return 0;
 520 }
 521
 522 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 523 {
 524         u64 gtod;
 525
 526         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 527                 return -EFAULT;
 528
 529         kvm_s390_set_tod_clock(kvm, gtod);
 530         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 531         return 0;
 532 }
 533
 534 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 535 {
 536         int ret;
 537
 538         if (attr->flags)
 539                 return -EINVAL;
 540
 541         switch (attr->attr) {
 542         case KVM_S390_VM_TOD_HIGH:
 543                 ret = kvm_s390_set_tod_high(kvm, attr);
 544                 break;
 545         case KVM_S390_VM_TOD_LOW:
 546                 ret = kvm_s390_set_tod_low(kvm, attr);
 547                 break;
 548         default:
 549                 ret = -ENXIO;
 550                 break;
 551         }
 552         return ret;
 553 }
 554
 555 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 556 {
 557         u8 gtod_high = 0;
 558
 559         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 560                                          sizeof(gtod_high)))
 561                 return -EFAULT;
 562         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 563
 564         return 0;
 565 }
 566
 567 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 568 {
 569         u64 gtod;
 570
 571         gtod = kvm_s390_get_tod_clock_fast(kvm);
 572         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 573                 return -EFAULT;
 574         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 575
 576         return 0;
 577 }
 578
 579 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 580 {
 581         int ret;
 582
 583         if (attr->flags)
 584                 return -EINVAL;
 585
 586         switch (attr->attr) {
 587         case KVM_S390_VM_TOD_HIGH:
 588                 ret = kvm_s390_get_tod_high(kvm, attr);
 589                 break;
 590         case KVM_S390_VM_TOD_LOW:
 591                 ret = kvm_s390_get_tod_low(kvm, attr);
 592                 break;
 593         default:
 594                 ret = -ENXIO;
 595                 break;
 596         }
 597         return ret;
 598 }
 599
 600 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 601 {
 602         struct kvm_s390_vm_cpu_processor *proc;
 603         int ret = 0;
 604
 605         mutex_lock(&kvm->lock);
 606         if (atomic_read(&kvm->online_vcpus)) {
 607                 ret = -EBUSY;
 608                 goto out;
 609         }
 610         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 611         if (!proc) {
 612                 ret = -ENOMEM;
 613                 goto out;
 614         }
 615         if (!copy_from_user(proc, (void __user *)attr->addr,
 616                             sizeof(*proc))) {
 617                 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
 618                        sizeof(struct cpuid));
 619                 kvm->arch.model.ibc = proc->ibc;
 620                 memcpy(kvm->arch.model.fac->list, proc->fac_list,
 621                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 622         } else
 623                 ret = -EFAULT;
 624         kfree(proc);
 625 out:
 626         mutex_unlock(&kvm->lock);
 627         return ret;
 628 }
 629
 630 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 631 {
 632         int ret = -ENXIO;
 633
 634         switch (attr->attr) {
 635         case KVM_S390_VM_CPU_PROCESSOR:
 636                 ret = kvm_s390_set_processor(kvm, attr);
 637                 break;
 638         }
 639         return ret;
 640 }
 641
 642 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 643 {
 644         struct kvm_s390_vm_cpu_processor *proc;
 645         int ret = 0;
 646
 647         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 648         if (!proc) {
 649                 ret = -ENOMEM;
 650                 goto out;
 651         }
 652         memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
 653         proc->ibc = kvm->arch.model.ibc;
 654         memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
 655         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 656                 ret = -EFAULT;
 657         kfree(proc);
 658 out:
 659         return ret;
 660 }
 661
 662 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 663 {
 664         struct kvm_s390_vm_cpu_machine *mach;
 665         int ret = 0;
 666
 667         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 668         if (!mach) {
 669                 ret = -ENOMEM;
 670                 goto out;
 671         }
 672         get_cpu_id((struct cpuid *) &mach->cpuid);
 673         mach->ibc = sclp.ibc;
 674         memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
 675                S390_ARCH_FAC_LIST_SIZE_BYTE);
 676         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 677                S390_ARCH_FAC_LIST_SIZE_BYTE);
 678         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 679                 ret = -EFAULT;
 680         kfree(mach);
 681 out:
 682         return ret;
 683 }
 684
 685 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 686 {
 687         int ret = -ENXIO;
 688
 689         switch (attr->attr) {
 690         case KVM_S390_VM_CPU_PROCESSOR:
 691                 ret = kvm_s390_get_processor(kvm, attr);
 692                 break;
 693         case KVM_S390_VM_CPU_MACHINE:
 694                 ret = kvm_s390_get_machine(kvm, attr);
 695                 break;
 696         }
 697         return ret;
 698 }
 699
 700 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 701 {
 702         int ret;
 703
 704         switch (attr->group) {
 705         case KVM_S390_VM_MEM_CTRL:
 706                 ret = kvm_s390_set_mem_control(kvm, attr);
 707                 break;
 708         case KVM_S390_VM_TOD:
 709                 ret = kvm_s390_set_tod(kvm, attr);
 710                 break;
 711         case KVM_S390_VM_CPU_MODEL:
 712                 ret = kvm_s390_set_cpu_model(kvm, attr);
 713                 break;
 714         case KVM_S390_VM_CRYPTO:
 715                 ret = kvm_s390_vm_set_crypto(kvm, attr);
 716                 break;
 717         default:
 718                 ret = -ENXIO;
 719                 break;
 720         }
 721
 722         return ret;
 723 }
 724
 725 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 726 {
 727         int ret;
 728
 729         switch (attr->group) {
 730         case KVM_S390_VM_MEM_CTRL:
 731                 ret = kvm_s390_get_mem_control(kvm, attr);
 732                 break;
 733         case KVM_S390_VM_TOD:
 734                 ret = kvm_s390_get_tod(kvm, attr);
 735                 break;
 736         case KVM_S390_VM_CPU_MODEL:
 737                 ret = kvm_s390_get_cpu_model(kvm, attr);
 738                 break;
 739         default:
 740                 ret = -ENXIO;
 741                 break;
 742         }
 743
 744         return ret;
 745 }
 746
 747 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 748 {
 749         int ret;
 750
 751         switch (attr->group) {
 752         case KVM_S390_VM_MEM_CTRL:
 753                 switch (attr->attr) {
 754                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 755                 case KVM_S390_VM_MEM_CLR_CMMA:
 756                 case KVM_S390_VM_MEM_LIMIT_SIZE:
 757                         ret = 0;
 758                         break;
 759                 default:
 760                         ret = -ENXIO;
 761                         break;
 762                 }
 763                 break;
 764         case KVM_S390_VM_TOD:
 765                 switch (attr->attr) {
 766                 case KVM_S390_VM_TOD_LOW:
 767                 case KVM_S390_VM_TOD_HIGH:
 768                         ret = 0;
 769                         break;
 770                 default:
 771                         ret = -ENXIO;
 772                         break;
 773                 }
 774                 break;
 775         case KVM_S390_VM_CPU_MODEL:
 776                 switch (attr->attr) {
 777                 case KVM_S390_VM_CPU_PROCESSOR:
 778                 case KVM_S390_VM_CPU_MACHINE:
 779                         ret = 0;
 780                         break;
 781                 default:
 782                         ret = -ENXIO;
 783                         break;
 784                 }
 785                 break;
 786         case KVM_S390_VM_CRYPTO:
 787                 switch (attr->attr) {
 788                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 789                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 790                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 791                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 792                         ret = 0;
 793                         break;
 794                 default:
 795                         ret = -ENXIO;
 796                         break;
 797                 }
 798                 break;
 799         default:
 800                 ret = -ENXIO;
 801                 break;
 802         }
 803
 804         return ret;
 805 }
 806
 807 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 808 {
 809         uint8_t *keys;
 810         uint64_t hva;
 811         unsigned long curkey;
 812         int i, r = 0;
 813
 814         if (args->flags != 0)
 815                 return -EINVAL;
 816
 817         /* Is this guest using storage keys? */
 818         if (!mm_use_skey(current->mm))
 819                 return KVM_S390_GET_SKEYS_NONE;
 820
 821         /* Enforce sane limit on memory allocation */
 822         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 823                 return -EINVAL;
 824
 825         keys = kmalloc_array(args->count, sizeof(uint8_t),
 826                              GFP_KERNEL | __GFP_NOWARN);
 827         if (!keys)
 828                 keys = vmalloc(sizeof(uint8_t) * args->count);
 829         if (!keys)
 830                 return -ENOMEM;
 831
 832         for (i = 0; i < args->count; i++) {
 833                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 834                 if (kvm_is_error_hva(hva)) {
 835                         r = -EFAULT;
 836                         goto out;
 837                 }
 838
 839                 curkey = get_guest_storage_key(current->mm, hva);
 840                 if (IS_ERR_VALUE(curkey)) {
 841                         r = curkey;
 842                         goto out;
 843                 }
 844                 keys[i] = curkey;
 845         }
 846
 847         r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
 848                          sizeof(uint8_t) * args->count);
 849         if (r)
 850                 r = -EFAULT;
 851 out:
 852         kvfree(keys);
 853         return r;
 854 }
 855
 856 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 857 {
 858         uint8_t *keys;
 859         uint64_t hva;
 860         int i, r = 0;
 861
 862         if (args->flags != 0)
 863                 return -EINVAL;
 864
 865         /* Enforce sane limit on memory allocation */
 866         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 867                 return -EINVAL;
 868
 869         keys = kmalloc_array(args->count, sizeof(uint8_t),
 870                              GFP_KERNEL | __GFP_NOWARN);
 871         if (!keys)
 872                 keys = vmalloc(sizeof(uint8_t) * args->count);
 873         if (!keys)
 874                 return -ENOMEM;
 875
 876         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
 877                            sizeof(uint8_t) * args->count);
 878         if (r) {
 879                 r = -EFAULT;
 880                 goto out;
 881         }
 882
 883         /* Enable storage key handling for the guest */
 884         r = s390_enable_skey();
 885         if (r)
 886                 goto out;
 887
 888         for (i = 0; i < args->count; i++) {
 889                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 890                 if (kvm_is_error_hva(hva)) {
 891                         r = -EFAULT;
 892                         goto out;
 893                 }
 894
 895                 /* Lowest order bit is reserved */
 896                 if (keys[i] & 0x01) {
 897                         r = -EINVAL;
 898                         goto out;
 899                 }
 900
 901                 r = set_guest_storage_key(current->mm, hva,
 902                                           (unsigned long)keys[i], 0);
 903                 if (r)
 904                         goto out;
 905         }
 906 out:
 907         kvfree(keys);
 908         return r;
 909 }
 910
 911 long kvm_arch_vm_ioctl(struct file *filp,
 912                        unsigned int ioctl, unsigned long arg)
 913 {
 914         struct kvm *kvm = filp->private_data;
 915         void __user *argp = (void __user *)arg;
 916         struct kvm_device_attr attr;
 917         int r;
 918
 919         switch (ioctl) {
 920         case KVM_S390_INTERRUPT: {
 921                 struct kvm_s390_interrupt s390int;
 922
 923                 r = -EFAULT;
 924                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 925                         break;
 926                 r = kvm_s390_inject_vm(kvm, &s390int);
 927                 break;
 928         }
 929         case KVM_ENABLE_CAP: {
 930                 struct kvm_enable_cap cap;
 931                 r = -EFAULT;
 932                 if (copy_from_user(&cap, argp, sizeof(cap)))
 933                         break;
 934                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 935                 break;
 936         }
 937         case KVM_CREATE_IRQCHIP: {
 938                 struct kvm_irq_routing_entry routing;
 939
 940                 r = -EINVAL;
 941                 if (kvm->arch.use_irqchip) {
 942                         /* Set up dummy routing. */
 943                         memset(&routing, 0, sizeof(routing));
 944                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
 945                 }
 946                 break;
 947         }
 948         case KVM_SET_DEVICE_ATTR: {
 949                 r = -EFAULT;
 950                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 951                         break;
 952                 r = kvm_s390_vm_set_attr(kvm, &attr);
 953                 break;
 954         }
 955         case KVM_GET_DEVICE_ATTR: {
 956                 r = -EFAULT;
 957                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 958                         break;
 959                 r = kvm_s390_vm_get_attr(kvm, &attr);
 960                 break;
 961         }
 962         case KVM_HAS_DEVICE_ATTR: {
 963                 r = -EFAULT;
 964                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 965                         break;
 966                 r = kvm_s390_vm_has_attr(kvm, &attr);
 967                 break;
 968         }
 969         case KVM_S390_GET_SKEYS: {
 970                 struct kvm_s390_skeys args;
 971
 972                 r = -EFAULT;
 973                 if (copy_from_user(&args, argp,
 974                                    sizeof(struct kvm_s390_skeys)))
 975                         break;
 976                 r = kvm_s390_get_skeys(kvm, &args);
 977                 break;
 978         }
 979         case KVM_S390_SET_SKEYS: {
 980                 struct kvm_s390_skeys args;
 981
 982                 r = -EFAULT;
 983                 if (copy_from_user(&args, argp,
 984                                    sizeof(struct kvm_s390_skeys)))
 985                         break;
 986                 r = kvm_s390_set_skeys(kvm, &args);
 987                 break;
 988         }
 989         default:
 990                 r = -ENOTTY;
 991         }
 992
 993         return r;
 994 }
 995
 996 static int kvm_s390_query_ap_config(u8 *config)
 997 {
 998         u32 fcn_code = 0x04000000UL;
 999         u32 cc = 0;
1000
1001         memset(config, 0, 128);
1002         asm volatile(
1003                 "lgr 0,%1\n"
1004                 "lgr 2,%2\n"
1005                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1006                 "0: ipm %0\n"
1007                 "srl %0,28\n"
1008                 "1:\n"
1009                 EX_TABLE(0b, 1b)
1010                 : "+r" (cc)
1011                 : "r" (fcn_code), "r" (config)
1012                 : "cc", "0", "2", "memory"
1013         );
1014
1015         return cc;
1016 }
1017
1018 static int kvm_s390_apxa_installed(void)
1019 {
1020         u8 config[128];
1021         int cc;
1022
1023         if (test_facility(2) && test_facility(12)) {
1024                 cc = kvm_s390_query_ap_config(config);
1025
1026                 if (cc)
1027                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1028                 else
1029                         return config[0] & 0x40;
1030         }
1031
1032         return 0;
1033 }
1034
1035 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1036 {
1037         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1038
1039         if (kvm_s390_apxa_installed())
1040                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1041         else
1042                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1043 }
1044
1045 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1046 {
1047         get_cpu_id(cpu_id);
1048         cpu_id->version = 0xff;
1049 }
1050
1051 static int kvm_s390_crypto_init(struct kvm *kvm)
1052 {
1053         if (!test_kvm_facility(kvm, 76))
1054                 return 0;
1055
1056         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1057                                          GFP_KERNEL | GFP_DMA);
1058         if (!kvm->arch.crypto.crycb)
1059                 return -ENOMEM;
1060
1061         kvm_s390_set_crycb_format(kvm);
1062
1063         /* Enable AES/DEA protected key functions by default */
1064         kvm->arch.crypto.aes_kw = 1;
1065         kvm->arch.crypto.dea_kw = 1;
1066         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1067                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1068         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1069                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1070
1071         return 0;
1072 }
1073
1074 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1075 {
1076         int i, rc;
1077         char debug_name[16];
1078         static unsigned long sca_offset;
1079
1080         rc = -EINVAL;
1081 #ifdef CONFIG_KVM_S390_UCONTROL
1082         if (type & ~KVM_VM_S390_UCONTROL)
1083                 goto out_err;
1084         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1085                 goto out_err;
1086 #else
1087         if (type)
1088                 goto out_err;
1089 #endif
1090
1091         rc = s390_enable_sie();
1092         if (rc)
1093                 goto out_err;
1094
1095         rc = -ENOMEM;
1096
1097         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1098         if (!kvm->arch.sca)
1099                 goto out_err;
1100         spin_lock(&kvm_lock);
1101         sca_offset += 16;
1102         if (sca_offset + sizeof(struct sca_block) > PAGE_SIZE)
1103                 sca_offset = 0;
1104         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1105         spin_unlock(&kvm_lock);
1106
1107         sprintf(debug_name, "kvm-%u", current->pid);
1108
1109         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1110         if (!kvm->arch.dbf)
1111                 goto out_err;
1112
1113         /*
1114          * The architectural maximum amount of facilities is 16 kbit. To store
1115          * this amount, 2 kbyte of memory is required. Thus we need a full
1116          * page to hold the guest facility list (arch.model.fac->list) and the
1117          * facility mask (arch.model.fac->mask). Its address size has to be
1118          * 31 bits and word aligned.
1119          */
1120         kvm->arch.model.fac =
1121                 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1122         if (!kvm->arch.model.fac)
1123                 goto out_err;
1124
1125         /* Populate the facility mask initially. */
1126         memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1127                S390_ARCH_FAC_LIST_SIZE_BYTE);
1128         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1129                 if (i < kvm_s390_fac_list_mask_size())
1130                         kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1131                 else
1132                         kvm->arch.model.fac->mask[i] = 0UL;
1133         }
1134
1135         /* Populate the facility list initially. */
1136         memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1137                S390_ARCH_FAC_LIST_SIZE_BYTE);
1138
1139         kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1140         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1141
1142         if (kvm_s390_crypto_init(kvm) < 0)
1143                 goto out_err;
1144
1145         spin_lock_init(&kvm->arch.float_int.lock);
1146         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1147                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1148         init_waitqueue_head(&kvm->arch.ipte_wq);
1149         mutex_init(&kvm->arch.ipte_mutex);
1150
1151         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1152         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1153
1154         if (type & KVM_VM_S390_UCONTROL) {
1155                 kvm->arch.gmap = NULL;
1156         } else {
1157                 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1158                 if (!kvm->arch.gmap)
1159                         goto out_err;
1160                 kvm->arch.gmap->private = kvm;
1161                 kvm->arch.gmap->pfault_enabled = 0;
1162         }
1163
1164         kvm->arch.css_support = 0;
1165         kvm->arch.use_irqchip = 0;
1166         kvm->arch.epoch = 0;
1167
1168         spin_lock_init(&kvm->arch.start_stop_lock);
1169         KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1170
1171         return 0;
1172 out_err:
1173         kfree(kvm->arch.crypto.crycb);
1174         free_page((unsigned long)kvm->arch.model.fac);
1175         debug_unregister(kvm->arch.dbf);
1176         free_page((unsigned long)(kvm->arch.sca));
1177         KVM_EVENT(3, "creation of vm failed: %d", rc);
1178         return rc;
1179 }
1180
1181 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1182 {
1183         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1184         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1185         kvm_s390_clear_local_irqs(vcpu);
1186         kvm_clear_async_pf_completion_queue(vcpu);
1187         if (!kvm_is_ucontrol(vcpu->kvm)) {
1188                 clear_bit(63 - vcpu->vcpu_id,
1189                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1190                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1191                     (__u64) vcpu->arch.sie_block)
1192                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1193         }
1194         smp_mb();
1195
1196         if (kvm_is_ucontrol(vcpu->kvm))
1197                 gmap_free(vcpu->arch.gmap);
1198
1199         if (vcpu->kvm->arch.use_cmma)
1200                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1201         free_page((unsigned long)(vcpu->arch.sie_block));
1202
1203         kvm_vcpu_uninit(vcpu);
1204         kmem_cache_free(kvm_vcpu_cache, vcpu);
1205 }
1206
1207 static void kvm_free_vcpus(struct kvm *kvm)
1208 {
1209         unsigned int i;
1210         struct kvm_vcpu *vcpu;
1211
1212         kvm_for_each_vcpu(i, vcpu, kvm)
1213                 kvm_arch_vcpu_destroy(vcpu);
1214
1215         mutex_lock(&kvm->lock);
1216         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1217                 kvm->vcpus[i] = NULL;
1218
1219         atomic_set(&kvm->online_vcpus, 0);
1220         mutex_unlock(&kvm->lock);
1221 }
1222
1223 void kvm_arch_destroy_vm(struct kvm *kvm)
1224 {
1225         kvm_free_vcpus(kvm);
1226         free_page((unsigned long)kvm->arch.model.fac);
1227         free_page((unsigned long)(kvm->arch.sca));
1228         debug_unregister(kvm->arch.dbf);
1229         kfree(kvm->arch.crypto.crycb);
1230         if (!kvm_is_ucontrol(kvm))
1231                 gmap_free(kvm->arch.gmap);
1232         kvm_s390_destroy_adapters(kvm);
1233         kvm_s390_clear_float_irqs(kvm);
1234         KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1235 }
1236
1237 /* Section: vcpu related */
1238 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1239 {
1240         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1241         if (!vcpu->arch.gmap)
1242                 return -ENOMEM;
1243         vcpu->arch.gmap->private = vcpu->kvm;
1244
1245         return 0;
1246 }
1247
1248 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1249 {
1250         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1251         kvm_clear_async_pf_completion_queue(vcpu);
1252         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1253                                     KVM_SYNC_GPRS |
1254                                     KVM_SYNC_ACRS |
1255                                     KVM_SYNC_CRS |
1256                                     KVM_SYNC_ARCH0 |
1257                                     KVM_SYNC_PFAULT;
1258         if (test_kvm_facility(vcpu->kvm, 129))
1259                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1260
1261         if (kvm_is_ucontrol(vcpu->kvm))
1262                 return __kvm_ucontrol_vcpu_init(vcpu);
1263
1264         return 0;
1265 }
1266
1267 /*
1268  * Backs up the current FP/VX register save area on a particular
1269  * destination.  Used to switch between different register save
1270  * areas.
1271  */
1272 static inline void save_fpu_to(struct fpu *dst)
1273 {
1274         dst->fpc = current->thread.fpu.fpc;
1275         dst->regs = current->thread.fpu.regs;
1276 }
1277
1278 /*
1279  * Switches the FP/VX register save area from which to lazy
1280  * restore register contents.
1281  */
1282 static inline void load_fpu_from(struct fpu *from)
1283 {
1284         current->thread.fpu.fpc = from->fpc;
1285         current->thread.fpu.regs = from->regs;
1286 }
1287
1288 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1289 {
1290         /* Save host register state */
1291         save_fpu_regs();
1292         save_fpu_to(&vcpu->arch.host_fpregs);
1293
1294         if (test_kvm_facility(vcpu->kvm, 129)) {
1295                 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1296                 /*
1297                  * Use the register save area in the SIE-control block
1298                  * for register restore and save in kvm_arch_vcpu_put()
1299                  */
1300                 current->thread.fpu.vxrs =
1301                         (__vector128 *)&vcpu->run->s.regs.vrs;
1302         } else
1303                 load_fpu_from(&vcpu->arch.guest_fpregs);
1304
1305         if (test_fp_ctl(current->thread.fpu.fpc))
1306                 /* User space provided an invalid FPC, let's clear it */
1307                 current->thread.fpu.fpc = 0;
1308
1309         save_access_regs(vcpu->arch.host_acrs);
1310         restore_access_regs(vcpu->run->s.regs.acrs);
1311         gmap_enable(vcpu->arch.gmap);
1312         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1313 }
1314
1315 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1316 {
1317         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1318         gmap_disable(vcpu->arch.gmap);
1319
1320         save_fpu_regs();
1321
1322         if (test_kvm_facility(vcpu->kvm, 129))
1323                 /*
1324                  * kvm_arch_vcpu_load() set up the register save area to
1325                  * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1326                  * are already saved.  Only the floating-point control must be
1327                  * copied.
1328                  */
1329                 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1330         else
1331                 save_fpu_to(&vcpu->arch.guest_fpregs);
1332         load_fpu_from(&vcpu->arch.host_fpregs);
1333
1334         save_access_regs(vcpu->run->s.regs.acrs);
1335         restore_access_regs(vcpu->arch.host_acrs);
1336 }
1337
1338 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1339 {
1340         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1341         vcpu->arch.sie_block->gpsw.mask = 0UL;
1342         vcpu->arch.sie_block->gpsw.addr = 0UL;
1343         kvm_s390_set_prefix(vcpu, 0);
1344         vcpu->arch.sie_block->cputm     = 0UL;
1345         vcpu->arch.sie_block->ckc       = 0UL;
1346         vcpu->arch.sie_block->todpr     = 0;
1347         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1348         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1349         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1350         vcpu->arch.guest_fpregs.fpc = 0;
1351         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1352         vcpu->arch.sie_block->gbea = 1;
1353         vcpu->arch.sie_block->pp = 0;
1354         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1355         kvm_clear_async_pf_completion_queue(vcpu);
1356         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1357                 kvm_s390_vcpu_stop(vcpu);
1358         kvm_s390_clear_local_irqs(vcpu);
1359 }
1360
1361 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1362 {
1363         mutex_lock(&vcpu->kvm->lock);
1364         preempt_disable();
1365         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1366         preempt_enable();
1367         mutex_unlock(&vcpu->kvm->lock);
1368         if (!kvm_is_ucontrol(vcpu->kvm))
1369                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1370 }
1371
1372 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1373 {
1374         if (!test_kvm_facility(vcpu->kvm, 76))
1375                 return;
1376
1377         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1378
1379         if (vcpu->kvm->arch.crypto.aes_kw)
1380                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1381         if (vcpu->kvm->arch.crypto.dea_kw)
1382                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1383
1384         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1385 }
1386
1387 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1388 {
1389         free_page(vcpu->arch.sie_block->cbrlo);
1390         vcpu->arch.sie_block->cbrlo = 0;
1391 }
1392
1393 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1394 {
1395         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1396         if (!vcpu->arch.sie_block->cbrlo)
1397                 return -ENOMEM;
1398
1399         vcpu->arch.sie_block->ecb2 |= 0x80;
1400         vcpu->arch.sie_block->ecb2 &= ~0x08;
1401         return 0;
1402 }
1403
1404 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1405 {
1406         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1407
1408         vcpu->arch.cpu_id = model->cpu_id;
1409         vcpu->arch.sie_block->ibc = model->ibc;
1410         vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1411 }
1412
1413 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1414 {
1415         int rc = 0;
1416
1417         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1418                                                     CPUSTAT_SM |
1419                                                     CPUSTAT_STOPPED);
1420
1421         if (test_kvm_facility(vcpu->kvm, 78))
1422                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1423         else if (test_kvm_facility(vcpu->kvm, 8))
1424                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1425
1426         kvm_s390_vcpu_setup_model(vcpu);
1427
1428         vcpu->arch.sie_block->ecb   = 6;
1429         if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1430                 vcpu->arch.sie_block->ecb |= 0x10;
1431
1432         vcpu->arch.sie_block->ecb2  = 8;
1433         vcpu->arch.sie_block->eca   = 0xC1002000U;
1434         if (sclp.has_siif)
1435                 vcpu->arch.sie_block->eca |= 1;
1436         if (sclp.has_sigpif)
1437                 vcpu->arch.sie_block->eca |= 0x10000000U;
1438         if (test_kvm_facility(vcpu->kvm, 129)) {
1439                 vcpu->arch.sie_block->eca |= 0x00020000;
1440                 vcpu->arch.sie_block->ecd |= 0x20000000;
1441         }
1442         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1443
1444         if (vcpu->kvm->arch.use_cmma) {
1445                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1446                 if (rc)
1447                         return rc;
1448         }
1449         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1450         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1451
1452         kvm_s390_vcpu_crypto_setup(vcpu);
1453
1454         return rc;
1455 }
1456
1457 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1458                                       unsigned int id)
1459 {
1460         struct kvm_vcpu *vcpu;
1461         struct sie_page *sie_page;
1462         int rc = -EINVAL;
1463
1464         if (id >= KVM_MAX_VCPUS)
1465                 goto out;
1466
1467         rc = -ENOMEM;
1468
1469         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1470         if (!vcpu)
1471                 goto out;
1472
1473         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1474         if (!sie_page)
1475                 goto out_free_cpu;
1476
1477         vcpu->arch.sie_block = &sie_page->sie_block;
1478         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1479
1480         vcpu->arch.sie_block->icpua = id;
1481         if (!kvm_is_ucontrol(kvm)) {
1482                 if (!kvm->arch.sca) {
1483                         WARN_ON_ONCE(1);
1484                         goto out_free_cpu;
1485                 }
1486                 if (!kvm->arch.sca->cpu[id].sda)
1487                         kvm->arch.sca->cpu[id].sda =
1488                                 (__u64) vcpu->arch.sie_block;
1489                 vcpu->arch.sie_block->scaoh =
1490                         (__u32)(((__u64)kvm->arch.sca) >> 32);
1491                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1492                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1493         }
1494
1495         spin_lock_init(&vcpu->arch.local_int.lock);
1496         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1497         vcpu->arch.local_int.wq = &vcpu->wq;
1498         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1499
1500         /*
1501          * Allocate a save area for floating-point registers.  If the vector
1502          * extension is available, register contents are saved in the SIE
1503          * control block.  The allocated save area is still required in
1504          * particular places, for example, in kvm_s390_vcpu_store_status().
1505          */
1506         vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1507                                                GFP_KERNEL);
1508         if (!vcpu->arch.guest_fpregs.fprs) {
1509                 rc = -ENOMEM;
1510                 goto out_free_sie_block;
1511         }
1512
1513         rc = kvm_vcpu_init(vcpu, kvm, id);
1514         if (rc)
1515                 goto out_free_sie_block;
1516         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1517                  vcpu->arch.sie_block);
1518         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1519
1520         return vcpu;
1521 out_free_sie_block:
1522         free_page((unsigned long)(vcpu->arch.sie_block));
1523 out_free_cpu:
1524         kmem_cache_free(kvm_vcpu_cache, vcpu);
1525 out:
1526         return ERR_PTR(rc);
1527 }
1528
1529 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1530 {
1531         return kvm_s390_vcpu_has_irq(vcpu, 0);
1532 }
1533
1534 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1535 {
1536         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1537         exit_sie(vcpu);
1538 }
1539
1540 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1541 {
1542         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1543 }
1544
1545 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1546 {
1547         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1548         exit_sie(vcpu);
1549 }
1550
1551 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1552 {
1553         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1554 }
1555
1556 /*
1557  * Kick a guest cpu out of SIE and wait until SIE is not running.
1558  * If the CPU is not running (e.g. waiting as idle) the function will
1559  * return immediately. */
1560 void exit_sie(struct kvm_vcpu *vcpu)
1561 {
1562         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1563         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1564                 cpu_relax();
1565 }
1566
1567 /* Kick a guest cpu out of SIE to process a request synchronously */
1568 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1569 {
1570         kvm_make_request(req, vcpu);
1571         kvm_s390_vcpu_request(vcpu);
1572 }
1573
1574 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1575 {
1576         int i;
1577         struct kvm *kvm = gmap->private;
1578         struct kvm_vcpu *vcpu;
1579
1580         kvm_for_each_vcpu(i, vcpu, kvm) {
1581                 /* match against both prefix pages */
1582                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1583                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1584                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1585                 }
1586         }
1587 }
1588
1589 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1590 {
1591         /* kvm common code refers to this, but never calls it */
1592         BUG();
1593         return 0;
1594 }
1595
1596 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1597                                            struct kvm_one_reg *reg)
1598 {
1599         int r = -EINVAL;
1600
1601         switch (reg->id) {
1602         case KVM_REG_S390_TODPR:
1603                 r = put_user(vcpu->arch.sie_block->todpr,
1604                              (u32 __user *)reg->addr);
1605                 break;
1606         case KVM_REG_S390_EPOCHDIFF:
1607                 r = put_user(vcpu->arch.sie_block->epoch,
1608                              (u64 __user *)reg->addr);
1609                 break;
1610         case KVM_REG_S390_CPU_TIMER:
1611                 r = put_user(vcpu->arch.sie_block->cputm,
1612                              (u64 __user *)reg->addr);
1613                 break;
1614         case KVM_REG_S390_CLOCK_COMP:
1615                 r = put_user(vcpu->arch.sie_block->ckc,
1616                              (u64 __user *)reg->addr);
1617                 break;
1618         case KVM_REG_S390_PFTOKEN:
1619                 r = put_user(vcpu->arch.pfault_token,
1620                              (u64 __user *)reg->addr);
1621                 break;
1622         case KVM_REG_S390_PFCOMPARE:
1623                 r = put_user(vcpu->arch.pfault_compare,
1624                              (u64 __user *)reg->addr);
1625                 break;
1626         case KVM_REG_S390_PFSELECT:
1627                 r = put_user(vcpu->arch.pfault_select,
1628                              (u64 __user *)reg->addr);
1629                 break;
1630         case KVM_REG_S390_PP:
1631                 r = put_user(vcpu->arch.sie_block->pp,
1632                              (u64 __user *)reg->addr);
1633                 break;
1634         case KVM_REG_S390_GBEA:
1635                 r = put_user(vcpu->arch.sie_block->gbea,
1636                              (u64 __user *)reg->addr);
1637                 break;
1638         default:
1639                 break;
1640         }
1641
1642         return r;
1643 }
1644
1645 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1646                                            struct kvm_one_reg *reg)
1647 {
1648         int r = -EINVAL;
1649
1650         switch (reg->id) {
1651         case KVM_REG_S390_TODPR:
1652                 r = get_user(vcpu->arch.sie_block->todpr,
1653                              (u32 __user *)reg->addr);
1654                 break;
1655         case KVM_REG_S390_EPOCHDIFF:
1656                 r = get_user(vcpu->arch.sie_block->epoch,
1657                              (u64 __user *)reg->addr);
1658                 break;
1659         case KVM_REG_S390_CPU_TIMER:
1660                 r = get_user(vcpu->arch.sie_block->cputm,
1661                              (u64 __user *)reg->addr);
1662                 break;
1663         case KVM_REG_S390_CLOCK_COMP:
1664                 r = get_user(vcpu->arch.sie_block->ckc,
1665                              (u64 __user *)reg->addr);
1666                 break;
1667         case KVM_REG_S390_PFTOKEN:
1668                 r = get_user(vcpu->arch.pfault_token,
1669                              (u64 __user *)reg->addr);
1670                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1671                         kvm_clear_async_pf_completion_queue(vcpu);
1672                 break;
1673         case KVM_REG_S390_PFCOMPARE:
1674                 r = get_user(vcpu->arch.pfault_compare,
1675                              (u64 __user *)reg->addr);
1676                 break;
1677         case KVM_REG_S390_PFSELECT:
1678                 r = get_user(vcpu->arch.pfault_select,
1679                              (u64 __user *)reg->addr);
1680                 break;
1681         case KVM_REG_S390_PP:
1682                 r = get_user(vcpu->arch.sie_block->pp,
1683                              (u64 __user *)reg->addr);
1684                 break;
1685         case KVM_REG_S390_GBEA:
1686                 r = get_user(vcpu->arch.sie_block->gbea,
1687                              (u64 __user *)reg->addr);
1688                 break;
1689         default:
1690                 break;
1691         }
1692
1693         return r;
1694 }
1695
1696 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1697 {
1698         kvm_s390_vcpu_initial_reset(vcpu);
1699         return 0;
1700 }
1701
1702 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1703 {
1704         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1705         return 0;
1706 }
1707
1708 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1709 {
1710         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1711         return 0;
1712 }
1713
1714 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1715                                   struct kvm_sregs *sregs)
1716 {
1717         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1718         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1719         restore_access_regs(vcpu->run->s.regs.acrs);
1720         return 0;
1721 }
1722
1723 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1724                                   struct kvm_sregs *sregs)
1725 {
1726         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1727         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1728         return 0;
1729 }
1730
1731 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1732 {
1733         if (test_fp_ctl(fpu->fpc))
1734                 return -EINVAL;
1735         memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1736         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1737         save_fpu_regs();
1738         load_fpu_from(&vcpu->arch.guest_fpregs);
1739         return 0;
1740 }
1741
1742 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1743 {
1744         memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1745         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1746         return 0;
1747 }
1748
1749 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1750 {
1751         int rc = 0;
1752
1753         if (!is_vcpu_stopped(vcpu))
1754                 rc = -EBUSY;
1755         else {
1756                 vcpu->run->psw_mask = psw.mask;
1757                 vcpu->run->psw_addr = psw.addr;
1758         }
1759         return rc;
1760 }
1761
1762 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1763                                   struct kvm_translation *tr)
1764 {
1765         return -EINVAL; /* not implemented yet */
1766 }
1767
1768 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1769                               KVM_GUESTDBG_USE_HW_BP | \
1770                               KVM_GUESTDBG_ENABLE)
1771
1772 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1773                                         struct kvm_guest_debug *dbg)
1774 {
1775         int rc = 0;
1776
1777         vcpu->guest_debug = 0;
1778         kvm_s390_clear_bp_data(vcpu);
1779
1780         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1781                 return -EINVAL;
1782
1783         if (dbg->control & KVM_GUESTDBG_ENABLE) {
1784                 vcpu->guest_debug = dbg->control;
1785                 /* enforce guest PER */
1786                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1787
1788                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1789                         rc = kvm_s390_import_bp_data(vcpu, dbg);
1790         } else {
1791                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1792                 vcpu->arch.guestdbg.last_bp = 0;
1793         }
1794
1795         if (rc) {
1796                 vcpu->guest_debug = 0;
1797                 kvm_s390_clear_bp_data(vcpu);
1798                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1799         }
1800
1801         return rc;
1802 }
1803
1804 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1805                                     struct kvm_mp_state *mp_state)
1806 {
1807         /* CHECK_STOP and LOAD are not supported yet */
1808         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1809                                        KVM_MP_STATE_OPERATING;
1810 }
1811
1812 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1813                                     struct kvm_mp_state *mp_state)
1814 {
1815         int rc = 0;
1816
1817         /* user space knows about this interface - let it control the state */
1818         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1819
1820         switch (mp_state->mp_state) {
1821         case KVM_MP_STATE_STOPPED:
1822                 kvm_s390_vcpu_stop(vcpu);
1823                 break;
1824         case KVM_MP_STATE_OPERATING:
1825                 kvm_s390_vcpu_start(vcpu);
1826                 break;
1827         case KVM_MP_STATE_LOAD:
1828         case KVM_MP_STATE_CHECK_STOP:
1829                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1830         default:
1831                 rc = -ENXIO;
1832         }
1833
1834         return rc;
1835 }
1836
1837 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1838 {
1839         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1840 }
1841
1842 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1843 {
1844 retry:
1845         kvm_s390_vcpu_request_handled(vcpu);
1846         if (!vcpu->requests)
1847                 return 0;
1848         /*
1849          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1850          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1851          * This ensures that the ipte instruction for this request has
1852          * already finished. We might race against a second unmapper that
1853          * wants to set the blocking bit. Lets just retry the request loop.
1854          */
1855         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1856                 int rc;
1857                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1858                                       kvm_s390_get_prefix(vcpu),
1859                                       PAGE_SIZE * 2);
1860                 if (rc)
1861                         return rc;
1862                 goto retry;
1863         }
1864
1865         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1866                 vcpu->arch.sie_block->ihcpu = 0xffff;
1867                 goto retry;
1868         }
1869
1870         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1871                 if (!ibs_enabled(vcpu)) {
1872                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1873                         atomic_or(CPUSTAT_IBS,
1874                                         &vcpu->arch.sie_block->cpuflags);
1875                 }
1876                 goto retry;
1877         }
1878
1879         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1880                 if (ibs_enabled(vcpu)) {
1881                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1882                         atomic_andnot(CPUSTAT_IBS,
1883                                           &vcpu->arch.sie_block->cpuflags);
1884                 }
1885                 goto retry;
1886         }
1887
1888         /* nothing to do, just clear the request */
1889         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1890
1891         return 0;
1892 }
1893
1894 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
1895 {
1896         struct kvm_vcpu *vcpu;
1897         int i;
1898
1899         mutex_lock(&kvm->lock);
1900         preempt_disable();
1901         kvm->arch.epoch = tod - get_tod_clock();
1902         kvm_s390_vcpu_block_all(kvm);
1903         kvm_for_each_vcpu(i, vcpu, kvm)
1904                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
1905         kvm_s390_vcpu_unblock_all(kvm);
1906         preempt_enable();
1907         mutex_unlock(&kvm->lock);
1908 }
1909
1910 /**
1911  * kvm_arch_fault_in_page - fault-in guest page if necessary
1912  * @vcpu: The corresponding virtual cpu
1913  * @gpa: Guest physical address
1914  * @writable: Whether the page should be writable or not
1915  *
1916  * Make sure that a guest page has been faulted-in on the host.
1917  *
1918  * Return: Zero on success, negative error code otherwise.
1919  */
1920 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1921 {
1922         return gmap_fault(vcpu->arch.gmap, gpa,
1923                           writable ? FAULT_FLAG_WRITE : 0);
1924 }
1925
1926 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1927                                       unsigned long token)
1928 {
1929         struct kvm_s390_interrupt inti;
1930         struct kvm_s390_irq irq;
1931
1932         if (start_token) {
1933                 irq.u.ext.ext_params2 = token;
1934                 irq.type = KVM_S390_INT_PFAULT_INIT;
1935                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1936         } else {
1937                 inti.type = KVM_S390_INT_PFAULT_DONE;
1938                 inti.parm64 = token;
1939                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1940         }
1941 }
1942
1943 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1944                                      struct kvm_async_pf *work)
1945 {
1946         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1947         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1948 }
1949
1950 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1951                                  struct kvm_async_pf *work)
1952 {
1953         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1954         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1955 }
1956
1957 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1958                                struct kvm_async_pf *work)
1959 {
1960         /* s390 will always inject the page directly */
1961 }
1962
1963 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1964 {
1965         /*
1966          * s390 will always inject the page directly,
1967          * but we still want check_async_completion to cleanup
1968          */
1969         return true;
1970 }
1971
1972 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1973 {
1974         hva_t hva;
1975         struct kvm_arch_async_pf arch;
1976         int rc;
1977
1978         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1979                 return 0;
1980         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1981             vcpu->arch.pfault_compare)
1982                 return 0;
1983         if (psw_extint_disabled(vcpu))
1984                 return 0;
1985         if (kvm_s390_vcpu_has_irq(vcpu, 0))
1986                 return 0;
1987         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1988                 return 0;
1989         if (!vcpu->arch.gmap->pfault_enabled)
1990                 return 0;
1991
1992         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1993         hva += current->thread.gmap_addr & ~PAGE_MASK;
1994         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1995                 return 0;
1996
1997         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1998         return rc;
1999 }
2000
2001 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2002 {
2003         int rc, cpuflags;
2004
2005         /*
2006          * On s390 notifications for arriving pages will be delivered directly
2007          * to the guest but the house keeping for completed pfaults is
2008          * handled outside the worker.
2009          */
2010         kvm_check_async_pf_completion(vcpu);
2011
2012         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
2013
2014         if (need_resched())
2015                 schedule();
2016
2017         if (test_cpu_flag(CIF_MCCK_PENDING))
2018                 s390_handle_mcck();
2019
2020         if (!kvm_is_ucontrol(vcpu->kvm)) {
2021                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2022                 if (rc)
2023                         return rc;
2024         }
2025
2026         rc = kvm_s390_handle_requests(vcpu);
2027         if (rc)
2028                 return rc;
2029
2030         if (guestdbg_enabled(vcpu)) {
2031                 kvm_s390_backup_guest_per_regs(vcpu);
2032                 kvm_s390_patch_guest_per_regs(vcpu);
2033         }
2034
2035         vcpu->arch.sie_block->icptcode = 0;
2036         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2037         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2038         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2039
2040         return 0;
2041 }
2042
2043 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2044 {
2045         psw_t *psw = &vcpu->arch.sie_block->gpsw;
2046         u8 opcode;
2047         int rc;
2048
2049         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2050         trace_kvm_s390_sie_fault(vcpu);
2051
2052         /*
2053          * We want to inject an addressing exception, which is defined as a
2054          * suppressing or terminating exception. However, since we came here
2055          * by a DAT access exception, the PSW still points to the faulting
2056          * instruction since DAT exceptions are nullifying. So we've got
2057          * to look up the current opcode to get the length of the instruction
2058          * to be able to forward the PSW.
2059          */
2060         rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2061         if (rc)
2062                 return kvm_s390_inject_prog_cond(vcpu, rc);
2063         psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2064
2065         return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2066 }
2067
2068 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2069 {
2070         int rc = -1;
2071
2072         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2073                    vcpu->arch.sie_block->icptcode);
2074         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2075
2076         if (guestdbg_enabled(vcpu))
2077                 kvm_s390_restore_guest_per_regs(vcpu);
2078
2079         if (exit_reason >= 0) {
2080                 rc = 0;
2081         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2082                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2083                 vcpu->run->s390_ucontrol.trans_exc_code =
2084                                                 current->thread.gmap_addr;
2085                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2086                 rc = -EREMOTE;
2087
2088         } else if (current->thread.gmap_pfault) {
2089                 trace_kvm_s390_major_guest_pfault(vcpu);
2090                 current->thread.gmap_pfault = 0;
2091                 if (kvm_arch_setup_async_pf(vcpu)) {
2092                         rc = 0;
2093                 } else {
2094                         gpa_t gpa = current->thread.gmap_addr;
2095                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
2096                 }
2097         }
2098
2099         if (rc == -1)
2100                 rc = vcpu_post_run_fault_in_sie(vcpu);
2101
2102         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2103
2104         if (rc == 0) {
2105                 if (kvm_is_ucontrol(vcpu->kvm))
2106                         /* Don't exit for host interrupts. */
2107                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
2108                 else
2109                         rc = kvm_handle_sie_intercept(vcpu);
2110         }
2111
2112         return rc;
2113 }
2114
2115 static int __vcpu_run(struct kvm_vcpu *vcpu)
2116 {
2117         int rc, exit_reason;
2118
2119         /*
2120          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2121          * ning the guest), so that memslots (and other stuff) are protected
2122          */
2123         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2124
2125         do {
2126                 rc = vcpu_pre_run(vcpu);
2127                 if (rc)
2128                         break;
2129
2130                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2131                 /*
2132                  * As PF_VCPU will be used in fault handler, between
2133                  * guest_enter and guest_exit should be no uaccess.
2134                  */
2135                 local_irq_disable();
2136                 __kvm_guest_enter();
2137                 local_irq_enable();
2138                 exit_reason = sie64a(vcpu->arch.sie_block,
2139                                      vcpu->run->s.regs.gprs);
2140                 local_irq_disable();
2141                 __kvm_guest_exit();
2142                 local_irq_enable();
2143                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2144
2145                 rc = vcpu_post_run(vcpu, exit_reason);
2146         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2147
2148         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2149         return rc;
2150 }
2151
2152 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2153 {
2154         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2155         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2156         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2157                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2158         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2159                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2160                 /* some control register changes require a tlb flush */
2161                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2162         }
2163         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2164                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2165                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2166                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2167                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2168                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2169         }
2170         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2171                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2172                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2173                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2174                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2175                         kvm_clear_async_pf_completion_queue(vcpu);
2176         }
2177         kvm_run->kvm_dirty_regs = 0;
2178 }
2179
2180 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2181 {
2182         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2183         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2184         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2185         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2186         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2187         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2188         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2189         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2190         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2191         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2192         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2193         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2194 }
2195
2196 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2197 {
2198         int rc;
2199         sigset_t sigsaved;
2200
2201         if (guestdbg_exit_pending(vcpu)) {
2202                 kvm_s390_prepare_debug_exit(vcpu);
2203                 return 0;
2204         }
2205
2206         if (vcpu->sigset_active)
2207                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2208
2209         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2210                 kvm_s390_vcpu_start(vcpu);
2211         } else if (is_vcpu_stopped(vcpu)) {
2212                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2213                                    vcpu->vcpu_id);
2214                 return -EINVAL;
2215         }
2216
2217         sync_regs(vcpu, kvm_run);
2218
2219         might_fault();
2220         rc = __vcpu_run(vcpu);
2221
2222         if (signal_pending(current) && !rc) {
2223                 kvm_run->exit_reason = KVM_EXIT_INTR;
2224                 rc = -EINTR;
2225         }
2226
2227         if (guestdbg_exit_pending(vcpu) && !rc)  {
2228                 kvm_s390_prepare_debug_exit(vcpu);
2229                 rc = 0;
2230         }
2231
2232         if (rc == -EOPNOTSUPP) {
2233                 /* intercept cannot be handled in-kernel, prepare kvm-run */
2234                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
2235                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2236                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
2237                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
2238                 rc = 0;
2239         }
2240
2241         if (rc == -EREMOTE) {
2242                 /* intercept was handled, but userspace support is needed
2243                  * kvm_run has been prepared by the handler */
2244                 rc = 0;
2245         }
2246
2247         store_regs(vcpu, kvm_run);
2248
2249         if (vcpu->sigset_active)
2250                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2251
2252         vcpu->stat.exit_userspace++;
2253         return rc;
2254 }
2255
2256 /*
2257  * store status at address
2258  * we use have two special cases:
2259  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2260  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2261  */
2262 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2263 {
2264         unsigned char archmode = 1;
2265         unsigned int px;
2266         u64 clkcomp;
2267         int rc;
2268
2269         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2270                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2271                         return -EFAULT;
2272                 gpa = SAVE_AREA_BASE;
2273         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2274                 if (write_guest_real(vcpu, 163, &archmode, 1))
2275                         return -EFAULT;
2276                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2277         }
2278         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2279                              vcpu->arch.guest_fpregs.fprs, 128);
2280         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2281                               vcpu->run->s.regs.gprs, 128);
2282         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2283                               &vcpu->arch.sie_block->gpsw, 16);
2284         px = kvm_s390_get_prefix(vcpu);
2285         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2286                               &px, 4);
2287         rc |= write_guest_abs(vcpu,
2288                               gpa + offsetof(struct save_area, fp_ctrl_reg),
2289                               &vcpu->arch.guest_fpregs.fpc, 4);
2290         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2291                               &vcpu->arch.sie_block->todpr, 4);
2292         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2293                               &vcpu->arch.sie_block->cputm, 8);
2294         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2295         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2296                               &clkcomp, 8);
2297         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2298                               &vcpu->run->s.regs.acrs, 64);
2299         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2300                               &vcpu->arch.sie_block->gcr, 128);
2301         return rc ? -EFAULT : 0;
2302 }
2303
2304 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2305 {
2306         /*
2307          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2308          * copying in vcpu load/put. Lets update our copies before we save
2309          * it into the save area
2310          */
2311         save_fpu_regs();
2312         if (test_kvm_facility(vcpu->kvm, 129)) {
2313                 /*
2314                  * If the vector extension is available, the vector registers
2315                  * which overlaps with floating-point registers are saved in
2316                  * the SIE-control block.  Hence, extract the floating-point
2317                  * registers and the FPC value and store them in the
2318                  * guest_fpregs structure.
2319                  */
2320                 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2321                 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2322                                  current->thread.fpu.vxrs);
2323         } else
2324                 save_fpu_to(&vcpu->arch.guest_fpregs);
2325         save_access_regs(vcpu->run->s.regs.acrs);
2326
2327         return kvm_s390_store_status_unloaded(vcpu, addr);
2328 }
2329
2330 /*
2331  * store additional status at address
2332  */
2333 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2334                                         unsigned long gpa)
2335 {
2336         /* Only bits 0-53 are used for address formation */
2337         if (!(gpa & ~0x3ff))
2338                 return 0;
2339
2340         return write_guest_abs(vcpu, gpa & ~0x3ff,
2341                                (void *)&vcpu->run->s.regs.vrs, 512);
2342 }
2343
2344 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2345 {
2346         if (!test_kvm_facility(vcpu->kvm, 129))
2347                 return 0;
2348
2349         /*
2350          * The guest VXRS are in the host VXRs due to the lazy
2351          * copying in vcpu load/put. We can simply call save_fpu_regs()
2352          * to save the current register state because we are in the
2353          * middle of a load/put cycle.
2354          *
2355          * Let's update our copies before we save it into the save area.
2356          */
2357         save_fpu_regs();
2358
2359         return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2360 }
2361
2362 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2363 {
2364         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2365         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2366 }
2367
2368 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2369 {
2370         unsigned int i;
2371         struct kvm_vcpu *vcpu;
2372
2373         kvm_for_each_vcpu(i, vcpu, kvm) {
2374                 __disable_ibs_on_vcpu(vcpu);
2375         }
2376 }
2377
2378 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2379 {
2380         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2381         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2382 }
2383
2384 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2385 {
2386         int i, online_vcpus, started_vcpus = 0;
2387
2388         if (!is_vcpu_stopped(vcpu))
2389                 return;
2390
2391         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2392         /* Only one cpu at a time may enter/leave the STOPPED state. */
2393         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2394         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2395
2396         for (i = 0; i < online_vcpus; i++) {
2397                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2398                         started_vcpus++;
2399         }
2400
2401         if (started_vcpus == 0) {
2402                 /* we're the only active VCPU -> speed it up */
2403                 __enable_ibs_on_vcpu(vcpu);
2404         } else if (started_vcpus == 1) {
2405                 /*
2406                  * As we are starting a second VCPU, we have to disable
2407                  * the IBS facility on all VCPUs to remove potentially
2408                  * oustanding ENABLE requests.
2409                  */
2410                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2411         }
2412
2413         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2414         /*
2415          * Another VCPU might have used IBS while we were offline.
2416          * Let's play safe and flush the VCPU at startup.
2417          */
2418         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2419         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2420         return;
2421 }
2422
2423 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2424 {
2425         int i, online_vcpus, started_vcpus = 0;
2426         struct kvm_vcpu *started_vcpu = NULL;
2427
2428         if (is_vcpu_stopped(vcpu))
2429                 return;
2430
2431         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2432         /* Only one cpu at a time may enter/leave the STOPPED state. */
2433         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2434         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2435
2436         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2437         kvm_s390_clear_stop_irq(vcpu);
2438
2439         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2440         __disable_ibs_on_vcpu(vcpu);
2441
2442         for (i = 0; i < online_vcpus; i++) {
2443                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2444                         started_vcpus++;
2445                         started_vcpu = vcpu->kvm->vcpus[i];
2446                 }
2447         }
2448
2449         if (started_vcpus == 1) {
2450                 /*
2451                  * As we only have one VCPU left, we want to enable the
2452                  * IBS facility for that VCPU to speed it up.
2453                  */
2454                 __enable_ibs_on_vcpu(started_vcpu);
2455         }
2456
2457         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2458         return;
2459 }
2460
2461 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2462                                      struct kvm_enable_cap *cap)
2463 {
2464         int r;
2465
2466         if (cap->flags)
2467                 return -EINVAL;
2468
2469         switch (cap->cap) {
2470         case KVM_CAP_S390_CSS_SUPPORT:
2471                 if (!vcpu->kvm->arch.css_support) {
2472                         vcpu->kvm->arch.css_support = 1;
2473                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2474                         trace_kvm_s390_enable_css(vcpu->kvm);
2475                 }
2476                 r = 0;
2477                 break;
2478         default:
2479                 r = -EINVAL;
2480                 break;
2481         }
2482         return r;
2483 }
2484
2485 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2486                                   struct kvm_s390_mem_op *mop)
2487 {
2488         void __user *uaddr = (void __user *)mop->buf;
2489         void *tmpbuf = NULL;
2490         int r, srcu_idx;
2491         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2492                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
2493
2494         if (mop->flags & ~supported_flags)
2495                 return -EINVAL;
2496
2497         if (mop->size > MEM_OP_MAX_SIZE)
2498                 return -E2BIG;
2499
2500         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2501                 tmpbuf = vmalloc(mop->size);
2502                 if (!tmpbuf)
2503                         return -ENOMEM;
2504         }
2505
2506         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2507
2508         switch (mop->op) {
2509         case KVM_S390_MEMOP_LOGICAL_READ:
2510                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2511                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2512                         break;
2513                 }
2514                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2515                 if (r == 0) {
2516                         if (copy_to_user(uaddr, tmpbuf, mop->size))
2517                                 r = -EFAULT;
2518                 }
2519                 break;
2520         case KVM_S390_MEMOP_LOGICAL_WRITE:
2521                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2522                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2523                         break;
2524                 }
2525                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2526                         r = -EFAULT;
2527                         break;
2528                 }
2529                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2530                 break;
2531         default:
2532                 r = -EINVAL;
2533         }
2534
2535         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2536
2537         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2538                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2539
2540         vfree(tmpbuf);
2541         return r;
2542 }
2543
2544 long kvm_arch_vcpu_ioctl(struct file *filp,
2545                          unsigned int ioctl, unsigned long arg)
2546 {
2547         struct kvm_vcpu *vcpu = filp->private_data;
2548         void __user *argp = (void __user *)arg;
2549         int idx;
2550         long r;
2551
2552         switch (ioctl) {
2553         case KVM_S390_IRQ: {
2554                 struct kvm_s390_irq s390irq;
2555
2556                 r = -EFAULT;
2557                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2558                         break;
2559                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2560                 break;
2561         }
2562         case KVM_S390_INTERRUPT: {
2563                 struct kvm_s390_interrupt s390int;
2564                 struct kvm_s390_irq s390irq;
2565
2566                 r = -EFAULT;
2567                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2568                         break;
2569                 if (s390int_to_s390irq(&s390int, &s390irq))
2570                         return -EINVAL;
2571                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2572                 break;
2573         }
2574         case KVM_S390_STORE_STATUS:
2575                 idx = srcu_read_lock(&vcpu->kvm->srcu);
2576                 r = kvm_s390_vcpu_store_status(vcpu, arg);
2577                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2578                 break;
2579         case KVM_S390_SET_INITIAL_PSW: {
2580                 psw_t psw;
2581
2582                 r = -EFAULT;
2583                 if (copy_from_user(&psw, argp, sizeof(psw)))
2584                         break;
2585                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2586                 break;
2587         }
2588         case KVM_S390_INITIAL_RESET:
2589                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2590                 break;
2591         case KVM_SET_ONE_REG:
2592         case KVM_GET_ONE_REG: {
2593                 struct kvm_one_reg reg;
2594                 r = -EFAULT;
2595                 if (copy_from_user(&reg, argp, sizeof(reg)))
2596                         break;
2597                 if (ioctl == KVM_SET_ONE_REG)
2598                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2599                 else
2600                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2601                 break;
2602         }
2603 #ifdef CONFIG_KVM_S390_UCONTROL
2604         case KVM_S390_UCAS_MAP: {
2605                 struct kvm_s390_ucas_mapping ucasmap;
2606
2607                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2608                         r = -EFAULT;
2609                         break;
2610                 }
2611
2612                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2613                         r = -EINVAL;
2614                         break;
2615                 }
2616
2617                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2618                                      ucasmap.vcpu_addr, ucasmap.length);
2619                 break;
2620         }
2621         case KVM_S390_UCAS_UNMAP: {
2622                 struct kvm_s390_ucas_mapping ucasmap;
2623
2624                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2625                         r = -EFAULT;
2626                         break;
2627                 }
2628
2629                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2630                         r = -EINVAL;
2631                         break;
2632                 }
2633
2634                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2635                         ucasmap.length);
2636                 break;
2637         }
2638 #endif
2639         case KVM_S390_VCPU_FAULT: {
2640                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2641                 break;
2642         }
2643         case KVM_ENABLE_CAP:
2644         {
2645                 struct kvm_enable_cap cap;
2646                 r = -EFAULT;
2647                 if (copy_from_user(&cap, argp, sizeof(cap)))
2648                         break;
2649                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2650                 break;
2651         }
2652         case KVM_S390_MEM_OP: {
2653                 struct kvm_s390_mem_op mem_op;
2654
2655                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2656                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2657                 else
2658                         r = -EFAULT;
2659                 break;
2660         }
2661         case KVM_S390_SET_IRQ_STATE: {
2662                 struct kvm_s390_irq_state irq_state;
2663
2664                 r = -EFAULT;
2665                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2666                         break;
2667                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2668                     irq_state.len == 0 ||
2669                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2670                         r = -EINVAL;
2671                         break;
2672                 }
2673                 r = kvm_s390_set_irq_state(vcpu,
2674                                            (void __user *) irq_state.buf,
2675                                            irq_state.len);
2676                 break;
2677         }
2678         case KVM_S390_GET_IRQ_STATE: {
2679                 struct kvm_s390_irq_state irq_state;
2680
2681                 r = -EFAULT;
2682                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2683                         break;
2684                 if (irq_state.len == 0) {
2685                         r = -EINVAL;
2686                         break;
2687                 }
2688                 r = kvm_s390_get_irq_state(vcpu,
2689                                            (__u8 __user *)  irq_state.buf,
2690                                            irq_state.len);
2691                 break;
2692         }
2693         default:
2694                 r = -ENOTTY;
2695         }
2696         return r;
2697 }
2698
2699 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2700 {
2701 #ifdef CONFIG_KVM_S390_UCONTROL
2702         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2703                  && (kvm_is_ucontrol(vcpu->kvm))) {
2704                 vmf->page = virt_to_page(vcpu->arch.sie_block);
2705                 get_page(vmf->page);
2706                 return 0;
2707         }
2708 #endif
2709         return VM_FAULT_SIGBUS;
2710 }
2711
2712 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2713                             unsigned long npages)
2714 {
2715         return 0;
2716 }
2717
2718 /* Section: memory related */
2719 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2720                                    struct kvm_memory_slot *memslot,
2721                                    const struct kvm_userspace_memory_region *mem,
2722                                    enum kvm_mr_change change)
2723 {
2724         /* A few sanity checks. We can have memory slots which have to be
2725            located/ended at a segment boundary (1MB). The memory in userland is
2726            ok to be fragmented into various different vmas. It is okay to mmap()
2727            and munmap() stuff in this slot after doing this call at any time */
2728
2729         if (mem->userspace_addr & 0xffffful)
2730                 return -EINVAL;
2731
2732         if (mem->memory_size & 0xffffful)
2733                 return -EINVAL;
2734
2735         return 0;
2736 }
2737
2738 void kvm_arch_commit_memory_region(struct kvm *kvm,
2739                                 const struct kvm_userspace_memory_region *mem,
2740                                 const struct kvm_memory_slot *old,
2741                                 const struct kvm_memory_slot *new,
2742                                 enum kvm_mr_change change)
2743 {
2744         int rc;
2745
2746         /* If the basics of the memslot do not change, we do not want
2747          * to update the gmap. Every update causes several unnecessary
2748          * segment translation exceptions. This is usually handled just
2749          * fine by the normal fault handler + gmap, but it will also
2750          * cause faults on the prefix page of running guest CPUs.
2751          */
2752         if (old->userspace_addr == mem->userspace_addr &&
2753             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2754             old->npages * PAGE_SIZE == mem->memory_size)
2755                 return;
2756
2757         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2758                 mem->guest_phys_addr, mem->memory_size);
2759         if (rc)
2760                 pr_warn("failed to commit memory region\n");
2761         return;
2762 }
2763
2764 static int __init kvm_s390_init(void)
2765 {
2766         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2767 }
2768
2769 static void __exit kvm_s390_exit(void)
2770 {
2771         kvm_exit();
2772 }
2773
2774 module_init(kvm_s390_init);
2775 module_exit(kvm_s390_exit);
2776
2777 /*
2778  * Enable autoloading of the kvm module.
2779  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2780  * since x86 takes a different approach.
2781  */
2782 #include <linux/miscdevice.h>
2783 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2784 MODULE_ALIAS("devname:kvm");