2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * KVM/MIPS: Support for hardware virtualization extensions
8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
9 * Authors: Yann Le Du <ledu@kymasys.com>
12 #include <linux/errno.h>
13 #include <linux/err.h>
14 #include <linux/module.h>
15 #include <linux/preempt.h>
16 #include <linux/vmalloc.h>
17 #include <asm/cacheflush.h>
18 #include <asm/cacheops.h>
19 #include <asm/cmpxchg.h>
21 #include <asm/hazards.h>
23 #include <asm/mmu_context.h>
24 #include <asm/r4kcache.h>
27 #include <asm/tlbex.h>
29 #include <linux/kvm_host.h>
31 #include "interrupt.h"
35 /* Pointers to last VCPU loaded on each physical CPU */
36 static struct kvm_vcpu *last_vcpu[NR_CPUS];
37 /* Pointers to last VCPU executed on each physical CPU */
38 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS];
41 * Number of guest VTLB entries to use, so we can catch inconsistency between
44 static unsigned int kvm_vz_guest_vtlb_size;
46 static inline long kvm_vz_read_gc0_ebase(void)
48 if (sizeof(long) == 8 && cpu_has_ebase_wg)
49 return read_gc0_ebase_64();
51 return read_gc0_ebase();
54 static inline void kvm_vz_write_gc0_ebase(long v)
57 * First write with WG=1 to write upper bits, then write again in case
58 * WG should be left at 0.
59 * write_gc0_ebase_64() is no longer UNDEFINED since R6.
61 if (sizeof(long) == 8 &&
62 (cpu_has_mips64r6 || cpu_has_ebase_wg)) {
63 write_gc0_ebase_64(v | MIPS_EBASE_WG);
64 write_gc0_ebase_64(v);
66 write_gc0_ebase(v | MIPS_EBASE_WG);
72 * These Config bits may be writable by the guest:
73 * Config: [K23, KU] (!TLB), K0
75 * Config2: [TU, SU] (impl)
77 * Config4: FTLBPageSize
78 * Config5: K, CV, MSAEn, UFE, FRE, SBRI, UFR
81 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu)
86 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu)
91 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu)
96 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu)
98 return MIPS_CONF3_ISA_OE;
101 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu)
103 /* no need to be exact */
104 return MIPS_CONF4_VFTLBPAGESIZE;
107 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu)
109 unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI;
111 /* Permit MSAEn changes if MSA supported and enabled */
112 if (kvm_mips_guest_has_msa(&vcpu->arch))
113 mask |= MIPS_CONF5_MSAEN;
116 * Permit guest FPU mode changes if FPU is enabled and the relevant
117 * feature exists according to FIR register.
119 if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
121 mask |= MIPS_CONF5_UFR;
123 mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE;
130 * VZ optionally allows these additional Config bits to be written by root:
132 * Config1: M, [MMUSize-1, C2, MD, PC, WR, CA], FP
134 * Config3: M, MSAP, [BPG], ULRI, [DSP2P, DSPP, CTXTC, ITL, LPA, VEIC,
135 * VInt, SP, CDMM, MT, SM, TL]
136 * Config4: M, [VTLBSizeExt, MMUSizeExt]
140 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu)
142 return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M;
145 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu)
147 unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M;
149 /* Permit FPU to be present if FPU is supported */
150 if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
151 mask |= MIPS_CONF1_FP;
156 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu)
158 return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M;
161 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu)
163 unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M |
166 /* Permit MSA to be present if MSA is supported */
167 if (kvm_mips_guest_can_have_msa(&vcpu->arch))
168 mask |= MIPS_CONF3_MSA;
173 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu)
175 return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M;
178 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu)
180 return kvm_vz_config5_guest_wrmask(vcpu);
183 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva)
185 /* VZ guest has already converted gva to gpa */
189 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority)
191 set_bit(priority, &vcpu->arch.pending_exceptions);
192 clear_bit(priority, &vcpu->arch.pending_exceptions_clr);
195 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority)
197 clear_bit(priority, &vcpu->arch.pending_exceptions);
198 set_bit(priority, &vcpu->arch.pending_exceptions_clr);
201 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu)
204 * timer expiry is asynchronous to vcpu execution therefore defer guest
207 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
210 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu)
213 * timer expiry is asynchronous to vcpu execution therefore defer guest
216 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER);
219 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu,
220 struct kvm_mips_interrupt *irq)
222 int intr = (int)irq->irq;
225 * interrupts are asynchronous to vcpu execution therefore defer guest
230 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IO);
234 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_1);
238 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_2);
247 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu,
248 struct kvm_mips_interrupt *irq)
250 int intr = (int)irq->irq;
253 * interrupts are asynchronous to vcpu execution therefore defer guest
258 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IO);
262 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_1);
266 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_2);
275 static u32 kvm_vz_priority_to_irq[MIPS_EXC_MAX] = {
276 [MIPS_EXC_INT_TIMER] = C_IRQ5,
277 [MIPS_EXC_INT_IO] = C_IRQ0,
278 [MIPS_EXC_INT_IPI_1] = C_IRQ1,
279 [MIPS_EXC_INT_IPI_2] = C_IRQ2,
282 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority,
285 u32 irq = (priority < MIPS_EXC_MAX) ?
286 kvm_vz_priority_to_irq[priority] : 0;
289 case MIPS_EXC_INT_TIMER:
293 case MIPS_EXC_INT_IO:
294 case MIPS_EXC_INT_IPI_1:
295 case MIPS_EXC_INT_IPI_2:
296 if (cpu_has_guestctl2)
297 set_c0_guestctl2(irq);
306 clear_bit(priority, &vcpu->arch.pending_exceptions);
310 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority,
313 u32 irq = (priority < MIPS_EXC_MAX) ?
314 kvm_vz_priority_to_irq[priority] : 0;
317 case MIPS_EXC_INT_TIMER:
319 * Call to kvm_write_c0_guest_compare() clears Cause.TI in
320 * kvm_mips_emulate_CP0(). Explicitly clear irq associated with
321 * Cause.IP[IPTI] if GuestCtl2 virtual interrupt register not
322 * supported or if not using GuestCtl2 Hardware Clear.
324 if (cpu_has_guestctl2) {
325 if (!(read_c0_guestctl2() & (irq << 14)))
326 clear_c0_guestctl2(irq);
328 clear_gc0_cause(irq);
332 case MIPS_EXC_INT_IO:
333 case MIPS_EXC_INT_IPI_1:
334 case MIPS_EXC_INT_IPI_2:
335 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */
336 if (cpu_has_guestctl2) {
337 if (!(read_c0_guestctl2() & (irq << 14)))
338 clear_c0_guestctl2(irq);
340 clear_gc0_cause(irq);
348 clear_bit(priority, &vcpu->arch.pending_exceptions_clr);
353 * VZ guest timer handling.
357 * _kvm_vz_restore_stimer() - Restore soft timer state.
358 * @vcpu: Virtual CPU.
359 * @compare: CP0_Compare register value, restored by caller.
360 * @cause: CP0_Cause register to restore.
362 * Restore VZ state relating to the soft timer.
364 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare,
368 * Avoid spurious counter interrupts by setting Guest CP0_Count to just
369 * after Guest CP0_Compare.
371 write_c0_gtoffset(compare - read_c0_count());
373 back_to_back_c0_hazard();
374 write_gc0_cause(cause);
378 * kvm_vz_restore_timer() - Restore guest timer state.
379 * @vcpu: Virtual CPU.
381 * Restore soft timer state from saved context.
383 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu)
385 struct mips_coproc *cop0 = vcpu->arch.cop0;
388 compare = kvm_read_sw_gc0_compare(cop0);
389 cause = kvm_read_sw_gc0_cause(cop0);
391 write_gc0_compare(compare);
392 _kvm_vz_restore_stimer(vcpu, compare, cause);
396 * kvm_vz_save_timer() - Save guest timer state.
397 * @vcpu: Virtual CPU.
399 * Save VZ guest timer state.
401 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu)
403 struct mips_coproc *cop0 = vcpu->arch.cop0;
406 compare = read_gc0_compare();
407 cause = read_gc0_cause();
409 /* save timer-related state to VCPU context */
410 kvm_write_sw_gc0_cause(cop0, cause);
411 kvm_write_sw_gc0_compare(cop0, compare);
415 * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA.
416 * @vcpu: KVM VCPU state.
417 * @gva: Guest virtual address to convert.
418 * @gpa: Output guest physical address.
420 * Convert a guest virtual address (GVA) which is valid according to the guest
421 * context, to a guest physical address (GPA).
423 * Returns: 0 on success.
426 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
431 if ((long)gva == (s32)gva32) {
432 /* Handle canonical 32-bit virtual address */
433 if ((s32)gva32 < (s32)0xc0000000) {
434 /* legacy unmapped KSeg0 or KSeg1 */
435 *gpa = gva32 & 0x1fffffff;
439 } else if ((gva & 0xc000000000000000) == 0x8000000000000000) {
442 * Traditionally fully unmapped.
443 * Bits 61:59 specify the CCA, which we can just mask off here.
444 * Bits 58:PABITS should be zero, but we shouldn't have got here
447 *gpa = gva & 0x07ffffffffffffff;
452 return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa);
456 * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA.
457 * @vcpu: KVM VCPU state.
458 * @badvaddr: Root BadVAddr.
459 * @gpa: Output guest physical address.
461 * VZ implementations are permitted to report guest virtual addresses (GVA) in
462 * BadVAddr on a root exception during guest execution, instead of the more
463 * convenient guest physical addresses (GPA). When we get a GVA, this function
464 * converts it to a GPA, taking into account guest segmentation and guest TLB
467 * Returns: 0 on success.
470 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr,
473 unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 &
474 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT;
476 /* If BadVAddr is GPA, then all is well in the world */
477 if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) {
482 /* Otherwise we'd expect it to be GVA ... */
483 if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA,
484 "Unexpected gexccode %#x\n", gexccode))
487 /* ... and we need to perform the GVA->GPA translation in software */
488 return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa);
491 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu)
493 u32 *opc = (u32 *) vcpu->arch.pc;
494 u32 cause = vcpu->arch.host_cp0_cause;
495 u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE;
496 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
500 * Fetch the instruction.
502 if (cause & CAUSEF_BD)
504 kvm_get_badinstr(opc, vcpu, &inst);
506 kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n",
507 exccode, opc, inst, badvaddr,
509 kvm_arch_vcpu_dump_regs(vcpu);
510 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
514 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst,
517 struct kvm_vcpu *vcpu)
519 struct mips_coproc *cop0 = vcpu->arch.cop0;
520 enum emulation_result er = EMULATE_DONE;
522 unsigned long curr_pc;
526 * Update PC and hold onto current PC in case there is
527 * an error and we want to rollback the PC
529 curr_pc = vcpu->arch.pc;
530 er = update_pc(vcpu, cause);
531 if (er == EMULATE_FAIL)
534 if (inst.co_format.co) {
535 switch (inst.co_format.func) {
537 er = kvm_mips_emul_wait(vcpu);
543 rt = inst.c0r_format.rt;
544 rd = inst.c0r_format.rd;
545 sel = inst.c0r_format.sel;
547 switch (inst.c0r_format.rs) {
550 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
551 cop0->stat[rd][sel]++;
553 if (rd == MIPS_CP0_COUNT &&
554 sel == 0) { /* Count */
555 val = kvm_mips_read_count(vcpu);
556 } else if (rd == MIPS_CP0_COMPARE &&
557 sel == 0) { /* Compare */
558 val = read_gc0_compare();
559 } else if ((rd == MIPS_CP0_PRID &&
560 (sel == 0 || /* PRid */
561 sel == 2 || /* CDMMBase */
562 sel == 3)) || /* CMGCRBase */
563 (rd == MIPS_CP0_STATUS &&
564 (sel == 2 || /* SRSCtl */
565 sel == 3)) || /* SRSMap */
566 (rd == MIPS_CP0_CONFIG &&
567 (sel == 7)) || /* Config7 */
568 (rd == MIPS_CP0_ERRCTL &&
569 (sel == 0))) { /* ErrCtl */
570 val = cop0->reg[rd][sel];
576 if (er != EMULATE_FAIL) {
578 if (inst.c0r_format.rs == mfc_op)
580 vcpu->arch.gprs[rt] = val;
583 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ?
584 KVM_TRACE_MFC0 : KVM_TRACE_DMFC0,
585 KVM_TRACE_COP0(rd, sel), val);
590 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
591 cop0->stat[rd][sel]++;
593 val = vcpu->arch.gprs[rt];
594 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ?
595 KVM_TRACE_MTC0 : KVM_TRACE_DMTC0,
596 KVM_TRACE_COP0(rd, sel), val);
598 if (rd == MIPS_CP0_COUNT &&
599 sel == 0) { /* Count */
600 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
601 } else if (rd == MIPS_CP0_COMPARE &&
602 sel == 0) { /* Compare */
603 kvm_mips_write_compare(vcpu,
606 } else if (rd == MIPS_CP0_ERRCTL &&
607 (sel == 0)) { /* ErrCtl */
608 /* ignore the written value */
619 /* Rollback PC only if emulation was unsuccessful */
620 if (er == EMULATE_FAIL) {
621 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n",
622 curr_pc, __func__, inst.word);
624 vcpu->arch.pc = curr_pc;
630 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst,
633 struct kvm_vcpu *vcpu)
635 enum emulation_result er = EMULATE_DONE;
636 u32 cache, op_inst, op, base;
638 struct kvm_vcpu_arch *arch = &vcpu->arch;
639 unsigned long va, curr_pc;
642 * Update PC and hold onto current PC in case there is
643 * an error and we want to rollback the PC
645 curr_pc = vcpu->arch.pc;
646 er = update_pc(vcpu, cause);
647 if (er == EMULATE_FAIL)
650 base = inst.i_format.rs;
651 op_inst = inst.i_format.rt;
653 offset = inst.spec3_format.simmediate;
655 offset = inst.i_format.simmediate;
656 cache = op_inst & CacheOp_Cache;
657 op = op_inst & CacheOp_Op;
659 va = arch->gprs[base] + offset;
661 kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
662 cache, op, base, arch->gprs[base], offset);
664 /* Secondary or tirtiary cache ops ignored */
665 if (cache != Cache_I && cache != Cache_D)
669 case Index_Invalidate_I:
670 flush_icache_line_indexed(va);
672 case Index_Writeback_Inv_D:
673 flush_dcache_line_indexed(va);
679 kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
680 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base],
683 vcpu->arch.pc = curr_pc;
688 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc,
689 struct kvm_vcpu *vcpu)
691 enum emulation_result er = EMULATE_DONE;
692 struct kvm_vcpu_arch *arch = &vcpu->arch;
693 struct kvm_run *run = vcpu->run;
694 union mips_instruction inst;
699 * Fetch the instruction.
701 if (cause & CAUSEF_BD)
703 err = kvm_get_badinstr(opc, vcpu, &inst.word);
707 switch (inst.r_format.opcode) {
709 er = kvm_vz_gpsi_cop0(inst, opc, cause, run, vcpu);
711 #ifndef CONFIG_CPU_MIPSR6
713 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
714 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu);
718 switch (inst.spec3_format.func) {
719 #ifdef CONFIG_CPU_MIPSR6
721 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
722 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu);
726 if (inst.r_format.rs || (inst.r_format.re >> 3))
729 rd = inst.r_format.rd;
730 rt = inst.r_format.rt;
731 sel = inst.r_format.re & 0x7;
734 case MIPS_HWR_CC: /* Read count register */
736 (long)(int)kvm_mips_read_count(vcpu);
739 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR,
740 KVM_TRACE_HWR(rd, sel), 0);
744 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR,
745 KVM_TRACE_HWR(rd, sel), arch->gprs[rt]);
747 er = update_pc(vcpu, cause);
756 kvm_err("GPSI exception not supported (%p/%#x)\n",
758 kvm_arch_vcpu_dump_regs(vcpu);
766 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc,
767 struct kvm_vcpu *vcpu)
769 enum emulation_result er = EMULATE_DONE;
770 struct kvm_vcpu_arch *arch = &vcpu->arch;
771 union mips_instruction inst;
775 * Fetch the instruction.
777 if (cause & CAUSEF_BD)
779 err = kvm_get_badinstr(opc, vcpu, &inst.word);
783 /* complete MTC0 on behalf of guest and advance EPC */
784 if (inst.c0r_format.opcode == cop0_op &&
785 inst.c0r_format.rs == mtc_op &&
786 inst.c0r_format.z == 0) {
787 int rt = inst.c0r_format.rt;
788 int rd = inst.c0r_format.rd;
789 int sel = inst.c0r_format.sel;
790 unsigned int val = arch->gprs[rt];
791 unsigned int old_val, change;
793 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel),
796 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
797 /* FR bit should read as zero if no FPU */
798 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
799 val &= ~(ST0_CU1 | ST0_FR);
802 * Also don't allow FR to be set if host doesn't support
805 if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64))
808 old_val = read_gc0_status();
809 change = val ^ old_val;
811 if (change & ST0_FR) {
813 * FPU and Vector register state is made
814 * UNPREDICTABLE by a change of FR, so don't
815 * even bother saving it.
821 * If MSA state is already live, it is undefined how it
822 * interacts with FR=0 FPU state, and we don't want to
823 * hit reserved instruction exceptions trying to save
824 * the MSA state later when CU=1 && FR=1, so play it
825 * safe and save it first.
827 if (change & ST0_CU1 && !(val & ST0_FR) &&
828 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
831 write_gc0_status(val);
832 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
833 u32 old_cause = read_gc0_cause();
834 u32 change = old_cause ^ val;
836 /* DC bit enabling/disabling timer? */
837 if (change & CAUSEF_DC) {
839 kvm_mips_count_disable_cause(vcpu);
841 kvm_mips_count_enable_cause(vcpu);
844 /* Only certain bits are RW to the guest */
845 change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP |
846 CAUSEF_IP0 | CAUSEF_IP1);
848 /* WP can only be cleared */
849 change &= ~CAUSEF_WP | old_cause;
851 write_gc0_cause(old_cause ^ change);
852 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */
853 write_gc0_intctl(val);
854 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
855 old_val = read_gc0_config5();
856 change = val ^ old_val;
857 /* Handle changes in FPU/MSA modes */
861 * Propagate FRE changes immediately if the FPU
862 * context is already loaded.
864 if (change & MIPS_CONF5_FRE &&
865 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
866 change_c0_config5(MIPS_CONF5_FRE, val);
871 (change & kvm_vz_config5_guest_wrmask(vcpu));
872 write_gc0_config5(val);
874 kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n",
879 if (er != EMULATE_FAIL)
880 er = update_pc(vcpu, cause);
882 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n",
890 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc,
891 struct kvm_vcpu *vcpu)
893 enum emulation_result er;
894 union mips_instruction inst;
895 unsigned long curr_pc;
898 if (cause & CAUSEF_BD)
900 err = kvm_get_badinstr(opc, vcpu, &inst.word);
905 * Update PC and hold onto current PC in case there is
906 * an error and we want to rollback the PC
908 curr_pc = vcpu->arch.pc;
909 er = update_pc(vcpu, cause);
910 if (er == EMULATE_FAIL)
913 er = kvm_mips_emul_hypcall(vcpu, inst);
914 if (er == EMULATE_FAIL)
915 vcpu->arch.pc = curr_pc;
920 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode,
923 struct kvm_vcpu *vcpu)
928 * Fetch the instruction.
930 if (cause & CAUSEF_BD)
932 kvm_get_badinstr(opc, vcpu, &inst);
934 kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x Status: %#x\n",
935 gexccode, opc, inst, read_gc0_status());
940 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu)
942 u32 *opc = (u32 *) vcpu->arch.pc;
943 u32 cause = vcpu->arch.host_cp0_cause;
944 enum emulation_result er = EMULATE_DONE;
945 u32 gexccode = (vcpu->arch.host_cp0_guestctl0 &
946 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT;
947 int ret = RESUME_GUEST;
949 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode);
951 case MIPS_GCTL0_GEXC_GPSI:
952 ++vcpu->stat.vz_gpsi_exits;
953 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu);
955 case MIPS_GCTL0_GEXC_GSFC:
956 ++vcpu->stat.vz_gsfc_exits;
957 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu);
959 case MIPS_GCTL0_GEXC_HC:
960 ++vcpu->stat.vz_hc_exits;
961 er = kvm_trap_vz_handle_hc(cause, opc, vcpu);
963 case MIPS_GCTL0_GEXC_GRR:
964 ++vcpu->stat.vz_grr_exits;
965 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
968 case MIPS_GCTL0_GEXC_GVA:
969 ++vcpu->stat.vz_gva_exits;
970 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
973 case MIPS_GCTL0_GEXC_GHFC:
974 ++vcpu->stat.vz_ghfc_exits;
975 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
978 case MIPS_GCTL0_GEXC_GPA:
979 ++vcpu->stat.vz_gpa_exits;
980 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
984 ++vcpu->stat.vz_resvd_exits;
985 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
991 if (er == EMULATE_DONE) {
993 } else if (er == EMULATE_HYPERCALL) {
994 ret = kvm_mips_handle_hypcall(vcpu);
996 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1003 * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor.
1004 * @vcpu: Virtual CPU context.
1006 * Handle when the guest attempts to use a coprocessor which hasn't been allowed
1007 * by the root context.
1009 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu)
1011 struct kvm_run *run = vcpu->run;
1012 u32 cause = vcpu->arch.host_cp0_cause;
1013 enum emulation_result er = EMULATE_FAIL;
1014 int ret = RESUME_GUEST;
1016 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
1018 * If guest FPU not present, the FPU operation should have been
1019 * treated as a reserved instruction!
1020 * If FPU already in use, we shouldn't get this at all.
1022 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) ||
1023 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) {
1025 return EMULATE_FAIL;
1031 /* other coprocessors not handled */
1039 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1050 * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root.
1051 * @vcpu: Virtual CPU context.
1053 * Handle when the guest attempts to use MSA when it is disabled in the root
1056 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu)
1058 struct kvm_run *run = vcpu->run;
1061 * If MSA not present or not exposed to guest or FR=0, the MSA operation
1062 * should have been treated as a reserved instruction!
1063 * Same if CU1=1, FR=0.
1064 * If MSA already in use, we shouldn't get this at all.
1066 if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
1067 (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 ||
1068 !(read_gc0_config5() & MIPS_CONF5_MSAEN) ||
1069 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1070 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1076 return RESUME_GUEST;
1079 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu)
1081 struct kvm_run *run = vcpu->run;
1082 u32 *opc = (u32 *) vcpu->arch.pc;
1083 u32 cause = vcpu->arch.host_cp0_cause;
1084 ulong badvaddr = vcpu->arch.host_cp0_badvaddr;
1085 union mips_instruction inst;
1086 enum emulation_result er = EMULATE_DONE;
1087 int err, ret = RESUME_GUEST;
1089 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) {
1090 /* A code fetch fault doesn't count as an MMIO */
1091 if (kvm_is_ifetch_fault(&vcpu->arch)) {
1092 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1096 /* Fetch the instruction */
1097 if (cause & CAUSEF_BD)
1099 err = kvm_get_badinstr(opc, vcpu, &inst.word);
1101 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1106 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1107 if (er == EMULATE_FAIL) {
1108 kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n",
1110 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1114 if (er == EMULATE_DONE) {
1116 } else if (er == EMULATE_DO_MMIO) {
1117 run->exit_reason = KVM_EXIT_MMIO;
1120 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1126 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu)
1128 struct kvm_run *run = vcpu->run;
1129 u32 *opc = (u32 *) vcpu->arch.pc;
1130 u32 cause = vcpu->arch.host_cp0_cause;
1131 ulong badvaddr = vcpu->arch.host_cp0_badvaddr;
1132 union mips_instruction inst;
1133 enum emulation_result er = EMULATE_DONE;
1135 int ret = RESUME_GUEST;
1137 /* Just try the access again if we couldn't do the translation */
1138 if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr))
1139 return RESUME_GUEST;
1140 vcpu->arch.host_cp0_badvaddr = badvaddr;
1142 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) {
1143 /* Fetch the instruction */
1144 if (cause & CAUSEF_BD)
1146 err = kvm_get_badinstr(opc, vcpu, &inst.word);
1148 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1153 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1154 if (er == EMULATE_FAIL) {
1155 kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n",
1157 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1161 if (er == EMULATE_DONE) {
1163 } else if (er == EMULATE_DO_MMIO) {
1164 run->exit_reason = KVM_EXIT_MMIO;
1167 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1173 static u64 kvm_vz_get_one_regs[] = {
1174 KVM_REG_MIPS_CP0_INDEX,
1175 KVM_REG_MIPS_CP0_ENTRYLO0,
1176 KVM_REG_MIPS_CP0_ENTRYLO1,
1177 KVM_REG_MIPS_CP0_CONTEXT,
1178 KVM_REG_MIPS_CP0_PAGEMASK,
1179 KVM_REG_MIPS_CP0_PAGEGRAIN,
1180 KVM_REG_MIPS_CP0_WIRED,
1181 KVM_REG_MIPS_CP0_HWRENA,
1182 KVM_REG_MIPS_CP0_BADVADDR,
1183 KVM_REG_MIPS_CP0_COUNT,
1184 KVM_REG_MIPS_CP0_ENTRYHI,
1185 KVM_REG_MIPS_CP0_COMPARE,
1186 KVM_REG_MIPS_CP0_STATUS,
1187 KVM_REG_MIPS_CP0_INTCTL,
1188 KVM_REG_MIPS_CP0_CAUSE,
1189 KVM_REG_MIPS_CP0_EPC,
1190 KVM_REG_MIPS_CP0_PRID,
1191 KVM_REG_MIPS_CP0_EBASE,
1192 KVM_REG_MIPS_CP0_CONFIG,
1193 KVM_REG_MIPS_CP0_CONFIG1,
1194 KVM_REG_MIPS_CP0_CONFIG2,
1195 KVM_REG_MIPS_CP0_CONFIG3,
1196 KVM_REG_MIPS_CP0_CONFIG4,
1197 KVM_REG_MIPS_CP0_CONFIG5,
1199 KVM_REG_MIPS_CP0_XCONTEXT,
1201 KVM_REG_MIPS_CP0_ERROREPC,
1203 KVM_REG_MIPS_COUNT_CTL,
1204 KVM_REG_MIPS_COUNT_RESUME,
1205 KVM_REG_MIPS_COUNT_HZ,
1208 static u64 kvm_vz_get_one_regs_kscratch[] = {
1209 KVM_REG_MIPS_CP0_KSCRATCH1,
1210 KVM_REG_MIPS_CP0_KSCRATCH2,
1211 KVM_REG_MIPS_CP0_KSCRATCH3,
1212 KVM_REG_MIPS_CP0_KSCRATCH4,
1213 KVM_REG_MIPS_CP0_KSCRATCH5,
1214 KVM_REG_MIPS_CP0_KSCRATCH6,
1217 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu)
1221 ret = ARRAY_SIZE(kvm_vz_get_one_regs);
1222 if (cpu_guest_has_userlocal)
1224 ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask);
1229 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices)
1234 if (copy_to_user(indices, kvm_vz_get_one_regs,
1235 sizeof(kvm_vz_get_one_regs)))
1237 indices += ARRAY_SIZE(kvm_vz_get_one_regs);
1239 if (cpu_guest_has_userlocal) {
1240 index = KVM_REG_MIPS_CP0_USERLOCAL;
1241 if (copy_to_user(indices, &index, sizeof(index)))
1245 for (i = 0; i < 6; ++i) {
1246 if (!cpu_guest_has_kscr(i + 2))
1249 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i],
1250 sizeof(kvm_vz_get_one_regs_kscratch[i])))
1258 static inline s64 entrylo_kvm_to_user(unsigned long v)
1262 if (BITS_PER_LONG == 32) {
1264 * KVM API exposes 64-bit version of the register, so move the
1265 * RI/XI bits up into place.
1267 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI;
1269 ret |= ((s64)v & mask) << 32;
1274 static inline unsigned long entrylo_user_to_kvm(s64 v)
1276 unsigned long mask, ret = v;
1278 if (BITS_PER_LONG == 32) {
1280 * KVM API exposes 64-bit versiono of the register, so move the
1281 * RI/XI bits down into place.
1283 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI;
1285 ret |= (v >> 32) & mask;
1290 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu,
1291 const struct kvm_one_reg *reg,
1294 struct mips_coproc *cop0 = vcpu->arch.cop0;
1298 case KVM_REG_MIPS_CP0_INDEX:
1299 *v = (long)read_gc0_index();
1301 case KVM_REG_MIPS_CP0_ENTRYLO0:
1302 *v = entrylo_kvm_to_user(read_gc0_entrylo0());
1304 case KVM_REG_MIPS_CP0_ENTRYLO1:
1305 *v = entrylo_kvm_to_user(read_gc0_entrylo1());
1307 case KVM_REG_MIPS_CP0_CONTEXT:
1308 *v = (long)read_gc0_context();
1310 case KVM_REG_MIPS_CP0_USERLOCAL:
1311 if (!cpu_guest_has_userlocal)
1313 *v = read_gc0_userlocal();
1315 case KVM_REG_MIPS_CP0_PAGEMASK:
1316 *v = (long)read_gc0_pagemask();
1318 case KVM_REG_MIPS_CP0_PAGEGRAIN:
1319 *v = (long)read_gc0_pagegrain();
1321 case KVM_REG_MIPS_CP0_WIRED:
1322 *v = (long)read_gc0_wired();
1324 case KVM_REG_MIPS_CP0_HWRENA:
1325 *v = (long)read_gc0_hwrena();
1327 case KVM_REG_MIPS_CP0_BADVADDR:
1328 *v = (long)read_gc0_badvaddr();
1330 case KVM_REG_MIPS_CP0_COUNT:
1331 *v = kvm_mips_read_count(vcpu);
1333 case KVM_REG_MIPS_CP0_ENTRYHI:
1334 *v = (long)read_gc0_entryhi();
1336 case KVM_REG_MIPS_CP0_COMPARE:
1337 *v = (long)read_gc0_compare();
1339 case KVM_REG_MIPS_CP0_STATUS:
1340 *v = (long)read_gc0_status();
1342 case KVM_REG_MIPS_CP0_INTCTL:
1343 *v = read_gc0_intctl();
1345 case KVM_REG_MIPS_CP0_CAUSE:
1346 *v = (long)read_gc0_cause();
1348 case KVM_REG_MIPS_CP0_EPC:
1349 *v = (long)read_gc0_epc();
1351 case KVM_REG_MIPS_CP0_PRID:
1352 *v = (long)kvm_read_c0_guest_prid(cop0);
1354 case KVM_REG_MIPS_CP0_EBASE:
1355 *v = kvm_vz_read_gc0_ebase();
1357 case KVM_REG_MIPS_CP0_CONFIG:
1358 *v = read_gc0_config();
1360 case KVM_REG_MIPS_CP0_CONFIG1:
1361 if (!cpu_guest_has_conf1)
1363 *v = read_gc0_config1();
1365 case KVM_REG_MIPS_CP0_CONFIG2:
1366 if (!cpu_guest_has_conf2)
1368 *v = read_gc0_config2();
1370 case KVM_REG_MIPS_CP0_CONFIG3:
1371 if (!cpu_guest_has_conf3)
1373 *v = read_gc0_config3();
1375 case KVM_REG_MIPS_CP0_CONFIG4:
1376 if (!cpu_guest_has_conf4)
1378 *v = read_gc0_config4();
1380 case KVM_REG_MIPS_CP0_CONFIG5:
1381 if (!cpu_guest_has_conf5)
1383 *v = read_gc0_config5();
1386 case KVM_REG_MIPS_CP0_XCONTEXT:
1387 *v = read_gc0_xcontext();
1390 case KVM_REG_MIPS_CP0_ERROREPC:
1391 *v = (long)read_gc0_errorepc();
1393 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
1394 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
1395 if (!cpu_guest_has_kscr(idx))
1399 *v = (long)read_gc0_kscratch1();
1402 *v = (long)read_gc0_kscratch2();
1405 *v = (long)read_gc0_kscratch3();
1408 *v = (long)read_gc0_kscratch4();
1411 *v = (long)read_gc0_kscratch5();
1414 *v = (long)read_gc0_kscratch6();
1418 case KVM_REG_MIPS_COUNT_CTL:
1419 *v = vcpu->arch.count_ctl;
1421 case KVM_REG_MIPS_COUNT_RESUME:
1422 *v = ktime_to_ns(vcpu->arch.count_resume);
1424 case KVM_REG_MIPS_COUNT_HZ:
1425 *v = vcpu->arch.count_hz;
1433 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu,
1434 const struct kvm_one_reg *reg,
1437 struct mips_coproc *cop0 = vcpu->arch.cop0;
1440 unsigned int cur, change;
1443 case KVM_REG_MIPS_CP0_INDEX:
1446 case KVM_REG_MIPS_CP0_ENTRYLO0:
1447 write_gc0_entrylo0(entrylo_user_to_kvm(v));
1449 case KVM_REG_MIPS_CP0_ENTRYLO1:
1450 write_gc0_entrylo1(entrylo_user_to_kvm(v));
1452 case KVM_REG_MIPS_CP0_CONTEXT:
1453 write_gc0_context(v);
1455 case KVM_REG_MIPS_CP0_USERLOCAL:
1456 if (!cpu_guest_has_userlocal)
1458 write_gc0_userlocal(v);
1460 case KVM_REG_MIPS_CP0_PAGEMASK:
1461 write_gc0_pagemask(v);
1463 case KVM_REG_MIPS_CP0_PAGEGRAIN:
1464 write_gc0_pagegrain(v);
1466 case KVM_REG_MIPS_CP0_WIRED:
1467 change_gc0_wired(MIPSR6_WIRED_WIRED, v);
1469 case KVM_REG_MIPS_CP0_HWRENA:
1470 write_gc0_hwrena(v);
1472 case KVM_REG_MIPS_CP0_BADVADDR:
1473 write_gc0_badvaddr(v);
1475 case KVM_REG_MIPS_CP0_COUNT:
1476 kvm_mips_write_count(vcpu, v);
1478 case KVM_REG_MIPS_CP0_ENTRYHI:
1479 write_gc0_entryhi(v);
1481 case KVM_REG_MIPS_CP0_COMPARE:
1482 kvm_mips_write_compare(vcpu, v, false);
1484 case KVM_REG_MIPS_CP0_STATUS:
1485 write_gc0_status(v);
1487 case KVM_REG_MIPS_CP0_INTCTL:
1488 write_gc0_intctl(v);
1490 case KVM_REG_MIPS_CP0_CAUSE:
1492 * If the timer is stopped or started (DC bit) it must look
1493 * atomic with changes to the timer interrupt pending bit (TI).
1494 * A timer interrupt should not happen in between.
1496 if ((read_gc0_cause() ^ v) & CAUSEF_DC) {
1497 if (v & CAUSEF_DC) {
1498 /* disable timer first */
1499 kvm_mips_count_disable_cause(vcpu);
1500 change_gc0_cause((u32)~CAUSEF_DC, v);
1502 /* enable timer last */
1503 change_gc0_cause((u32)~CAUSEF_DC, v);
1504 kvm_mips_count_enable_cause(vcpu);
1510 case KVM_REG_MIPS_CP0_EPC:
1513 case KVM_REG_MIPS_CP0_PRID:
1514 kvm_write_c0_guest_prid(cop0, v);
1516 case KVM_REG_MIPS_CP0_EBASE:
1517 kvm_vz_write_gc0_ebase(v);
1519 case KVM_REG_MIPS_CP0_CONFIG:
1520 cur = read_gc0_config();
1521 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu);
1524 write_gc0_config(v);
1527 case KVM_REG_MIPS_CP0_CONFIG1:
1528 if (!cpu_guest_has_conf1)
1530 cur = read_gc0_config1();
1531 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu);
1534 write_gc0_config1(v);
1537 case KVM_REG_MIPS_CP0_CONFIG2:
1538 if (!cpu_guest_has_conf2)
1540 cur = read_gc0_config2();
1541 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu);
1544 write_gc0_config2(v);
1547 case KVM_REG_MIPS_CP0_CONFIG3:
1548 if (!cpu_guest_has_conf3)
1550 cur = read_gc0_config3();
1551 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu);
1554 write_gc0_config3(v);
1557 case KVM_REG_MIPS_CP0_CONFIG4:
1558 if (!cpu_guest_has_conf4)
1560 cur = read_gc0_config4();
1561 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu);
1564 write_gc0_config4(v);
1567 case KVM_REG_MIPS_CP0_CONFIG5:
1568 if (!cpu_guest_has_conf5)
1570 cur = read_gc0_config5();
1571 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu);
1574 write_gc0_config5(v);
1578 case KVM_REG_MIPS_CP0_XCONTEXT:
1579 write_gc0_xcontext(v);
1582 case KVM_REG_MIPS_CP0_ERROREPC:
1583 write_gc0_errorepc(v);
1585 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
1586 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
1587 if (!cpu_guest_has_kscr(idx))
1591 write_gc0_kscratch1(v);
1594 write_gc0_kscratch2(v);
1597 write_gc0_kscratch3(v);
1600 write_gc0_kscratch4(v);
1603 write_gc0_kscratch5(v);
1606 write_gc0_kscratch6(v);
1610 case KVM_REG_MIPS_COUNT_CTL:
1611 ret = kvm_mips_set_count_ctl(vcpu, v);
1613 case KVM_REG_MIPS_COUNT_RESUME:
1614 ret = kvm_mips_set_count_resume(vcpu, v);
1616 case KVM_REG_MIPS_COUNT_HZ:
1617 ret = kvm_mips_set_count_hz(vcpu, v);
1625 #define guestid_cache(cpu) (cpu_data[cpu].guestid_cache)
1626 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu)
1628 unsigned long guestid = guestid_cache(cpu);
1630 if (!(++guestid & GUESTID_MASK)) {
1631 if (cpu_has_vtag_icache)
1634 if (!guestid) /* fix version if needed */
1635 guestid = GUESTID_FIRST_VERSION;
1637 ++guestid; /* guestid 0 reserved for root */
1639 /* start new guestid cycle */
1640 kvm_vz_local_flush_roottlb_all_guests();
1641 kvm_vz_local_flush_guesttlb_all();
1644 guestid_cache(cpu) = guestid;
1647 /* Returns 1 if the guest TLB may be clobbered */
1648 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu)
1653 if (!vcpu->requests)
1656 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1657 if (cpu_has_guestid) {
1658 /* Drop all GuestIDs for this VCPU */
1659 for_each_possible_cpu(i)
1660 vcpu->arch.vzguestid[i] = 0;
1661 /* This will clobber guest TLB contents too */
1665 * For Root ASID Dealias (RAD) we don't do anything here, but we
1666 * still need the request to ensure we recheck asid_flush_mask.
1667 * We can still return 0 as only the root TLB will be affected
1668 * by a root ASID flush.
1675 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu)
1677 unsigned int wired = read_gc0_wired();
1678 struct kvm_mips_tlb *tlbs;
1681 /* Expand the wired TLB array if necessary */
1682 wired &= MIPSR6_WIRED_WIRED;
1683 if (wired > vcpu->arch.wired_tlb_limit) {
1684 tlbs = krealloc(vcpu->arch.wired_tlb, wired *
1685 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC);
1686 if (WARN_ON(!tlbs)) {
1687 /* Save whatever we can */
1688 wired = vcpu->arch.wired_tlb_limit;
1690 vcpu->arch.wired_tlb = tlbs;
1691 vcpu->arch.wired_tlb_limit = wired;
1696 /* Save wired entries from the guest TLB */
1697 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired);
1698 /* Invalidate any dropped entries since last time */
1699 for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) {
1700 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
1701 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0;
1702 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0;
1703 vcpu->arch.wired_tlb[i].tlb_mask = 0;
1705 vcpu->arch.wired_tlb_used = wired;
1708 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu)
1710 /* Load wired entries into the guest TLB */
1711 if (vcpu->arch.wired_tlb)
1712 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0,
1713 vcpu->arch.wired_tlb_used);
1716 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu)
1718 struct kvm *kvm = vcpu->kvm;
1719 struct mm_struct *gpa_mm = &kvm->arch.gpa_mm;
1723 * Are we entering guest context on a different CPU to last time?
1724 * If so, the VCPU's guest TLB state on this CPU may be stale.
1726 migrated = (vcpu->arch.last_exec_cpu != cpu);
1727 vcpu->arch.last_exec_cpu = cpu;
1730 * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and
1731 * remains set until another vcpu is loaded in. As a rule GuestRID
1732 * remains zeroed when in root context unless the kernel is busy
1733 * manipulating guest tlb entries.
1735 if (cpu_has_guestid) {
1737 * Check if our GuestID is of an older version and thus invalid.
1739 * We also discard the stored GuestID if we've executed on
1740 * another CPU, as the guest mappings may have changed without
1741 * hypervisor knowledge.
1744 (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) &
1745 GUESTID_VERSION_MASK) {
1746 kvm_vz_get_new_guestid(cpu, vcpu);
1747 vcpu->arch.vzguestid[cpu] = guestid_cache(cpu);
1748 trace_kvm_guestid_change(vcpu,
1749 vcpu->arch.vzguestid[cpu]);
1752 /* Restore GuestID */
1753 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]);
1756 * The Guest TLB only stores a single guest's TLB state, so
1757 * flush it if another VCPU has executed on this CPU.
1759 * We also flush if we've executed on another CPU, as the guest
1760 * mappings may have changed without hypervisor knowledge.
1762 if (migrated || last_exec_vcpu[cpu] != vcpu)
1763 kvm_vz_local_flush_guesttlb_all();
1764 last_exec_vcpu[cpu] = vcpu;
1767 * Root ASID dealiases guest GPA mappings in the root TLB.
1768 * Allocate new root ASID if needed.
1770 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask)
1771 || (cpu_context(cpu, gpa_mm) ^ asid_cache(cpu)) &
1772 asid_version_mask(cpu))
1773 get_new_mmu_context(gpa_mm, cpu);
1777 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1779 struct mips_coproc *cop0 = vcpu->arch.cop0;
1783 * Have we migrated to a different CPU?
1784 * If so, any old guest TLB state may be stale.
1786 migrated = (vcpu->arch.last_sched_cpu != cpu);
1789 * Was this the last VCPU to run on this CPU?
1790 * If not, any old guest state from this VCPU will have been clobbered.
1792 all = migrated || (last_vcpu[cpu] != vcpu);
1793 last_vcpu[cpu] = vcpu;
1796 * Restore CP0_Wired unconditionally as we clear it after use, and
1797 * restore wired guest TLB entries (while in guest context).
1799 kvm_restore_gc0_wired(cop0);
1800 if (current->flags & PF_VCPU) {
1802 kvm_vz_vcpu_load_tlb(vcpu, cpu);
1803 kvm_vz_vcpu_load_wired(vcpu);
1807 * Restore timer state regardless, as e.g. Cause.TI can change over time
1808 * if left unmaintained.
1810 kvm_vz_restore_timer(vcpu);
1812 /* Don't bother restoring registers multiple times unless necessary */
1817 * Restore config registers first, as some implementations restrict
1818 * writes to other registers when the corresponding feature bits aren't
1819 * set. For example Status.CU1 cannot be set unless Config1.FP is set.
1821 kvm_restore_gc0_config(cop0);
1822 if (cpu_guest_has_conf1)
1823 kvm_restore_gc0_config1(cop0);
1824 if (cpu_guest_has_conf2)
1825 kvm_restore_gc0_config2(cop0);
1826 if (cpu_guest_has_conf3)
1827 kvm_restore_gc0_config3(cop0);
1828 if (cpu_guest_has_conf4)
1829 kvm_restore_gc0_config4(cop0);
1830 if (cpu_guest_has_conf5)
1831 kvm_restore_gc0_config5(cop0);
1832 if (cpu_guest_has_conf6)
1833 kvm_restore_gc0_config6(cop0);
1834 if (cpu_guest_has_conf7)
1835 kvm_restore_gc0_config7(cop0);
1837 kvm_restore_gc0_index(cop0);
1838 kvm_restore_gc0_entrylo0(cop0);
1839 kvm_restore_gc0_entrylo1(cop0);
1840 kvm_restore_gc0_context(cop0);
1842 kvm_restore_gc0_xcontext(cop0);
1844 kvm_restore_gc0_pagemask(cop0);
1845 kvm_restore_gc0_pagegrain(cop0);
1846 kvm_restore_gc0_hwrena(cop0);
1847 kvm_restore_gc0_badvaddr(cop0);
1848 kvm_restore_gc0_entryhi(cop0);
1849 kvm_restore_gc0_status(cop0);
1850 kvm_restore_gc0_intctl(cop0);
1851 kvm_restore_gc0_epc(cop0);
1852 kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0));
1853 if (cpu_guest_has_userlocal)
1854 kvm_restore_gc0_userlocal(cop0);
1856 kvm_restore_gc0_errorepc(cop0);
1858 /* restore KScratch registers if enabled in guest */
1859 if (cpu_guest_has_conf4) {
1860 if (cpu_guest_has_kscr(2))
1861 kvm_restore_gc0_kscratch1(cop0);
1862 if (cpu_guest_has_kscr(3))
1863 kvm_restore_gc0_kscratch2(cop0);
1864 if (cpu_guest_has_kscr(4))
1865 kvm_restore_gc0_kscratch3(cop0);
1866 if (cpu_guest_has_kscr(5))
1867 kvm_restore_gc0_kscratch4(cop0);
1868 if (cpu_guest_has_kscr(6))
1869 kvm_restore_gc0_kscratch5(cop0);
1870 if (cpu_guest_has_kscr(7))
1871 kvm_restore_gc0_kscratch6(cop0);
1874 /* restore Root.GuestCtl2 from unused Guest guestctl2 register */
1875 if (cpu_has_guestctl2)
1877 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]);
1882 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
1884 struct mips_coproc *cop0 = vcpu->arch.cop0;
1886 if (current->flags & PF_VCPU)
1887 kvm_vz_vcpu_save_wired(vcpu);
1891 kvm_save_gc0_index(cop0);
1892 kvm_save_gc0_entrylo0(cop0);
1893 kvm_save_gc0_entrylo1(cop0);
1894 kvm_save_gc0_context(cop0);
1896 kvm_save_gc0_xcontext(cop0);
1898 kvm_save_gc0_pagemask(cop0);
1899 kvm_save_gc0_pagegrain(cop0);
1900 kvm_save_gc0_wired(cop0);
1901 /* allow wired TLB entries to be overwritten */
1902 clear_gc0_wired(MIPSR6_WIRED_WIRED);
1903 kvm_save_gc0_hwrena(cop0);
1904 kvm_save_gc0_badvaddr(cop0);
1905 kvm_save_gc0_entryhi(cop0);
1906 kvm_save_gc0_status(cop0);
1907 kvm_save_gc0_intctl(cop0);
1908 kvm_save_gc0_epc(cop0);
1909 kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase());
1910 if (cpu_guest_has_userlocal)
1911 kvm_save_gc0_userlocal(cop0);
1913 /* only save implemented config registers */
1914 kvm_save_gc0_config(cop0);
1915 if (cpu_guest_has_conf1)
1916 kvm_save_gc0_config1(cop0);
1917 if (cpu_guest_has_conf2)
1918 kvm_save_gc0_config2(cop0);
1919 if (cpu_guest_has_conf3)
1920 kvm_save_gc0_config3(cop0);
1921 if (cpu_guest_has_conf4)
1922 kvm_save_gc0_config4(cop0);
1923 if (cpu_guest_has_conf5)
1924 kvm_save_gc0_config5(cop0);
1925 if (cpu_guest_has_conf6)
1926 kvm_save_gc0_config6(cop0);
1927 if (cpu_guest_has_conf7)
1928 kvm_save_gc0_config7(cop0);
1930 kvm_save_gc0_errorepc(cop0);
1932 /* save KScratch registers if enabled in guest */
1933 if (cpu_guest_has_conf4) {
1934 if (cpu_guest_has_kscr(2))
1935 kvm_save_gc0_kscratch1(cop0);
1936 if (cpu_guest_has_kscr(3))
1937 kvm_save_gc0_kscratch2(cop0);
1938 if (cpu_guest_has_kscr(4))
1939 kvm_save_gc0_kscratch3(cop0);
1940 if (cpu_guest_has_kscr(5))
1941 kvm_save_gc0_kscratch4(cop0);
1942 if (cpu_guest_has_kscr(6))
1943 kvm_save_gc0_kscratch5(cop0);
1944 if (cpu_guest_has_kscr(7))
1945 kvm_save_gc0_kscratch6(cop0);
1948 kvm_vz_save_timer(vcpu);
1950 /* save Root.GuestCtl2 in unused Guest guestctl2 register */
1951 if (cpu_has_guestctl2)
1952 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] =
1953 read_c0_guestctl2();
1959 * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB.
1960 * @size: Number of guest VTLB entries (0 < @size <= root VTLB entries).
1962 * Attempt to resize the guest VTLB by writing guest Config registers. This is
1963 * necessary for cores with a shared root/guest TLB to avoid overlap with wired
1964 * entries in the root VTLB.
1966 * Returns: The resulting guest VTLB size.
1968 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size)
1970 unsigned int config4 = 0, ret = 0, limit;
1972 /* Write MMUSize - 1 into guest Config registers */
1973 if (cpu_guest_has_conf1)
1974 change_gc0_config1(MIPS_CONF1_TLBS,
1975 (size - 1) << MIPS_CONF1_TLBS_SHIFT);
1976 if (cpu_guest_has_conf4) {
1977 config4 = read_gc0_config4();
1978 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
1979 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) {
1980 config4 &= ~MIPS_CONF4_VTLBSIZEEXT;
1981 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
1982 MIPS_CONF4_VTLBSIZEEXT_SHIFT;
1983 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
1984 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) {
1985 config4 &= ~MIPS_CONF4_MMUSIZEEXT;
1986 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
1987 MIPS_CONF4_MMUSIZEEXT_SHIFT;
1989 write_gc0_config4(config4);
1993 * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it
1994 * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write
1997 if (cpu_has_mips_r6) {
1998 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >>
1999 MIPSR6_WIRED_LIMIT_SHIFT;
2000 if (size - 1 <= limit)
2002 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT);
2005 /* Read back MMUSize - 1 */
2006 back_to_back_c0_hazard();
2007 if (cpu_guest_has_conf1)
2008 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >>
2009 MIPS_CONF1_TLBS_SHIFT;
2011 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
2012 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT)
2013 ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >>
2014 MIPS_CONF4_VTLBSIZEEXT_SHIFT) <<
2015 MIPS_CONF1_TLBS_SIZE;
2016 else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
2017 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT)
2018 ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >>
2019 MIPS_CONF4_MMUSIZEEXT_SHIFT) <<
2020 MIPS_CONF1_TLBS_SIZE;
2025 static int kvm_vz_hardware_enable(void)
2027 unsigned int mmu_size, guest_mmu_size, ftlb_size;
2030 * ImgTec cores tend to use a shared root/guest TLB. To avoid overlap of
2031 * root wired and guest entries, the guest TLB may need resizing.
2033 mmu_size = current_cpu_data.tlbsizevtlb;
2034 ftlb_size = current_cpu_data.tlbsize - mmu_size;
2036 /* Try switching to maximum guest VTLB size for flush */
2037 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size);
2038 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
2039 kvm_vz_local_flush_guesttlb_all();
2042 * Reduce to make space for root wired entries and at least 2 root
2043 * non-wired entries. This does assume that long-term wired entries
2044 * won't be added later.
2046 guest_mmu_size = mmu_size - num_wired_entries() - 2;
2047 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size);
2048 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
2051 * Write the VTLB size, but if another CPU has already written, check it
2052 * matches or we won't provide a consistent view to the guest. If this
2053 * ever happens it suggests an asymmetric number of wired entries.
2055 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) &&
2056 WARN(guest_mmu_size != kvm_vz_guest_vtlb_size,
2057 "Available guest VTLB size mismatch"))
2061 * Enable virtualization features granting guest direct control of
2063 * CP0=1: Guest coprocessor 0 context.
2064 * AT=Guest: Guest MMU.
2065 * CG=1: Hit (virtual address) CACHE operations (optional).
2066 * CF=1: Guest Config registers.
2067 * CGI=1: Indexed flush CACHE operations (optional).
2069 write_c0_guestctl0(MIPS_GCTL0_CP0 |
2070 (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) |
2071 MIPS_GCTL0_CG | MIPS_GCTL0_CF);
2072 if (cpu_has_guestctl0ext)
2073 set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI);
2075 if (cpu_has_guestid) {
2076 write_c0_guestctl1(0);
2077 kvm_vz_local_flush_roottlb_all_guests();
2079 GUESTID_MASK = current_cpu_data.guestid_mask;
2080 GUESTID_FIRST_VERSION = GUESTID_MASK + 1;
2081 GUESTID_VERSION_MASK = ~GUESTID_MASK;
2083 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION;
2086 /* clear any pending injected virtual guest interrupts */
2087 if (cpu_has_guestctl2)
2088 clear_c0_guestctl2(0x3f << 10);
2093 static void kvm_vz_hardware_disable(void)
2095 kvm_vz_local_flush_guesttlb_all();
2097 if (cpu_has_guestid) {
2098 write_c0_guestctl1(0);
2099 kvm_vz_local_flush_roottlb_all_guests();
2103 static int kvm_vz_check_extension(struct kvm *kvm, long ext)
2108 case KVM_CAP_MIPS_VZ:
2109 /* we wouldn't be here unless cpu_has_vz */
2113 case KVM_CAP_MIPS_64BIT:
2114 /* We support 64-bit registers/operations and addresses */
2126 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu)
2130 for_each_possible_cpu(i)
2131 vcpu->arch.vzguestid[i] = 0;
2136 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu)
2141 * If the VCPU is freed and reused as another VCPU, we don't want the
2142 * matching pointer wrongly hanging around in last_vcpu[] or
2145 for_each_possible_cpu(cpu) {
2146 if (last_vcpu[cpu] == vcpu)
2147 last_vcpu[cpu] = NULL;
2148 if (last_exec_vcpu[cpu] == vcpu)
2149 last_exec_vcpu[cpu] = NULL;
2153 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu)
2155 struct mips_coproc *cop0 = vcpu->arch.cop0;
2156 unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */
2159 * Start off the timer at the same frequency as the host timer, but the
2160 * soft timer doesn't handle frequencies greater than 1GHz yet.
2162 if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC)
2163 count_hz = mips_hpt_frequency;
2164 kvm_mips_init_count(vcpu, count_hz);
2167 * Initialize guest register state to valid architectural reset state.
2171 if (cpu_has_mips_r6)
2172 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC);
2174 if (cpu_has_mips_r6)
2175 kvm_write_sw_gc0_wired(cop0,
2176 read_gc0_wired() & MIPSR6_WIRED_LIMIT);
2178 kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL);
2179 if (cpu_has_mips_r6)
2180 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status());
2182 kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() &
2183 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI));
2185 kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id);
2187 kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id);
2189 kvm_save_gc0_config(cop0);
2190 /* architecturally writable (e.g. from guest) */
2191 kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK,
2192 _page_cachable_default >> _CACHE_SHIFT);
2193 /* architecturally read only, but maybe writable from root */
2194 kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config());
2195 if (cpu_guest_has_conf1) {
2196 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M);
2198 kvm_save_gc0_config1(cop0);
2199 /* architecturally read only, but maybe writable from root */
2200 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2 |
2207 if (cpu_guest_has_conf2) {
2208 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M);
2210 kvm_save_gc0_config2(cop0);
2212 if (cpu_guest_has_conf3) {
2213 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M);
2215 kvm_save_gc0_config3(cop0);
2216 /* architecturally writable (e.g. from guest) */
2217 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE);
2218 /* architecturally read only, but maybe writable from root */
2219 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA |
2234 if (cpu_guest_has_conf4) {
2235 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M);
2237 kvm_save_gc0_config4(cop0);
2239 if (cpu_guest_has_conf5) {
2240 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M);
2242 kvm_save_gc0_config5(cop0);
2243 /* architecturally writable (e.g. from guest) */
2244 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K |
2251 /* architecturally read only, but maybe writable from root */
2252 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP);
2255 /* start with no pending virtual guest interrupts */
2256 if (cpu_has_guestctl2)
2257 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0;
2259 /* Put PC at reset vector */
2260 vcpu->arch.pc = CKSEG1ADDR(0x1fc00000);
2265 static void kvm_vz_flush_shadow_all(struct kvm *kvm)
2267 if (cpu_has_guestid) {
2268 /* Flush GuestID for each VCPU individually */
2269 kvm_flush_remote_tlbs(kvm);
2272 * For each CPU there is a single GPA ASID used by all VCPUs in
2273 * the VM, so it doesn't make sense for the VCPUs to handle
2274 * invalidation of these ASIDs individually.
2276 * Instead mark all CPUs as needing ASID invalidation in
2277 * asid_flush_mask, and just use kvm_flush_remote_tlbs(kvm) to
2278 * kick any running VCPUs so they check asid_flush_mask.
2280 cpumask_setall(&kvm->arch.asid_flush_mask);
2281 kvm_flush_remote_tlbs(kvm);
2285 static void kvm_vz_flush_shadow_memslot(struct kvm *kvm,
2286 const struct kvm_memory_slot *slot)
2288 kvm_vz_flush_shadow_all(kvm);
2291 static void kvm_vz_vcpu_reenter(struct kvm_run *run, struct kvm_vcpu *vcpu)
2293 int cpu = smp_processor_id();
2294 int preserve_guest_tlb;
2296 preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu);
2298 if (preserve_guest_tlb)
2299 kvm_vz_vcpu_save_wired(vcpu);
2301 kvm_vz_vcpu_load_tlb(vcpu, cpu);
2303 if (preserve_guest_tlb)
2304 kvm_vz_vcpu_load_wired(vcpu);
2307 static int kvm_vz_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
2309 int cpu = smp_processor_id();
2312 /* Check if we have any exceptions/interrupts pending */
2313 kvm_mips_deliver_interrupts(vcpu, read_gc0_cause());
2315 kvm_vz_check_requests(vcpu, cpu);
2316 kvm_vz_vcpu_load_tlb(vcpu, cpu);
2317 kvm_vz_vcpu_load_wired(vcpu);
2319 r = vcpu->arch.vcpu_run(run, vcpu);
2321 kvm_vz_vcpu_save_wired(vcpu);
2326 static struct kvm_mips_callbacks kvm_vz_callbacks = {
2327 .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable,
2328 .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss,
2329 .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss,
2330 .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss,
2331 .handle_addr_err_st = kvm_trap_vz_no_handler,
2332 .handle_addr_err_ld = kvm_trap_vz_no_handler,
2333 .handle_syscall = kvm_trap_vz_no_handler,
2334 .handle_res_inst = kvm_trap_vz_no_handler,
2335 .handle_break = kvm_trap_vz_no_handler,
2336 .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled,
2337 .handle_guest_exit = kvm_trap_vz_handle_guest_exit,
2339 .hardware_enable = kvm_vz_hardware_enable,
2340 .hardware_disable = kvm_vz_hardware_disable,
2341 .check_extension = kvm_vz_check_extension,
2342 .vcpu_init = kvm_vz_vcpu_init,
2343 .vcpu_uninit = kvm_vz_vcpu_uninit,
2344 .vcpu_setup = kvm_vz_vcpu_setup,
2345 .flush_shadow_all = kvm_vz_flush_shadow_all,
2346 .flush_shadow_memslot = kvm_vz_flush_shadow_memslot,
2347 .gva_to_gpa = kvm_vz_gva_to_gpa_cb,
2348 .queue_timer_int = kvm_vz_queue_timer_int_cb,
2349 .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb,
2350 .queue_io_int = kvm_vz_queue_io_int_cb,
2351 .dequeue_io_int = kvm_vz_dequeue_io_int_cb,
2352 .irq_deliver = kvm_vz_irq_deliver_cb,
2353 .irq_clear = kvm_vz_irq_clear_cb,
2354 .num_regs = kvm_vz_num_regs,
2355 .copy_reg_indices = kvm_vz_copy_reg_indices,
2356 .get_one_reg = kvm_vz_get_one_reg,
2357 .set_one_reg = kvm_vz_set_one_reg,
2358 .vcpu_load = kvm_vz_vcpu_load,
2359 .vcpu_put = kvm_vz_vcpu_put,
2360 .vcpu_run = kvm_vz_vcpu_run,
2361 .vcpu_reenter = kvm_vz_vcpu_reenter,
2364 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
2370 * VZ requires at least 2 KScratch registers, so it should have been
2371 * possible to allocate pgd_reg.
2373 if (WARN(pgd_reg == -1,
2374 "pgd_reg not allocated even though cpu_has_vz\n"))
2377 pr_info("Starting KVM with MIPS VZ extensions\n");
2379 *install_callbacks = &kvm_vz_callbacks;