2 * Copyright 2012 Michael Ellerman, IBM Corporation.
3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
10 #include <linux/kernel.h>
11 #include <linux/kvm_host.h>
12 #include <linux/err.h>
13 #include <linux/gfp.h>
14 #include <linux/anon_inodes.h>
16 #include <asm/uaccess.h>
17 #include <asm/kvm_book3s.h>
18 #include <asm/kvm_ppc.h>
19 #include <asm/hvcall.h>
21 #include <asm/debug.h>
24 #include <linux/debugfs.h>
25 #include <linux/seq_file.h>
27 #include "book3s_xics.h"
30 #define XICS_DBG(fmt...) do { } while (0)
32 #define XICS_DBG(fmt...) trace_printk(fmt)
35 #define ENABLE_REALMODE true
36 #define DEBUG_REALMODE false
42 * Each ICS has a mutex protecting the information about the IRQ
43 * sources and avoiding simultaneous deliveries if the same interrupt.
45 * ICP operations are done via a single compare & swap transaction
46 * (most ICP state fits in the union kvmppc_icp_state)
53 * - To speed up resends, keep a bitmap of "resend" set bits in the
56 * - Speed up server# -> ICP lookup (array ? hash table ?)
58 * - Make ICS lockless as well, or at least a per-interrupt lock or hashed
59 * locks array to improve scalability
62 /* -- ICS routines -- */
64 static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
68 * Return value ideally indicates how the interrupt was handled, but no
69 * callers look at it (given that we don't implement KVM_IRQ_LINE_STATUS),
72 static int ics_deliver_irq(struct kvmppc_xics *xics, u32 irq, u32 level)
74 struct ics_irq_state *state;
75 struct kvmppc_ics *ics;
78 XICS_DBG("ics deliver %#x (level: %d)\n", irq, level);
80 ics = kvmppc_xics_find_ics(xics, irq, &src);
82 XICS_DBG("ics_deliver_irq: IRQ 0x%06x not found !\n", irq);
85 state = &ics->irq_state[src];
90 * We set state->asserted locklessly. This should be fine as
91 * we are the only setter, thus concurrent access is undefined
94 if (level == 1 || level == KVM_INTERRUPT_SET_LEVEL)
96 else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
101 /* Attempt delivery */
102 icp_deliver_irq(xics, NULL, irq);
107 static void ics_check_resend(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
108 struct kvmppc_icp *icp)
112 mutex_lock(&ics->lock);
114 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
115 struct ics_irq_state *state = &ics->irq_state[i];
120 XICS_DBG("resend %#x prio %#x\n", state->number,
123 mutex_unlock(&ics->lock);
124 icp_deliver_irq(xics, icp, state->number);
125 mutex_lock(&ics->lock);
128 mutex_unlock(&ics->lock);
131 static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
132 struct ics_irq_state *state,
133 u32 server, u32 priority, u32 saved_priority)
137 mutex_lock(&ics->lock);
139 state->server = server;
140 state->priority = priority;
141 state->saved_priority = saved_priority;
143 if ((state->masked_pending || state->resend) && priority != MASKED) {
144 state->masked_pending = 0;
148 mutex_unlock(&ics->lock);
153 int kvmppc_xics_set_xive(struct kvm *kvm, u32 irq, u32 server, u32 priority)
155 struct kvmppc_xics *xics = kvm->arch.xics;
156 struct kvmppc_icp *icp;
157 struct kvmppc_ics *ics;
158 struct ics_irq_state *state;
164 ics = kvmppc_xics_find_ics(xics, irq, &src);
167 state = &ics->irq_state[src];
169 icp = kvmppc_xics_find_server(kvm, server);
173 XICS_DBG("set_xive %#x server %#x prio %#x MP:%d RS:%d\n",
174 irq, server, priority,
175 state->masked_pending, state->resend);
177 if (write_xive(xics, ics, state, server, priority, priority))
178 icp_deliver_irq(xics, icp, irq);
183 int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority)
185 struct kvmppc_xics *xics = kvm->arch.xics;
186 struct kvmppc_ics *ics;
187 struct ics_irq_state *state;
193 ics = kvmppc_xics_find_ics(xics, irq, &src);
196 state = &ics->irq_state[src];
198 mutex_lock(&ics->lock);
199 *server = state->server;
200 *priority = state->priority;
201 mutex_unlock(&ics->lock);
206 int kvmppc_xics_int_on(struct kvm *kvm, u32 irq)
208 struct kvmppc_xics *xics = kvm->arch.xics;
209 struct kvmppc_icp *icp;
210 struct kvmppc_ics *ics;
211 struct ics_irq_state *state;
217 ics = kvmppc_xics_find_ics(xics, irq, &src);
220 state = &ics->irq_state[src];
222 icp = kvmppc_xics_find_server(kvm, state->server);
226 if (write_xive(xics, ics, state, state->server, state->saved_priority,
227 state->saved_priority))
228 icp_deliver_irq(xics, icp, irq);
233 int kvmppc_xics_int_off(struct kvm *kvm, u32 irq)
235 struct kvmppc_xics *xics = kvm->arch.xics;
236 struct kvmppc_ics *ics;
237 struct ics_irq_state *state;
243 ics = kvmppc_xics_find_ics(xics, irq, &src);
246 state = &ics->irq_state[src];
248 write_xive(xics, ics, state, state->server, MASKED, state->priority);
253 /* -- ICP routines, including hcalls -- */
255 static inline bool icp_try_update(struct kvmppc_icp *icp,
256 union kvmppc_icp_state old,
257 union kvmppc_icp_state new,
262 /* Calculate new output value */
263 new.out_ee = (new.xisr && (new.pending_pri < new.cppr));
265 /* Attempt atomic update */
266 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;
270 XICS_DBG("UPD [%04x] - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
272 old.cppr, old.mfrr, old.pending_pri, old.xisr,
273 old.need_resend, old.out_ee);
274 XICS_DBG("UPD - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
275 new.cppr, new.mfrr, new.pending_pri, new.xisr,
276 new.need_resend, new.out_ee);
278 * Check for output state update
280 * Note that this is racy since another processor could be updating
281 * the state already. This is why we never clear the interrupt output
282 * here, we only ever set it. The clear only happens prior to doing
283 * an update and only by the processor itself. Currently we do it
284 * in Accept (H_XIRR) and Up_Cppr (H_XPPR).
286 * We also do not try to figure out whether the EE state has changed,
287 * we unconditionally set it if the new state calls for it. The reason
288 * for that is that we opportunistically remove the pending interrupt
289 * flag when raising CPPR, so we need to set it back here if an
290 * interrupt is still pending.
293 kvmppc_book3s_queue_irqprio(icp->vcpu,
294 BOOK3S_INTERRUPT_EXTERNAL_LEVEL);
296 kvmppc_fast_vcpu_kick(icp->vcpu);
302 static void icp_check_resend(struct kvmppc_xics *xics,
303 struct kvmppc_icp *icp)
307 /* Order this load with the test for need_resend in the caller */
309 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {
310 struct kvmppc_ics *ics = xics->ics[icsid];
312 if (!test_and_clear_bit(icsid, icp->resend_map))
316 ics_check_resend(xics, ics, icp);
320 static bool icp_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,
323 union kvmppc_icp_state old_state, new_state;
326 XICS_DBG("try deliver %#x(P:%#x) to server %#x\n", irq, priority,
330 old_state = new_state = ACCESS_ONCE(icp->state);
334 /* See if we can deliver */
335 success = new_state.cppr > priority &&
336 new_state.mfrr > priority &&
337 new_state.pending_pri > priority;
340 * If we can, check for a rejection and perform the
344 *reject = new_state.xisr;
345 new_state.xisr = irq;
346 new_state.pending_pri = priority;
349 * If we failed to deliver we set need_resend
350 * so a subsequent CPPR state change causes us
351 * to try a new delivery.
353 new_state.need_resend = true;
356 } while (!icp_try_update(icp, old_state, new_state, false));
361 static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
364 struct ics_irq_state *state;
365 struct kvmppc_ics *ics;
370 * This is used both for initial delivery of an interrupt and
371 * for subsequent rejection.
373 * Rejection can be racy vs. resends. We have evaluated the
374 * rejection in an atomic ICP transaction which is now complete,
375 * so potentially the ICP can already accept the interrupt again.
377 * So we need to retry the delivery. Essentially the reject path
378 * boils down to a failed delivery. Always.
380 * Now the interrupt could also have moved to a different target,
381 * thus we may need to re-do the ICP lookup as well
385 /* Get the ICS state and lock it */
386 ics = kvmppc_xics_find_ics(xics, new_irq, &src);
388 XICS_DBG("icp_deliver_irq: IRQ 0x%06x not found !\n", new_irq);
391 state = &ics->irq_state[src];
393 /* Get a lock on the ICS */
394 mutex_lock(&ics->lock);
397 if (!icp || state->server != icp->server_num) {
398 icp = kvmppc_xics_find_server(xics->kvm, state->server);
400 pr_warn("icp_deliver_irq: IRQ 0x%06x server 0x%x not found !\n",
401 new_irq, state->server);
406 /* Clear the resend bit of that interrupt */
410 * If masked, bail out
412 * Note: PAPR doesn't mention anything about masked pending
413 * when doing a resend, only when doing a delivery.
415 * However that would have the effect of losing a masked
416 * interrupt that was rejected and isn't consistent with
417 * the whole masked_pending business which is about not
418 * losing interrupts that occur while masked.
420 * I don't differenciate normal deliveries and resends, this
421 * implementation will differ from PAPR and not lose such
424 if (state->priority == MASKED) {
425 XICS_DBG("irq %#x masked pending\n", new_irq);
426 state->masked_pending = 1;
431 * Try the delivery, this will set the need_resend flag
432 * in the ICP as part of the atomic transaction if the
433 * delivery is not possible.
435 * Note that if successful, the new delivery might have itself
436 * rejected an interrupt that was "delivered" before we took the
439 * In this case we do the whole sequence all over again for the
440 * new guy. We cannot assume that the rejected interrupt is less
441 * favored than the new one, and thus doesn't need to be delivered,
442 * because by the time we exit icp_try_to_deliver() the target
443 * processor may well have alrady consumed & completed it, and thus
444 * the rejected interrupt might actually be already acceptable.
446 if (icp_try_to_deliver(icp, new_irq, state->priority, &reject)) {
448 * Delivery was successful, did we reject somebody else ?
450 if (reject && reject != XICS_IPI) {
451 mutex_unlock(&ics->lock);
457 * We failed to deliver the interrupt we need to set the
458 * resend map bit and mark the ICS state as needing a resend
460 set_bit(ics->icsid, icp->resend_map);
464 * If the need_resend flag got cleared in the ICP some time
465 * between icp_try_to_deliver() atomic update and now, then
466 * we know it might have missed the resend_map bit. So we
470 if (!icp->state.need_resend) {
471 mutex_unlock(&ics->lock);
476 mutex_unlock(&ics->lock);
479 static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
482 union kvmppc_icp_state old_state, new_state;
486 * This handles several related states in one operation:
488 * ICP State: Down_CPPR
490 * Load CPPR with new value and if the XISR is 0
491 * then check for resends:
495 * If MFRR is more favored than CPPR, check for IPIs
496 * and notify ICS of a potential resend. This is done
497 * asynchronously (when used in real mode, we will have
500 * We do not handle the complete Check_IPI as documented
501 * here. In the PAPR, this state will be used for both
502 * Set_MFRR and Down_CPPR. However, we know that we aren't
503 * changing the MFRR state here so we don't need to handle
504 * the case of an MFRR causing a reject of a pending irq,
505 * this will have been handled when the MFRR was set in the
508 * Thus we don't have to handle rejects, only resends.
510 * When implementing real mode for HV KVM, resend will lead to
511 * a H_TOO_HARD return and the whole transaction will be handled
515 old_state = new_state = ACCESS_ONCE(icp->state);
518 new_state.cppr = new_cppr;
521 * Cut down Resend / Check_IPI / IPI
523 * The logic is that we cannot have a pending interrupt
524 * trumped by an IPI at this point (see above), so we
525 * know that either the pending interrupt is already an
526 * IPI (in which case we don't care to override it) or
527 * it's either more favored than us or non existent
529 if (new_state.mfrr < new_cppr &&
530 new_state.mfrr <= new_state.pending_pri) {
531 WARN_ON(new_state.xisr != XICS_IPI &&
532 new_state.xisr != 0);
533 new_state.pending_pri = new_state.mfrr;
534 new_state.xisr = XICS_IPI;
537 /* Latch/clear resend bit */
538 resend = new_state.need_resend;
539 new_state.need_resend = 0;
541 } while (!icp_try_update(icp, old_state, new_state, true));
544 * Now handle resend checks. Those are asynchronous to the ICP
545 * state update in HW (ie bus transactions) so we can handle them
546 * separately here too
549 icp_check_resend(xics, icp);
552 static noinline unsigned long kvmppc_h_xirr(struct kvm_vcpu *vcpu)
554 union kvmppc_icp_state old_state, new_state;
555 struct kvmppc_icp *icp = vcpu->arch.icp;
558 /* First, remove EE from the processor */
559 kvmppc_book3s_dequeue_irqprio(icp->vcpu,
560 BOOK3S_INTERRUPT_EXTERNAL_LEVEL);
563 * ICP State: Accept_Interrupt
565 * Return the pending interrupt (if any) along with the
566 * current CPPR, then clear the XISR & set CPPR to the
570 old_state = new_state = ACCESS_ONCE(icp->state);
572 xirr = old_state.xisr | (((u32)old_state.cppr) << 24);
575 new_state.cppr = new_state.pending_pri;
576 new_state.pending_pri = 0xff;
579 } while (!icp_try_update(icp, old_state, new_state, true));
581 XICS_DBG("h_xirr vcpu %d xirr %#x\n", vcpu->vcpu_id, xirr);
586 static noinline int kvmppc_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
589 union kvmppc_icp_state old_state, new_state;
590 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
591 struct kvmppc_icp *icp;
596 XICS_DBG("h_ipi vcpu %d to server %lu mfrr %#lx\n",
597 vcpu->vcpu_id, server, mfrr);
599 icp = vcpu->arch.icp;
600 local = icp->server_num == server;
602 icp = kvmppc_xics_find_server(vcpu->kvm, server);
608 * ICP state: Set_MFRR
610 * If the CPPR is more favored than the new MFRR, then
611 * nothing needs to be rejected as there can be no XISR to
612 * reject. If the MFRR is being made less favored then
613 * there might be a previously-rejected interrupt needing
616 * If the CPPR is less favored, then we might be replacing
617 * an interrupt, and thus need to possibly reject it as in
619 * ICP state: Check_IPI
622 old_state = new_state = ACCESS_ONCE(icp->state);
625 new_state.mfrr = mfrr;
630 if (mfrr < new_state.cppr) {
631 /* Reject a pending interrupt if not an IPI */
632 if (mfrr <= new_state.pending_pri)
633 reject = new_state.xisr;
634 new_state.pending_pri = mfrr;
635 new_state.xisr = XICS_IPI;
638 if (mfrr > old_state.mfrr && mfrr > new_state.cppr) {
639 resend = new_state.need_resend;
640 new_state.need_resend = 0;
642 } while (!icp_try_update(icp, old_state, new_state, local));
645 if (reject && reject != XICS_IPI)
646 icp_deliver_irq(xics, icp, reject);
650 icp_check_resend(xics, icp);
655 static int kvmppc_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
657 union kvmppc_icp_state state;
658 struct kvmppc_icp *icp;
660 icp = vcpu->arch.icp;
661 if (icp->server_num != server) {
662 icp = kvmppc_xics_find_server(vcpu->kvm, server);
666 state = ACCESS_ONCE(icp->state);
667 kvmppc_set_gpr(vcpu, 4, ((u32)state.cppr << 24) | state.xisr);
668 kvmppc_set_gpr(vcpu, 5, state.mfrr);
672 static noinline void kvmppc_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
674 union kvmppc_icp_state old_state, new_state;
675 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
676 struct kvmppc_icp *icp = vcpu->arch.icp;
679 XICS_DBG("h_cppr vcpu %d cppr %#lx\n", vcpu->vcpu_id, cppr);
682 * ICP State: Set_CPPR
684 * We can safely compare the new value with the current
685 * value outside of the transaction as the CPPR is only
686 * ever changed by the processor on itself
688 if (cppr > icp->state.cppr)
689 icp_down_cppr(xics, icp, cppr);
690 else if (cppr == icp->state.cppr)
696 * The processor is raising its priority, this can result
697 * in a rejection of a pending interrupt:
699 * ICP State: Reject_Current
701 * We can remove EE from the current processor, the update
702 * transaction will set it again if needed
704 kvmppc_book3s_dequeue_irqprio(icp->vcpu,
705 BOOK3S_INTERRUPT_EXTERNAL_LEVEL);
708 old_state = new_state = ACCESS_ONCE(icp->state);
711 new_state.cppr = cppr;
713 if (cppr <= new_state.pending_pri) {
714 reject = new_state.xisr;
716 new_state.pending_pri = 0xff;
719 } while (!icp_try_update(icp, old_state, new_state, true));
722 * Check for rejects. They are handled by doing a new delivery
723 * attempt (see comments in icp_deliver_irq).
725 if (reject && reject != XICS_IPI)
726 icp_deliver_irq(xics, icp, reject);
729 static noinline int kvmppc_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
731 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
732 struct kvmppc_icp *icp = vcpu->arch.icp;
733 struct kvmppc_ics *ics;
734 struct ics_irq_state *state;
735 u32 irq = xirr & 0x00ffffff;
738 XICS_DBG("h_eoi vcpu %d eoi %#lx\n", vcpu->vcpu_id, xirr);
743 * Note: If EOI is incorrectly used by SW to lower the CPPR
744 * value (ie more favored), we do not check for rejection of
745 * a pending interrupt, this is a SW error and PAPR sepcifies
746 * that we don't have to deal with it.
748 * The sending of an EOI to the ICS is handled after the
751 * ICP State: Down_CPPR which we handle
752 * in a separate function as it's shared with H_CPPR.
754 icp_down_cppr(xics, icp, xirr >> 24);
756 /* IPIs have no EOI */
760 * EOI handling: If the interrupt is still asserted, we need to
761 * resend it. We can take a lockless "peek" at the ICS state here.
763 * "Message" interrupts will never have "asserted" set
765 ics = kvmppc_xics_find_ics(xics, irq, &src);
767 XICS_DBG("h_eoi: IRQ 0x%06x not found !\n", irq);
770 state = &ics->irq_state[src];
772 /* Still asserted, resend it */
774 icp_deliver_irq(xics, icp, irq);
776 kvm_notify_acked_irq(vcpu->kvm, 0, irq);
781 static noinline int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall)
783 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
784 struct kvmppc_icp *icp = vcpu->arch.icp;
786 XICS_DBG("XICS_RM: H_%x completing, act: %x state: %lx tgt: %p\n",
787 hcall, icp->rm_action, icp->rm_dbgstate.raw, icp->rm_dbgtgt);
789 if (icp->rm_action & XICS_RM_KICK_VCPU)
790 kvmppc_fast_vcpu_kick(icp->rm_kick_target);
791 if (icp->rm_action & XICS_RM_CHECK_RESEND)
792 icp_check_resend(xics, icp);
793 if (icp->rm_action & XICS_RM_REJECT)
794 icp_deliver_irq(xics, icp, icp->rm_reject);
795 if (icp->rm_action & XICS_RM_NOTIFY_EOI)
796 kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq);
803 int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 req)
805 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
809 /* Check if we have an ICP */
810 if (!xics || !vcpu->arch.icp)
813 /* These requests don't have real-mode implementations at present */
816 res = kvmppc_h_xirr(vcpu);
817 kvmppc_set_gpr(vcpu, 4, res);
818 kvmppc_set_gpr(vcpu, 5, get_tb());
821 rc = kvmppc_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4));
825 /* Check for real mode returning too hard */
826 if (xics->real_mode && is_kvmppc_hv_enabled(vcpu->kvm))
827 return kvmppc_xics_rm_complete(vcpu, req);
831 res = kvmppc_h_xirr(vcpu);
832 kvmppc_set_gpr(vcpu, 4, res);
835 kvmppc_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4));
838 rc = kvmppc_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4));
841 rc = kvmppc_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4),
842 kvmppc_get_gpr(vcpu, 5));
848 EXPORT_SYMBOL_GPL(kvmppc_xics_hcall);
851 /* -- Initialisation code etc. -- */
853 static int xics_debug_show(struct seq_file *m, void *private)
855 struct kvmppc_xics *xics = m->private;
856 struct kvm *kvm = xics->kvm;
857 struct kvm_vcpu *vcpu;
863 seq_printf(m, "=========\nICP state\n=========\n");
865 kvm_for_each_vcpu(i, vcpu, kvm) {
866 struct kvmppc_icp *icp = vcpu->arch.icp;
867 union kvmppc_icp_state state;
872 state.raw = ACCESS_ONCE(icp->state.raw);
873 seq_printf(m, "cpu server %#lx XIRR:%#x PPRI:%#x CPPR:%#x MFRR:%#x OUT:%d NR:%d\n",
874 icp->server_num, state.xisr,
875 state.pending_pri, state.cppr, state.mfrr,
876 state.out_ee, state.need_resend);
879 for (icsid = 0; icsid <= KVMPPC_XICS_MAX_ICS_ID; icsid++) {
880 struct kvmppc_ics *ics = xics->ics[icsid];
885 seq_printf(m, "=========\nICS state for ICS 0x%x\n=========\n",
888 mutex_lock(&ics->lock);
890 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
891 struct ics_irq_state *irq = &ics->irq_state[i];
893 seq_printf(m, "irq 0x%06x: server %#x prio %#x save prio %#x asserted %d resend %d masked pending %d\n",
894 irq->number, irq->server, irq->priority,
895 irq->saved_priority, irq->asserted,
896 irq->resend, irq->masked_pending);
899 mutex_unlock(&ics->lock);
904 static int xics_debug_open(struct inode *inode, struct file *file)
906 return single_open(file, xics_debug_show, inode->i_private);
909 static const struct file_operations xics_debug_fops = {
910 .open = xics_debug_open,
913 .release = single_release,
916 static void xics_debugfs_init(struct kvmppc_xics *xics)
920 name = kasprintf(GFP_KERNEL, "kvm-xics-%p", xics);
922 pr_err("%s: no memory for name\n", __func__);
926 xics->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root,
927 xics, &xics_debug_fops);
929 pr_debug("%s: created %s\n", __func__, name);
933 static struct kvmppc_ics *kvmppc_xics_create_ics(struct kvm *kvm,
934 struct kvmppc_xics *xics, int irq)
936 struct kvmppc_ics *ics;
939 icsid = irq >> KVMPPC_XICS_ICS_SHIFT;
941 mutex_lock(&kvm->lock);
943 /* ICS already exists - somebody else got here first */
944 if (xics->ics[icsid])
948 ics = kzalloc(sizeof(struct kvmppc_ics), GFP_KERNEL);
952 mutex_init(&ics->lock);
955 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
956 ics->irq_state[i].number = (icsid << KVMPPC_XICS_ICS_SHIFT) | i;
957 ics->irq_state[i].priority = MASKED;
958 ics->irq_state[i].saved_priority = MASKED;
961 xics->ics[icsid] = ics;
963 if (icsid > xics->max_icsid)
964 xics->max_icsid = icsid;
967 mutex_unlock(&kvm->lock);
968 return xics->ics[icsid];
971 int kvmppc_xics_create_icp(struct kvm_vcpu *vcpu, unsigned long server_num)
973 struct kvmppc_icp *icp;
975 if (!vcpu->kvm->arch.xics)
978 if (kvmppc_xics_find_server(vcpu->kvm, server_num))
981 icp = kzalloc(sizeof(struct kvmppc_icp), GFP_KERNEL);
986 icp->server_num = server_num;
987 icp->state.mfrr = MASKED;
988 icp->state.pending_pri = MASKED;
989 vcpu->arch.icp = icp;
991 XICS_DBG("created server for vcpu %d\n", vcpu->vcpu_id);
996 u64 kvmppc_xics_get_icp(struct kvm_vcpu *vcpu)
998 struct kvmppc_icp *icp = vcpu->arch.icp;
999 union kvmppc_icp_state state;
1004 return ((u64)state.cppr << KVM_REG_PPC_ICP_CPPR_SHIFT) |
1005 ((u64)state.xisr << KVM_REG_PPC_ICP_XISR_SHIFT) |
1006 ((u64)state.mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT) |
1007 ((u64)state.pending_pri << KVM_REG_PPC_ICP_PPRI_SHIFT);
1010 int kvmppc_xics_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
1012 struct kvmppc_icp *icp = vcpu->arch.icp;
1013 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
1014 union kvmppc_icp_state old_state, new_state;
1015 struct kvmppc_ics *ics;
1016 u8 cppr, mfrr, pending_pri;
1024 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
1025 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
1026 KVM_REG_PPC_ICP_XISR_MASK;
1027 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
1028 pending_pri = icpval >> KVM_REG_PPC_ICP_PPRI_SHIFT;
1030 /* Require the new state to be internally consistent */
1032 if (pending_pri != 0xff)
1034 } else if (xisr == XICS_IPI) {
1035 if (pending_pri != mfrr || pending_pri >= cppr)
1038 if (pending_pri >= mfrr || pending_pri >= cppr)
1040 ics = kvmppc_xics_find_ics(xics, xisr, &src);
1046 new_state.cppr = cppr;
1047 new_state.xisr = xisr;
1048 new_state.mfrr = mfrr;
1049 new_state.pending_pri = pending_pri;
1052 * Deassert the CPU interrupt request.
1053 * icp_try_update will reassert it if necessary.
1055 kvmppc_book3s_dequeue_irqprio(icp->vcpu,
1056 BOOK3S_INTERRUPT_EXTERNAL_LEVEL);
1059 * Note that if we displace an interrupt from old_state.xisr,
1060 * we don't mark it as rejected. We expect userspace to set
1061 * the state of the interrupt sources to be consistent with
1062 * the ICP states (either before or afterwards, which doesn't
1063 * matter). We do handle resends due to CPPR becoming less
1064 * favoured because that is necessary to end up with a
1065 * consistent state in the situation where userspace restores
1066 * the ICS states before the ICP states.
1069 old_state = ACCESS_ONCE(icp->state);
1071 if (new_state.mfrr <= old_state.mfrr) {
1073 new_state.need_resend = old_state.need_resend;
1075 resend = old_state.need_resend;
1076 new_state.need_resend = 0;
1078 } while (!icp_try_update(icp, old_state, new_state, false));
1081 icp_check_resend(xics, icp);
1086 static int xics_get_source(struct kvmppc_xics *xics, long irq, u64 addr)
1089 struct kvmppc_ics *ics;
1090 struct ics_irq_state *irqp;
1091 u64 __user *ubufp = (u64 __user *) addr;
1095 ics = kvmppc_xics_find_ics(xics, irq, &idx);
1099 irqp = &ics->irq_state[idx];
1100 mutex_lock(&ics->lock);
1104 prio = irqp->priority;
1105 if (prio == MASKED) {
1106 val |= KVM_XICS_MASKED;
1107 prio = irqp->saved_priority;
1109 val |= prio << KVM_XICS_PRIORITY_SHIFT;
1111 val |= KVM_XICS_LEVEL_SENSITIVE | KVM_XICS_PENDING;
1112 else if (irqp->masked_pending || irqp->resend)
1113 val |= KVM_XICS_PENDING;
1116 mutex_unlock(&ics->lock);
1118 if (!ret && put_user(val, ubufp))
1124 static int xics_set_source(struct kvmppc_xics *xics, long irq, u64 addr)
1126 struct kvmppc_ics *ics;
1127 struct ics_irq_state *irqp;
1128 u64 __user *ubufp = (u64 __user *) addr;
1134 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
1137 ics = kvmppc_xics_find_ics(xics, irq, &idx);
1139 ics = kvmppc_xics_create_ics(xics->kvm, xics, irq);
1143 irqp = &ics->irq_state[idx];
1144 if (get_user(val, ubufp))
1147 server = val & KVM_XICS_DESTINATION_MASK;
1148 prio = val >> KVM_XICS_PRIORITY_SHIFT;
1149 if (prio != MASKED &&
1150 kvmppc_xics_find_server(xics->kvm, server) == NULL)
1153 mutex_lock(&ics->lock);
1154 irqp->server = server;
1155 irqp->saved_priority = prio;
1156 if (val & KVM_XICS_MASKED)
1158 irqp->priority = prio;
1160 irqp->masked_pending = 0;
1162 if ((val & KVM_XICS_PENDING) && (val & KVM_XICS_LEVEL_SENSITIVE))
1165 mutex_unlock(&ics->lock);
1167 if (val & KVM_XICS_PENDING)
1168 icp_deliver_irq(xics, NULL, irqp->number);
1173 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
1176 struct kvmppc_xics *xics = kvm->arch.xics;
1178 return ics_deliver_irq(xics, irq, level);
1181 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
1182 int irq_source_id, int level, bool line_status)
1186 return kvm_set_irq(kvm, irq_source_id, irq_entry->gsi,
1187 level, line_status);
1190 static int xics_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1192 struct kvmppc_xics *xics = dev->private;
1194 switch (attr->group) {
1195 case KVM_DEV_XICS_GRP_SOURCES:
1196 return xics_set_source(xics, attr->attr, attr->addr);
1201 static int xics_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1203 struct kvmppc_xics *xics = dev->private;
1205 switch (attr->group) {
1206 case KVM_DEV_XICS_GRP_SOURCES:
1207 return xics_get_source(xics, attr->attr, attr->addr);
1212 static int xics_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1214 switch (attr->group) {
1215 case KVM_DEV_XICS_GRP_SOURCES:
1216 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
1217 attr->attr < KVMPPC_XICS_NR_IRQS)
1224 static void kvmppc_xics_free(struct kvm_device *dev)
1226 struct kvmppc_xics *xics = dev->private;
1228 struct kvm *kvm = xics->kvm;
1230 debugfs_remove(xics->dentry);
1233 kvm->arch.xics = NULL;
1235 for (i = 0; i <= xics->max_icsid; i++)
1236 kfree(xics->ics[i]);
1241 static int kvmppc_xics_create(struct kvm_device *dev, u32 type)
1243 struct kvmppc_xics *xics;
1244 struct kvm *kvm = dev->kvm;
1247 xics = kzalloc(sizeof(*xics), GFP_KERNEL);
1251 dev->private = xics;
1255 /* Already there ? */
1256 mutex_lock(&kvm->lock);
1260 kvm->arch.xics = xics;
1261 mutex_unlock(&kvm->lock);
1268 xics_debugfs_init(xics);
1270 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1271 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
1272 /* Enable real mode support */
1273 xics->real_mode = ENABLE_REALMODE;
1274 xics->real_mode_dbg = DEBUG_REALMODE;
1276 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
1281 struct kvm_device_ops kvm_xics_ops = {
1283 .create = kvmppc_xics_create,
1284 .destroy = kvmppc_xics_free,
1285 .set_attr = xics_set_attr,
1286 .get_attr = xics_get_attr,
1287 .has_attr = xics_has_attr,
1290 int kvmppc_xics_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu,
1293 struct kvmppc_xics *xics = dev->private;
1296 if (dev->ops != &kvm_xics_ops)
1298 if (xics->kvm != vcpu->kvm)
1300 if (vcpu->arch.irq_type)
1303 r = kvmppc_xics_create_icp(vcpu, xcpu);
1305 vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
1310 void kvmppc_xics_free_icp(struct kvm_vcpu *vcpu)
1312 if (!vcpu->arch.icp)
1314 kfree(vcpu->arch.icp);
1315 vcpu->arch.icp = NULL;
1316 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
1319 static int xics_set_irq(struct kvm_kernel_irq_routing_entry *e,
1320 struct kvm *kvm, int irq_source_id, int level,
1323 return kvm_set_irq(kvm, irq_source_id, e->gsi, level, line_status);
1326 int kvm_irq_map_gsi(struct kvm *kvm,
1327 struct kvm_kernel_irq_routing_entry *entries, int gsi)
1330 entries->type = KVM_IRQ_ROUTING_IRQCHIP;
1331 entries->set = xics_set_irq;
1332 entries->irqchip.irqchip = 0;
1333 entries->irqchip.pin = gsi;
1337 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)