2 * Copyright (C) 2015, 2016 ARM Ltd.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 #include <linux/irqchip/arm-gic.h>
18 #include <linux/kvm.h>
19 #include <linux/kvm_host.h>
20 #include <kvm/arm_vgic.h>
21 #include <asm/kvm_mmu.h>
26 * Call this function to convert a u64 value to an unsigned long * bitmask
27 * in a way that works on both 32-bit and 64-bit LE and BE platforms.
29 * Warning: Calling this function may modify *val.
31 static unsigned long *u64_to_bitmask(u64 *val)
33 #if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
34 *val = (*val >> 32) | (*val << 32);
36 return (unsigned long *)val;
39 static inline void vgic_v2_write_lr(int lr, u32 val)
41 void __iomem *base = kvm_vgic_global_state.vctrl_base;
43 writel_relaxed(val, base + GICH_LR0 + (lr * 4));
46 void vgic_v2_init_lrs(void)
50 for (i = 0; i < kvm_vgic_global_state.nr_lr; i++)
51 vgic_v2_write_lr(i, 0);
54 void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu)
56 struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
58 if (cpuif->vgic_misr & GICH_MISR_EOI) {
59 u64 eisr = cpuif->vgic_eisr;
60 unsigned long *eisr_bmap = u64_to_bitmask(&eisr);
63 for_each_set_bit(lr, eisr_bmap, kvm_vgic_global_state.nr_lr) {
64 u32 intid = cpuif->vgic_lr[lr] & GICH_LR_VIRTUALID;
66 WARN_ON(cpuif->vgic_lr[lr] & GICH_LR_STATE);
68 /* Only SPIs require notification */
69 if (vgic_valid_spi(vcpu->kvm, intid))
70 kvm_notify_acked_irq(vcpu->kvm, 0,
71 intid - VGIC_NR_PRIVATE_IRQS);
75 /* check and disable underflow maintenance IRQ */
76 cpuif->vgic_hcr &= ~GICH_HCR_UIE;
79 * In the next iterations of the vcpu loop, if we sync the
80 * vgic state after flushing it, but before entering the guest
81 * (this happens for pending signals and vmid rollovers), then
82 * make sure we don't pick up any old maintenance interrupts
88 void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)
90 struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
92 cpuif->vgic_hcr |= GICH_HCR_UIE;
96 * transfer the content of the LRs back into the corresponding ap_list:
97 * - active bit is transferred as is
99 * - transferred as is in case of edge sensitive IRQs
100 * - set to the line-level (resample time) for level sensitive IRQs
102 void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
104 struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
107 for (lr = 0; lr < vcpu->arch.vgic_cpu.used_lrs; lr++) {
108 u32 val = cpuif->vgic_lr[lr];
109 u32 intid = val & GICH_LR_VIRTUALID;
110 struct vgic_irq *irq;
112 irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
114 spin_lock(&irq->irq_lock);
116 /* Always preserve the active bit */
117 irq->active = !!(val & GICH_LR_ACTIVE_BIT);
119 /* Edge is the only case where we preserve the pending bit */
120 if (irq->config == VGIC_CONFIG_EDGE &&
121 (val & GICH_LR_PENDING_BIT)) {
122 irq->pending_latch = true;
124 if (vgic_irq_is_sgi(intid)) {
125 u32 cpuid = val & GICH_LR_PHYSID_CPUID;
127 cpuid >>= GICH_LR_PHYSID_CPUID_SHIFT;
128 irq->source |= (1 << cpuid);
133 * Clear soft pending state when level irqs have been acked.
134 * Always regenerate the pending state.
136 if (irq->config == VGIC_CONFIG_LEVEL) {
137 if (!(val & GICH_LR_PENDING_BIT))
138 irq->pending_latch = false;
141 spin_unlock(&irq->irq_lock);
142 vgic_put_irq(vcpu->kvm, irq);
147 * Populates the particular LR with the state of a given IRQ:
148 * - for an edge sensitive IRQ the pending state is cleared in struct vgic_irq
149 * - for a level sensitive IRQ the pending state value is unchanged;
150 * it is dictated directly by the input level
152 * If @irq describes an SGI with multiple sources, we choose the
153 * lowest-numbered source VCPU and clear that bit in the source bitmap.
155 * The irq_lock must be held by the caller.
157 void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr)
159 u32 val = irq->intid;
161 if (irq_is_pending(irq)) {
162 val |= GICH_LR_PENDING_BIT;
164 if (irq->config == VGIC_CONFIG_EDGE)
165 irq->pending_latch = false;
167 if (vgic_irq_is_sgi(irq->intid)) {
168 u32 src = ffs(irq->source);
171 val |= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;
172 irq->source &= ~(1 << (src - 1));
174 irq->pending_latch = true;
179 val |= GICH_LR_ACTIVE_BIT;
183 val |= irq->hwintid << GICH_LR_PHYSID_CPUID_SHIFT;
185 if (irq->config == VGIC_CONFIG_LEVEL)
189 /* The GICv2 LR only holds five bits of priority. */
190 val |= (irq->priority >> 3) << GICH_LR_PRIORITY_SHIFT;
192 vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = val;
195 void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr)
197 vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = 0;
200 void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
204 vmcr = (vmcrp->ctlr << GICH_VMCR_CTRL_SHIFT) & GICH_VMCR_CTRL_MASK;
205 vmcr |= (vmcrp->abpr << GICH_VMCR_ALIAS_BINPOINT_SHIFT) &
206 GICH_VMCR_ALIAS_BINPOINT_MASK;
207 vmcr |= (vmcrp->bpr << GICH_VMCR_BINPOINT_SHIFT) &
208 GICH_VMCR_BINPOINT_MASK;
209 vmcr |= ((vmcrp->pmr >> GICV_PMR_PRIORITY_SHIFT) <<
210 GICH_VMCR_PRIMASK_SHIFT) & GICH_VMCR_PRIMASK_MASK;
212 vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = vmcr;
215 void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
217 u32 vmcr = vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr;
219 vmcrp->ctlr = (vmcr & GICH_VMCR_CTRL_MASK) >>
220 GICH_VMCR_CTRL_SHIFT;
221 vmcrp->abpr = (vmcr & GICH_VMCR_ALIAS_BINPOINT_MASK) >>
222 GICH_VMCR_ALIAS_BINPOINT_SHIFT;
223 vmcrp->bpr = (vmcr & GICH_VMCR_BINPOINT_MASK) >>
224 GICH_VMCR_BINPOINT_SHIFT;
225 vmcrp->pmr = ((vmcr & GICH_VMCR_PRIMASK_MASK) >>
226 GICH_VMCR_PRIMASK_SHIFT) << GICV_PMR_PRIORITY_SHIFT;
229 void vgic_v2_enable(struct kvm_vcpu *vcpu)
232 * By forcing VMCR to zero, the GIC will restore the binary
233 * points to their reset values. Anything else resets to zero
236 vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = 0;
237 vcpu->arch.vgic_cpu.vgic_v2.vgic_elrsr = ~0;
239 /* Get the show on the road... */
240 vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr = GICH_HCR_EN;
243 /* check for overlapping regions and for regions crossing the end of memory */
244 static bool vgic_v2_check_base(gpa_t dist_base, gpa_t cpu_base)
246 if (dist_base + KVM_VGIC_V2_DIST_SIZE < dist_base)
248 if (cpu_base + KVM_VGIC_V2_CPU_SIZE < cpu_base)
251 if (dist_base + KVM_VGIC_V2_DIST_SIZE <= cpu_base)
253 if (cpu_base + KVM_VGIC_V2_CPU_SIZE <= dist_base)
259 int vgic_v2_map_resources(struct kvm *kvm)
261 struct vgic_dist *dist = &kvm->arch.vgic;
267 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
268 IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) {
269 kvm_err("Need to set vgic cpu and dist addresses first\n");
274 if (!vgic_v2_check_base(dist->vgic_dist_base, dist->vgic_cpu_base)) {
275 kvm_err("VGIC CPU and dist frames overlap\n");
281 * Initialize the vgic if this hasn't already been done on demand by
282 * accessing the vgic state from userspace.
284 ret = vgic_init(kvm);
286 kvm_err("Unable to initialize VGIC dynamic data structures\n");
290 ret = vgic_register_dist_iodev(kvm, dist->vgic_dist_base, VGIC_V2);
292 kvm_err("Unable to register VGIC MMIO regions\n");
296 if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) {
297 ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
298 kvm_vgic_global_state.vcpu_base,
299 KVM_VGIC_V2_CPU_SIZE, true);
301 kvm_err("Unable to remap VGIC CPU to VCPU\n");
312 DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap);
315 * vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT
316 * @node: pointer to the DT node
318 * Returns 0 if a GICv2 has been found, returns an error code otherwise
320 int vgic_v2_probe(const struct gic_kvm_info *info)
325 if (!info->vctrl.start) {
326 kvm_err("GICH not present in the firmware table\n");
330 if (!PAGE_ALIGNED(info->vcpu.start) ||
331 !PAGE_ALIGNED(resource_size(&info->vcpu))) {
332 kvm_info("GICV region size/alignment is unsafe, using trapping (reduced performance)\n");
333 kvm_vgic_global_state.vcpu_base_va = ioremap(info->vcpu.start,
334 resource_size(&info->vcpu));
335 if (!kvm_vgic_global_state.vcpu_base_va) {
336 kvm_err("Cannot ioremap GICV\n");
340 ret = create_hyp_io_mappings(kvm_vgic_global_state.vcpu_base_va,
341 kvm_vgic_global_state.vcpu_base_va + resource_size(&info->vcpu),
344 kvm_err("Cannot map GICV into hyp\n");
348 static_branch_enable(&vgic_v2_cpuif_trap);
351 kvm_vgic_global_state.vctrl_base = ioremap(info->vctrl.start,
352 resource_size(&info->vctrl));
353 if (!kvm_vgic_global_state.vctrl_base) {
354 kvm_err("Cannot ioremap GICH\n");
359 vtr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VTR);
360 kvm_vgic_global_state.nr_lr = (vtr & 0x3f) + 1;
362 ret = create_hyp_io_mappings(kvm_vgic_global_state.vctrl_base,
363 kvm_vgic_global_state.vctrl_base +
364 resource_size(&info->vctrl),
367 kvm_err("Cannot map VCTRL into hyp\n");
371 ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);
373 kvm_err("Cannot register GICv2 KVM device\n");
377 kvm_vgic_global_state.can_emulate_gicv2 = true;
378 kvm_vgic_global_state.vcpu_base = info->vcpu.start;
379 kvm_vgic_global_state.type = VGIC_V2;
380 kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS;
382 kvm_info("vgic-v2@%llx\n", info->vctrl.start);
386 if (kvm_vgic_global_state.vctrl_base)
387 iounmap(kvm_vgic_global_state.vctrl_base);
388 if (kvm_vgic_global_state.vcpu_base_va)
389 iounmap(kvm_vgic_global_state.vcpu_base_va);