Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
1a89dd91 | 2 | /* |
50926d82 | 3 | * Copyright (C) 2015, 2016 ARM Ltd. |
1a89dd91 | 4 | */ |
50926d82 MZ |
5 | #ifndef __KVM_ARM_VGIC_H |
6 | #define __KVM_ARM_VGIC_H | |
b18b5778 | 7 | |
6c9eeb5f | 8 | #include <linux/bits.h> |
b47ef92a | 9 | #include <linux/kvm.h> |
b47ef92a | 10 | #include <linux/irqreturn.h> |
6c9eeb5f AS |
11 | #include <linux/kref.h> |
12 | #include <linux/mutex.h> | |
b47ef92a | 13 | #include <linux/spinlock.h> |
fb5ee369 | 14 | #include <linux/static_key.h> |
b47ef92a | 15 | #include <linux/types.h> |
6777f77f | 16 | #include <kvm/iodev.h> |
424c3383 | 17 | #include <linux/list.h> |
5a7a8426 | 18 | #include <linux/jump_label.h> |
1a89dd91 | 19 | |
74fe55dc MZ |
20 | #include <linux/irqchip/arm-gic-v4.h> |
21 | ||
e25028c8 | 22 | #define VGIC_V3_MAX_CPUS 512 |
50926d82 MZ |
23 | #define VGIC_V2_MAX_CPUS 8 |
24 | #define VGIC_NR_IRQS_LEGACY 256 | |
b47ef92a MZ |
25 | #define VGIC_NR_SGIS 16 |
26 | #define VGIC_NR_PPIS 16 | |
27 | #define VGIC_NR_PRIVATE_IRQS (VGIC_NR_SGIS + VGIC_NR_PPIS) | |
50926d82 MZ |
28 | #define VGIC_MAX_PRIVATE (VGIC_NR_PRIVATE_IRQS - 1) |
29 | #define VGIC_MAX_SPI 1019 | |
30 | #define VGIC_MAX_RESERVED 1023 | |
31 | #define VGIC_MIN_LPI 8192 | |
180ae7b1 | 32 | #define KVM_IRQCHIP_NUM_PINS (1020 - 32) |
8f186d52 | 33 | |
3cba4af3 | 34 | #define irq_is_ppi(irq) ((irq) >= VGIC_NR_SGIS && (irq) < VGIC_NR_PRIVATE_IRQS) |
ebb127f2 CD |
35 | #define irq_is_spi(irq) ((irq) >= VGIC_NR_PRIVATE_IRQS && \ |
36 | (irq) <= VGIC_MAX_SPI) | |
3cba4af3 | 37 | |
50926d82 MZ |
38 | enum vgic_type { |
39 | VGIC_V2, /* Good ol' GICv2 */ | |
40 | VGIC_V3, /* New fancy GICv3 */ | |
41 | }; | |
b47ef92a | 42 | |
50926d82 MZ |
43 | /* same for all guests, as depending only on the _host's_ GIC model */ |
44 | struct vgic_global { | |
45 | /* type of the host GIC */ | |
46 | enum vgic_type type; | |
b47ef92a | 47 | |
50926d82 MZ |
48 | /* Physical address of vgic virtual cpu interface */ |
49 | phys_addr_t vcpu_base; | |
b47ef92a | 50 | |
1bb32a44 | 51 | /* GICV mapping, kernel VA */ |
bf8feb39 | 52 | void __iomem *vcpu_base_va; |
1bb32a44 MZ |
53 | /* GICV mapping, HYP VA */ |
54 | void __iomem *vcpu_hyp_va; | |
bf8feb39 | 55 | |
1bb32a44 | 56 | /* virtual control interface mapping, kernel VA */ |
50926d82 | 57 | void __iomem *vctrl_base; |
1bb32a44 MZ |
58 | /* virtual control interface mapping, HYP VA */ |
59 | void __iomem *vctrl_hyp; | |
b47ef92a | 60 | |
50926d82 MZ |
61 | /* Number of implemented list registers */ |
62 | int nr_lr; | |
8d5c6b06 | 63 | |
50926d82 MZ |
64 | /* Maintenance IRQ number */ |
65 | unsigned int maint_irq; | |
1a9b1305 | 66 | |
50926d82 MZ |
67 | /* maximum number of VCPUs allowed (GICv2 limits us to 8) */ |
68 | int max_gic_vcpus; | |
8d5c6b06 | 69 | |
50926d82 MZ |
70 | /* Only needed for the legacy KVM_CREATE_IRQCHIP */ |
71 | bool can_emulate_gicv2; | |
5a7a8426 | 72 | |
e7c48059 MZ |
73 | /* Hardware has GICv4? */ |
74 | bool has_gicv4; | |
ae699ad3 | 75 | bool has_gicv4_1; |
e7c48059 | 76 | |
f6c3e24f MZ |
77 | /* Pseudo GICv3 from outer space */ |
78 | bool no_hw_deactivation; | |
79 | ||
5a7a8426 VM |
80 | /* GIC system register CPU interface */ |
81 | struct static_key_false gicv3_cpuif; | |
d017d7b0 VK |
82 | |
83 | u32 ich_vtr_el2; | |
8d5c6b06 MZ |
84 | }; |
85 | ||
50926d82 | 86 | extern struct vgic_global kvm_vgic_global_state; |
beee38b9 | 87 | |
50926d82 MZ |
88 | #define VGIC_V2_MAX_LRS (1 << 6) |
89 | #define VGIC_V3_MAX_LRS 16 | |
90 | #define VGIC_V3_LR_INDEX(lr) (VGIC_V3_MAX_LRS - 1 - lr) | |
8d5c6b06 | 91 | |
50926d82 MZ |
92 | enum vgic_irq_config { |
93 | VGIC_CONFIG_EDGE = 0, | |
94 | VGIC_CONFIG_LEVEL | |
ca85f623 MZ |
95 | }; |
96 | ||
db75f1a3 MZ |
97 | /* |
98 | * Per-irq ops overriding some common behavious. | |
99 | * | |
100 | * Always called in non-preemptible section and the functions can use | |
101 | * kvm_arm_get_running_vcpu() to get the vcpu pointer for private IRQs. | |
102 | */ | |
103 | struct irq_ops { | |
354920e7 MZ |
104 | /* Per interrupt flags for special-cased interrupts */ |
105 | unsigned long flags; | |
106 | ||
107 | #define VGIC_IRQ_SW_RESAMPLE BIT(0) /* Clear the active state for resampling */ | |
108 | ||
db75f1a3 MZ |
109 | /* |
110 | * Callback function pointer to in-kernel devices that can tell us the | |
111 | * state of the input level of mapped level-triggered IRQ faster than | |
112 | * peaking into the physical GIC. | |
113 | */ | |
114 | bool (*get_input_level)(int vintid); | |
115 | }; | |
116 | ||
50926d82 | 117 | struct vgic_irq { |
8fa3adb8 | 118 | raw_spinlock_t irq_lock; /* Protects the content of the struct */ |
3802411d | 119 | struct list_head lpi_list; /* Used to link all LPIs together */ |
50926d82 MZ |
120 | struct list_head ap_list; |
121 | ||
122 | struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU | |
123 | * SPIs and LPIs: The VCPU whose ap_list | |
124 | * this is queued on. | |
125 | */ | |
126 | ||
127 | struct kvm_vcpu *target_vcpu; /* The VCPU that this interrupt should | |
128 | * be sent to, as a result of the | |
129 | * targets reg (v2) or the | |
130 | * affinity reg (v3). | |
131 | */ | |
132 | ||
133 | u32 intid; /* Guest visible INTID */ | |
50926d82 | 134 | bool line_level; /* Level only */ |
8694e4da CD |
135 | bool pending_latch; /* The pending latch state used to calculate |
136 | * the pending state for both level | |
137 | * and edge triggered IRQs. */ | |
50926d82 MZ |
138 | bool active; /* not used for LPIs */ |
139 | bool enabled; | |
140 | bool hw; /* Tied to HW IRQ */ | |
5dd4b924 | 141 | struct kref refcount; /* Used for LPIs */ |
50926d82 | 142 | u32 hwintid; /* HW INTID number */ |
47bbd31f | 143 | unsigned int host_irq; /* linux irq corresponding to hwintid */ |
50926d82 MZ |
144 | union { |
145 | u8 targets; /* GICv2 target VCPUs mask */ | |
146 | u32 mpidr; /* GICv3 target VCPU */ | |
147 | }; | |
148 | u8 source; /* GICv2 SGIs only */ | |
53692908 | 149 | u8 active_source; /* GICv2 SGIs only */ |
50926d82 | 150 | u8 priority; |
8df3c8f3 | 151 | u8 group; /* 0 == group 0, 1 == group 1 */ |
50926d82 | 152 | enum vgic_irq_config config; /* Level or edge */ |
c6ccd30e | 153 | |
db75f1a3 | 154 | struct irq_ops *ops; |
b6909a65 | 155 | |
c6ccd30e CD |
156 | void *owner; /* Opaque pointer to reserve an interrupt |
157 | for in-kernel devices. */ | |
b26e5fda AP |
158 | }; |
159 | ||
354920e7 MZ |
160 | static inline bool vgic_irq_needs_resampling(struct vgic_irq *irq) |
161 | { | |
162 | return irq->ops && (irq->ops->flags & VGIC_IRQ_SW_RESAMPLE); | |
163 | } | |
164 | ||
50926d82 | 165 | struct vgic_register_region; |
59c5ab40 AP |
166 | struct vgic_its; |
167 | ||
168 | enum iodev_type { | |
169 | IODEV_CPUIF, | |
170 | IODEV_DIST, | |
171 | IODEV_REDIST, | |
172 | IODEV_ITS | |
173 | }; | |
50926d82 | 174 | |
6777f77f | 175 | struct vgic_io_device { |
50926d82 | 176 | gpa_t base_addr; |
59c5ab40 AP |
177 | union { |
178 | struct kvm_vcpu *redist_vcpu; | |
179 | struct vgic_its *its; | |
180 | }; | |
50926d82 | 181 | const struct vgic_register_region *regions; |
59c5ab40 | 182 | enum iodev_type iodev_type; |
50926d82 | 183 | int nr_regions; |
6777f77f AP |
184 | struct kvm_io_device dev; |
185 | }; | |
186 | ||
59c5ab40 AP |
187 | struct vgic_its { |
188 | /* The base address of the ITS control register frame */ | |
189 | gpa_t vgic_its_base; | |
190 | ||
191 | bool enabled; | |
192 | struct vgic_io_device iodev; | |
bb717644 | 193 | struct kvm_device *dev; |
424c3383 AP |
194 | |
195 | /* These registers correspond to GITS_BASER{0,1} */ | |
196 | u64 baser_device_table; | |
197 | u64 baser_coll_table; | |
198 | ||
199 | /* Protects the command queue */ | |
200 | struct mutex cmd_lock; | |
201 | u64 cbaser; | |
202 | u32 creadr; | |
203 | u32 cwriter; | |
204 | ||
71afe470 EA |
205 | /* migration ABI revision in use */ |
206 | u32 abi_rev; | |
207 | ||
424c3383 AP |
208 | /* Protects the device and collection lists */ |
209 | struct mutex its_lock; | |
210 | struct list_head device_list; | |
211 | struct list_head collection_list; | |
59c5ab40 AP |
212 | }; |
213 | ||
10f92c4c CD |
214 | struct vgic_state_iter; |
215 | ||
dbd9733a EA |
216 | struct vgic_redist_region { |
217 | u32 index; | |
218 | gpa_t base; | |
219 | u32 count; /* number of redistributors or 0 if single region */ | |
220 | u32 free_index; /* index of the next free redistributor */ | |
221 | struct list_head list; | |
222 | }; | |
223 | ||
1a89dd91 | 224 | struct vgic_dist { |
f982cf4e | 225 | bool in_kernel; |
01ac5e34 | 226 | bool ready; |
50926d82 | 227 | bool initialized; |
b47ef92a | 228 | |
59892136 AP |
229 | /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */ |
230 | u32 vgic_model; | |
231 | ||
aa075b0f CD |
232 | /* Implementation revision as reported in the GICD_IIDR */ |
233 | u32 implementation_rev; | |
234 | ||
32f8777e CD |
235 | /* Userspace can write to GICv2 IGROUPR */ |
236 | bool v2_groups_user_writable; | |
237 | ||
0e4e82f1 AP |
238 | /* Do injected MSIs require an additional device ID? */ |
239 | bool msis_require_devid; | |
240 | ||
50926d82 | 241 | int nr_spis; |
c1bfb577 | 242 | |
50926d82 MZ |
243 | /* base addresses in guest physical address space: */ |
244 | gpa_t vgic_dist_base; /* distributor */ | |
a0675c25 | 245 | union { |
50926d82 MZ |
246 | /* either a GICv2 CPU interface */ |
247 | gpa_t vgic_cpu_base; | |
248 | /* or a number of GICv3 redistributor regions */ | |
dbd9733a | 249 | struct list_head rd_regions; |
a0675c25 | 250 | }; |
b47ef92a | 251 | |
50926d82 MZ |
252 | /* distributor enabled */ |
253 | bool enabled; | |
47a98b15 | 254 | |
bacf2c60 MZ |
255 | /* Wants SGIs without active state */ |
256 | bool nassgireq; | |
257 | ||
50926d82 | 258 | struct vgic_irq *spis; |
b47ef92a | 259 | |
a9cf86f6 | 260 | struct vgic_io_device dist_iodev; |
0aa1de57 | 261 | |
1085fdc6 AP |
262 | bool has_its; |
263 | ||
0aa1de57 AP |
264 | /* |
265 | * Contains the attributes and gpa of the LPI configuration table. | |
266 | * Since we report GICR_TYPER.CommonLPIAff as 0b00, we can share | |
267 | * one address across all redistributors. | |
bad36e4e | 268 | * GICv3 spec: IHI 0069E 6.1.1 "LPI Configuration tables" |
0aa1de57 AP |
269 | */ |
270 | u64 propbaser; | |
3802411d AP |
271 | |
272 | /* Protects the lpi_list and the count value below. */ | |
fc3bc475 | 273 | raw_spinlock_t lpi_list_lock; |
3802411d AP |
274 | struct list_head lpi_list_head; |
275 | int lpi_list_count; | |
10f92c4c | 276 | |
24cab82c MZ |
277 | /* LPI translation cache */ |
278 | struct list_head lpi_translation_cache; | |
279 | ||
10f92c4c CD |
280 | /* used by vgic-debug */ |
281 | struct vgic_state_iter *iter; | |
74fe55dc MZ |
282 | |
283 | /* | |
284 | * GICv4 ITS per-VM data, containing the IRQ domain, the VPE | |
285 | * array, the property table pointer as well as allocation | |
286 | * data. This essentially ties the Linux IRQ core and ITS | |
287 | * together, and avoids leaking KVM's data structures anywhere | |
288 | * else. | |
289 | */ | |
290 | struct its_vm its_vm; | |
1a89dd91 MZ |
291 | }; |
292 | ||
eede821d MZ |
293 | struct vgic_v2_cpu_if { |
294 | u32 vgic_hcr; | |
295 | u32 vgic_vmcr; | |
eede821d | 296 | u32 vgic_apr; |
8f186d52 | 297 | u32 vgic_lr[VGIC_V2_MAX_LRS]; |
fc5d1f1a CD |
298 | |
299 | unsigned int used_lrs; | |
eede821d MZ |
300 | }; |
301 | ||
b2fb1c0d | 302 | struct vgic_v3_cpu_if { |
b2fb1c0d MZ |
303 | u32 vgic_hcr; |
304 | u32 vgic_vmcr; | |
2f5fa41a | 305 | u32 vgic_sre; /* Restored only, change ignored */ |
b2fb1c0d MZ |
306 | u32 vgic_ap0r[4]; |
307 | u32 vgic_ap1r[4]; | |
308 | u64 vgic_lr[VGIC_V3_MAX_LRS]; | |
74fe55dc MZ |
309 | |
310 | /* | |
311 | * GICv4 ITS per-VPE data, containing the doorbell IRQ, the | |
312 | * pending table pointer, the its_vm pointer and a few other | |
313 | * HW specific things. As for the its_vm structure, this is | |
314 | * linking the Linux IRQ subsystem and the ITS together. | |
315 | */ | |
316 | struct its_vpe its_vpe; | |
fc5d1f1a CD |
317 | |
318 | unsigned int used_lrs; | |
b2fb1c0d MZ |
319 | }; |
320 | ||
1a89dd91 | 321 | struct vgic_cpu { |
9d949dce | 322 | /* CPU vif control registers for world switch */ |
eede821d MZ |
323 | union { |
324 | struct vgic_v2_cpu_if vgic_v2; | |
b2fb1c0d | 325 | struct vgic_v3_cpu_if vgic_v3; |
eede821d | 326 | }; |
6c3d63c9 | 327 | |
50926d82 | 328 | struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS]; |
1a89dd91 | 329 | |
e08d8d29 | 330 | raw_spinlock_t ap_list_lock; /* Protects the ap_list */ |
9d949dce | 331 | |
50926d82 MZ |
332 | /* |
333 | * List of IRQs that this VCPU should consider because they are either | |
334 | * Active or Pending (hence the name; AP list), or because they recently | |
335 | * were one of the two and need to be migrated off this list to another | |
336 | * VCPU. | |
337 | */ | |
338 | struct list_head ap_list_head; | |
495dd859 | 339 | |
8f6cdc1c AP |
340 | /* |
341 | * Members below are used with GICv3 emulation only and represent | |
342 | * parts of the redistributor. | |
343 | */ | |
344 | struct vgic_io_device rd_iodev; | |
dbd9733a | 345 | struct vgic_redist_region *rdreg; |
28e9d4bc | 346 | u32 rdreg_index; |
0aa1de57 AP |
347 | |
348 | /* Contains the attributes and gpa of the LPI pending tables. */ | |
349 | u64 pendbaser; | |
350 | ||
351 | bool lpis_enabled; | |
d017d7b0 VK |
352 | |
353 | /* Cache guest priority bits */ | |
354 | u32 num_pri_bits; | |
355 | ||
356 | /* Cache guest interrupt ID bits */ | |
357 | u32 num_id_bits; | |
50926d82 | 358 | }; |
1a89dd91 | 359 | |
fb5ee369 | 360 | extern struct static_key_false vgic_v2_cpuif_trap; |
59da1cbf | 361 | extern struct static_key_false vgic_v3_cpuif_trap; |
fb5ee369 | 362 | |
ce01e4e8 | 363 | int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write); |
6c3d63c9 | 364 | void kvm_vgic_early_init(struct kvm *kvm); |
1aab6f46 | 365 | int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu); |
59892136 | 366 | int kvm_vgic_create(struct kvm *kvm, u32 type); |
c1bfb577 | 367 | void kvm_vgic_destroy(struct kvm *kvm); |
c1bfb577 | 368 | void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu); |
50926d82 MZ |
369 | int kvm_vgic_map_resources(struct kvm *kvm); |
370 | int kvm_vgic_hyp_init(void); | |
5b0d2cc2 | 371 | void kvm_vgic_init_cpu_hardware(void); |
50926d82 MZ |
372 | |
373 | int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid, | |
cb3f0ad8 | 374 | bool level, void *owner); |
47bbd31f | 375 | int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq, |
db75f1a3 | 376 | u32 vintid, struct irq_ops *ops); |
47bbd31f EA |
377 | int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid); |
378 | bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid); | |
1a89dd91 | 379 | |
50926d82 MZ |
380 | int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu); |
381 | ||
328e5664 CD |
382 | void kvm_vgic_load(struct kvm_vcpu *vcpu); |
383 | void kvm_vgic_put(struct kvm_vcpu *vcpu); | |
5eeaf10e | 384 | void kvm_vgic_vmcr_sync(struct kvm_vcpu *vcpu); |
328e5664 | 385 | |
f982cf4e | 386 | #define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel)) |
50926d82 | 387 | #define vgic_initialized(k) ((k)->arch.vgic.initialized) |
c52edf5f | 388 | #define vgic_ready(k) ((k)->arch.vgic.ready) |
2defaff4 | 389 | #define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \ |
50926d82 MZ |
390 | ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) |
391 | ||
392 | bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu); | |
393 | void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu); | |
394 | void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu); | |
413aa807 | 395 | void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid); |
9d949dce | 396 | |
6249f2a4 | 397 | void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1); |
8f186d52 | 398 | |
50926d82 MZ |
399 | /** |
400 | * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW | |
401 | * | |
402 | * The host's GIC naturally limits the maximum amount of VCPUs a guest | |
403 | * can use. | |
404 | */ | |
405 | static inline int kvm_vgic_get_max_vcpus(void) | |
406 | { | |
407 | return kvm_vgic_global_state.max_gic_vcpus; | |
408 | } | |
409 | ||
180ae7b1 EA |
410 | /** |
411 | * kvm_vgic_setup_default_irq_routing: | |
412 | * Setup a default flat gsi routing table mapping all SPIs | |
413 | */ | |
414 | int kvm_vgic_setup_default_irq_routing(struct kvm *kvm); | |
415 | ||
c6ccd30e CD |
416 | int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner); |
417 | ||
196b1364 MZ |
418 | struct kvm_kernel_irq_routing_entry; |
419 | ||
420 | int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq, | |
421 | struct kvm_kernel_irq_routing_entry *irq_entry); | |
422 | ||
423 | int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int irq, | |
424 | struct kvm_kernel_irq_routing_entry *irq_entry); | |
425 | ||
8e01d9a3 | 426 | int vgic_v4_load(struct kvm_vcpu *vcpu); |
57e3cebd | 427 | void vgic_v4_commit(struct kvm_vcpu *vcpu); |
8e01d9a3 | 428 | int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db); |
df9ba959 | 429 | |
50926d82 | 430 | #endif /* __KVM_ARM_VGIC_H */ |