Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7de5c0af MZ |
2 | /* |
3 | * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved. | |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
7de5c0af MZ |
5 | */ |
6 | ||
7 | #include <linux/interrupt.h> | |
8 | #include <linux/irq.h> | |
9 | #include <linux/irqdomain.h> | |
10 | #include <linux/msi.h> | |
11 | #include <linux/sched.h> | |
12 | ||
13 | #include <linux/irqchip/arm-gic-v4.h> | |
14 | ||
7954907b MZ |
15 | /* |
16 | * WARNING: The blurb below assumes that you understand the | |
17 | * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets | |
18 | * translated into GICv4 commands. So it effectively targets at most | |
19 | * two individuals. You know who you are. | |
20 | * | |
21 | * The core GICv4 code is designed to *avoid* exposing too much of the | |
22 | * core GIC code (that would in turn leak into the hypervisor code), | |
23 | * and instead provide a hypervisor agnostic interface to the HW (of | |
24 | * course, the astute reader will quickly realize that hypervisor | |
25 | * agnostic actually means KVM-specific - what were you thinking?). | |
26 | * | |
27 | * In order to achieve a modicum of isolation, we try to hide most of | |
28 | * the GICv4 "stuff" behind normal irqchip operations: | |
29 | * | |
30 | * - Any guest-visible VLPI is backed by a Linux interrupt (and a | |
31 | * physical LPI which gets unmapped when the guest maps the | |
32 | * VLPI). This allows the same DevID/EventID pair to be either | |
33 | * mapped to the LPI (host) or the VLPI (guest). Note that this is | |
34 | * exclusive, and you cannot have both. | |
35 | * | |
36 | * - Enabling/disabling a VLPI is done by issuing mask/unmask calls. | |
37 | * | |
38 | * - Guest INT/CLEAR commands are implemented through | |
39 | * irq_set_irqchip_state(). | |
40 | * | |
41 | * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or | |
42 | * issuing an INV after changing a priority) gets shoved into the | |
43 | * irq_set_vcpu_affinity() method. While this is quite horrible | |
44 | * (let's face it, this is the irqchip version of an ioctl), it | |
45 | * confines the crap to a single location. And map/unmap really is | |
46 | * about setting the affinity of a VLPI to a vcpu, so only INV is | |
47 | * majorly out of place. So there. | |
48 | * | |
49 | * A number of commands are simply not provided by this interface, as | |
50 | * they do not make direct sense. For example, MAPD is purely local to | |
51 | * the virtual ITS (because it references a virtual device, and the | |
52 | * physical ITS is still very much in charge of the physical | |
53 | * device). Same goes for things like MAPC (the physical ITS deals | |
54 | * with the actual vPE affinity, and not the braindead concept of | |
55 | * collection). SYNC is not provided either, as each and every command | |
56 | * is followed by a VSYNC. This could be relaxed in the future, should | |
57 | * this be seen as a bottleneck (yes, this means *never*). | |
58 | * | |
59 | * But handling VLPIs is only one side of the job of the GICv4 | |
60 | * code. The other (darker) side is to take care of the doorbell | |
61 | * interrupts which are delivered when a VLPI targeting a non-running | |
62 | * vcpu is being made pending. | |
63 | * | |
64 | * The choice made here is that each vcpu (VPE in old northern GICv4 | |
65 | * dialect) gets a single doorbell LPI, no matter how many interrupts | |
66 | * are targeting it. This has a nice property, which is that the | |
67 | * interrupt becomes a handle for the VPE, and that the hypervisor | |
68 | * code can manipulate it through the normal interrupt API: | |
69 | * | |
70 | * - VMs (or rather the VM abstraction that matters to the GIC) | |
71 | * contain an irq domain where each interrupt maps to a VPE. In | |
72 | * turn, this domain sits on top of the normal LPI allocator, and a | |
73 | * specially crafted irq_chip implementation. | |
74 | * | |
75 | * - mask/unmask do what is expected on the doorbell interrupt. | |
76 | * | |
77 | * - irq_set_affinity is used to move a VPE from one redistributor to | |
78 | * another. | |
79 | * | |
80 | * - irq_set_vcpu_affinity once again gets hijacked for the purpose of | |
81 | * creating a new sub-API, namely scheduling/descheduling a VPE | |
82 | * (which involves programming GICR_V{PROP,PEND}BASER) and | |
83 | * performing INVALL operations. | |
84 | */ | |
85 | ||
7de5c0af MZ |
86 | static struct irq_domain *gic_domain; |
87 | static const struct irq_domain_ops *vpe_domain_ops; | |
166cba71 | 88 | static const struct irq_domain_ops *sgi_domain_ops; |
7de5c0af | 89 | |
46135d6f LP |
90 | #ifdef CONFIG_ARM64 |
91 | #include <asm/cpufeature.h> | |
92 | ||
93 | bool gic_cpuif_has_vsgi(void) | |
94 | { | |
95 | unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); | |
96 | ||
97 | fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT); | |
98 | ||
99 | return fld >= 0x3; | |
100 | } | |
101 | #else | |
102 | bool gic_cpuif_has_vsgi(void) | |
103 | { | |
104 | return false; | |
105 | } | |
106 | #endif | |
107 | ||
ae699ad3 MZ |
108 | static bool has_v4_1(void) |
109 | { | |
110 | return !!sgi_domain_ops; | |
111 | } | |
112 | ||
46135d6f LP |
113 | static bool has_v4_1_sgi(void) |
114 | { | |
115 | return has_v4_1() && gic_cpuif_has_vsgi(); | |
116 | } | |
117 | ||
6d31b6ff MZ |
118 | static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) |
119 | { | |
120 | char *name; | |
121 | int sgi_base; | |
122 | ||
46135d6f | 123 | if (!has_v4_1_sgi()) |
6d31b6ff MZ |
124 | return 0; |
125 | ||
126 | name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current)); | |
127 | if (!name) | |
128 | goto err; | |
129 | ||
130 | vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx); | |
131 | if (!vpe->fwnode) | |
132 | goto err; | |
133 | ||
134 | kfree(name); | |
135 | name = NULL; | |
136 | ||
137 | vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16, | |
138 | sgi_domain_ops, vpe); | |
139 | if (!vpe->sgi_domain) | |
140 | goto err; | |
141 | ||
142 | sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16, | |
143 | NUMA_NO_NODE, vpe, | |
144 | false, NULL); | |
145 | if (sgi_base <= 0) | |
146 | goto err; | |
147 | ||
148 | return 0; | |
149 | ||
150 | err: | |
151 | if (vpe->sgi_domain) | |
152 | irq_domain_remove(vpe->sgi_domain); | |
153 | if (vpe->fwnode) | |
154 | irq_domain_free_fwnode(vpe->fwnode); | |
155 | kfree(name); | |
156 | return -ENOMEM; | |
157 | } | |
158 | ||
7de5c0af MZ |
159 | int its_alloc_vcpu_irqs(struct its_vm *vm) |
160 | { | |
161 | int vpe_base_irq, i; | |
162 | ||
163 | vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe", | |
164 | task_pid_nr(current)); | |
165 | if (!vm->fwnode) | |
166 | goto err; | |
167 | ||
168 | vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes, | |
169 | vm->fwnode, vpe_domain_ops, | |
170 | vm); | |
171 | if (!vm->domain) | |
172 | goto err; | |
173 | ||
174 | for (i = 0; i < vm->nr_vpes; i++) { | |
175 | vm->vpes[i]->its_vm = vm; | |
176 | vm->vpes[i]->idai = true; | |
177 | } | |
178 | ||
179 | vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes, | |
180 | NUMA_NO_NODE, vm, | |
181 | false, NULL); | |
182 | if (vpe_base_irq <= 0) | |
183 | goto err; | |
184 | ||
6d31b6ff MZ |
185 | for (i = 0; i < vm->nr_vpes; i++) { |
186 | int ret; | |
7de5c0af | 187 | vm->vpes[i]->irq = vpe_base_irq + i; |
6d31b6ff MZ |
188 | ret = its_alloc_vcpu_sgis(vm->vpes[i], i); |
189 | if (ret) | |
190 | goto err; | |
191 | } | |
7de5c0af MZ |
192 | |
193 | return 0; | |
194 | ||
195 | err: | |
196 | if (vm->domain) | |
197 | irq_domain_remove(vm->domain); | |
198 | if (vm->fwnode) | |
199 | irq_domain_free_fwnode(vm->fwnode); | |
200 | ||
201 | return -ENOMEM; | |
202 | } | |
203 | ||
6d31b6ff MZ |
204 | static void its_free_sgi_irqs(struct its_vm *vm) |
205 | { | |
206 | int i; | |
207 | ||
46135d6f | 208 | if (!has_v4_1_sgi()) |
6d31b6ff MZ |
209 | return; |
210 | ||
211 | for (i = 0; i < vm->nr_vpes; i++) { | |
212 | unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0); | |
213 | ||
214 | if (WARN_ON(!irq)) | |
215 | continue; | |
216 | ||
217 | irq_domain_free_irqs(irq, 16); | |
218 | irq_domain_remove(vm->vpes[i]->sgi_domain); | |
219 | irq_domain_free_fwnode(vm->vpes[i]->fwnode); | |
220 | } | |
221 | } | |
222 | ||
7de5c0af MZ |
223 | void its_free_vcpu_irqs(struct its_vm *vm) |
224 | { | |
6d31b6ff | 225 | its_free_sgi_irqs(vm); |
7de5c0af MZ |
226 | irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes); |
227 | irq_domain_remove(vm->domain); | |
228 | irq_domain_free_fwnode(vm->fwnode); | |
229 | } | |
eab84318 MZ |
230 | |
231 | static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info) | |
232 | { | |
233 | return irq_set_vcpu_affinity(vpe->irq, info); | |
234 | } | |
235 | ||
ae699ad3 | 236 | int its_make_vpe_non_resident(struct its_vpe *vpe, bool db) |
eab84318 | 237 | { |
ae699ad3 MZ |
238 | struct irq_desc *desc = irq_to_desc(vpe->irq); |
239 | struct its_cmd_info info = { }; | |
8e01d9a3 | 240 | int ret; |
eab84318 MZ |
241 | |
242 | WARN_ON(preemptible()); | |
243 | ||
ae699ad3 MZ |
244 | info.cmd_type = DESCHEDULE_VPE; |
245 | if (has_v4_1()) { | |
246 | /* GICv4.1 can directly deal with doorbells */ | |
247 | info.req_db = db; | |
248 | } else { | |
249 | /* Undo the nested disable_irq() calls... */ | |
250 | while (db && irqd_irq_disabled(&desc->irq_data)) | |
251 | enable_irq(vpe->irq); | |
252 | } | |
253 | ||
254 | ret = its_send_vpe_cmd(vpe, &info); | |
255 | if (!ret) | |
256 | vpe->resident = false; | |
257 | ||
57e3cebd SL |
258 | vpe->ready = false; |
259 | ||
ae699ad3 MZ |
260 | return ret; |
261 | } | |
262 | ||
263 | int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en) | |
264 | { | |
265 | struct its_cmd_info info = { }; | |
266 | int ret; | |
267 | ||
268 | WARN_ON(preemptible()); | |
269 | ||
270 | info.cmd_type = SCHEDULE_VPE; | |
271 | if (has_v4_1()) { | |
272 | info.g0en = g0en; | |
273 | info.g1en = g1en; | |
274 | } else { | |
275 | /* Disabled the doorbell, as we're about to enter the guest */ | |
276 | disable_irq_nosync(vpe->irq); | |
277 | } | |
eab84318 | 278 | |
8e01d9a3 MZ |
279 | ret = its_send_vpe_cmd(vpe, &info); |
280 | if (!ret) | |
ae699ad3 | 281 | vpe->resident = true; |
8e01d9a3 MZ |
282 | |
283 | return ret; | |
eab84318 MZ |
284 | } |
285 | ||
57e3cebd SL |
286 | int its_commit_vpe(struct its_vpe *vpe) |
287 | { | |
288 | struct its_cmd_info info = { | |
289 | .cmd_type = COMMIT_VPE, | |
290 | }; | |
291 | int ret; | |
292 | ||
293 | WARN_ON(preemptible()); | |
294 | ||
295 | ret = its_send_vpe_cmd(vpe, &info); | |
296 | if (!ret) | |
297 | vpe->ready = true; | |
298 | ||
299 | return ret; | |
300 | } | |
301 | ||
302 | ||
eab84318 MZ |
303 | int its_invall_vpe(struct its_vpe *vpe) |
304 | { | |
305 | struct its_cmd_info info = { | |
306 | .cmd_type = INVALL_VPE, | |
307 | }; | |
308 | ||
309 | return its_send_vpe_cmd(vpe, &info); | |
310 | } | |
f2eac75d MZ |
311 | |
312 | int its_map_vlpi(int irq, struct its_vlpi_map *map) | |
313 | { | |
314 | struct its_cmd_info info = { | |
315 | .cmd_type = MAP_VLPI, | |
6c09ffd0 AB |
316 | { |
317 | .map = map, | |
318 | }, | |
f2eac75d | 319 | }; |
90dc7122 | 320 | int ret; |
f2eac75d MZ |
321 | |
322 | /* | |
323 | * The host will never see that interrupt firing again, so it | |
324 | * is vital that we don't do any lazy masking. | |
325 | */ | |
326 | irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY); | |
327 | ||
90dc7122 MZ |
328 | ret = irq_set_vcpu_affinity(irq, &info); |
329 | if (ret) | |
330 | irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); | |
331 | ||
332 | return ret; | |
f2eac75d MZ |
333 | } |
334 | ||
335 | int its_get_vlpi(int irq, struct its_vlpi_map *map) | |
336 | { | |
337 | struct its_cmd_info info = { | |
338 | .cmd_type = GET_VLPI, | |
6c09ffd0 AB |
339 | { |
340 | .map = map, | |
341 | }, | |
f2eac75d MZ |
342 | }; |
343 | ||
344 | return irq_set_vcpu_affinity(irq, &info); | |
345 | } | |
346 | ||
347 | int its_unmap_vlpi(int irq) | |
348 | { | |
349 | irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); | |
350 | return irq_set_vcpu_affinity(irq, NULL); | |
351 | } | |
352 | ||
353 | int its_prop_update_vlpi(int irq, u8 config, bool inv) | |
354 | { | |
355 | struct its_cmd_info info = { | |
356 | .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI, | |
6c09ffd0 AB |
357 | { |
358 | .config = config, | |
359 | }, | |
f2eac75d MZ |
360 | }; |
361 | ||
362 | return irq_set_vcpu_affinity(irq, &info); | |
363 | } | |
3d63cb53 | 364 | |
d50676f5 MZ |
365 | int its_prop_update_vsgi(int irq, u8 priority, bool group) |
366 | { | |
367 | struct its_cmd_info info = { | |
368 | .cmd_type = PROP_UPDATE_VSGI, | |
369 | { | |
370 | .priority = priority, | |
371 | .group = group, | |
372 | }, | |
373 | }; | |
374 | ||
375 | return irq_set_vcpu_affinity(irq, &info); | |
376 | } | |
377 | ||
166cba71 MZ |
378 | int its_init_v4(struct irq_domain *domain, |
379 | const struct irq_domain_ops *vpe_ops, | |
380 | const struct irq_domain_ops *sgi_ops) | |
3d63cb53 MZ |
381 | { |
382 | if (domain) { | |
383 | pr_info("ITS: Enabling GICv4 support\n"); | |
384 | gic_domain = domain; | |
166cba71 MZ |
385 | vpe_domain_ops = vpe_ops; |
386 | sgi_domain_ops = sgi_ops; | |
3d63cb53 MZ |
387 | return 0; |
388 | } | |
389 | ||
390 | pr_err("ITS: No GICv4 VPE domain allocated\n"); | |
391 | return -ENODEV; | |
392 | } |