Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
0b05ac6e BH |
2 | /* |
3 | * Copyright 2011 IBM Corporation. | |
0b05ac6e BH |
4 | */ |
5 | #include <linux/types.h> | |
6 | #include <linux/threads.h> | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/irq.h> | |
9 | #include <linux/debugfs.h> | |
10 | #include <linux/smp.h> | |
11 | #include <linux/interrupt.h> | |
12 | #include <linux/seq_file.h> | |
13 | #include <linux/init.h> | |
14 | #include <linux/cpu.h> | |
15 | #include <linux/of.h> | |
16 | #include <linux/slab.h> | |
17 | #include <linux/spinlock.h> | |
a69e2fb7 | 18 | #include <linux/delay.h> |
0b05ac6e BH |
19 | |
20 | #include <asm/prom.h> | |
21 | #include <asm/io.h> | |
22 | #include <asm/smp.h> | |
23 | #include <asm/machdep.h> | |
24 | #include <asm/irq.h> | |
25 | #include <asm/errno.h> | |
26 | #include <asm/rtas.h> | |
27 | #include <asm/xics.h> | |
28 | #include <asm/firmware.h> | |
29 | ||
30 | /* Globals common to all ICP/ICS implementations */ | |
31 | const struct icp_ops *icp_ops; | |
32 | ||
33 | unsigned int xics_default_server = 0xff; | |
34 | unsigned int xics_default_distrib_server = 0; | |
35 | unsigned int xics_interrupt_server_size = 8; | |
36 | ||
37 | DEFINE_PER_CPU(struct xics_cppr, xics_cppr); | |
38 | ||
bae1d8f1 | 39 | struct irq_domain *xics_host; |
0b05ac6e | 40 | |
298f6f95 | 41 | static struct ics *xics_ics; |
0b05ac6e BH |
42 | |
43 | void xics_update_irq_servers(void) | |
44 | { | |
45 | int i, j; | |
46 | struct device_node *np; | |
47 | u32 ilen; | |
d213dd53 | 48 | const __be32 *ireg; |
0b05ac6e BH |
49 | u32 hcpuid; |
50 | ||
51 | /* Find the server numbers for the boot cpu. */ | |
52 | np = of_get_cpu_node(boot_cpuid, NULL); | |
53 | BUG_ON(!np); | |
54 | ||
55 | hcpuid = get_hard_smp_processor_id(boot_cpuid); | |
f6e17f9b BH |
56 | xics_default_server = xics_default_distrib_server = hcpuid; |
57 | ||
58 | pr_devel("xics: xics_default_server = 0x%x\n", xics_default_server); | |
0b05ac6e BH |
59 | |
60 | ireg = of_get_property(np, "ibm,ppc-interrupt-gserver#s", &ilen); | |
61 | if (!ireg) { | |
62 | of_node_put(np); | |
63 | return; | |
64 | } | |
65 | ||
66 | i = ilen / sizeof(int); | |
67 | ||
68 | /* Global interrupt distribution server is specified in the last | |
69 | * entry of "ibm,ppc-interrupt-gserver#s" property. Get the last | |
70 | * entry fom this property for current boot cpu id and use it as | |
71 | * default distribution server | |
72 | */ | |
73 | for (j = 0; j < i; j += 2) { | |
d213dd53 AB |
74 | if (be32_to_cpu(ireg[j]) == hcpuid) { |
75 | xics_default_distrib_server = be32_to_cpu(ireg[j+1]); | |
f6e17f9b | 76 | break; |
0b05ac6e BH |
77 | } |
78 | } | |
f6e17f9b BH |
79 | pr_devel("xics: xics_default_distrib_server = 0x%x\n", |
80 | xics_default_distrib_server); | |
0b05ac6e BH |
81 | of_node_put(np); |
82 | } | |
83 | ||
84 | /* GIQ stuff, currently only supported on RTAS setups, will have | |
85 | * to be sorted properly for bare metal | |
86 | */ | |
87 | void xics_set_cpu_giq(unsigned int gserver, unsigned int join) | |
88 | { | |
89 | #ifdef CONFIG_PPC_RTAS | |
90 | int index; | |
91 | int status; | |
92 | ||
93 | if (!rtas_indicator_present(GLOBAL_INTERRUPT_QUEUE, NULL)) | |
94 | return; | |
95 | ||
96 | index = (1UL << xics_interrupt_server_size) - 1 - gserver; | |
97 | ||
98 | status = rtas_set_indicator_fast(GLOBAL_INTERRUPT_QUEUE, index, join); | |
99 | ||
100 | WARN(status < 0, "set-indicator(%d, %d, %u) returned %d\n", | |
101 | GLOBAL_INTERRUPT_QUEUE, index, join, status); | |
102 | #endif | |
103 | } | |
104 | ||
105 | void xics_setup_cpu(void) | |
106 | { | |
107 | icp_ops->set_priority(LOWEST_PRIORITY); | |
108 | ||
109 | xics_set_cpu_giq(xics_default_distrib_server, 1); | |
110 | } | |
111 | ||
112 | void xics_mask_unknown_vec(unsigned int vec) | |
113 | { | |
f6e17f9b | 114 | pr_err("Interrupt 0x%x (real) is invalid, disabling it.\n", vec); |
0b05ac6e | 115 | |
298f6f95 CLG |
116 | if (WARN_ON(!xics_ics)) |
117 | return; | |
118 | xics_ics->mask_unknown(xics_ics, vec); | |
0b05ac6e BH |
119 | } |
120 | ||
121 | ||
122 | #ifdef CONFIG_SMP | |
123 | ||
0b05ac6e BH |
124 | static void xics_request_ipi(void) |
125 | { | |
126 | unsigned int ipi; | |
127 | ||
128 | ipi = irq_create_mapping(xics_host, XICS_IPI); | |
ef24ba70 | 129 | BUG_ON(!ipi); |
0b05ac6e BH |
130 | |
131 | /* | |
a3a9f3b4 | 132 | * IPIs are marked IRQF_PERCPU. The handler was set in map. |
0b05ac6e | 133 | */ |
0b05ac6e | 134 | BUG_ON(request_irq(ipi, icp_ops->ipi_action, |
3b5e16d7 | 135 | IRQF_PERCPU | IRQF_NO_THREAD, "IPI", NULL)); |
0b05ac6e BH |
136 | } |
137 | ||
a7f4ee1f | 138 | void __init xics_smp_probe(void) |
0b05ac6e | 139 | { |
0b05ac6e BH |
140 | /* Register all the IPIs */ |
141 | xics_request_ipi(); | |
45b21cfe ME |
142 | |
143 | /* Setup cause_ipi callback based on which ICP is used */ | |
144 | smp_ops->cause_ipi = icp_ops->cause_ipi; | |
0b05ac6e BH |
145 | } |
146 | ||
147 | #endif /* CONFIG_SMP */ | |
148 | ||
149 | void xics_teardown_cpu(void) | |
150 | { | |
69111bac | 151 | struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); |
0b05ac6e BH |
152 | |
153 | /* | |
154 | * we have to reset the cppr index to 0 because we're | |
155 | * not going to return from the IPI | |
156 | */ | |
157 | os_cppr->index = 0; | |
158 | icp_ops->set_priority(0); | |
159 | icp_ops->teardown_cpu(); | |
160 | } | |
161 | ||
162 | void xics_kexec_teardown_cpu(int secondary) | |
163 | { | |
164 | xics_teardown_cpu(); | |
165 | ||
166 | icp_ops->flush_ipi(); | |
167 | ||
168 | /* | |
169 | * Some machines need to have at least one cpu in the GIQ, | |
170 | * so leave the master cpu in the group. | |
171 | */ | |
172 | if (secondary) | |
173 | xics_set_cpu_giq(xics_default_distrib_server, 0); | |
174 | } | |
175 | ||
176 | ||
177 | #ifdef CONFIG_HOTPLUG_CPU | |
178 | ||
179 | /* Interrupts are disabled. */ | |
180 | void xics_migrate_irqs_away(void) | |
181 | { | |
182 | int cpu = smp_processor_id(), hw_cpu = hard_smp_processor_id(); | |
183 | unsigned int irq, virq; | |
4013369f | 184 | struct irq_desc *desc; |
0b05ac6e BH |
185 | |
186 | /* If we used to be the default server, move to the new "boot_cpuid" */ | |
187 | if (hw_cpu == xics_default_server) | |
188 | xics_update_irq_servers(); | |
189 | ||
190 | /* Reject any interrupt that was queued to us... */ | |
191 | icp_ops->set_priority(0); | |
192 | ||
193 | /* Remove ourselves from the global interrupt queue */ | |
194 | xics_set_cpu_giq(xics_default_distrib_server, 0); | |
195 | ||
4013369f | 196 | for_each_irq_desc(virq, desc) { |
0b05ac6e BH |
197 | struct irq_chip *chip; |
198 | long server; | |
199 | unsigned long flags; | |
0b05ac6e BH |
200 | |
201 | /* We can't set affinity on ISA interrupts */ | |
7c576f4d | 202 | if (virq < NR_IRQS_LEGACY) |
0b05ac6e | 203 | continue; |
0b05ac6e | 204 | /* We only need to migrate enabled IRQS */ |
4013369f | 205 | if (!desc->action) |
0b05ac6e | 206 | continue; |
6d9285b0 GL |
207 | if (desc->irq_data.domain != xics_host) |
208 | continue; | |
209 | irq = desc->irq_data.hwirq; | |
210 | /* We need to get IPIs still. */ | |
211 | if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) | |
212 | continue; | |
0b05ac6e BH |
213 | chip = irq_desc_get_chip(desc); |
214 | if (!chip || !chip->irq_set_affinity) | |
215 | continue; | |
216 | ||
217 | raw_spin_lock_irqsave(&desc->lock, flags); | |
218 | ||
219 | /* Locate interrupt server */ | |
298f6f95 | 220 | server = xics_ics->get_server(xics_ics, irq); |
0b05ac6e | 221 | if (server < 0) { |
298f6f95 CLG |
222 | pr_err("%s: Can't find server for irq %d/%x\n", |
223 | __func__, virq, irq); | |
0b05ac6e BH |
224 | goto unlock; |
225 | } | |
226 | ||
227 | /* We only support delivery to all cpus or to one cpu. | |
228 | * The irq has to be migrated only in the single cpu | |
229 | * case. | |
230 | */ | |
231 | if (server != hw_cpu) | |
232 | goto unlock; | |
233 | ||
234 | /* This is expected during cpu offline. */ | |
235 | if (cpu_online(cpu)) | |
f2c2cbcc JP |
236 | pr_warn("IRQ %u affinity broken off cpu %u\n", |
237 | virq, cpu); | |
0b05ac6e BH |
238 | |
239 | /* Reset affinity to all cpus */ | |
240 | raw_spin_unlock_irqrestore(&desc->lock, flags); | |
241 | irq_set_affinity(virq, cpu_all_mask); | |
242 | continue; | |
243 | unlock: | |
244 | raw_spin_unlock_irqrestore(&desc->lock, flags); | |
245 | } | |
a69e2fb7 BS |
246 | |
247 | /* Allow "sufficient" time to drop any inflight IRQ's */ | |
248 | mdelay(5); | |
249 | ||
250 | /* | |
251 | * Allow IPIs again. This is done at the very end, after migrating all | |
252 | * interrupts, the expectation is that we'll only get woken up by an IPI | |
253 | * interrupt beyond this point, but leave externals masked just to be | |
254 | * safe. If we're using icp-opal this may actually allow all | |
255 | * interrupts anyway, but that should be OK. | |
256 | */ | |
257 | icp_ops->set_priority(DEFAULT_PRIORITY); | |
258 | ||
0b05ac6e BH |
259 | } |
260 | #endif /* CONFIG_HOTPLUG_CPU */ | |
261 | ||
262 | #ifdef CONFIG_SMP | |
263 | /* | |
264 | * For the moment we only implement delivery to all cpus or one cpu. | |
265 | * | |
266 | * If the requested affinity is cpu_all_mask, we set global affinity. | |
267 | * If not we set it to the first cpu in the mask, even if multiple cpus | |
268 | * are set. This is so things like irqbalance (which set core and package | |
269 | * wide affinities) do the right thing. | |
f6e17f9b BH |
270 | * |
271 | * We need to fix this to implement support for the links | |
0b05ac6e BH |
272 | */ |
273 | int xics_get_irq_server(unsigned int virq, const struct cpumask *cpumask, | |
274 | unsigned int strict_check) | |
275 | { | |
276 | ||
277 | if (!distribute_irqs) | |
278 | return xics_default_server; | |
279 | ||
280 | if (!cpumask_subset(cpu_possible_mask, cpumask)) { | |
281 | int server = cpumask_first_and(cpu_online_mask, cpumask); | |
282 | ||
283 | if (server < nr_cpu_ids) | |
284 | return get_hard_smp_processor_id(server); | |
285 | ||
286 | if (strict_check) | |
287 | return -1; | |
288 | } | |
289 | ||
290 | /* | |
291 | * Workaround issue with some versions of JS20 firmware that | |
292 | * deliver interrupts to cpus which haven't been started. This | |
293 | * happens when using the maxcpus= boot option. | |
294 | */ | |
295 | if (cpumask_equal(cpu_online_mask, cpu_present_mask)) | |
296 | return xics_default_distrib_server; | |
297 | ||
298 | return xics_default_server; | |
299 | } | |
300 | #endif /* CONFIG_SMP */ | |
301 | ||
ad3aedfb MZ |
302 | static int xics_host_match(struct irq_domain *h, struct device_node *node, |
303 | enum irq_domain_bus_token bus_token) | |
0b05ac6e | 304 | { |
298f6f95 CLG |
305 | if (WARN_ON(!xics_ics)) |
306 | return 0; | |
307 | return xics_ics->host_match(xics_ics, node) ? 1 : 0; | |
0b05ac6e BH |
308 | } |
309 | ||
310 | /* Dummies */ | |
311 | static void xics_ipi_unmask(struct irq_data *d) { } | |
312 | static void xics_ipi_mask(struct irq_data *d) { } | |
313 | ||
314 | static struct irq_chip xics_ipi_chip = { | |
315 | .name = "XICS", | |
316 | .irq_eoi = NULL, /* Patched at init time */ | |
317 | .irq_mask = xics_ipi_mask, | |
318 | .irq_unmask = xics_ipi_unmask, | |
319 | }; | |
320 | ||
bae1d8f1 | 321 | static int xics_host_map(struct irq_domain *h, unsigned int virq, |
0b05ac6e BH |
322 | irq_hw_number_t hw) |
323 | { | |
0b05ac6e BH |
324 | pr_devel("xics: map virq %d, hwirq 0x%lx\n", virq, hw); |
325 | ||
880a3d6a BH |
326 | /* |
327 | * Mark interrupts as edge sensitive by default so that resend | |
328 | * actually works. The device-tree parsing will turn the LSIs | |
329 | * back to level. | |
330 | */ | |
331 | irq_clear_status_flags(virq, IRQ_LEVEL); | |
0b05ac6e BH |
332 | |
333 | /* Don't call into ICS for IPIs */ | |
334 | if (hw == XICS_IPI) { | |
335 | irq_set_chip_and_handler(virq, &xics_ipi_chip, | |
e085255e | 336 | handle_percpu_irq); |
0b05ac6e BH |
337 | return 0; |
338 | } | |
339 | ||
298f6f95 CLG |
340 | if (WARN_ON(!xics_ics)) |
341 | return -EINVAL; | |
342 | ||
0b05ac6e | 343 | /* Let the ICS setup the chip data */ |
298f6f95 CLG |
344 | if (xics_ics->map(xics_ics, virq)) |
345 | return -EINVAL; | |
e085255e | 346 | |
298f6f95 | 347 | return 0; |
0b05ac6e BH |
348 | } |
349 | ||
bae1d8f1 | 350 | static int xics_host_xlate(struct irq_domain *h, struct device_node *ct, |
0b05ac6e BH |
351 | const u32 *intspec, unsigned int intsize, |
352 | irq_hw_number_t *out_hwirq, unsigned int *out_flags) | |
353 | ||
354 | { | |
0b05ac6e | 355 | *out_hwirq = intspec[0]; |
0b05ac6e | 356 | |
880a3d6a BH |
357 | /* |
358 | * If intsize is at least 2, we look for the type in the second cell, | |
359 | * we assume the LSB indicates a level interrupt. | |
360 | */ | |
361 | if (intsize > 1) { | |
362 | if (intspec[1] & 1) | |
363 | *out_flags = IRQ_TYPE_LEVEL_LOW; | |
364 | else | |
365 | *out_flags = IRQ_TYPE_EDGE_RISING; | |
366 | } else | |
367 | *out_flags = IRQ_TYPE_LEVEL_LOW; | |
368 | ||
369 | return 0; | |
370 | } | |
371 | ||
372 | int xics_set_irq_type(struct irq_data *d, unsigned int flow_type) | |
373 | { | |
374 | /* | |
375 | * We only support these. This has really no effect other than setting | |
376 | * the corresponding descriptor bits mind you but those will in turn | |
377 | * affect the resend function when re-enabling an edge interrupt. | |
378 | * | |
379 | * Set set the default to edge as explained in map(). | |
380 | */ | |
381 | if (flow_type == IRQ_TYPE_DEFAULT || flow_type == IRQ_TYPE_NONE) | |
382 | flow_type = IRQ_TYPE_EDGE_RISING; | |
383 | ||
384 | if (flow_type != IRQ_TYPE_EDGE_RISING && | |
385 | flow_type != IRQ_TYPE_LEVEL_LOW) | |
386 | return -EINVAL; | |
387 | ||
388 | irqd_set_trigger_type(d, flow_type); | |
389 | ||
390 | return IRQ_SET_MASK_OK_NOCOPY; | |
391 | } | |
392 | ||
393 | int xics_retrigger(struct irq_data *data) | |
394 | { | |
395 | /* | |
396 | * We need to push a dummy CPPR when retriggering, since the subsequent | |
397 | * EOI will try to pop it. Passing 0 works, as the function hard codes | |
398 | * the priority value anyway. | |
399 | */ | |
400 | xics_push_cppr(0); | |
401 | ||
402 | /* Tell the core to do a soft retrigger */ | |
0b05ac6e BH |
403 | return 0; |
404 | } | |
405 | ||
202648a6 | 406 | static const struct irq_domain_ops xics_host_ops = { |
0b05ac6e BH |
407 | .match = xics_host_match, |
408 | .map = xics_host_map, | |
409 | .xlate = xics_host_xlate, | |
410 | }; | |
411 | ||
412 | static void __init xics_init_host(void) | |
413 | { | |
a8db8cf0 | 414 | xics_host = irq_domain_add_tree(NULL, &xics_host_ops, NULL); |
0b05ac6e BH |
415 | BUG_ON(xics_host == NULL); |
416 | irq_set_default_host(xics_host); | |
417 | } | |
418 | ||
419 | void __init xics_register_ics(struct ics *ics) | |
420 | { | |
298f6f95 CLG |
421 | if (WARN_ONCE(xics_ics, "XICS: Source Controller is already defined !")) |
422 | return; | |
423 | xics_ics = ics; | |
0b05ac6e BH |
424 | } |
425 | ||
426 | static void __init xics_get_server_size(void) | |
427 | { | |
428 | struct device_node *np; | |
d213dd53 | 429 | const __be32 *isize; |
0b05ac6e BH |
430 | |
431 | /* We fetch the interrupt server size from the first ICS node | |
432 | * we find if any | |
433 | */ | |
434 | np = of_find_compatible_node(NULL, NULL, "ibm,ppc-xics"); | |
435 | if (!np) | |
436 | return; | |
589b1f7e | 437 | |
0b05ac6e | 438 | isize = of_get_property(np, "ibm,interrupt-server#-size", NULL); |
589b1f7e Y |
439 | if (isize) |
440 | xics_interrupt_server_size = be32_to_cpu(*isize); | |
441 | ||
0b05ac6e BH |
442 | of_node_put(np); |
443 | } | |
444 | ||
445 | void __init xics_init(void) | |
446 | { | |
447 | int rc = -1; | |
448 | ||
449 | /* Fist locate ICP */ | |
0b05ac6e BH |
450 | if (firmware_has_feature(FW_FEATURE_LPAR)) |
451 | rc = icp_hv_init(); | |
d7436188 | 452 | if (rc < 0) { |
0b05ac6e | 453 | rc = icp_native_init(); |
d7436188 BH |
454 | if (rc == -ENODEV) |
455 | rc = icp_opal_init(); | |
456 | } | |
0b05ac6e | 457 | if (rc < 0) { |
f2c2cbcc | 458 | pr_warn("XICS: Cannot find a Presentation Controller !\n"); |
0b05ac6e BH |
459 | return; |
460 | } | |
461 | ||
462 | /* Copy get_irq callback over to ppc_md */ | |
463 | ppc_md.get_irq = icp_ops->get_irq; | |
464 | ||
465 | /* Patch up IPI chip EOI */ | |
466 | xics_ipi_chip.irq_eoi = icp_ops->eoi; | |
467 | ||
468 | /* Now locate ICS */ | |
0b05ac6e | 469 | rc = ics_rtas_init(); |
5c7c1e94 BH |
470 | if (rc < 0) |
471 | rc = ics_opal_init(); | |
aa9c5adf BH |
472 | if (rc < 0) |
473 | rc = ics_native_init(); | |
0b05ac6e | 474 | if (rc < 0) |
f2c2cbcc | 475 | pr_warn("XICS: Cannot find a Source Controller !\n"); |
0b05ac6e BH |
476 | |
477 | /* Initialize common bits */ | |
478 | xics_get_server_size(); | |
479 | xics_update_irq_servers(); | |
480 | xics_init_host(); | |
481 | xics_setup_cpu(); | |
482 | } |