Commit | Line | Data |
---|---|---|
2299c49d SH |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org) | |
7 | * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. | |
8 | */ | |
1f19aee0 MR |
9 | |
10 | #define pr_fmt(fmt) "irq-mips-gic: " fmt | |
11 | ||
357a9c4b | 12 | #include <linux/bitfield.h> |
39b8d525 | 13 | #include <linux/bitmap.h> |
fb8f7be1 | 14 | #include <linux/clocksource.h> |
da61fcf9 | 15 | #include <linux/cpuhotplug.h> |
39b8d525 | 16 | #include <linux/init.h> |
18743d27 | 17 | #include <linux/interrupt.h> |
fb8f7be1 | 18 | #include <linux/irq.h> |
41a83e06 | 19 | #include <linux/irqchip.h> |
1982752f | 20 | #include <linux/irqdomain.h> |
a7057270 | 21 | #include <linux/of_address.h> |
aa493737 | 22 | #include <linux/percpu.h> |
18743d27 | 23 | #include <linux/sched.h> |
631330f5 | 24 | #include <linux/smp.h> |
39b8d525 | 25 | |
e83f7e02 | 26 | #include <asm/mips-cps.h> |
98b67c37 SH |
27 | #include <asm/setup.h> |
28 | #include <asm/traps.h> | |
39b8d525 | 29 | |
a7057270 AB |
30 | #include <dt-bindings/interrupt-controller/mips-gic.h> |
31 | ||
b11d4c1f | 32 | #define GIC_MAX_INTRS 256 |
aa493737 | 33 | #define GIC_MAX_LONGS BITS_TO_LONGS(GIC_MAX_INTRS) |
98b67c37 | 34 | |
b11d4c1f PB |
35 | /* Add 2 to convert GIC CPU pin to core interrupt */ |
36 | #define GIC_CPU_PIN_OFFSET 2 | |
37 | ||
38 | /* Mapped interrupt to pin X, then GIC will generate the vector (X+1). */ | |
39 | #define GIC_PIN_TO_VEC_OFFSET 1 | |
40 | ||
41 | /* Convert between local/shared IRQ number and GIC HW IRQ number. */ | |
42 | #define GIC_LOCAL_HWIRQ_BASE 0 | |
43 | #define GIC_LOCAL_TO_HWIRQ(x) (GIC_LOCAL_HWIRQ_BASE + (x)) | |
44 | #define GIC_HWIRQ_TO_LOCAL(x) ((x) - GIC_LOCAL_HWIRQ_BASE) | |
45 | #define GIC_SHARED_HWIRQ_BASE GIC_NUM_LOCAL_INTRS | |
46 | #define GIC_SHARED_TO_HWIRQ(x) (GIC_SHARED_HWIRQ_BASE + (x)) | |
47 | #define GIC_HWIRQ_TO_SHARED(x) ((x) - GIC_SHARED_HWIRQ_BASE) | |
48 | ||
582e2b4a | 49 | void __iomem *mips_gic_base; |
822350bc | 50 | |
b0e453ff | 51 | static DEFINE_PER_CPU_READ_MOSTLY(unsigned long[GIC_MAX_LONGS], pcpu_masks); |
2af70a96 | 52 | |
95150ae8 | 53 | static DEFINE_SPINLOCK(gic_lock); |
c49581a4 | 54 | static struct irq_domain *gic_irq_domain; |
fbd55241 | 55 | static int gic_shared_intrs; |
3263d085 | 56 | static unsigned int gic_cpu_pin; |
1b6af71a | 57 | static unsigned int timer_cpu_pin; |
4a6a3ea3 | 58 | static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller; |
8190cc57 SH |
59 | |
60 | #ifdef CONFIG_GENERIC_IRQ_IPI | |
61dc367e PB |
61 | static DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS); |
62 | static DECLARE_BITMAP(ipi_available, GIC_MAX_INTRS); | |
8190cc57 | 63 | #endif /* CONFIG_GENERIC_IRQ_IPI */ |
39b8d525 | 64 | |
da61fcf9 PB |
65 | static struct gic_all_vpes_chip_data { |
66 | u32 map; | |
67 | bool mask; | |
68 | } gic_all_vpes_chip_data[GIC_NUM_LOCAL_INTRS]; | |
69 | ||
7778c4b2 | 70 | static void gic_clear_pcpu_masks(unsigned int intr) |
8fa4b930 | 71 | { |
7778c4b2 | 72 | unsigned int i; |
835d2b45 | 73 | |
7778c4b2 PB |
74 | /* Clear the interrupt's bit in all pcpu_masks */ |
75 | for_each_possible_cpu(i) | |
76 | clear_bit(intr, per_cpu_ptr(pcpu_masks, i)); | |
835d2b45 PB |
77 | } |
78 | ||
e9de688d AB |
79 | static bool gic_local_irq_is_routable(int intr) |
80 | { | |
81 | u32 vpe_ctl; | |
82 | ||
83 | /* All local interrupts are routable in EIC mode. */ | |
84 | if (cpu_has_veic) | |
85 | return true; | |
86 | ||
0d0cf58c | 87 | vpe_ctl = read_gic_vl_ctl(); |
e9de688d AB |
88 | switch (intr) { |
89 | case GIC_LOCAL_INT_TIMER: | |
0d0cf58c | 90 | return vpe_ctl & GIC_VX_CTL_TIMER_ROUTABLE; |
e9de688d | 91 | case GIC_LOCAL_INT_PERFCTR: |
0d0cf58c | 92 | return vpe_ctl & GIC_VX_CTL_PERFCNT_ROUTABLE; |
e9de688d | 93 | case GIC_LOCAL_INT_FDC: |
0d0cf58c | 94 | return vpe_ctl & GIC_VX_CTL_FDC_ROUTABLE; |
e9de688d AB |
95 | case GIC_LOCAL_INT_SWINT0: |
96 | case GIC_LOCAL_INT_SWINT1: | |
0d0cf58c | 97 | return vpe_ctl & GIC_VX_CTL_SWINT_ROUTABLE; |
e9de688d AB |
98 | default: |
99 | return true; | |
100 | } | |
101 | } | |
102 | ||
3263d085 | 103 | static void gic_bind_eic_interrupt(int irq, int set) |
98b67c37 SH |
104 | { |
105 | /* Convert irq vector # to hw int # */ | |
106 | irq -= GIC_PIN_TO_VEC_OFFSET; | |
107 | ||
108 | /* Set irq to use shadow set */ | |
0d0cf58c | 109 | write_gic_vl_eic_shadow_set(irq, set); |
98b67c37 SH |
110 | } |
111 | ||
bb11cff3 | 112 | static void gic_send_ipi(struct irq_data *d, unsigned int cpu) |
39b8d525 | 113 | { |
bb11cff3 QY |
114 | irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d)); |
115 | ||
3680746a | 116 | write_gic_wedge(GIC_WEDGE_RW | hwirq); |
39b8d525 RB |
117 | } |
118 | ||
e9de688d AB |
119 | int gic_get_c0_compare_int(void) |
120 | { | |
121 | if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) | |
122 | return MIPS_CPU_IRQ_BASE + cp0_compare_irq; | |
123 | return irq_create_mapping(gic_irq_domain, | |
124 | GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER)); | |
125 | } | |
126 | ||
127 | int gic_get_c0_perfcount_int(void) | |
128 | { | |
129 | if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) { | |
7e3e6cb2 | 130 | /* Is the performance counter shared with the timer? */ |
e9de688d AB |
131 | if (cp0_perfcount_irq < 0) |
132 | return -1; | |
133 | return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; | |
134 | } | |
135 | return irq_create_mapping(gic_irq_domain, | |
136 | GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR)); | |
137 | } | |
138 | ||
6429e2b6 JH |
139 | int gic_get_c0_fdc_int(void) |
140 | { | |
141 | if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) { | |
142 | /* Is the FDC IRQ even present? */ | |
143 | if (cp0_fdc_irq < 0) | |
144 | return -1; | |
145 | return MIPS_CPU_IRQ_BASE + cp0_fdc_irq; | |
146 | } | |
147 | ||
6429e2b6 JH |
148 | return irq_create_mapping(gic_irq_domain, |
149 | GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC)); | |
150 | } | |
151 | ||
1b3ed367 | 152 | static void gic_handle_shared_int(bool chained) |
39b8d525 | 153 | { |
046a6ee2 | 154 | unsigned int intr; |
8f5ee79c | 155 | unsigned long *pcpu_mask; |
8f5ee79c | 156 | DECLARE_BITMAP(pending, GIC_MAX_INTRS); |
39b8d525 RB |
157 | |
158 | /* Get per-cpu bitmaps */ | |
aa493737 | 159 | pcpu_mask = this_cpu_ptr(pcpu_masks); |
d77d5ac9 | 160 | |
7778c4b2 | 161 | if (mips_cm_is64) |
e98fcb2a PB |
162 | __ioread64_copy(pending, addr_gic_pend(), |
163 | DIV_ROUND_UP(gic_shared_intrs, 64)); | |
7778c4b2 | 164 | else |
e98fcb2a PB |
165 | __ioread32_copy(pending, addr_gic_pend(), |
166 | DIV_ROUND_UP(gic_shared_intrs, 32)); | |
39b8d525 | 167 | |
fbd55241 | 168 | bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs); |
39b8d525 | 169 | |
cae750ba | 170 | for_each_set_bit(intr, pending, gic_shared_intrs) { |
1b3ed367 | 171 | if (chained) |
046a6ee2 MZ |
172 | generic_handle_domain_irq(gic_irq_domain, |
173 | GIC_SHARED_TO_HWIRQ(intr)); | |
1b3ed367 | 174 | else |
1fee9db9 MZ |
175 | do_domain_IRQ(gic_irq_domain, |
176 | GIC_SHARED_TO_HWIRQ(intr)); | |
d7eb4f2e | 177 | } |
39b8d525 RB |
178 | } |
179 | ||
161d049e | 180 | static void gic_mask_irq(struct irq_data *d) |
39b8d525 | 181 | { |
7778c4b2 PB |
182 | unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq); |
183 | ||
90019f8f | 184 | write_gic_rmask(intr); |
7778c4b2 | 185 | gic_clear_pcpu_masks(intr); |
39b8d525 RB |
186 | } |
187 | ||
161d049e | 188 | static void gic_unmask_irq(struct irq_data *d) |
39b8d525 | 189 | { |
7778c4b2 PB |
190 | unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq); |
191 | unsigned int cpu; | |
192 | ||
90019f8f | 193 | write_gic_smask(intr); |
7778c4b2 PB |
194 | |
195 | gic_clear_pcpu_masks(intr); | |
d9f82930 | 196 | cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); |
7778c4b2 | 197 | set_bit(intr, per_cpu_ptr(pcpu_masks, cpu)); |
39b8d525 RB |
198 | } |
199 | ||
5561c9e4 AB |
200 | static void gic_ack_irq(struct irq_data *d) |
201 | { | |
e9de688d | 202 | unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq); |
c49581a4 | 203 | |
3680746a | 204 | write_gic_wedge(irq); |
5561c9e4 AB |
205 | } |
206 | ||
95150ae8 AB |
207 | static int gic_set_type(struct irq_data *d, unsigned int type) |
208 | { | |
5af3e93e | 209 | unsigned int irq, pol, trig, dual; |
95150ae8 | 210 | unsigned long flags; |
5af3e93e PB |
211 | |
212 | irq = GIC_HWIRQ_TO_SHARED(d->hwirq); | |
95150ae8 AB |
213 | |
214 | spin_lock_irqsave(&gic_lock, flags); | |
215 | switch (type & IRQ_TYPE_SENSE_MASK) { | |
216 | case IRQ_TYPE_EDGE_FALLING: | |
5af3e93e PB |
217 | pol = GIC_POL_FALLING_EDGE; |
218 | trig = GIC_TRIG_EDGE; | |
219 | dual = GIC_DUAL_SINGLE; | |
95150ae8 AB |
220 | break; |
221 | case IRQ_TYPE_EDGE_RISING: | |
5af3e93e PB |
222 | pol = GIC_POL_RISING_EDGE; |
223 | trig = GIC_TRIG_EDGE; | |
224 | dual = GIC_DUAL_SINGLE; | |
95150ae8 AB |
225 | break; |
226 | case IRQ_TYPE_EDGE_BOTH: | |
5af3e93e PB |
227 | pol = 0; /* Doesn't matter */ |
228 | trig = GIC_TRIG_EDGE; | |
229 | dual = GIC_DUAL_DUAL; | |
95150ae8 AB |
230 | break; |
231 | case IRQ_TYPE_LEVEL_LOW: | |
5af3e93e PB |
232 | pol = GIC_POL_ACTIVE_LOW; |
233 | trig = GIC_TRIG_LEVEL; | |
234 | dual = GIC_DUAL_SINGLE; | |
95150ae8 AB |
235 | break; |
236 | case IRQ_TYPE_LEVEL_HIGH: | |
237 | default: | |
5af3e93e PB |
238 | pol = GIC_POL_ACTIVE_HIGH; |
239 | trig = GIC_TRIG_LEVEL; | |
240 | dual = GIC_DUAL_SINGLE; | |
95150ae8 AB |
241 | break; |
242 | } | |
243 | ||
5af3e93e PB |
244 | change_gic_pol(irq, pol); |
245 | change_gic_trig(irq, trig); | |
246 | change_gic_dual(irq, dual); | |
247 | ||
248 | if (trig == GIC_TRIG_EDGE) | |
a595fc51 TG |
249 | irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller, |
250 | handle_edge_irq, NULL); | |
251 | else | |
252 | irq_set_chip_handler_name_locked(d, &gic_level_irq_controller, | |
253 | handle_level_irq, NULL); | |
95150ae8 | 254 | spin_unlock_irqrestore(&gic_lock, flags); |
39b8d525 | 255 | |
95150ae8 AB |
256 | return 0; |
257 | } | |
258 | ||
259 | #ifdef CONFIG_SMP | |
161d049e TG |
260 | static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask, |
261 | bool force) | |
39b8d525 | 262 | { |
e9de688d | 263 | unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq); |
07df8bfe PB |
264 | unsigned long flags; |
265 | unsigned int cpu; | |
39b8d525 | 266 | |
07df8bfe PB |
267 | cpu = cpumask_first_and(cpumask, cpu_online_mask); |
268 | if (cpu >= NR_CPUS) | |
14d160ab | 269 | return -EINVAL; |
39b8d525 RB |
270 | |
271 | /* Assumption : cpumask refers to a single CPU */ | |
272 | spin_lock_irqsave(&gic_lock, flags); | |
39b8d525 | 273 | |
c214c035 | 274 | /* Re-route this IRQ */ |
07df8bfe | 275 | write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu))); |
c214c035 TW |
276 | |
277 | /* Update the pcpu_masks */ | |
7778c4b2 PB |
278 | gic_clear_pcpu_masks(irq); |
279 | if (read_gic_mask(irq)) | |
07df8bfe | 280 | set_bit(irq, per_cpu_ptr(pcpu_masks, cpu)); |
39b8d525 | 281 | |
18416e45 | 282 | irq_data_update_effective_affinity(d, cpumask_of(cpu)); |
39b8d525 RB |
283 | spin_unlock_irqrestore(&gic_lock, flags); |
284 | ||
7f15a648 | 285 | return IRQ_SET_MASK_OK; |
39b8d525 RB |
286 | } |
287 | #endif | |
288 | ||
4a6a3ea3 AB |
289 | static struct irq_chip gic_level_irq_controller = { |
290 | .name = "MIPS GIC", | |
291 | .irq_mask = gic_mask_irq, | |
292 | .irq_unmask = gic_unmask_irq, | |
293 | .irq_set_type = gic_set_type, | |
294 | #ifdef CONFIG_SMP | |
295 | .irq_set_affinity = gic_set_affinity, | |
296 | #endif | |
297 | }; | |
298 | ||
299 | static struct irq_chip gic_edge_irq_controller = { | |
161d049e | 300 | .name = "MIPS GIC", |
5561c9e4 | 301 | .irq_ack = gic_ack_irq, |
161d049e | 302 | .irq_mask = gic_mask_irq, |
161d049e | 303 | .irq_unmask = gic_unmask_irq, |
95150ae8 | 304 | .irq_set_type = gic_set_type, |
39b8d525 | 305 | #ifdef CONFIG_SMP |
161d049e | 306 | .irq_set_affinity = gic_set_affinity, |
39b8d525 | 307 | #endif |
bb11cff3 | 308 | .ipi_send_single = gic_send_ipi, |
39b8d525 RB |
309 | }; |
310 | ||
1b3ed367 | 311 | static void gic_handle_local_int(bool chained) |
e9de688d AB |
312 | { |
313 | unsigned long pending, masked; | |
046a6ee2 | 314 | unsigned int intr; |
e9de688d | 315 | |
9da3c645 PB |
316 | pending = read_gic_vl_pend(); |
317 | masked = read_gic_vl_mask(); | |
e9de688d AB |
318 | |
319 | bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS); | |
320 | ||
0f4ed158 | 321 | for_each_set_bit(intr, &pending, GIC_NUM_LOCAL_INTRS) { |
1b3ed367 | 322 | if (chained) |
046a6ee2 MZ |
323 | generic_handle_domain_irq(gic_irq_domain, |
324 | GIC_LOCAL_TO_HWIRQ(intr)); | |
1b3ed367 | 325 | else |
1fee9db9 MZ |
326 | do_domain_IRQ(gic_irq_domain, |
327 | GIC_LOCAL_TO_HWIRQ(intr)); | |
d7eb4f2e | 328 | } |
e9de688d AB |
329 | } |
330 | ||
331 | static void gic_mask_local_irq(struct irq_data *d) | |
332 | { | |
333 | int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); | |
334 | ||
9da3c645 | 335 | write_gic_vl_rmask(BIT(intr)); |
e9de688d AB |
336 | } |
337 | ||
338 | static void gic_unmask_local_irq(struct irq_data *d) | |
339 | { | |
340 | int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); | |
341 | ||
9da3c645 | 342 | write_gic_vl_smask(BIT(intr)); |
e9de688d AB |
343 | } |
344 | ||
345 | static struct irq_chip gic_local_irq_controller = { | |
346 | .name = "MIPS GIC Local", | |
347 | .irq_mask = gic_mask_local_irq, | |
348 | .irq_unmask = gic_unmask_local_irq, | |
349 | }; | |
350 | ||
351 | static void gic_mask_local_irq_all_vpes(struct irq_data *d) | |
352 | { | |
da61fcf9 | 353 | struct gic_all_vpes_chip_data *cd; |
e9de688d | 354 | unsigned long flags; |
da61fcf9 PB |
355 | int intr, cpu; |
356 | ||
357 | intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); | |
358 | cd = irq_data_get_irq_chip_data(d); | |
359 | cd->mask = false; | |
e9de688d AB |
360 | |
361 | spin_lock_irqsave(&gic_lock, flags); | |
da61fcf9 PB |
362 | for_each_online_cpu(cpu) { |
363 | write_gic_vl_other(mips_cm_vp_id(cpu)); | |
9da3c645 | 364 | write_gic_vo_rmask(BIT(intr)); |
e9de688d AB |
365 | } |
366 | spin_unlock_irqrestore(&gic_lock, flags); | |
367 | } | |
368 | ||
369 | static void gic_unmask_local_irq_all_vpes(struct irq_data *d) | |
370 | { | |
da61fcf9 | 371 | struct gic_all_vpes_chip_data *cd; |
e9de688d | 372 | unsigned long flags; |
da61fcf9 PB |
373 | int intr, cpu; |
374 | ||
375 | intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); | |
376 | cd = irq_data_get_irq_chip_data(d); | |
377 | cd->mask = true; | |
e9de688d AB |
378 | |
379 | spin_lock_irqsave(&gic_lock, flags); | |
da61fcf9 PB |
380 | for_each_online_cpu(cpu) { |
381 | write_gic_vl_other(mips_cm_vp_id(cpu)); | |
9da3c645 | 382 | write_gic_vo_smask(BIT(intr)); |
e9de688d AB |
383 | } |
384 | spin_unlock_irqrestore(&gic_lock, flags); | |
385 | } | |
386 | ||
dd098a0e | 387 | static void gic_all_vpes_irq_cpu_online(void) |
da61fcf9 | 388 | { |
dd098a0e MZ |
389 | static const unsigned int local_intrs[] = { |
390 | GIC_LOCAL_INT_TIMER, | |
391 | GIC_LOCAL_INT_PERFCTR, | |
392 | GIC_LOCAL_INT_FDC, | |
393 | }; | |
394 | unsigned long flags; | |
395 | int i; | |
da61fcf9 | 396 | |
dd098a0e | 397 | spin_lock_irqsave(&gic_lock, flags); |
da61fcf9 | 398 | |
dd098a0e MZ |
399 | for (i = 0; i < ARRAY_SIZE(local_intrs); i++) { |
400 | unsigned int intr = local_intrs[i]; | |
401 | struct gic_all_vpes_chip_data *cd; | |
402 | ||
403 | cd = &gic_all_vpes_chip_data[intr]; | |
404 | write_gic_vl_map(mips_gic_vx_map_reg(intr), cd->map); | |
405 | if (cd->mask) | |
406 | write_gic_vl_smask(BIT(intr)); | |
407 | } | |
408 | ||
409 | spin_unlock_irqrestore(&gic_lock, flags); | |
da61fcf9 PB |
410 | } |
411 | ||
e9de688d | 412 | static struct irq_chip gic_all_vpes_local_irq_controller = { |
da61fcf9 PB |
413 | .name = "MIPS GIC Local", |
414 | .irq_mask = gic_mask_local_irq_all_vpes, | |
415 | .irq_unmask = gic_unmask_local_irq_all_vpes, | |
e9de688d AB |
416 | }; |
417 | ||
18743d27 | 418 | static void __gic_irq_dispatch(void) |
39b8d525 | 419 | { |
1b3ed367 RV |
420 | gic_handle_local_int(false); |
421 | gic_handle_shared_int(false); | |
18743d27 | 422 | } |
39b8d525 | 423 | |
bd0b9ac4 | 424 | static void gic_irq_dispatch(struct irq_desc *desc) |
18743d27 | 425 | { |
1b3ed367 RV |
426 | gic_handle_local_int(true); |
427 | gic_handle_shared_int(true); | |
18743d27 AB |
428 | } |
429 | ||
e9de688d | 430 | static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq, |
7778c4b2 | 431 | irq_hw_number_t hw, unsigned int cpu) |
e9de688d AB |
432 | { |
433 | int intr = GIC_HWIRQ_TO_SHARED(hw); | |
d9f82930 | 434 | struct irq_data *data; |
c49581a4 AB |
435 | unsigned long flags; |
436 | ||
d9f82930 PB |
437 | data = irq_get_irq_data(virq); |
438 | ||
c49581a4 | 439 | spin_lock_irqsave(&gic_lock, flags); |
d3e8cf44 | 440 | write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); |
7778c4b2 | 441 | write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu))); |
d9f82930 | 442 | irq_data_update_effective_affinity(data, cpumask_of(cpu)); |
c49581a4 AB |
443 | spin_unlock_irqrestore(&gic_lock, flags); |
444 | ||
445 | return 0; | |
446 | } | |
447 | ||
b87281e7 | 448 | static int gic_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, |
c98c1822 QY |
449 | const u32 *intspec, unsigned int intsize, |
450 | irq_hw_number_t *out_hwirq, | |
451 | unsigned int *out_type) | |
452 | { | |
453 | if (intsize != 3) | |
454 | return -EINVAL; | |
455 | ||
456 | if (intspec[0] == GIC_SHARED) | |
457 | *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]); | |
458 | else if (intspec[0] == GIC_LOCAL) | |
459 | *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]); | |
460 | else | |
461 | return -EINVAL; | |
462 | *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; | |
463 | ||
464 | return 0; | |
465 | } | |
466 | ||
8ada00a6 MR |
467 | static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq, |
468 | irq_hw_number_t hwirq) | |
c98c1822 | 469 | { |
da61fcf9 | 470 | struct gic_all_vpes_chip_data *cd; |
63b746b1 PB |
471 | unsigned long flags; |
472 | unsigned int intr; | |
da61fcf9 | 473 | int err, cpu; |
63b746b1 | 474 | u32 map; |
c98c1822 | 475 | |
8ada00a6 | 476 | if (hwirq >= GIC_SHARED_HWIRQ_BASE) { |
8190cc57 | 477 | #ifdef CONFIG_GENERIC_IRQ_IPI |
b87281e7 PB |
478 | /* verify that shared irqs don't conflict with an IPI irq */ |
479 | if (test_bit(GIC_HWIRQ_TO_SHARED(hwirq), ipi_resrv)) | |
480 | return -EBUSY; | |
8190cc57 | 481 | #endif /* CONFIG_GENERIC_IRQ_IPI */ |
c98c1822 | 482 | |
b87281e7 PB |
483 | err = irq_domain_set_hwirq_and_chip(d, virq, hwirq, |
484 | &gic_level_irq_controller, | |
485 | NULL); | |
486 | if (err) | |
487 | return err; | |
488 | ||
18416e45 | 489 | irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); |
b87281e7 | 490 | return gic_shared_irq_domain_map(d, virq, hwirq, 0); |
c98c1822 QY |
491 | } |
492 | ||
63b746b1 PB |
493 | intr = GIC_HWIRQ_TO_LOCAL(hwirq); |
494 | map = GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin; | |
495 | ||
dd098a0e | 496 | /* |
915649da | 497 | * If adding support for more per-cpu interrupts, keep the |
dd098a0e MZ |
498 | * array in gic_all_vpes_irq_cpu_online() in sync. |
499 | */ | |
63b746b1 | 500 | switch (intr) { |
b87281e7 | 501 | case GIC_LOCAL_INT_TIMER: |
63b746b1 PB |
502 | /* CONFIG_MIPS_CMP workaround (see __gic_init) */ |
503 | map = GIC_MAP_PIN_MAP_TO_PIN | timer_cpu_pin; | |
df561f66 | 504 | fallthrough; |
b87281e7 PB |
505 | case GIC_LOCAL_INT_PERFCTR: |
506 | case GIC_LOCAL_INT_FDC: | |
507 | /* | |
508 | * HACK: These are all really percpu interrupts, but | |
509 | * the rest of the MIPS kernel code does not use the | |
510 | * percpu IRQ API for them. | |
511 | */ | |
da61fcf9 PB |
512 | cd = &gic_all_vpes_chip_data[intr]; |
513 | cd->map = map; | |
b87281e7 PB |
514 | err = irq_domain_set_hwirq_and_chip(d, virq, hwirq, |
515 | &gic_all_vpes_local_irq_controller, | |
da61fcf9 | 516 | cd); |
b87281e7 PB |
517 | if (err) |
518 | return err; | |
c98c1822 | 519 | |
b87281e7 PB |
520 | irq_set_handler(virq, handle_percpu_irq); |
521 | break; | |
522 | ||
523 | default: | |
524 | err = irq_domain_set_hwirq_and_chip(d, virq, hwirq, | |
525 | &gic_local_irq_controller, | |
526 | NULL); | |
527 | if (err) | |
528 | return err; | |
529 | ||
530 | irq_set_handler(virq, handle_percpu_devid_irq); | |
531 | irq_set_percpu_devid(virq); | |
532 | break; | |
533 | } | |
534 | ||
63b746b1 PB |
535 | if (!gic_local_irq_is_routable(intr)) |
536 | return -EPERM; | |
537 | ||
538 | spin_lock_irqsave(&gic_lock, flags); | |
da61fcf9 PB |
539 | for_each_online_cpu(cpu) { |
540 | write_gic_vl_other(mips_cm_vp_id(cpu)); | |
6d4d367d | 541 | write_gic_vo_map(mips_gic_vx_map_reg(intr), map); |
63b746b1 PB |
542 | } |
543 | spin_unlock_irqrestore(&gic_lock, flags); | |
544 | ||
545 | return 0; | |
c98c1822 QY |
546 | } |
547 | ||
8ada00a6 MR |
548 | static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq, |
549 | unsigned int nr_irqs, void *arg) | |
550 | { | |
551 | struct irq_fwspec *fwspec = arg; | |
552 | irq_hw_number_t hwirq; | |
553 | ||
554 | if (fwspec->param[0] == GIC_SHARED) | |
555 | hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]); | |
556 | else | |
557 | hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]); | |
558 | ||
559 | return gic_irq_domain_map(d, virq, hwirq); | |
560 | } | |
561 | ||
b87281e7 PB |
562 | void gic_irq_domain_free(struct irq_domain *d, unsigned int virq, |
563 | unsigned int nr_irqs) | |
2564970a | 564 | { |
2564970a PB |
565 | } |
566 | ||
b87281e7 PB |
567 | static const struct irq_domain_ops gic_irq_domain_ops = { |
568 | .xlate = gic_irq_domain_xlate, | |
569 | .alloc = gic_irq_domain_alloc, | |
570 | .free = gic_irq_domain_free, | |
8ada00a6 | 571 | .map = gic_irq_domain_map, |
2af70a96 QY |
572 | }; |
573 | ||
8190cc57 SH |
574 | #ifdef CONFIG_GENERIC_IRQ_IPI |
575 | ||
2af70a96 QY |
576 | static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, |
577 | const u32 *intspec, unsigned int intsize, | |
578 | irq_hw_number_t *out_hwirq, | |
579 | unsigned int *out_type) | |
580 | { | |
581 | /* | |
582 | * There's nothing to translate here. hwirq is dynamically allocated and | |
583 | * the irq type is always edge triggered. | |
584 | * */ | |
585 | *out_hwirq = 0; | |
586 | *out_type = IRQ_TYPE_EDGE_RISING; | |
587 | ||
588 | return 0; | |
589 | } | |
590 | ||
591 | static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq, | |
592 | unsigned int nr_irqs, void *arg) | |
593 | { | |
594 | struct cpumask *ipimask = arg; | |
b87281e7 PB |
595 | irq_hw_number_t hwirq, base_hwirq; |
596 | int cpu, ret, i; | |
2af70a96 | 597 | |
b87281e7 PB |
598 | base_hwirq = find_first_bit(ipi_available, gic_shared_intrs); |
599 | if (base_hwirq == gic_shared_intrs) | |
600 | return -ENOMEM; | |
601 | ||
602 | /* check that we have enough space */ | |
603 | for (i = base_hwirq; i < nr_irqs; i++) { | |
604 | if (!test_bit(i, ipi_available)) | |
605 | return -EBUSY; | |
606 | } | |
607 | bitmap_clear(ipi_available, base_hwirq, nr_irqs); | |
608 | ||
609 | /* map the hwirq for each cpu consecutively */ | |
610 | i = 0; | |
611 | for_each_cpu(cpu, ipimask) { | |
612 | hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i); | |
613 | ||
614 | ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq, | |
615 | &gic_edge_irq_controller, | |
616 | NULL); | |
617 | if (ret) | |
618 | goto error; | |
2af70a96 | 619 | |
b87281e7 | 620 | ret = irq_domain_set_hwirq_and_chip(d->parent, virq + i, hwirq, |
2af70a96 QY |
621 | &gic_edge_irq_controller, |
622 | NULL); | |
623 | if (ret) | |
624 | goto error; | |
625 | ||
626 | ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING); | |
627 | if (ret) | |
628 | goto error; | |
b87281e7 PB |
629 | |
630 | ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu); | |
631 | if (ret) | |
632 | goto error; | |
633 | ||
634 | i++; | |
2af70a96 QY |
635 | } |
636 | ||
637 | return 0; | |
638 | error: | |
b87281e7 | 639 | bitmap_set(ipi_available, base_hwirq, nr_irqs); |
2af70a96 QY |
640 | return ret; |
641 | } | |
642 | ||
b0e453ff WY |
643 | static void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq, |
644 | unsigned int nr_irqs) | |
2af70a96 | 645 | { |
b87281e7 PB |
646 | irq_hw_number_t base_hwirq; |
647 | struct irq_data *data; | |
648 | ||
649 | data = irq_get_irq_data(virq); | |
650 | if (!data) | |
651 | return; | |
652 | ||
653 | base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data)); | |
654 | bitmap_set(ipi_available, base_hwirq, nr_irqs); | |
2af70a96 QY |
655 | } |
656 | ||
b0e453ff WY |
657 | static int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node, |
658 | enum irq_domain_bus_token bus_token) | |
2af70a96 QY |
659 | { |
660 | bool is_ipi; | |
661 | ||
662 | switch (bus_token) { | |
663 | case DOMAIN_BUS_IPI: | |
664 | is_ipi = d->bus_token == bus_token; | |
547aefc4 | 665 | return (!node || to_of_node(d->fwnode) == node) && is_ipi; |
2af70a96 QY |
666 | break; |
667 | default: | |
668 | return 0; | |
669 | } | |
670 | } | |
671 | ||
0b7e815a | 672 | static const struct irq_domain_ops gic_ipi_domain_ops = { |
2af70a96 QY |
673 | .xlate = gic_ipi_domain_xlate, |
674 | .alloc = gic_ipi_domain_alloc, | |
675 | .free = gic_ipi_domain_free, | |
676 | .match = gic_ipi_domain_match, | |
c49581a4 AB |
677 | }; |
678 | ||
8190cc57 SH |
679 | static int gic_register_ipi_domain(struct device_node *node) |
680 | { | |
681 | struct irq_domain *gic_ipi_domain; | |
682 | unsigned int v[2], num_ipis; | |
683 | ||
684 | gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain, | |
685 | IRQ_DOMAIN_FLAG_IPI_PER_CPU, | |
686 | GIC_NUM_LOCAL_INTRS + gic_shared_intrs, | |
687 | node, &gic_ipi_domain_ops, NULL); | |
688 | if (!gic_ipi_domain) { | |
689 | pr_err("Failed to add IPI domain"); | |
690 | return -ENXIO; | |
691 | } | |
692 | ||
693 | irq_domain_update_bus_token(gic_ipi_domain, DOMAIN_BUS_IPI); | |
694 | ||
695 | if (node && | |
696 | !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) { | |
697 | bitmap_set(ipi_resrv, v[0], v[1]); | |
698 | } else { | |
699 | /* | |
700 | * Reserve 2 interrupts per possible CPU/VP for use as IPIs, | |
701 | * meeting the requirements of arch/mips SMP. | |
702 | */ | |
703 | num_ipis = 2 * num_possible_cpus(); | |
704 | bitmap_set(ipi_resrv, gic_shared_intrs - num_ipis, num_ipis); | |
705 | } | |
706 | ||
707 | bitmap_copy(ipi_available, ipi_resrv, GIC_MAX_INTRS); | |
708 | ||
709 | return 0; | |
710 | } | |
711 | ||
712 | #else /* !CONFIG_GENERIC_IRQ_IPI */ | |
713 | ||
714 | static inline int gic_register_ipi_domain(struct device_node *node) | |
715 | { | |
716 | return 0; | |
717 | } | |
718 | ||
719 | #endif /* !CONFIG_GENERIC_IRQ_IPI */ | |
720 | ||
da61fcf9 PB |
721 | static int gic_cpu_startup(unsigned int cpu) |
722 | { | |
890f6b55 PB |
723 | /* Enable or disable EIC */ |
724 | change_gic_vl_ctl(GIC_VX_CTL_EIC, | |
725 | cpu_has_veic ? GIC_VX_CTL_EIC : 0); | |
726 | ||
25ac19e1 PB |
727 | /* Clear all local IRQ masks (ie. disable all local interrupts) */ |
728 | write_gic_vl_rmask(~0); | |
729 | ||
dd098a0e MZ |
730 | /* Enable desired interrupts */ |
731 | gic_all_vpes_irq_cpu_online(); | |
da61fcf9 PB |
732 | |
733 | return 0; | |
734 | } | |
fbea7541 PB |
735 | |
736 | static int __init gic_of_init(struct device_node *node, | |
737 | struct device_node *parent) | |
39b8d525 | 738 | { |
8190cc57 | 739 | unsigned int cpu_vec, i, gicconfig; |
b2b2e584 | 740 | unsigned long reserved; |
fbea7541 PB |
741 | phys_addr_t gic_base; |
742 | struct resource res; | |
743 | size_t gic_len; | |
8190cc57 | 744 | int ret; |
fbea7541 PB |
745 | |
746 | /* Find the first available CPU vector. */ | |
b2b2e584 | 747 | i = 0; |
a08588ea | 748 | reserved = (C_SW0 | C_SW1) >> __ffs(C_SW0); |
fbea7541 PB |
749 | while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors", |
750 | i++, &cpu_vec)) | |
751 | reserved |= BIT(cpu_vec); | |
b2b2e584 PB |
752 | |
753 | cpu_vec = find_first_zero_bit(&reserved, hweight_long(ST0_IM)); | |
754 | if (cpu_vec == hweight_long(ST0_IM)) { | |
1f19aee0 | 755 | pr_err("No CPU vectors available\n"); |
fbea7541 PB |
756 | return -ENODEV; |
757 | } | |
758 | ||
759 | if (of_address_to_resource(node, 0, &res)) { | |
760 | /* | |
761 | * Probe the CM for the GIC base address if not specified | |
762 | * in the device-tree. | |
763 | */ | |
764 | if (mips_cm_present()) { | |
765 | gic_base = read_gcr_gic_base() & | |
766 | ~CM_GCR_GIC_BASE_GICEN; | |
767 | gic_len = 0x20000; | |
666740fd MR |
768 | pr_warn("Using inherited base address %pa\n", |
769 | &gic_base); | |
fbea7541 | 770 | } else { |
1f19aee0 | 771 | pr_err("Failed to get memory range\n"); |
fbea7541 PB |
772 | return -ENODEV; |
773 | } | |
774 | } else { | |
775 | gic_base = res.start; | |
776 | gic_len = resource_size(&res); | |
777 | } | |
39b8d525 | 778 | |
fbea7541 PB |
779 | if (mips_cm_present()) { |
780 | write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN); | |
781 | /* Ensure GIC region is enabled before trying to access it */ | |
782 | __sync(); | |
783 | } | |
c0a9f72c | 784 | |
4bdc0d67 | 785 | mips_gic_base = ioremap(gic_base, gic_len); |
71349cc8 WD |
786 | if (!mips_gic_base) { |
787 | pr_err("Failed to ioremap gic_base\n"); | |
788 | return -ENOMEM; | |
789 | } | |
39b8d525 | 790 | |
3680746a | 791 | gicconfig = read_gic_config(); |
357a9c4b | 792 | gic_shared_intrs = FIELD_GET(GIC_CONFIG_NUMINTERRUPTS, gicconfig); |
3680746a | 793 | gic_shared_intrs = (gic_shared_intrs + 1) * 8; |
39b8d525 | 794 | |
18743d27 AB |
795 | if (cpu_has_veic) { |
796 | /* Always use vector 1 in EIC mode */ | |
797 | gic_cpu_pin = 0; | |
1b6af71a | 798 | timer_cpu_pin = gic_cpu_pin; |
18743d27 AB |
799 | set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET, |
800 | __gic_irq_dispatch); | |
801 | } else { | |
802 | gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET; | |
803 | irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec, | |
804 | gic_irq_dispatch); | |
1b6af71a JH |
805 | /* |
806 | * With the CMP implementation of SMP (deprecated), other CPUs | |
807 | * are started by the bootloader and put into a timer based | |
808 | * waiting poll loop. We must not re-route those CPU's local | |
809 | * timer interrupts as the wait instruction will never finish, | |
810 | * so just handle whatever CPU interrupt it is routed to by | |
811 | * default. | |
812 | * | |
813 | * This workaround should be removed when CMP support is | |
814 | * dropped. | |
815 | */ | |
816 | if (IS_ENABLED(CONFIG_MIPS_CMP) && | |
817 | gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) { | |
0d0cf58c | 818 | timer_cpu_pin = read_gic_vl_timer_map() & GIC_MAP_PIN_MAP; |
1b6af71a JH |
819 | irq_set_chained_handler(MIPS_CPU_IRQ_BASE + |
820 | GIC_CPU_PIN_OFFSET + | |
821 | timer_cpu_pin, | |
822 | gic_irq_dispatch); | |
823 | } else { | |
824 | timer_cpu_pin = gic_cpu_pin; | |
825 | } | |
18743d27 AB |
826 | } |
827 | ||
a7057270 | 828 | gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS + |
fbea7541 | 829 | gic_shared_intrs, 0, |
c49581a4 | 830 | &gic_irq_domain_ops, NULL); |
fbea7541 | 831 | if (!gic_irq_domain) { |
1f19aee0 | 832 | pr_err("Failed to add IRQ domain"); |
fbea7541 PB |
833 | return -ENXIO; |
834 | } | |
0b271f56 | 835 | |
8190cc57 SH |
836 | ret = gic_register_ipi_domain(node); |
837 | if (ret) | |
838 | return ret; | |
a7057270 | 839 | |
87888bcb | 840 | board_bind_eic_interrupt = &gic_bind_eic_interrupt; |
a7057270 | 841 | |
87888bcb PB |
842 | /* Setup defaults */ |
843 | for (i = 0; i < gic_shared_intrs; i++) { | |
844 | change_gic_pol(i, GIC_POL_ACTIVE_HIGH); | |
845 | change_gic_trig(i, GIC_TRIG_LEVEL); | |
90019f8f | 846 | write_gic_rmask(i); |
a7057270 AB |
847 | } |
848 | ||
da61fcf9 PB |
849 | return cpuhp_setup_state(CPUHP_AP_IRQ_MIPS_GIC_STARTING, |
850 | "irqchip/mips/gic:starting", | |
851 | gic_cpu_startup, NULL); | |
a7057270 AB |
852 | } |
853 | IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init); |