Commit | Line | Data |
---|---|---|
9ae6f740 TP |
1 | /* |
2 | * Marvell Armada 370 and Armada XP SoC IRQ handling | |
3 | * | |
4 | * Copyright (C) 2012 Marvell | |
5 | * | |
6 | * Lior Amsalem <alior@marvell.com> | |
7 | * Gregory CLEMENT <gregory.clement@free-electrons.com> | |
8 | * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> | |
9 | * Ben Dooks <ben.dooks@codethink.co.uk> | |
10 | * | |
11 | * This file is licensed under the terms of the GNU General Public | |
12 | * License version 2. This program is licensed "as is" without any | |
13 | * warranty of any kind, whether express or implied. | |
14 | */ | |
15 | ||
16 | #include <linux/kernel.h> | |
17 | #include <linux/module.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/irq.h> | |
20 | #include <linux/interrupt.h> | |
41a83e06 | 21 | #include <linux/irqchip.h> |
bc69b8ad | 22 | #include <linux/irqchip/chained_irq.h> |
d7df84b3 | 23 | #include <linux/cpu.h> |
9ae6f740 TP |
24 | #include <linux/io.h> |
25 | #include <linux/of_address.h> | |
26 | #include <linux/of_irq.h> | |
31f614ed | 27 | #include <linux/of_pci.h> |
9ae6f740 | 28 | #include <linux/irqdomain.h> |
31f614ed | 29 | #include <linux/slab.h> |
0f077eb5 | 30 | #include <linux/syscore_ops.h> |
31f614ed | 31 | #include <linux/msi.h> |
9ae6f740 TP |
32 | #include <asm/mach/arch.h> |
33 | #include <asm/exception.h> | |
344e873e | 34 | #include <asm/smp_plat.h> |
9339d432 TP |
35 | #include <asm/mach/irq.h> |
36 | ||
054ea4ce TP |
37 | /* |
38 | * Overall diagram of the Armada XP interrupt controller: | |
39 | * | |
40 | * To CPU 0 To CPU 1 | |
41 | * | |
42 | * /\ /\ | |
43 | * || || | |
44 | * +---------------+ +---------------+ | |
45 | * | | | | | |
46 | * | per-CPU | | per-CPU | | |
47 | * | mask/unmask | | mask/unmask | | |
48 | * | CPU0 | | CPU1 | | |
49 | * | | | | | |
50 | * +---------------+ +---------------+ | |
51 | * /\ /\ | |
52 | * || || | |
53 | * \\_______________________// | |
54 | * || | |
55 | * +-------------------+ | |
56 | * | | | |
57 | * | Global interrupt | | |
58 | * | mask/unmask | | |
59 | * | | | |
60 | * +-------------------+ | |
61 | * /\ | |
62 | * || | |
63 | * interrupt from | |
64 | * device | |
65 | * | |
66 | * The "global interrupt mask/unmask" is modified using the | |
67 | * ARMADA_370_XP_INT_SET_ENABLE_OFFS and | |
68 | * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative | |
69 | * to "main_int_base". | |
70 | * | |
71 | * The "per-CPU mask/unmask" is modified using the | |
72 | * ARMADA_370_XP_INT_SET_MASK_OFFS and | |
73 | * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to | |
74 | * "per_cpu_int_base". This base address points to a special address, | |
75 | * which automatically accesses the registers of the current CPU. | |
76 | * | |
77 | * The per-CPU mask/unmask can also be adjusted using the global | |
78 | * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use | |
79 | * to configure interrupt affinity. | |
80 | * | |
81 | * Due to this model, all interrupts need to be mask/unmasked at two | |
82 | * different levels: at the global level and at the per-CPU level. | |
83 | * | |
84 | * This driver takes the following approach to deal with this: | |
85 | * | |
86 | * - For global interrupts: | |
87 | * | |
88 | * At ->map() time, a global interrupt is unmasked at the per-CPU | |
89 | * mask/unmask level. It is therefore unmasked at this level for | |
90 | * the current CPU, running the ->map() code. This allows to have | |
91 | * the interrupt unmasked at this level in non-SMP | |
92 | * configurations. In SMP configurations, the ->set_affinity() | |
93 | * callback is called, which using the | |
94 | * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask | |
95 | * for the interrupt. | |
96 | * | |
97 | * The ->mask() and ->unmask() operations only mask/unmask the | |
98 | * interrupt at the "global" level. | |
99 | * | |
100 | * So, a global interrupt is enabled at the per-CPU level as soon | |
101 | * as it is mapped. At run time, the masking/unmasking takes place | |
102 | * at the global level. | |
103 | * | |
104 | * - For per-CPU interrupts | |
105 | * | |
106 | * At ->map() time, a per-CPU interrupt is unmasked at the global | |
107 | * mask/unmask level. | |
108 | * | |
109 | * The ->mask() and ->unmask() operations mask/unmask the interrupt | |
110 | * at the per-CPU level. | |
111 | * | |
112 | * So, a per-CPU interrupt is enabled at the global level as soon | |
113 | * as it is mapped. At run time, the masking/unmasking takes place | |
114 | * at the per-CPU level. | |
115 | */ | |
9ae6f740 | 116 | |
9a234c9c | 117 | /* Registers relative to main_int_base */ |
f3e16ccd | 118 | #define ARMADA_370_XP_INT_CONTROL (0x00) |
9a234c9c | 119 | #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04) |
9ae6f740 TP |
120 | #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) |
121 | #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) | |
3202bf01 | 122 | #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) |
8cc3cfc5 | 123 | #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF |
758e8366 | 124 | #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) |
9ae6f740 | 125 | |
9a234c9c TP |
126 | /* Registers relative to per_cpu_int_base */ |
127 | #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08) | |
128 | #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c) | |
bc69b8ad | 129 | #define ARMADA_375_PPI_CAUSE (0x10) |
9a234c9c TP |
130 | #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) |
131 | #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) | |
132 | #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) | |
133 | #define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54) | |
134 | #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) | |
344e873e | 135 | |
3202bf01 GC |
136 | #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) |
137 | ||
5ec69017 TP |
138 | #define IPI_DOORBELL_START (0) |
139 | #define IPI_DOORBELL_END (8) | |
140 | #define IPI_DOORBELL_MASK 0xFF | |
31f614ed TP |
141 | #define PCI_MSI_DOORBELL_START (16) |
142 | #define PCI_MSI_DOORBELL_NR (16) | |
143 | #define PCI_MSI_DOORBELL_END (32) | |
144 | #define PCI_MSI_DOORBELL_MASK 0xFFFF0000 | |
344e873e | 145 | |
9ae6f740 TP |
146 | static void __iomem *per_cpu_int_base; |
147 | static void __iomem *main_int_base; | |
148 | static struct irq_domain *armada_370_xp_mpic_domain; | |
0f077eb5 | 149 | static u32 doorbell_mask_reg; |
5724be84 | 150 | static int parent_irq; |
31f614ed TP |
151 | #ifdef CONFIG_PCI_MSI |
152 | static struct irq_domain *armada_370_xp_msi_domain; | |
fcc392d5 | 153 | static struct irq_domain *armada_370_xp_msi_inner_domain; |
31f614ed TP |
154 | static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR); |
155 | static DEFINE_MUTEX(msi_used_lock); | |
156 | static phys_addr_t msi_doorbell_addr; | |
157 | #endif | |
9ae6f740 | 158 | |
2c299de5 EG |
159 | static inline bool is_percpu_irq(irq_hw_number_t irq) |
160 | { | |
080481f9 | 161 | if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS) |
2c299de5 | 162 | return true; |
080481f9 MR |
163 | |
164 | return false; | |
2c299de5 EG |
165 | } |
166 | ||
3202bf01 GC |
167 | /* |
168 | * In SMP mode: | |
169 | * For shared global interrupts, mask/unmask global enable bit | |
097ef18d | 170 | * For CPU interrupts, mask/unmask the calling CPU's bit |
3202bf01 | 171 | */ |
9ae6f740 TP |
172 | static void armada_370_xp_irq_mask(struct irq_data *d) |
173 | { | |
3202bf01 GC |
174 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
175 | ||
2c299de5 | 176 | if (!is_percpu_irq(hwirq)) |
3202bf01 GC |
177 | writel(hwirq, main_int_base + |
178 | ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); | |
179 | else | |
180 | writel(hwirq, per_cpu_int_base + | |
181 | ARMADA_370_XP_INT_SET_MASK_OFFS); | |
9ae6f740 TP |
182 | } |
183 | ||
184 | static void armada_370_xp_irq_unmask(struct irq_data *d) | |
185 | { | |
3202bf01 GC |
186 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
187 | ||
2c299de5 | 188 | if (!is_percpu_irq(hwirq)) |
3202bf01 GC |
189 | writel(hwirq, main_int_base + |
190 | ARMADA_370_XP_INT_SET_ENABLE_OFFS); | |
191 | else | |
192 | writel(hwirq, per_cpu_int_base + | |
193 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
9ae6f740 TP |
194 | } |
195 | ||
31f614ed TP |
196 | #ifdef CONFIG_PCI_MSI |
197 | ||
fcc392d5 | 198 | static struct irq_chip armada_370_xp_msi_irq_chip = { |
f692a172 | 199 | .name = "MPIC MSI", |
fcc392d5 TP |
200 | .irq_mask = pci_msi_mask_irq, |
201 | .irq_unmask = pci_msi_unmask_irq, | |
202 | }; | |
31f614ed | 203 | |
fcc392d5 | 204 | static struct msi_domain_info armada_370_xp_msi_domain_info = { |
a71b9412 | 205 | .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
319ec8b3 | 206 | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), |
fcc392d5 TP |
207 | .chip = &armada_370_xp_msi_irq_chip, |
208 | }; | |
31f614ed | 209 | |
fcc392d5 | 210 | static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) |
31f614ed | 211 | { |
fcc392d5 TP |
212 | msg->address_lo = lower_32_bits(msi_doorbell_addr); |
213 | msg->address_hi = upper_32_bits(msi_doorbell_addr); | |
214 | msg->data = 0xf00 | (data->hwirq + PCI_MSI_DOORBELL_START); | |
31f614ed TP |
215 | } |
216 | ||
fcc392d5 TP |
217 | static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, |
218 | const struct cpumask *mask, bool force) | |
31f614ed | 219 | { |
fcc392d5 TP |
220 | return -EINVAL; |
221 | } | |
31f614ed | 222 | |
fcc392d5 | 223 | static struct irq_chip armada_370_xp_msi_bottom_irq_chip = { |
f692a172 | 224 | .name = "MPIC MSI", |
fcc392d5 TP |
225 | .irq_compose_msi_msg = armada_370_xp_compose_msi_msg, |
226 | .irq_set_affinity = armada_370_xp_msi_set_affinity, | |
227 | }; | |
3930115e | 228 | |
fcc392d5 TP |
229 | static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, |
230 | unsigned int nr_irqs, void *args) | |
231 | { | |
a71b9412 | 232 | int hwirq, i; |
31f614ed | 233 | |
fcc392d5 | 234 | mutex_lock(&msi_used_lock); |
a71b9412 TP |
235 | |
236 | hwirq = bitmap_find_next_zero_area(msi_used, PCI_MSI_DOORBELL_NR, | |
237 | 0, nr_irqs, 0); | |
fcc392d5 TP |
238 | if (hwirq >= PCI_MSI_DOORBELL_NR) { |
239 | mutex_unlock(&msi_used_lock); | |
240 | return -ENOSPC; | |
31f614ed TP |
241 | } |
242 | ||
a71b9412 | 243 | bitmap_set(msi_used, hwirq, nr_irqs); |
fcc392d5 | 244 | mutex_unlock(&msi_used_lock); |
31f614ed | 245 | |
a71b9412 TP |
246 | for (i = 0; i < nr_irqs; i++) { |
247 | irq_domain_set_info(domain, virq + i, hwirq + i, | |
248 | &armada_370_xp_msi_bottom_irq_chip, | |
249 | domain->host_data, handle_simple_irq, | |
250 | NULL, NULL); | |
251 | } | |
31f614ed | 252 | |
fcc392d5 | 253 | return hwirq; |
31f614ed TP |
254 | } |
255 | ||
fcc392d5 TP |
256 | static void armada_370_xp_msi_free(struct irq_domain *domain, |
257 | unsigned int virq, unsigned int nr_irqs) | |
31f614ed | 258 | { |
fcc392d5 | 259 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); |
31f614ed | 260 | |
fcc392d5 | 261 | mutex_lock(&msi_used_lock); |
a71b9412 | 262 | bitmap_clear(msi_used, d->hwirq, nr_irqs); |
fcc392d5 | 263 | mutex_unlock(&msi_used_lock); |
31f614ed TP |
264 | } |
265 | ||
fcc392d5 TP |
266 | static const struct irq_domain_ops armada_370_xp_msi_domain_ops = { |
267 | .alloc = armada_370_xp_msi_alloc, | |
268 | .free = armada_370_xp_msi_free, | |
31f614ed TP |
269 | }; |
270 | ||
271 | static int armada_370_xp_msi_init(struct device_node *node, | |
272 | phys_addr_t main_int_phys_base) | |
273 | { | |
31f614ed | 274 | u32 reg; |
31f614ed TP |
275 | |
276 | msi_doorbell_addr = main_int_phys_base + | |
277 | ARMADA_370_XP_SW_TRIG_INT_OFFS; | |
278 | ||
fcc392d5 TP |
279 | armada_370_xp_msi_inner_domain = |
280 | irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR, | |
281 | &armada_370_xp_msi_domain_ops, NULL); | |
282 | if (!armada_370_xp_msi_inner_domain) | |
31f614ed TP |
283 | return -ENOMEM; |
284 | ||
31f614ed | 285 | armada_370_xp_msi_domain = |
fcc392d5 TP |
286 | pci_msi_create_irq_domain(of_node_to_fwnode(node), |
287 | &armada_370_xp_msi_domain_info, | |
288 | armada_370_xp_msi_inner_domain); | |
31f614ed | 289 | if (!armada_370_xp_msi_domain) { |
fcc392d5 | 290 | irq_domain_remove(armada_370_xp_msi_inner_domain); |
31f614ed TP |
291 | return -ENOMEM; |
292 | } | |
293 | ||
31f614ed TP |
294 | reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS) |
295 | | PCI_MSI_DOORBELL_MASK; | |
296 | ||
297 | writel(reg, per_cpu_int_base + | |
298 | ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
299 | ||
300 | /* Unmask IPI interrupt */ | |
301 | writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
302 | ||
303 | return 0; | |
304 | } | |
305 | #else | |
306 | static inline int armada_370_xp_msi_init(struct device_node *node, | |
307 | phys_addr_t main_int_phys_base) | |
308 | { | |
309 | return 0; | |
310 | } | |
311 | #endif | |
312 | ||
f02147dd MZ |
313 | static void armada_xp_mpic_perf_init(void) |
314 | { | |
315 | unsigned long cpuid = cpu_logical_map(smp_processor_id()); | |
316 | ||
317 | /* Enable Performance Counter Overflow interrupts */ | |
318 | writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), | |
319 | per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); | |
320 | } | |
321 | ||
344e873e | 322 | #ifdef CONFIG_SMP |
f02147dd MZ |
323 | static struct irq_domain *ipi_domain; |
324 | ||
325 | static void armada_370_xp_ipi_mask(struct irq_data *d) | |
326 | { | |
327 | u32 reg; | |
328 | reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
329 | reg &= ~BIT(d->hwirq); | |
330 | writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
331 | } | |
332 | ||
333 | static void armada_370_xp_ipi_unmask(struct irq_data *d) | |
334 | { | |
335 | u32 reg; | |
336 | reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
337 | reg |= BIT(d->hwirq); | |
338 | writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
339 | } | |
340 | ||
341 | static void armada_370_xp_ipi_send_mask(struct irq_data *d, | |
342 | const struct cpumask *mask) | |
343 | { | |
344 | unsigned long map = 0; | |
345 | int cpu; | |
346 | ||
347 | /* Convert our logical CPU mask into a physical one. */ | |
348 | for_each_cpu(cpu, mask) | |
349 | map |= 1 << cpu_logical_map(cpu); | |
350 | ||
351 | /* | |
352 | * Ensure that stores to Normal memory are visible to the | |
353 | * other CPUs before issuing the IPI. | |
354 | */ | |
355 | dsb(); | |
356 | ||
357 | /* submit softirq */ | |
358 | writel((map << 8) | d->hwirq, main_int_base + | |
359 | ARMADA_370_XP_SW_TRIG_INT_OFFS); | |
360 | } | |
361 | ||
362 | static void armada_370_xp_ipi_eoi(struct irq_data *d) | |
363 | { | |
364 | writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); | |
365 | } | |
366 | ||
367 | static struct irq_chip ipi_irqchip = { | |
368 | .name = "IPI", | |
369 | .irq_mask = armada_370_xp_ipi_mask, | |
370 | .irq_unmask = armada_370_xp_ipi_unmask, | |
371 | .irq_eoi = armada_370_xp_ipi_eoi, | |
372 | .ipi_send_mask = armada_370_xp_ipi_send_mask, | |
373 | }; | |
374 | ||
375 | static int armada_370_xp_ipi_alloc(struct irq_domain *d, | |
376 | unsigned int virq, | |
377 | unsigned int nr_irqs, void *args) | |
378 | { | |
379 | int i; | |
380 | ||
381 | for (i = 0; i < nr_irqs; i++) { | |
382 | irq_set_percpu_devid(virq + i); | |
383 | irq_domain_set_info(d, virq + i, i, &ipi_irqchip, | |
384 | d->host_data, | |
e52e73b7 | 385 | handle_percpu_devid_irq, |
f02147dd MZ |
386 | NULL, NULL); |
387 | } | |
388 | ||
389 | return 0; | |
390 | } | |
391 | ||
392 | static void armada_370_xp_ipi_free(struct irq_domain *d, | |
393 | unsigned int virq, | |
394 | unsigned int nr_irqs) | |
395 | { | |
396 | /* Not freeing IPIs */ | |
397 | } | |
398 | ||
399 | static const struct irq_domain_ops ipi_domain_ops = { | |
400 | .alloc = armada_370_xp_ipi_alloc, | |
401 | .free = armada_370_xp_ipi_free, | |
402 | }; | |
403 | ||
404 | static void ipi_resume(void) | |
405 | { | |
406 | int i; | |
407 | ||
408 | for (i = 0; i < IPI_DOORBELL_END; i++) { | |
409 | int irq; | |
410 | ||
411 | irq = irq_find_mapping(ipi_domain, i); | |
412 | if (irq <= 0) | |
413 | continue; | |
414 | if (irq_percpu_is_enabled(irq)) { | |
415 | struct irq_data *d; | |
416 | d = irq_domain_get_irq_data(ipi_domain, irq); | |
417 | armada_370_xp_ipi_unmask(d); | |
418 | } | |
419 | } | |
420 | } | |
421 | ||
422 | static __init void armada_xp_ipi_init(struct device_node *node) | |
423 | { | |
424 | int base_ipi; | |
425 | ||
426 | ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), | |
427 | IPI_DOORBELL_END, | |
428 | &ipi_domain_ops, NULL); | |
429 | if (WARN_ON(!ipi_domain)) | |
430 | return; | |
431 | ||
432 | irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); | |
433 | base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, IPI_DOORBELL_END, | |
434 | NUMA_NO_NODE, NULL, false, NULL); | |
435 | if (WARN_ON(!base_ipi)) | |
436 | return; | |
437 | ||
438 | set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); | |
439 | } | |
440 | ||
19e61d41 AE |
441 | static DEFINE_RAW_SPINLOCK(irq_controller_lock); |
442 | ||
344e873e GC |
443 | static int armada_xp_set_affinity(struct irq_data *d, |
444 | const struct cpumask *mask_val, bool force) | |
445 | { | |
3202bf01 | 446 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
8cc3cfc5 | 447 | unsigned long reg, mask; |
3202bf01 GC |
448 | int cpu; |
449 | ||
8cc3cfc5 TG |
450 | /* Select a single core from the affinity mask which is online */ |
451 | cpu = cpumask_any_and(mask_val, cpu_online_mask); | |
452 | mask = 1UL << cpu_logical_map(cpu); | |
3202bf01 GC |
453 | |
454 | raw_spin_lock(&irq_controller_lock); | |
3202bf01 | 455 | reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); |
8cc3cfc5 | 456 | reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask; |
3202bf01 | 457 | writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); |
3202bf01 GC |
458 | raw_spin_unlock(&irq_controller_lock); |
459 | ||
e31793a3 MZ |
460 | irq_data_update_effective_affinity(d, cpumask_of(cpu)); |
461 | ||
1dacf194 | 462 | return IRQ_SET_MASK_OK; |
344e873e | 463 | } |
9ae6f740 | 464 | |
d7df84b3 | 465 | static void armada_xp_mpic_smp_cpu_init(void) |
344e873e | 466 | { |
b73842b7 TP |
467 | u32 control; |
468 | int nr_irqs, i; | |
469 | ||
470 | control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); | |
471 | nr_irqs = (control >> 2) & 0x3ff; | |
472 | ||
473 | for (i = 0; i < nr_irqs; i++) | |
474 | writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); | |
475 | ||
f02147dd MZ |
476 | /* Disable all IPIs */ |
477 | writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
478 | ||
344e873e GC |
479 | /* Clear pending IPIs */ |
480 | writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); | |
481 | ||
344e873e GC |
482 | /* Unmask IPI interrupt */ |
483 | writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
484 | } | |
d7df84b3 | 485 | |
0fa4ce74 TP |
486 | static void armada_xp_mpic_reenable_percpu(void) |
487 | { | |
488 | unsigned int irq; | |
489 | ||
490 | /* Re-enable per-CPU interrupts that were enabled before suspend */ | |
491 | for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { | |
492 | struct irq_data *data; | |
493 | int virq; | |
494 | ||
495 | virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); | |
496 | if (virq == 0) | |
497 | continue; | |
498 | ||
499 | data = irq_get_irq_data(virq); | |
500 | ||
501 | if (!irq_percpu_is_enabled(virq)) | |
502 | continue; | |
503 | ||
504 | armada_370_xp_irq_unmask(data); | |
505 | } | |
f02147dd MZ |
506 | |
507 | ipi_resume(); | |
0fa4ce74 TP |
508 | } |
509 | ||
cb5ff2d2 | 510 | static int armada_xp_mpic_starting_cpu(unsigned int cpu) |
d7df84b3 | 511 | { |
cb5ff2d2 RC |
512 | armada_xp_mpic_perf_init(); |
513 | armada_xp_mpic_smp_cpu_init(); | |
0fa4ce74 | 514 | armada_xp_mpic_reenable_percpu(); |
cb5ff2d2 | 515 | return 0; |
d7df84b3 TP |
516 | } |
517 | ||
cb5ff2d2 | 518 | static int mpic_cascaded_starting_cpu(unsigned int cpu) |
5724be84 | 519 | { |
cb5ff2d2 | 520 | armada_xp_mpic_perf_init(); |
0fa4ce74 | 521 | armada_xp_mpic_reenable_percpu(); |
cb5ff2d2 RC |
522 | enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); |
523 | return 0; | |
5724be84 | 524 | } |
f02147dd MZ |
525 | #else |
526 | static void armada_xp_mpic_smp_cpu_init(void) {} | |
527 | static void ipi_resume(void) {} | |
528 | #endif | |
529 | ||
530 | static struct irq_chip armada_370_xp_irq_chip = { | |
531 | .name = "MPIC", | |
532 | .irq_mask = armada_370_xp_irq_mask, | |
533 | .irq_mask_ack = armada_370_xp_irq_mask, | |
534 | .irq_unmask = armada_370_xp_irq_unmask, | |
535 | #ifdef CONFIG_SMP | |
536 | .irq_set_affinity = armada_xp_set_affinity, | |
c76c15e6 | 537 | #endif |
f02147dd MZ |
538 | .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, |
539 | }; | |
540 | ||
541 | static int armada_370_xp_mpic_irq_map(struct irq_domain *h, | |
542 | unsigned int virq, irq_hw_number_t hw) | |
543 | { | |
544 | armada_370_xp_irq_mask(irq_get_irq_data(virq)); | |
545 | if (!is_percpu_irq(hw)) | |
546 | writel(hw, per_cpu_int_base + | |
547 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
548 | else | |
549 | writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); | |
550 | irq_set_status_flags(virq, IRQ_LEVEL); | |
551 | ||
552 | if (is_percpu_irq(hw)) { | |
553 | irq_set_percpu_devid(virq); | |
554 | irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, | |
555 | handle_percpu_devid_irq); | |
556 | } else { | |
557 | irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, | |
558 | handle_level_irq); | |
559 | irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); | |
560 | } | |
561 | irq_set_probe(virq); | |
562 | ||
563 | return 0; | |
564 | } | |
344e873e | 565 | |
96009736 | 566 | static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { |
9ae6f740 TP |
567 | .map = armada_370_xp_mpic_irq_map, |
568 | .xlate = irq_domain_xlate_onecell, | |
569 | }; | |
570 | ||
9b8cf779 | 571 | #ifdef CONFIG_PCI_MSI |
bc69b8ad | 572 | static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) |
9b8cf779 EG |
573 | { |
574 | u32 msimask, msinr; | |
575 | ||
576 | msimask = readl_relaxed(per_cpu_int_base + | |
577 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) | |
578 | & PCI_MSI_DOORBELL_MASK; | |
579 | ||
580 | writel(~msimask, per_cpu_int_base + | |
581 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); | |
582 | ||
583 | for (msinr = PCI_MSI_DOORBELL_START; | |
584 | msinr < PCI_MSI_DOORBELL_END; msinr++) { | |
585 | int irq; | |
586 | ||
587 | if (!(msimask & BIT(msinr))) | |
588 | continue; | |
589 | ||
e89c6a06 | 590 | if (is_chained) { |
fcc392d5 | 591 | irq = irq_find_mapping(armada_370_xp_msi_inner_domain, |
0636bab6 | 592 | msinr - PCI_MSI_DOORBELL_START); |
bc69b8ad | 593 | generic_handle_irq(irq); |
e89c6a06 | 594 | } else { |
0636bab6 | 595 | irq = msinr - PCI_MSI_DOORBELL_START; |
fcc392d5 | 596 | handle_domain_irq(armada_370_xp_msi_inner_domain, |
e89c6a06 MZ |
597 | irq, regs); |
598 | } | |
9b8cf779 EG |
599 | } |
600 | } | |
601 | #else | |
bc69b8ad | 602 | static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} |
9b8cf779 EG |
603 | #endif |
604 | ||
bd0b9ac4 | 605 | static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) |
bc69b8ad | 606 | { |
5b29264c | 607 | struct irq_chip *chip = irq_desc_get_chip(desc); |
758e8366 | 608 | unsigned long irqmap, irqn, irqsrc, cpuid; |
bc69b8ad EG |
609 | unsigned int cascade_irq; |
610 | ||
611 | chained_irq_enter(chip, desc); | |
612 | ||
613 | irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); | |
758e8366 | 614 | cpuid = cpu_logical_map(smp_processor_id()); |
bc69b8ad EG |
615 | |
616 | for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { | |
758e8366 GJ |
617 | irqsrc = readl_relaxed(main_int_base + |
618 | ARMADA_370_XP_INT_SOURCE_CTL(irqn)); | |
619 | ||
620 | /* Check if the interrupt is not masked on current CPU. | |
621 | * Test IRQ (0-1) and FIQ (8-9) mask bits. | |
622 | */ | |
623 | if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid))) | |
624 | continue; | |
625 | ||
626 | if (irqn == 1) { | |
627 | armada_370_xp_handle_msi_irq(NULL, true); | |
628 | continue; | |
629 | } | |
630 | ||
bc69b8ad EG |
631 | cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn); |
632 | generic_handle_irq(cascade_irq); | |
633 | } | |
634 | ||
635 | chained_irq_exit(chip, desc); | |
636 | } | |
637 | ||
8783dd3a | 638 | static void __exception_irq_entry |
9339d432 | 639 | armada_370_xp_handle_irq(struct pt_regs *regs) |
9ae6f740 TP |
640 | { |
641 | u32 irqstat, irqnr; | |
642 | ||
643 | do { | |
644 | irqstat = readl_relaxed(per_cpu_int_base + | |
645 | ARMADA_370_XP_CPU_INTACK_OFFS); | |
646 | irqnr = irqstat & 0x3FF; | |
647 | ||
344e873e GC |
648 | if (irqnr > 1022) |
649 | break; | |
650 | ||
31f614ed | 651 | if (irqnr > 1) { |
e89c6a06 MZ |
652 | handle_domain_irq(armada_370_xp_mpic_domain, |
653 | irqnr, regs); | |
9ae6f740 TP |
654 | continue; |
655 | } | |
31f614ed | 656 | |
31f614ed | 657 | /* MSI handling */ |
9b8cf779 | 658 | if (irqnr == 1) |
bc69b8ad | 659 | armada_370_xp_handle_msi_irq(regs, false); |
31f614ed | 660 | |
344e873e GC |
661 | #ifdef CONFIG_SMP |
662 | /* IPI Handling */ | |
663 | if (irqnr == 0) { | |
f02147dd MZ |
664 | unsigned long ipimask; |
665 | int ipi; | |
344e873e GC |
666 | |
667 | ipimask = readl_relaxed(per_cpu_int_base + | |
668 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) | |
5ec69017 | 669 | & IPI_DOORBELL_MASK; |
344e873e | 670 | |
f02147dd MZ |
671 | for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) |
672 | handle_domain_irq(ipi_domain, ipi, regs); | |
344e873e GC |
673 | } |
674 | #endif | |
9ae6f740 | 675 | |
9ae6f740 TP |
676 | } while (1); |
677 | } | |
678 | ||
0f077eb5 TP |
679 | static int armada_370_xp_mpic_suspend(void) |
680 | { | |
681 | doorbell_mask_reg = readl(per_cpu_int_base + | |
682 | ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
683 | return 0; | |
684 | } | |
685 | ||
686 | static void armada_370_xp_mpic_resume(void) | |
687 | { | |
688 | int nirqs; | |
689 | irq_hw_number_t irq; | |
690 | ||
691 | /* Re-enable interrupts */ | |
692 | nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff; | |
693 | for (irq = 0; irq < nirqs; irq++) { | |
694 | struct irq_data *data; | |
695 | int virq; | |
696 | ||
697 | virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); | |
698 | if (virq == 0) | |
699 | continue; | |
700 | ||
0fa4ce74 TP |
701 | data = irq_get_irq_data(virq); |
702 | ||
703 | if (!is_percpu_irq(irq)) { | |
704 | /* Non per-CPU interrupts */ | |
0f077eb5 TP |
705 | writel(irq, per_cpu_int_base + |
706 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
0fa4ce74 TP |
707 | if (!irqd_irq_disabled(data)) |
708 | armada_370_xp_irq_unmask(data); | |
709 | } else { | |
710 | /* Per-CPU interrupts */ | |
0f077eb5 TP |
711 | writel(irq, main_int_base + |
712 | ARMADA_370_XP_INT_SET_ENABLE_OFFS); | |
713 | ||
0fa4ce74 TP |
714 | /* |
715 | * Re-enable on the current CPU, | |
716 | * armada_xp_mpic_reenable_percpu() will take | |
717 | * care of secondary CPUs when they come up. | |
718 | */ | |
719 | if (irq_percpu_is_enabled(virq)) | |
720 | armada_370_xp_irq_unmask(data); | |
721 | } | |
0f077eb5 TP |
722 | } |
723 | ||
724 | /* Reconfigure doorbells for IPIs and MSIs */ | |
725 | writel(doorbell_mask_reg, | |
726 | per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
727 | if (doorbell_mask_reg & IPI_DOORBELL_MASK) | |
728 | writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
729 | if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK) | |
730 | writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
f02147dd MZ |
731 | |
732 | ipi_resume(); | |
0f077eb5 TP |
733 | } |
734 | ||
6c880902 | 735 | static struct syscore_ops armada_370_xp_mpic_syscore_ops = { |
0f077eb5 TP |
736 | .suspend = armada_370_xp_mpic_suspend, |
737 | .resume = armada_370_xp_mpic_resume, | |
738 | }; | |
739 | ||
b313ada8 TP |
740 | static int __init armada_370_xp_mpic_of_init(struct device_node *node, |
741 | struct device_node *parent) | |
9ae6f740 | 742 | { |
627dfcc2 | 743 | struct resource main_int_res, per_cpu_int_res; |
5724be84 | 744 | int nr_irqs, i; |
b313ada8 TP |
745 | u32 control; |
746 | ||
627dfcc2 TP |
747 | BUG_ON(of_address_to_resource(node, 0, &main_int_res)); |
748 | BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); | |
b313ada8 | 749 | |
627dfcc2 TP |
750 | BUG_ON(!request_mem_region(main_int_res.start, |
751 | resource_size(&main_int_res), | |
752 | node->full_name)); | |
753 | BUG_ON(!request_mem_region(per_cpu_int_res.start, | |
754 | resource_size(&per_cpu_int_res), | |
755 | node->full_name)); | |
756 | ||
757 | main_int_base = ioremap(main_int_res.start, | |
758 | resource_size(&main_int_res)); | |
b313ada8 | 759 | BUG_ON(!main_int_base); |
627dfcc2 TP |
760 | |
761 | per_cpu_int_base = ioremap(per_cpu_int_res.start, | |
762 | resource_size(&per_cpu_int_res)); | |
b313ada8 TP |
763 | BUG_ON(!per_cpu_int_base); |
764 | ||
765 | control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); | |
b73842b7 TP |
766 | nr_irqs = (control >> 2) & 0x3ff; |
767 | ||
768 | for (i = 0; i < nr_irqs; i++) | |
769 | writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); | |
b313ada8 TP |
770 | |
771 | armada_370_xp_mpic_domain = | |
b73842b7 | 772 | irq_domain_add_linear(node, nr_irqs, |
b313ada8 | 773 | &armada_370_xp_mpic_irq_ops, NULL); |
627dfcc2 | 774 | BUG_ON(!armada_370_xp_mpic_domain); |
96f0d93a | 775 | irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED); |
b313ada8 | 776 | |
933a24b0 | 777 | /* Setup for the boot CPU */ |
28da06df | 778 | armada_xp_mpic_perf_init(); |
b313ada8 | 779 | armada_xp_mpic_smp_cpu_init(); |
b313ada8 | 780 | |
31f614ed TP |
781 | armada_370_xp_msi_init(node, main_int_res.start); |
782 | ||
bc69b8ad EG |
783 | parent_irq = irq_of_parse_and_map(node, 0); |
784 | if (parent_irq <= 0) { | |
785 | irq_set_default_host(armada_370_xp_mpic_domain); | |
786 | set_handle_irq(armada_370_xp_handle_irq); | |
ef37d337 | 787 | #ifdef CONFIG_SMP |
f02147dd | 788 | armada_xp_ipi_init(node); |
cb5ff2d2 | 789 | cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, |
73c1b41e | 790 | "irqchip/armada/ipi:starting", |
cb5ff2d2 | 791 | armada_xp_mpic_starting_cpu, NULL); |
ef37d337 | 792 | #endif |
bc69b8ad | 793 | } else { |
5724be84 | 794 | #ifdef CONFIG_SMP |
008b69e4 | 795 | cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, |
73c1b41e | 796 | "irqchip/armada/cascade:starting", |
cb5ff2d2 | 797 | mpic_cascaded_starting_cpu, NULL); |
5724be84 | 798 | #endif |
bc69b8ad EG |
799 | irq_set_chained_handler(parent_irq, |
800 | armada_370_xp_mpic_handle_cascade_irq); | |
801 | } | |
b313ada8 | 802 | |
0f077eb5 TP |
803 | register_syscore_ops(&armada_370_xp_mpic_syscore_ops); |
804 | ||
b313ada8 | 805 | return 0; |
9ae6f740 | 806 | } |
b313ada8 | 807 | |
9339d432 | 808 | IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init); |