Commit | Line | Data |
---|---|---|
9ae6f740 TP |
1 | /* |
2 | * Marvell Armada 370 and Armada XP SoC IRQ handling | |
3 | * | |
4 | * Copyright (C) 2012 Marvell | |
5 | * | |
6 | * Lior Amsalem <alior@marvell.com> | |
7 | * Gregory CLEMENT <gregory.clement@free-electrons.com> | |
8 | * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> | |
9 | * Ben Dooks <ben.dooks@codethink.co.uk> | |
10 | * | |
11 | * This file is licensed under the terms of the GNU General Public | |
12 | * License version 2. This program is licensed "as is" without any | |
13 | * warranty of any kind, whether express or implied. | |
14 | */ | |
15 | ||
16 | #include <linux/kernel.h> | |
17 | #include <linux/module.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/irq.h> | |
20 | #include <linux/interrupt.h> | |
41a83e06 | 21 | #include <linux/irqchip.h> |
bc69b8ad | 22 | #include <linux/irqchip/chained_irq.h> |
d7df84b3 | 23 | #include <linux/cpu.h> |
9ae6f740 TP |
24 | #include <linux/io.h> |
25 | #include <linux/of_address.h> | |
26 | #include <linux/of_irq.h> | |
31f614ed | 27 | #include <linux/of_pci.h> |
9ae6f740 | 28 | #include <linux/irqdomain.h> |
31f614ed | 29 | #include <linux/slab.h> |
0f077eb5 | 30 | #include <linux/syscore_ops.h> |
31f614ed | 31 | #include <linux/msi.h> |
9ae6f740 TP |
32 | #include <asm/mach/arch.h> |
33 | #include <asm/exception.h> | |
344e873e | 34 | #include <asm/smp_plat.h> |
9339d432 TP |
35 | #include <asm/mach/irq.h> |
36 | ||
054ea4ce TP |
37 | /* |
38 | * Overall diagram of the Armada XP interrupt controller: | |
39 | * | |
40 | * To CPU 0 To CPU 1 | |
41 | * | |
42 | * /\ /\ | |
43 | * || || | |
44 | * +---------------+ +---------------+ | |
45 | * | | | | | |
46 | * | per-CPU | | per-CPU | | |
47 | * | mask/unmask | | mask/unmask | | |
48 | * | CPU0 | | CPU1 | | |
49 | * | | | | | |
50 | * +---------------+ +---------------+ | |
51 | * /\ /\ | |
52 | * || || | |
53 | * \\_______________________// | |
54 | * || | |
55 | * +-------------------+ | |
56 | * | | | |
57 | * | Global interrupt | | |
58 | * | mask/unmask | | |
59 | * | | | |
60 | * +-------------------+ | |
61 | * /\ | |
62 | * || | |
63 | * interrupt from | |
64 | * device | |
65 | * | |
66 | * The "global interrupt mask/unmask" is modified using the | |
67 | * ARMADA_370_XP_INT_SET_ENABLE_OFFS and | |
68 | * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative | |
69 | * to "main_int_base". | |
70 | * | |
71 | * The "per-CPU mask/unmask" is modified using the | |
72 | * ARMADA_370_XP_INT_SET_MASK_OFFS and | |
73 | * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to | |
74 | * "per_cpu_int_base". This base address points to a special address, | |
75 | * which automatically accesses the registers of the current CPU. | |
76 | * | |
77 | * The per-CPU mask/unmask can also be adjusted using the global | |
78 | * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use | |
79 | * to configure interrupt affinity. | |
80 | * | |
81 | * Due to this model, all interrupts need to be mask/unmasked at two | |
82 | * different levels: at the global level and at the per-CPU level. | |
83 | * | |
84 | * This driver takes the following approach to deal with this: | |
85 | * | |
86 | * - For global interrupts: | |
87 | * | |
88 | * At ->map() time, a global interrupt is unmasked at the per-CPU | |
89 | * mask/unmask level. It is therefore unmasked at this level for | |
90 | * the current CPU, running the ->map() code. This allows to have | |
91 | * the interrupt unmasked at this level in non-SMP | |
92 | * configurations. In SMP configurations, the ->set_affinity() | |
93 | * callback is called, which using the | |
94 | * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask | |
95 | * for the interrupt. | |
96 | * | |
97 | * The ->mask() and ->unmask() operations only mask/unmask the | |
98 | * interrupt at the "global" level. | |
99 | * | |
100 | * So, a global interrupt is enabled at the per-CPU level as soon | |
101 | * as it is mapped. At run time, the masking/unmasking takes place | |
102 | * at the global level. | |
103 | * | |
104 | * - For per-CPU interrupts | |
105 | * | |
106 | * At ->map() time, a per-CPU interrupt is unmasked at the global | |
107 | * mask/unmask level. | |
108 | * | |
109 | * The ->mask() and ->unmask() operations mask/unmask the interrupt | |
110 | * at the per-CPU level. | |
111 | * | |
112 | * So, a per-CPU interrupt is enabled at the global level as soon | |
113 | * as it is mapped. At run time, the masking/unmasking takes place | |
114 | * at the per-CPU level. | |
115 | */ | |
9ae6f740 | 116 | |
9a234c9c | 117 | /* Registers relative to main_int_base */ |
f3e16ccd | 118 | #define ARMADA_370_XP_INT_CONTROL (0x00) |
9a234c9c | 119 | #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04) |
9ae6f740 TP |
120 | #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) |
121 | #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) | |
3202bf01 | 122 | #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) |
8cc3cfc5 | 123 | #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF |
758e8366 | 124 | #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) |
9ae6f740 | 125 | |
9a234c9c TP |
126 | /* Registers relative to per_cpu_int_base */ |
127 | #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08) | |
128 | #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c) | |
bc69b8ad | 129 | #define ARMADA_375_PPI_CAUSE (0x10) |
9a234c9c TP |
130 | #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) |
131 | #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) | |
132 | #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) | |
133 | #define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54) | |
134 | #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) | |
344e873e | 135 | |
3202bf01 GC |
136 | #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) |
137 | ||
5ec69017 TP |
138 | #define IPI_DOORBELL_START (0) |
139 | #define IPI_DOORBELL_END (8) | |
140 | #define IPI_DOORBELL_MASK 0xFF | |
31f614ed TP |
141 | #define PCI_MSI_DOORBELL_START (16) |
142 | #define PCI_MSI_DOORBELL_NR (16) | |
143 | #define PCI_MSI_DOORBELL_END (32) | |
144 | #define PCI_MSI_DOORBELL_MASK 0xFFFF0000 | |
344e873e | 145 | |
9ae6f740 TP |
146 | static void __iomem *per_cpu_int_base; |
147 | static void __iomem *main_int_base; | |
148 | static struct irq_domain *armada_370_xp_mpic_domain; | |
0f077eb5 | 149 | static u32 doorbell_mask_reg; |
5724be84 | 150 | static int parent_irq; |
31f614ed TP |
151 | #ifdef CONFIG_PCI_MSI |
152 | static struct irq_domain *armada_370_xp_msi_domain; | |
fcc392d5 | 153 | static struct irq_domain *armada_370_xp_msi_inner_domain; |
31f614ed TP |
154 | static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR); |
155 | static DEFINE_MUTEX(msi_used_lock); | |
156 | static phys_addr_t msi_doorbell_addr; | |
157 | #endif | |
9ae6f740 | 158 | |
2c299de5 EG |
159 | static inline bool is_percpu_irq(irq_hw_number_t irq) |
160 | { | |
080481f9 | 161 | if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS) |
2c299de5 | 162 | return true; |
080481f9 MR |
163 | |
164 | return false; | |
2c299de5 EG |
165 | } |
166 | ||
3202bf01 GC |
167 | /* |
168 | * In SMP mode: | |
169 | * For shared global interrupts, mask/unmask global enable bit | |
097ef18d | 170 | * For CPU interrupts, mask/unmask the calling CPU's bit |
3202bf01 | 171 | */ |
9ae6f740 TP |
172 | static void armada_370_xp_irq_mask(struct irq_data *d) |
173 | { | |
3202bf01 GC |
174 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
175 | ||
2c299de5 | 176 | if (!is_percpu_irq(hwirq)) |
3202bf01 GC |
177 | writel(hwirq, main_int_base + |
178 | ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); | |
179 | else | |
180 | writel(hwirq, per_cpu_int_base + | |
181 | ARMADA_370_XP_INT_SET_MASK_OFFS); | |
9ae6f740 TP |
182 | } |
183 | ||
184 | static void armada_370_xp_irq_unmask(struct irq_data *d) | |
185 | { | |
3202bf01 GC |
186 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
187 | ||
2c299de5 | 188 | if (!is_percpu_irq(hwirq)) |
3202bf01 GC |
189 | writel(hwirq, main_int_base + |
190 | ARMADA_370_XP_INT_SET_ENABLE_OFFS); | |
191 | else | |
192 | writel(hwirq, per_cpu_int_base + | |
193 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
9ae6f740 TP |
194 | } |
195 | ||
31f614ed TP |
196 | #ifdef CONFIG_PCI_MSI |
197 | ||
fcc392d5 | 198 | static struct irq_chip armada_370_xp_msi_irq_chip = { |
f692a172 | 199 | .name = "MPIC MSI", |
fcc392d5 TP |
200 | .irq_mask = pci_msi_mask_irq, |
201 | .irq_unmask = pci_msi_unmask_irq, | |
202 | }; | |
31f614ed | 203 | |
fcc392d5 | 204 | static struct msi_domain_info armada_370_xp_msi_domain_info = { |
a71b9412 | 205 | .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
319ec8b3 | 206 | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), |
fcc392d5 TP |
207 | .chip = &armada_370_xp_msi_irq_chip, |
208 | }; | |
31f614ed | 209 | |
fcc392d5 | 210 | static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) |
31f614ed | 211 | { |
fcc392d5 TP |
212 | msg->address_lo = lower_32_bits(msi_doorbell_addr); |
213 | msg->address_hi = upper_32_bits(msi_doorbell_addr); | |
214 | msg->data = 0xf00 | (data->hwirq + PCI_MSI_DOORBELL_START); | |
31f614ed TP |
215 | } |
216 | ||
fcc392d5 TP |
217 | static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, |
218 | const struct cpumask *mask, bool force) | |
31f614ed | 219 | { |
fcc392d5 TP |
220 | return -EINVAL; |
221 | } | |
31f614ed | 222 | |
fcc392d5 | 223 | static struct irq_chip armada_370_xp_msi_bottom_irq_chip = { |
f692a172 | 224 | .name = "MPIC MSI", |
fcc392d5 TP |
225 | .irq_compose_msi_msg = armada_370_xp_compose_msi_msg, |
226 | .irq_set_affinity = armada_370_xp_msi_set_affinity, | |
227 | }; | |
3930115e | 228 | |
fcc392d5 TP |
229 | static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, |
230 | unsigned int nr_irqs, void *args) | |
231 | { | |
a71b9412 | 232 | int hwirq, i; |
31f614ed | 233 | |
fcc392d5 | 234 | mutex_lock(&msi_used_lock); |
a71b9412 TP |
235 | |
236 | hwirq = bitmap_find_next_zero_area(msi_used, PCI_MSI_DOORBELL_NR, | |
237 | 0, nr_irqs, 0); | |
fcc392d5 TP |
238 | if (hwirq >= PCI_MSI_DOORBELL_NR) { |
239 | mutex_unlock(&msi_used_lock); | |
240 | return -ENOSPC; | |
31f614ed TP |
241 | } |
242 | ||
a71b9412 | 243 | bitmap_set(msi_used, hwirq, nr_irqs); |
fcc392d5 | 244 | mutex_unlock(&msi_used_lock); |
31f614ed | 245 | |
a71b9412 TP |
246 | for (i = 0; i < nr_irqs; i++) { |
247 | irq_domain_set_info(domain, virq + i, hwirq + i, | |
248 | &armada_370_xp_msi_bottom_irq_chip, | |
249 | domain->host_data, handle_simple_irq, | |
250 | NULL, NULL); | |
251 | } | |
31f614ed | 252 | |
fcc392d5 | 253 | return hwirq; |
31f614ed TP |
254 | } |
255 | ||
fcc392d5 TP |
256 | static void armada_370_xp_msi_free(struct irq_domain *domain, |
257 | unsigned int virq, unsigned int nr_irqs) | |
31f614ed | 258 | { |
fcc392d5 | 259 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); |
31f614ed | 260 | |
fcc392d5 | 261 | mutex_lock(&msi_used_lock); |
a71b9412 | 262 | bitmap_clear(msi_used, d->hwirq, nr_irqs); |
fcc392d5 | 263 | mutex_unlock(&msi_used_lock); |
31f614ed TP |
264 | } |
265 | ||
fcc392d5 TP |
266 | static const struct irq_domain_ops armada_370_xp_msi_domain_ops = { |
267 | .alloc = armada_370_xp_msi_alloc, | |
268 | .free = armada_370_xp_msi_free, | |
31f614ed TP |
269 | }; |
270 | ||
271 | static int armada_370_xp_msi_init(struct device_node *node, | |
272 | phys_addr_t main_int_phys_base) | |
273 | { | |
31f614ed | 274 | u32 reg; |
31f614ed TP |
275 | |
276 | msi_doorbell_addr = main_int_phys_base + | |
277 | ARMADA_370_XP_SW_TRIG_INT_OFFS; | |
278 | ||
fcc392d5 TP |
279 | armada_370_xp_msi_inner_domain = |
280 | irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR, | |
281 | &armada_370_xp_msi_domain_ops, NULL); | |
282 | if (!armada_370_xp_msi_inner_domain) | |
31f614ed TP |
283 | return -ENOMEM; |
284 | ||
31f614ed | 285 | armada_370_xp_msi_domain = |
fcc392d5 TP |
286 | pci_msi_create_irq_domain(of_node_to_fwnode(node), |
287 | &armada_370_xp_msi_domain_info, | |
288 | armada_370_xp_msi_inner_domain); | |
31f614ed | 289 | if (!armada_370_xp_msi_domain) { |
fcc392d5 | 290 | irq_domain_remove(armada_370_xp_msi_inner_domain); |
31f614ed TP |
291 | return -ENOMEM; |
292 | } | |
293 | ||
31f614ed TP |
294 | reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS) |
295 | | PCI_MSI_DOORBELL_MASK; | |
296 | ||
297 | writel(reg, per_cpu_int_base + | |
298 | ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
299 | ||
300 | /* Unmask IPI interrupt */ | |
301 | writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
302 | ||
303 | return 0; | |
304 | } | |
305 | #else | |
306 | static inline int armada_370_xp_msi_init(struct device_node *node, | |
307 | phys_addr_t main_int_phys_base) | |
308 | { | |
309 | return 0; | |
310 | } | |
311 | #endif | |
312 | ||
344e873e | 313 | #ifdef CONFIG_SMP |
19e61d41 AE |
314 | static DEFINE_RAW_SPINLOCK(irq_controller_lock); |
315 | ||
344e873e GC |
316 | static int armada_xp_set_affinity(struct irq_data *d, |
317 | const struct cpumask *mask_val, bool force) | |
318 | { | |
3202bf01 | 319 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
8cc3cfc5 | 320 | unsigned long reg, mask; |
3202bf01 GC |
321 | int cpu; |
322 | ||
8cc3cfc5 TG |
323 | /* Select a single core from the affinity mask which is online */ |
324 | cpu = cpumask_any_and(mask_val, cpu_online_mask); | |
325 | mask = 1UL << cpu_logical_map(cpu); | |
3202bf01 GC |
326 | |
327 | raw_spin_lock(&irq_controller_lock); | |
3202bf01 | 328 | reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); |
8cc3cfc5 | 329 | reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask; |
3202bf01 | 330 | writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); |
3202bf01 GC |
331 | raw_spin_unlock(&irq_controller_lock); |
332 | ||
e31793a3 MZ |
333 | irq_data_update_effective_affinity(d, cpumask_of(cpu)); |
334 | ||
1dacf194 | 335 | return IRQ_SET_MASK_OK; |
344e873e GC |
336 | } |
337 | #endif | |
338 | ||
9ae6f740 | 339 | static struct irq_chip armada_370_xp_irq_chip = { |
f692a172 | 340 | .name = "MPIC", |
9ae6f740 TP |
341 | .irq_mask = armada_370_xp_irq_mask, |
342 | .irq_mask_ack = armada_370_xp_irq_mask, | |
343 | .irq_unmask = armada_370_xp_irq_unmask, | |
344e873e GC |
344 | #ifdef CONFIG_SMP |
345 | .irq_set_affinity = armada_xp_set_affinity, | |
346 | #endif | |
0d8e1d80 | 347 | .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, |
9ae6f740 TP |
348 | }; |
349 | ||
350 | static int armada_370_xp_mpic_irq_map(struct irq_domain *h, | |
351 | unsigned int virq, irq_hw_number_t hw) | |
352 | { | |
353 | armada_370_xp_irq_mask(irq_get_irq_data(virq)); | |
2c299de5 | 354 | if (!is_percpu_irq(hw)) |
600468d0 GC |
355 | writel(hw, per_cpu_int_base + |
356 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
357 | else | |
358 | writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); | |
9ae6f740 | 359 | irq_set_status_flags(virq, IRQ_LEVEL); |
3a6f08a3 | 360 | |
2c299de5 | 361 | if (is_percpu_irq(hw)) { |
3a6f08a3 GC |
362 | irq_set_percpu_devid(virq); |
363 | irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, | |
364 | handle_percpu_devid_irq); | |
3a6f08a3 GC |
365 | } else { |
366 | irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, | |
367 | handle_level_irq); | |
e31793a3 | 368 | irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); |
3a6f08a3 | 369 | } |
d17cab44 | 370 | irq_set_probe(virq); |
9ae6f740 TP |
371 | |
372 | return 0; | |
373 | } | |
374 | ||
d7df84b3 | 375 | static void armada_xp_mpic_smp_cpu_init(void) |
344e873e | 376 | { |
b73842b7 TP |
377 | u32 control; |
378 | int nr_irqs, i; | |
379 | ||
380 | control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); | |
381 | nr_irqs = (control >> 2) & 0x3ff; | |
382 | ||
383 | for (i = 0; i < nr_irqs; i++) | |
384 | writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); | |
385 | ||
344e873e GC |
386 | /* Clear pending IPIs */ |
387 | writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); | |
388 | ||
389 | /* Enable first 8 IPIs */ | |
5ec69017 | 390 | writel(IPI_DOORBELL_MASK, per_cpu_int_base + |
344e873e GC |
391 | ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
392 | ||
393 | /* Unmask IPI interrupt */ | |
394 | writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
395 | } | |
d7df84b3 | 396 | |
28da06df MR |
397 | static void armada_xp_mpic_perf_init(void) |
398 | { | |
399 | unsigned long cpuid = cpu_logical_map(smp_processor_id()); | |
400 | ||
401 | /* Enable Performance Counter Overflow interrupts */ | |
402 | writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), | |
403 | per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); | |
404 | } | |
405 | ||
933a24b0 EG |
406 | #ifdef CONFIG_SMP |
407 | static void armada_mpic_send_doorbell(const struct cpumask *mask, | |
408 | unsigned int irq) | |
409 | { | |
410 | int cpu; | |
411 | unsigned long map = 0; | |
412 | ||
413 | /* Convert our logical CPU mask into a physical one. */ | |
414 | for_each_cpu(cpu, mask) | |
415 | map |= 1 << cpu_logical_map(cpu); | |
416 | ||
417 | /* | |
418 | * Ensure that stores to Normal memory are visible to the | |
419 | * other CPUs before issuing the IPI. | |
420 | */ | |
421 | dsb(); | |
422 | ||
423 | /* submit softirq */ | |
424 | writel((map << 8) | irq, main_int_base + | |
425 | ARMADA_370_XP_SW_TRIG_INT_OFFS); | |
426 | } | |
427 | ||
0fa4ce74 TP |
428 | static void armada_xp_mpic_reenable_percpu(void) |
429 | { | |
430 | unsigned int irq; | |
431 | ||
432 | /* Re-enable per-CPU interrupts that were enabled before suspend */ | |
433 | for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { | |
434 | struct irq_data *data; | |
435 | int virq; | |
436 | ||
437 | virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); | |
438 | if (virq == 0) | |
439 | continue; | |
440 | ||
441 | data = irq_get_irq_data(virq); | |
442 | ||
443 | if (!irq_percpu_is_enabled(virq)) | |
444 | continue; | |
445 | ||
446 | armada_370_xp_irq_unmask(data); | |
447 | } | |
448 | } | |
449 | ||
cb5ff2d2 | 450 | static int armada_xp_mpic_starting_cpu(unsigned int cpu) |
d7df84b3 | 451 | { |
cb5ff2d2 RC |
452 | armada_xp_mpic_perf_init(); |
453 | armada_xp_mpic_smp_cpu_init(); | |
0fa4ce74 | 454 | armada_xp_mpic_reenable_percpu(); |
cb5ff2d2 | 455 | return 0; |
d7df84b3 TP |
456 | } |
457 | ||
cb5ff2d2 | 458 | static int mpic_cascaded_starting_cpu(unsigned int cpu) |
5724be84 | 459 | { |
cb5ff2d2 | 460 | armada_xp_mpic_perf_init(); |
0fa4ce74 | 461 | armada_xp_mpic_reenable_percpu(); |
cb5ff2d2 RC |
462 | enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); |
463 | return 0; | |
5724be84 | 464 | } |
c76c15e6 | 465 | #endif |
344e873e | 466 | |
96009736 | 467 | static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { |
9ae6f740 TP |
468 | .map = armada_370_xp_mpic_irq_map, |
469 | .xlate = irq_domain_xlate_onecell, | |
470 | }; | |
471 | ||
9b8cf779 | 472 | #ifdef CONFIG_PCI_MSI |
bc69b8ad | 473 | static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) |
9b8cf779 EG |
474 | { |
475 | u32 msimask, msinr; | |
476 | ||
477 | msimask = readl_relaxed(per_cpu_int_base + | |
478 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) | |
479 | & PCI_MSI_DOORBELL_MASK; | |
480 | ||
481 | writel(~msimask, per_cpu_int_base + | |
482 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); | |
483 | ||
484 | for (msinr = PCI_MSI_DOORBELL_START; | |
485 | msinr < PCI_MSI_DOORBELL_END; msinr++) { | |
486 | int irq; | |
487 | ||
488 | if (!(msimask & BIT(msinr))) | |
489 | continue; | |
490 | ||
e89c6a06 | 491 | if (is_chained) { |
fcc392d5 | 492 | irq = irq_find_mapping(armada_370_xp_msi_inner_domain, |
0636bab6 | 493 | msinr - PCI_MSI_DOORBELL_START); |
bc69b8ad | 494 | generic_handle_irq(irq); |
e89c6a06 | 495 | } else { |
0636bab6 | 496 | irq = msinr - PCI_MSI_DOORBELL_START; |
fcc392d5 | 497 | handle_domain_irq(armada_370_xp_msi_inner_domain, |
e89c6a06 MZ |
498 | irq, regs); |
499 | } | |
9b8cf779 EG |
500 | } |
501 | } | |
502 | #else | |
bc69b8ad | 503 | static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} |
9b8cf779 EG |
504 | #endif |
505 | ||
bd0b9ac4 | 506 | static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) |
bc69b8ad | 507 | { |
5b29264c | 508 | struct irq_chip *chip = irq_desc_get_chip(desc); |
758e8366 | 509 | unsigned long irqmap, irqn, irqsrc, cpuid; |
bc69b8ad EG |
510 | unsigned int cascade_irq; |
511 | ||
512 | chained_irq_enter(chip, desc); | |
513 | ||
514 | irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); | |
758e8366 | 515 | cpuid = cpu_logical_map(smp_processor_id()); |
bc69b8ad EG |
516 | |
517 | for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { | |
758e8366 GJ |
518 | irqsrc = readl_relaxed(main_int_base + |
519 | ARMADA_370_XP_INT_SOURCE_CTL(irqn)); | |
520 | ||
521 | /* Check if the interrupt is not masked on current CPU. | |
522 | * Test IRQ (0-1) and FIQ (8-9) mask bits. | |
523 | */ | |
524 | if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid))) | |
525 | continue; | |
526 | ||
527 | if (irqn == 1) { | |
528 | armada_370_xp_handle_msi_irq(NULL, true); | |
529 | continue; | |
530 | } | |
531 | ||
bc69b8ad EG |
532 | cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn); |
533 | generic_handle_irq(cascade_irq); | |
534 | } | |
535 | ||
536 | chained_irq_exit(chip, desc); | |
537 | } | |
538 | ||
8783dd3a | 539 | static void __exception_irq_entry |
9339d432 | 540 | armada_370_xp_handle_irq(struct pt_regs *regs) |
9ae6f740 TP |
541 | { |
542 | u32 irqstat, irqnr; | |
543 | ||
544 | do { | |
545 | irqstat = readl_relaxed(per_cpu_int_base + | |
546 | ARMADA_370_XP_CPU_INTACK_OFFS); | |
547 | irqnr = irqstat & 0x3FF; | |
548 | ||
344e873e GC |
549 | if (irqnr > 1022) |
550 | break; | |
551 | ||
31f614ed | 552 | if (irqnr > 1) { |
e89c6a06 MZ |
553 | handle_domain_irq(armada_370_xp_mpic_domain, |
554 | irqnr, regs); | |
9ae6f740 TP |
555 | continue; |
556 | } | |
31f614ed | 557 | |
31f614ed | 558 | /* MSI handling */ |
9b8cf779 | 559 | if (irqnr == 1) |
bc69b8ad | 560 | armada_370_xp_handle_msi_irq(regs, false); |
31f614ed | 561 | |
344e873e GC |
562 | #ifdef CONFIG_SMP |
563 | /* IPI Handling */ | |
564 | if (irqnr == 0) { | |
565 | u32 ipimask, ipinr; | |
566 | ||
567 | ipimask = readl_relaxed(per_cpu_int_base + | |
568 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) | |
5ec69017 | 569 | & IPI_DOORBELL_MASK; |
344e873e | 570 | |
a6f089e9 | 571 | writel(~ipimask, per_cpu_int_base + |
344e873e GC |
572 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); |
573 | ||
574 | /* Handle all pending doorbells */ | |
5ec69017 TP |
575 | for (ipinr = IPI_DOORBELL_START; |
576 | ipinr < IPI_DOORBELL_END; ipinr++) { | |
344e873e GC |
577 | if (ipimask & (0x1 << ipinr)) |
578 | handle_IPI(ipinr, regs); | |
579 | } | |
580 | continue; | |
581 | } | |
582 | #endif | |
9ae6f740 | 583 | |
9ae6f740 TP |
584 | } while (1); |
585 | } | |
586 | ||
0f077eb5 TP |
587 | static int armada_370_xp_mpic_suspend(void) |
588 | { | |
589 | doorbell_mask_reg = readl(per_cpu_int_base + | |
590 | ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
591 | return 0; | |
592 | } | |
593 | ||
594 | static void armada_370_xp_mpic_resume(void) | |
595 | { | |
596 | int nirqs; | |
597 | irq_hw_number_t irq; | |
598 | ||
599 | /* Re-enable interrupts */ | |
600 | nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff; | |
601 | for (irq = 0; irq < nirqs; irq++) { | |
602 | struct irq_data *data; | |
603 | int virq; | |
604 | ||
605 | virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); | |
606 | if (virq == 0) | |
607 | continue; | |
608 | ||
0fa4ce74 TP |
609 | data = irq_get_irq_data(virq); |
610 | ||
611 | if (!is_percpu_irq(irq)) { | |
612 | /* Non per-CPU interrupts */ | |
0f077eb5 TP |
613 | writel(irq, per_cpu_int_base + |
614 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
0fa4ce74 TP |
615 | if (!irqd_irq_disabled(data)) |
616 | armada_370_xp_irq_unmask(data); | |
617 | } else { | |
618 | /* Per-CPU interrupts */ | |
0f077eb5 TP |
619 | writel(irq, main_int_base + |
620 | ARMADA_370_XP_INT_SET_ENABLE_OFFS); | |
621 | ||
0fa4ce74 TP |
622 | /* |
623 | * Re-enable on the current CPU, | |
624 | * armada_xp_mpic_reenable_percpu() will take | |
625 | * care of secondary CPUs when they come up. | |
626 | */ | |
627 | if (irq_percpu_is_enabled(virq)) | |
628 | armada_370_xp_irq_unmask(data); | |
629 | } | |
0f077eb5 TP |
630 | } |
631 | ||
632 | /* Reconfigure doorbells for IPIs and MSIs */ | |
633 | writel(doorbell_mask_reg, | |
634 | per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); | |
635 | if (doorbell_mask_reg & IPI_DOORBELL_MASK) | |
636 | writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
637 | if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK) | |
638 | writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); | |
639 | } | |
640 | ||
6c880902 | 641 | static struct syscore_ops armada_370_xp_mpic_syscore_ops = { |
0f077eb5 TP |
642 | .suspend = armada_370_xp_mpic_suspend, |
643 | .resume = armada_370_xp_mpic_resume, | |
644 | }; | |
645 | ||
b313ada8 TP |
646 | static int __init armada_370_xp_mpic_of_init(struct device_node *node, |
647 | struct device_node *parent) | |
9ae6f740 | 648 | { |
627dfcc2 | 649 | struct resource main_int_res, per_cpu_int_res; |
5724be84 | 650 | int nr_irqs, i; |
b313ada8 TP |
651 | u32 control; |
652 | ||
627dfcc2 TP |
653 | BUG_ON(of_address_to_resource(node, 0, &main_int_res)); |
654 | BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); | |
b313ada8 | 655 | |
627dfcc2 TP |
656 | BUG_ON(!request_mem_region(main_int_res.start, |
657 | resource_size(&main_int_res), | |
658 | node->full_name)); | |
659 | BUG_ON(!request_mem_region(per_cpu_int_res.start, | |
660 | resource_size(&per_cpu_int_res), | |
661 | node->full_name)); | |
662 | ||
663 | main_int_base = ioremap(main_int_res.start, | |
664 | resource_size(&main_int_res)); | |
b313ada8 | 665 | BUG_ON(!main_int_base); |
627dfcc2 TP |
666 | |
667 | per_cpu_int_base = ioremap(per_cpu_int_res.start, | |
668 | resource_size(&per_cpu_int_res)); | |
b313ada8 TP |
669 | BUG_ON(!per_cpu_int_base); |
670 | ||
671 | control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); | |
b73842b7 TP |
672 | nr_irqs = (control >> 2) & 0x3ff; |
673 | ||
674 | for (i = 0; i < nr_irqs; i++) | |
675 | writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); | |
b313ada8 TP |
676 | |
677 | armada_370_xp_mpic_domain = | |
b73842b7 | 678 | irq_domain_add_linear(node, nr_irqs, |
b313ada8 | 679 | &armada_370_xp_mpic_irq_ops, NULL); |
627dfcc2 | 680 | BUG_ON(!armada_370_xp_mpic_domain); |
96f0d93a | 681 | irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED); |
b313ada8 | 682 | |
933a24b0 | 683 | /* Setup for the boot CPU */ |
28da06df | 684 | armada_xp_mpic_perf_init(); |
b313ada8 | 685 | armada_xp_mpic_smp_cpu_init(); |
b313ada8 | 686 | |
31f614ed TP |
687 | armada_370_xp_msi_init(node, main_int_res.start); |
688 | ||
bc69b8ad EG |
689 | parent_irq = irq_of_parse_and_map(node, 0); |
690 | if (parent_irq <= 0) { | |
691 | irq_set_default_host(armada_370_xp_mpic_domain); | |
692 | set_handle_irq(armada_370_xp_handle_irq); | |
ef37d337 TP |
693 | #ifdef CONFIG_SMP |
694 | set_smp_cross_call(armada_mpic_send_doorbell); | |
cb5ff2d2 | 695 | cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, |
73c1b41e | 696 | "irqchip/armada/ipi:starting", |
cb5ff2d2 | 697 | armada_xp_mpic_starting_cpu, NULL); |
ef37d337 | 698 | #endif |
bc69b8ad | 699 | } else { |
5724be84 | 700 | #ifdef CONFIG_SMP |
008b69e4 | 701 | cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, |
73c1b41e | 702 | "irqchip/armada/cascade:starting", |
cb5ff2d2 | 703 | mpic_cascaded_starting_cpu, NULL); |
5724be84 | 704 | #endif |
bc69b8ad EG |
705 | irq_set_chained_handler(parent_irq, |
706 | armada_370_xp_mpic_handle_cascade_irq); | |
707 | } | |
b313ada8 | 708 | |
0f077eb5 TP |
709 | register_syscore_ops(&armada_370_xp_mpic_syscore_ops); |
710 | ||
b313ada8 | 711 | return 0; |
9ae6f740 | 712 | } |
b313ada8 | 713 | |
9339d432 | 714 | IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init); |