Commit | Line | Data |
---|---|---|
1e33888f AR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * PCIe host bridge driver for Apple system-on-chips. | |
4 | * | |
5 | * The HW is ECAM compliant, so once the controller is initialized, | |
6 | * the driver mostly deals MSI mapping and handling of per-port | |
7 | * interrupts (INTx, management and error signals). | |
8 | * | |
9 | * Initialization requires enabling power and clocks, along with a | |
10 | * number of register pokes. | |
11 | * | |
12 | * Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io> | |
13 | * Copyright (C) 2021 Google LLC | |
14 | * Copyright (C) 2021 Corellium LLC | |
15 | * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org> | |
16 | * | |
17 | * Author: Alyssa Rosenzweig <alyssa@rosenzweig.io> | |
18 | * Author: Marc Zyngier <maz@kernel.org> | |
19 | */ | |
20 | ||
21 | #include <linux/gpio/consumer.h> | |
22 | #include <linux/kernel.h> | |
23 | #include <linux/iopoll.h> | |
d8fcbe52 | 24 | #include <linux/irqchip/chained_irq.h> |
1e33888f | 25 | #include <linux/irqdomain.h> |
468c8d52 | 26 | #include <linux/list.h> |
1e33888f AR |
27 | #include <linux/module.h> |
28 | #include <linux/msi.h> | |
468c8d52 | 29 | #include <linux/notifier.h> |
1e33888f AR |
30 | #include <linux/of_irq.h> |
31 | #include <linux/pci-ecam.h> | |
32 | ||
33 | #define CORE_RC_PHYIF_CTL 0x00024 | |
34 | #define CORE_RC_PHYIF_CTL_RUN BIT(0) | |
35 | #define CORE_RC_PHYIF_STAT 0x00028 | |
36 | #define CORE_RC_PHYIF_STAT_REFCLK BIT(4) | |
37 | #define CORE_RC_CTL 0x00050 | |
38 | #define CORE_RC_CTL_RUN BIT(0) | |
39 | #define CORE_RC_STAT 0x00058 | |
40 | #define CORE_RC_STAT_READY BIT(0) | |
41 | #define CORE_FABRIC_STAT 0x04000 | |
42 | #define CORE_FABRIC_STAT_MASK 0x001F001F | |
43 | #define CORE_LANE_CFG(port) (0x84000 + 0x4000 * (port)) | |
44 | #define CORE_LANE_CFG_REFCLK0REQ BIT(0) | |
75d36df6 | 45 | #define CORE_LANE_CFG_REFCLK1REQ BIT(1) |
1e33888f | 46 | #define CORE_LANE_CFG_REFCLK0ACK BIT(2) |
75d36df6 | 47 | #define CORE_LANE_CFG_REFCLK1ACK BIT(3) |
1e33888f AR |
48 | #define CORE_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) |
49 | #define CORE_LANE_CTL(port) (0x84004 + 0x4000 * (port)) | |
50 | #define CORE_LANE_CTL_CFGACC BIT(15) | |
51 | ||
52 | #define PORT_LTSSMCTL 0x00080 | |
53 | #define PORT_LTSSMCTL_START BIT(0) | |
54 | #define PORT_INTSTAT 0x00100 | |
55 | #define PORT_INT_TUNNEL_ERR 31 | |
56 | #define PORT_INT_CPL_TIMEOUT 23 | |
57 | #define PORT_INT_RID2SID_MAPERR 22 | |
58 | #define PORT_INT_CPL_ABORT 21 | |
59 | #define PORT_INT_MSI_BAD_DATA 19 | |
60 | #define PORT_INT_MSI_ERR 18 | |
61 | #define PORT_INT_REQADDR_GT32 17 | |
62 | #define PORT_INT_AF_TIMEOUT 15 | |
63 | #define PORT_INT_LINK_DOWN 14 | |
64 | #define PORT_INT_LINK_UP 12 | |
65 | #define PORT_INT_LINK_BWMGMT 11 | |
66 | #define PORT_INT_AER_MASK (15 << 4) | |
67 | #define PORT_INT_PORT_ERR 4 | |
68 | #define PORT_INT_INTx(i) i | |
69 | #define PORT_INT_INTx_MASK 15 | |
70 | #define PORT_INTMSK 0x00104 | |
71 | #define PORT_INTMSKSET 0x00108 | |
72 | #define PORT_INTMSKCLR 0x0010c | |
73 | #define PORT_MSICFG 0x00124 | |
74 | #define PORT_MSICFG_EN BIT(0) | |
75 | #define PORT_MSICFG_L2MSINUM_SHIFT 4 | |
76 | #define PORT_MSIBASE 0x00128 | |
77 | #define PORT_MSIBASE_1_SHIFT 16 | |
78 | #define PORT_MSIADDR 0x00168 | |
79 | #define PORT_LINKSTS 0x00208 | |
80 | #define PORT_LINKSTS_UP BIT(0) | |
81 | #define PORT_LINKSTS_BUSY BIT(2) | |
82 | #define PORT_LINKCMDSTS 0x00210 | |
83 | #define PORT_OUTS_NPREQS 0x00284 | |
84 | #define PORT_OUTS_NPREQS_REQ BIT(24) | |
85 | #define PORT_OUTS_NPREQS_CPL BIT(16) | |
86 | #define PORT_RXWR_FIFO 0x00288 | |
87 | #define PORT_RXWR_FIFO_HDR GENMASK(15, 10) | |
88 | #define PORT_RXWR_FIFO_DATA GENMASK(9, 0) | |
89 | #define PORT_RXRD_FIFO 0x0028C | |
90 | #define PORT_RXRD_FIFO_REQ GENMASK(6, 0) | |
91 | #define PORT_OUTS_CPLS 0x00290 | |
92 | #define PORT_OUTS_CPLS_SHRD GENMASK(14, 8) | |
93 | #define PORT_OUTS_CPLS_WAIT GENMASK(6, 0) | |
94 | #define PORT_APPCLK 0x00800 | |
95 | #define PORT_APPCLK_EN BIT(0) | |
96 | #define PORT_APPCLK_CGDIS BIT(8) | |
97 | #define PORT_STATUS 0x00804 | |
98 | #define PORT_STATUS_READY BIT(0) | |
99 | #define PORT_REFCLK 0x00810 | |
100 | #define PORT_REFCLK_EN BIT(0) | |
101 | #define PORT_REFCLK_CGDIS BIT(8) | |
102 | #define PORT_PERST 0x00814 | |
103 | #define PORT_PERST_OFF BIT(0) | |
104 | #define PORT_RID2SID(i16) (0x00828 + 4 * (i16)) | |
105 | #define PORT_RID2SID_VALID BIT(31) | |
106 | #define PORT_RID2SID_SID_SHIFT 16 | |
107 | #define PORT_RID2SID_BUS_SHIFT 8 | |
108 | #define PORT_RID2SID_DEV_SHIFT 3 | |
109 | #define PORT_RID2SID_FUNC_SHIFT 0 | |
110 | #define PORT_OUTS_PREQS_HDR 0x00980 | |
111 | #define PORT_OUTS_PREQS_HDR_MASK GENMASK(9, 0) | |
112 | #define PORT_OUTS_PREQS_DATA 0x00984 | |
113 | #define PORT_OUTS_PREQS_DATA_MASK GENMASK(15, 0) | |
114 | #define PORT_TUNCTRL 0x00988 | |
115 | #define PORT_TUNCTRL_PERST_ON BIT(0) | |
116 | #define PORT_TUNCTRL_PERST_ACK_REQ BIT(1) | |
117 | #define PORT_TUNSTAT 0x0098c | |
118 | #define PORT_TUNSTAT_PERST_ON BIT(0) | |
119 | #define PORT_TUNSTAT_PERST_ACK_PEND BIT(1) | |
120 | #define PORT_PREFMEM_ENABLE 0x00994 | |
121 | ||
468c8d52 MZ |
122 | #define MAX_RID2SID 64 |
123 | ||
476c41ed MZ |
124 | /* |
125 | * The doorbell address is set to 0xfffff000, which by convention | |
126 | * matches what MacOS does, and it is possible to use any other | |
127 | * address (in the bottom 4GB, as the base register is only 32bit). | |
946d619f MZ |
128 | * However, it has to be excluded from the IOVA range, and the DART |
129 | * driver has to know about it. | |
476c41ed | 130 | */ |
946d619f | 131 | #define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR |
476c41ed | 132 | |
1e33888f | 133 | struct apple_pcie { |
476c41ed | 134 | struct mutex lock; |
1e33888f AR |
135 | struct device *dev; |
136 | void __iomem *base; | |
476c41ed MZ |
137 | struct irq_domain *domain; |
138 | unsigned long *bitmap; | |
468c8d52 | 139 | struct list_head ports; |
d8fcbe52 | 140 | struct completion event; |
476c41ed MZ |
141 | struct irq_fwspec fwspec; |
142 | u32 nvecs; | |
1e33888f AR |
143 | }; |
144 | ||
145 | struct apple_pcie_port { | |
146 | struct apple_pcie *pcie; | |
147 | struct device_node *np; | |
148 | void __iomem *base; | |
d8fcbe52 | 149 | struct irq_domain *domain; |
468c8d52 MZ |
150 | struct list_head entry; |
151 | DECLARE_BITMAP(sid_map, MAX_RID2SID); | |
152 | int sid_map_sz; | |
1e33888f AR |
153 | int idx; |
154 | }; | |
155 | ||
156 | static void rmw_set(u32 set, void __iomem *addr) | |
157 | { | |
158 | writel_relaxed(readl_relaxed(addr) | set, addr); | |
159 | } | |
160 | ||
1512f908 AR |
161 | static void rmw_clear(u32 clr, void __iomem *addr) |
162 | { | |
163 | writel_relaxed(readl_relaxed(addr) & ~clr, addr); | |
164 | } | |
165 | ||
476c41ed MZ |
166 | static void apple_msi_top_irq_mask(struct irq_data *d) |
167 | { | |
168 | pci_msi_mask_irq(d); | |
169 | irq_chip_mask_parent(d); | |
170 | } | |
171 | ||
172 | static void apple_msi_top_irq_unmask(struct irq_data *d) | |
173 | { | |
174 | pci_msi_unmask_irq(d); | |
175 | irq_chip_unmask_parent(d); | |
176 | } | |
177 | ||
178 | static struct irq_chip apple_msi_top_chip = { | |
179 | .name = "PCIe MSI", | |
180 | .irq_mask = apple_msi_top_irq_mask, | |
181 | .irq_unmask = apple_msi_top_irq_unmask, | |
182 | .irq_eoi = irq_chip_eoi_parent, | |
183 | .irq_set_affinity = irq_chip_set_affinity_parent, | |
184 | .irq_set_type = irq_chip_set_type_parent, | |
185 | }; | |
186 | ||
187 | static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) | |
188 | { | |
189 | msg->address_hi = upper_32_bits(DOORBELL_ADDR); | |
190 | msg->address_lo = lower_32_bits(DOORBELL_ADDR); | |
191 | msg->data = data->hwirq; | |
192 | } | |
193 | ||
194 | static struct irq_chip apple_msi_bottom_chip = { | |
195 | .name = "MSI", | |
196 | .irq_mask = irq_chip_mask_parent, | |
197 | .irq_unmask = irq_chip_unmask_parent, | |
198 | .irq_eoi = irq_chip_eoi_parent, | |
199 | .irq_set_affinity = irq_chip_set_affinity_parent, | |
200 | .irq_set_type = irq_chip_set_type_parent, | |
201 | .irq_compose_msi_msg = apple_msi_compose_msg, | |
202 | }; | |
203 | ||
204 | static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, | |
205 | unsigned int nr_irqs, void *args) | |
206 | { | |
207 | struct apple_pcie *pcie = domain->host_data; | |
208 | struct irq_fwspec fwspec = pcie->fwspec; | |
209 | unsigned int i; | |
210 | int ret, hwirq; | |
211 | ||
212 | mutex_lock(&pcie->lock); | |
213 | ||
214 | hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs, | |
215 | order_base_2(nr_irqs)); | |
216 | ||
217 | mutex_unlock(&pcie->lock); | |
218 | ||
219 | if (hwirq < 0) | |
220 | return -ENOSPC; | |
221 | ||
89eb1681 | 222 | fwspec.param[fwspec.param_count - 2] += hwirq; |
476c41ed MZ |
223 | |
224 | ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); | |
225 | if (ret) | |
226 | return ret; | |
227 | ||
228 | for (i = 0; i < nr_irqs; i++) { | |
229 | irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, | |
230 | &apple_msi_bottom_chip, | |
231 | domain->host_data); | |
232 | } | |
233 | ||
234 | return 0; | |
235 | } | |
236 | ||
237 | static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, | |
238 | unsigned int nr_irqs) | |
239 | { | |
240 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); | |
241 | struct apple_pcie *pcie = domain->host_data; | |
242 | ||
243 | mutex_lock(&pcie->lock); | |
244 | ||
245 | bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs)); | |
246 | ||
247 | mutex_unlock(&pcie->lock); | |
248 | } | |
249 | ||
250 | static const struct irq_domain_ops apple_msi_domain_ops = { | |
251 | .alloc = apple_msi_domain_alloc, | |
252 | .free = apple_msi_domain_free, | |
253 | }; | |
254 | ||
255 | static struct msi_domain_info apple_msi_info = { | |
256 | .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | | |
257 | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), | |
258 | .chip = &apple_msi_top_chip, | |
259 | }; | |
260 | ||
d8fcbe52 MZ |
261 | static void apple_port_irq_mask(struct irq_data *data) |
262 | { | |
263 | struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); | |
264 | ||
265 | writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKSET); | |
266 | } | |
267 | ||
268 | static void apple_port_irq_unmask(struct irq_data *data) | |
269 | { | |
270 | struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); | |
271 | ||
272 | writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKCLR); | |
273 | } | |
274 | ||
275 | static bool hwirq_is_intx(unsigned int hwirq) | |
276 | { | |
277 | return BIT(hwirq) & PORT_INT_INTx_MASK; | |
278 | } | |
279 | ||
280 | static void apple_port_irq_ack(struct irq_data *data) | |
281 | { | |
282 | struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); | |
283 | ||
284 | if (!hwirq_is_intx(data->hwirq)) | |
285 | writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT); | |
286 | } | |
287 | ||
288 | static int apple_port_irq_set_type(struct irq_data *data, unsigned int type) | |
289 | { | |
290 | /* | |
291 | * It doesn't seem that there is any way to configure the | |
292 | * trigger, so assume INTx have to be level (as per the spec), | |
293 | * and the rest is edge (which looks likely). | |
294 | */ | |
295 | if (hwirq_is_intx(data->hwirq) ^ !!(type & IRQ_TYPE_LEVEL_MASK)) | |
296 | return -EINVAL; | |
297 | ||
298 | irqd_set_trigger_type(data, type); | |
299 | return 0; | |
300 | } | |
301 | ||
302 | static struct irq_chip apple_port_irqchip = { | |
303 | .name = "PCIe", | |
304 | .irq_ack = apple_port_irq_ack, | |
305 | .irq_mask = apple_port_irq_mask, | |
306 | .irq_unmask = apple_port_irq_unmask, | |
307 | .irq_set_type = apple_port_irq_set_type, | |
308 | }; | |
309 | ||
310 | static int apple_port_irq_domain_alloc(struct irq_domain *domain, | |
311 | unsigned int virq, unsigned int nr_irqs, | |
312 | void *args) | |
313 | { | |
314 | struct apple_pcie_port *port = domain->host_data; | |
315 | struct irq_fwspec *fwspec = args; | |
316 | int i; | |
317 | ||
318 | for (i = 0; i < nr_irqs; i++) { | |
319 | irq_flow_handler_t flow = handle_edge_irq; | |
320 | unsigned int type = IRQ_TYPE_EDGE_RISING; | |
321 | ||
322 | if (hwirq_is_intx(fwspec->param[0] + i)) { | |
323 | flow = handle_level_irq; | |
324 | type = IRQ_TYPE_LEVEL_HIGH; | |
325 | } | |
326 | ||
327 | irq_domain_set_info(domain, virq + i, fwspec->param[0] + i, | |
328 | &apple_port_irqchip, port, flow, | |
329 | NULL, NULL); | |
330 | ||
331 | irq_set_irq_type(virq + i, type); | |
332 | } | |
333 | ||
334 | return 0; | |
335 | } | |
336 | ||
337 | static void apple_port_irq_domain_free(struct irq_domain *domain, | |
338 | unsigned int virq, unsigned int nr_irqs) | |
339 | { | |
340 | int i; | |
341 | ||
342 | for (i = 0; i < nr_irqs; i++) { | |
343 | struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); | |
344 | ||
345 | irq_set_handler(virq + i, NULL); | |
346 | irq_domain_reset_irq_data(d); | |
347 | } | |
348 | } | |
349 | ||
350 | static const struct irq_domain_ops apple_port_irq_domain_ops = { | |
351 | .translate = irq_domain_translate_onecell, | |
352 | .alloc = apple_port_irq_domain_alloc, | |
353 | .free = apple_port_irq_domain_free, | |
354 | }; | |
355 | ||
356 | static void apple_port_irq_handler(struct irq_desc *desc) | |
357 | { | |
358 | struct apple_pcie_port *port = irq_desc_get_handler_data(desc); | |
359 | struct irq_chip *chip = irq_desc_get_chip(desc); | |
360 | unsigned long stat; | |
361 | int i; | |
362 | ||
363 | chained_irq_enter(chip, desc); | |
364 | ||
365 | stat = readl_relaxed(port->base + PORT_INTSTAT); | |
366 | ||
367 | for_each_set_bit(i, &stat, 32) | |
368 | generic_handle_domain_irq(port->domain, i); | |
369 | ||
370 | chained_irq_exit(chip, desc); | |
371 | } | |
372 | ||
373 | static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) | |
374 | { | |
375 | struct fwnode_handle *fwnode = &port->np->fwnode; | |
376 | unsigned int irq; | |
377 | ||
378 | /* FIXME: consider moving each interrupt under each port */ | |
379 | irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), | |
380 | port->idx); | |
381 | if (!irq) | |
382 | return -ENXIO; | |
383 | ||
384 | port->domain = irq_domain_create_linear(fwnode, 32, | |
385 | &apple_port_irq_domain_ops, | |
386 | port); | |
387 | if (!port->domain) | |
388 | return -ENOMEM; | |
389 | ||
390 | /* Disable all interrupts */ | |
391 | writel_relaxed(~0, port->base + PORT_INTMSKSET); | |
392 | writel_relaxed(~0, port->base + PORT_INTSTAT); | |
393 | ||
394 | irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port); | |
395 | ||
476c41ed MZ |
396 | /* Configure MSI base address */ |
397 | BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); | |
398 | writel_relaxed(lower_32_bits(DOORBELL_ADDR), port->base + PORT_MSIADDR); | |
399 | ||
400 | /* Enable MSIs, shared between all ports */ | |
401 | writel_relaxed(0, port->base + PORT_MSIBASE); | |
402 | writel_relaxed((ilog2(port->pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT) | | |
403 | PORT_MSICFG_EN, port->base + PORT_MSICFG); | |
404 | ||
d8fcbe52 MZ |
405 | return 0; |
406 | } | |
407 | ||
408 | static irqreturn_t apple_pcie_port_irq(int irq, void *data) | |
409 | { | |
410 | struct apple_pcie_port *port = data; | |
411 | unsigned int hwirq = irq_domain_get_irq_data(port->domain, irq)->hwirq; | |
412 | ||
413 | switch (hwirq) { | |
414 | case PORT_INT_LINK_UP: | |
415 | dev_info_ratelimited(port->pcie->dev, "Link up on %pOF\n", | |
416 | port->np); | |
417 | complete_all(&port->pcie->event); | |
418 | break; | |
419 | case PORT_INT_LINK_DOWN: | |
420 | dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n", | |
421 | port->np); | |
422 | break; | |
423 | default: | |
424 | return IRQ_NONE; | |
425 | } | |
426 | ||
427 | return IRQ_HANDLED; | |
428 | } | |
429 | ||
430 | static int apple_pcie_port_register_irqs(struct apple_pcie_port *port) | |
431 | { | |
432 | static struct { | |
433 | unsigned int hwirq; | |
434 | const char *name; | |
435 | } port_irqs[] = { | |
436 | { PORT_INT_LINK_UP, "Link up", }, | |
437 | { PORT_INT_LINK_DOWN, "Link down", }, | |
438 | }; | |
439 | int i; | |
440 | ||
441 | for (i = 0; i < ARRAY_SIZE(port_irqs); i++) { | |
442 | struct irq_fwspec fwspec = { | |
443 | .fwnode = &port->np->fwnode, | |
444 | .param_count = 1, | |
445 | .param = { | |
446 | [0] = port_irqs[i].hwirq, | |
447 | }, | |
448 | }; | |
449 | unsigned int irq; | |
450 | int ret; | |
451 | ||
452 | irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE, | |
453 | &fwspec); | |
454 | if (WARN_ON(!irq)) | |
455 | continue; | |
456 | ||
457 | ret = request_irq(irq, apple_pcie_port_irq, 0, | |
458 | port_irqs[i].name, port); | |
459 | WARN_ON(ret); | |
460 | } | |
461 | ||
462 | return 0; | |
463 | } | |
464 | ||
1512f908 AR |
465 | static int apple_pcie_setup_refclk(struct apple_pcie *pcie, |
466 | struct apple_pcie_port *port) | |
467 | { | |
468 | u32 stat; | |
469 | int res; | |
470 | ||
471 | res = readl_relaxed_poll_timeout(pcie->base + CORE_RC_PHYIF_STAT, stat, | |
472 | stat & CORE_RC_PHYIF_STAT_REFCLK, | |
473 | 100, 50000); | |
474 | if (res < 0) | |
475 | return res; | |
476 | ||
477 | rmw_set(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx)); | |
478 | rmw_set(CORE_LANE_CFG_REFCLK0REQ, pcie->base + CORE_LANE_CFG(port->idx)); | |
479 | ||
480 | res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx), | |
481 | stat, stat & CORE_LANE_CFG_REFCLK0ACK, | |
482 | 100, 50000); | |
483 | if (res < 0) | |
484 | return res; | |
485 | ||
75d36df6 | 486 | rmw_set(CORE_LANE_CFG_REFCLK1REQ, pcie->base + CORE_LANE_CFG(port->idx)); |
1512f908 | 487 | res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx), |
75d36df6 | 488 | stat, stat & CORE_LANE_CFG_REFCLK1ACK, |
1512f908 AR |
489 | 100, 50000); |
490 | ||
491 | if (res < 0) | |
492 | return res; | |
493 | ||
494 | rmw_clear(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx)); | |
495 | ||
496 | rmw_set(CORE_LANE_CFG_REFCLKEN, pcie->base + CORE_LANE_CFG(port->idx)); | |
497 | rmw_set(PORT_REFCLK_EN, port->base + PORT_REFCLK); | |
498 | ||
499 | return 0; | |
500 | } | |
501 | ||
468c8d52 MZ |
502 | static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, |
503 | int idx, u32 val) | |
504 | { | |
505 | writel_relaxed(val, port->base + PORT_RID2SID(idx)); | |
506 | /* Read back to ensure completion of the write */ | |
507 | return readl_relaxed(port->base + PORT_RID2SID(idx)); | |
508 | } | |
509 | ||
1e33888f AR |
510 | static int apple_pcie_setup_port(struct apple_pcie *pcie, |
511 | struct device_node *np) | |
512 | { | |
513 | struct platform_device *platform = to_platform_device(pcie->dev); | |
514 | struct apple_pcie_port *port; | |
515 | struct gpio_desc *reset; | |
516 | u32 stat, idx; | |
468c8d52 | 517 | int ret, i; |
1e33888f | 518 | |
a6b9ede1 DT |
519 | reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", |
520 | GPIOD_OUT_LOW, "PERST#"); | |
1e33888f AR |
521 | if (IS_ERR(reset)) |
522 | return PTR_ERR(reset); | |
523 | ||
524 | port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); | |
525 | if (!port) | |
526 | return -ENOMEM; | |
527 | ||
528 | ret = of_property_read_u32_index(np, "reg", 0, &idx); | |
529 | if (ret) | |
530 | return ret; | |
531 | ||
532 | /* Use the first reg entry to work out the port index */ | |
533 | port->idx = idx >> 11; | |
534 | port->pcie = pcie; | |
535 | port->np = np; | |
536 | ||
537 | port->base = devm_platform_ioremap_resource(platform, port->idx + 2); | |
538 | if (IS_ERR(port->base)) | |
539 | return PTR_ERR(port->base); | |
540 | ||
541 | rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); | |
542 | ||
ee91cb57 | 543 | /* Assert PERST# before setting up the clock */ |
87620512 | 544 | gpiod_set_value(reset, 1); |
ee91cb57 | 545 | |
1512f908 AR |
546 | ret = apple_pcie_setup_refclk(pcie, port); |
547 | if (ret < 0) | |
548 | return ret; | |
549 | ||
ee91cb57 MZ |
550 | /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ |
551 | usleep_range(100, 200); | |
552 | ||
553 | /* Deassert PERST# */ | |
1e33888f | 554 | rmw_set(PORT_PERST_OFF, port->base + PORT_PERST); |
87620512 | 555 | gpiod_set_value(reset, 0); |
1e33888f | 556 | |
ee91cb57 MZ |
557 | /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ |
558 | msleep(100); | |
559 | ||
1e33888f AR |
560 | ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, |
561 | stat & PORT_STATUS_READY, 100, 250000); | |
562 | if (ret < 0) { | |
563 | dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); | |
564 | return ret; | |
565 | } | |
566 | ||
754bb7ad HM |
567 | rmw_clear(PORT_REFCLK_CGDIS, port->base + PORT_REFCLK); |
568 | rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); | |
569 | ||
d8fcbe52 MZ |
570 | ret = apple_pcie_port_setup_irq(port); |
571 | if (ret) | |
572 | return ret; | |
573 | ||
468c8d52 MZ |
574 | /* Reset all RID/SID mappings, and check for RAZ/WI registers */ |
575 | for (i = 0; i < MAX_RID2SID; i++) { | |
576 | if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d) | |
577 | break; | |
578 | apple_pcie_rid2sid_write(port, i, 0); | |
579 | } | |
580 | ||
581 | dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i); | |
582 | ||
583 | port->sid_map_sz = i; | |
584 | ||
585 | list_add_tail(&port->entry, &pcie->ports); | |
d8fcbe52 MZ |
586 | init_completion(&pcie->event); |
587 | ||
588 | ret = apple_pcie_port_register_irqs(port); | |
589 | WARN_ON(ret); | |
590 | ||
1e33888f AR |
591 | writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); |
592 | ||
d8fcbe52 MZ |
593 | if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) |
594 | dev_warn(pcie->dev, "%pOF link didn't come up\n", np); | |
595 | ||
1e33888f AR |
596 | return 0; |
597 | } | |
598 | ||
476c41ed MZ |
599 | static int apple_msi_init(struct apple_pcie *pcie) |
600 | { | |
601 | struct fwnode_handle *fwnode = dev_fwnode(pcie->dev); | |
602 | struct of_phandle_args args = {}; | |
603 | struct irq_domain *parent; | |
604 | int ret; | |
605 | ||
606 | ret = of_parse_phandle_with_args(to_of_node(fwnode), "msi-ranges", | |
607 | "#interrupt-cells", 0, &args); | |
608 | if (ret) | |
609 | return ret; | |
610 | ||
611 | ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges", | |
612 | args.args_count + 1, &pcie->nvecs); | |
613 | if (ret) | |
614 | return ret; | |
615 | ||
616 | of_phandle_args_to_fwspec(args.np, args.args, args.args_count, | |
617 | &pcie->fwspec); | |
618 | ||
619 | pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL); | |
620 | if (!pcie->bitmap) | |
621 | return -ENOMEM; | |
622 | ||
623 | parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED); | |
624 | if (!parent) { | |
625 | dev_err(pcie->dev, "failed to find parent domain\n"); | |
626 | return -ENXIO; | |
627 | } | |
628 | ||
629 | parent = irq_domain_create_hierarchy(parent, 0, pcie->nvecs, fwnode, | |
630 | &apple_msi_domain_ops, pcie); | |
631 | if (!parent) { | |
632 | dev_err(pcie->dev, "failed to create IRQ domain\n"); | |
633 | return -ENOMEM; | |
634 | } | |
635 | irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); | |
636 | ||
637 | pcie->domain = pci_msi_create_irq_domain(fwnode, &apple_msi_info, | |
638 | parent); | |
639 | if (!pcie->domain) { | |
640 | dev_err(pcie->dev, "failed to create MSI domain\n"); | |
641 | irq_domain_remove(parent); | |
642 | return -ENOMEM; | |
643 | } | |
644 | ||
645 | return 0; | |
646 | } | |
647 | ||
468c8d52 MZ |
648 | static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev) |
649 | { | |
650 | struct pci_config_window *cfg = pdev->sysdata; | |
651 | struct apple_pcie *pcie = cfg->priv; | |
652 | struct pci_dev *port_pdev; | |
653 | struct apple_pcie_port *port; | |
654 | ||
655 | /* Find the root port this device is on */ | |
656 | port_pdev = pcie_find_root_port(pdev); | |
657 | ||
658 | /* If finding the port itself, nothing to do */ | |
659 | if (WARN_ON(!port_pdev) || pdev == port_pdev) | |
660 | return NULL; | |
661 | ||
662 | list_for_each_entry(port, &pcie->ports, entry) { | |
663 | if (port->idx == PCI_SLOT(port_pdev->devfn)) | |
664 | return port; | |
665 | } | |
666 | ||
667 | return NULL; | |
668 | } | |
669 | ||
670 | static int apple_pcie_add_device(struct apple_pcie_port *port, | |
671 | struct pci_dev *pdev) | |
672 | { | |
673 | u32 sid, rid = PCI_DEVID(pdev->bus->number, pdev->devfn); | |
674 | int idx, err; | |
675 | ||
676 | dev_dbg(&pdev->dev, "added to bus %s, index %d\n", | |
677 | pci_name(pdev->bus->self), port->idx); | |
678 | ||
679 | err = of_map_id(port->pcie->dev->of_node, rid, "iommu-map", | |
680 | "iommu-map-mask", NULL, &sid); | |
681 | if (err) | |
682 | return err; | |
683 | ||
684 | mutex_lock(&port->pcie->lock); | |
685 | ||
686 | idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); | |
687 | if (idx >= 0) { | |
688 | apple_pcie_rid2sid_write(port, idx, | |
689 | PORT_RID2SID_VALID | | |
690 | (sid << PORT_RID2SID_SID_SHIFT) | rid); | |
691 | ||
692 | dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n", | |
693 | rid, sid, idx); | |
694 | } | |
695 | ||
696 | mutex_unlock(&port->pcie->lock); | |
697 | ||
698 | return idx >= 0 ? 0 : -ENOSPC; | |
699 | } | |
700 | ||
701 | static void apple_pcie_release_device(struct apple_pcie_port *port, | |
702 | struct pci_dev *pdev) | |
703 | { | |
704 | u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn); | |
705 | int idx; | |
706 | ||
707 | mutex_lock(&port->pcie->lock); | |
708 | ||
709 | for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { | |
710 | u32 val; | |
711 | ||
712 | val = readl_relaxed(port->base + PORT_RID2SID(idx)); | |
713 | if ((val & 0xffff) == rid) { | |
714 | apple_pcie_rid2sid_write(port, idx, 0); | |
715 | bitmap_release_region(port->sid_map, idx, 0); | |
716 | dev_dbg(&pdev->dev, "Released %x (%d)\n", val, idx); | |
717 | break; | |
718 | } | |
719 | } | |
720 | ||
721 | mutex_unlock(&port->pcie->lock); | |
722 | } | |
723 | ||
724 | static int apple_pcie_bus_notifier(struct notifier_block *nb, | |
725 | unsigned long action, | |
726 | void *data) | |
727 | { | |
728 | struct device *dev = data; | |
729 | struct pci_dev *pdev = to_pci_dev(dev); | |
730 | struct apple_pcie_port *port; | |
731 | int err; | |
732 | ||
733 | /* | |
734 | * This is a bit ugly. We assume that if we get notified for | |
735 | * any PCI device, we must be in charge of it, and that there | |
736 | * is no other PCI controller in the whole system. It probably | |
737 | * holds for now, but who knows for how long? | |
738 | */ | |
739 | port = apple_pcie_get_port(pdev); | |
740 | if (!port) | |
741 | return NOTIFY_DONE; | |
742 | ||
743 | switch (action) { | |
744 | case BUS_NOTIFY_ADD_DEVICE: | |
745 | err = apple_pcie_add_device(port, pdev); | |
746 | if (err) | |
747 | return notifier_from_errno(err); | |
748 | break; | |
749 | case BUS_NOTIFY_DEL_DEVICE: | |
750 | apple_pcie_release_device(port, pdev); | |
751 | break; | |
752 | default: | |
753 | return NOTIFY_DONE; | |
754 | } | |
755 | ||
756 | return NOTIFY_OK; | |
757 | } | |
758 | ||
759 | static struct notifier_block apple_pcie_nb = { | |
760 | .notifier_call = apple_pcie_bus_notifier, | |
761 | }; | |
762 | ||
1e33888f AR |
763 | static int apple_pcie_init(struct pci_config_window *cfg) |
764 | { | |
765 | struct device *dev = cfg->parent; | |
766 | struct platform_device *platform = to_platform_device(dev); | |
767 | struct device_node *of_port; | |
768 | struct apple_pcie *pcie; | |
769 | int ret; | |
770 | ||
771 | pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); | |
772 | if (!pcie) | |
773 | return -ENOMEM; | |
774 | ||
775 | pcie->dev = dev; | |
776 | ||
476c41ed MZ |
777 | mutex_init(&pcie->lock); |
778 | ||
1e33888f AR |
779 | pcie->base = devm_platform_ioremap_resource(platform, 1); |
780 | if (IS_ERR(pcie->base)) | |
781 | return PTR_ERR(pcie->base); | |
782 | ||
468c8d52 MZ |
783 | cfg->priv = pcie; |
784 | INIT_LIST_HEAD(&pcie->ports); | |
785 | ||
1e33888f AR |
786 | for_each_child_of_node(dev->of_node, of_port) { |
787 | ret = apple_pcie_setup_port(pcie, of_port); | |
788 | if (ret) { | |
789 | dev_err(pcie->dev, "Port %pOF setup fail: %d\n", of_port, ret); | |
790 | of_node_put(of_port); | |
791 | return ret; | |
792 | } | |
793 | } | |
794 | ||
476c41ed | 795 | return apple_msi_init(pcie); |
1e33888f AR |
796 | } |
797 | ||
468c8d52 MZ |
798 | static int apple_pcie_probe(struct platform_device *pdev) |
799 | { | |
800 | int ret; | |
801 | ||
802 | ret = bus_register_notifier(&pci_bus_type, &apple_pcie_nb); | |
803 | if (ret) | |
804 | return ret; | |
805 | ||
806 | ret = pci_host_common_probe(pdev); | |
807 | if (ret) | |
808 | bus_unregister_notifier(&pci_bus_type, &apple_pcie_nb); | |
809 | ||
810 | return ret; | |
811 | } | |
812 | ||
1e33888f AR |
813 | static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { |
814 | .init = apple_pcie_init, | |
815 | .pci_ops = { | |
816 | .map_bus = pci_ecam_map_bus, | |
817 | .read = pci_generic_config_read, | |
818 | .write = pci_generic_config_write, | |
819 | } | |
820 | }; | |
821 | ||
822 | static const struct of_device_id apple_pcie_of_match[] = { | |
823 | { .compatible = "apple,pcie", .data = &apple_pcie_cfg_ecam_ops }, | |
824 | { } | |
825 | }; | |
826 | MODULE_DEVICE_TABLE(of, apple_pcie_of_match); | |
827 | ||
828 | static struct platform_driver apple_pcie_driver = { | |
468c8d52 | 829 | .probe = apple_pcie_probe, |
1e33888f AR |
830 | .driver = { |
831 | .name = "pcie-apple", | |
832 | .of_match_table = apple_pcie_of_match, | |
833 | .suppress_bind_attrs = true, | |
834 | }, | |
835 | }; | |
836 | module_platform_driver(apple_pcie_driver); | |
837 | ||
838 | MODULE_LICENSE("GPL v2"); |