projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/iommu/powernv: Release replaced TCE
[linux-2.6-block.git] / arch / powerpc / platforms / powernv / pci-ioda.c
CommitLineData
184cd4a3
BH		 1/*
2 * Support PCI/PCIe on PowerNV platforms
3 *
4 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
cee72d5b 12#undef DEBUG
184cd4a3
BH		 13
14#include <linux/kernel.h>
15#include <linux/pci.h>
361f2a2a 16#include <linux/crash_dump.h>
37c367f2 17#include <linux/debugfs.h>
184cd4a3
BH		 18#include <linux/delay.h>
19#include <linux/string.h>
20#include <linux/init.h>
21#include <linux/bootmem.h>
22#include <linux/irq.h>
23#include <linux/io.h>
24#include <linux/msi.h>
cd15b048 25#include <linux/memblock.h>
ac9a5889 26#include <linux/iommu.h>
e57080f1 27#include <linux/rculist.h>
184cd4a3
BH		 28
29#include <asm/sections.h>
30#include <asm/io.h>
31#include <asm/prom.h>
32#include <asm/pci-bridge.h>
33#include <asm/machdep.h>
fb1b55d6 34#include <asm/msi_bitmap.h>
184cd4a3
BH		 35#include <asm/ppc-pci.h>
36#include <asm/opal.h>
37#include <asm/iommu.h>
38#include <asm/tce.h>
137436c9 39#include <asm/xics.h>
37c367f2 40#include <asm/debug.h>
262af557 41#include <asm/firmware.h>
80c49c7e
IM		 42#include <asm/pnv-pci.h>
43
ec249dd8 44#include <misc/cxl-base.h>
184cd4a3
BH		 45
46#include "powernv.h"
47#include "pci.h"
48
781a868f
WY		 49/* 256M DMA window, 4K TCE pages, 8 bytes TCE */
50#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8)
51
6d31c2fa
JP		 52static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
53 const char *fmt, ...)
54{
55 struct va_format vaf;
56 va_list args;
57 char pfix[32];
58
59 va_start(args, fmt);
60
61 vaf.fmt = fmt;
62 vaf.va = &args;
63
781a868f 64 if (pe->flags & PNV_IODA_PE_DEV)
6d31c2fa 65 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
781a868f 66 else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
6d31c2fa
JP		 67 sprintf(pfix, "%04x:%02x ",
68 pci_domain_nr(pe->pbus), pe->pbus->number);
781a868f
WY		 69#ifdef CONFIG_PCI_IOV
70 else if (pe->flags & PNV_IODA_PE_VF)
71 sprintf(pfix, "%04x:%02x:%2x.%d",
72 pci_domain_nr(pe->parent_dev->bus),
73 (pe->rid & 0xff00) >> 8,
74 PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
75#endif /* CONFIG_PCI_IOV*/
6d31c2fa
JP		 76
77 printk("%spci %s: [PE# %.3d] %pV",
78 level, pfix, pe->pe_number, &vaf);
79
80 va_end(args);
81}
184cd4a3 82
6d31c2fa
JP		 83#define pe_err(pe, fmt, ...) \
84 pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
85#define pe_warn(pe, fmt, ...) \
86 pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
87#define pe_info(pe, fmt, ...) \
88 pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
184cd4a3 89
4e287840
TLSC		 90static bool pnv_iommu_bypass_disabled __read_mostly;
91
92static int __init iommu_setup(char *str)
93{
94 if (!str)
95 return -EINVAL;
96
97 while (*str) {
98 if (!strncmp(str, "nobypass", 8)) {
99 pnv_iommu_bypass_disabled = true;
100 pr_info("PowerNV: IOMMU bypass window disabled.\n");
101 break;
102 }
103 str += strcspn(str, ",");
104 if (*str == ',')
105 str++;
106 }
107
108 return 0;
109}
110early_param("iommu", iommu_setup);
111
8e0a1611
AK		 112/*
113 * stdcix is only supposed to be used in hypervisor real mode as per
114 * the architecture spec
115 */
116static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
117{
118 __asm__ __volatile__("stdcix %0,0,%1"
119 : : "r" (val), "r" (paddr) : "memory");
120}
121
262af557
GC		 122static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
123{
124 return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
125 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
126}
127
4b82ab18
GS		 128static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
129{
130 if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
131 pr_warn("%s: Invalid PE %d on PHB#%x\n",
132 __func__, pe_no, phb->hose->global_number);
133 return;
134 }
135
136 if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
137 pr_warn("%s: PE %d was assigned on PHB#%x\n",
138 __func__, pe_no, phb->hose->global_number);
139 return;
140 }
141
142 phb->ioda.pe_array[pe_no].phb = phb;
143 phb->ioda.pe_array[pe_no].pe_number = pe_no;
144}
145
cad5cef6 146static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
184cd4a3
BH		 147{
148 unsigned long pe;
149
150 do {
151 pe = find_next_zero_bit(phb->ioda.pe_alloc,
152 phb->ioda.total_pe, 0);
153 if (pe >= phb->ioda.total_pe)
154 return IODA_INVALID_PE;
155 } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
156
4cce9550 157 phb->ioda.pe_array[pe].phb = phb;
184cd4a3
BH		 158 phb->ioda.pe_array[pe].pe_number = pe;
159 return pe;
160}
161
cad5cef6 162static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
184cd4a3
BH		 163{
164 WARN_ON(phb->ioda.pe_array[pe].pdev);
165
166 memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
167 clear_bit(pe, phb->ioda.pe_alloc);
168}
169
262af557
GC		 170/* The default M64 BAR is shared by all PEs */
171static int pnv_ioda2_init_m64(struct pnv_phb *phb)
172{
173 const char *desc;
174 struct resource *r;
175 s64 rc;
176
177 /* Configure the default M64 BAR */
178 rc = opal_pci_set_phb_mem_window(phb->opal_id,
179 OPAL_M64_WINDOW_TYPE,
180 phb->ioda.m64_bar_idx,
181 phb->ioda.m64_base,
182 0, /* unused */
183 phb->ioda.m64_size);
184 if (rc != OPAL_SUCCESS) {
185 desc = "configuring";
186 goto fail;
187 }
188
189 /* Enable the default M64 BAR */
190 rc = opal_pci_phb_mmio_enable(phb->opal_id,
191 OPAL_M64_WINDOW_TYPE,
192 phb->ioda.m64_bar_idx,
193 OPAL_ENABLE_M64_SPLIT);
194 if (rc != OPAL_SUCCESS) {
195 desc = "enabling";
196 goto fail;
197 }
198
199 /* Mark the M64 BAR assigned */
200 set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
201
202 /*
203 * Strip off the segment used by the reserved PE, which is
204 * expected to be 0 or last one of PE capabicity.
205 */
206 r = &phb->hose->mem_resources[1];
207 if (phb->ioda.reserved_pe == 0)
208 r->start += phb->ioda.m64_segsize;
209 else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
210 r->end -= phb->ioda.m64_segsize;
211 else
212 pr_warn(" Cannot strip M64 segment for reserved PE#%d\n",
213 phb->ioda.reserved_pe);
214
215 return 0;
216
217fail:
218 pr_warn(" Failure %lld %s M64 BAR#%d\n",
219 rc, desc, phb->ioda.m64_bar_idx);
220 opal_pci_phb_mmio_enable(phb->opal_id,
221 OPAL_M64_WINDOW_TYPE,
222 phb->ioda.m64_bar_idx,
223 OPAL_DISABLE_M64);
224 return -EIO;
225}
226
5ef73567 227static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
262af557
GC		 228{
229 resource_size_t sgsz = phb->ioda.m64_segsize;
230 struct pci_dev *pdev;
231 struct resource *r;
232 int base, step, i;
233
234 /*
235 * Root bus always has full M64 range and root port has
236 * M64 range used in reality. So we're checking root port
237 * instead of root bus.
238 */
239 list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
4b82ab18
GS		 240 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
241 r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
262af557
GC		 242 if (!r->parent ||
243 !pnv_pci_is_mem_pref_64(r->flags))
244 continue;
245
246 base = (r->start - phb->ioda.m64_base) / sgsz;
247 for (step = 0; step < resource_size(r) / sgsz; step++)
4b82ab18 248 pnv_ioda_reserve_pe(phb, base + step);
262af557
GC		 249 }
250 }
251}
252
253static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
254 struct pci_bus *bus, int all)
255{
256 resource_size_t segsz = phb->ioda.m64_segsize;
257 struct pci_dev *pdev;
258 struct resource *r;
259 struct pnv_ioda_pe *master_pe, *pe;
260 unsigned long size, *pe_alloc;
261 bool found;
262 int start, i, j;
263
264 /* Root bus shouldn't use M64 */
265 if (pci_is_root_bus(bus))
266 return IODA_INVALID_PE;
267
268 /* We support only one M64 window on each bus */
269 found = false;
270 pci_bus_for_each_resource(bus, r, i) {
271 if (r && r->parent &&
272 pnv_pci_is_mem_pref_64(r->flags)) {
273 found = true;
274 break;
275 }
276 }
277
278 /* No M64 window found ? */
279 if (!found)
280 return IODA_INVALID_PE;
281
282 /* Allocate bitmap */
283 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
284 pe_alloc = kzalloc(size, GFP_KERNEL);
285 if (!pe_alloc) {
286 pr_warn("%s: Out of memory !\n",
287 __func__);
288 return IODA_INVALID_PE;
289 }
290
291 /*
292 * Figure out reserved PE numbers by the PE
293 * the its child PEs.
294 */
295 start = (r->start - phb->ioda.m64_base) / segsz;
296 for (i = 0; i < resource_size(r) / segsz; i++)
297 set_bit(start + i, pe_alloc);
298
299 if (all)
300 goto done;
301
302 /*
303 * If the PE doesn't cover all subordinate buses,
304 * we need subtract from reserved PEs for children.
305 */
306 list_for_each_entry(pdev, &bus->devices, bus_list) {
307 if (!pdev->subordinate)
308 continue;
309
310 pci_bus_for_each_resource(pdev->subordinate, r, i) {
311 if (!r || !r->parent ||
312 !pnv_pci_is_mem_pref_64(r->flags))
313 continue;
314
315 start = (r->start - phb->ioda.m64_base) / segsz;
316 for (j = 0; j < resource_size(r) / segsz ; j++)
317 clear_bit(start + j, pe_alloc);
318 }
319 }
320
321 /*
322 * the current bus might not own M64 window and that's all
323 * contributed by its child buses. For the case, we needn't
324 * pick M64 dependent PE#.
325 */
326 if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
327 kfree(pe_alloc);
328 return IODA_INVALID_PE;
329 }
330
331 /*
332 * Figure out the master PE and put all slave PEs to master
333 * PE's list to form compound PE.
334 */
335done:
336 master_pe = NULL;
337 i = -1;
338 while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
339 phb->ioda.total_pe) {
340 pe = &phb->ioda.pe_array[i];
262af557
GC		 341
342 if (!master_pe) {
343 pe->flags |= PNV_IODA_PE_MASTER;
344 INIT_LIST_HEAD(&pe->slaves);
345 master_pe = pe;
346 } else {
347 pe->flags |= PNV_IODA_PE_SLAVE;
348 pe->master = master_pe;
349 list_add_tail(&pe->list, &master_pe->slaves);
350 }
351 }
352
353 kfree(pe_alloc);
354 return master_pe->pe_number;
355}
356
357static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
358{
359 struct pci_controller *hose = phb->hose;
360 struct device_node *dn = hose->dn;
361 struct resource *res;
362 const u32 *r;
363 u64 pci_addr;
364
1665c4a8
GS		 365 /* FIXME: Support M64 for P7IOC */
366 if (phb->type != PNV_PHB_IODA2) {
367 pr_info(" Not support M64 window\n");
368 return;
369 }
370
262af557
GC		 371 if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
372 pr_info(" Firmware too old to support M64 window\n");
373 return;
374 }
375
376 r = of_get_property(dn, "ibm,opal-m64-window", NULL);
377 if (!r) {
378 pr_info(" No <ibm,opal-m64-window> on %s\n",
379 dn->full_name);
380 return;
381 }
382
262af557
GC		 383 res = &hose->mem_resources[1];
384 res->start = of_translate_address(dn, r + 2);
385 res->end = res->start + of_read_number(r + 4, 2) - 1;
386 res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
387 pci_addr = of_read_number(r, 2);
388 hose->mem_offset[1] = res->start - pci_addr;
389
390 phb->ioda.m64_size = resource_size(res);
391 phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
392 phb->ioda.m64_base = pci_addr;
393
e9863e68
WY		 394 pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
395 res->start, res->end, pci_addr);
396
262af557
GC		 397 /* Use last M64 BAR to cover M64 window */
398 phb->ioda.m64_bar_idx = 15;
399 phb->init_m64 = pnv_ioda2_init_m64;
5ef73567 400 phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
262af557
GC		 401 phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
402}
403
49dec922
GS		 404static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
405{
406 struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
407 struct pnv_ioda_pe *slave;
408 s64 rc;
409
410 /* Fetch master PE */
411 if (pe->flags & PNV_IODA_PE_SLAVE) {
412 pe = pe->master;
ec8e4e9d
GS		 413 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
414 return;
415
49dec922
GS		 416 pe_no = pe->pe_number;
417 }
418
419 /* Freeze master PE */
420 rc = opal_pci_eeh_freeze_set(phb->opal_id,
421 pe_no,
422 OPAL_EEH_ACTION_SET_FREEZE_ALL);
423 if (rc != OPAL_SUCCESS) {
424 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
425 __func__, rc, phb->hose->global_number, pe_no);
426 return;
427 }
428
429 /* Freeze slave PEs */
430 if (!(pe->flags & PNV_IODA_PE_MASTER))
431 return;
432
433 list_for_each_entry(slave, &pe->slaves, list) {
434 rc = opal_pci_eeh_freeze_set(phb->opal_id,
435 slave->pe_number,
436 OPAL_EEH_ACTION_SET_FREEZE_ALL);
437 if (rc != OPAL_SUCCESS)
438 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
439 __func__, rc, phb->hose->global_number,
440 slave->pe_number);
441 }
442}
443
e51df2c1 444static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
49dec922
GS		 445{
446 struct pnv_ioda_pe *pe, *slave;
447 s64 rc;
448
449 /* Find master PE */
450 pe = &phb->ioda.pe_array[pe_no];
451 if (pe->flags & PNV_IODA_PE_SLAVE) {
452 pe = pe->master;
453 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
454 pe_no = pe->pe_number;
455 }
456
457 /* Clear frozen state for master PE */
458 rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
459 if (rc != OPAL_SUCCESS) {
460 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
461 __func__, rc, opt, phb->hose->global_number, pe_no);
462 return -EIO;
463 }
464
465 if (!(pe->flags & PNV_IODA_PE_MASTER))
466 return 0;
467
468 /* Clear frozen state for slave PEs */
469 list_for_each_entry(slave, &pe->slaves, list) {
470 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
471 slave->pe_number,
472 opt);
473 if (rc != OPAL_SUCCESS) {
474 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
475 __func__, rc, opt, phb->hose->global_number,
476 slave->pe_number);
477 return -EIO;
478 }
479 }
480
481 return 0;
482}
483
484static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
485{
486 struct pnv_ioda_pe *slave, *pe;
487 u8 fstate, state;
488 __be16 pcierr;
489 s64 rc;
490
491 /* Sanity check on PE number */
492 if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
493 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
494
495 /*
496 * Fetch the master PE and the PE instance might be
497 * not initialized yet.
498 */
499 pe = &phb->ioda.pe_array[pe_no];
500 if (pe->flags & PNV_IODA_PE_SLAVE) {
501 pe = pe->master;
502 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
503 pe_no = pe->pe_number;
504 }
505
506 /* Check the master PE */
507 rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
508 &state, &pcierr, NULL);
509 if (rc != OPAL_SUCCESS) {
510 pr_warn("%s: Failure %lld getting "
511 "PHB#%x-PE#%x state\n",
512 __func__, rc,
513 phb->hose->global_number, pe_no);
514 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
515 }
516
517 /* Check the slave PE */
518 if (!(pe->flags & PNV_IODA_PE_MASTER))
519 return state;
520
521 list_for_each_entry(slave, &pe->slaves, list) {
522 rc = opal_pci_eeh_freeze_status(phb->opal_id,
523 slave->pe_number,
524 &fstate,
525 &pcierr,
526 NULL);
527 if (rc != OPAL_SUCCESS) {
528 pr_warn("%s: Failure %lld getting "
529 "PHB#%x-PE#%x state\n",
530 __func__, rc,
531 phb->hose->global_number, slave->pe_number);
532 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
533 }
534
535 /*
536 * Override the result based on the ascending
537 * priority.
538 */
539 if (fstate > state)
540 state = fstate;
541 }
542
543 return state;
544}
545
184cd4a3
BH		 546/* Currently those 2 are only used when MSIs are enabled, this will change
547 * but in the meantime, we need to protect them to avoid warnings
548 */
549#ifdef CONFIG_PCI_MSI
cad5cef6 550static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
184cd4a3
BH		 551{
552 struct pci_controller *hose = pci_bus_to_host(dev->bus);
553 struct pnv_phb *phb = hose->private_data;
b72c1f65 554 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 555
556 if (!pdn)
557 return NULL;
558 if (pdn->pe_number == IODA_INVALID_PE)
559 return NULL;
560 return &phb->ioda.pe_array[pdn->pe_number];
561}
184cd4a3
BH		 562#endif /* CONFIG_PCI_MSI */
563
b131a842
GS		 564static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
565 struct pnv_ioda_pe *parent,
566 struct pnv_ioda_pe *child,
567 bool is_add)
568{
569 const char *desc = is_add ? "adding" : "removing";
570 uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
571 OPAL_REMOVE_PE_FROM_DOMAIN;
572 struct pnv_ioda_pe *slave;
573 long rc;
574
575 /* Parent PE affects child PE */
576 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
577 child->pe_number, op);
578 if (rc != OPAL_SUCCESS) {
579 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
580 rc, desc);
581 return -ENXIO;
582 }
583
584 if (!(child->flags & PNV_IODA_PE_MASTER))
585 return 0;
586
587 /* Compound case: parent PE affects slave PEs */
588 list_for_each_entry(slave, &child->slaves, list) {
589 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
590 slave->pe_number, op);
591 if (rc != OPAL_SUCCESS) {
592 pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
593 rc, desc);
594 return -ENXIO;
595 }
596 }
597
598 return 0;
599}
600
601static int pnv_ioda_set_peltv(struct pnv_phb *phb,
602 struct pnv_ioda_pe *pe,
603 bool is_add)
604{
605 struct pnv_ioda_pe *slave;
781a868f 606 struct pci_dev *pdev = NULL;
b131a842
GS		 607 int ret;
608
609 /*
610 * Clear PE frozen state. If it's master PE, we need
611 * clear slave PE frozen state as well.
612 */
613 if (is_add) {
614 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
615 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
616 if (pe->flags & PNV_IODA_PE_MASTER) {
617 list_for_each_entry(slave, &pe->slaves, list)
618 opal_pci_eeh_freeze_clear(phb->opal_id,
619 slave->pe_number,
620 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
621 }
622 }
623
624 /*
625 * Associate PE in PELT. We need add the PE into the
626 * corresponding PELT-V as well. Otherwise, the error
627 * originated from the PE might contribute to other
628 * PEs.
629 */
630 ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
631 if (ret)
632 return ret;
633
634 /* For compound PEs, any one affects all of them */
635 if (pe->flags & PNV_IODA_PE_MASTER) {
636 list_for_each_entry(slave, &pe->slaves, list) {
637 ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
638 if (ret)
639 return ret;
640 }
641 }
642
643 if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
644 pdev = pe->pbus->self;
781a868f 645 else if (pe->flags & PNV_IODA_PE_DEV)
b131a842 646 pdev = pe->pdev->bus->self;
781a868f
WY		 647#ifdef CONFIG_PCI_IOV
648 else if (pe->flags & PNV_IODA_PE_VF)
649 pdev = pe->parent_dev->bus->self;
650#endif /* CONFIG_PCI_IOV */
b131a842
GS		 651 while (pdev) {
652 struct pci_dn *pdn = pci_get_pdn(pdev);
653 struct pnv_ioda_pe *parent;
654
655 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
656 parent = &phb->ioda.pe_array[pdn->pe_number];
657 ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
658 if (ret)
659 return ret;
660 }
661
662 pdev = pdev->bus->self;
663 }
664
665 return 0;
666}
667
781a868f
WY		 668#ifdef CONFIG_PCI_IOV
669static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
670{
671 struct pci_dev *parent;
672 uint8_t bcomp, dcomp, fcomp;
673 int64_t rc;
674 long rid_end, rid;
675
676 /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
677 if (pe->pbus) {
678 int count;
679
680 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
681 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
682 parent = pe->pbus->self;
683 if (pe->flags & PNV_IODA_PE_BUS_ALL)
684 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
685 else
686 count = 1;
687
688 switch(count) {
689 case 1: bcomp = OpalPciBusAll; break;
690 case 2: bcomp = OpalPciBus7Bits; break;
691 case 4: bcomp = OpalPciBus6Bits; break;
692 case 8: bcomp = OpalPciBus5Bits; break;
693 case 16: bcomp = OpalPciBus4Bits; break;
694 case 32: bcomp = OpalPciBus3Bits; break;
695 default:
696 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
697 count);
698 /* Do an exact match only */
699 bcomp = OpalPciBusAll;
700 }
701 rid_end = pe->rid + (count << 8);
702 } else {
703 if (pe->flags & PNV_IODA_PE_VF)
704 parent = pe->parent_dev;
705 else
706 parent = pe->pdev->bus->self;
707 bcomp = OpalPciBusAll;
708 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
709 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
710 rid_end = pe->rid + 1;
711 }
712
713 /* Clear the reverse map */
714 for (rid = pe->rid; rid < rid_end; rid++)
715 phb->ioda.pe_rmap[rid] = 0;
716
717 /* Release from all parents PELT-V */
718 while (parent) {
719 struct pci_dn *pdn = pci_get_pdn(parent);
720 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
721 rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
722 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
723 /* XXX What to do in case of error ? */
724 }
725 parent = parent->bus->self;
726 }
727
728 opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
729 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
730
731 /* Disassociate PE in PELT */
732 rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
733 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
734 if (rc)
735 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
736 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
737 bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
738 if (rc)
739 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
740
741 pe->pbus = NULL;
742 pe->pdev = NULL;
743 pe->parent_dev = NULL;
744
745 return 0;
746}
747#endif /* CONFIG_PCI_IOV */
748
cad5cef6 749static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
184cd4a3
BH		 750{
751 struct pci_dev *parent;
752 uint8_t bcomp, dcomp, fcomp;
753 long rc, rid_end, rid;
754
755 /* Bus validation ? */
756 if (pe->pbus) {
757 int count;
758
759 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
760 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
761 parent = pe->pbus->self;
fb446ad0
GS		 762 if (pe->flags & PNV_IODA_PE_BUS_ALL)
763 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
764 else
765 count = 1;
766
184cd4a3
BH		 767 switch(count) {
768 case 1: bcomp = OpalPciBusAll; break;
769 case 2: bcomp = OpalPciBus7Bits; break;
770 case 4: bcomp = OpalPciBus6Bits; break;
771 case 8: bcomp = OpalPciBus5Bits; break;
772 case 16: bcomp = OpalPciBus4Bits; break;
773 case 32: bcomp = OpalPciBus3Bits; break;
774 default:
781a868f
WY		 775 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
776 count);
184cd4a3
BH		 777 /* Do an exact match only */
778 bcomp = OpalPciBusAll;
779 }
780 rid_end = pe->rid + (count << 8);
781 } else {
781a868f
WY		 782#ifdef CONFIG_PCI_IOV
783 if (pe->flags & PNV_IODA_PE_VF)
784 parent = pe->parent_dev;
785 else
786#endif /* CONFIG_PCI_IOV */
787 parent = pe->pdev->bus->self;
184cd4a3
BH		 788 bcomp = OpalPciBusAll;
789 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
790 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
791 rid_end = pe->rid + 1;
792 }
793
631ad691
GS		 794 /*
795 * Associate PE in PELT. We need add the PE into the
796 * corresponding PELT-V as well. Otherwise, the error
797 * originated from the PE might contribute to other
798 * PEs.
799 */
184cd4a3
BH		 800 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
801 bcomp, dcomp, fcomp, OPAL_MAP_PE);
802 if (rc) {
803 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
804 return -ENXIO;
805 }
631ad691 806
b131a842
GS		 807 /* Configure PELTV */
808 pnv_ioda_set_peltv(phb, pe, true);
184cd4a3 809
184cd4a3
BH		 810 /* Setup reverse map */
811 for (rid = pe->rid; rid < rid_end; rid++)
812 phb->ioda.pe_rmap[rid] = pe->pe_number;
813
814 /* Setup one MVTs on IODA1 */
4773f76b
GS		 815 if (phb->type != PNV_PHB_IODA1) {
816 pe->mve_number = 0;
817 goto out;
818 }
819
820 pe->mve_number = pe->pe_number;
821 rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
822 if (rc != OPAL_SUCCESS) {
823 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
824 rc, pe->mve_number);
825 pe->mve_number = -1;
826 } else {
827 rc = opal_pci_set_mve_enable(phb->opal_id,
828 pe->mve_number, OPAL_ENABLE_MVE);
184cd4a3 829 if (rc) {
4773f76b 830 pe_err(pe, "OPAL error %ld enabling MVE %d\n",
184cd4a3
BH		 831 rc, pe->mve_number);
832 pe->mve_number = -1;
184cd4a3 833 }
4773f76b 834 }
184cd4a3 835
4773f76b 836out:
184cd4a3
BH		 837 return 0;
838}
839
cad5cef6
GKH		 840static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
841 struct pnv_ioda_pe *pe)
184cd4a3
BH		 842{
843 struct pnv_ioda_pe *lpe;
844
7ebdf956 845 list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
184cd4a3 846 if (lpe->dma_weight < pe->dma_weight) {
7ebdf956 847 list_add_tail(&pe->dma_link, &lpe->dma_link);
184cd4a3
BH		 848 return;
849 }
850 }
7ebdf956 851 list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
184cd4a3
BH		 852}
853
854static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
855{
856 /* This is quite simplistic. The "base" weight of a device
857 * is 10. 0 means no DMA is to be accounted for it.
858 */
859
860 /* If it's a bridge, no DMA */
861 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
862 return 0;
863
864 /* Reduce the weight of slow USB controllers */
865 if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
866 dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
867 dev->class == PCI_CLASS_SERIAL_USB_EHCI)
868 return 3;
869
870 /* Increase the weight of RAID (includes Obsidian) */
871 if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
872 return 15;
873
874 /* Default */
875 return 10;
876}
877
781a868f
WY		 878#ifdef CONFIG_PCI_IOV
879static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
880{
881 struct pci_dn *pdn = pci_get_pdn(dev);
882 int i;
883 struct resource *res, res2;
884 resource_size_t size;
885 u16 num_vfs;
886
887 if (!dev->is_physfn)
888 return -EINVAL;
889
890 /*
891 * "offset" is in VFs. The M64 windows are sized so that when they
892 * are segmented, each segment is the same size as the IOV BAR.
893 * Each segment is in a separate PE, and the high order bits of the
894 * address are the PE number. Therefore, each VF's BAR is in a
895 * separate PE, and changing the IOV BAR start address changes the
896 * range of PEs the VFs are in.
897 */
898 num_vfs = pdn->num_vfs;
899 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
900 res = &dev->resource[i + PCI_IOV_RESOURCES];
901 if (!res->flags || !res->parent)
902 continue;
903
904 if (!pnv_pci_is_mem_pref_64(res->flags))
905 continue;
906
907 /*
908 * The actual IOV BAR range is determined by the start address
909 * and the actual size for num_vfs VFs BAR. This check is to
910 * make sure that after shifting, the range will not overlap
911 * with another device.
912 */
913 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
914 res2.flags = res->flags;
915 res2.start = res->start + (size * offset);
916 res2.end = res2.start + (size * num_vfs) - 1;
917
918 if (res2.end > res->end) {
919 dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
920 i, &res2, res, num_vfs, offset);
921 return -EBUSY;
922 }
923 }
924
925 /*
926 * After doing so, there would be a "hole" in the /proc/iomem when
927 * offset is a positive value. It looks like the device return some
928 * mmio back to the system, which actually no one could use it.
929 */
930 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
931 res = &dev->resource[i + PCI_IOV_RESOURCES];
932 if (!res->flags || !res->parent)
933 continue;
934
935 if (!pnv_pci_is_mem_pref_64(res->flags))
936 continue;
937
938 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
939 res2 = *res;
940 res->start += size * offset;
941
942 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
943 i, &res2, res, num_vfs, offset);
944 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
945 }
946 return 0;
947}
948#endif /* CONFIG_PCI_IOV */
949
fb446ad0 950#if 0
cad5cef6 951static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
184cd4a3
BH		 952{
953 struct pci_controller *hose = pci_bus_to_host(dev->bus);
954 struct pnv_phb *phb = hose->private_data;
b72c1f65 955 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 956 struct pnv_ioda_pe *pe;
957 int pe_num;
958
959 if (!pdn) {
960 pr_err("%s: Device tree node not associated properly\n",
961 pci_name(dev));
962 return NULL;
963 }
964 if (pdn->pe_number != IODA_INVALID_PE)
965 return NULL;
966
967 /* PE#0 has been pre-set */
968 if (dev->bus->number == 0)
969 pe_num = 0;
970 else
971 pe_num = pnv_ioda_alloc_pe(phb);
972 if (pe_num == IODA_INVALID_PE) {
973 pr_warning("%s: Not enough PE# available, disabling device\n",
974 pci_name(dev));
975 return NULL;
976 }
977
978 /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
979 * pointer in the PE data structure, both should be destroyed at the
980 * same time. However, this needs to be looked at more closely again
981 * once we actually start removing things (Hotplug, SR-IOV, ...)
982 *
983 * At some point we want to remove the PDN completely anyways
984 */
985 pe = &phb->ioda.pe_array[pe_num];
986 pci_dev_get(dev);
987 pdn->pcidev = dev;
988 pdn->pe_number = pe_num;
989 pe->pdev = dev;
990 pe->pbus = NULL;
991 pe->tce32_seg = -1;
992 pe->mve_number = -1;
993 pe->rid = dev->bus->number << 8 | pdn->devfn;
994
995 pe_info(pe, "Associated device to PE\n");
996
997 if (pnv_ioda_configure_pe(phb, pe)) {
998 /* XXX What do we do here ? */
999 if (pe_num)
1000 pnv_ioda_free_pe(phb, pe_num);
1001 pdn->pe_number = IODA_INVALID_PE;
1002 pe->pdev = NULL;
1003 pci_dev_put(dev);
1004 return NULL;
1005 }
1006
1007 /* Assign a DMA weight to the device */
1008 pe->dma_weight = pnv_ioda_dma_weight(dev);
1009 if (pe->dma_weight != 0) {
1010 phb->ioda.dma_weight += pe->dma_weight;
1011 phb->ioda.dma_pe_count++;
1012 }
1013
1014 /* Link the PE */
1015 pnv_ioda_link_pe_by_weight(phb, pe);
1016
1017 return pe;
1018}
fb446ad0 1019#endif /* Useful for SRIOV case */
184cd4a3
BH		 1020
1021static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1022{
1023 struct pci_dev *dev;
1024
1025 list_for_each_entry(dev, &bus->devices, bus_list) {
b72c1f65 1026 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 1027
1028 if (pdn == NULL) {
1029 pr_warn("%s: No device node associated with device !\n",
1030 pci_name(dev));
1031 continue;
1032 }
184cd4a3
BH		 1033 pdn->pe_number = pe->pe_number;
1034 pe->dma_weight += pnv_ioda_dma_weight(dev);
fb446ad0 1035 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
184cd4a3
BH		 1036 pnv_ioda_setup_same_PE(dev->subordinate, pe);
1037 }
1038}
1039
fb446ad0
GS		 1040/*
1041 * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1042 * single PCI bus. Another one that contains the primary PCI bus and its
1043 * subordinate PCI devices and buses. The second type of PE is normally
1044 * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1045 */
cad5cef6 1046static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
184cd4a3 1047{
fb446ad0 1048 struct pci_controller *hose = pci_bus_to_host(bus);
184cd4a3 1049 struct pnv_phb *phb = hose->private_data;
184cd4a3 1050 struct pnv_ioda_pe *pe;
262af557
GC		 1051 int pe_num = IODA_INVALID_PE;
1052
1053 /* Check if PE is determined by M64 */
1054 if (phb->pick_m64_pe)
1055 pe_num = phb->pick_m64_pe(phb, bus, all);
1056
1057 /* The PE number isn't pinned by M64 */
1058 if (pe_num == IODA_INVALID_PE)
1059 pe_num = pnv_ioda_alloc_pe(phb);
184cd4a3 1060
184cd4a3 1061 if (pe_num == IODA_INVALID_PE) {
fb446ad0
GS		 1062 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1063 __func__, pci_domain_nr(bus), bus->number);
184cd4a3
BH		 1064 return;
1065 }
1066
1067 pe = &phb->ioda.pe_array[pe_num];
262af557 1068 pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
184cd4a3
BH		 1069 pe->pbus = bus;
1070 pe->pdev = NULL;
1071 pe->tce32_seg = -1;
1072 pe->mve_number = -1;
b918c62e 1073 pe->rid = bus->busn_res.start << 8;
184cd4a3
BH		 1074 pe->dma_weight = 0;
1075
fb446ad0
GS		 1076 if (all)
1077 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1078 bus->busn_res.start, bus->busn_res.end, pe_num);
1079 else
1080 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1081 bus->busn_res.start, pe_num);
184cd4a3
BH		 1082
1083 if (pnv_ioda_configure_pe(phb, pe)) {
1084 /* XXX What do we do here ? */
1085 if (pe_num)
1086 pnv_ioda_free_pe(phb, pe_num);
1087 pe->pbus = NULL;
1088 return;
1089 }
1090
1091 /* Associate it with all child devices */
1092 pnv_ioda_setup_same_PE(bus, pe);
1093
7ebdf956
GS		 1094 /* Put PE to the list */
1095 list_add_tail(&pe->list, &phb->ioda.pe_list);
1096
184cd4a3
BH		 1097 /* Account for one DMA PE if at least one DMA capable device exist
1098 * below the bridge
1099 */
1100 if (pe->dma_weight != 0) {
1101 phb->ioda.dma_weight += pe->dma_weight;
1102 phb->ioda.dma_pe_count++;
1103 }
1104
1105 /* Link the PE */
1106 pnv_ioda_link_pe_by_weight(phb, pe);
1107}
1108
cad5cef6 1109static void pnv_ioda_setup_PEs(struct pci_bus *bus)
184cd4a3
BH		 1110{
1111 struct pci_dev *dev;
fb446ad0
GS		 1112
1113 pnv_ioda_setup_bus_PE(bus, 0);
184cd4a3
BH		 1114
1115 list_for_each_entry(dev, &bus->devices, bus_list) {
fb446ad0
GS		 1116 if (dev->subordinate) {
1117 if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1118 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1119 else
1120 pnv_ioda_setup_PEs(dev->subordinate);
1121 }
1122 }
1123}
1124
1125/*
1126 * Configure PEs so that the downstream PCI buses and devices
1127 * could have their associated PE#. Unfortunately, we didn't
1128 * figure out the way to identify the PLX bridge yet. So we
1129 * simply put the PCI bus and the subordinate behind the root
1130 * port to PE# here. The game rule here is expected to be changed
1131 * as soon as we can detected PLX bridge correctly.
1132 */
cad5cef6 1133static void pnv_pci_ioda_setup_PEs(void)
fb446ad0
GS		 1134{
1135 struct pci_controller *hose, *tmp;
262af557 1136 struct pnv_phb *phb;
fb446ad0
GS		 1137
1138 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
262af557
GC		 1139 phb = hose->private_data;
1140
1141 /* M64 layout might affect PE allocation */
5ef73567
GS		 1142 if (phb->reserve_m64_pe)
1143 phb->reserve_m64_pe(phb);
262af557 1144
fb446ad0 1145 pnv_ioda_setup_PEs(hose->bus);
184cd4a3
BH		 1146 }
1147}
1148
a8b2f828 1149#ifdef CONFIG_PCI_IOV
781a868f
WY		 1150static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1151{
1152 struct pci_bus *bus;
1153 struct pci_controller *hose;
1154 struct pnv_phb *phb;
1155 struct pci_dn *pdn;
02639b0e 1156 int i, j;
781a868f
WY		 1157
1158 bus = pdev->bus;
1159 hose = pci_bus_to_host(bus);
1160 phb = hose->private_data;
1161 pdn = pci_get_pdn(pdev);
1162
02639b0e
WY		 1163 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1164 for (j = 0; j < M64_PER_IOV; j++) {
1165 if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1166 continue;
1167 opal_pci_phb_mmio_enable(phb->opal_id,
1168 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1169 clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1170 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1171 }
781a868f
WY		 1172
1173 return 0;
1174}
1175
02639b0e 1176static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
781a868f
WY		 1177{
1178 struct pci_bus *bus;
1179 struct pci_controller *hose;
1180 struct pnv_phb *phb;
1181 struct pci_dn *pdn;
1182 unsigned int win;
1183 struct resource *res;
02639b0e 1184 int i, j;
781a868f 1185 int64_t rc;
02639b0e
WY		 1186 int total_vfs;
1187 resource_size_t size, start;
1188 int pe_num;
1189 int vf_groups;
1190 int vf_per_group;
781a868f
WY		 1191
1192 bus = pdev->bus;
1193 hose = pci_bus_to_host(bus);
1194 phb = hose->private_data;
1195 pdn = pci_get_pdn(pdev);
02639b0e 1196 total_vfs = pci_sriov_get_totalvfs(pdev);
781a868f
WY		 1197
1198 /* Initialize the m64_wins to IODA_INVALID_M64 */
1199 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
02639b0e
WY		 1200 for (j = 0; j < M64_PER_IOV; j++)
1201 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1202
1203 if (pdn->m64_per_iov == M64_PER_IOV) {
1204 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1205 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1206 roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1207 } else {
1208 vf_groups = 1;
1209 vf_per_group = 1;
1210 }
781a868f
WY		 1211
1212 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1213 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1214 if (!res->flags || !res->parent)
1215 continue;
1216
1217 if (!pnv_pci_is_mem_pref_64(res->flags))
1218 continue;
1219
02639b0e
WY		 1220 for (j = 0; j < vf_groups; j++) {
1221 do {
1222 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1223 phb->ioda.m64_bar_idx + 1, 0);
1224
1225 if (win >= phb->ioda.m64_bar_idx + 1)
1226 goto m64_failed;
1227 } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1228
1229 pdn->m64_wins[i][j] = win;
1230
1231 if (pdn->m64_per_iov == M64_PER_IOV) {
1232 size = pci_iov_resource_size(pdev,
1233 PCI_IOV_RESOURCES + i);
1234 size = size * vf_per_group;
1235 start = res->start + size * j;
1236 } else {
1237 size = resource_size(res);
1238 start = res->start;
1239 }
1240
1241 /* Map the M64 here */
1242 if (pdn->m64_per_iov == M64_PER_IOV) {
1243 pe_num = pdn->offset + j;
1244 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1245 pe_num, OPAL_M64_WINDOW_TYPE,
1246 pdn->m64_wins[i][j], 0);
1247 }
1248
1249 rc = opal_pci_set_phb_mem_window(phb->opal_id,
1250 OPAL_M64_WINDOW_TYPE,
1251 pdn->m64_wins[i][j],
1252 start,
1253 0, /* unused */
1254 size);
781a868f 1255
781a868f 1256
02639b0e
WY		 1257 if (rc != OPAL_SUCCESS) {
1258 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1259 win, rc);
1260 goto m64_failed;
1261 }
781a868f 1262
02639b0e
WY		 1263 if (pdn->m64_per_iov == M64_PER_IOV)
1264 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1265 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1266 else
1267 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1268 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
781a868f 1269
02639b0e
WY		 1270 if (rc != OPAL_SUCCESS) {
1271 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1272 win, rc);
1273 goto m64_failed;
1274 }
781a868f
WY		 1275 }
1276 }
1277 return 0;
1278
1279m64_failed:
1280 pnv_pci_vf_release_m64(pdev);
1281 return -EBUSY;
1282}
1283
1284static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1285{
1286 struct pci_bus *bus;
1287 struct pci_controller *hose;
1288 struct pnv_phb *phb;
1289 struct iommu_table *tbl;
1290 unsigned long addr;
1291 int64_t rc;
1292
1293 bus = dev->bus;
1294 hose = pci_bus_to_host(bus);
1295 phb = hose->private_data;
b348aa65 1296 tbl = pe->table_group.tables[0];
781a868f
WY		 1297 addr = tbl->it_base;
1298
1299 opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1300 pe->pe_number << 1, 1, __pa(addr),
1301 0, 0x1000);
1302
1303 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1304 pe->pe_number,
1305 (pe->pe_number << 1) + 1,
1306 pe->tce_bypass_base,
1307 0);
1308 if (rc)
1309 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1310
0eaf4def
AK		 1311 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1312 if (pe->table_group.group) {
1313 iommu_group_put(pe->table_group.group);
1314 BUG_ON(pe->table_group.group);
ac9a5889 1315 }
781a868f
WY		 1316 iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
1317 free_pages(addr, get_order(TCE32_TABLE_SIZE));
781a868f
WY		 1318}
1319
02639b0e 1320static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
781a868f
WY		 1321{
1322 struct pci_bus *bus;
1323 struct pci_controller *hose;
1324 struct pnv_phb *phb;
1325 struct pnv_ioda_pe *pe, *pe_n;
1326 struct pci_dn *pdn;
02639b0e
WY		 1327 u16 vf_index;
1328 int64_t rc;
781a868f
WY		 1329
1330 bus = pdev->bus;
1331 hose = pci_bus_to_host(bus);
1332 phb = hose->private_data;
02639b0e 1333 pdn = pci_get_pdn(pdev);
781a868f
WY		 1334
1335 if (!pdev->is_physfn)
1336 return;
1337
02639b0e
WY		 1338 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1339 int vf_group;
1340 int vf_per_group;
1341 int vf_index1;
1342
1343 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1344
1345 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1346 for (vf_index = vf_group * vf_per_group;
1347 vf_index < (vf_group + 1) * vf_per_group &&
1348 vf_index < num_vfs;
1349 vf_index++)
1350 for (vf_index1 = vf_group * vf_per_group;
1351 vf_index1 < (vf_group + 1) * vf_per_group &&
1352 vf_index1 < num_vfs;
1353 vf_index1++){
1354
1355 rc = opal_pci_set_peltv(phb->opal_id,
1356 pdn->offset + vf_index,
1357 pdn->offset + vf_index1,
1358 OPAL_REMOVE_PE_FROM_DOMAIN);
1359
1360 if (rc)
1361 dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1362 __func__,
1363 pdn->offset + vf_index1, rc);
1364 }
1365 }
1366
781a868f
WY		 1367 list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1368 if (pe->parent_dev != pdev)
1369 continue;
1370
1371 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1372
1373 /* Remove from list */
1374 mutex_lock(&phb->ioda.pe_list_mutex);
1375 list_del(&pe->list);
1376 mutex_unlock(&phb->ioda.pe_list_mutex);
1377
1378 pnv_ioda_deconfigure_pe(phb, pe);
1379
1380 pnv_ioda_free_pe(phb, pe->pe_number);
1381 }
1382}
1383
1384void pnv_pci_sriov_disable(struct pci_dev *pdev)
1385{
1386 struct pci_bus *bus;
1387 struct pci_controller *hose;
1388 struct pnv_phb *phb;
1389 struct pci_dn *pdn;
1390 struct pci_sriov *iov;
1391 u16 num_vfs;
1392
1393 bus = pdev->bus;
1394 hose = pci_bus_to_host(bus);
1395 phb = hose->private_data;
1396 pdn = pci_get_pdn(pdev);
1397 iov = pdev->sriov;
1398 num_vfs = pdn->num_vfs;
1399
1400 /* Release VF PEs */
02639b0e 1401 pnv_ioda_release_vf_PE(pdev, num_vfs);
781a868f
WY		 1402
1403 if (phb->type == PNV_PHB_IODA2) {
02639b0e
WY		 1404 if (pdn->m64_per_iov == 1)
1405 pnv_pci_vf_resource_shift(pdev, -pdn->offset);
781a868f
WY		 1406
1407 /* Release M64 windows */
1408 pnv_pci_vf_release_m64(pdev);
1409
1410 /* Release PE numbers */
1411 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1412 pdn->offset = 0;
1413 }
1414}
1415
1416static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1417 struct pnv_ioda_pe *pe);
1418static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1419{
1420 struct pci_bus *bus;
1421 struct pci_controller *hose;
1422 struct pnv_phb *phb;
1423 struct pnv_ioda_pe *pe;
1424 int pe_num;
1425 u16 vf_index;
1426 struct pci_dn *pdn;
02639b0e 1427 int64_t rc;
781a868f
WY		 1428
1429 bus = pdev->bus;
1430 hose = pci_bus_to_host(bus);
1431 phb = hose->private_data;
1432 pdn = pci_get_pdn(pdev);
1433
1434 if (!pdev->is_physfn)
1435 return;
1436
1437 /* Reserve PE for each VF */
1438 for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1439 pe_num = pdn->offset + vf_index;
1440
1441 pe = &phb->ioda.pe_array[pe_num];
1442 pe->pe_number = pe_num;
1443 pe->phb = phb;
1444 pe->flags = PNV_IODA_PE_VF;
1445 pe->pbus = NULL;
1446 pe->parent_dev = pdev;
1447 pe->tce32_seg = -1;
1448 pe->mve_number = -1;
1449 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1450 pci_iov_virtfn_devfn(pdev, vf_index);
1451
1452 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1453 hose->global_number, pdev->bus->number,
1454 PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1455 PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1456
1457 if (pnv_ioda_configure_pe(phb, pe)) {
1458 /* XXX What do we do here ? */
1459 if (pe_num)
1460 pnv_ioda_free_pe(phb, pe_num);
1461 pe->pdev = NULL;
1462 continue;
1463 }
1464
781a868f
WY		 1465 /* Put PE to the list */
1466 mutex_lock(&phb->ioda.pe_list_mutex);
1467 list_add_tail(&pe->list, &phb->ioda.pe_list);
1468 mutex_unlock(&phb->ioda.pe_list_mutex);
1469
1470 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1471 }
02639b0e
WY		 1472
1473 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1474 int vf_group;
1475 int vf_per_group;
1476 int vf_index1;
1477
1478 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1479
1480 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1481 for (vf_index = vf_group * vf_per_group;
1482 vf_index < (vf_group + 1) * vf_per_group &&
1483 vf_index < num_vfs;
1484 vf_index++) {
1485 for (vf_index1 = vf_group * vf_per_group;
1486 vf_index1 < (vf_group + 1) * vf_per_group &&
1487 vf_index1 < num_vfs;
1488 vf_index1++) {
1489
1490 rc = opal_pci_set_peltv(phb->opal_id,
1491 pdn->offset + vf_index,
1492 pdn->offset + vf_index1,
1493 OPAL_ADD_PE_TO_DOMAIN);
1494
1495 if (rc)
1496 dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1497 __func__,
1498 pdn->offset + vf_index1, rc);
1499 }
1500 }
1501 }
1502 }
781a868f
WY		 1503}
1504
1505int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1506{
1507 struct pci_bus *bus;
1508 struct pci_controller *hose;
1509 struct pnv_phb *phb;
1510 struct pci_dn *pdn;
1511 int ret;
1512
1513 bus = pdev->bus;
1514 hose = pci_bus_to_host(bus);
1515 phb = hose->private_data;
1516 pdn = pci_get_pdn(pdev);
1517
1518 if (phb->type == PNV_PHB_IODA2) {
1519 /* Calculate available PE for required VFs */
1520 mutex_lock(&phb->ioda.pe_alloc_mutex);
1521 pdn->offset = bitmap_find_next_zero_area(
1522 phb->ioda.pe_alloc, phb->ioda.total_pe,
1523 0, num_vfs, 0);
1524 if (pdn->offset >= phb->ioda.total_pe) {
1525 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1526 dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1527 pdn->offset = 0;
1528 return -EBUSY;
1529 }
1530 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1531 pdn->num_vfs = num_vfs;
1532 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1533
1534 /* Assign M64 window accordingly */
02639b0e 1535 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
781a868f
WY		 1536 if (ret) {
1537 dev_info(&pdev->dev, "Not enough M64 window resources\n");
1538 goto m64_failed;
1539 }
1540
1541 /*
1542 * When using one M64 BAR to map one IOV BAR, we need to shift
1543 * the IOV BAR according to the PE# allocated to the VFs.
1544 * Otherwise, the PE# for the VF will conflict with others.
1545 */
02639b0e
WY		 1546 if (pdn->m64_per_iov == 1) {
1547 ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1548 if (ret)
1549 goto m64_failed;
1550 }
781a868f
WY		 1551 }
1552
1553 /* Setup VF PEs */
1554 pnv_ioda_setup_vf_PE(pdev, num_vfs);
1555
1556 return 0;
1557
1558m64_failed:
1559 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1560 pdn->offset = 0;
1561
1562 return ret;
1563}
1564
a8b2f828
GS		 1565int pcibios_sriov_disable(struct pci_dev *pdev)
1566{
781a868f
WY		 1567 pnv_pci_sriov_disable(pdev);
1568
a8b2f828
GS		 1569 /* Release PCI data */
1570 remove_dev_pci_data(pdev);
1571 return 0;
1572}
1573
1574int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1575{
1576 /* Allocate PCI data */
1577 add_dev_pci_data(pdev);
781a868f
WY		 1578
1579 pnv_pci_sriov_enable(pdev, num_vfs);
a8b2f828
GS		 1580 return 0;
1581}
1582#endif /* CONFIG_PCI_IOV */
1583
959c9bdd 1584static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
184cd4a3 1585{
b72c1f65 1586 struct pci_dn *pdn = pci_get_pdn(pdev);
959c9bdd 1587 struct pnv_ioda_pe *pe;
184cd4a3 1588
959c9bdd
GS		 1589 /*
1590 * The function can be called while the PE#
1591 * hasn't been assigned. Do nothing for the
1592 * case.
1593 */
1594 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1595 return;
184cd4a3 1596
959c9bdd 1597 pe = &phb->ioda.pe_array[pdn->pe_number];
cd15b048 1598 WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
b348aa65 1599 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
4617082e
AK		 1600 /*
1601 * Note: iommu_add_device() will fail here as
1602 * for physical PE: the device is already added by now;
1603 * for virtual PE: sysfs entries are not ready yet and
1604 * tce_iommu_bus_notifier will add the device to a group later.
1605 */
184cd4a3
BH		 1606}
1607
763d2d8d 1608static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
cd15b048 1609{
763d2d8d
DA		 1610 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1611 struct pnv_phb *phb = hose->private_data;
cd15b048
BH		 1612 struct pci_dn *pdn = pci_get_pdn(pdev);
1613 struct pnv_ioda_pe *pe;
1614 uint64_t top;
1615 bool bypass = false;
1616
1617 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1618 return -ENODEV;;
1619
1620 pe = &phb->ioda.pe_array[pdn->pe_number];
1621 if (pe->tce_bypass_enabled) {
1622 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1623 bypass = (dma_mask >= top);
1624 }
1625
1626 if (bypass) {
1627 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1628 set_dma_ops(&pdev->dev, &dma_direct_ops);
1629 set_dma_offset(&pdev->dev, pe->tce_bypass_base);
1630 } else {
1631 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1632 set_dma_ops(&pdev->dev, &dma_iommu_ops);
b348aa65 1633 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
cd15b048 1634 }
a32305bf 1635 *pdev->dev.dma_mask = dma_mask;
cd15b048
BH		 1636 return 0;
1637}
1638
fe7e85c6
GS		 1639static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1640 struct pci_dev *pdev)
1641{
1642 struct pci_dn *pdn = pci_get_pdn(pdev);
1643 struct pnv_ioda_pe *pe;
1644 u64 end, mask;
1645
1646 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1647 return 0;
1648
1649 pe = &phb->ioda.pe_array[pdn->pe_number];
1650 if (!pe->tce_bypass_enabled)
1651 return __dma_get_required_mask(&pdev->dev);
1652
1653
1654 end = pe->tce_bypass_base + memblock_end_of_DRAM();
1655 mask = 1ULL << (fls64(end) - 1);
1656 mask += mask - 1;
1657
1658 return mask;
1659}
1660
dff4a39e 1661static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
ea30e99e 1662 struct pci_bus *bus)
74251fe2
BH		 1663{
1664 struct pci_dev *dev;
1665
1666 list_for_each_entry(dev, &bus->devices, bus_list) {
b348aa65 1667 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
4617082e 1668 iommu_add_device(&dev->dev);
dff4a39e 1669
74251fe2 1670 if (dev->subordinate)
ea30e99e 1671 pnv_ioda_setup_bus_dma(pe, dev->subordinate);
74251fe2
BH		 1672 }
1673}
1674
decbda25
AK		 1675static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1676 unsigned long index, unsigned long npages, bool rm)
4cce9550 1677{
0eaf4def
AK		 1678 struct iommu_table_group_link *tgl = list_first_entry_or_null(
1679 &tbl->it_group_list, struct iommu_table_group_link,
1680 next);
1681 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
b348aa65 1682 struct pnv_ioda_pe, table_group);
3ad26e5c 1683 __be64 __iomem *invalidate = rm ?
5780fb04
AK		 1684 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1685 pe->phb->ioda.tce_inval_reg;
4cce9550 1686 unsigned long start, end, inc;
b0376c9b 1687 const unsigned shift = tbl->it_page_shift;
4cce9550 1688
decbda25
AK		 1689 start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1690 end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1691 npages - 1);
4cce9550
GS		 1692
1693 /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1694 if (tbl->it_busno) {
b0376c9b
AK		 1695 start <<= shift;
1696 end <<= shift;
1697 inc = 128ull << shift;
4cce9550
GS		 1698 start |= tbl->it_busno;
1699 end |= tbl->it_busno;
1700 } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1701 /* p7ioc-style invalidation, 2 TCEs per write */
1702 start |= (1ull << 63);
1703 end |= (1ull << 63);
1704 inc = 16;
1705 } else {
1706 /* Default (older HW) */
1707 inc = 128;
1708 }
1709
1710 end |= inc - 1; /* round up end to be different than start */
1711
1712 mb(); /* Ensure above stores are visible */
1713 while (start <= end) {
8e0a1611 1714 if (rm)
3ad26e5c 1715 __raw_rm_writeq(cpu_to_be64(start), invalidate);
8e0a1611 1716 else
3ad26e5c 1717 __raw_writeq(cpu_to_be64(start), invalidate);
4cce9550
GS		 1718 start += inc;
1719 }
1720
1721 /*
1722 * The iommu layer will do another mb() for us on build()
1723 * and we don't care on free()
1724 */
1725}
1726
decbda25
AK		 1727static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1728 long npages, unsigned long uaddr,
1729 enum dma_data_direction direction,
1730 struct dma_attrs *attrs)
1731{
1732 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1733 attrs);
1734
1735 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1736 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1737
1738 return ret;
1739}
1740
05c6cfb9
AK		 1741#ifdef CONFIG_IOMMU_API
1742static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
1743 unsigned long *hpa, enum dma_data_direction *direction)
1744{
1745 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1746
1747 if (!ret && (tbl->it_type &
1748 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1749 pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
1750
1751 return ret;
1752}
1753#endif
1754
decbda25
AK		 1755static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1756 long npages)
1757{
1758 pnv_tce_free(tbl, index, npages);
1759
1760 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1761 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1762}
1763
da004c36 1764static struct iommu_table_ops pnv_ioda1_iommu_ops = {
decbda25 1765 .set = pnv_ioda1_tce_build,
05c6cfb9
AK		 1766#ifdef CONFIG_IOMMU_API
1767 .exchange = pnv_ioda1_tce_xchg,
1768#endif
decbda25 1769 .clear = pnv_ioda1_tce_free,
da004c36
AK		 1770 .get = pnv_tce_get,
1771};
1772
5780fb04
AK		 1773static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1774{
1775 /* 01xb - invalidate TCEs that match the specified PE# */
1776 unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1777 struct pnv_phb *phb = pe->phb;
1778
1779 if (!phb->ioda.tce_inval_reg)
1780 return;
1781
1782 mb(); /* Ensure above stores are visible */
1783 __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1784}
1785
e57080f1
AK		 1786static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
1787 __be64 __iomem *invalidate, unsigned shift,
1788 unsigned long index, unsigned long npages)
4cce9550
GS		 1789{
1790 unsigned long start, end, inc;
4cce9550
GS		 1791
1792 /* We'll invalidate DMA address in PE scope */
b0376c9b 1793 start = 0x2ull << 60;
e57080f1 1794 start |= (pe_number & 0xFF);
4cce9550
GS		 1795 end = start;
1796
1797 /* Figure out the start, end and step */
decbda25
AK		 1798 start |= (index << shift);
1799 end |= ((index + npages - 1) << shift);
b0376c9b 1800 inc = (0x1ull << shift);
4cce9550
GS		 1801 mb();
1802
1803 while (start <= end) {
8e0a1611 1804 if (rm)
3ad26e5c 1805 __raw_rm_writeq(cpu_to_be64(start), invalidate);
8e0a1611 1806 else
3ad26e5c 1807 __raw_writeq(cpu_to_be64(start), invalidate);
4cce9550
GS		 1808 start += inc;
1809 }
1810}
1811
e57080f1
AK		 1812static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1813 unsigned long index, unsigned long npages, bool rm)
1814{
1815 struct iommu_table_group_link *tgl;
1816
1817 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1818 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1819 struct pnv_ioda_pe, table_group);
1820 __be64 __iomem *invalidate = rm ?
1821 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1822 pe->phb->ioda.tce_inval_reg;
1823
1824 pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
1825 invalidate, tbl->it_page_shift,
1826 index, npages);
1827 }
1828}
1829
decbda25
AK		 1830static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1831 long npages, unsigned long uaddr,
1832 enum dma_data_direction direction,
1833 struct dma_attrs *attrs)
4cce9550 1834{
decbda25
AK		 1835 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1836 attrs);
4cce9550 1837
decbda25
AK		 1838 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1839 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1840
1841 return ret;
1842}
1843
05c6cfb9
AK		 1844#ifdef CONFIG_IOMMU_API
1845static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
1846 unsigned long *hpa, enum dma_data_direction *direction)
1847{
1848 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1849
1850 if (!ret && (tbl->it_type &
1851 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1852 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
1853
1854 return ret;
1855}
1856#endif
1857
decbda25
AK		 1858static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1859 long npages)
1860{
1861 pnv_tce_free(tbl, index, npages);
1862
1863 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1864 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
4cce9550
GS		 1865}
1866
da004c36 1867static struct iommu_table_ops pnv_ioda2_iommu_ops = {
decbda25 1868 .set = pnv_ioda2_tce_build,
05c6cfb9
AK		 1869#ifdef CONFIG_IOMMU_API
1870 .exchange = pnv_ioda2_tce_xchg,
1871#endif
decbda25 1872 .clear = pnv_ioda2_tce_free,
da004c36
AK		 1873 .get = pnv_tce_get,
1874};
1875
cad5cef6
GKH		 1876static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1877 struct pnv_ioda_pe *pe, unsigned int base,
1878 unsigned int segs)
184cd4a3
BH		 1879{
1880
1881 struct page *tce_mem = NULL;
184cd4a3
BH		 1882 struct iommu_table *tbl;
1883 unsigned int i;
1884 int64_t rc;
1885 void *addr;
1886
184cd4a3
BH		 1887 /* XXX FIXME: Handle 64-bit only DMA devices */
1888 /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1889 /* XXX FIXME: Allocate multi-level tables on PHB3 */
1890
1891 /* We shouldn't already have a 32-bit DMA associated */
1892 if (WARN_ON(pe->tce32_seg >= 0))
1893 return;
1894
0eaf4def 1895 tbl = pnv_pci_table_alloc(phb->hose->node);
b348aa65
AK		 1896 iommu_register_group(&pe->table_group, phb->hose->global_number,
1897 pe->pe_number);
0eaf4def 1898 pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
c5773822 1899
184cd4a3
BH		 1900 /* Grab a 32-bit TCE table */
1901 pe->tce32_seg = base;
1902 pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1903 (base << 28), ((base + segs) << 28) - 1);
1904
1905 /* XXX Currently, we allocate one big contiguous table for the
1906 * TCEs. We only really need one chunk per 256M of TCE space
1907 * (ie per segment) but that's an optimization for later, it
1908 * requires some added smarts with our get/put_tce implementation
1909 */
1910 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1911 get_order(TCE32_TABLE_SIZE * segs));
1912 if (!tce_mem) {
1913 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1914 goto fail;
1915 }
1916 addr = page_address(tce_mem);
1917 memset(addr, 0, TCE32_TABLE_SIZE * segs);
1918
1919 /* Configure HW */
1920 for (i = 0; i < segs; i++) {
1921 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1922 pe->pe_number,
1923 base + i, 1,
1924 __pa(addr) + TCE32_TABLE_SIZE * i,
1925 TCE32_TABLE_SIZE, 0x1000);
1926 if (rc) {
1927 pe_err(pe, " Failed to configure 32-bit TCE table,"
1928 " err %ld\n", rc);
1929 goto fail;
1930 }
1931 }
1932
1933 /* Setup linux iommu table */
184cd4a3 1934 pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
8fa5d454 1935 base << 28, IOMMU_PAGE_SHIFT_4K);
184cd4a3
BH		 1936
1937 /* OPAL variant of P7IOC SW invalidated TCEs */
5780fb04 1938 if (phb->ioda.tce_inval_reg)
65fd766b
GS		 1939 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1940 TCE_PCI_SWINV_FREE |
1941 TCE_PCI_SWINV_PAIR);
5780fb04 1942
da004c36 1943 tbl->it_ops = &pnv_ioda1_iommu_ops;
184cd4a3
BH		 1944 iommu_init_table(tbl, phb->hose->node);
1945
781a868f 1946 if (pe->flags & PNV_IODA_PE_DEV) {
4617082e
AK		 1947 /*
1948 * Setting table base here only for carrying iommu_group
1949 * further down to let iommu_add_device() do the job.
1950 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1951 */
1952 set_iommu_table_base(&pe->pdev->dev, tbl);
1953 iommu_add_device(&pe->pdev->dev);
c5773822 1954 } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
ea30e99e 1955 pnv_ioda_setup_bus_dma(pe, pe->pbus);
74251fe2 1956
184cd4a3
BH		 1957 return;
1958 fail:
1959 /* XXX Failure: Try to fallback to 64-bit only ? */
1960 if (pe->tce32_seg >= 0)
1961 pe->tce32_seg = -1;
1962 if (tce_mem)
1963 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
0eaf4def
AK		 1964 if (tbl) {
1965 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1966 iommu_free_table(tbl, "pnv");
1967 }
184cd4a3
BH		 1968}
1969
f87a8864 1970static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
cd15b048 1971{
cd15b048
BH		 1972 uint16_t window_id = (pe->pe_number << 1 ) + 1;
1973 int64_t rc;
1974
1975 pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
1976 if (enable) {
1977 phys_addr_t top = memblock_end_of_DRAM();
1978
1979 top = roundup_pow_of_two(top);
1980 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1981 pe->pe_number,
1982 window_id,
1983 pe->tce_bypass_base,
1984 top);
1985 } else {
1986 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1987 pe->pe_number,
1988 window_id,
1989 pe->tce_bypass_base,
1990 0);
cd15b048
BH		 1991 }
1992 if (rc)
1993 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
1994 else
1995 pe->tce_bypass_enabled = enable;
1996}
1997
f87a8864
AK		 1998#ifdef CONFIG_IOMMU_API
1999static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
cd15b048 2000{
f87a8864
AK		 2001 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2002 table_group);
cd15b048 2003
f87a8864
AK		 2004 iommu_take_ownership(table_group->tables[0]);
2005 pnv_pci_ioda2_set_bypass(pe, false);
2006}
cd15b048 2007
f87a8864
AK		 2008static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2009{
2010 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2011 table_group);
2012
2013 iommu_release_ownership(table_group->tables[0]);
2014 pnv_pci_ioda2_set_bypass(pe, true);
cd15b048
BH		 2015}
2016
f87a8864
AK		 2017static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
2018 .take_ownership = pnv_ioda2_take_ownership,
2019 .release_ownership = pnv_ioda2_release_ownership,
2020};
2021#endif
2022
5780fb04
AK		 2023static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
2024{
2025 const __be64 *swinvp;
2026
2027 /* OPAL variant of PHB3 invalidated TCEs */
2028 swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
2029 if (!swinvp)
2030 return;
2031
2032 phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
2033 phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
2034}
2035
373f5657
GS		 2036static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2037 struct pnv_ioda_pe *pe)
2038{
2039 struct page *tce_mem = NULL;
2040 void *addr;
373f5657
GS		 2041 struct iommu_table *tbl;
2042 unsigned int tce_table_size, end;
2043 int64_t rc;
2044
2045 /* We shouldn't already have a 32-bit DMA associated */
2046 if (WARN_ON(pe->tce32_seg >= 0))
2047 return;
2048
f87a8864
AK		 2049 /* TVE #1 is selected by PCI address bit 59 */
2050 pe->tce_bypass_base = 1ull << 59;
2051
0eaf4def 2052 tbl = pnv_pci_table_alloc(phb->hose->node);
b348aa65
AK		 2053 iommu_register_group(&pe->table_group, phb->hose->global_number,
2054 pe->pe_number);
0eaf4def 2055 pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
c5773822 2056
373f5657
GS		 2057 /* The PE will reserve all possible 32-bits space */
2058 pe->tce32_seg = 0;
2059 end = (1 << ilog2(phb->ioda.m32_pci_base));
2060 tce_table_size = (end / 0x1000) * 8;
2061 pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
2062 end);
2063
2064 /* Allocate TCE table */
2065 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
2066 get_order(tce_table_size));
2067 if (!tce_mem) {
2068 pe_err(pe, "Failed to allocate a 32-bit TCE memory\n");
2069 goto fail;
2070 }
2071 addr = page_address(tce_mem);
2072 memset(addr, 0, tce_table_size);
2073
2074 /*
2075 * Map TCE table through TVT. The TVE index is the PE number
2076 * shifted by 1 bit for 32-bits DMA space.
2077 */
2078 rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2079 pe->pe_number << 1, 1, __pa(addr),
2080 tce_table_size, 0x1000);
2081 if (rc) {
2082 pe_err(pe, "Failed to configure 32-bit TCE table,"
2083 " err %ld\n", rc);
2084 goto fail;
2085 }
2086
5780fb04
AK		 2087 pnv_pci_ioda2_tce_invalidate_entire(pe);
2088
373f5657 2089 /* Setup linux iommu table */
8fa5d454
AK		 2090 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0,
2091 IOMMU_PAGE_SHIFT_4K);
373f5657
GS		 2092
2093 /* OPAL variant of PHB3 invalidated TCEs */
5780fb04 2094 if (phb->ioda.tce_inval_reg)
65fd766b 2095 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
5780fb04 2096
da004c36 2097 tbl->it_ops = &pnv_ioda2_iommu_ops;
373f5657 2098 iommu_init_table(tbl, phb->hose->node);
f87a8864
AK		 2099#ifdef CONFIG_IOMMU_API
2100 pe->table_group.ops = &pnv_pci_ioda2_ops;
2101#endif
373f5657 2102
781a868f 2103 if (pe->flags & PNV_IODA_PE_DEV) {
4617082e
AK		 2104 /*
2105 * Setting table base here only for carrying iommu_group
2106 * further down to let iommu_add_device() do the job.
2107 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2108 */
2109 set_iommu_table_base(&pe->pdev->dev, tbl);
2110 iommu_add_device(&pe->pdev->dev);
c5773822 2111 } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
ea30e99e 2112 pnv_ioda_setup_bus_dma(pe, pe->pbus);
74251fe2 2113
cd15b048 2114 /* Also create a bypass window */
4e287840 2115 if (!pnv_iommu_bypass_disabled)
f87a8864 2116 pnv_pci_ioda2_set_bypass(pe, true);
4e287840 2117
373f5657
GS		 2118 return;
2119fail:
2120 if (pe->tce32_seg >= 0)
2121 pe->tce32_seg = -1;
2122 if (tce_mem)
2123 __free_pages(tce_mem, get_order(tce_table_size));
0eaf4def
AK		 2124 if (tbl) {
2125 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
2126 iommu_free_table(tbl, "pnv");
2127 }
373f5657
GS		 2128}
2129
cad5cef6 2130static void pnv_ioda_setup_dma(struct pnv_phb *phb)
184cd4a3
BH		 2131{
2132 struct pci_controller *hose = phb->hose;
2133 unsigned int residual, remaining, segs, tw, base;
2134 struct pnv_ioda_pe *pe;
2135
2136 /* If we have more PE# than segments available, hand out one
2137 * per PE until we run out and let the rest fail. If not,
2138 * then we assign at least one segment per PE, plus more based
2139 * on the amount of devices under that PE
2140 */
2141 if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2142 residual = 0;
2143 else
2144 residual = phb->ioda.tce32_count -
2145 phb->ioda.dma_pe_count;
2146
2147 pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2148 hose->global_number, phb->ioda.tce32_count);
2149 pr_info("PCI: %d PE# for a total weight of %d\n",
2150 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2151
5780fb04
AK		 2152 pnv_pci_ioda_setup_opal_tce_kill(phb);
2153
184cd4a3
BH		 2154 /* Walk our PE list and configure their DMA segments, hand them
2155 * out one base segment plus any residual segments based on
2156 * weight
2157 */
2158 remaining = phb->ioda.tce32_count;
2159 tw = phb->ioda.dma_weight;
2160 base = 0;
7ebdf956 2161 list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
184cd4a3
BH		 2162 if (!pe->dma_weight)
2163 continue;
2164 if (!remaining) {
2165 pe_warn(pe, "No DMA32 resources available\n");
2166 continue;
2167 }
2168 segs = 1;
2169 if (residual) {
2170 segs += ((pe->dma_weight * residual) + (tw / 2)) / tw;
2171 if (segs > remaining)
2172 segs = remaining;
2173 }
373f5657
GS		 2174
2175 /*
2176 * For IODA2 compliant PHB3, we needn't care about the weight.
2177 * The all available 32-bits DMA space will be assigned to
2178 * the specific PE.
2179 */
2180 if (phb->type == PNV_PHB_IODA1) {
2181 pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2182 pe->dma_weight, segs);
2183 pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2184 } else {
2185 pe_info(pe, "Assign DMA32 space\n");
2186 segs = 0;
2187 pnv_pci_ioda2_setup_dma_pe(phb, pe);
2188 }
2189
184cd4a3
BH		 2190 remaining -= segs;
2191 base += segs;
2192 }
2193}
2194
2195#ifdef CONFIG_PCI_MSI
137436c9
GS		 2196static void pnv_ioda2_msi_eoi(struct irq_data *d)
2197{
2198 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2199 struct irq_chip *chip = irq_data_get_irq_chip(d);
2200 struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2201 ioda.irq_chip);
2202 int64_t rc;
2203
2204 rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2205 WARN_ON_ONCE(rc);
2206
2207 icp_native_eoi(d);
2208}
2209
fd9a1c26
IM		 2210
2211static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2212{
2213 struct irq_data *idata;
2214 struct irq_chip *ichip;
2215
2216 if (phb->type != PNV_PHB_IODA2)
2217 return;
2218
2219 if (!phb->ioda.irq_chip_init) {
2220 /*
2221 * First time we setup an MSI IRQ, we need to setup the
2222 * corresponding IRQ chip to route correctly.
2223 */
2224 idata = irq_get_irq_data(virq);
2225 ichip = irq_data_get_irq_chip(idata);
2226 phb->ioda.irq_chip_init = 1;
2227 phb->ioda.irq_chip = *ichip;
2228 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2229 }
2230 irq_set_chip(virq, &phb->ioda.irq_chip);
2231}
2232
80c49c7e
IM		 2233#ifdef CONFIG_CXL_BASE
2234
6f963ec2 2235struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
80c49c7e
IM		 2236{
2237 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2238
6f963ec2 2239 return of_node_get(hose->dn);
80c49c7e 2240}
6f963ec2 2241EXPORT_SYMBOL(pnv_pci_get_phb_node);
80c49c7e 2242
1212aa1c 2243int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
80c49c7e
IM		 2244{
2245 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2246 struct pnv_phb *phb = hose->private_data;
2247 struct pnv_ioda_pe *pe;
2248 int rc;
2249
2250 pe = pnv_ioda_get_pe(dev);
2251 if (!pe)
2252 return -ENODEV;
2253
2254 pe_info(pe, "Switching PHB to CXL\n");
2255
1212aa1c 2256 rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
80c49c7e
IM		 2257 if (rc)
2258 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2259
2260 return rc;
2261}
1212aa1c 2262EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
80c49c7e
IM		 2263
2264/* Find PHB for cxl dev and allocate MSI hwirqs?
2265 * Returns the absolute hardware IRQ number
2266 */
2267int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2268{
2269 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2270 struct pnv_phb *phb = hose->private_data;
2271 int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2272
2273 if (hwirq < 0) {
2274 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2275 return -ENOSPC;
2276 }
2277
2278 return phb->msi_base + hwirq;
2279}
2280EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2281
2282void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2283{
2284 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2285 struct pnv_phb *phb = hose->private_data;
2286
2287 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2288}
2289EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2290
2291void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2292 struct pci_dev *dev)
2293{
2294 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2295 struct pnv_phb *phb = hose->private_data;
2296 int i, hwirq;
2297
2298 for (i = 1; i < CXL_IRQ_RANGES; i++) {
2299 if (!irqs->range[i])
2300 continue;
2301 pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n",
2302 i, irqs->offset[i],
2303 irqs->range[i]);
2304 hwirq = irqs->offset[i] - phb->msi_base;
2305 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2306 irqs->range[i]);
2307 }
2308}
2309EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2310
2311int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2312 struct pci_dev *dev, int num)
2313{
2314 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2315 struct pnv_phb *phb = hose->private_data;
2316 int i, hwirq, try;
2317
2318 memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2319
2320 /* 0 is reserved for the multiplexed PSL DSI interrupt */
2321 for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2322 try = num;
2323 while (try) {
2324 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2325 if (hwirq >= 0)
2326 break;
2327 try /= 2;
2328 }
2329 if (!try)
2330 goto fail;
2331
2332 irqs->offset[i] = phb->msi_base + hwirq;
2333 irqs->range[i] = try;
2334 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n",
2335 i, irqs->offset[i], irqs->range[i]);
2336 num -= try;
2337 }
2338 if (num)
2339 goto fail;
2340
2341 return 0;
2342fail:
2343 pnv_cxl_release_hwirq_ranges(irqs, dev);
2344 return -ENOSPC;
2345}
2346EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2347
2348int pnv_cxl_get_irq_count(struct pci_dev *dev)
2349{
2350 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2351 struct pnv_phb *phb = hose->private_data;
2352
2353 return phb->msi_bmp.irq_count;
2354}
2355EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2356
2357int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2358 unsigned int virq)
2359{
2360 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2361 struct pnv_phb *phb = hose->private_data;
2362 unsigned int xive_num = hwirq - phb->msi_base;
2363 struct pnv_ioda_pe *pe;
2364 int rc;
2365
2366 if (!(pe = pnv_ioda_get_pe(dev)))
2367 return -ENODEV;
2368
2369 /* Assign XIVE to PE */
2370 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2371 if (rc) {
2372 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2373 "hwirq 0x%x XIVE 0x%x PE\n",
2374 pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2375 return -EIO;
2376 }
2377 set_msi_irq_chip(phb, virq);
2378
2379 return 0;
2380}
2381EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2382#endif
2383
184cd4a3 2384static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
137436c9
GS		 2385 unsigned int hwirq, unsigned int virq,
2386 unsigned int is_64, struct msi_msg *msg)
184cd4a3
BH		 2387{
2388 struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2389 unsigned int xive_num = hwirq - phb->msi_base;
3a1a4661 2390 __be32 data;
184cd4a3
BH		 2391 int rc;
2392
2393 /* No PE assigned ? bail out ... no MSI for you ! */
2394 if (pe == NULL)
2395 return -ENXIO;
2396
2397 /* Check if we have an MVE */
2398 if (pe->mve_number < 0)
2399 return -ENXIO;
2400
b72c1f65 2401 /* Force 32-bit MSI on some broken devices */
36074381 2402 if (dev->no_64bit_msi)
b72c1f65
BH		 2403 is_64 = 0;
2404
184cd4a3
BH		 2405 /* Assign XIVE to PE */
2406 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2407 if (rc) {
2408 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2409 pci_name(dev), rc, xive_num);
2410 return -EIO;
2411 }
2412
2413 if (is_64) {
3a1a4661
BH		 2414 __be64 addr64;
2415
184cd4a3
BH		 2416 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2417 &addr64, &data);
2418 if (rc) {
2419 pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2420 pci_name(dev), rc);
2421 return -EIO;
2422 }
3a1a4661
BH		 2423 msg->address_hi = be64_to_cpu(addr64) >> 32;
2424 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
184cd4a3 2425 } else {
3a1a4661
BH		 2426 __be32 addr32;
2427
184cd4a3
BH		 2428 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2429 &addr32, &data);
2430 if (rc) {
2431 pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2432 pci_name(dev), rc);
2433 return -EIO;
2434 }
2435 msg->address_hi = 0;
3a1a4661 2436 msg->address_lo = be32_to_cpu(addr32);
184cd4a3 2437 }
3a1a4661 2438 msg->data = be32_to_cpu(data);
184cd4a3 2439
fd9a1c26 2440 set_msi_irq_chip(phb, virq);
137436c9 2441
184cd4a3
BH		 2442 pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2443 " address=%x_%08x data=%x PE# %d\n",
2444 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2445 msg->address_hi, msg->address_lo, data, pe->pe_number);
2446
2447 return 0;
2448}
2449
2450static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2451{
fb1b55d6 2452 unsigned int count;
184cd4a3
BH		 2453 const __be32 *prop = of_get_property(phb->hose->dn,
2454 "ibm,opal-msi-ranges", NULL);
2455 if (!prop) {
2456 /* BML Fallback */
2457 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2458 }
2459 if (!prop)
2460 return;
2461
2462 phb->msi_base = be32_to_cpup(prop);
fb1b55d6
GS		 2463 count = be32_to_cpup(prop + 1);
2464 if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
184cd4a3
BH		 2465 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2466 phb->hose->global_number);
2467 return;
2468 }
fb1b55d6 2469
184cd4a3
BH		 2470 phb->msi_setup = pnv_pci_ioda_msi_setup;
2471 phb->msi32_support = 1;
2472 pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
fb1b55d6 2473 count, phb->msi_base);
184cd4a3
BH		 2474}
2475#else
2476static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2477#endif /* CONFIG_PCI_MSI */
2478
6e628c7d
WY		 2479#ifdef CONFIG_PCI_IOV
2480static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2481{
2482 struct pci_controller *hose;
2483 struct pnv_phb *phb;
2484 struct resource *res;
2485 int i;
2486 resource_size_t size;
2487 struct pci_dn *pdn;
5b88ec22 2488 int mul, total_vfs;
6e628c7d
WY		 2489
2490 if (!pdev->is_physfn || pdev->is_added)
2491 return;
2492
2493 hose = pci_bus_to_host(pdev->bus);
2494 phb = hose->private_data;
2495
2496 pdn = pci_get_pdn(pdev);
2497 pdn->vfs_expanded = 0;
2498
5b88ec22
WY		 2499 total_vfs = pci_sriov_get_totalvfs(pdev);
2500 pdn->m64_per_iov = 1;
2501 mul = phb->ioda.total_pe;
2502
2503 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2504 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2505 if (!res->flags || res->parent)
2506 continue;
2507 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2508 dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2509 i, res);
2510 continue;
2511 }
2512
2513 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2514
2515 /* bigger than 64M */
2516 if (size > (1 << 26)) {
2517 dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2518 i, res);
2519 pdn->m64_per_iov = M64_PER_IOV;
2520 mul = roundup_pow_of_two(total_vfs);
2521 break;
2522 }
2523 }
2524
6e628c7d
WY		 2525 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2526 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2527 if (!res->flags || res->parent)
2528 continue;
2529 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2530 dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2531 i, res);
2532 continue;
2533 }
2534
2535 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2536 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
5b88ec22 2537 res->end = res->start + size * mul - 1;
6e628c7d
WY		 2538 dev_dbg(&pdev->dev, " %pR\n", res);
2539 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
5b88ec22 2540 i, res, mul);
6e628c7d 2541 }
5b88ec22 2542 pdn->vfs_expanded = mul;
6e628c7d
WY		 2543}
2544#endif /* CONFIG_PCI_IOV */
2545
11685bec
GS		 2546/*
2547 * This function is supposed to be called on basis of PE from top
2548 * to bottom style. So the the I/O or MMIO segment assigned to
2549 * parent PE could be overrided by its child PEs if necessary.
2550 */
cad5cef6
GKH		 2551static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2552 struct pnv_ioda_pe *pe)
11685bec
GS		 2553{
2554 struct pnv_phb *phb = hose->private_data;
2555 struct pci_bus_region region;
2556 struct resource *res;
2557 int i, index;
2558 int rc;
2559
2560 /*
2561 * NOTE: We only care PCI bus based PE for now. For PCI
2562 * device based PE, for example SRIOV sensitive VF should
2563 * be figured out later.
2564 */
2565 BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2566
2567 pci_bus_for_each_resource(pe->pbus, res, i) {
2568 if (!res || !res->flags ||
2569 res->start > res->end)
2570 continue;
2571
2572 if (res->flags & IORESOURCE_IO) {
2573 region.start = res->start - phb->ioda.io_pci_base;
2574 region.end = res->end - phb->ioda.io_pci_base;
2575 index = region.start / phb->ioda.io_segsize;
2576
2577 while (index < phb->ioda.total_pe &&
2578 region.start <= region.end) {
2579 phb->ioda.io_segmap[index] = pe->pe_number;
2580 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2581 pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2582 if (rc != OPAL_SUCCESS) {
2583 pr_err("%s: OPAL error %d when mapping IO "
2584 "segment #%d to PE#%d\n",
2585 __func__, rc, index, pe->pe_number);
2586 break;
2587 }
2588
2589 region.start += phb->ioda.io_segsize;
2590 index++;
2591 }
027fa02f
GS		 2592 } else if ((res->flags & IORESOURCE_MEM) &&
2593 !pnv_pci_is_mem_pref_64(res->flags)) {
11685bec 2594 region.start = res->start -
3fd47f06 2595 hose->mem_offset[0] -
11685bec
GS		 2596 phb->ioda.m32_pci_base;
2597 region.end = res->end -
3fd47f06 2598 hose->mem_offset[0] -
11685bec
GS		 2599 phb->ioda.m32_pci_base;
2600 index = region.start / phb->ioda.m32_segsize;
2601
2602 while (index < phb->ioda.total_pe &&
2603 region.start <= region.end) {
2604 phb->ioda.m32_segmap[index] = pe->pe_number;
2605 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2606 pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2607 if (rc != OPAL_SUCCESS) {
2608 pr_err("%s: OPAL error %d when mapping M32 "
2609 "segment#%d to PE#%d",
2610 __func__, rc, index, pe->pe_number);
2611 break;
2612 }
2613
2614 region.start += phb->ioda.m32_segsize;
2615 index++;
2616 }
2617 }
2618 }
2619}
2620
cad5cef6 2621static void pnv_pci_ioda_setup_seg(void)
11685bec
GS		 2622{
2623 struct pci_controller *tmp, *hose;
2624 struct pnv_phb *phb;
2625 struct pnv_ioda_pe *pe;
2626
2627 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2628 phb = hose->private_data;
2629 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2630 pnv_ioda_setup_pe_seg(hose, pe);
2631 }
2632 }
2633}
2634
cad5cef6 2635static void pnv_pci_ioda_setup_DMA(void)
13395c48
GS		 2636{
2637 struct pci_controller *hose, *tmp;
db1266c8 2638 struct pnv_phb *phb;
13395c48
GS		 2639
2640 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2641 pnv_ioda_setup_dma(hose->private_data);
db1266c8
GS		 2642
2643 /* Mark the PHB initialization done */
2644 phb = hose->private_data;
2645 phb->initialized = 1;
13395c48
GS		 2646 }
2647}
2648
37c367f2
GS		 2649static void pnv_pci_ioda_create_dbgfs(void)
2650{
2651#ifdef CONFIG_DEBUG_FS
2652 struct pci_controller *hose, *tmp;
2653 struct pnv_phb *phb;
2654 char name[16];
2655
2656 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2657 phb = hose->private_data;
2658
2659 sprintf(name, "PCI%04x", hose->global_number);
2660 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2661 if (!phb->dbgfs)
2662 pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2663 __func__, hose->global_number);
2664 }
2665#endif /* CONFIG_DEBUG_FS */
2666}
2667
cad5cef6 2668static void pnv_pci_ioda_fixup(void)
fb446ad0
GS		 2669{
2670 pnv_pci_ioda_setup_PEs();
11685bec 2671 pnv_pci_ioda_setup_seg();
13395c48 2672 pnv_pci_ioda_setup_DMA();
e9cc17d4 2673
37c367f2
GS		 2674 pnv_pci_ioda_create_dbgfs();
2675
e9cc17d4 2676#ifdef CONFIG_EEH
e9cc17d4 2677 eeh_init();
dadcd6d6 2678 eeh_addr_cache_build();
e9cc17d4 2679#endif
fb446ad0
GS		 2680}
2681
271fd03a
GS		 2682/*
2683 * Returns the alignment for I/O or memory windows for P2P
2684 * bridges. That actually depends on how PEs are segmented.
2685 * For now, we return I/O or M32 segment size for PE sensitive
2686 * P2P bridges. Otherwise, the default values (4KiB for I/O,
2687 * 1MiB for memory) will be returned.
2688 *
2689 * The current PCI bus might be put into one PE, which was
2690 * create against the parent PCI bridge. For that case, we
2691 * needn't enlarge the alignment so that we can save some
2692 * resources.
2693 */
2694static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2695 unsigned long type)
2696{
2697 struct pci_dev *bridge;
2698 struct pci_controller *hose = pci_bus_to_host(bus);
2699 struct pnv_phb *phb = hose->private_data;
2700 int num_pci_bridges = 0;
2701
2702 bridge = bus->self;
2703 while (bridge) {
2704 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2705 num_pci_bridges++;
2706 if (num_pci_bridges >= 2)
2707 return 1;
2708 }
2709
2710 bridge = bridge->bus->self;
2711 }
2712
262af557
GC		 2713 /* We fail back to M32 if M64 isn't supported */
2714 if (phb->ioda.m64_segsize &&
2715 pnv_pci_is_mem_pref_64(type))
2716 return phb->ioda.m64_segsize;
271fd03a
GS		 2717 if (type & IORESOURCE_MEM)
2718 return phb->ioda.m32_segsize;
2719
2720 return phb->ioda.io_segsize;
2721}
2722
5350ab3f
WY		 2723#ifdef CONFIG_PCI_IOV
2724static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2725 int resno)
2726{
2727 struct pci_dn *pdn = pci_get_pdn(pdev);
2728 resource_size_t align, iov_align;
2729
2730 iov_align = resource_size(&pdev->resource[resno]);
2731 if (iov_align)
2732 return iov_align;
2733
2734 align = pci_iov_resource_size(pdev, resno);
2735 if (pdn->vfs_expanded)
2736 return pdn->vfs_expanded * align;
2737
2738 return align;
2739}
2740#endif /* CONFIG_PCI_IOV */
2741
184cd4a3
BH		 2742/* Prevent enabling devices for which we couldn't properly
2743 * assign a PE
2744 */
c88c2a18 2745static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
184cd4a3 2746{
db1266c8
GS		 2747 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2748 struct pnv_phb *phb = hose->private_data;
2749 struct pci_dn *pdn;
184cd4a3 2750
db1266c8
GS		 2751 /* The function is probably called while the PEs have
2752 * not be created yet. For example, resource reassignment
2753 * during PCI probe period. We just skip the check if
2754 * PEs isn't ready.
2755 */
2756 if (!phb->initialized)
c88c2a18 2757 return true;
db1266c8 2758
b72c1f65 2759 pdn = pci_get_pdn(dev);
184cd4a3 2760 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
c88c2a18 2761 return false;
db1266c8 2762
c88c2a18 2763 return true;
184cd4a3
BH		 2764}
2765
2766static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
2767 u32 devfn)
2768{
2769 return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
2770}
2771
7a8e6bbf 2772static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
73ed148a 2773{
7a8e6bbf
MN		 2774 struct pnv_phb *phb = hose->private_data;
2775
d1a85eee 2776 opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
73ed148a
BH		 2777 OPAL_ASSERT_RESET);
2778}
2779
92ae0353
DA		 2780static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2781 .dma_dev_setup = pnv_pci_dma_dev_setup,
2782#ifdef CONFIG_PCI_MSI
2783 .setup_msi_irqs = pnv_setup_msi_irqs,
2784 .teardown_msi_irqs = pnv_teardown_msi_irqs,
2785#endif
2786 .enable_device_hook = pnv_pci_enable_device_hook,
2787 .window_alignment = pnv_pci_window_alignment,
2788 .reset_secondary_bus = pnv_pci_reset_secondary_bus,
763d2d8d 2789 .dma_set_mask = pnv_pci_ioda_dma_set_mask,
7a8e6bbf 2790 .shutdown = pnv_pci_ioda_shutdown,
92ae0353
DA		 2791};
2792
e51df2c1
AB		 2793static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2794 u64 hub_id, int ioda_type)
184cd4a3
BH		 2795{
2796 struct pci_controller *hose;
184cd4a3 2797 struct pnv_phb *phb;
8184616f 2798 unsigned long size, m32map_off, pemap_off, iomap_off = 0;
c681b93c 2799 const __be64 *prop64;
3a1a4661 2800 const __be32 *prop32;
f1b7cc3e 2801 int len;
184cd4a3
BH		 2802 u64 phb_id;
2803 void *aux;
2804 long rc;
2805
58d714ec 2806 pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
184cd4a3
BH		 2807
2808 prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2809 if (!prop64) {
2810 pr_err(" Missing \"ibm,opal-phbid\" property !\n");
2811 return;
2812 }
2813 phb_id = be64_to_cpup(prop64);
2814 pr_debug(" PHB-ID : 0x%016llx\n", phb_id);
2815
e39f223f 2816 phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
58d714ec
GS		 2817
2818 /* Allocate PCI controller */
58d714ec
GS		 2819 phb->hose = hose = pcibios_alloc_controller(np);
2820 if (!phb->hose) {
2821 pr_err(" Can't allocate PCI controller for %s\n",
184cd4a3 2822 np->full_name);
e39f223f 2823 memblock_free(__pa(phb), sizeof(struct pnv_phb));
184cd4a3
BH		 2824 return;
2825 }
2826
2827 spin_lock_init(&phb->lock);
f1b7cc3e
GS		 2828 prop32 = of_get_property(np, "bus-range", &len);
2829 if (prop32 && len == 8) {
3a1a4661
BH		 2830 hose->first_busno = be32_to_cpu(prop32[0]);
2831 hose->last_busno = be32_to_cpu(prop32[1]);
f1b7cc3e
GS		 2832 } else {
2833 pr_warn(" Broken <bus-range> on %s\n", np->full_name);
2834 hose->first_busno = 0;
2835 hose->last_busno = 0xff;
2836 }
184cd4a3 2837 hose->private_data = phb;
e9cc17d4 2838 phb->hub_id = hub_id;
184cd4a3 2839 phb->opal_id = phb_id;
aa0c033f 2840 phb->type = ioda_type;
781a868f 2841 mutex_init(&phb->ioda.pe_alloc_mutex);
184cd4a3 2842
cee72d5b
BH		 2843 /* Detect specific models for error handling */
2844 if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2845 phb->model = PNV_PHB_MODEL_P7IOC;
f3d40c25 2846 else if (of_device_is_compatible(np, "ibm,power8-pciex"))
aa0c033f 2847 phb->model = PNV_PHB_MODEL_PHB3;
cee72d5b
BH		 2848 else
2849 phb->model = PNV_PHB_MODEL_UNKNOWN;
2850
aa0c033f 2851 /* Parse 32-bit and IO ranges (if any) */
2f1ec02e 2852 pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
184cd4a3 2853
aa0c033f 2854 /* Get registers */
184cd4a3
BH		 2855 phb->regs = of_iomap(np, 0);
2856 if (phb->regs == NULL)
2857 pr_err(" Failed to map registers !\n");
2858
184cd4a3 2859 /* Initialize more IODA stuff */
36954dc7 2860 phb->ioda.total_pe = 1;
aa0c033f 2861 prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
36954dc7 2862 if (prop32)
3a1a4661 2863 phb->ioda.total_pe = be32_to_cpup(prop32);
36954dc7
GS		 2864 prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
2865 if (prop32)
2866 phb->ioda.reserved_pe = be32_to_cpup(prop32);
262af557
GC		 2867
2868 /* Parse 64-bit MMIO range */
2869 pnv_ioda_parse_m64_window(phb);
2870
184cd4a3 2871 phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
aa0c033f 2872 /* FW Has already off top 64k of M32 space (MSI space) */
184cd4a3
BH		 2873 phb->ioda.m32_size += 0x10000;
2874
2875 phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
3fd47f06 2876 phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
184cd4a3
BH		 2877 phb->ioda.io_size = hose->pci_io_size;
2878 phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
2879 phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
2880
c35d2a8c 2881 /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
184cd4a3
BH		 2882 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
2883 m32map_off = size;
e47747f4 2884 size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
c35d2a8c
GS		 2885 if (phb->type == PNV_PHB_IODA1) {
2886 iomap_off = size;
2887 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
2888 }
184cd4a3
BH		 2889 pemap_off = size;
2890 size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
e39f223f 2891 aux = memblock_virt_alloc(size, 0);
184cd4a3
BH		 2892 phb->ioda.pe_alloc = aux;
2893 phb->ioda.m32_segmap = aux + m32map_off;
c35d2a8c
GS		 2894 if (phb->type == PNV_PHB_IODA1)
2895 phb->ioda.io_segmap = aux + iomap_off;
184cd4a3 2896 phb->ioda.pe_array = aux + pemap_off;
36954dc7 2897 set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
184cd4a3 2898
7ebdf956 2899 INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
184cd4a3 2900 INIT_LIST_HEAD(&phb->ioda.pe_list);
781a868f 2901 mutex_init(&phb->ioda.pe_list_mutex);
184cd4a3
BH		 2902
2903 /* Calculate how many 32-bit TCE segments we have */
2904 phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
2905
aa0c033f 2906#if 0 /* We should really do that ... */
184cd4a3
BH		 2907 rc = opal_pci_set_phb_mem_window(opal->phb_id,
2908 window_type,
2909 window_num,
2910 starting_real_address,
2911 starting_pci_address,
2912 segment_size);
2913#endif
2914
262af557
GC		 2915 pr_info(" %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
2916 phb->ioda.total_pe, phb->ioda.reserved_pe,
2917 phb->ioda.m32_size, phb->ioda.m32_segsize);
2918 if (phb->ioda.m64_size)
2919 pr_info(" M64: 0x%lx [segment=0x%lx]\n",
2920 phb->ioda.m64_size, phb->ioda.m64_segsize);
2921 if (phb->ioda.io_size)
2922 pr_info(" IO: 0x%x [segment=0x%x]\n",
2923 phb->ioda.io_size, phb->ioda.io_segsize);
2924
184cd4a3 2925
184cd4a3 2926 phb->hose->ops = &pnv_pci_ops;
49dec922
GS		 2927 phb->get_pe_state = pnv_ioda_get_pe_state;
2928 phb->freeze_pe = pnv_ioda_freeze_pe;
2929 phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
184cd4a3
BH		 2930
2931 /* Setup RID -> PE mapping function */
2932 phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
2933
2934 /* Setup TCEs */
2935 phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
fe7e85c6 2936 phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
184cd4a3
BH		 2937
2938 /* Setup MSI support */
2939 pnv_pci_init_ioda_msis(phb);
2940
c40a4210
GS		 2941 /*
2942 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
2943 * to let the PCI core do resource assignment. It's supposed
2944 * that the PCI core will do correct I/O and MMIO alignment
2945 * for the P2P bridge bars so that each PCI bus (excluding
2946 * the child P2P bridges) can form individual PE.
184cd4a3 2947 */
fb446ad0 2948 ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
92ae0353 2949 hose->controller_ops = pnv_pci_ioda_controller_ops;
ad30cb99 2950
6e628c7d
WY		 2951#ifdef CONFIG_PCI_IOV
2952 ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
5350ab3f 2953 ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
ad30cb99
ME		 2954#endif
2955
c40a4210 2956 pci_add_flags(PCI_REASSIGN_ALL_RSRC);
184cd4a3
BH		 2957
2958 /* Reset IODA tables to a clean state */
d1a85eee 2959 rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
184cd4a3 2960 if (rc)
f11fe552 2961 pr_warning(" OPAL Error %ld performing IODA table reset !\n", rc);
361f2a2a
GS		 2962
2963 /* If we're running in kdump kerenl, the previous kerenl never
2964 * shutdown PCI devices correctly. We already got IODA table
2965 * cleaned out. So we have to issue PHB reset to stop all PCI
2966 * transactions from previous kerenl.
2967 */
2968 if (is_kdump_kernel()) {
2969 pr_info(" Issue PHB reset ...\n");
cadf364d
GS		 2970 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
2971 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
361f2a2a 2972 }
262af557 2973
9e9e8935
GS		 2974 /* Remove M64 resource if we can't configure it successfully */
2975 if (!phb->init_m64 || phb->init_m64(phb))
262af557 2976 hose->mem_resources[1].flags = 0;
aa0c033f
GS		 2977}
2978
67975005 2979void __init pnv_pci_init_ioda2_phb(struct device_node *np)
aa0c033f 2980{
e9cc17d4 2981 pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
184cd4a3
BH		 2982}
2983
2984void __init pnv_pci_init_ioda_hub(struct device_node *np)
2985{
2986 struct device_node *phbn;
c681b93c 2987 const __be64 *prop64;
184cd4a3
BH		 2988 u64 hub_id;
2989
2990 pr_info("Probing IODA IO-Hub %s\n", np->full_name);
2991
2992 prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
2993 if (!prop64) {
2994 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
2995 return;
2996 }
2997 hub_id = be64_to_cpup(prop64);
2998 pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
2999
3000 /* Count child PHBs */
3001 for_each_child_of_node(np, phbn) {
3002 /* Look for IODA1 PHBs */
3003 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
e9cc17d4 3004 pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
184cd4a3
BH		 3005 }
3006}
Linux 6.x block layer and io_uring tree(s)