projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/powernv: Allow to reserve one PE for multiple times
[linux-2.6-block.git] / arch / powerpc / platforms / powernv / pci-ioda.c
CommitLineData
184cd4a3
BH		 1/*
2 * Support PCI/PCIe on PowerNV platforms
3 *
4 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
cee72d5b 12#undef DEBUG
184cd4a3
BH		 13
14#include <linux/kernel.h>
15#include <linux/pci.h>
361f2a2a 16#include <linux/crash_dump.h>
37c367f2 17#include <linux/debugfs.h>
184cd4a3
BH		 18#include <linux/delay.h>
19#include <linux/string.h>
20#include <linux/init.h>
21#include <linux/bootmem.h>
22#include <linux/irq.h>
23#include <linux/io.h>
24#include <linux/msi.h>
cd15b048 25#include <linux/memblock.h>
ac9a5889 26#include <linux/iommu.h>
e57080f1 27#include <linux/rculist.h>
4793d65d 28#include <linux/sizes.h>
184cd4a3
BH		 29
30#include <asm/sections.h>
31#include <asm/io.h>
32#include <asm/prom.h>
33#include <asm/pci-bridge.h>
34#include <asm/machdep.h>
fb1b55d6 35#include <asm/msi_bitmap.h>
184cd4a3
BH		 36#include <asm/ppc-pci.h>
37#include <asm/opal.h>
38#include <asm/iommu.h>
39#include <asm/tce.h>
137436c9 40#include <asm/xics.h>
37c367f2 41#include <asm/debug.h>
262af557 42#include <asm/firmware.h>
80c49c7e 43#include <asm/pnv-pci.h>
aca6913f 44#include <asm/mmzone.h>
80c49c7e 45
ec249dd8 46#include <misc/cxl-base.h>
184cd4a3
BH		 47
48#include "powernv.h"
49#include "pci.h"
50
781a868f
WY		 51/* 256M DMA window, 4K TCE pages, 8 bytes TCE */
52#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8)
53
bbb845c4
AK		 54#define POWERNV_IOMMU_DEFAULT_LEVELS 1
55#define POWERNV_IOMMU_MAX_LEVELS 5
56
aca6913f
AK		 57static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
58
6d31c2fa
JP		 59static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
60 const char *fmt, ...)
61{
62 struct va_format vaf;
63 va_list args;
64 char pfix[32];
65
66 va_start(args, fmt);
67
68 vaf.fmt = fmt;
69 vaf.va = &args;
70
781a868f 71 if (pe->flags & PNV_IODA_PE_DEV)
6d31c2fa 72 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
781a868f 73 else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
6d31c2fa
JP		 74 sprintf(pfix, "%04x:%02x ",
75 pci_domain_nr(pe->pbus), pe->pbus->number);
781a868f
WY		 76#ifdef CONFIG_PCI_IOV
77 else if (pe->flags & PNV_IODA_PE_VF)
78 sprintf(pfix, "%04x:%02x:%2x.%d",
79 pci_domain_nr(pe->parent_dev->bus),
80 (pe->rid & 0xff00) >> 8,
81 PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
82#endif /* CONFIG_PCI_IOV*/
6d31c2fa
JP		 83
84 printk("%spci %s: [PE# %.3d] %pV",
85 level, pfix, pe->pe_number, &vaf);
86
87 va_end(args);
88}
184cd4a3 89
6d31c2fa
JP		 90#define pe_err(pe, fmt, ...) \
91 pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
92#define pe_warn(pe, fmt, ...) \
93 pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
94#define pe_info(pe, fmt, ...) \
95 pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
184cd4a3 96
4e287840
TLSC		 97static bool pnv_iommu_bypass_disabled __read_mostly;
98
99static int __init iommu_setup(char *str)
100{
101 if (!str)
102 return -EINVAL;
103
104 while (*str) {
105 if (!strncmp(str, "nobypass", 8)) {
106 pnv_iommu_bypass_disabled = true;
107 pr_info("PowerNV: IOMMU bypass window disabled.\n");
108 break;
109 }
110 str += strcspn(str, ",");
111 if (*str == ',')
112 str++;
113 }
114
115 return 0;
116}
117early_param("iommu", iommu_setup);
118
8e0a1611
AK		 119/*
120 * stdcix is only supposed to be used in hypervisor real mode as per
121 * the architecture spec
122 */
123static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
124{
125 __asm__ __volatile__("stdcix %0,0,%1"
126 : : "r" (val), "r" (paddr) : "memory");
127}
128
262af557
GC		 129static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
130{
131 return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
132 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
133}
134
4b82ab18
GS		 135static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
136{
137 if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
138 pr_warn("%s: Invalid PE %d on PHB#%x\n",
139 __func__, pe_no, phb->hose->global_number);
140 return;
141 }
142
e9dc4d7f
GS		 143 if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
144 pr_debug("%s: PE %d was reserved on PHB#%x\n",
145 __func__, pe_no, phb->hose->global_number);
4b82ab18
GS		 146
147 phb->ioda.pe_array[pe_no].phb = phb;
148 phb->ioda.pe_array[pe_no].pe_number = pe_no;
149}
150
cad5cef6 151static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
184cd4a3
BH		 152{
153 unsigned long pe;
154
155 do {
156 pe = find_next_zero_bit(phb->ioda.pe_alloc,
157 phb->ioda.total_pe, 0);
158 if (pe >= phb->ioda.total_pe)
159 return IODA_INVALID_PE;
160 } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
161
4cce9550 162 phb->ioda.pe_array[pe].phb = phb;
184cd4a3
BH		 163 phb->ioda.pe_array[pe].pe_number = pe;
164 return pe;
165}
166
cad5cef6 167static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
184cd4a3
BH		 168{
169 WARN_ON(phb->ioda.pe_array[pe].pdev);
170
171 memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
172 clear_bit(pe, phb->ioda.pe_alloc);
173}
174
262af557
GC		 175/* The default M64 BAR is shared by all PEs */
176static int pnv_ioda2_init_m64(struct pnv_phb *phb)
177{
178 const char *desc;
179 struct resource *r;
180 s64 rc;
181
182 /* Configure the default M64 BAR */
183 rc = opal_pci_set_phb_mem_window(phb->opal_id,
184 OPAL_M64_WINDOW_TYPE,
185 phb->ioda.m64_bar_idx,
186 phb->ioda.m64_base,
187 0, /* unused */
188 phb->ioda.m64_size);
189 if (rc != OPAL_SUCCESS) {
190 desc = "configuring";
191 goto fail;
192 }
193
194 /* Enable the default M64 BAR */
195 rc = opal_pci_phb_mmio_enable(phb->opal_id,
196 OPAL_M64_WINDOW_TYPE,
197 phb->ioda.m64_bar_idx,
198 OPAL_ENABLE_M64_SPLIT);
199 if (rc != OPAL_SUCCESS) {
200 desc = "enabling";
201 goto fail;
202 }
203
204 /* Mark the M64 BAR assigned */
205 set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
206
207 /*
208 * Strip off the segment used by the reserved PE, which is
209 * expected to be 0 or last one of PE capabicity.
210 */
211 r = &phb->hose->mem_resources[1];
212 if (phb->ioda.reserved_pe == 0)
213 r->start += phb->ioda.m64_segsize;
214 else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
215 r->end -= phb->ioda.m64_segsize;
216 else
217 pr_warn(" Cannot strip M64 segment for reserved PE#%d\n",
218 phb->ioda.reserved_pe);
219
220 return 0;
221
222fail:
223 pr_warn(" Failure %lld %s M64 BAR#%d\n",
224 rc, desc, phb->ioda.m64_bar_idx);
225 opal_pci_phb_mmio_enable(phb->opal_id,
226 OPAL_M64_WINDOW_TYPE,
227 phb->ioda.m64_bar_idx,
228 OPAL_DISABLE_M64);
229 return -EIO;
230}
231
5ef73567 232static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
262af557
GC		 233{
234 resource_size_t sgsz = phb->ioda.m64_segsize;
235 struct pci_dev *pdev;
236 struct resource *r;
237 int base, step, i;
238
239 /*
240 * Root bus always has full M64 range and root port has
241 * M64 range used in reality. So we're checking root port
242 * instead of root bus.
243 */
244 list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
4b82ab18
GS		 245 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
246 r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
262af557
GC		 247 if (!r->parent ||
248 !pnv_pci_is_mem_pref_64(r->flags))
249 continue;
250
251 base = (r->start - phb->ioda.m64_base) / sgsz;
252 for (step = 0; step < resource_size(r) / sgsz; step++)
4b82ab18 253 pnv_ioda_reserve_pe(phb, base + step);
262af557
GC		 254 }
255 }
256}
257
258static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
259 struct pci_bus *bus, int all)
260{
261 resource_size_t segsz = phb->ioda.m64_segsize;
262 struct pci_dev *pdev;
263 struct resource *r;
264 struct pnv_ioda_pe *master_pe, *pe;
265 unsigned long size, *pe_alloc;
266 bool found;
267 int start, i, j;
268
269 /* Root bus shouldn't use M64 */
270 if (pci_is_root_bus(bus))
271 return IODA_INVALID_PE;
272
273 /* We support only one M64 window on each bus */
274 found = false;
275 pci_bus_for_each_resource(bus, r, i) {
276 if (r && r->parent &&
277 pnv_pci_is_mem_pref_64(r->flags)) {
278 found = true;
279 break;
280 }
281 }
282
283 /* No M64 window found ? */
284 if (!found)
285 return IODA_INVALID_PE;
286
287 /* Allocate bitmap */
288 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
289 pe_alloc = kzalloc(size, GFP_KERNEL);
290 if (!pe_alloc) {
291 pr_warn("%s: Out of memory !\n",
292 __func__);
293 return IODA_INVALID_PE;
294 }
295
296 /*
297 * Figure out reserved PE numbers by the PE
298 * the its child PEs.
299 */
300 start = (r->start - phb->ioda.m64_base) / segsz;
301 for (i = 0; i < resource_size(r) / segsz; i++)
302 set_bit(start + i, pe_alloc);
303
304 if (all)
305 goto done;
306
307 /*
308 * If the PE doesn't cover all subordinate buses,
309 * we need subtract from reserved PEs for children.
310 */
311 list_for_each_entry(pdev, &bus->devices, bus_list) {
312 if (!pdev->subordinate)
313 continue;
314
315 pci_bus_for_each_resource(pdev->subordinate, r, i) {
316 if (!r || !r->parent ||
317 !pnv_pci_is_mem_pref_64(r->flags))
318 continue;
319
320 start = (r->start - phb->ioda.m64_base) / segsz;
321 for (j = 0; j < resource_size(r) / segsz ; j++)
322 clear_bit(start + j, pe_alloc);
323 }
324 }
325
326 /*
327 * the current bus might not own M64 window and that's all
328 * contributed by its child buses. For the case, we needn't
329 * pick M64 dependent PE#.
330 */
331 if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
332 kfree(pe_alloc);
333 return IODA_INVALID_PE;
334 }
335
336 /*
337 * Figure out the master PE and put all slave PEs to master
338 * PE's list to form compound PE.
339 */
340done:
341 master_pe = NULL;
342 i = -1;
343 while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
344 phb->ioda.total_pe) {
345 pe = &phb->ioda.pe_array[i];
262af557
GC		 346
347 if (!master_pe) {
348 pe->flags |= PNV_IODA_PE_MASTER;
349 INIT_LIST_HEAD(&pe->slaves);
350 master_pe = pe;
351 } else {
352 pe->flags |= PNV_IODA_PE_SLAVE;
353 pe->master = master_pe;
354 list_add_tail(&pe->list, &master_pe->slaves);
355 }
356 }
357
358 kfree(pe_alloc);
359 return master_pe->pe_number;
360}
361
362static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
363{
364 struct pci_controller *hose = phb->hose;
365 struct device_node *dn = hose->dn;
366 struct resource *res;
367 const u32 *r;
368 u64 pci_addr;
369
1665c4a8
GS		 370 /* FIXME: Support M64 for P7IOC */
371 if (phb->type != PNV_PHB_IODA2) {
372 pr_info(" Not support M64 window\n");
373 return;
374 }
375
262af557
GC		 376 if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
377 pr_info(" Firmware too old to support M64 window\n");
378 return;
379 }
380
381 r = of_get_property(dn, "ibm,opal-m64-window", NULL);
382 if (!r) {
383 pr_info(" No <ibm,opal-m64-window> on %s\n",
384 dn->full_name);
385 return;
386 }
387
262af557
GC		 388 res = &hose->mem_resources[1];
389 res->start = of_translate_address(dn, r + 2);
390 res->end = res->start + of_read_number(r + 4, 2) - 1;
391 res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
392 pci_addr = of_read_number(r, 2);
393 hose->mem_offset[1] = res->start - pci_addr;
394
395 phb->ioda.m64_size = resource_size(res);
396 phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
397 phb->ioda.m64_base = pci_addr;
398
e9863e68
WY		 399 pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
400 res->start, res->end, pci_addr);
401
262af557
GC		 402 /* Use last M64 BAR to cover M64 window */
403 phb->ioda.m64_bar_idx = 15;
404 phb->init_m64 = pnv_ioda2_init_m64;
5ef73567 405 phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
262af557
GC		 406 phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
407}
408
49dec922
GS		 409static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
410{
411 struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
412 struct pnv_ioda_pe *slave;
413 s64 rc;
414
415 /* Fetch master PE */
416 if (pe->flags & PNV_IODA_PE_SLAVE) {
417 pe = pe->master;
ec8e4e9d
GS		 418 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
419 return;
420
49dec922
GS		 421 pe_no = pe->pe_number;
422 }
423
424 /* Freeze master PE */
425 rc = opal_pci_eeh_freeze_set(phb->opal_id,
426 pe_no,
427 OPAL_EEH_ACTION_SET_FREEZE_ALL);
428 if (rc != OPAL_SUCCESS) {
429 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
430 __func__, rc, phb->hose->global_number, pe_no);
431 return;
432 }
433
434 /* Freeze slave PEs */
435 if (!(pe->flags & PNV_IODA_PE_MASTER))
436 return;
437
438 list_for_each_entry(slave, &pe->slaves, list) {
439 rc = opal_pci_eeh_freeze_set(phb->opal_id,
440 slave->pe_number,
441 OPAL_EEH_ACTION_SET_FREEZE_ALL);
442 if (rc != OPAL_SUCCESS)
443 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
444 __func__, rc, phb->hose->global_number,
445 slave->pe_number);
446 }
447}
448
e51df2c1 449static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
49dec922
GS		 450{
451 struct pnv_ioda_pe *pe, *slave;
452 s64 rc;
453
454 /* Find master PE */
455 pe = &phb->ioda.pe_array[pe_no];
456 if (pe->flags & PNV_IODA_PE_SLAVE) {
457 pe = pe->master;
458 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
459 pe_no = pe->pe_number;
460 }
461
462 /* Clear frozen state for master PE */
463 rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
464 if (rc != OPAL_SUCCESS) {
465 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
466 __func__, rc, opt, phb->hose->global_number, pe_no);
467 return -EIO;
468 }
469
470 if (!(pe->flags & PNV_IODA_PE_MASTER))
471 return 0;
472
473 /* Clear frozen state for slave PEs */
474 list_for_each_entry(slave, &pe->slaves, list) {
475 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
476 slave->pe_number,
477 opt);
478 if (rc != OPAL_SUCCESS) {
479 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
480 __func__, rc, opt, phb->hose->global_number,
481 slave->pe_number);
482 return -EIO;
483 }
484 }
485
486 return 0;
487}
488
489static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
490{
491 struct pnv_ioda_pe *slave, *pe;
492 u8 fstate, state;
493 __be16 pcierr;
494 s64 rc;
495
496 /* Sanity check on PE number */
497 if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
498 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
499
500 /*
501 * Fetch the master PE and the PE instance might be
502 * not initialized yet.
503 */
504 pe = &phb->ioda.pe_array[pe_no];
505 if (pe->flags & PNV_IODA_PE_SLAVE) {
506 pe = pe->master;
507 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
508 pe_no = pe->pe_number;
509 }
510
511 /* Check the master PE */
512 rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
513 &state, &pcierr, NULL);
514 if (rc != OPAL_SUCCESS) {
515 pr_warn("%s: Failure %lld getting "
516 "PHB#%x-PE#%x state\n",
517 __func__, rc,
518 phb->hose->global_number, pe_no);
519 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
520 }
521
522 /* Check the slave PE */
523 if (!(pe->flags & PNV_IODA_PE_MASTER))
524 return state;
525
526 list_for_each_entry(slave, &pe->slaves, list) {
527 rc = opal_pci_eeh_freeze_status(phb->opal_id,
528 slave->pe_number,
529 &fstate,
530 &pcierr,
531 NULL);
532 if (rc != OPAL_SUCCESS) {
533 pr_warn("%s: Failure %lld getting "
534 "PHB#%x-PE#%x state\n",
535 __func__, rc,
536 phb->hose->global_number, slave->pe_number);
537 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
538 }
539
540 /*
541 * Override the result based on the ascending
542 * priority.
543 */
544 if (fstate > state)
545 state = fstate;
546 }
547
548 return state;
549}
550
184cd4a3
BH		 551/* Currently those 2 are only used when MSIs are enabled, this will change
552 * but in the meantime, we need to protect them to avoid warnings
553 */
554#ifdef CONFIG_PCI_MSI
cad5cef6 555static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
184cd4a3
BH		 556{
557 struct pci_controller *hose = pci_bus_to_host(dev->bus);
558 struct pnv_phb *phb = hose->private_data;
b72c1f65 559 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 560
561 if (!pdn)
562 return NULL;
563 if (pdn->pe_number == IODA_INVALID_PE)
564 return NULL;
565 return &phb->ioda.pe_array[pdn->pe_number];
566}
184cd4a3
BH		 567#endif /* CONFIG_PCI_MSI */
568
b131a842
GS		 569static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
570 struct pnv_ioda_pe *parent,
571 struct pnv_ioda_pe *child,
572 bool is_add)
573{
574 const char *desc = is_add ? "adding" : "removing";
575 uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
576 OPAL_REMOVE_PE_FROM_DOMAIN;
577 struct pnv_ioda_pe *slave;
578 long rc;
579
580 /* Parent PE affects child PE */
581 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
582 child->pe_number, op);
583 if (rc != OPAL_SUCCESS) {
584 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
585 rc, desc);
586 return -ENXIO;
587 }
588
589 if (!(child->flags & PNV_IODA_PE_MASTER))
590 return 0;
591
592 /* Compound case: parent PE affects slave PEs */
593 list_for_each_entry(slave, &child->slaves, list) {
594 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
595 slave->pe_number, op);
596 if (rc != OPAL_SUCCESS) {
597 pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
598 rc, desc);
599 return -ENXIO;
600 }
601 }
602
603 return 0;
604}
605
606static int pnv_ioda_set_peltv(struct pnv_phb *phb,
607 struct pnv_ioda_pe *pe,
608 bool is_add)
609{
610 struct pnv_ioda_pe *slave;
781a868f 611 struct pci_dev *pdev = NULL;
b131a842
GS		 612 int ret;
613
614 /*
615 * Clear PE frozen state. If it's master PE, we need
616 * clear slave PE frozen state as well.
617 */
618 if (is_add) {
619 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
620 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
621 if (pe->flags & PNV_IODA_PE_MASTER) {
622 list_for_each_entry(slave, &pe->slaves, list)
623 opal_pci_eeh_freeze_clear(phb->opal_id,
624 slave->pe_number,
625 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
626 }
627 }
628
629 /*
630 * Associate PE in PELT. We need add the PE into the
631 * corresponding PELT-V as well. Otherwise, the error
632 * originated from the PE might contribute to other
633 * PEs.
634 */
635 ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
636 if (ret)
637 return ret;
638
639 /* For compound PEs, any one affects all of them */
640 if (pe->flags & PNV_IODA_PE_MASTER) {
641 list_for_each_entry(slave, &pe->slaves, list) {
642 ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
643 if (ret)
644 return ret;
645 }
646 }
647
648 if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
649 pdev = pe->pbus->self;
781a868f 650 else if (pe->flags & PNV_IODA_PE_DEV)
b131a842 651 pdev = pe->pdev->bus->self;
781a868f
WY		 652#ifdef CONFIG_PCI_IOV
653 else if (pe->flags & PNV_IODA_PE_VF)
654 pdev = pe->parent_dev->bus->self;
655#endif /* CONFIG_PCI_IOV */
b131a842
GS		 656 while (pdev) {
657 struct pci_dn *pdn = pci_get_pdn(pdev);
658 struct pnv_ioda_pe *parent;
659
660 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
661 parent = &phb->ioda.pe_array[pdn->pe_number];
662 ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
663 if (ret)
664 return ret;
665 }
666
667 pdev = pdev->bus->self;
668 }
669
670 return 0;
671}
672
781a868f
WY		 673#ifdef CONFIG_PCI_IOV
674static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
675{
676 struct pci_dev *parent;
677 uint8_t bcomp, dcomp, fcomp;
678 int64_t rc;
679 long rid_end, rid;
680
681 /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
682 if (pe->pbus) {
683 int count;
684
685 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
686 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
687 parent = pe->pbus->self;
688 if (pe->flags & PNV_IODA_PE_BUS_ALL)
689 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
690 else
691 count = 1;
692
693 switch(count) {
694 case 1: bcomp = OpalPciBusAll; break;
695 case 2: bcomp = OpalPciBus7Bits; break;
696 case 4: bcomp = OpalPciBus6Bits; break;
697 case 8: bcomp = OpalPciBus5Bits; break;
698 case 16: bcomp = OpalPciBus4Bits; break;
699 case 32: bcomp = OpalPciBus3Bits; break;
700 default:
701 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
702 count);
703 /* Do an exact match only */
704 bcomp = OpalPciBusAll;
705 }
706 rid_end = pe->rid + (count << 8);
707 } else {
708 if (pe->flags & PNV_IODA_PE_VF)
709 parent = pe->parent_dev;
710 else
711 parent = pe->pdev->bus->self;
712 bcomp = OpalPciBusAll;
713 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
714 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
715 rid_end = pe->rid + 1;
716 }
717
718 /* Clear the reverse map */
719 for (rid = pe->rid; rid < rid_end; rid++)
720 phb->ioda.pe_rmap[rid] = 0;
721
722 /* Release from all parents PELT-V */
723 while (parent) {
724 struct pci_dn *pdn = pci_get_pdn(parent);
725 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
726 rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
727 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
728 /* XXX What to do in case of error ? */
729 }
730 parent = parent->bus->self;
731 }
732
733 opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
734 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
735
736 /* Disassociate PE in PELT */
737 rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
738 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
739 if (rc)
740 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
741 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
742 bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
743 if (rc)
744 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
745
746 pe->pbus = NULL;
747 pe->pdev = NULL;
748 pe->parent_dev = NULL;
749
750 return 0;
751}
752#endif /* CONFIG_PCI_IOV */
753
cad5cef6 754static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
184cd4a3
BH		 755{
756 struct pci_dev *parent;
757 uint8_t bcomp, dcomp, fcomp;
758 long rc, rid_end, rid;
759
760 /* Bus validation ? */
761 if (pe->pbus) {
762 int count;
763
764 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
765 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
766 parent = pe->pbus->self;
fb446ad0
GS		 767 if (pe->flags & PNV_IODA_PE_BUS_ALL)
768 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
769 else
770 count = 1;
771
184cd4a3
BH		 772 switch(count) {
773 case 1: bcomp = OpalPciBusAll; break;
774 case 2: bcomp = OpalPciBus7Bits; break;
775 case 4: bcomp = OpalPciBus6Bits; break;
776 case 8: bcomp = OpalPciBus5Bits; break;
777 case 16: bcomp = OpalPciBus4Bits; break;
778 case 32: bcomp = OpalPciBus3Bits; break;
779 default:
781a868f
WY		 780 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
781 count);
184cd4a3
BH		 782 /* Do an exact match only */
783 bcomp = OpalPciBusAll;
784 }
785 rid_end = pe->rid + (count << 8);
786 } else {
781a868f
WY		 787#ifdef CONFIG_PCI_IOV
788 if (pe->flags & PNV_IODA_PE_VF)
789 parent = pe->parent_dev;
790 else
791#endif /* CONFIG_PCI_IOV */
792 parent = pe->pdev->bus->self;
184cd4a3
BH		 793 bcomp = OpalPciBusAll;
794 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
795 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
796 rid_end = pe->rid + 1;
797 }
798
631ad691
GS		 799 /*
800 * Associate PE in PELT. We need add the PE into the
801 * corresponding PELT-V as well. Otherwise, the error
802 * originated from the PE might contribute to other
803 * PEs.
804 */
184cd4a3
BH		 805 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
806 bcomp, dcomp, fcomp, OPAL_MAP_PE);
807 if (rc) {
808 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
809 return -ENXIO;
810 }
631ad691 811
b131a842
GS		 812 /* Configure PELTV */
813 pnv_ioda_set_peltv(phb, pe, true);
184cd4a3 814
184cd4a3
BH		 815 /* Setup reverse map */
816 for (rid = pe->rid; rid < rid_end; rid++)
817 phb->ioda.pe_rmap[rid] = pe->pe_number;
818
819 /* Setup one MVTs on IODA1 */
4773f76b
GS		 820 if (phb->type != PNV_PHB_IODA1) {
821 pe->mve_number = 0;
822 goto out;
823 }
824
825 pe->mve_number = pe->pe_number;
826 rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
827 if (rc != OPAL_SUCCESS) {
828 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
829 rc, pe->mve_number);
830 pe->mve_number = -1;
831 } else {
832 rc = opal_pci_set_mve_enable(phb->opal_id,
833 pe->mve_number, OPAL_ENABLE_MVE);
184cd4a3 834 if (rc) {
4773f76b 835 pe_err(pe, "OPAL error %ld enabling MVE %d\n",
184cd4a3
BH		 836 rc, pe->mve_number);
837 pe->mve_number = -1;
184cd4a3 838 }
4773f76b 839 }
184cd4a3 840
4773f76b 841out:
184cd4a3
BH		 842 return 0;
843}
844
cad5cef6
GKH		 845static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
846 struct pnv_ioda_pe *pe)
184cd4a3
BH		 847{
848 struct pnv_ioda_pe *lpe;
849
7ebdf956 850 list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
184cd4a3 851 if (lpe->dma_weight < pe->dma_weight) {
7ebdf956 852 list_add_tail(&pe->dma_link, &lpe->dma_link);
184cd4a3
BH		 853 return;
854 }
855 }
7ebdf956 856 list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
184cd4a3
BH		 857}
858
859static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
860{
861 /* This is quite simplistic. The "base" weight of a device
862 * is 10. 0 means no DMA is to be accounted for it.
863 */
864
865 /* If it's a bridge, no DMA */
866 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
867 return 0;
868
869 /* Reduce the weight of slow USB controllers */
870 if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
871 dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
872 dev->class == PCI_CLASS_SERIAL_USB_EHCI)
873 return 3;
874
875 /* Increase the weight of RAID (includes Obsidian) */
876 if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
877 return 15;
878
879 /* Default */
880 return 10;
881}
882
781a868f
WY		 883#ifdef CONFIG_PCI_IOV
884static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
885{
886 struct pci_dn *pdn = pci_get_pdn(dev);
887 int i;
888 struct resource *res, res2;
889 resource_size_t size;
890 u16 num_vfs;
891
892 if (!dev->is_physfn)
893 return -EINVAL;
894
895 /*
896 * "offset" is in VFs. The M64 windows are sized so that when they
897 * are segmented, each segment is the same size as the IOV BAR.
898 * Each segment is in a separate PE, and the high order bits of the
899 * address are the PE number. Therefore, each VF's BAR is in a
900 * separate PE, and changing the IOV BAR start address changes the
901 * range of PEs the VFs are in.
902 */
903 num_vfs = pdn->num_vfs;
904 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
905 res = &dev->resource[i + PCI_IOV_RESOURCES];
906 if (!res->flags || !res->parent)
907 continue;
908
909 if (!pnv_pci_is_mem_pref_64(res->flags))
910 continue;
911
912 /*
913 * The actual IOV BAR range is determined by the start address
914 * and the actual size for num_vfs VFs BAR. This check is to
915 * make sure that after shifting, the range will not overlap
916 * with another device.
917 */
918 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
919 res2.flags = res->flags;
920 res2.start = res->start + (size * offset);
921 res2.end = res2.start + (size * num_vfs) - 1;
922
923 if (res2.end > res->end) {
924 dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
925 i, &res2, res, num_vfs, offset);
926 return -EBUSY;
927 }
928 }
929
930 /*
931 * After doing so, there would be a "hole" in the /proc/iomem when
932 * offset is a positive value. It looks like the device return some
933 * mmio back to the system, which actually no one could use it.
934 */
935 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
936 res = &dev->resource[i + PCI_IOV_RESOURCES];
937 if (!res->flags || !res->parent)
938 continue;
939
940 if (!pnv_pci_is_mem_pref_64(res->flags))
941 continue;
942
943 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
944 res2 = *res;
945 res->start += size * offset;
946
947 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
948 i, &res2, res, num_vfs, offset);
949 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
950 }
951 return 0;
952}
953#endif /* CONFIG_PCI_IOV */
954
fb446ad0 955#if 0
cad5cef6 956static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
184cd4a3
BH		 957{
958 struct pci_controller *hose = pci_bus_to_host(dev->bus);
959 struct pnv_phb *phb = hose->private_data;
b72c1f65 960 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 961 struct pnv_ioda_pe *pe;
962 int pe_num;
963
964 if (!pdn) {
965 pr_err("%s: Device tree node not associated properly\n",
966 pci_name(dev));
967 return NULL;
968 }
969 if (pdn->pe_number != IODA_INVALID_PE)
970 return NULL;
971
972 /* PE#0 has been pre-set */
973 if (dev->bus->number == 0)
974 pe_num = 0;
975 else
976 pe_num = pnv_ioda_alloc_pe(phb);
977 if (pe_num == IODA_INVALID_PE) {
978 pr_warning("%s: Not enough PE# available, disabling device\n",
979 pci_name(dev));
980 return NULL;
981 }
982
983 /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
984 * pointer in the PE data structure, both should be destroyed at the
985 * same time. However, this needs to be looked at more closely again
986 * once we actually start removing things (Hotplug, SR-IOV, ...)
987 *
988 * At some point we want to remove the PDN completely anyways
989 */
990 pe = &phb->ioda.pe_array[pe_num];
991 pci_dev_get(dev);
992 pdn->pcidev = dev;
993 pdn->pe_number = pe_num;
994 pe->pdev = dev;
995 pe->pbus = NULL;
996 pe->tce32_seg = -1;
997 pe->mve_number = -1;
998 pe->rid = dev->bus->number << 8 | pdn->devfn;
999
1000 pe_info(pe, "Associated device to PE\n");
1001
1002 if (pnv_ioda_configure_pe(phb, pe)) {
1003 /* XXX What do we do here ? */
1004 if (pe_num)
1005 pnv_ioda_free_pe(phb, pe_num);
1006 pdn->pe_number = IODA_INVALID_PE;
1007 pe->pdev = NULL;
1008 pci_dev_put(dev);
1009 return NULL;
1010 }
1011
1012 /* Assign a DMA weight to the device */
1013 pe->dma_weight = pnv_ioda_dma_weight(dev);
1014 if (pe->dma_weight != 0) {
1015 phb->ioda.dma_weight += pe->dma_weight;
1016 phb->ioda.dma_pe_count++;
1017 }
1018
1019 /* Link the PE */
1020 pnv_ioda_link_pe_by_weight(phb, pe);
1021
1022 return pe;
1023}
fb446ad0 1024#endif /* Useful for SRIOV case */
184cd4a3
BH		 1025
1026static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1027{
1028 struct pci_dev *dev;
1029
1030 list_for_each_entry(dev, &bus->devices, bus_list) {
b72c1f65 1031 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 1032
1033 if (pdn == NULL) {
1034 pr_warn("%s: No device node associated with device !\n",
1035 pci_name(dev));
1036 continue;
1037 }
184cd4a3
BH		 1038 pdn->pe_number = pe->pe_number;
1039 pe->dma_weight += pnv_ioda_dma_weight(dev);
fb446ad0 1040 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
184cd4a3
BH		 1041 pnv_ioda_setup_same_PE(dev->subordinate, pe);
1042 }
1043}
1044
fb446ad0
GS		 1045/*
1046 * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1047 * single PCI bus. Another one that contains the primary PCI bus and its
1048 * subordinate PCI devices and buses. The second type of PE is normally
1049 * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1050 */
cad5cef6 1051static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
184cd4a3 1052{
fb446ad0 1053 struct pci_controller *hose = pci_bus_to_host(bus);
184cd4a3 1054 struct pnv_phb *phb = hose->private_data;
184cd4a3 1055 struct pnv_ioda_pe *pe;
262af557
GC		 1056 int pe_num = IODA_INVALID_PE;
1057
1058 /* Check if PE is determined by M64 */
1059 if (phb->pick_m64_pe)
1060 pe_num = phb->pick_m64_pe(phb, bus, all);
1061
1062 /* The PE number isn't pinned by M64 */
1063 if (pe_num == IODA_INVALID_PE)
1064 pe_num = pnv_ioda_alloc_pe(phb);
184cd4a3 1065
184cd4a3 1066 if (pe_num == IODA_INVALID_PE) {
fb446ad0
GS		 1067 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1068 __func__, pci_domain_nr(bus), bus->number);
184cd4a3
BH		 1069 return;
1070 }
1071
1072 pe = &phb->ioda.pe_array[pe_num];
262af557 1073 pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
184cd4a3
BH		 1074 pe->pbus = bus;
1075 pe->pdev = NULL;
1076 pe->tce32_seg = -1;
1077 pe->mve_number = -1;
b918c62e 1078 pe->rid = bus->busn_res.start << 8;
184cd4a3
BH		 1079 pe->dma_weight = 0;
1080
fb446ad0
GS		 1081 if (all)
1082 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1083 bus->busn_res.start, bus->busn_res.end, pe_num);
1084 else
1085 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1086 bus->busn_res.start, pe_num);
184cd4a3
BH		 1087
1088 if (pnv_ioda_configure_pe(phb, pe)) {
1089 /* XXX What do we do here ? */
1090 if (pe_num)
1091 pnv_ioda_free_pe(phb, pe_num);
1092 pe->pbus = NULL;
1093 return;
1094 }
1095
1096 /* Associate it with all child devices */
1097 pnv_ioda_setup_same_PE(bus, pe);
1098
7ebdf956
GS		 1099 /* Put PE to the list */
1100 list_add_tail(&pe->list, &phb->ioda.pe_list);
1101
184cd4a3
BH		 1102 /* Account for one DMA PE if at least one DMA capable device exist
1103 * below the bridge
1104 */
1105 if (pe->dma_weight != 0) {
1106 phb->ioda.dma_weight += pe->dma_weight;
1107 phb->ioda.dma_pe_count++;
1108 }
1109
1110 /* Link the PE */
1111 pnv_ioda_link_pe_by_weight(phb, pe);
1112}
1113
cad5cef6 1114static void pnv_ioda_setup_PEs(struct pci_bus *bus)
184cd4a3
BH		 1115{
1116 struct pci_dev *dev;
fb446ad0
GS		 1117
1118 pnv_ioda_setup_bus_PE(bus, 0);
184cd4a3
BH		 1119
1120 list_for_each_entry(dev, &bus->devices, bus_list) {
fb446ad0
GS		 1121 if (dev->subordinate) {
1122 if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1123 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1124 else
1125 pnv_ioda_setup_PEs(dev->subordinate);
1126 }
1127 }
1128}
1129
1130/*
1131 * Configure PEs so that the downstream PCI buses and devices
1132 * could have their associated PE#. Unfortunately, we didn't
1133 * figure out the way to identify the PLX bridge yet. So we
1134 * simply put the PCI bus and the subordinate behind the root
1135 * port to PE# here. The game rule here is expected to be changed
1136 * as soon as we can detected PLX bridge correctly.
1137 */
cad5cef6 1138static void pnv_pci_ioda_setup_PEs(void)
fb446ad0
GS		 1139{
1140 struct pci_controller *hose, *tmp;
262af557 1141 struct pnv_phb *phb;
fb446ad0
GS		 1142
1143 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
262af557
GC		 1144 phb = hose->private_data;
1145
1146 /* M64 layout might affect PE allocation */
5ef73567
GS		 1147 if (phb->reserve_m64_pe)
1148 phb->reserve_m64_pe(phb);
262af557 1149
fb446ad0 1150 pnv_ioda_setup_PEs(hose->bus);
184cd4a3
BH		 1151 }
1152}
1153
a8b2f828 1154#ifdef CONFIG_PCI_IOV
781a868f
WY		 1155static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1156{
1157 struct pci_bus *bus;
1158 struct pci_controller *hose;
1159 struct pnv_phb *phb;
1160 struct pci_dn *pdn;
02639b0e 1161 int i, j;
781a868f
WY		 1162
1163 bus = pdev->bus;
1164 hose = pci_bus_to_host(bus);
1165 phb = hose->private_data;
1166 pdn = pci_get_pdn(pdev);
1167
02639b0e
WY		 1168 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1169 for (j = 0; j < M64_PER_IOV; j++) {
1170 if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1171 continue;
1172 opal_pci_phb_mmio_enable(phb->opal_id,
1173 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1174 clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1175 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1176 }
781a868f
WY		 1177
1178 return 0;
1179}
1180
02639b0e 1181static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
781a868f
WY		 1182{
1183 struct pci_bus *bus;
1184 struct pci_controller *hose;
1185 struct pnv_phb *phb;
1186 struct pci_dn *pdn;
1187 unsigned int win;
1188 struct resource *res;
02639b0e 1189 int i, j;
781a868f 1190 int64_t rc;
02639b0e
WY		 1191 int total_vfs;
1192 resource_size_t size, start;
1193 int pe_num;
1194 int vf_groups;
1195 int vf_per_group;
781a868f
WY		 1196
1197 bus = pdev->bus;
1198 hose = pci_bus_to_host(bus);
1199 phb = hose->private_data;
1200 pdn = pci_get_pdn(pdev);
02639b0e 1201 total_vfs = pci_sriov_get_totalvfs(pdev);
781a868f
WY		 1202
1203 /* Initialize the m64_wins to IODA_INVALID_M64 */
1204 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
02639b0e
WY		 1205 for (j = 0; j < M64_PER_IOV; j++)
1206 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1207
1208 if (pdn->m64_per_iov == M64_PER_IOV) {
1209 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1210 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1211 roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1212 } else {
1213 vf_groups = 1;
1214 vf_per_group = 1;
1215 }
781a868f
WY		 1216
1217 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1218 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1219 if (!res->flags || !res->parent)
1220 continue;
1221
1222 if (!pnv_pci_is_mem_pref_64(res->flags))
1223 continue;
1224
02639b0e
WY		 1225 for (j = 0; j < vf_groups; j++) {
1226 do {
1227 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1228 phb->ioda.m64_bar_idx + 1, 0);
1229
1230 if (win >= phb->ioda.m64_bar_idx + 1)
1231 goto m64_failed;
1232 } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1233
1234 pdn->m64_wins[i][j] = win;
1235
1236 if (pdn->m64_per_iov == M64_PER_IOV) {
1237 size = pci_iov_resource_size(pdev,
1238 PCI_IOV_RESOURCES + i);
1239 size = size * vf_per_group;
1240 start = res->start + size * j;
1241 } else {
1242 size = resource_size(res);
1243 start = res->start;
1244 }
1245
1246 /* Map the M64 here */
1247 if (pdn->m64_per_iov == M64_PER_IOV) {
1248 pe_num = pdn->offset + j;
1249 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1250 pe_num, OPAL_M64_WINDOW_TYPE,
1251 pdn->m64_wins[i][j], 0);
1252 }
1253
1254 rc = opal_pci_set_phb_mem_window(phb->opal_id,
1255 OPAL_M64_WINDOW_TYPE,
1256 pdn->m64_wins[i][j],
1257 start,
1258 0, /* unused */
1259 size);
781a868f 1260
781a868f 1261
02639b0e
WY		 1262 if (rc != OPAL_SUCCESS) {
1263 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1264 win, rc);
1265 goto m64_failed;
1266 }
781a868f 1267
02639b0e
WY		 1268 if (pdn->m64_per_iov == M64_PER_IOV)
1269 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1270 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1271 else
1272 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1273 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
781a868f 1274
02639b0e
WY		 1275 if (rc != OPAL_SUCCESS) {
1276 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1277 win, rc);
1278 goto m64_failed;
1279 }
781a868f
WY		 1280 }
1281 }
1282 return 0;
1283
1284m64_failed:
1285 pnv_pci_vf_release_m64(pdev);
1286 return -EBUSY;
1287}
1288
c035e37b
AK		 1289static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1290 int num);
1291static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
1292
781a868f
WY		 1293static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1294{
781a868f 1295 struct iommu_table *tbl;
781a868f
WY		 1296 int64_t rc;
1297
b348aa65 1298 tbl = pe->table_group.tables[0];
c035e37b 1299 rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
781a868f
WY		 1300 if (rc)
1301 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1302
c035e37b 1303 pnv_pci_ioda2_set_bypass(pe, false);
0eaf4def
AK		 1304 if (pe->table_group.group) {
1305 iommu_group_put(pe->table_group.group);
1306 BUG_ON(pe->table_group.group);
ac9a5889 1307 }
aca6913f 1308 pnv_pci_ioda2_table_free_pages(tbl);
781a868f 1309 iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
781a868f
WY		 1310}
1311
02639b0e 1312static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
781a868f
WY		 1313{
1314 struct pci_bus *bus;
1315 struct pci_controller *hose;
1316 struct pnv_phb *phb;
1317 struct pnv_ioda_pe *pe, *pe_n;
1318 struct pci_dn *pdn;
02639b0e
WY		 1319 u16 vf_index;
1320 int64_t rc;
781a868f
WY		 1321
1322 bus = pdev->bus;
1323 hose = pci_bus_to_host(bus);
1324 phb = hose->private_data;
02639b0e 1325 pdn = pci_get_pdn(pdev);
781a868f
WY		 1326
1327 if (!pdev->is_physfn)
1328 return;
1329
02639b0e
WY		 1330 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1331 int vf_group;
1332 int vf_per_group;
1333 int vf_index1;
1334
1335 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1336
1337 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1338 for (vf_index = vf_group * vf_per_group;
1339 vf_index < (vf_group + 1) * vf_per_group &&
1340 vf_index < num_vfs;
1341 vf_index++)
1342 for (vf_index1 = vf_group * vf_per_group;
1343 vf_index1 < (vf_group + 1) * vf_per_group &&
1344 vf_index1 < num_vfs;
1345 vf_index1++){
1346
1347 rc = opal_pci_set_peltv(phb->opal_id,
1348 pdn->offset + vf_index,
1349 pdn->offset + vf_index1,
1350 OPAL_REMOVE_PE_FROM_DOMAIN);
1351
1352 if (rc)
1353 dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1354 __func__,
1355 pdn->offset + vf_index1, rc);
1356 }
1357 }
1358
781a868f
WY		 1359 list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1360 if (pe->parent_dev != pdev)
1361 continue;
1362
1363 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1364
1365 /* Remove from list */
1366 mutex_lock(&phb->ioda.pe_list_mutex);
1367 list_del(&pe->list);
1368 mutex_unlock(&phb->ioda.pe_list_mutex);
1369
1370 pnv_ioda_deconfigure_pe(phb, pe);
1371
1372 pnv_ioda_free_pe(phb, pe->pe_number);
1373 }
1374}
1375
1376void pnv_pci_sriov_disable(struct pci_dev *pdev)
1377{
1378 struct pci_bus *bus;
1379 struct pci_controller *hose;
1380 struct pnv_phb *phb;
1381 struct pci_dn *pdn;
1382 struct pci_sriov *iov;
1383 u16 num_vfs;
1384
1385 bus = pdev->bus;
1386 hose = pci_bus_to_host(bus);
1387 phb = hose->private_data;
1388 pdn = pci_get_pdn(pdev);
1389 iov = pdev->sriov;
1390 num_vfs = pdn->num_vfs;
1391
1392 /* Release VF PEs */
02639b0e 1393 pnv_ioda_release_vf_PE(pdev, num_vfs);
781a868f
WY		 1394
1395 if (phb->type == PNV_PHB_IODA2) {
02639b0e
WY		 1396 if (pdn->m64_per_iov == 1)
1397 pnv_pci_vf_resource_shift(pdev, -pdn->offset);
781a868f
WY		 1398
1399 /* Release M64 windows */
1400 pnv_pci_vf_release_m64(pdev);
1401
1402 /* Release PE numbers */
1403 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1404 pdn->offset = 0;
1405 }
1406}
1407
1408static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1409 struct pnv_ioda_pe *pe);
1410static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1411{
1412 struct pci_bus *bus;
1413 struct pci_controller *hose;
1414 struct pnv_phb *phb;
1415 struct pnv_ioda_pe *pe;
1416 int pe_num;
1417 u16 vf_index;
1418 struct pci_dn *pdn;
02639b0e 1419 int64_t rc;
781a868f
WY		 1420
1421 bus = pdev->bus;
1422 hose = pci_bus_to_host(bus);
1423 phb = hose->private_data;
1424 pdn = pci_get_pdn(pdev);
1425
1426 if (!pdev->is_physfn)
1427 return;
1428
1429 /* Reserve PE for each VF */
1430 for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1431 pe_num = pdn->offset + vf_index;
1432
1433 pe = &phb->ioda.pe_array[pe_num];
1434 pe->pe_number = pe_num;
1435 pe->phb = phb;
1436 pe->flags = PNV_IODA_PE_VF;
1437 pe->pbus = NULL;
1438 pe->parent_dev = pdev;
1439 pe->tce32_seg = -1;
1440 pe->mve_number = -1;
1441 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1442 pci_iov_virtfn_devfn(pdev, vf_index);
1443
1444 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1445 hose->global_number, pdev->bus->number,
1446 PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1447 PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1448
1449 if (pnv_ioda_configure_pe(phb, pe)) {
1450 /* XXX What do we do here ? */
1451 if (pe_num)
1452 pnv_ioda_free_pe(phb, pe_num);
1453 pe->pdev = NULL;
1454 continue;
1455 }
1456
781a868f
WY		 1457 /* Put PE to the list */
1458 mutex_lock(&phb->ioda.pe_list_mutex);
1459 list_add_tail(&pe->list, &phb->ioda.pe_list);
1460 mutex_unlock(&phb->ioda.pe_list_mutex);
1461
1462 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1463 }
02639b0e
WY		 1464
1465 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1466 int vf_group;
1467 int vf_per_group;
1468 int vf_index1;
1469
1470 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1471
1472 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1473 for (vf_index = vf_group * vf_per_group;
1474 vf_index < (vf_group + 1) * vf_per_group &&
1475 vf_index < num_vfs;
1476 vf_index++) {
1477 for (vf_index1 = vf_group * vf_per_group;
1478 vf_index1 < (vf_group + 1) * vf_per_group &&
1479 vf_index1 < num_vfs;
1480 vf_index1++) {
1481
1482 rc = opal_pci_set_peltv(phb->opal_id,
1483 pdn->offset + vf_index,
1484 pdn->offset + vf_index1,
1485 OPAL_ADD_PE_TO_DOMAIN);
1486
1487 if (rc)
1488 dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1489 __func__,
1490 pdn->offset + vf_index1, rc);
1491 }
1492 }
1493 }
1494 }
781a868f
WY		 1495}
1496
1497int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1498{
1499 struct pci_bus *bus;
1500 struct pci_controller *hose;
1501 struct pnv_phb *phb;
1502 struct pci_dn *pdn;
1503 int ret;
1504
1505 bus = pdev->bus;
1506 hose = pci_bus_to_host(bus);
1507 phb = hose->private_data;
1508 pdn = pci_get_pdn(pdev);
1509
1510 if (phb->type == PNV_PHB_IODA2) {
1511 /* Calculate available PE for required VFs */
1512 mutex_lock(&phb->ioda.pe_alloc_mutex);
1513 pdn->offset = bitmap_find_next_zero_area(
1514 phb->ioda.pe_alloc, phb->ioda.total_pe,
1515 0, num_vfs, 0);
1516 if (pdn->offset >= phb->ioda.total_pe) {
1517 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1518 dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1519 pdn->offset = 0;
1520 return -EBUSY;
1521 }
1522 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1523 pdn->num_vfs = num_vfs;
1524 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1525
1526 /* Assign M64 window accordingly */
02639b0e 1527 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
781a868f
WY		 1528 if (ret) {
1529 dev_info(&pdev->dev, "Not enough M64 window resources\n");
1530 goto m64_failed;
1531 }
1532
1533 /*
1534 * When using one M64 BAR to map one IOV BAR, we need to shift
1535 * the IOV BAR according to the PE# allocated to the VFs.
1536 * Otherwise, the PE# for the VF will conflict with others.
1537 */
02639b0e
WY		 1538 if (pdn->m64_per_iov == 1) {
1539 ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1540 if (ret)
1541 goto m64_failed;
1542 }
781a868f
WY		 1543 }
1544
1545 /* Setup VF PEs */
1546 pnv_ioda_setup_vf_PE(pdev, num_vfs);
1547
1548 return 0;
1549
1550m64_failed:
1551 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1552 pdn->offset = 0;
1553
1554 return ret;
1555}
1556
a8b2f828
GS		 1557int pcibios_sriov_disable(struct pci_dev *pdev)
1558{
781a868f
WY		 1559 pnv_pci_sriov_disable(pdev);
1560
a8b2f828
GS		 1561 /* Release PCI data */
1562 remove_dev_pci_data(pdev);
1563 return 0;
1564}
1565
1566int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1567{
1568 /* Allocate PCI data */
1569 add_dev_pci_data(pdev);
781a868f
WY		 1570
1571 pnv_pci_sriov_enable(pdev, num_vfs);
a8b2f828
GS		 1572 return 0;
1573}
1574#endif /* CONFIG_PCI_IOV */
1575
959c9bdd 1576static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
184cd4a3 1577{
b72c1f65 1578 struct pci_dn *pdn = pci_get_pdn(pdev);
959c9bdd 1579 struct pnv_ioda_pe *pe;
184cd4a3 1580
959c9bdd
GS		 1581 /*
1582 * The function can be called while the PE#
1583 * hasn't been assigned. Do nothing for the
1584 * case.
1585 */
1586 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1587 return;
184cd4a3 1588
959c9bdd 1589 pe = &phb->ioda.pe_array[pdn->pe_number];
cd15b048 1590 WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
b348aa65 1591 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
4617082e
AK		 1592 /*
1593 * Note: iommu_add_device() will fail here as
1594 * for physical PE: the device is already added by now;
1595 * for virtual PE: sysfs entries are not ready yet and
1596 * tce_iommu_bus_notifier will add the device to a group later.
1597 */
184cd4a3
BH		 1598}
1599
763d2d8d 1600static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
cd15b048 1601{
763d2d8d
DA		 1602 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1603 struct pnv_phb *phb = hose->private_data;
cd15b048
BH		 1604 struct pci_dn *pdn = pci_get_pdn(pdev);
1605 struct pnv_ioda_pe *pe;
1606 uint64_t top;
1607 bool bypass = false;
1608
1609 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1610 return -ENODEV;;
1611
1612 pe = &phb->ioda.pe_array[pdn->pe_number];
1613 if (pe->tce_bypass_enabled) {
1614 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1615 bypass = (dma_mask >= top);
1616 }
1617
1618 if (bypass) {
1619 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1620 set_dma_ops(&pdev->dev, &dma_direct_ops);
cd15b048
BH		 1621 } else {
1622 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1623 set_dma_ops(&pdev->dev, &dma_iommu_ops);
cd15b048 1624 }
a32305bf 1625 *pdev->dev.dma_mask = dma_mask;
cd15b048
BH		 1626 return 0;
1627}
1628
fe7e85c6
GS		 1629static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1630 struct pci_dev *pdev)
1631{
1632 struct pci_dn *pdn = pci_get_pdn(pdev);
1633 struct pnv_ioda_pe *pe;
1634 u64 end, mask;
1635
1636 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1637 return 0;
1638
1639 pe = &phb->ioda.pe_array[pdn->pe_number];
1640 if (!pe->tce_bypass_enabled)
1641 return __dma_get_required_mask(&pdev->dev);
1642
1643
1644 end = pe->tce_bypass_base + memblock_end_of_DRAM();
1645 mask = 1ULL << (fls64(end) - 1);
1646 mask += mask - 1;
1647
1648 return mask;
1649}
1650
dff4a39e 1651static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
ea30e99e 1652 struct pci_bus *bus)
74251fe2
BH		 1653{
1654 struct pci_dev *dev;
1655
1656 list_for_each_entry(dev, &bus->devices, bus_list) {
b348aa65 1657 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
e91c2511 1658 set_dma_offset(&dev->dev, pe->tce_bypass_base);
4617082e 1659 iommu_add_device(&dev->dev);
dff4a39e 1660
5c89a87d 1661 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
ea30e99e 1662 pnv_ioda_setup_bus_dma(pe, dev->subordinate);
74251fe2
BH		 1663 }
1664}
1665
decbda25
AK		 1666static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1667 unsigned long index, unsigned long npages, bool rm)
4cce9550 1668{
0eaf4def
AK		 1669 struct iommu_table_group_link *tgl = list_first_entry_or_null(
1670 &tbl->it_group_list, struct iommu_table_group_link,
1671 next);
1672 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
b348aa65 1673 struct pnv_ioda_pe, table_group);
3ad26e5c 1674 __be64 __iomem *invalidate = rm ?
5780fb04
AK		 1675 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1676 pe->phb->ioda.tce_inval_reg;
4cce9550 1677 unsigned long start, end, inc;
b0376c9b 1678 const unsigned shift = tbl->it_page_shift;
4cce9550 1679
decbda25
AK		 1680 start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1681 end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1682 npages - 1);
4cce9550
GS		 1683
1684 /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1685 if (tbl->it_busno) {
b0376c9b
AK		 1686 start <<= shift;
1687 end <<= shift;
1688 inc = 128ull << shift;
4cce9550
GS		 1689 start |= tbl->it_busno;
1690 end |= tbl->it_busno;
1691 } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1692 /* p7ioc-style invalidation, 2 TCEs per write */
1693 start |= (1ull << 63);
1694 end |= (1ull << 63);
1695 inc = 16;
1696 } else {
1697 /* Default (older HW) */
1698 inc = 128;
1699 }
1700
1701 end |= inc - 1; /* round up end to be different than start */
1702
1703 mb(); /* Ensure above stores are visible */
1704 while (start <= end) {
8e0a1611 1705 if (rm)
3ad26e5c 1706 __raw_rm_writeq(cpu_to_be64(start), invalidate);
8e0a1611 1707 else
3ad26e5c 1708 __raw_writeq(cpu_to_be64(start), invalidate);
4cce9550
GS		 1709 start += inc;
1710 }
1711
1712 /*
1713 * The iommu layer will do another mb() for us on build()
1714 * and we don't care on free()
1715 */
1716}
1717
decbda25
AK		 1718static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1719 long npages, unsigned long uaddr,
1720 enum dma_data_direction direction,
1721 struct dma_attrs *attrs)
1722{
1723 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1724 attrs);
1725
1726 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1727 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1728
1729 return ret;
1730}
1731
05c6cfb9
AK		 1732#ifdef CONFIG_IOMMU_API
1733static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
1734 unsigned long *hpa, enum dma_data_direction *direction)
1735{
1736 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1737
1738 if (!ret && (tbl->it_type &
1739 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1740 pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
1741
1742 return ret;
1743}
1744#endif
1745
decbda25
AK		 1746static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1747 long npages)
1748{
1749 pnv_tce_free(tbl, index, npages);
1750
1751 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1752 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1753}
1754
da004c36 1755static struct iommu_table_ops pnv_ioda1_iommu_ops = {
decbda25 1756 .set = pnv_ioda1_tce_build,
05c6cfb9
AK		 1757#ifdef CONFIG_IOMMU_API
1758 .exchange = pnv_ioda1_tce_xchg,
1759#endif
decbda25 1760 .clear = pnv_ioda1_tce_free,
da004c36
AK		 1761 .get = pnv_tce_get,
1762};
1763
5780fb04
AK		 1764static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1765{
1766 /* 01xb - invalidate TCEs that match the specified PE# */
1767 unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1768 struct pnv_phb *phb = pe->phb;
1769
1770 if (!phb->ioda.tce_inval_reg)
1771 return;
1772
1773 mb(); /* Ensure above stores are visible */
1774 __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1775}
1776
e57080f1
AK		 1777static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
1778 __be64 __iomem *invalidate, unsigned shift,
1779 unsigned long index, unsigned long npages)
4cce9550
GS		 1780{
1781 unsigned long start, end, inc;
4cce9550
GS		 1782
1783 /* We'll invalidate DMA address in PE scope */
b0376c9b 1784 start = 0x2ull << 60;
e57080f1 1785 start |= (pe_number & 0xFF);
4cce9550
GS		 1786 end = start;
1787
1788 /* Figure out the start, end and step */
decbda25
AK		 1789 start |= (index << shift);
1790 end |= ((index + npages - 1) << shift);
b0376c9b 1791 inc = (0x1ull << shift);
4cce9550
GS		 1792 mb();
1793
1794 while (start <= end) {
8e0a1611 1795 if (rm)
3ad26e5c 1796 __raw_rm_writeq(cpu_to_be64(start), invalidate);
8e0a1611 1797 else
3ad26e5c 1798 __raw_writeq(cpu_to_be64(start), invalidate);
4cce9550
GS		 1799 start += inc;
1800 }
1801}
1802
e57080f1
AK		 1803static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1804 unsigned long index, unsigned long npages, bool rm)
1805{
1806 struct iommu_table_group_link *tgl;
1807
1808 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1809 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1810 struct pnv_ioda_pe, table_group);
1811 __be64 __iomem *invalidate = rm ?
1812 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1813 pe->phb->ioda.tce_inval_reg;
1814
1815 pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
1816 invalidate, tbl->it_page_shift,
1817 index, npages);
1818 }
1819}
1820
decbda25
AK		 1821static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1822 long npages, unsigned long uaddr,
1823 enum dma_data_direction direction,
1824 struct dma_attrs *attrs)
4cce9550 1825{
decbda25
AK		 1826 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1827 attrs);
4cce9550 1828
decbda25
AK		 1829 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1830 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1831
1832 return ret;
1833}
1834
05c6cfb9
AK		 1835#ifdef CONFIG_IOMMU_API
1836static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
1837 unsigned long *hpa, enum dma_data_direction *direction)
1838{
1839 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1840
1841 if (!ret && (tbl->it_type &
1842 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1843 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
1844
1845 return ret;
1846}
1847#endif
1848
decbda25
AK		 1849static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1850 long npages)
1851{
1852 pnv_tce_free(tbl, index, npages);
1853
1854 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1855 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
4cce9550
GS		 1856}
1857
4793d65d
AK		 1858static void pnv_ioda2_table_free(struct iommu_table *tbl)
1859{
1860 pnv_pci_ioda2_table_free_pages(tbl);
1861 iommu_free_table(tbl, "pnv");
1862}
1863
da004c36 1864static struct iommu_table_ops pnv_ioda2_iommu_ops = {
decbda25 1865 .set = pnv_ioda2_tce_build,
05c6cfb9
AK		 1866#ifdef CONFIG_IOMMU_API
1867 .exchange = pnv_ioda2_tce_xchg,
1868#endif
decbda25 1869 .clear = pnv_ioda2_tce_free,
da004c36 1870 .get = pnv_tce_get,
4793d65d 1871 .free = pnv_ioda2_table_free,
da004c36
AK		 1872};
1873
cad5cef6
GKH		 1874static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1875 struct pnv_ioda_pe *pe, unsigned int base,
1876 unsigned int segs)
184cd4a3
BH		 1877{
1878
1879 struct page *tce_mem = NULL;
184cd4a3
BH		 1880 struct iommu_table *tbl;
1881 unsigned int i;
1882 int64_t rc;
1883 void *addr;
1884
184cd4a3
BH		 1885 /* XXX FIXME: Handle 64-bit only DMA devices */
1886 /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1887 /* XXX FIXME: Allocate multi-level tables on PHB3 */
1888
1889 /* We shouldn't already have a 32-bit DMA associated */
1890 if (WARN_ON(pe->tce32_seg >= 0))
1891 return;
1892
0eaf4def 1893 tbl = pnv_pci_table_alloc(phb->hose->node);
b348aa65
AK		 1894 iommu_register_group(&pe->table_group, phb->hose->global_number,
1895 pe->pe_number);
0eaf4def 1896 pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
c5773822 1897
184cd4a3
BH		 1898 /* Grab a 32-bit TCE table */
1899 pe->tce32_seg = base;
1900 pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1901 (base << 28), ((base + segs) << 28) - 1);
1902
1903 /* XXX Currently, we allocate one big contiguous table for the
1904 * TCEs. We only really need one chunk per 256M of TCE space
1905 * (ie per segment) but that's an optimization for later, it
1906 * requires some added smarts with our get/put_tce implementation
1907 */
1908 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1909 get_order(TCE32_TABLE_SIZE * segs));
1910 if (!tce_mem) {
1911 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1912 goto fail;
1913 }
1914 addr = page_address(tce_mem);
1915 memset(addr, 0, TCE32_TABLE_SIZE * segs);
1916
1917 /* Configure HW */
1918 for (i = 0; i < segs; i++) {
1919 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1920 pe->pe_number,
1921 base + i, 1,
1922 __pa(addr) + TCE32_TABLE_SIZE * i,
1923 TCE32_TABLE_SIZE, 0x1000);
1924 if (rc) {
1925 pe_err(pe, " Failed to configure 32-bit TCE table,"
1926 " err %ld\n", rc);
1927 goto fail;
1928 }
1929 }
1930
1931 /* Setup linux iommu table */
184cd4a3 1932 pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
8fa5d454 1933 base << 28, IOMMU_PAGE_SHIFT_4K);
184cd4a3
BH		 1934
1935 /* OPAL variant of P7IOC SW invalidated TCEs */
5780fb04 1936 if (phb->ioda.tce_inval_reg)
65fd766b
GS		 1937 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1938 TCE_PCI_SWINV_FREE |
1939 TCE_PCI_SWINV_PAIR);
5780fb04 1940
da004c36 1941 tbl->it_ops = &pnv_ioda1_iommu_ops;
4793d65d
AK		 1942 pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
1943 pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
184cd4a3
BH		 1944 iommu_init_table(tbl, phb->hose->node);
1945
781a868f 1946 if (pe->flags & PNV_IODA_PE_DEV) {
4617082e
AK		 1947 /*
1948 * Setting table base here only for carrying iommu_group
1949 * further down to let iommu_add_device() do the job.
1950 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1951 */
1952 set_iommu_table_base(&pe->pdev->dev, tbl);
1953 iommu_add_device(&pe->pdev->dev);
c5773822 1954 } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
ea30e99e 1955 pnv_ioda_setup_bus_dma(pe, pe->pbus);
74251fe2 1956
184cd4a3
BH		 1957 return;
1958 fail:
1959 /* XXX Failure: Try to fallback to 64-bit only ? */
1960 if (pe->tce32_seg >= 0)
1961 pe->tce32_seg = -1;
1962 if (tce_mem)
1963 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
0eaf4def
AK		 1964 if (tbl) {
1965 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1966 iommu_free_table(tbl, "pnv");
1967 }
184cd4a3
BH		 1968}
1969
43cb60ab
AK		 1970static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1971 int num, struct iommu_table *tbl)
1972{
1973 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1974 table_group);
1975 struct pnv_phb *phb = pe->phb;
1976 int64_t rc;
bbb845c4
AK		 1977 const unsigned long size = tbl->it_indirect_levels ?
1978 tbl->it_level_size : tbl->it_size;
43cb60ab
AK		 1979 const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1980 const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1981
4793d65d 1982 pe_info(pe, "Setting up window#%d %llx..%llx pg=%x\n", num,
43cb60ab
AK		 1983 start_addr, start_addr + win_size - 1,
1984 IOMMU_PAGE_SIZE(tbl));
1985
1986 /*
1987 * Map TCE table through TVT. The TVE index is the PE number
1988 * shifted by 1 bit for 32-bits DMA space.
1989 */
1990 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1991 pe->pe_number,
4793d65d 1992 (pe->pe_number << 1) + num,
bbb845c4 1993 tbl->it_indirect_levels + 1,
43cb60ab 1994 __pa(tbl->it_base),
bbb845c4 1995 size << 3,
43cb60ab
AK		 1996 IOMMU_PAGE_SIZE(tbl));
1997 if (rc) {
1998 pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
1999 return rc;
2000 }
2001
2002 pnv_pci_link_table_and_group(phb->hose->node, num,
2003 tbl, &pe->table_group);
2004 pnv_pci_ioda2_tce_invalidate_entire(pe);
2005
2006 return 0;
2007}
2008
f87a8864 2009static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
cd15b048 2010{
cd15b048
BH		 2011 uint16_t window_id = (pe->pe_number << 1 ) + 1;
2012 int64_t rc;
2013
2014 pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
2015 if (enable) {
2016 phys_addr_t top = memblock_end_of_DRAM();
2017
2018 top = roundup_pow_of_two(top);
2019 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2020 pe->pe_number,
2021 window_id,
2022 pe->tce_bypass_base,
2023 top);
2024 } else {
2025 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2026 pe->pe_number,
2027 window_id,
2028 pe->tce_bypass_base,
2029 0);
cd15b048
BH		 2030 }
2031 if (rc)
2032 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
2033 else
2034 pe->tce_bypass_enabled = enable;
2035}
2036
4793d65d
AK		 2037static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2038 __u32 page_shift, __u64 window_size, __u32 levels,
2039 struct iommu_table *tbl);
2040
2041static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
2042 int num, __u32 page_shift, __u64 window_size, __u32 levels,
2043 struct iommu_table **ptbl)
2044{
2045 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2046 table_group);
2047 int nid = pe->phb->hose->node;
2048 __u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
2049 long ret;
2050 struct iommu_table *tbl;
2051
2052 tbl = pnv_pci_table_alloc(nid);
2053 if (!tbl)
2054 return -ENOMEM;
2055
2056 ret = pnv_pci_ioda2_table_alloc_pages(nid,
2057 bus_offset, page_shift, window_size,
2058 levels, tbl);
2059 if (ret) {
2060 iommu_free_table(tbl, "pnv");
2061 return ret;
2062 }
2063
2064 tbl->it_ops = &pnv_ioda2_iommu_ops;
2065 if (pe->phb->ioda.tce_inval_reg)
2066 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2067
2068 *ptbl = tbl;
2069
2070 return 0;
2071}
2072
46d3e1e1
AK		 2073static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
2074{
2075 struct iommu_table *tbl = NULL;
2076 long rc;
2077
2078 rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
2079 IOMMU_PAGE_SHIFT_4K,
2080 pe->table_group.tce32_size,
2081 POWERNV_IOMMU_DEFAULT_LEVELS, &tbl);
2082 if (rc) {
2083 pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
2084 rc);
2085 return rc;
2086 }
2087
2088 iommu_init_table(tbl, pe->phb->hose->node);
2089
2090 rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
2091 if (rc) {
2092 pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
2093 rc);
2094 pnv_ioda2_table_free(tbl);
2095 return rc;
2096 }
2097
2098 if (!pnv_iommu_bypass_disabled)
2099 pnv_pci_ioda2_set_bypass(pe, true);
2100
2101 /* OPAL variant of PHB3 invalidated TCEs */
2102 if (pe->phb->ioda.tce_inval_reg)
2103 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2104
2105 /*
2106 * Setting table base here only for carrying iommu_group
2107 * further down to let iommu_add_device() do the job.
2108 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2109 */
2110 if (pe->flags & PNV_IODA_PE_DEV)
2111 set_iommu_table_base(&pe->pdev->dev, tbl);
2112
2113 return 0;
2114}
2115
b5926430
AK		 2116#if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
2117static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
2118 int num)
2119{
2120 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2121 table_group);
2122 struct pnv_phb *phb = pe->phb;
2123 long ret;
2124
2125 pe_info(pe, "Removing DMA window #%d\n", num);
2126
2127 ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2128 (pe->pe_number << 1) + num,
2129 0/* levels */, 0/* table address */,
2130 0/* table size */, 0/* page size */);
2131 if (ret)
2132 pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
2133 else
2134 pnv_pci_ioda2_tce_invalidate_entire(pe);
2135
2136 pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
2137
2138 return ret;
2139}
2140#endif
2141
f87a8864 2142#ifdef CONFIG_IOMMU_API
00547193
AK		 2143static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
2144 __u64 window_size, __u32 levels)
2145{
2146 unsigned long bytes = 0;
2147 const unsigned window_shift = ilog2(window_size);
2148 unsigned entries_shift = window_shift - page_shift;
2149 unsigned table_shift = entries_shift + 3;
2150 unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
2151 unsigned long direct_table_size;
2152
2153 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
2154 (window_size > memory_hotplug_max()) ||
2155 !is_power_of_2(window_size))
2156 return 0;
2157
2158 /* Calculate a direct table size from window_size and levels */
2159 entries_shift = (entries_shift + levels - 1) / levels;
2160 table_shift = entries_shift + 3;
2161 table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
2162 direct_table_size = 1UL << table_shift;
2163
2164 for ( ; levels; --levels) {
2165 bytes += _ALIGN_UP(tce_table_size, direct_table_size);
2166
2167 tce_table_size /= direct_table_size;
2168 tce_table_size <<= 3;
2169 tce_table_size = _ALIGN_UP(tce_table_size, direct_table_size);
2170 }
2171
2172 return bytes;
2173}
2174
f87a8864 2175static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
cd15b048 2176{
f87a8864
AK		 2177 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2178 table_group);
46d3e1e1
AK		 2179 /* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
2180 struct iommu_table *tbl = pe->table_group.tables[0];
cd15b048 2181
f87a8864 2182 pnv_pci_ioda2_set_bypass(pe, false);
46d3e1e1
AK		 2183 pnv_pci_ioda2_unset_window(&pe->table_group, 0);
2184 pnv_ioda2_table_free(tbl);
f87a8864 2185}
cd15b048 2186
f87a8864
AK		 2187static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2188{
2189 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2190 table_group);
2191
46d3e1e1 2192 pnv_pci_ioda2_setup_default_config(pe);
cd15b048
BH		 2193}
2194
f87a8864 2195static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
00547193 2196 .get_table_size = pnv_pci_ioda2_get_table_size,
4793d65d
AK		 2197 .create_table = pnv_pci_ioda2_create_table,
2198 .set_window = pnv_pci_ioda2_set_window,
2199 .unset_window = pnv_pci_ioda2_unset_window,
f87a8864
AK		 2200 .take_ownership = pnv_ioda2_take_ownership,
2201 .release_ownership = pnv_ioda2_release_ownership,
2202};
2203#endif
2204
5780fb04
AK		 2205static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
2206{
2207 const __be64 *swinvp;
2208
2209 /* OPAL variant of PHB3 invalidated TCEs */
2210 swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
2211 if (!swinvp)
2212 return;
2213
2214 phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
2215 phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
2216}
2217
bbb845c4
AK		 2218static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift,
2219 unsigned levels, unsigned long limit,
2220 unsigned long *current_offset)
373f5657
GS		 2221{
2222 struct page *tce_mem = NULL;
bbb845c4 2223 __be64 *addr, *tmp;
aca6913f 2224 unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
bbb845c4
AK		 2225 unsigned long allocated = 1UL << (order + PAGE_SHIFT);
2226 unsigned entries = 1UL << (shift - 3);
2227 long i;
aca6913f
AK		 2228
2229 tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
2230 if (!tce_mem) {
2231 pr_err("Failed to allocate a TCE memory, order=%d\n", order);
2232 return NULL;
2233 }
2234 addr = page_address(tce_mem);
bbb845c4
AK		 2235 memset(addr, 0, allocated);
2236
2237 --levels;
2238 if (!levels) {
2239 *current_offset += allocated;
2240 return addr;
2241 }
2242
2243 for (i = 0; i < entries; ++i) {
2244 tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
2245 levels, limit, current_offset);
2246 if (!tmp)
2247 break;
2248
2249 addr[i] = cpu_to_be64(__pa(tmp) |
2250 TCE_PCI_READ | TCE_PCI_WRITE);
2251
2252 if (*current_offset >= limit)
2253 break;
2254 }
aca6913f
AK		 2255
2256 return addr;
2257}
2258
bbb845c4
AK		 2259static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2260 unsigned long size, unsigned level);
2261
aca6913f 2262static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
bbb845c4
AK		 2263 __u32 page_shift, __u64 window_size, __u32 levels,
2264 struct iommu_table *tbl)
aca6913f 2265{
373f5657 2266 void *addr;
bbb845c4 2267 unsigned long offset = 0, level_shift;
aca6913f
AK		 2268 const unsigned window_shift = ilog2(window_size);
2269 unsigned entries_shift = window_shift - page_shift;
2270 unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
2271 const unsigned long tce_table_size = 1UL << table_shift;
2272
bbb845c4
AK		 2273 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
2274 return -EINVAL;
2275
aca6913f
AK		 2276 if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
2277 return -EINVAL;
2278
bbb845c4
AK		 2279 /* Adjust direct table size from window_size and levels */
2280 entries_shift = (entries_shift + levels - 1) / levels;
2281 level_shift = entries_shift + 3;
2282 level_shift = max_t(unsigned, level_shift, PAGE_SHIFT);
2283
aca6913f 2284 /* Allocate TCE table */
bbb845c4
AK		 2285 addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
2286 levels, tce_table_size, &offset);
2287
2288 /* addr==NULL means that the first level allocation failed */
aca6913f
AK		 2289 if (!addr)
2290 return -ENOMEM;
2291
bbb845c4
AK		 2292 /*
2293 * First level was allocated but some lower level failed as
2294 * we did not allocate as much as we wanted,
2295 * release partially allocated table.
2296 */
2297 if (offset < tce_table_size) {
2298 pnv_pci_ioda2_table_do_free_pages(addr,
2299 1ULL << (level_shift - 3), levels - 1);
2300 return -ENOMEM;
2301 }
2302
aca6913f
AK		 2303 /* Setup linux iommu table */
2304 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
2305 page_shift);
bbb845c4
AK		 2306 tbl->it_level_size = 1ULL << (level_shift - 3);
2307 tbl->it_indirect_levels = levels - 1;
00547193 2308 tbl->it_allocated_size = offset;
aca6913f
AK		 2309
2310 pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
2311 window_size, tce_table_size, bus_offset);
2312
2313 return 0;
2314}
2315
bbb845c4
AK		 2316static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2317 unsigned long size, unsigned level)
2318{
2319 const unsigned long addr_ul = (unsigned long) addr &
2320 ~(TCE_PCI_READ | TCE_PCI_WRITE);
2321
2322 if (level) {
2323 long i;
2324 u64 *tmp = (u64 *) addr_ul;
2325
2326 for (i = 0; i < size; ++i) {
2327 unsigned long hpa = be64_to_cpu(tmp[i]);
2328
2329 if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
2330 continue;
2331
2332 pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
2333 level - 1);
2334 }
2335 }
2336
2337 free_pages(addr_ul, get_order(size << 3));
2338}
2339
aca6913f
AK		 2340static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
2341{
bbb845c4
AK		 2342 const unsigned long size = tbl->it_indirect_levels ?
2343 tbl->it_level_size : tbl->it_size;
2344
aca6913f
AK		 2345 if (!tbl->it_size)
2346 return;
2347
bbb845c4
AK		 2348 pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
2349 tbl->it_indirect_levels);
aca6913f
AK		 2350}
2351
2352static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2353 struct pnv_ioda_pe *pe)
2354{
373f5657
GS		 2355 int64_t rc;
2356
2357 /* We shouldn't already have a 32-bit DMA associated */
2358 if (WARN_ON(pe->tce32_seg >= 0))
2359 return;
2360
f87a8864
AK		 2361 /* TVE #1 is selected by PCI address bit 59 */
2362 pe->tce_bypass_base = 1ull << 59;
2363
b348aa65
AK		 2364 iommu_register_group(&pe->table_group, phb->hose->global_number,
2365 pe->pe_number);
c5773822 2366
373f5657
GS		 2367 /* The PE will reserve all possible 32-bits space */
2368 pe->tce32_seg = 0;
373f5657 2369 pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
aca6913f 2370 phb->ioda.m32_pci_base);
373f5657 2371
aca6913f 2372 /* Setup linux iommu table */
4793d65d
AK		 2373 pe->table_group.tce32_start = 0;
2374 pe->table_group.tce32_size = phb->ioda.m32_pci_base;
2375 pe->table_group.max_dynamic_windows_supported =
2376 IOMMU_TABLE_GROUP_MAX_TABLES;
2377 pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
2378 pe->table_group.pgsizes = SZ_4K | SZ_64K | SZ_16M;
e5aad1e6
AK		 2379#ifdef CONFIG_IOMMU_API
2380 pe->table_group.ops = &pnv_pci_ioda2_ops;
2381#endif
2382
46d3e1e1 2383 rc = pnv_pci_ioda2_setup_default_config(pe);
373f5657 2384 if (rc) {
46d3e1e1
AK		 2385 if (pe->tce32_seg >= 0)
2386 pe->tce32_seg = -1;
2387 return;
373f5657
GS		 2388 }
2389
46d3e1e1 2390 if (pe->flags & PNV_IODA_PE_DEV)
4617082e 2391 iommu_add_device(&pe->pdev->dev);
46d3e1e1 2392 else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
ea30e99e 2393 pnv_ioda_setup_bus_dma(pe, pe->pbus);
373f5657
GS		 2394}
2395
cad5cef6 2396static void pnv_ioda_setup_dma(struct pnv_phb *phb)
184cd4a3
BH		 2397{
2398 struct pci_controller *hose = phb->hose;
2399 unsigned int residual, remaining, segs, tw, base;
2400 struct pnv_ioda_pe *pe;
2401
2402 /* If we have more PE# than segments available, hand out one
2403 * per PE until we run out and let the rest fail. If not,
2404 * then we assign at least one segment per PE, plus more based
2405 * on the amount of devices under that PE
2406 */
2407 if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2408 residual = 0;
2409 else
2410 residual = phb->ioda.tce32_count -
2411 phb->ioda.dma_pe_count;
2412
2413 pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2414 hose->global_number, phb->ioda.tce32_count);
2415 pr_info("PCI: %d PE# for a total weight of %d\n",
2416 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2417
5780fb04
AK		 2418 pnv_pci_ioda_setup_opal_tce_kill(phb);
2419
184cd4a3
BH		 2420 /* Walk our PE list and configure their DMA segments, hand them
2421 * out one base segment plus any residual segments based on
2422 * weight
2423 */
2424 remaining = phb->ioda.tce32_count;
2425 tw = phb->ioda.dma_weight;
2426 base = 0;
7ebdf956 2427 list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
184cd4a3
BH		 2428 if (!pe->dma_weight)
2429 continue;
2430 if (!remaining) {
2431 pe_warn(pe, "No DMA32 resources available\n");
2432 continue;
2433 }
2434 segs = 1;
2435 if (residual) {
2436 segs += ((pe->dma_weight * residual) + (tw / 2)) / tw;
2437 if (segs > remaining)
2438 segs = remaining;
2439 }
373f5657
GS		 2440
2441 /*
2442 * For IODA2 compliant PHB3, we needn't care about the weight.
2443 * The all available 32-bits DMA space will be assigned to
2444 * the specific PE.
2445 */
2446 if (phb->type == PNV_PHB_IODA1) {
2447 pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2448 pe->dma_weight, segs);
2449 pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2450 } else {
2451 pe_info(pe, "Assign DMA32 space\n");
2452 segs = 0;
2453 pnv_pci_ioda2_setup_dma_pe(phb, pe);
2454 }
2455
184cd4a3
BH		 2456 remaining -= segs;
2457 base += segs;
2458 }
2459}
2460
2461#ifdef CONFIG_PCI_MSI
137436c9
GS		 2462static void pnv_ioda2_msi_eoi(struct irq_data *d)
2463{
2464 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2465 struct irq_chip *chip = irq_data_get_irq_chip(d);
2466 struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2467 ioda.irq_chip);
2468 int64_t rc;
2469
2470 rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2471 WARN_ON_ONCE(rc);
2472
2473 icp_native_eoi(d);
2474}
2475
fd9a1c26
IM		 2476
2477static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2478{
2479 struct irq_data *idata;
2480 struct irq_chip *ichip;
2481
2482 if (phb->type != PNV_PHB_IODA2)
2483 return;
2484
2485 if (!phb->ioda.irq_chip_init) {
2486 /*
2487 * First time we setup an MSI IRQ, we need to setup the
2488 * corresponding IRQ chip to route correctly.
2489 */
2490 idata = irq_get_irq_data(virq);
2491 ichip = irq_data_get_irq_chip(idata);
2492 phb->ioda.irq_chip_init = 1;
2493 phb->ioda.irq_chip = *ichip;
2494 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2495 }
2496 irq_set_chip(virq, &phb->ioda.irq_chip);
2497}
2498
80c49c7e
IM		 2499#ifdef CONFIG_CXL_BASE
2500
6f963ec2 2501struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
80c49c7e
IM		 2502{
2503 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2504
6f963ec2 2505 return of_node_get(hose->dn);
80c49c7e 2506}
6f963ec2 2507EXPORT_SYMBOL(pnv_pci_get_phb_node);
80c49c7e 2508
1212aa1c 2509int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
80c49c7e
IM		 2510{
2511 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2512 struct pnv_phb *phb = hose->private_data;
2513 struct pnv_ioda_pe *pe;
2514 int rc;
2515
2516 pe = pnv_ioda_get_pe(dev);
2517 if (!pe)
2518 return -ENODEV;
2519
2520 pe_info(pe, "Switching PHB to CXL\n");
2521
1212aa1c 2522 rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
80c49c7e
IM		 2523 if (rc)
2524 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2525
2526 return rc;
2527}
1212aa1c 2528EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
80c49c7e
IM		 2529
2530/* Find PHB for cxl dev and allocate MSI hwirqs?
2531 * Returns the absolute hardware IRQ number
2532 */
2533int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2534{
2535 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2536 struct pnv_phb *phb = hose->private_data;
2537 int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2538
2539 if (hwirq < 0) {
2540 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2541 return -ENOSPC;
2542 }
2543
2544 return phb->msi_base + hwirq;
2545}
2546EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2547
2548void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2549{
2550 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2551 struct pnv_phb *phb = hose->private_data;
2552
2553 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2554}
2555EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2556
2557void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2558 struct pci_dev *dev)
2559{
2560 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2561 struct pnv_phb *phb = hose->private_data;
2562 int i, hwirq;
2563
2564 for (i = 1; i < CXL_IRQ_RANGES; i++) {
2565 if (!irqs->range[i])
2566 continue;
2567 pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n",
2568 i, irqs->offset[i],
2569 irqs->range[i]);
2570 hwirq = irqs->offset[i] - phb->msi_base;
2571 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2572 irqs->range[i]);
2573 }
2574}
2575EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2576
2577int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2578 struct pci_dev *dev, int num)
2579{
2580 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2581 struct pnv_phb *phb = hose->private_data;
2582 int i, hwirq, try;
2583
2584 memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2585
2586 /* 0 is reserved for the multiplexed PSL DSI interrupt */
2587 for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2588 try = num;
2589 while (try) {
2590 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2591 if (hwirq >= 0)
2592 break;
2593 try /= 2;
2594 }
2595 if (!try)
2596 goto fail;
2597
2598 irqs->offset[i] = phb->msi_base + hwirq;
2599 irqs->range[i] = try;
2600 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n",
2601 i, irqs->offset[i], irqs->range[i]);
2602 num -= try;
2603 }
2604 if (num)
2605 goto fail;
2606
2607 return 0;
2608fail:
2609 pnv_cxl_release_hwirq_ranges(irqs, dev);
2610 return -ENOSPC;
2611}
2612EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2613
2614int pnv_cxl_get_irq_count(struct pci_dev *dev)
2615{
2616 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2617 struct pnv_phb *phb = hose->private_data;
2618
2619 return phb->msi_bmp.irq_count;
2620}
2621EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2622
2623int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2624 unsigned int virq)
2625{
2626 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2627 struct pnv_phb *phb = hose->private_data;
2628 unsigned int xive_num = hwirq - phb->msi_base;
2629 struct pnv_ioda_pe *pe;
2630 int rc;
2631
2632 if (!(pe = pnv_ioda_get_pe(dev)))
2633 return -ENODEV;
2634
2635 /* Assign XIVE to PE */
2636 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2637 if (rc) {
2638 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2639 "hwirq 0x%x XIVE 0x%x PE\n",
2640 pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2641 return -EIO;
2642 }
2643 set_msi_irq_chip(phb, virq);
2644
2645 return 0;
2646}
2647EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2648#endif
2649
184cd4a3 2650static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
137436c9
GS		 2651 unsigned int hwirq, unsigned int virq,
2652 unsigned int is_64, struct msi_msg *msg)
184cd4a3
BH		 2653{
2654 struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2655 unsigned int xive_num = hwirq - phb->msi_base;
3a1a4661 2656 __be32 data;
184cd4a3
BH		 2657 int rc;
2658
2659 /* No PE assigned ? bail out ... no MSI for you ! */
2660 if (pe == NULL)
2661 return -ENXIO;
2662
2663 /* Check if we have an MVE */
2664 if (pe->mve_number < 0)
2665 return -ENXIO;
2666
b72c1f65 2667 /* Force 32-bit MSI on some broken devices */
36074381 2668 if (dev->no_64bit_msi)
b72c1f65
BH		 2669 is_64 = 0;
2670
184cd4a3
BH		 2671 /* Assign XIVE to PE */
2672 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2673 if (rc) {
2674 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2675 pci_name(dev), rc, xive_num);
2676 return -EIO;
2677 }
2678
2679 if (is_64) {
3a1a4661
BH		 2680 __be64 addr64;
2681
184cd4a3
BH		 2682 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2683 &addr64, &data);
2684 if (rc) {
2685 pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2686 pci_name(dev), rc);
2687 return -EIO;
2688 }
3a1a4661
BH		 2689 msg->address_hi = be64_to_cpu(addr64) >> 32;
2690 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
184cd4a3 2691 } else {
3a1a4661
BH		 2692 __be32 addr32;
2693
184cd4a3
BH		 2694 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2695 &addr32, &data);
2696 if (rc) {
2697 pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2698 pci_name(dev), rc);
2699 return -EIO;
2700 }
2701 msg->address_hi = 0;
3a1a4661 2702 msg->address_lo = be32_to_cpu(addr32);
184cd4a3 2703 }
3a1a4661 2704 msg->data = be32_to_cpu(data);
184cd4a3 2705
fd9a1c26 2706 set_msi_irq_chip(phb, virq);
137436c9 2707
184cd4a3
BH		 2708 pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2709 " address=%x_%08x data=%x PE# %d\n",
2710 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2711 msg->address_hi, msg->address_lo, data, pe->pe_number);
2712
2713 return 0;
2714}
2715
2716static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2717{
fb1b55d6 2718 unsigned int count;
184cd4a3
BH		 2719 const __be32 *prop = of_get_property(phb->hose->dn,
2720 "ibm,opal-msi-ranges", NULL);
2721 if (!prop) {
2722 /* BML Fallback */
2723 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2724 }
2725 if (!prop)
2726 return;
2727
2728 phb->msi_base = be32_to_cpup(prop);
fb1b55d6
GS		 2729 count = be32_to_cpup(prop + 1);
2730 if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
184cd4a3
BH		 2731 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2732 phb->hose->global_number);
2733 return;
2734 }
fb1b55d6 2735
184cd4a3
BH		 2736 phb->msi_setup = pnv_pci_ioda_msi_setup;
2737 phb->msi32_support = 1;
2738 pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
fb1b55d6 2739 count, phb->msi_base);
184cd4a3
BH		 2740}
2741#else
2742static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2743#endif /* CONFIG_PCI_MSI */
2744
6e628c7d
WY		 2745#ifdef CONFIG_PCI_IOV
2746static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2747{
2748 struct pci_controller *hose;
2749 struct pnv_phb *phb;
2750 struct resource *res;
2751 int i;
2752 resource_size_t size;
2753 struct pci_dn *pdn;
5b88ec22 2754 int mul, total_vfs;
6e628c7d
WY		 2755
2756 if (!pdev->is_physfn || pdev->is_added)
2757 return;
2758
2759 hose = pci_bus_to_host(pdev->bus);
2760 phb = hose->private_data;
2761
2762 pdn = pci_get_pdn(pdev);
2763 pdn->vfs_expanded = 0;
2764
5b88ec22
WY		 2765 total_vfs = pci_sriov_get_totalvfs(pdev);
2766 pdn->m64_per_iov = 1;
2767 mul = phb->ioda.total_pe;
2768
2769 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2770 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2771 if (!res->flags || res->parent)
2772 continue;
2773 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2774 dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2775 i, res);
2776 continue;
2777 }
2778
2779 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2780
2781 /* bigger than 64M */
2782 if (size > (1 << 26)) {
2783 dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2784 i, res);
2785 pdn->m64_per_iov = M64_PER_IOV;
2786 mul = roundup_pow_of_two(total_vfs);
2787 break;
2788 }
2789 }
2790
6e628c7d
WY		 2791 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2792 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2793 if (!res->flags || res->parent)
2794 continue;
2795 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2796 dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2797 i, res);
2798 continue;
2799 }
2800
2801 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2802 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
5b88ec22 2803 res->end = res->start + size * mul - 1;
6e628c7d
WY		 2804 dev_dbg(&pdev->dev, " %pR\n", res);
2805 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
5b88ec22 2806 i, res, mul);
6e628c7d 2807 }
5b88ec22 2808 pdn->vfs_expanded = mul;
6e628c7d
WY		 2809}
2810#endif /* CONFIG_PCI_IOV */
2811
11685bec
GS		 2812/*
2813 * This function is supposed to be called on basis of PE from top
2814 * to bottom style. So the the I/O or MMIO segment assigned to
2815 * parent PE could be overrided by its child PEs if necessary.
2816 */
cad5cef6
GKH		 2817static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2818 struct pnv_ioda_pe *pe)
11685bec
GS		 2819{
2820 struct pnv_phb *phb = hose->private_data;
2821 struct pci_bus_region region;
2822 struct resource *res;
2823 int i, index;
2824 int rc;
2825
2826 /*
2827 * NOTE: We only care PCI bus based PE for now. For PCI
2828 * device based PE, for example SRIOV sensitive VF should
2829 * be figured out later.
2830 */
2831 BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2832
2833 pci_bus_for_each_resource(pe->pbus, res, i) {
2834 if (!res || !res->flags ||
2835 res->start > res->end)
2836 continue;
2837
2838 if (res->flags & IORESOURCE_IO) {
2839 region.start = res->start - phb->ioda.io_pci_base;
2840 region.end = res->end - phb->ioda.io_pci_base;
2841 index = region.start / phb->ioda.io_segsize;
2842
2843 while (index < phb->ioda.total_pe &&
2844 region.start <= region.end) {
2845 phb->ioda.io_segmap[index] = pe->pe_number;
2846 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2847 pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2848 if (rc != OPAL_SUCCESS) {
2849 pr_err("%s: OPAL error %d when mapping IO "
2850 "segment #%d to PE#%d\n",
2851 __func__, rc, index, pe->pe_number);
2852 break;
2853 }
2854
2855 region.start += phb->ioda.io_segsize;
2856 index++;
2857 }
027fa02f
GS		 2858 } else if ((res->flags & IORESOURCE_MEM) &&
2859 !pnv_pci_is_mem_pref_64(res->flags)) {
11685bec 2860 region.start = res->start -
3fd47f06 2861 hose->mem_offset[0] -
11685bec
GS		 2862 phb->ioda.m32_pci_base;
2863 region.end = res->end -
3fd47f06 2864 hose->mem_offset[0] -
11685bec
GS		 2865 phb->ioda.m32_pci_base;
2866 index = region.start / phb->ioda.m32_segsize;
2867
2868 while (index < phb->ioda.total_pe &&
2869 region.start <= region.end) {
2870 phb->ioda.m32_segmap[index] = pe->pe_number;
2871 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2872 pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2873 if (rc != OPAL_SUCCESS) {
2874 pr_err("%s: OPAL error %d when mapping M32 "
2875 "segment#%d to PE#%d",
2876 __func__, rc, index, pe->pe_number);
2877 break;
2878 }
2879
2880 region.start += phb->ioda.m32_segsize;
2881 index++;
2882 }
2883 }
2884 }
2885}
2886
cad5cef6 2887static void pnv_pci_ioda_setup_seg(void)
11685bec
GS		 2888{
2889 struct pci_controller *tmp, *hose;
2890 struct pnv_phb *phb;
2891 struct pnv_ioda_pe *pe;
2892
2893 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2894 phb = hose->private_data;
2895 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2896 pnv_ioda_setup_pe_seg(hose, pe);
2897 }
2898 }
2899}
2900
cad5cef6 2901static void pnv_pci_ioda_setup_DMA(void)
13395c48
GS		 2902{
2903 struct pci_controller *hose, *tmp;
db1266c8 2904 struct pnv_phb *phb;
13395c48
GS		 2905
2906 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2907 pnv_ioda_setup_dma(hose->private_data);
db1266c8
GS		 2908
2909 /* Mark the PHB initialization done */
2910 phb = hose->private_data;
2911 phb->initialized = 1;
13395c48
GS		 2912 }
2913}
2914
37c367f2
GS		 2915static void pnv_pci_ioda_create_dbgfs(void)
2916{
2917#ifdef CONFIG_DEBUG_FS
2918 struct pci_controller *hose, *tmp;
2919 struct pnv_phb *phb;
2920 char name[16];
2921
2922 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2923 phb = hose->private_data;
2924
2925 sprintf(name, "PCI%04x", hose->global_number);
2926 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2927 if (!phb->dbgfs)
2928 pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2929 __func__, hose->global_number);
2930 }
2931#endif /* CONFIG_DEBUG_FS */
2932}
2933
cad5cef6 2934static void pnv_pci_ioda_fixup(void)
fb446ad0
GS		 2935{
2936 pnv_pci_ioda_setup_PEs();
11685bec 2937 pnv_pci_ioda_setup_seg();
13395c48 2938 pnv_pci_ioda_setup_DMA();
e9cc17d4 2939
37c367f2
GS		 2940 pnv_pci_ioda_create_dbgfs();
2941
e9cc17d4 2942#ifdef CONFIG_EEH
e9cc17d4 2943 eeh_init();
dadcd6d6 2944 eeh_addr_cache_build();
e9cc17d4 2945#endif
fb446ad0
GS		 2946}
2947
271fd03a
GS		 2948/*
2949 * Returns the alignment for I/O or memory windows for P2P
2950 * bridges. That actually depends on how PEs are segmented.
2951 * For now, we return I/O or M32 segment size for PE sensitive
2952 * P2P bridges. Otherwise, the default values (4KiB for I/O,
2953 * 1MiB for memory) will be returned.
2954 *
2955 * The current PCI bus might be put into one PE, which was
2956 * create against the parent PCI bridge. For that case, we
2957 * needn't enlarge the alignment so that we can save some
2958 * resources.
2959 */
2960static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2961 unsigned long type)
2962{
2963 struct pci_dev *bridge;
2964 struct pci_controller *hose = pci_bus_to_host(bus);
2965 struct pnv_phb *phb = hose->private_data;
2966 int num_pci_bridges = 0;
2967
2968 bridge = bus->self;
2969 while (bridge) {
2970 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2971 num_pci_bridges++;
2972 if (num_pci_bridges >= 2)
2973 return 1;
2974 }
2975
2976 bridge = bridge->bus->self;
2977 }
2978
262af557
GC		 2979 /* We fail back to M32 if M64 isn't supported */
2980 if (phb->ioda.m64_segsize &&
2981 pnv_pci_is_mem_pref_64(type))
2982 return phb->ioda.m64_segsize;
271fd03a
GS		 2983 if (type & IORESOURCE_MEM)
2984 return phb->ioda.m32_segsize;
2985
2986 return phb->ioda.io_segsize;
2987}
2988
5350ab3f
WY		 2989#ifdef CONFIG_PCI_IOV
2990static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2991 int resno)
2992{
2993 struct pci_dn *pdn = pci_get_pdn(pdev);
2994 resource_size_t align, iov_align;
2995
2996 iov_align = resource_size(&pdev->resource[resno]);
2997 if (iov_align)
2998 return iov_align;
2999
3000 align = pci_iov_resource_size(pdev, resno);
3001 if (pdn->vfs_expanded)
3002 return pdn->vfs_expanded * align;
3003
3004 return align;
3005}
3006#endif /* CONFIG_PCI_IOV */
3007
184cd4a3
BH		 3008/* Prevent enabling devices for which we couldn't properly
3009 * assign a PE
3010 */
c88c2a18 3011static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
184cd4a3 3012{
db1266c8
GS		 3013 struct pci_controller *hose = pci_bus_to_host(dev->bus);
3014 struct pnv_phb *phb = hose->private_data;
3015 struct pci_dn *pdn;
184cd4a3 3016
db1266c8
GS		 3017 /* The function is probably called while the PEs have
3018 * not be created yet. For example, resource reassignment
3019 * during PCI probe period. We just skip the check if
3020 * PEs isn't ready.
3021 */
3022 if (!phb->initialized)
c88c2a18 3023 return true;
db1266c8 3024
b72c1f65 3025 pdn = pci_get_pdn(dev);
184cd4a3 3026 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
c88c2a18 3027 return false;
db1266c8 3028
c88c2a18 3029 return true;
184cd4a3
BH		 3030}
3031
3032static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
3033 u32 devfn)
3034{
3035 return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
3036}
3037
7a8e6bbf 3038static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
73ed148a 3039{
7a8e6bbf
MN		 3040 struct pnv_phb *phb = hose->private_data;
3041
d1a85eee 3042 opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
73ed148a
BH		 3043 OPAL_ASSERT_RESET);
3044}
3045
92ae0353
DA		 3046static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
3047 .dma_dev_setup = pnv_pci_dma_dev_setup,
3048#ifdef CONFIG_PCI_MSI
3049 .setup_msi_irqs = pnv_setup_msi_irqs,
3050 .teardown_msi_irqs = pnv_teardown_msi_irqs,
3051#endif
3052 .enable_device_hook = pnv_pci_enable_device_hook,
3053 .window_alignment = pnv_pci_window_alignment,
3054 .reset_secondary_bus = pnv_pci_reset_secondary_bus,
763d2d8d 3055 .dma_set_mask = pnv_pci_ioda_dma_set_mask,
7a8e6bbf 3056 .shutdown = pnv_pci_ioda_shutdown,
92ae0353
DA		 3057};
3058
e51df2c1
AB		 3059static void __init pnv_pci_init_ioda_phb(struct device_node *np,
3060 u64 hub_id, int ioda_type)
184cd4a3
BH		 3061{
3062 struct pci_controller *hose;
184cd4a3 3063 struct pnv_phb *phb;
8184616f 3064 unsigned long size, m32map_off, pemap_off, iomap_off = 0;
c681b93c 3065 const __be64 *prop64;
3a1a4661 3066 const __be32 *prop32;
f1b7cc3e 3067 int len;
184cd4a3
BH		 3068 u64 phb_id;
3069 void *aux;
3070 long rc;
3071
58d714ec 3072 pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
184cd4a3
BH		 3073
3074 prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
3075 if (!prop64) {
3076 pr_err(" Missing \"ibm,opal-phbid\" property !\n");
3077 return;
3078 }
3079 phb_id = be64_to_cpup(prop64);
3080 pr_debug(" PHB-ID : 0x%016llx\n", phb_id);
3081
e39f223f 3082 phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
58d714ec
GS		 3083
3084 /* Allocate PCI controller */
58d714ec
GS		 3085 phb->hose = hose = pcibios_alloc_controller(np);
3086 if (!phb->hose) {
3087 pr_err(" Can't allocate PCI controller for %s\n",
184cd4a3 3088 np->full_name);
e39f223f 3089 memblock_free(__pa(phb), sizeof(struct pnv_phb));
184cd4a3
BH		 3090 return;
3091 }
3092
3093 spin_lock_init(&phb->lock);
f1b7cc3e
GS		 3094 prop32 = of_get_property(np, "bus-range", &len);
3095 if (prop32 && len == 8) {
3a1a4661
BH		 3096 hose->first_busno = be32_to_cpu(prop32[0]);
3097 hose->last_busno = be32_to_cpu(prop32[1]);
f1b7cc3e
GS		 3098 } else {
3099 pr_warn(" Broken <bus-range> on %s\n", np->full_name);
3100 hose->first_busno = 0;
3101 hose->last_busno = 0xff;
3102 }
184cd4a3 3103 hose->private_data = phb;
e9cc17d4 3104 phb->hub_id = hub_id;
184cd4a3 3105 phb->opal_id = phb_id;
aa0c033f 3106 phb->type = ioda_type;
781a868f 3107 mutex_init(&phb->ioda.pe_alloc_mutex);
184cd4a3 3108
cee72d5b
BH		 3109 /* Detect specific models for error handling */
3110 if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
3111 phb->model = PNV_PHB_MODEL_P7IOC;
f3d40c25 3112 else if (of_device_is_compatible(np, "ibm,power8-pciex"))
aa0c033f 3113 phb->model = PNV_PHB_MODEL_PHB3;
cee72d5b
BH		 3114 else
3115 phb->model = PNV_PHB_MODEL_UNKNOWN;
3116
aa0c033f 3117 /* Parse 32-bit and IO ranges (if any) */
2f1ec02e 3118 pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
184cd4a3 3119
aa0c033f 3120 /* Get registers */
184cd4a3
BH		 3121 phb->regs = of_iomap(np, 0);
3122 if (phb->regs == NULL)
3123 pr_err(" Failed to map registers !\n");
3124
184cd4a3 3125 /* Initialize more IODA stuff */
36954dc7 3126 phb->ioda.total_pe = 1;
aa0c033f 3127 prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
36954dc7 3128 if (prop32)
3a1a4661 3129 phb->ioda.total_pe = be32_to_cpup(prop32);
36954dc7
GS		 3130 prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
3131 if (prop32)
3132 phb->ioda.reserved_pe = be32_to_cpup(prop32);
262af557
GC		 3133
3134 /* Parse 64-bit MMIO range */
3135 pnv_ioda_parse_m64_window(phb);
3136
184cd4a3 3137 phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
aa0c033f 3138 /* FW Has already off top 64k of M32 space (MSI space) */
184cd4a3
BH		 3139 phb->ioda.m32_size += 0x10000;
3140
3141 phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
3fd47f06 3142 phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
184cd4a3
BH		 3143 phb->ioda.io_size = hose->pci_io_size;
3144 phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
3145 phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
3146
c35d2a8c 3147 /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
184cd4a3
BH		 3148 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
3149 m32map_off = size;
e47747f4 3150 size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
c35d2a8c
GS		 3151 if (phb->type == PNV_PHB_IODA1) {
3152 iomap_off = size;
3153 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
3154 }
184cd4a3
BH		 3155 pemap_off = size;
3156 size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
e39f223f 3157 aux = memblock_virt_alloc(size, 0);
184cd4a3
BH		 3158 phb->ioda.pe_alloc = aux;
3159 phb->ioda.m32_segmap = aux + m32map_off;
c35d2a8c
GS		 3160 if (phb->type == PNV_PHB_IODA1)
3161 phb->ioda.io_segmap = aux + iomap_off;
184cd4a3 3162 phb->ioda.pe_array = aux + pemap_off;
36954dc7 3163 set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
184cd4a3 3164
7ebdf956 3165 INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
184cd4a3 3166 INIT_LIST_HEAD(&phb->ioda.pe_list);
781a868f 3167 mutex_init(&phb->ioda.pe_list_mutex);
184cd4a3
BH		 3168
3169 /* Calculate how many 32-bit TCE segments we have */
3170 phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
3171
aa0c033f 3172#if 0 /* We should really do that ... */
184cd4a3
BH		 3173 rc = opal_pci_set_phb_mem_window(opal->phb_id,
3174 window_type,
3175 window_num,
3176 starting_real_address,
3177 starting_pci_address,
3178 segment_size);
3179#endif
3180
262af557
GC		 3181 pr_info(" %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
3182 phb->ioda.total_pe, phb->ioda.reserved_pe,
3183 phb->ioda.m32_size, phb->ioda.m32_segsize);
3184 if (phb->ioda.m64_size)
3185 pr_info(" M64: 0x%lx [segment=0x%lx]\n",
3186 phb->ioda.m64_size, phb->ioda.m64_segsize);
3187 if (phb->ioda.io_size)
3188 pr_info(" IO: 0x%x [segment=0x%x]\n",
3189 phb->ioda.io_size, phb->ioda.io_segsize);
3190
184cd4a3 3191
184cd4a3 3192 phb->hose->ops = &pnv_pci_ops;
49dec922
GS		 3193 phb->get_pe_state = pnv_ioda_get_pe_state;
3194 phb->freeze_pe = pnv_ioda_freeze_pe;
3195 phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
184cd4a3
BH		 3196
3197 /* Setup RID -> PE mapping function */
3198 phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
3199
3200 /* Setup TCEs */
3201 phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
fe7e85c6 3202 phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
184cd4a3
BH		 3203
3204 /* Setup MSI support */
3205 pnv_pci_init_ioda_msis(phb);
3206
c40a4210
GS		 3207 /*
3208 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3209 * to let the PCI core do resource assignment. It's supposed
3210 * that the PCI core will do correct I/O and MMIO alignment
3211 * for the P2P bridge bars so that each PCI bus (excluding
3212 * the child P2P bridges) can form individual PE.
184cd4a3 3213 */
fb446ad0 3214 ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
92ae0353 3215 hose->controller_ops = pnv_pci_ioda_controller_ops;
ad30cb99 3216
6e628c7d
WY		 3217#ifdef CONFIG_PCI_IOV
3218 ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
5350ab3f 3219 ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
ad30cb99
ME		 3220#endif
3221
c40a4210 3222 pci_add_flags(PCI_REASSIGN_ALL_RSRC);
184cd4a3
BH		 3223
3224 /* Reset IODA tables to a clean state */
d1a85eee 3225 rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
184cd4a3 3226 if (rc)
f11fe552 3227 pr_warning(" OPAL Error %ld performing IODA table reset !\n", rc);
361f2a2a
GS		 3228
3229 /* If we're running in kdump kerenl, the previous kerenl never
3230 * shutdown PCI devices correctly. We already got IODA table
3231 * cleaned out. So we have to issue PHB reset to stop all PCI
3232 * transactions from previous kerenl.
3233 */
3234 if (is_kdump_kernel()) {
3235 pr_info(" Issue PHB reset ...\n");
cadf364d
GS		 3236 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3237 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
361f2a2a 3238 }
262af557 3239
9e9e8935
GS		 3240 /* Remove M64 resource if we can't configure it successfully */
3241 if (!phb->init_m64 || phb->init_m64(phb))
262af557 3242 hose->mem_resources[1].flags = 0;
aa0c033f
GS		 3243}
3244
67975005 3245void __init pnv_pci_init_ioda2_phb(struct device_node *np)
aa0c033f 3246{
e9cc17d4 3247 pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
184cd4a3
BH		 3248}
3249
3250void __init pnv_pci_init_ioda_hub(struct device_node *np)
3251{
3252 struct device_node *phbn;
c681b93c 3253 const __be64 *prop64;
184cd4a3
BH		 3254 u64 hub_id;
3255
3256 pr_info("Probing IODA IO-Hub %s\n", np->full_name);
3257
3258 prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3259 if (!prop64) {
3260 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3261 return;
3262 }
3263 hub_id = be64_to_cpup(prop64);
3264 pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3265
3266 /* Count child PHBs */
3267 for_each_child_of_node(np, phbn) {
3268 /* Look for IODA1 PHBs */
3269 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
e9cc17d4 3270 pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
184cd4a3
BH		 3271 }
3272}
Linux 6.x block layer and io_uring tree(s)