arch/powerpc/platforms/powernv/pci-ioda.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #undef DEBUG
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/crash_dump.h>
  17 #include <linux/debugfs.h>
  18 #include <linux/delay.h>
  19 #include <linux/string.h>
  20 #include <linux/init.h>
  21 #include <linux/bootmem.h>
  22 #include <linux/irq.h>
  23 #include <linux/io.h>
  24 #include <linux/msi.h>
  25 #include <linux/memblock.h>
  26 #include <linux/iommu.h>
  27 #include <linux/rculist.h>
  28 #include <linux/sizes.h>
  29
  30 #include <asm/sections.h>
  31 #include <asm/io.h>
  32 #include <asm/prom.h>
  33 #include <asm/pci-bridge.h>
  34 #include <asm/machdep.h>
  35 #include <asm/msi_bitmap.h>
  36 #include <asm/ppc-pci.h>
  37 #include <asm/opal.h>
  38 #include <asm/iommu.h>
  39 #include <asm/tce.h>
  40 #include <asm/xics.h>
  41 #include <asm/debug.h>
  42 #include <asm/firmware.h>
  43 #include <asm/pnv-pci.h>
  44 #include <asm/mmzone.h>
  45
  46 #include <misc/cxl-base.h>
  47
  48 #include "powernv.h"
  49 #include "pci.h"
  50
  51 /* 256M DMA window, 4K TCE pages, 8 bytes TCE */
  52 #define TCE32_TABLE_SIZE        ((0x10000000 / 0x1000) * 8)
  53
  54 #define POWERNV_IOMMU_DEFAULT_LEVELS    1
  55 #define POWERNV_IOMMU_MAX_LEVELS        5
  56
  57 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
  58
  59 static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  60                             const char *fmt, ...)
  61 {
  62         struct va_format vaf;
  63         va_list args;
  64         char pfix[32];
  65
  66         va_start(args, fmt);
  67
  68         vaf.fmt = fmt;
  69         vaf.va = &args;
  70
  71         if (pe->flags & PNV_IODA_PE_DEV)
  72                 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  73         else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  74                 sprintf(pfix, "%04x:%02x     ",
  75                         pci_domain_nr(pe->pbus), pe->pbus->number);
  76 #ifdef CONFIG_PCI_IOV
  77         else if (pe->flags & PNV_IODA_PE_VF)
  78                 sprintf(pfix, "%04x:%02x:%2x.%d",
  79                         pci_domain_nr(pe->parent_dev->bus),
  80                         (pe->rid & 0xff00) >> 8,
  81                         PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  82 #endif /* CONFIG_PCI_IOV*/
  83
  84         printk("%spci %s: [PE# %.3d] %pV",
  85                level, pfix, pe->pe_number, &vaf);
  86
  87         va_end(args);
  88 }
  89
  90 #define pe_err(pe, fmt, ...)                                    \
  91         pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
  92 #define pe_warn(pe, fmt, ...)                                   \
  93         pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
  94 #define pe_info(pe, fmt, ...)                                   \
  95         pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
  96
  97 static bool pnv_iommu_bypass_disabled __read_mostly;
  98
  99 static int __init iommu_setup(char *str)
 100 {
 101         if (!str)
 102                 return -EINVAL;
 103
 104         while (*str) {
 105                 if (!strncmp(str, "nobypass", 8)) {
 106                         pnv_iommu_bypass_disabled = true;
 107                         pr_info("PowerNV: IOMMU bypass window disabled.\n");
 108                         break;
 109                 }
 110                 str += strcspn(str, ",");
 111                 if (*str == ',')
 112                         str++;
 113         }
 114
 115         return 0;
 116 }
 117 early_param("iommu", iommu_setup);
 118
 119 /*
 120  * stdcix is only supposed to be used in hypervisor real mode as per
 121  * the architecture spec
 122  */
 123 static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
 124 {
 125         __asm__ __volatile__("stdcix %0,0,%1"
 126                 : : "r" (val), "r" (paddr) : "memory");
 127 }
 128
 129 static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
 130 {
 131         return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
 132                 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
 133 }
 134
 135 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
 136 {
 137         if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
 138                 pr_warn("%s: Invalid PE %d on PHB#%x\n",
 139                         __func__, pe_no, phb->hose->global_number);
 140                 return;
 141         }
 142
 143         if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
 144                 pr_debug("%s: PE %d was reserved on PHB#%x\n",
 145                          __func__, pe_no, phb->hose->global_number);
 146
 147         phb->ioda.pe_array[pe_no].phb = phb;
 148         phb->ioda.pe_array[pe_no].pe_number = pe_no;
 149 }
 150
 151 static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
 152 {
 153         unsigned long pe;
 154
 155         do {
 156                 pe = find_next_zero_bit(phb->ioda.pe_alloc,
 157                                         phb->ioda.total_pe, 0);
 158                 if (pe >= phb->ioda.total_pe)
 159                         return IODA_INVALID_PE;
 160         } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
 161
 162         phb->ioda.pe_array[pe].phb = phb;
 163         phb->ioda.pe_array[pe].pe_number = pe;
 164         return pe;
 165 }
 166
 167 static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
 168 {
 169         WARN_ON(phb->ioda.pe_array[pe].pdev);
 170
 171         memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
 172         clear_bit(pe, phb->ioda.pe_alloc);
 173 }
 174
 175 /* The default M64 BAR is shared by all PEs */
 176 static int pnv_ioda2_init_m64(struct pnv_phb *phb)
 177 {
 178         const char *desc;
 179         struct resource *r;
 180         s64 rc;
 181
 182         /* Configure the default M64 BAR */
 183         rc = opal_pci_set_phb_mem_window(phb->opal_id,
 184                                          OPAL_M64_WINDOW_TYPE,
 185                                          phb->ioda.m64_bar_idx,
 186                                          phb->ioda.m64_base,
 187                                          0, /* unused */
 188                                          phb->ioda.m64_size);
 189         if (rc != OPAL_SUCCESS) {
 190                 desc = "configuring";
 191                 goto fail;
 192         }
 193
 194         /* Enable the default M64 BAR */
 195         rc = opal_pci_phb_mmio_enable(phb->opal_id,
 196                                       OPAL_M64_WINDOW_TYPE,
 197                                       phb->ioda.m64_bar_idx,
 198                                       OPAL_ENABLE_M64_SPLIT);
 199         if (rc != OPAL_SUCCESS) {
 200                 desc = "enabling";
 201                 goto fail;
 202         }
 203
 204         /* Mark the M64 BAR assigned */
 205         set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
 206
 207         /*
 208          * Strip off the segment used by the reserved PE, which is
 209          * expected to be 0 or last one of PE capabicity.
 210          */
 211         r = &phb->hose->mem_resources[1];
 212         if (phb->ioda.reserved_pe == 0)
 213                 r->start += phb->ioda.m64_segsize;
 214         else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
 215                 r->end -= phb->ioda.m64_segsize;
 216         else
 217                 pr_warn("  Cannot strip M64 segment for reserved PE#%d\n",
 218                         phb->ioda.reserved_pe);
 219
 220         return 0;
 221
 222 fail:
 223         pr_warn("  Failure %lld %s M64 BAR#%d\n",
 224                 rc, desc, phb->ioda.m64_bar_idx);
 225         opal_pci_phb_mmio_enable(phb->opal_id,
 226                                  OPAL_M64_WINDOW_TYPE,
 227                                  phb->ioda.m64_bar_idx,
 228                                  OPAL_DISABLE_M64);
 229         return -EIO;
 230 }
 231
 232 static void pnv_ioda2_reserve_dev_m64_pe(struct pci_dev *pdev,
 233                                          unsigned long *pe_bitmap)
 234 {
 235         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 236         struct pnv_phb *phb = hose->private_data;
 237         struct resource *r;
 238         resource_size_t base, sgsz, start, end;
 239         int segno, i;
 240
 241         base = phb->ioda.m64_base;
 242         sgsz = phb->ioda.m64_segsize;
 243         for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
 244                 r = &pdev->resource[i];
 245                 if (!r->parent || !pnv_pci_is_mem_pref_64(r->flags))
 246                         continue;
 247
 248                 start = _ALIGN_DOWN(r->start - base, sgsz);
 249                 end = _ALIGN_UP(r->end - base, sgsz);
 250                 for (segno = start / sgsz; segno < end / sgsz; segno++) {
 251                         if (pe_bitmap)
 252                                 set_bit(segno, pe_bitmap);
 253                         else
 254                                 pnv_ioda_reserve_pe(phb, segno);
 255                 }
 256         }
 257 }
 258
 259 static void pnv_ioda2_reserve_m64_pe(struct pci_bus *bus,
 260                                      unsigned long *pe_bitmap,
 261                                      bool all)
 262 {
 263         struct pci_dev *pdev;
 264
 265         list_for_each_entry(pdev, &bus->devices, bus_list) {
 266                 pnv_ioda2_reserve_dev_m64_pe(pdev, pe_bitmap);
 267
 268                 if (all && pdev->subordinate)
 269                         pnv_ioda2_reserve_m64_pe(pdev->subordinate,
 270                                                  pe_bitmap, all);
 271         }
 272 }
 273
 274 static int pnv_ioda2_pick_m64_pe(struct pci_bus *bus, bool all)
 275 {
 276         struct pci_controller *hose = pci_bus_to_host(bus);
 277         struct pnv_phb *phb = hose->private_data;
 278         struct pnv_ioda_pe *master_pe, *pe;
 279         unsigned long size, *pe_alloc;
 280         int i;
 281
 282         /* Root bus shouldn't use M64 */
 283         if (pci_is_root_bus(bus))
 284                 return IODA_INVALID_PE;
 285
 286         /* Allocate bitmap */
 287         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
 288         pe_alloc = kzalloc(size, GFP_KERNEL);
 289         if (!pe_alloc) {
 290                 pr_warn("%s: Out of memory !\n",
 291                         __func__);
 292                 return IODA_INVALID_PE;
 293         }
 294
 295         /* Figure out reserved PE numbers by the PE */
 296         pnv_ioda2_reserve_m64_pe(bus, pe_alloc, all);
 297
 298         /*
 299          * the current bus might not own M64 window and that's all
 300          * contributed by its child buses. For the case, we needn't
 301          * pick M64 dependent PE#.
 302          */
 303         if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
 304                 kfree(pe_alloc);
 305                 return IODA_INVALID_PE;
 306         }
 307
 308         /*
 309          * Figure out the master PE and put all slave PEs to master
 310          * PE's list to form compound PE.
 311          */
 312         master_pe = NULL;
 313         i = -1;
 314         while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
 315                 phb->ioda.total_pe) {
 316                 pe = &phb->ioda.pe_array[i];
 317
 318                 if (!master_pe) {
 319                         pe->flags |= PNV_IODA_PE_MASTER;
 320                         INIT_LIST_HEAD(&pe->slaves);
 321                         master_pe = pe;
 322                 } else {
 323                         pe->flags |= PNV_IODA_PE_SLAVE;
 324                         pe->master = master_pe;
 325                         list_add_tail(&pe->list, &master_pe->slaves);
 326                 }
 327         }
 328
 329         kfree(pe_alloc);
 330         return master_pe->pe_number;
 331 }
 332
 333 static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
 334 {
 335         struct pci_controller *hose = phb->hose;
 336         struct device_node *dn = hose->dn;
 337         struct resource *res;
 338         const u32 *r;
 339         u64 pci_addr;
 340
 341         /* FIXME: Support M64 for P7IOC */
 342         if (phb->type != PNV_PHB_IODA2) {
 343                 pr_info("  Not support M64 window\n");
 344                 return;
 345         }
 346
 347         if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
 348                 pr_info("  Firmware too old to support M64 window\n");
 349                 return;
 350         }
 351
 352         r = of_get_property(dn, "ibm,opal-m64-window", NULL);
 353         if (!r) {
 354                 pr_info("  No <ibm,opal-m64-window> on %s\n",
 355                         dn->full_name);
 356                 return;
 357         }
 358
 359         res = &hose->mem_resources[1];
 360         res->start = of_translate_address(dn, r + 2);
 361         res->end = res->start + of_read_number(r + 4, 2) - 1;
 362         res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
 363         pci_addr = of_read_number(r, 2);
 364         hose->mem_offset[1] = res->start - pci_addr;
 365
 366         phb->ioda.m64_size = resource_size(res);
 367         phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
 368         phb->ioda.m64_base = pci_addr;
 369
 370         pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
 371                         res->start, res->end, pci_addr);
 372
 373         /* Use last M64 BAR to cover M64 window */
 374         phb->ioda.m64_bar_idx = 15;
 375         phb->init_m64 = pnv_ioda2_init_m64;
 376         phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
 377         phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
 378 }
 379
 380 static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
 381 {
 382         struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
 383         struct pnv_ioda_pe *slave;
 384         s64 rc;
 385
 386         /* Fetch master PE */
 387         if (pe->flags & PNV_IODA_PE_SLAVE) {
 388                 pe = pe->master;
 389                 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
 390                         return;
 391
 392                 pe_no = pe->pe_number;
 393         }
 394
 395         /* Freeze master PE */
 396         rc = opal_pci_eeh_freeze_set(phb->opal_id,
 397                                      pe_no,
 398                                      OPAL_EEH_ACTION_SET_FREEZE_ALL);
 399         if (rc != OPAL_SUCCESS) {
 400                 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 401                         __func__, rc, phb->hose->global_number, pe_no);
 402                 return;
 403         }
 404
 405         /* Freeze slave PEs */
 406         if (!(pe->flags & PNV_IODA_PE_MASTER))
 407                 return;
 408
 409         list_for_each_entry(slave, &pe->slaves, list) {
 410                 rc = opal_pci_eeh_freeze_set(phb->opal_id,
 411                                              slave->pe_number,
 412                                              OPAL_EEH_ACTION_SET_FREEZE_ALL);
 413                 if (rc != OPAL_SUCCESS)
 414                         pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 415                                 __func__, rc, phb->hose->global_number,
 416                                 slave->pe_number);
 417         }
 418 }
 419
 420 static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
 421 {
 422         struct pnv_ioda_pe *pe, *slave;
 423         s64 rc;
 424
 425         /* Find master PE */
 426         pe = &phb->ioda.pe_array[pe_no];
 427         if (pe->flags & PNV_IODA_PE_SLAVE) {
 428                 pe = pe->master;
 429                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 430                 pe_no = pe->pe_number;
 431         }
 432
 433         /* Clear frozen state for master PE */
 434         rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
 435         if (rc != OPAL_SUCCESS) {
 436                 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 437                         __func__, rc, opt, phb->hose->global_number, pe_no);
 438                 return -EIO;
 439         }
 440
 441         if (!(pe->flags & PNV_IODA_PE_MASTER))
 442                 return 0;
 443
 444         /* Clear frozen state for slave PEs */
 445         list_for_each_entry(slave, &pe->slaves, list) {
 446                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 447                                              slave->pe_number,
 448                                              opt);
 449                 if (rc != OPAL_SUCCESS) {
 450                         pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 451                                 __func__, rc, opt, phb->hose->global_number,
 452                                 slave->pe_number);
 453                         return -EIO;
 454                 }
 455         }
 456
 457         return 0;
 458 }
 459
 460 static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
 461 {
 462         struct pnv_ioda_pe *slave, *pe;
 463         u8 fstate, state;
 464         __be16 pcierr;
 465         s64 rc;
 466
 467         /* Sanity check on PE number */
 468         if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
 469                 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
 470
 471         /*
 472          * Fetch the master PE and the PE instance might be
 473          * not initialized yet.
 474          */
 475         pe = &phb->ioda.pe_array[pe_no];
 476         if (pe->flags & PNV_IODA_PE_SLAVE) {
 477                 pe = pe->master;
 478                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 479                 pe_no = pe->pe_number;
 480         }
 481
 482         /* Check the master PE */
 483         rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
 484                                         &state, &pcierr, NULL);
 485         if (rc != OPAL_SUCCESS) {
 486                 pr_warn("%s: Failure %lld getting "
 487                         "PHB#%x-PE#%x state\n",
 488                         __func__, rc,
 489                         phb->hose->global_number, pe_no);
 490                 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 491         }
 492
 493         /* Check the slave PE */
 494         if (!(pe->flags & PNV_IODA_PE_MASTER))
 495                 return state;
 496
 497         list_for_each_entry(slave, &pe->slaves, list) {
 498                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 499                                                 slave->pe_number,
 500                                                 &fstate,
 501                                                 &pcierr,
 502                                                 NULL);
 503                 if (rc != OPAL_SUCCESS) {
 504                         pr_warn("%s: Failure %lld getting "
 505                                 "PHB#%x-PE#%x state\n",
 506                                 __func__, rc,
 507                                 phb->hose->global_number, slave->pe_number);
 508                         return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 509                 }
 510
 511                 /*
 512                  * Override the result based on the ascending
 513                  * priority.
 514                  */
 515                 if (fstate > state)
 516                         state = fstate;
 517         }
 518
 519         return state;
 520 }
 521
 522 /* Currently those 2 are only used when MSIs are enabled, this will change
 523  * but in the meantime, we need to protect them to avoid warnings
 524  */
 525 #ifdef CONFIG_PCI_MSI
 526 static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
 527 {
 528         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 529         struct pnv_phb *phb = hose->private_data;
 530         struct pci_dn *pdn = pci_get_pdn(dev);
 531
 532         if (!pdn)
 533                 return NULL;
 534         if (pdn->pe_number == IODA_INVALID_PE)
 535                 return NULL;
 536         return &phb->ioda.pe_array[pdn->pe_number];
 537 }
 538 #endif /* CONFIG_PCI_MSI */
 539
 540 static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
 541                                   struct pnv_ioda_pe *parent,
 542                                   struct pnv_ioda_pe *child,
 543                                   bool is_add)
 544 {
 545         const char *desc = is_add ? "adding" : "removing";
 546         uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
 547                               OPAL_REMOVE_PE_FROM_DOMAIN;
 548         struct pnv_ioda_pe *slave;
 549         long rc;
 550
 551         /* Parent PE affects child PE */
 552         rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 553                                 child->pe_number, op);
 554         if (rc != OPAL_SUCCESS) {
 555                 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
 556                         rc, desc);
 557                 return -ENXIO;
 558         }
 559
 560         if (!(child->flags & PNV_IODA_PE_MASTER))
 561                 return 0;
 562
 563         /* Compound case: parent PE affects slave PEs */
 564         list_for_each_entry(slave, &child->slaves, list) {
 565                 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 566                                         slave->pe_number, op);
 567                 if (rc != OPAL_SUCCESS) {
 568                         pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
 569                                 rc, desc);
 570                         return -ENXIO;
 571                 }
 572         }
 573
 574         return 0;
 575 }
 576
 577 static int pnv_ioda_set_peltv(struct pnv_phb *phb,
 578                               struct pnv_ioda_pe *pe,
 579                               bool is_add)
 580 {
 581         struct pnv_ioda_pe *slave;
 582         struct pci_dev *pdev = NULL;
 583         int ret;
 584
 585         /*
 586          * Clear PE frozen state. If it's master PE, we need
 587          * clear slave PE frozen state as well.
 588          */
 589         if (is_add) {
 590                 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 591                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 592                 if (pe->flags & PNV_IODA_PE_MASTER) {
 593                         list_for_each_entry(slave, &pe->slaves, list)
 594                                 opal_pci_eeh_freeze_clear(phb->opal_id,
 595                                                           slave->pe_number,
 596                                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 597                 }
 598         }
 599
 600         /*
 601          * Associate PE in PELT. We need add the PE into the
 602          * corresponding PELT-V as well. Otherwise, the error
 603          * originated from the PE might contribute to other
 604          * PEs.
 605          */
 606         ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
 607         if (ret)
 608                 return ret;
 609
 610         /* For compound PEs, any one affects all of them */
 611         if (pe->flags & PNV_IODA_PE_MASTER) {
 612                 list_for_each_entry(slave, &pe->slaves, list) {
 613                         ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
 614                         if (ret)
 615                                 return ret;
 616                 }
 617         }
 618
 619         if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
 620                 pdev = pe->pbus->self;
 621         else if (pe->flags & PNV_IODA_PE_DEV)
 622                 pdev = pe->pdev->bus->self;
 623 #ifdef CONFIG_PCI_IOV
 624         else if (pe->flags & PNV_IODA_PE_VF)
 625                 pdev = pe->parent_dev;
 626 #endif /* CONFIG_PCI_IOV */
 627         while (pdev) {
 628                 struct pci_dn *pdn = pci_get_pdn(pdev);
 629                 struct pnv_ioda_pe *parent;
 630
 631                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 632                         parent = &phb->ioda.pe_array[pdn->pe_number];
 633                         ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
 634                         if (ret)
 635                                 return ret;
 636                 }
 637
 638                 pdev = pdev->bus->self;
 639         }
 640
 641         return 0;
 642 }
 643
 644 #ifdef CONFIG_PCI_IOV
 645 static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 646 {
 647         struct pci_dev *parent;
 648         uint8_t bcomp, dcomp, fcomp;
 649         int64_t rc;
 650         long rid_end, rid;
 651
 652         /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
 653         if (pe->pbus) {
 654                 int count;
 655
 656                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 657                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 658                 parent = pe->pbus->self;
 659                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 660                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 661                 else
 662                         count = 1;
 663
 664                 switch(count) {
 665                 case  1: bcomp = OpalPciBusAll;         break;
 666                 case  2: bcomp = OpalPciBus7Bits;       break;
 667                 case  4: bcomp = OpalPciBus6Bits;       break;
 668                 case  8: bcomp = OpalPciBus5Bits;       break;
 669                 case 16: bcomp = OpalPciBus4Bits;       break;
 670                 case 32: bcomp = OpalPciBus3Bits;       break;
 671                 default:
 672                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 673                                 count);
 674                         /* Do an exact match only */
 675                         bcomp = OpalPciBusAll;
 676                 }
 677                 rid_end = pe->rid + (count << 8);
 678         } else {
 679                 if (pe->flags & PNV_IODA_PE_VF)
 680                         parent = pe->parent_dev;
 681                 else
 682                         parent = pe->pdev->bus->self;
 683                 bcomp = OpalPciBusAll;
 684                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 685                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 686                 rid_end = pe->rid + 1;
 687         }
 688
 689         /* Clear the reverse map */
 690         for (rid = pe->rid; rid < rid_end; rid++)
 691                 phb->ioda.pe_rmap[rid] = 0;
 692
 693         /* Release from all parents PELT-V */
 694         while (parent) {
 695                 struct pci_dn *pdn = pci_get_pdn(parent);
 696                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 697                         rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
 698                                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 699                         /* XXX What to do in case of error ? */
 700                 }
 701                 parent = parent->bus->self;
 702         }
 703
 704         opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 705                                   OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 706
 707         /* Disassociate PE in PELT */
 708         rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
 709                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 710         if (rc)
 711                 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
 712         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 713                              bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
 714         if (rc)
 715                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 716
 717         pe->pbus = NULL;
 718         pe->pdev = NULL;
 719         pe->parent_dev = NULL;
 720
 721         return 0;
 722 }
 723 #endif /* CONFIG_PCI_IOV */
 724
 725 static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 726 {
 727         struct pci_dev *parent;
 728         uint8_t bcomp, dcomp, fcomp;
 729         long rc, rid_end, rid;
 730
 731         /* Bus validation ? */
 732         if (pe->pbus) {
 733                 int count;
 734
 735                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 736                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 737                 parent = pe->pbus->self;
 738                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 739                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 740                 else
 741                         count = 1;
 742
 743                 switch(count) {
 744                 case  1: bcomp = OpalPciBusAll;         break;
 745                 case  2: bcomp = OpalPciBus7Bits;       break;
 746                 case  4: bcomp = OpalPciBus6Bits;       break;
 747                 case  8: bcomp = OpalPciBus5Bits;       break;
 748                 case 16: bcomp = OpalPciBus4Bits;       break;
 749                 case 32: bcomp = OpalPciBus3Bits;       break;
 750                 default:
 751                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 752                                 count);
 753                         /* Do an exact match only */
 754                         bcomp = OpalPciBusAll;
 755                 }
 756                 rid_end = pe->rid + (count << 8);
 757         } else {
 758 #ifdef CONFIG_PCI_IOV
 759                 if (pe->flags & PNV_IODA_PE_VF)
 760                         parent = pe->parent_dev;
 761                 else
 762 #endif /* CONFIG_PCI_IOV */
 763                         parent = pe->pdev->bus->self;
 764                 bcomp = OpalPciBusAll;
 765                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 766                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 767                 rid_end = pe->rid + 1;
 768         }
 769
 770         /*
 771          * Associate PE in PELT. We need add the PE into the
 772          * corresponding PELT-V as well. Otherwise, the error
 773          * originated from the PE might contribute to other
 774          * PEs.
 775          */
 776         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 777                              bcomp, dcomp, fcomp, OPAL_MAP_PE);
 778         if (rc) {
 779                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 780                 return -ENXIO;
 781         }
 782
 783         /* Configure PELTV */
 784         pnv_ioda_set_peltv(phb, pe, true);
 785
 786         /* Setup reverse map */
 787         for (rid = pe->rid; rid < rid_end; rid++)
 788                 phb->ioda.pe_rmap[rid] = pe->pe_number;
 789
 790         /* Setup one MVTs on IODA1 */
 791         if (phb->type != PNV_PHB_IODA1) {
 792                 pe->mve_number = 0;
 793                 goto out;
 794         }
 795
 796         pe->mve_number = pe->pe_number;
 797         rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
 798         if (rc != OPAL_SUCCESS) {
 799                 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
 800                        rc, pe->mve_number);
 801                 pe->mve_number = -1;
 802         } else {
 803                 rc = opal_pci_set_mve_enable(phb->opal_id,
 804                                              pe->mve_number, OPAL_ENABLE_MVE);
 805                 if (rc) {
 806                         pe_err(pe, "OPAL error %ld enabling MVE %d\n",
 807                                rc, pe->mve_number);
 808                         pe->mve_number = -1;
 809                 }
 810         }
 811
 812 out:
 813         return 0;
 814 }
 815
 816 static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
 817                                        struct pnv_ioda_pe *pe)
 818 {
 819         struct pnv_ioda_pe *lpe;
 820
 821         list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
 822                 if (lpe->dma_weight < pe->dma_weight) {
 823                         list_add_tail(&pe->dma_link, &lpe->dma_link);
 824                         return;
 825                 }
 826         }
 827         list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
 828 }
 829
 830 static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
 831 {
 832         /* This is quite simplistic. The "base" weight of a device
 833          * is 10. 0 means no DMA is to be accounted for it.
 834          */
 835
 836         /* If it's a bridge, no DMA */
 837         if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
 838                 return 0;
 839
 840         /* Reduce the weight of slow USB controllers */
 841         if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
 842             dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
 843             dev->class == PCI_CLASS_SERIAL_USB_EHCI)
 844                 return 3;
 845
 846         /* Increase the weight of RAID (includes Obsidian) */
 847         if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
 848                 return 15;
 849
 850         /* Default */
 851         return 10;
 852 }
 853
 854 #ifdef CONFIG_PCI_IOV
 855 static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
 856 {
 857         struct pci_dn *pdn = pci_get_pdn(dev);
 858         int i;
 859         struct resource *res, res2;
 860         resource_size_t size;
 861         u16 num_vfs;
 862
 863         if (!dev->is_physfn)
 864                 return -EINVAL;
 865
 866         /*
 867          * "offset" is in VFs.  The M64 windows are sized so that when they
 868          * are segmented, each segment is the same size as the IOV BAR.
 869          * Each segment is in a separate PE, and the high order bits of the
 870          * address are the PE number.  Therefore, each VF's BAR is in a
 871          * separate PE, and changing the IOV BAR start address changes the
 872          * range of PEs the VFs are in.
 873          */
 874         num_vfs = pdn->num_vfs;
 875         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 876                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 877                 if (!res->flags || !res->parent)
 878                         continue;
 879
 880                 if (!pnv_pci_is_mem_pref_64(res->flags))
 881                         continue;
 882
 883                 /*
 884                  * The actual IOV BAR range is determined by the start address
 885                  * and the actual size for num_vfs VFs BAR.  This check is to
 886                  * make sure that after shifting, the range will not overlap
 887                  * with another device.
 888                  */
 889                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 890                 res2.flags = res->flags;
 891                 res2.start = res->start + (size * offset);
 892                 res2.end = res2.start + (size * num_vfs) - 1;
 893
 894                 if (res2.end > res->end) {
 895                         dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
 896                                 i, &res2, res, num_vfs, offset);
 897                         return -EBUSY;
 898                 }
 899         }
 900
 901         /*
 902          * After doing so, there would be a "hole" in the /proc/iomem when
 903          * offset is a positive value. It looks like the device return some
 904          * mmio back to the system, which actually no one could use it.
 905          */
 906         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 907                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 908                 if (!res->flags || !res->parent)
 909                         continue;
 910
 911                 if (!pnv_pci_is_mem_pref_64(res->flags))
 912                         continue;
 913
 914                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 915                 res2 = *res;
 916                 res->start += size * offset;
 917
 918                 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n",
 919                          i, &res2, res, (offset > 0) ? "En" : "Dis",
 920                          num_vfs, offset);
 921                 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
 922         }
 923         return 0;
 924 }
 925 #endif /* CONFIG_PCI_IOV */
 926
 927 #if 0
 928 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 929 {
 930         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 931         struct pnv_phb *phb = hose->private_data;
 932         struct pci_dn *pdn = pci_get_pdn(dev);
 933         struct pnv_ioda_pe *pe;
 934         int pe_num;
 935
 936         if (!pdn) {
 937                 pr_err("%s: Device tree node not associated properly\n",
 938                            pci_name(dev));
 939                 return NULL;
 940         }
 941         if (pdn->pe_number != IODA_INVALID_PE)
 942                 return NULL;
 943
 944         /* PE#0 has been pre-set */
 945         if (dev->bus->number == 0)
 946                 pe_num = 0;
 947         else
 948                 pe_num = pnv_ioda_alloc_pe(phb);
 949         if (pe_num == IODA_INVALID_PE) {
 950                 pr_warning("%s: Not enough PE# available, disabling device\n",
 951                            pci_name(dev));
 952                 return NULL;
 953         }
 954
 955         /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
 956          * pointer in the PE data structure, both should be destroyed at the
 957          * same time. However, this needs to be looked at more closely again
 958          * once we actually start removing things (Hotplug, SR-IOV, ...)
 959          *
 960          * At some point we want to remove the PDN completely anyways
 961          */
 962         pe = &phb->ioda.pe_array[pe_num];
 963         pci_dev_get(dev);
 964         pdn->pcidev = dev;
 965         pdn->pe_number = pe_num;
 966         pe->pdev = dev;
 967         pe->pbus = NULL;
 968         pe->tce32_seg = -1;
 969         pe->mve_number = -1;
 970         pe->rid = dev->bus->number << 8 | pdn->devfn;
 971
 972         pe_info(pe, "Associated device to PE\n");
 973
 974         if (pnv_ioda_configure_pe(phb, pe)) {
 975                 /* XXX What do we do here ? */
 976                 if (pe_num)
 977                         pnv_ioda_free_pe(phb, pe_num);
 978                 pdn->pe_number = IODA_INVALID_PE;
 979                 pe->pdev = NULL;
 980                 pci_dev_put(dev);
 981                 return NULL;
 982         }
 983
 984         /* Assign a DMA weight to the device */
 985         pe->dma_weight = pnv_ioda_dma_weight(dev);
 986         if (pe->dma_weight != 0) {
 987                 phb->ioda.dma_weight += pe->dma_weight;
 988                 phb->ioda.dma_pe_count++;
 989         }
 990
 991         /* Link the PE */
 992         pnv_ioda_link_pe_by_weight(phb, pe);
 993
 994         return pe;
 995 }
 996 #endif /* Useful for SRIOV case */
 997
 998 static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
 999 {
1000         struct pci_dev *dev;
1001
1002         list_for_each_entry(dev, &bus->devices, bus_list) {
1003                 struct pci_dn *pdn = pci_get_pdn(dev);
1004
1005                 if (pdn == NULL) {
1006                         pr_warn("%s: No device node associated with device !\n",
1007                                 pci_name(dev));
1008                         continue;
1009                 }
1010                 pdn->pe_number = pe->pe_number;
1011                 pe->dma_weight += pnv_ioda_dma_weight(dev);
1012                 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1013                         pnv_ioda_setup_same_PE(dev->subordinate, pe);
1014         }
1015 }
1016
1017 /*
1018  * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1019  * single PCI bus. Another one that contains the primary PCI bus and its
1020  * subordinate PCI devices and buses. The second type of PE is normally
1021  * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1022  */
1023 static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
1024 {
1025         struct pci_controller *hose = pci_bus_to_host(bus);
1026         struct pnv_phb *phb = hose->private_data;
1027         struct pnv_ioda_pe *pe;
1028         int pe_num = IODA_INVALID_PE;
1029
1030         /* Check if PE is determined by M64 */
1031         if (phb->pick_m64_pe)
1032                 pe_num = phb->pick_m64_pe(bus, all);
1033
1034         /* The PE number isn't pinned by M64 */
1035         if (pe_num == IODA_INVALID_PE)
1036                 pe_num = pnv_ioda_alloc_pe(phb);
1037
1038         if (pe_num == IODA_INVALID_PE) {
1039                 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1040                         __func__, pci_domain_nr(bus), bus->number);
1041                 return;
1042         }
1043
1044         pe = &phb->ioda.pe_array[pe_num];
1045         pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
1046         pe->pbus = bus;
1047         pe->pdev = NULL;
1048         pe->tce32_seg = -1;
1049         pe->mve_number = -1;
1050         pe->rid = bus->busn_res.start << 8;
1051         pe->dma_weight = 0;
1052
1053         if (all)
1054                 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1055                         bus->busn_res.start, bus->busn_res.end, pe_num);
1056         else
1057                 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1058                         bus->busn_res.start, pe_num);
1059
1060         if (pnv_ioda_configure_pe(phb, pe)) {
1061                 /* XXX What do we do here ? */
1062                 if (pe_num)
1063                         pnv_ioda_free_pe(phb, pe_num);
1064                 pe->pbus = NULL;
1065                 return;
1066         }
1067
1068         /* Associate it with all child devices */
1069         pnv_ioda_setup_same_PE(bus, pe);
1070
1071         /* Put PE to the list */
1072         list_add_tail(&pe->list, &phb->ioda.pe_list);
1073
1074         /* Account for one DMA PE if at least one DMA capable device exist
1075          * below the bridge
1076          */
1077         if (pe->dma_weight != 0) {
1078                 phb->ioda.dma_weight += pe->dma_weight;
1079                 phb->ioda.dma_pe_count++;
1080         }
1081
1082         /* Link the PE */
1083         pnv_ioda_link_pe_by_weight(phb, pe);
1084 }
1085
1086 static void pnv_ioda_setup_PEs(struct pci_bus *bus)
1087 {
1088         struct pci_dev *dev;
1089
1090         pnv_ioda_setup_bus_PE(bus, false);
1091
1092         list_for_each_entry(dev, &bus->devices, bus_list) {
1093                 if (dev->subordinate) {
1094                         if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1095                                 pnv_ioda_setup_bus_PE(dev->subordinate, true);
1096                         else
1097                                 pnv_ioda_setup_PEs(dev->subordinate);
1098                 }
1099         }
1100 }
1101
1102 /*
1103  * Configure PEs so that the downstream PCI buses and devices
1104  * could have their associated PE#. Unfortunately, we didn't
1105  * figure out the way to identify the PLX bridge yet. So we
1106  * simply put the PCI bus and the subordinate behind the root
1107  * port to PE# here. The game rule here is expected to be changed
1108  * as soon as we can detected PLX bridge correctly.
1109  */
1110 static void pnv_pci_ioda_setup_PEs(void)
1111 {
1112         struct pci_controller *hose, *tmp;
1113         struct pnv_phb *phb;
1114
1115         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
1116                 phb = hose->private_data;
1117
1118                 /* M64 layout might affect PE allocation */
1119                 if (phb->reserve_m64_pe)
1120                         phb->reserve_m64_pe(hose->bus, NULL, true);
1121
1122                 pnv_ioda_setup_PEs(hose->bus);
1123         }
1124 }
1125
1126 #ifdef CONFIG_PCI_IOV
1127 static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1128 {
1129         struct pci_bus        *bus;
1130         struct pci_controller *hose;
1131         struct pnv_phb        *phb;
1132         struct pci_dn         *pdn;
1133         int                    i, j;
1134
1135         bus = pdev->bus;
1136         hose = pci_bus_to_host(bus);
1137         phb = hose->private_data;
1138         pdn = pci_get_pdn(pdev);
1139
1140         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1141                 for (j = 0; j < M64_PER_IOV; j++) {
1142                         if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1143                                 continue;
1144                         opal_pci_phb_mmio_enable(phb->opal_id,
1145                                 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1146                         clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1147                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1148                 }
1149
1150         return 0;
1151 }
1152
1153 static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
1154 {
1155         struct pci_bus        *bus;
1156         struct pci_controller *hose;
1157         struct pnv_phb        *phb;
1158         struct pci_dn         *pdn;
1159         unsigned int           win;
1160         struct resource       *res;
1161         int                    i, j;
1162         int64_t                rc;
1163         int                    total_vfs;
1164         resource_size_t        size, start;
1165         int                    pe_num;
1166         int                    vf_groups;
1167         int                    vf_per_group;
1168
1169         bus = pdev->bus;
1170         hose = pci_bus_to_host(bus);
1171         phb = hose->private_data;
1172         pdn = pci_get_pdn(pdev);
1173         total_vfs = pci_sriov_get_totalvfs(pdev);
1174
1175         /* Initialize the m64_wins to IODA_INVALID_M64 */
1176         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1177                 for (j = 0; j < M64_PER_IOV; j++)
1178                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1179
1180         if (pdn->m64_per_iov == M64_PER_IOV) {
1181                 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1182                 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1183                         roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1184         } else {
1185                 vf_groups = 1;
1186                 vf_per_group = 1;
1187         }
1188
1189         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1190                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1191                 if (!res->flags || !res->parent)
1192                         continue;
1193
1194                 if (!pnv_pci_is_mem_pref_64(res->flags))
1195                         continue;
1196
1197                 for (j = 0; j < vf_groups; j++) {
1198                         do {
1199                                 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1200                                                 phb->ioda.m64_bar_idx + 1, 0);
1201
1202                                 if (win >= phb->ioda.m64_bar_idx + 1)
1203                                         goto m64_failed;
1204                         } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1205
1206                         pdn->m64_wins[i][j] = win;
1207
1208                         if (pdn->m64_per_iov == M64_PER_IOV) {
1209                                 size = pci_iov_resource_size(pdev,
1210                                                         PCI_IOV_RESOURCES + i);
1211                                 size = size * vf_per_group;
1212                                 start = res->start + size * j;
1213                         } else {
1214                                 size = resource_size(res);
1215                                 start = res->start;
1216                         }
1217
1218                         /* Map the M64 here */
1219                         if (pdn->m64_per_iov == M64_PER_IOV) {
1220                                 pe_num = pdn->offset + j;
1221                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1222                                                 pe_num, OPAL_M64_WINDOW_TYPE,
1223                                                 pdn->m64_wins[i][j], 0);
1224                         }
1225
1226                         rc = opal_pci_set_phb_mem_window(phb->opal_id,
1227                                                  OPAL_M64_WINDOW_TYPE,
1228                                                  pdn->m64_wins[i][j],
1229                                                  start,
1230                                                  0, /* unused */
1231                                                  size);
1232
1233
1234                         if (rc != OPAL_SUCCESS) {
1235                                 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1236                                         win, rc);
1237                                 goto m64_failed;
1238                         }
1239
1240                         if (pdn->m64_per_iov == M64_PER_IOV)
1241                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1242                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1243                         else
1244                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1245                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
1246
1247                         if (rc != OPAL_SUCCESS) {
1248                                 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1249                                         win, rc);
1250                                 goto m64_failed;
1251                         }
1252                 }
1253         }
1254         return 0;
1255
1256 m64_failed:
1257         pnv_pci_vf_release_m64(pdev);
1258         return -EBUSY;
1259 }
1260
1261 static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1262                 int num);
1263 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
1264
1265 static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1266 {
1267         struct iommu_table    *tbl;
1268         int64_t               rc;
1269
1270         tbl = pe->table_group.tables[0];
1271         rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
1272         if (rc)
1273                 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1274
1275         pnv_pci_ioda2_set_bypass(pe, false);
1276         if (pe->table_group.group) {
1277                 iommu_group_put(pe->table_group.group);
1278                 BUG_ON(pe->table_group.group);
1279         }
1280         pnv_pci_ioda2_table_free_pages(tbl);
1281         iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
1282 }
1283
1284 static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1285 {
1286         struct pci_bus        *bus;
1287         struct pci_controller *hose;
1288         struct pnv_phb        *phb;
1289         struct pnv_ioda_pe    *pe, *pe_n;
1290         struct pci_dn         *pdn;
1291         u16                    vf_index;
1292         int64_t                rc;
1293
1294         bus = pdev->bus;
1295         hose = pci_bus_to_host(bus);
1296         phb = hose->private_data;
1297         pdn = pci_get_pdn(pdev);
1298
1299         if (!pdev->is_physfn)
1300                 return;
1301
1302         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1303                 int   vf_group;
1304                 int   vf_per_group;
1305                 int   vf_index1;
1306
1307                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1308
1309                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1310                         for (vf_index = vf_group * vf_per_group;
1311                                 vf_index < (vf_group + 1) * vf_per_group &&
1312                                 vf_index < num_vfs;
1313                                 vf_index++)
1314                                 for (vf_index1 = vf_group * vf_per_group;
1315                                         vf_index1 < (vf_group + 1) * vf_per_group &&
1316                                         vf_index1 < num_vfs;
1317                                         vf_index1++){
1318
1319                                         rc = opal_pci_set_peltv(phb->opal_id,
1320                                                 pdn->offset + vf_index,
1321                                                 pdn->offset + vf_index1,
1322                                                 OPAL_REMOVE_PE_FROM_DOMAIN);
1323
1324                                         if (rc)
1325                                             dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1326                                                 __func__,
1327                                                 pdn->offset + vf_index1, rc);
1328                                 }
1329         }
1330
1331         list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1332                 if (pe->parent_dev != pdev)
1333                         continue;
1334
1335                 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1336
1337                 /* Remove from list */
1338                 mutex_lock(&phb->ioda.pe_list_mutex);
1339                 list_del(&pe->list);
1340                 mutex_unlock(&phb->ioda.pe_list_mutex);
1341
1342                 pnv_ioda_deconfigure_pe(phb, pe);
1343
1344                 pnv_ioda_free_pe(phb, pe->pe_number);
1345         }
1346 }
1347
1348 void pnv_pci_sriov_disable(struct pci_dev *pdev)
1349 {
1350         struct pci_bus        *bus;
1351         struct pci_controller *hose;
1352         struct pnv_phb        *phb;
1353         struct pci_dn         *pdn;
1354         struct pci_sriov      *iov;
1355         u16 num_vfs;
1356
1357         bus = pdev->bus;
1358         hose = pci_bus_to_host(bus);
1359         phb = hose->private_data;
1360         pdn = pci_get_pdn(pdev);
1361         iov = pdev->sriov;
1362         num_vfs = pdn->num_vfs;
1363
1364         /* Release VF PEs */
1365         pnv_ioda_release_vf_PE(pdev, num_vfs);
1366
1367         if (phb->type == PNV_PHB_IODA2) {
1368                 if (pdn->m64_per_iov == 1)
1369                         pnv_pci_vf_resource_shift(pdev, -pdn->offset);
1370
1371                 /* Release M64 windows */
1372                 pnv_pci_vf_release_m64(pdev);
1373
1374                 /* Release PE numbers */
1375                 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1376                 pdn->offset = 0;
1377         }
1378 }
1379
1380 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1381                                        struct pnv_ioda_pe *pe);
1382 static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1383 {
1384         struct pci_bus        *bus;
1385         struct pci_controller *hose;
1386         struct pnv_phb        *phb;
1387         struct pnv_ioda_pe    *pe;
1388         int                    pe_num;
1389         u16                    vf_index;
1390         struct pci_dn         *pdn;
1391         int64_t                rc;
1392
1393         bus = pdev->bus;
1394         hose = pci_bus_to_host(bus);
1395         phb = hose->private_data;
1396         pdn = pci_get_pdn(pdev);
1397
1398         if (!pdev->is_physfn)
1399                 return;
1400
1401         /* Reserve PE for each VF */
1402         for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1403                 pe_num = pdn->offset + vf_index;
1404
1405                 pe = &phb->ioda.pe_array[pe_num];
1406                 pe->pe_number = pe_num;
1407                 pe->phb = phb;
1408                 pe->flags = PNV_IODA_PE_VF;
1409                 pe->pbus = NULL;
1410                 pe->parent_dev = pdev;
1411                 pe->tce32_seg = -1;
1412                 pe->mve_number = -1;
1413                 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1414                            pci_iov_virtfn_devfn(pdev, vf_index);
1415
1416                 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1417                         hose->global_number, pdev->bus->number,
1418                         PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1419                         PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1420
1421                 if (pnv_ioda_configure_pe(phb, pe)) {
1422                         /* XXX What do we do here ? */
1423                         if (pe_num)
1424                                 pnv_ioda_free_pe(phb, pe_num);
1425                         pe->pdev = NULL;
1426                         continue;
1427                 }
1428
1429                 /* Put PE to the list */
1430                 mutex_lock(&phb->ioda.pe_list_mutex);
1431                 list_add_tail(&pe->list, &phb->ioda.pe_list);
1432                 mutex_unlock(&phb->ioda.pe_list_mutex);
1433
1434                 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1435         }
1436
1437         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1438                 int   vf_group;
1439                 int   vf_per_group;
1440                 int   vf_index1;
1441
1442                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1443
1444                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1445                         for (vf_index = vf_group * vf_per_group;
1446                              vf_index < (vf_group + 1) * vf_per_group &&
1447                              vf_index < num_vfs;
1448                              vf_index++) {
1449                                 for (vf_index1 = vf_group * vf_per_group;
1450                                      vf_index1 < (vf_group + 1) * vf_per_group &&
1451                                      vf_index1 < num_vfs;
1452                                      vf_index1++) {
1453
1454                                         rc = opal_pci_set_peltv(phb->opal_id,
1455                                                 pdn->offset + vf_index,
1456                                                 pdn->offset + vf_index1,
1457                                                 OPAL_ADD_PE_TO_DOMAIN);
1458
1459                                         if (rc)
1460                                             dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1461                                                 __func__,
1462                                                 pdn->offset + vf_index1, rc);
1463                                 }
1464                         }
1465                 }
1466         }
1467 }
1468
1469 int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1470 {
1471         struct pci_bus        *bus;
1472         struct pci_controller *hose;
1473         struct pnv_phb        *phb;
1474         struct pci_dn         *pdn;
1475         int                    ret;
1476
1477         bus = pdev->bus;
1478         hose = pci_bus_to_host(bus);
1479         phb = hose->private_data;
1480         pdn = pci_get_pdn(pdev);
1481
1482         if (phb->type == PNV_PHB_IODA2) {
1483                 /* Calculate available PE for required VFs */
1484                 mutex_lock(&phb->ioda.pe_alloc_mutex);
1485                 pdn->offset = bitmap_find_next_zero_area(
1486                         phb->ioda.pe_alloc, phb->ioda.total_pe,
1487                         0, num_vfs, 0);
1488                 if (pdn->offset >= phb->ioda.total_pe) {
1489                         mutex_unlock(&phb->ioda.pe_alloc_mutex);
1490                         dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1491                         pdn->offset = 0;
1492                         return -EBUSY;
1493                 }
1494                 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1495                 pdn->num_vfs = num_vfs;
1496                 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1497
1498                 /* Assign M64 window accordingly */
1499                 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
1500                 if (ret) {
1501                         dev_info(&pdev->dev, "Not enough M64 window resources\n");
1502                         goto m64_failed;
1503                 }
1504
1505                 /*
1506                  * When using one M64 BAR to map one IOV BAR, we need to shift
1507                  * the IOV BAR according to the PE# allocated to the VFs.
1508                  * Otherwise, the PE# for the VF will conflict with others.
1509                  */
1510                 if (pdn->m64_per_iov == 1) {
1511                         ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1512                         if (ret)
1513                                 goto m64_failed;
1514                 }
1515         }
1516
1517         /* Setup VF PEs */
1518         pnv_ioda_setup_vf_PE(pdev, num_vfs);
1519
1520         return 0;
1521
1522 m64_failed:
1523         bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1524         pdn->offset = 0;
1525
1526         return ret;
1527 }
1528
1529 int pcibios_sriov_disable(struct pci_dev *pdev)
1530 {
1531         pnv_pci_sriov_disable(pdev);
1532
1533         /* Release PCI data */
1534         remove_dev_pci_data(pdev);
1535         return 0;
1536 }
1537
1538 int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1539 {
1540         /* Allocate PCI data */
1541         add_dev_pci_data(pdev);
1542
1543         pnv_pci_sriov_enable(pdev, num_vfs);
1544         return 0;
1545 }
1546 #endif /* CONFIG_PCI_IOV */
1547
1548 static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
1549 {
1550         struct pci_dn *pdn = pci_get_pdn(pdev);
1551         struct pnv_ioda_pe *pe;
1552
1553         /*
1554          * The function can be called while the PE#
1555          * hasn't been assigned. Do nothing for the
1556          * case.
1557          */
1558         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1559                 return;
1560
1561         pe = &phb->ioda.pe_array[pdn->pe_number];
1562         WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1563         set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1564         /*
1565          * Note: iommu_add_device() will fail here as
1566          * for physical PE: the device is already added by now;
1567          * for virtual PE: sysfs entries are not ready yet and
1568          * tce_iommu_bus_notifier will add the device to a group later.
1569          */
1570 }
1571
1572 static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
1573 {
1574         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1575         struct pnv_phb *phb = hose->private_data;
1576         struct pci_dn *pdn = pci_get_pdn(pdev);
1577         struct pnv_ioda_pe *pe;
1578         uint64_t top;
1579         bool bypass = false;
1580
1581         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1582                 return -ENODEV;;
1583
1584         pe = &phb->ioda.pe_array[pdn->pe_number];
1585         if (pe->tce_bypass_enabled) {
1586                 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1587                 bypass = (dma_mask >= top);
1588         }
1589
1590         if (bypass) {
1591                 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1592                 set_dma_ops(&pdev->dev, &dma_direct_ops);
1593         } else {
1594                 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1595                 set_dma_ops(&pdev->dev, &dma_iommu_ops);
1596         }
1597         *pdev->dev.dma_mask = dma_mask;
1598         return 0;
1599 }
1600
1601 static u64 pnv_pci_ioda_dma_get_required_mask(struct pci_dev *pdev)
1602 {
1603         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1604         struct pnv_phb *phb = hose->private_data;
1605         struct pci_dn *pdn = pci_get_pdn(pdev);
1606         struct pnv_ioda_pe *pe;
1607         u64 end, mask;
1608
1609         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1610                 return 0;
1611
1612         pe = &phb->ioda.pe_array[pdn->pe_number];
1613         if (!pe->tce_bypass_enabled)
1614                 return __dma_get_required_mask(&pdev->dev);
1615
1616
1617         end = pe->tce_bypass_base + memblock_end_of_DRAM();
1618         mask = 1ULL << (fls64(end) - 1);
1619         mask += mask - 1;
1620
1621         return mask;
1622 }
1623
1624 static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
1625                                    struct pci_bus *bus)
1626 {
1627         struct pci_dev *dev;
1628
1629         list_for_each_entry(dev, &bus->devices, bus_list) {
1630                 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
1631                 set_dma_offset(&dev->dev, pe->tce_bypass_base);
1632                 iommu_add_device(&dev->dev);
1633
1634                 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1635                         pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1636         }
1637 }
1638
1639 static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1640                 unsigned long index, unsigned long npages, bool rm)
1641 {
1642         struct iommu_table_group_link *tgl = list_first_entry_or_null(
1643                         &tbl->it_group_list, struct iommu_table_group_link,
1644                         next);
1645         struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1646                         struct pnv_ioda_pe, table_group);
1647         __be64 __iomem *invalidate = rm ?
1648                 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1649                 pe->phb->ioda.tce_inval_reg;
1650         unsigned long start, end, inc;
1651         const unsigned shift = tbl->it_page_shift;
1652
1653         start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1654         end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1655                         npages - 1);
1656
1657         /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1658         if (tbl->it_busno) {
1659                 start <<= shift;
1660                 end <<= shift;
1661                 inc = 128ull << shift;
1662                 start |= tbl->it_busno;
1663                 end |= tbl->it_busno;
1664         } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1665                 /* p7ioc-style invalidation, 2 TCEs per write */
1666                 start |= (1ull << 63);
1667                 end |= (1ull << 63);
1668                 inc = 16;
1669         } else {
1670                 /* Default (older HW) */
1671                 inc = 128;
1672         }
1673
1674         end |= inc - 1; /* round up end to be different than start */
1675
1676         mb(); /* Ensure above stores are visible */
1677         while (start <= end) {
1678                 if (rm)
1679                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1680                 else
1681                         __raw_writeq(cpu_to_be64(start), invalidate);
1682                 start += inc;
1683         }
1684
1685         /*
1686          * The iommu layer will do another mb() for us on build()
1687          * and we don't care on free()
1688          */
1689 }
1690
1691 static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1692                 long npages, unsigned long uaddr,
1693                 enum dma_data_direction direction,
1694                 struct dma_attrs *attrs)
1695 {
1696         int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1697                         attrs);
1698
1699         if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1700                 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1701
1702         return ret;
1703 }
1704
1705 #ifdef CONFIG_IOMMU_API
1706 static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
1707                 unsigned long *hpa, enum dma_data_direction *direction)
1708 {
1709         long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1710
1711         if (!ret && (tbl->it_type &
1712                         (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1713                 pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
1714
1715         return ret;
1716 }
1717 #endif
1718
1719 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1720                 long npages)
1721 {
1722         pnv_tce_free(tbl, index, npages);
1723
1724         if (tbl->it_type & TCE_PCI_SWINV_FREE)
1725                 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1726 }
1727
1728 static struct iommu_table_ops pnv_ioda1_iommu_ops = {
1729         .set = pnv_ioda1_tce_build,
1730 #ifdef CONFIG_IOMMU_API
1731         .exchange = pnv_ioda1_tce_xchg,
1732 #endif
1733         .clear = pnv_ioda1_tce_free,
1734         .get = pnv_tce_get,
1735 };
1736
1737 static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1738 {
1739         /* 01xb - invalidate TCEs that match the specified PE# */
1740         unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1741         struct pnv_phb *phb = pe->phb;
1742
1743         if (!phb->ioda.tce_inval_reg)
1744                 return;
1745
1746         mb(); /* Ensure above stores are visible */
1747         __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1748 }
1749
1750 static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
1751                 __be64 __iomem *invalidate, unsigned shift,
1752                 unsigned long index, unsigned long npages)
1753 {
1754         unsigned long start, end, inc;
1755
1756         /* We'll invalidate DMA address in PE scope */
1757         start = 0x2ull << 60;
1758         start |= (pe_number & 0xFF);
1759         end = start;
1760
1761         /* Figure out the start, end and step */
1762         start |= (index << shift);
1763         end |= ((index + npages - 1) << shift);
1764         inc = (0x1ull << shift);
1765         mb();
1766
1767         while (start <= end) {
1768                 if (rm)
1769                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1770                 else
1771                         __raw_writeq(cpu_to_be64(start), invalidate);
1772                 start += inc;
1773         }
1774 }
1775
1776 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1777                 unsigned long index, unsigned long npages, bool rm)
1778 {
1779         struct iommu_table_group_link *tgl;
1780
1781         list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1782                 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1783                                 struct pnv_ioda_pe, table_group);
1784                 __be64 __iomem *invalidate = rm ?
1785                         (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1786                         pe->phb->ioda.tce_inval_reg;
1787
1788                 pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
1789                         invalidate, tbl->it_page_shift,
1790                         index, npages);
1791         }
1792 }
1793
1794 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1795                 long npages, unsigned long uaddr,
1796                 enum dma_data_direction direction,
1797                 struct dma_attrs *attrs)
1798 {
1799         int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1800                         attrs);
1801
1802         if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1803                 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1804
1805         return ret;
1806 }
1807
1808 #ifdef CONFIG_IOMMU_API
1809 static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
1810                 unsigned long *hpa, enum dma_data_direction *direction)
1811 {
1812         long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1813
1814         if (!ret && (tbl->it_type &
1815                         (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1816                 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
1817
1818         return ret;
1819 }
1820 #endif
1821
1822 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1823                 long npages)
1824 {
1825         pnv_tce_free(tbl, index, npages);
1826
1827         if (tbl->it_type & TCE_PCI_SWINV_FREE)
1828                 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1829 }
1830
1831 static void pnv_ioda2_table_free(struct iommu_table *tbl)
1832 {
1833         pnv_pci_ioda2_table_free_pages(tbl);
1834         iommu_free_table(tbl, "pnv");
1835 }
1836
1837 static struct iommu_table_ops pnv_ioda2_iommu_ops = {
1838         .set = pnv_ioda2_tce_build,
1839 #ifdef CONFIG_IOMMU_API
1840         .exchange = pnv_ioda2_tce_xchg,
1841 #endif
1842         .clear = pnv_ioda2_tce_free,
1843         .get = pnv_tce_get,
1844         .free = pnv_ioda2_table_free,
1845 };
1846
1847 static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1848                                       struct pnv_ioda_pe *pe, unsigned int base,
1849                                       unsigned int segs)
1850 {
1851
1852         struct page *tce_mem = NULL;
1853         struct iommu_table *tbl;
1854         unsigned int i;
1855         int64_t rc;
1856         void *addr;
1857
1858         /* XXX FIXME: Handle 64-bit only DMA devices */
1859         /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1860         /* XXX FIXME: Allocate multi-level tables on PHB3 */
1861
1862         /* We shouldn't already have a 32-bit DMA associated */
1863         if (WARN_ON(pe->tce32_seg >= 0))
1864                 return;
1865
1866         tbl = pnv_pci_table_alloc(phb->hose->node);
1867         iommu_register_group(&pe->table_group, phb->hose->global_number,
1868                         pe->pe_number);
1869         pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
1870
1871         /* Grab a 32-bit TCE table */
1872         pe->tce32_seg = base;
1873         pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1874                 (base << 28), ((base + segs) << 28) - 1);
1875
1876         /* XXX Currently, we allocate one big contiguous table for the
1877          * TCEs. We only really need one chunk per 256M of TCE space
1878          * (ie per segment) but that's an optimization for later, it
1879          * requires some added smarts with our get/put_tce implementation
1880          */
1881         tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1882                                    get_order(TCE32_TABLE_SIZE * segs));
1883         if (!tce_mem) {
1884                 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1885                 goto fail;
1886         }
1887         addr = page_address(tce_mem);
1888         memset(addr, 0, TCE32_TABLE_SIZE * segs);
1889
1890         /* Configure HW */
1891         for (i = 0; i < segs; i++) {
1892                 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1893                                               pe->pe_number,
1894                                               base + i, 1,
1895                                               __pa(addr) + TCE32_TABLE_SIZE * i,
1896                                               TCE32_TABLE_SIZE, 0x1000);
1897                 if (rc) {
1898                         pe_err(pe, " Failed to configure 32-bit TCE table,"
1899                                " err %ld\n", rc);
1900                         goto fail;
1901                 }
1902         }
1903
1904         /* Setup linux iommu table */
1905         pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
1906                                   base << 28, IOMMU_PAGE_SHIFT_4K);
1907
1908         /* OPAL variant of P7IOC SW invalidated TCEs */
1909         if (phb->ioda.tce_inval_reg)
1910                 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1911                                  TCE_PCI_SWINV_FREE   |
1912                                  TCE_PCI_SWINV_PAIR);
1913
1914         tbl->it_ops = &pnv_ioda1_iommu_ops;
1915         pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
1916         pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
1917         iommu_init_table(tbl, phb->hose->node);
1918
1919         if (pe->flags & PNV_IODA_PE_DEV) {
1920                 /*
1921                  * Setting table base here only for carrying iommu_group
1922                  * further down to let iommu_add_device() do the job.
1923                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1924                  */
1925                 set_iommu_table_base(&pe->pdev->dev, tbl);
1926                 iommu_add_device(&pe->pdev->dev);
1927         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
1928                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
1929
1930         return;
1931  fail:
1932         /* XXX Failure: Try to fallback to 64-bit only ? */
1933         if (pe->tce32_seg >= 0)
1934                 pe->tce32_seg = -1;
1935         if (tce_mem)
1936                 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
1937         if (tbl) {
1938                 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1939                 iommu_free_table(tbl, "pnv");
1940         }
1941 }
1942
1943 static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1944                 int num, struct iommu_table *tbl)
1945 {
1946         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1947                         table_group);
1948         struct pnv_phb *phb = pe->phb;
1949         int64_t rc;
1950         const unsigned long size = tbl->it_indirect_levels ?
1951                         tbl->it_level_size : tbl->it_size;
1952         const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1953         const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1954
1955         pe_info(pe, "Setting up window#%d %llx..%llx pg=%x\n", num,
1956                         start_addr, start_addr + win_size - 1,
1957                         IOMMU_PAGE_SIZE(tbl));
1958
1959         /*
1960          * Map TCE table through TVT. The TVE index is the PE number
1961          * shifted by 1 bit for 32-bits DMA space.
1962          */
1963         rc = opal_pci_map_pe_dma_window(phb->opal_id,
1964                         pe->pe_number,
1965                         (pe->pe_number << 1) + num,
1966                         tbl->it_indirect_levels + 1,
1967                         __pa(tbl->it_base),
1968                         size << 3,
1969                         IOMMU_PAGE_SIZE(tbl));
1970         if (rc) {
1971                 pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
1972                 return rc;
1973         }
1974
1975         pnv_pci_link_table_and_group(phb->hose->node, num,
1976                         tbl, &pe->table_group);
1977         pnv_pci_ioda2_tce_invalidate_entire(pe);
1978
1979         return 0;
1980 }
1981
1982 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
1983 {
1984         uint16_t window_id = (pe->pe_number << 1 ) + 1;
1985         int64_t rc;
1986
1987         pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
1988         if (enable) {
1989                 phys_addr_t top = memblock_end_of_DRAM();
1990
1991                 top = roundup_pow_of_two(top);
1992                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1993                                                      pe->pe_number,
1994                                                      window_id,
1995                                                      pe->tce_bypass_base,
1996                                                      top);
1997         } else {
1998                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1999                                                      pe->pe_number,
2000                                                      window_id,
2001                                                      pe->tce_bypass_base,
2002                                                      0);
2003         }
2004         if (rc)
2005                 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
2006         else
2007                 pe->tce_bypass_enabled = enable;
2008 }
2009
2010 static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2011                 __u32 page_shift, __u64 window_size, __u32 levels,
2012                 struct iommu_table *tbl);
2013
2014 static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
2015                 int num, __u32 page_shift, __u64 window_size, __u32 levels,
2016                 struct iommu_table **ptbl)
2017 {
2018         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2019                         table_group);
2020         int nid = pe->phb->hose->node;
2021         __u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
2022         long ret;
2023         struct iommu_table *tbl;
2024
2025         tbl = pnv_pci_table_alloc(nid);
2026         if (!tbl)
2027                 return -ENOMEM;
2028
2029         ret = pnv_pci_ioda2_table_alloc_pages(nid,
2030                         bus_offset, page_shift, window_size,
2031                         levels, tbl);
2032         if (ret) {
2033                 iommu_free_table(tbl, "pnv");
2034                 return ret;
2035         }
2036
2037         tbl->it_ops = &pnv_ioda2_iommu_ops;
2038         if (pe->phb->ioda.tce_inval_reg)
2039                 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2040
2041         *ptbl = tbl;
2042
2043         return 0;
2044 }
2045
2046 static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
2047 {
2048         struct iommu_table *tbl = NULL;
2049         long rc;
2050
2051         rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
2052                         IOMMU_PAGE_SHIFT_4K,
2053                         pe->table_group.tce32_size,
2054                         POWERNV_IOMMU_DEFAULT_LEVELS, &tbl);
2055         if (rc) {
2056                 pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
2057                                 rc);
2058                 return rc;
2059         }
2060
2061         iommu_init_table(tbl, pe->phb->hose->node);
2062
2063         rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
2064         if (rc) {
2065                 pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
2066                                 rc);
2067                 pnv_ioda2_table_free(tbl);
2068                 return rc;
2069         }
2070
2071         if (!pnv_iommu_bypass_disabled)
2072                 pnv_pci_ioda2_set_bypass(pe, true);
2073
2074         /* OPAL variant of PHB3 invalidated TCEs */
2075         if (pe->phb->ioda.tce_inval_reg)
2076                 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2077
2078         /*
2079          * Setting table base here only for carrying iommu_group
2080          * further down to let iommu_add_device() do the job.
2081          * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2082          */
2083         if (pe->flags & PNV_IODA_PE_DEV)
2084                 set_iommu_table_base(&pe->pdev->dev, tbl);
2085
2086         return 0;
2087 }
2088
2089 #if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
2090 static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
2091                 int num)
2092 {
2093         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2094                         table_group);
2095         struct pnv_phb *phb = pe->phb;
2096         long ret;
2097
2098         pe_info(pe, "Removing DMA window #%d\n", num);
2099
2100         ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2101                         (pe->pe_number << 1) + num,
2102                         0/* levels */, 0/* table address */,
2103                         0/* table size */, 0/* page size */);
2104         if (ret)
2105                 pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
2106         else
2107                 pnv_pci_ioda2_tce_invalidate_entire(pe);
2108
2109         pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
2110
2111         return ret;
2112 }
2113 #endif
2114
2115 #ifdef CONFIG_IOMMU_API
2116 static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
2117                 __u64 window_size, __u32 levels)
2118 {
2119         unsigned long bytes = 0;
2120         const unsigned window_shift = ilog2(window_size);
2121         unsigned entries_shift = window_shift - page_shift;
2122         unsigned table_shift = entries_shift + 3;
2123         unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
2124         unsigned long direct_table_size;
2125
2126         if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
2127                         (window_size > memory_hotplug_max()) ||
2128                         !is_power_of_2(window_size))
2129                 return 0;
2130
2131         /* Calculate a direct table size from window_size and levels */
2132         entries_shift = (entries_shift + levels - 1) / levels;
2133         table_shift = entries_shift + 3;
2134         table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
2135         direct_table_size =  1UL << table_shift;
2136
2137         for ( ; levels; --levels) {
2138                 bytes += _ALIGN_UP(tce_table_size, direct_table_size);
2139
2140                 tce_table_size /= direct_table_size;
2141                 tce_table_size <<= 3;
2142                 tce_table_size = _ALIGN_UP(tce_table_size, direct_table_size);
2143         }
2144
2145         return bytes;
2146 }
2147
2148 static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
2149 {
2150         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2151                                                 table_group);
2152         /* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
2153         struct iommu_table *tbl = pe->table_group.tables[0];
2154
2155         pnv_pci_ioda2_set_bypass(pe, false);
2156         pnv_pci_ioda2_unset_window(&pe->table_group, 0);
2157         pnv_ioda2_table_free(tbl);
2158 }
2159
2160 static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2161 {
2162         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2163                                                 table_group);
2164
2165         pnv_pci_ioda2_setup_default_config(pe);
2166 }
2167
2168 static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
2169         .get_table_size = pnv_pci_ioda2_get_table_size,
2170         .create_table = pnv_pci_ioda2_create_table,
2171         .set_window = pnv_pci_ioda2_set_window,
2172         .unset_window = pnv_pci_ioda2_unset_window,
2173         .take_ownership = pnv_ioda2_take_ownership,
2174         .release_ownership = pnv_ioda2_release_ownership,
2175 };
2176 #endif
2177
2178 static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
2179 {
2180         const __be64 *swinvp;
2181
2182         /* OPAL variant of PHB3 invalidated TCEs */
2183         swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
2184         if (!swinvp)
2185                 return;
2186
2187         phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
2188         phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
2189 }
2190
2191 static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift,
2192                 unsigned levels, unsigned long limit,
2193                 unsigned long *current_offset)
2194 {
2195         struct page *tce_mem = NULL;
2196         __be64 *addr, *tmp;
2197         unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
2198         unsigned long allocated = 1UL << (order + PAGE_SHIFT);
2199         unsigned entries = 1UL << (shift - 3);
2200         long i;
2201
2202         tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
2203         if (!tce_mem) {
2204                 pr_err("Failed to allocate a TCE memory, order=%d\n", order);
2205                 return NULL;
2206         }
2207         addr = page_address(tce_mem);
2208         memset(addr, 0, allocated);
2209
2210         --levels;
2211         if (!levels) {
2212                 *current_offset += allocated;
2213                 return addr;
2214         }
2215
2216         for (i = 0; i < entries; ++i) {
2217                 tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
2218                                 levels, limit, current_offset);
2219                 if (!tmp)
2220                         break;
2221
2222                 addr[i] = cpu_to_be64(__pa(tmp) |
2223                                 TCE_PCI_READ | TCE_PCI_WRITE);
2224
2225                 if (*current_offset >= limit)
2226                         break;
2227         }
2228
2229         return addr;
2230 }
2231
2232 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2233                 unsigned long size, unsigned level);
2234
2235 static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2236                 __u32 page_shift, __u64 window_size, __u32 levels,
2237                 struct iommu_table *tbl)
2238 {
2239         void *addr;
2240         unsigned long offset = 0, level_shift;
2241         const unsigned window_shift = ilog2(window_size);
2242         unsigned entries_shift = window_shift - page_shift;
2243         unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
2244         const unsigned long tce_table_size = 1UL << table_shift;
2245
2246         if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
2247                 return -EINVAL;
2248
2249         if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
2250                 return -EINVAL;
2251
2252         /* Adjust direct table size from window_size and levels */
2253         entries_shift = (entries_shift + levels - 1) / levels;
2254         level_shift = entries_shift + 3;
2255         level_shift = max_t(unsigned, level_shift, PAGE_SHIFT);
2256
2257         /* Allocate TCE table */
2258         addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
2259                         levels, tce_table_size, &offset);
2260
2261         /* addr==NULL means that the first level allocation failed */
2262         if (!addr)
2263                 return -ENOMEM;
2264
2265         /*
2266          * First level was allocated but some lower level failed as
2267          * we did not allocate as much as we wanted,
2268          * release partially allocated table.
2269          */
2270         if (offset < tce_table_size) {
2271                 pnv_pci_ioda2_table_do_free_pages(addr,
2272                                 1ULL << (level_shift - 3), levels - 1);
2273                 return -ENOMEM;
2274         }
2275
2276         /* Setup linux iommu table */
2277         pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
2278                         page_shift);
2279         tbl->it_level_size = 1ULL << (level_shift - 3);
2280         tbl->it_indirect_levels = levels - 1;
2281         tbl->it_allocated_size = offset;
2282
2283         pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
2284                         window_size, tce_table_size, bus_offset);
2285
2286         return 0;
2287 }
2288
2289 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
2290                 unsigned long size, unsigned level)
2291 {
2292         const unsigned long addr_ul = (unsigned long) addr &
2293                         ~(TCE_PCI_READ | TCE_PCI_WRITE);
2294
2295         if (level) {
2296                 long i;
2297                 u64 *tmp = (u64 *) addr_ul;
2298
2299                 for (i = 0; i < size; ++i) {
2300                         unsigned long hpa = be64_to_cpu(tmp[i]);
2301
2302                         if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
2303                                 continue;
2304
2305                         pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
2306                                         level - 1);
2307                 }
2308         }
2309
2310         free_pages(addr_ul, get_order(size << 3));
2311 }
2312
2313 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
2314 {
2315         const unsigned long size = tbl->it_indirect_levels ?
2316                         tbl->it_level_size : tbl->it_size;
2317
2318         if (!tbl->it_size)
2319                 return;
2320
2321         pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
2322                         tbl->it_indirect_levels);
2323 }
2324
2325 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2326                                        struct pnv_ioda_pe *pe)
2327 {
2328         int64_t rc;
2329
2330         /* We shouldn't already have a 32-bit DMA associated */
2331         if (WARN_ON(pe->tce32_seg >= 0))
2332                 return;
2333
2334         /* TVE #1 is selected by PCI address bit 59 */
2335         pe->tce_bypass_base = 1ull << 59;
2336
2337         iommu_register_group(&pe->table_group, phb->hose->global_number,
2338                         pe->pe_number);
2339
2340         /* The PE will reserve all possible 32-bits space */
2341         pe->tce32_seg = 0;
2342         pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
2343                 phb->ioda.m32_pci_base);
2344
2345         /* Setup linux iommu table */
2346         pe->table_group.tce32_start = 0;
2347         pe->table_group.tce32_size = phb->ioda.m32_pci_base;
2348         pe->table_group.max_dynamic_windows_supported =
2349                         IOMMU_TABLE_GROUP_MAX_TABLES;
2350         pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
2351         pe->table_group.pgsizes = SZ_4K | SZ_64K | SZ_16M;
2352 #ifdef CONFIG_IOMMU_API
2353         pe->table_group.ops = &pnv_pci_ioda2_ops;
2354 #endif
2355
2356         rc = pnv_pci_ioda2_setup_default_config(pe);
2357         if (rc) {
2358                 if (pe->tce32_seg >= 0)
2359                         pe->tce32_seg = -1;
2360                 return;
2361         }
2362
2363         if (pe->flags & PNV_IODA_PE_DEV)
2364                 iommu_add_device(&pe->pdev->dev);
2365         else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
2366                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
2367 }
2368
2369 static void pnv_ioda_setup_dma(struct pnv_phb *phb)
2370 {
2371         struct pci_controller *hose = phb->hose;
2372         unsigned int residual, remaining, segs, tw, base;
2373         struct pnv_ioda_pe *pe;
2374
2375         /* If we have more PE# than segments available, hand out one
2376          * per PE until we run out and let the rest fail. If not,
2377          * then we assign at least one segment per PE, plus more based
2378          * on the amount of devices under that PE
2379          */
2380         if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2381                 residual = 0;
2382         else
2383                 residual = phb->ioda.tce32_count -
2384                         phb->ioda.dma_pe_count;
2385
2386         pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2387                 hose->global_number, phb->ioda.tce32_count);
2388         pr_info("PCI: %d PE# for a total weight of %d\n",
2389                 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2390
2391         pnv_pci_ioda_setup_opal_tce_kill(phb);
2392
2393         /* Walk our PE list and configure their DMA segments, hand them
2394          * out one base segment plus any residual segments based on
2395          * weight
2396          */
2397         remaining = phb->ioda.tce32_count;
2398         tw = phb->ioda.dma_weight;
2399         base = 0;
2400         list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
2401                 if (!pe->dma_weight)
2402                         continue;
2403                 if (!remaining) {
2404                         pe_warn(pe, "No DMA32 resources available\n");
2405                         continue;
2406                 }
2407                 segs = 1;
2408                 if (residual) {
2409                         segs += ((pe->dma_weight * residual)  + (tw / 2)) / tw;
2410                         if (segs > remaining)
2411                                 segs = remaining;
2412                 }
2413
2414                 /*
2415                  * For IODA2 compliant PHB3, we needn't care about the weight.
2416                  * The all available 32-bits DMA space will be assigned to
2417                  * the specific PE.
2418                  */
2419                 if (phb->type == PNV_PHB_IODA1) {
2420                         pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2421                                 pe->dma_weight, segs);
2422                         pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2423                 } else {
2424                         pe_info(pe, "Assign DMA32 space\n");
2425                         segs = 0;
2426                         pnv_pci_ioda2_setup_dma_pe(phb, pe);
2427                 }
2428
2429                 remaining -= segs;
2430                 base += segs;
2431         }
2432 }
2433
2434 #ifdef CONFIG_PCI_MSI
2435 static void pnv_ioda2_msi_eoi(struct irq_data *d)
2436 {
2437         unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2438         struct irq_chip *chip = irq_data_get_irq_chip(d);
2439         struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2440                                            ioda.irq_chip);
2441         int64_t rc;
2442
2443         rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2444         WARN_ON_ONCE(rc);
2445
2446         icp_native_eoi(d);
2447 }
2448
2449
2450 static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2451 {
2452         struct irq_data *idata;
2453         struct irq_chip *ichip;
2454
2455         if (phb->type != PNV_PHB_IODA2)
2456                 return;
2457
2458         if (!phb->ioda.irq_chip_init) {
2459                 /*
2460                  * First time we setup an MSI IRQ, we need to setup the
2461                  * corresponding IRQ chip to route correctly.
2462                  */
2463                 idata = irq_get_irq_data(virq);
2464                 ichip = irq_data_get_irq_chip(idata);
2465                 phb->ioda.irq_chip_init = 1;
2466                 phb->ioda.irq_chip = *ichip;
2467                 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2468         }
2469         irq_set_chip(virq, &phb->ioda.irq_chip);
2470 }
2471
2472 #ifdef CONFIG_CXL_BASE
2473
2474 struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
2475 {
2476         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2477
2478         return of_node_get(hose->dn);
2479 }
2480 EXPORT_SYMBOL(pnv_pci_get_phb_node);
2481
2482 int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
2483 {
2484         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2485         struct pnv_phb *phb = hose->private_data;
2486         struct pnv_ioda_pe *pe;
2487         int rc;
2488
2489         pe = pnv_ioda_get_pe(dev);
2490         if (!pe)
2491                 return -ENODEV;
2492
2493         pe_info(pe, "Switching PHB to CXL\n");
2494
2495         rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
2496         if (rc)
2497                 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2498
2499         return rc;
2500 }
2501 EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
2502
2503 /* Find PHB for cxl dev and allocate MSI hwirqs?
2504  * Returns the absolute hardware IRQ number
2505  */
2506 int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2507 {
2508         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2509         struct pnv_phb *phb = hose->private_data;
2510         int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2511
2512         if (hwirq < 0) {
2513                 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2514                 return -ENOSPC;
2515         }
2516
2517         return phb->msi_base + hwirq;
2518 }
2519 EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2520
2521 void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2522 {
2523         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2524         struct pnv_phb *phb = hose->private_data;
2525
2526         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2527 }
2528 EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2529
2530 void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2531                                   struct pci_dev *dev)
2532 {
2533         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2534         struct pnv_phb *phb = hose->private_data;
2535         int i, hwirq;
2536
2537         for (i = 1; i < CXL_IRQ_RANGES; i++) {
2538                 if (!irqs->range[i])
2539                         continue;
2540                 pr_devel("cxl release irq range 0x%x: offset: 0x%lx  limit: %ld\n",
2541                          i, irqs->offset[i],
2542                          irqs->range[i]);
2543                 hwirq = irqs->offset[i] - phb->msi_base;
2544                 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2545                                        irqs->range[i]);
2546         }
2547 }
2548 EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2549
2550 int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2551                                struct pci_dev *dev, int num)
2552 {
2553         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2554         struct pnv_phb *phb = hose->private_data;
2555         int i, hwirq, try;
2556
2557         memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2558
2559         /* 0 is reserved for the multiplexed PSL DSI interrupt */
2560         for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2561                 try = num;
2562                 while (try) {
2563                         hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2564                         if (hwirq >= 0)
2565                                 break;
2566                         try /= 2;
2567                 }
2568                 if (!try)
2569                         goto fail;
2570
2571                 irqs->offset[i] = phb->msi_base + hwirq;
2572                 irqs->range[i] = try;
2573                 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx  limit: %li\n",
2574                          i, irqs->offset[i], irqs->range[i]);
2575                 num -= try;
2576         }
2577         if (num)
2578                 goto fail;
2579
2580         return 0;
2581 fail:
2582         pnv_cxl_release_hwirq_ranges(irqs, dev);
2583         return -ENOSPC;
2584 }
2585 EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2586
2587 int pnv_cxl_get_irq_count(struct pci_dev *dev)
2588 {
2589         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2590         struct pnv_phb *phb = hose->private_data;
2591
2592         return phb->msi_bmp.irq_count;
2593 }
2594 EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2595
2596 int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2597                            unsigned int virq)
2598 {
2599         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2600         struct pnv_phb *phb = hose->private_data;
2601         unsigned int xive_num = hwirq - phb->msi_base;
2602         struct pnv_ioda_pe *pe;
2603         int rc;
2604
2605         if (!(pe = pnv_ioda_get_pe(dev)))
2606                 return -ENODEV;
2607
2608         /* Assign XIVE to PE */
2609         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2610         if (rc) {
2611                 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2612                         "hwirq 0x%x XIVE 0x%x PE\n",
2613                         pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2614                 return -EIO;
2615         }
2616         set_msi_irq_chip(phb, virq);
2617
2618         return 0;
2619 }
2620 EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2621 #endif
2622
2623 static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
2624                                   unsigned int hwirq, unsigned int virq,
2625                                   unsigned int is_64, struct msi_msg *msg)
2626 {
2627         struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2628         unsigned int xive_num = hwirq - phb->msi_base;
2629         __be32 data;
2630         int rc;
2631
2632         /* No PE assigned ? bail out ... no MSI for you ! */
2633         if (pe == NULL)
2634                 return -ENXIO;
2635
2636         /* Check if we have an MVE */
2637         if (pe->mve_number < 0)
2638                 return -ENXIO;
2639
2640         /* Force 32-bit MSI on some broken devices */
2641         if (dev->no_64bit_msi)
2642                 is_64 = 0;
2643
2644         /* Assign XIVE to PE */
2645         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2646         if (rc) {
2647                 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2648                         pci_name(dev), rc, xive_num);
2649                 return -EIO;
2650         }
2651
2652         if (is_64) {
2653                 __be64 addr64;
2654
2655                 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2656                                      &addr64, &data);
2657                 if (rc) {
2658                         pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2659                                 pci_name(dev), rc);
2660                         return -EIO;
2661                 }
2662                 msg->address_hi = be64_to_cpu(addr64) >> 32;
2663                 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
2664         } else {
2665                 __be32 addr32;
2666
2667                 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2668                                      &addr32, &data);
2669                 if (rc) {
2670                         pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2671                                 pci_name(dev), rc);
2672                         return -EIO;
2673                 }
2674                 msg->address_hi = 0;
2675                 msg->address_lo = be32_to_cpu(addr32);
2676         }
2677         msg->data = be32_to_cpu(data);
2678
2679         set_msi_irq_chip(phb, virq);
2680
2681         pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2682                  " address=%x_%08x data=%x PE# %d\n",
2683                  pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2684                  msg->address_hi, msg->address_lo, data, pe->pe_number);
2685
2686         return 0;
2687 }
2688
2689 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2690 {
2691         unsigned int count;
2692         const __be32 *prop = of_get_property(phb->hose->dn,
2693                                              "ibm,opal-msi-ranges", NULL);
2694         if (!prop) {
2695                 /* BML Fallback */
2696                 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2697         }
2698         if (!prop)
2699                 return;
2700
2701         phb->msi_base = be32_to_cpup(prop);
2702         count = be32_to_cpup(prop + 1);
2703         if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
2704                 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2705                        phb->hose->global_number);
2706                 return;
2707         }
2708
2709         phb->msi_setup = pnv_pci_ioda_msi_setup;
2710         phb->msi32_support = 1;
2711         pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
2712                 count, phb->msi_base);
2713 }
2714 #else
2715 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2716 #endif /* CONFIG_PCI_MSI */
2717
2718 #ifdef CONFIG_PCI_IOV
2719 static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2720 {
2721         struct pci_controller *hose;
2722         struct pnv_phb *phb;
2723         struct resource *res;
2724         int i;
2725         resource_size_t size;
2726         struct pci_dn *pdn;
2727         int mul, total_vfs;
2728
2729         if (!pdev->is_physfn || pdev->is_added)
2730                 return;
2731
2732         hose = pci_bus_to_host(pdev->bus);
2733         phb = hose->private_data;
2734
2735         pdn = pci_get_pdn(pdev);
2736         pdn->vfs_expanded = 0;
2737
2738         total_vfs = pci_sriov_get_totalvfs(pdev);
2739         pdn->m64_per_iov = 1;
2740         mul = phb->ioda.total_pe;
2741
2742         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2743                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2744                 if (!res->flags || res->parent)
2745                         continue;
2746                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2747                         dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2748                                  i, res);
2749                         continue;
2750                 }
2751
2752                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2753
2754                 /* bigger than 64M */
2755                 if (size > (1 << 26)) {
2756                         dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2757                                  i, res);
2758                         pdn->m64_per_iov = M64_PER_IOV;
2759                         mul = roundup_pow_of_two(total_vfs);
2760                         break;
2761                 }
2762         }
2763
2764         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2765                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2766                 if (!res->flags || res->parent)
2767                         continue;
2768                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2769                         dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2770                                  i, res);
2771                         continue;
2772                 }
2773
2774                 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2775                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2776                 res->end = res->start + size * mul - 1;
2777                 dev_dbg(&pdev->dev, "                       %pR\n", res);
2778                 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
2779                          i, res, mul);
2780         }
2781         pdn->vfs_expanded = mul;
2782 }
2783 #endif /* CONFIG_PCI_IOV */
2784
2785 /*
2786  * This function is supposed to be called on basis of PE from top
2787  * to bottom style. So the the I/O or MMIO segment assigned to
2788  * parent PE could be overrided by its child PEs if necessary.
2789  */
2790 static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2791                                   struct pnv_ioda_pe *pe)
2792 {
2793         struct pnv_phb *phb = hose->private_data;
2794         struct pci_bus_region region;
2795         struct resource *res;
2796         int i, index;
2797         int rc;
2798
2799         /*
2800          * NOTE: We only care PCI bus based PE for now. For PCI
2801          * device based PE, for example SRIOV sensitive VF should
2802          * be figured out later.
2803          */
2804         BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2805
2806         pci_bus_for_each_resource(pe->pbus, res, i) {
2807                 if (!res || !res->flags ||
2808                     res->start > res->end)
2809                         continue;
2810
2811                 if (res->flags & IORESOURCE_IO) {
2812                         region.start = res->start - phb->ioda.io_pci_base;
2813                         region.end   = res->end - phb->ioda.io_pci_base;
2814                         index = region.start / phb->ioda.io_segsize;
2815
2816                         while (index < phb->ioda.total_pe &&
2817                                region.start <= region.end) {
2818                                 phb->ioda.io_segmap[index] = pe->pe_number;
2819                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2820                                         pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2821                                 if (rc != OPAL_SUCCESS) {
2822                                         pr_err("%s: OPAL error %d when mapping IO "
2823                                                "segment #%d to PE#%d\n",
2824                                                __func__, rc, index, pe->pe_number);
2825                                         break;
2826                                 }
2827
2828                                 region.start += phb->ioda.io_segsize;
2829                                 index++;
2830                         }
2831                 } else if ((res->flags & IORESOURCE_MEM) &&
2832                            !pnv_pci_is_mem_pref_64(res->flags)) {
2833                         region.start = res->start -
2834                                        hose->mem_offset[0] -
2835                                        phb->ioda.m32_pci_base;
2836                         region.end   = res->end -
2837                                        hose->mem_offset[0] -
2838                                        phb->ioda.m32_pci_base;
2839                         index = region.start / phb->ioda.m32_segsize;
2840
2841                         while (index < phb->ioda.total_pe &&
2842                                region.start <= region.end) {
2843                                 phb->ioda.m32_segmap[index] = pe->pe_number;
2844                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2845                                         pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2846                                 if (rc != OPAL_SUCCESS) {
2847                                         pr_err("%s: OPAL error %d when mapping M32 "
2848                                                "segment#%d to PE#%d",
2849                                                __func__, rc, index, pe->pe_number);
2850                                         break;
2851                                 }
2852
2853                                 region.start += phb->ioda.m32_segsize;
2854                                 index++;
2855                         }
2856                 }
2857         }
2858 }
2859
2860 static void pnv_pci_ioda_setup_seg(void)
2861 {
2862         struct pci_controller *tmp, *hose;
2863         struct pnv_phb *phb;
2864         struct pnv_ioda_pe *pe;
2865
2866         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2867                 phb = hose->private_data;
2868                 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2869                         pnv_ioda_setup_pe_seg(hose, pe);
2870                 }
2871         }
2872 }
2873
2874 static void pnv_pci_ioda_setup_DMA(void)
2875 {
2876         struct pci_controller *hose, *tmp;
2877         struct pnv_phb *phb;
2878
2879         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2880                 pnv_ioda_setup_dma(hose->private_data);
2881
2882                 /* Mark the PHB initialization done */
2883                 phb = hose->private_data;
2884                 phb->initialized = 1;
2885         }
2886 }
2887
2888 static void pnv_pci_ioda_create_dbgfs(void)
2889 {
2890 #ifdef CONFIG_DEBUG_FS
2891         struct pci_controller *hose, *tmp;
2892         struct pnv_phb *phb;
2893         char name[16];
2894
2895         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2896                 phb = hose->private_data;
2897
2898                 sprintf(name, "PCI%04x", hose->global_number);
2899                 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2900                 if (!phb->dbgfs)
2901                         pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2902                                 __func__, hose->global_number);
2903         }
2904 #endif /* CONFIG_DEBUG_FS */
2905 }
2906
2907 static void pnv_pci_ioda_fixup(void)
2908 {
2909         pnv_pci_ioda_setup_PEs();
2910         pnv_pci_ioda_setup_seg();
2911         pnv_pci_ioda_setup_DMA();
2912
2913         pnv_pci_ioda_create_dbgfs();
2914
2915 #ifdef CONFIG_EEH
2916         eeh_init();
2917         eeh_addr_cache_build();
2918 #endif
2919 }
2920
2921 /*
2922  * Returns the alignment for I/O or memory windows for P2P
2923  * bridges. That actually depends on how PEs are segmented.
2924  * For now, we return I/O or M32 segment size for PE sensitive
2925  * P2P bridges. Otherwise, the default values (4KiB for I/O,
2926  * 1MiB for memory) will be returned.
2927  *
2928  * The current PCI bus might be put into one PE, which was
2929  * create against the parent PCI bridge. For that case, we
2930  * needn't enlarge the alignment so that we can save some
2931  * resources.
2932  */
2933 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2934                                                 unsigned long type)
2935 {
2936         struct pci_dev *bridge;
2937         struct pci_controller *hose = pci_bus_to_host(bus);
2938         struct pnv_phb *phb = hose->private_data;
2939         int num_pci_bridges = 0;
2940
2941         bridge = bus->self;
2942         while (bridge) {
2943                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2944                         num_pci_bridges++;
2945                         if (num_pci_bridges >= 2)
2946                                 return 1;
2947                 }
2948
2949                 bridge = bridge->bus->self;
2950         }
2951
2952         /* We fail back to M32 if M64 isn't supported */
2953         if (phb->ioda.m64_segsize &&
2954             pnv_pci_is_mem_pref_64(type))
2955                 return phb->ioda.m64_segsize;
2956         if (type & IORESOURCE_MEM)
2957                 return phb->ioda.m32_segsize;
2958
2959         return phb->ioda.io_segsize;
2960 }
2961
2962 #ifdef CONFIG_PCI_IOV
2963 static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2964                                                       int resno)
2965 {
2966         struct pci_dn *pdn = pci_get_pdn(pdev);
2967         resource_size_t align, iov_align;
2968
2969         iov_align = resource_size(&pdev->resource[resno]);
2970         if (iov_align)
2971                 return iov_align;
2972
2973         align = pci_iov_resource_size(pdev, resno);
2974         if (pdn->vfs_expanded)
2975                 return pdn->vfs_expanded * align;
2976
2977         return align;
2978 }
2979 #endif /* CONFIG_PCI_IOV */
2980
2981 /* Prevent enabling devices for which we couldn't properly
2982  * assign a PE
2983  */
2984 static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
2985 {
2986         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2987         struct pnv_phb *phb = hose->private_data;
2988         struct pci_dn *pdn;
2989
2990         /* The function is probably called while the PEs have
2991          * not be created yet. For example, resource reassignment
2992          * during PCI probe period. We just skip the check if
2993          * PEs isn't ready.
2994          */
2995         if (!phb->initialized)
2996                 return true;
2997
2998         pdn = pci_get_pdn(dev);
2999         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
3000                 return false;
3001
3002         return true;
3003 }
3004
3005 static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
3006                                u32 devfn)
3007 {
3008         return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
3009 }
3010
3011 static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
3012 {
3013         struct pnv_phb *phb = hose->private_data;
3014
3015         opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
3016                        OPAL_ASSERT_RESET);
3017 }
3018
3019 static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
3020        .dma_dev_setup = pnv_pci_dma_dev_setup,
3021 #ifdef CONFIG_PCI_MSI
3022        .setup_msi_irqs = pnv_setup_msi_irqs,
3023        .teardown_msi_irqs = pnv_teardown_msi_irqs,
3024 #endif
3025        .enable_device_hook = pnv_pci_enable_device_hook,
3026        .window_alignment = pnv_pci_window_alignment,
3027        .reset_secondary_bus = pnv_pci_reset_secondary_bus,
3028        .dma_set_mask = pnv_pci_ioda_dma_set_mask,
3029        .dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask,
3030        .shutdown = pnv_pci_ioda_shutdown,
3031 };
3032
3033 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
3034                                          u64 hub_id, int ioda_type)
3035 {
3036         struct pci_controller *hose;
3037         struct pnv_phb *phb;
3038         unsigned long size, m32map_off, pemap_off, iomap_off = 0;
3039         const __be64 *prop64;
3040         const __be32 *prop32;
3041         int len;
3042         u64 phb_id;
3043         void *aux;
3044         long rc;
3045
3046         pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
3047
3048         prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
3049         if (!prop64) {
3050                 pr_err("  Missing \"ibm,opal-phbid\" property !\n");
3051                 return;
3052         }
3053         phb_id = be64_to_cpup(prop64);
3054         pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
3055
3056         phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
3057
3058         /* Allocate PCI controller */
3059         phb->hose = hose = pcibios_alloc_controller(np);
3060         if (!phb->hose) {
3061                 pr_err("  Can't allocate PCI controller for %s\n",
3062                        np->full_name);
3063                 memblock_free(__pa(phb), sizeof(struct pnv_phb));
3064                 return;
3065         }
3066
3067         spin_lock_init(&phb->lock);
3068         prop32 = of_get_property(np, "bus-range", &len);
3069         if (prop32 && len == 8) {
3070                 hose->first_busno = be32_to_cpu(prop32[0]);
3071                 hose->last_busno = be32_to_cpu(prop32[1]);
3072         } else {
3073                 pr_warn("  Broken <bus-range> on %s\n", np->full_name);
3074                 hose->first_busno = 0;
3075                 hose->last_busno = 0xff;
3076         }
3077         hose->private_data = phb;
3078         phb->hub_id = hub_id;
3079         phb->opal_id = phb_id;
3080         phb->type = ioda_type;
3081         mutex_init(&phb->ioda.pe_alloc_mutex);
3082
3083         /* Detect specific models for error handling */
3084         if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
3085                 phb->model = PNV_PHB_MODEL_P7IOC;
3086         else if (of_device_is_compatible(np, "ibm,power8-pciex"))
3087                 phb->model = PNV_PHB_MODEL_PHB3;
3088         else
3089                 phb->model = PNV_PHB_MODEL_UNKNOWN;
3090
3091         /* Parse 32-bit and IO ranges (if any) */
3092         pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
3093
3094         /* Get registers */
3095         phb->regs = of_iomap(np, 0);
3096         if (phb->regs == NULL)
3097                 pr_err("  Failed to map registers !\n");
3098
3099         /* Initialize more IODA stuff */
3100         phb->ioda.total_pe = 1;
3101         prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
3102         if (prop32)
3103                 phb->ioda.total_pe = be32_to_cpup(prop32);
3104         prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
3105         if (prop32)
3106                 phb->ioda.reserved_pe = be32_to_cpup(prop32);
3107
3108         /* Parse 64-bit MMIO range */
3109         pnv_ioda_parse_m64_window(phb);
3110
3111         phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
3112         /* FW Has already off top 64k of M32 space (MSI space) */
3113         phb->ioda.m32_size += 0x10000;
3114
3115         phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
3116         phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
3117         phb->ioda.io_size = hose->pci_io_size;
3118         phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
3119         phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
3120
3121         /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
3122         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
3123         m32map_off = size;
3124         size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
3125         if (phb->type == PNV_PHB_IODA1) {
3126                 iomap_off = size;
3127                 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
3128         }
3129         pemap_off = size;
3130         size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
3131         aux = memblock_virt_alloc(size, 0);
3132         phb->ioda.pe_alloc = aux;
3133         phb->ioda.m32_segmap = aux + m32map_off;
3134         if (phb->type == PNV_PHB_IODA1)
3135                 phb->ioda.io_segmap = aux + iomap_off;
3136         phb->ioda.pe_array = aux + pemap_off;
3137         set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
3138
3139         INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
3140         INIT_LIST_HEAD(&phb->ioda.pe_list);
3141         mutex_init(&phb->ioda.pe_list_mutex);
3142
3143         /* Calculate how many 32-bit TCE segments we have */
3144         phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
3145
3146 #if 0 /* We should really do that ... */
3147         rc = opal_pci_set_phb_mem_window(opal->phb_id,
3148                                          window_type,
3149                                          window_num,
3150                                          starting_real_address,
3151                                          starting_pci_address,
3152                                          segment_size);
3153 #endif
3154
3155         pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
3156                 phb->ioda.total_pe, phb->ioda.reserved_pe,
3157                 phb->ioda.m32_size, phb->ioda.m32_segsize);
3158         if (phb->ioda.m64_size)
3159                 pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
3160                         phb->ioda.m64_size, phb->ioda.m64_segsize);
3161         if (phb->ioda.io_size)
3162                 pr_info("                  IO: 0x%x [segment=0x%x]\n",
3163                         phb->ioda.io_size, phb->ioda.io_segsize);
3164
3165
3166         phb->hose->ops = &pnv_pci_ops;
3167         phb->get_pe_state = pnv_ioda_get_pe_state;
3168         phb->freeze_pe = pnv_ioda_freeze_pe;
3169         phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
3170
3171         /* Setup RID -> PE mapping function */
3172         phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
3173
3174         /* Setup TCEs */
3175         phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
3176
3177         /* Setup MSI support */
3178         pnv_pci_init_ioda_msis(phb);
3179
3180         /*
3181          * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3182          * to let the PCI core do resource assignment. It's supposed
3183          * that the PCI core will do correct I/O and MMIO alignment
3184          * for the P2P bridge bars so that each PCI bus (excluding
3185          * the child P2P bridges) can form individual PE.
3186          */
3187         ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
3188         hose->controller_ops = pnv_pci_ioda_controller_ops;
3189
3190 #ifdef CONFIG_PCI_IOV
3191         ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
3192         ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
3193 #endif
3194
3195         pci_add_flags(PCI_REASSIGN_ALL_RSRC);
3196
3197         /* Reset IODA tables to a clean state */
3198         rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
3199         if (rc)
3200                 pr_warning("  OPAL Error %ld performing IODA table reset !\n", rc);
3201
3202         /* If we're running in kdump kerenl, the previous kerenl never
3203          * shutdown PCI devices correctly. We already got IODA table
3204          * cleaned out. So we have to issue PHB reset to stop all PCI
3205          * transactions from previous kerenl.
3206          */
3207         if (is_kdump_kernel()) {
3208                 pr_info("  Issue PHB reset ...\n");
3209                 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3210                 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
3211         }
3212
3213         /* Remove M64 resource if we can't configure it successfully */
3214         if (!phb->init_m64 || phb->init_m64(phb))
3215                 hose->mem_resources[1].flags = 0;
3216 }
3217
3218 void __init pnv_pci_init_ioda2_phb(struct device_node *np)
3219 {
3220         pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
3221 }
3222
3223 void __init pnv_pci_init_ioda_hub(struct device_node *np)
3224 {
3225         struct device_node *phbn;
3226         const __be64 *prop64;
3227         u64 hub_id;
3228
3229         pr_info("Probing IODA IO-Hub %s\n", np->full_name);
3230
3231         prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3232         if (!prop64) {
3233                 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3234                 return;
3235         }
3236         hub_id = be64_to_cpup(prop64);
3237         pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3238
3239         /* Count child PHBs */
3240         for_each_child_of_node(np, phbn) {
3241                 /* Look for IODA1 PHBs */
3242                 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
3243                         pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
3244         }
3245 }