arch/powerpc/platforms/powernv/pci-ioda.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #undef DEBUG
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/crash_dump.h>
  17 #include <linux/debugfs.h>
  18 #include <linux/delay.h>
  19 #include <linux/string.h>
  20 #include <linux/init.h>
  21 #include <linux/bootmem.h>
  22 #include <linux/irq.h>
  23 #include <linux/io.h>
  24 #include <linux/msi.h>
  25 #include <linux/memblock.h>
  26 #include <linux/iommu.h>
  27
  28 #include <asm/sections.h>
  29 #include <asm/io.h>
  30 #include <asm/prom.h>
  31 #include <asm/pci-bridge.h>
  32 #include <asm/machdep.h>
  33 #include <asm/msi_bitmap.h>
  34 #include <asm/ppc-pci.h>
  35 #include <asm/opal.h>
  36 #include <asm/iommu.h>
  37 #include <asm/tce.h>
  38 #include <asm/xics.h>
  39 #include <asm/debug.h>
  40 #include <asm/firmware.h>
  41 #include <asm/pnv-pci.h>
  42
  43 #include <misc/cxl-base.h>
  44
  45 #include "powernv.h"
  46 #include "pci.h"
  47
  48 /* 256M DMA window, 4K TCE pages, 8 bytes TCE */
  49 #define TCE32_TABLE_SIZE        ((0x10000000 / 0x1000) * 8)
  50
  51 static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  52                             const char *fmt, ...)
  53 {
  54         struct va_format vaf;
  55         va_list args;
  56         char pfix[32];
  57
  58         va_start(args, fmt);
  59
  60         vaf.fmt = fmt;
  61         vaf.va = &args;
  62
  63         if (pe->flags & PNV_IODA_PE_DEV)
  64                 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  65         else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  66                 sprintf(pfix, "%04x:%02x     ",
  67                         pci_domain_nr(pe->pbus), pe->pbus->number);
  68 #ifdef CONFIG_PCI_IOV
  69         else if (pe->flags & PNV_IODA_PE_VF)
  70                 sprintf(pfix, "%04x:%02x:%2x.%d",
  71                         pci_domain_nr(pe->parent_dev->bus),
  72                         (pe->rid & 0xff00) >> 8,
  73                         PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  74 #endif /* CONFIG_PCI_IOV*/
  75
  76         printk("%spci %s: [PE# %.3d] %pV",
  77                level, pfix, pe->pe_number, &vaf);
  78
  79         va_end(args);
  80 }
  81
  82 #define pe_err(pe, fmt, ...)                                    \
  83         pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
  84 #define pe_warn(pe, fmt, ...)                                   \
  85         pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
  86 #define pe_info(pe, fmt, ...)                                   \
  87         pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
  88
  89 static bool pnv_iommu_bypass_disabled __read_mostly;
  90
  91 static int __init iommu_setup(char *str)
  92 {
  93         if (!str)
  94                 return -EINVAL;
  95
  96         while (*str) {
  97                 if (!strncmp(str, "nobypass", 8)) {
  98                         pnv_iommu_bypass_disabled = true;
  99                         pr_info("PowerNV: IOMMU bypass window disabled.\n");
 100                         break;
 101                 }
 102                 str += strcspn(str, ",");
 103                 if (*str == ',')
 104                         str++;
 105         }
 106
 107         return 0;
 108 }
 109 early_param("iommu", iommu_setup);
 110
 111 /*
 112  * stdcix is only supposed to be used in hypervisor real mode as per
 113  * the architecture spec
 114  */
 115 static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
 116 {
 117         __asm__ __volatile__("stdcix %0,0,%1"
 118                 : : "r" (val), "r" (paddr) : "memory");
 119 }
 120
 121 static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
 122 {
 123         return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
 124                 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
 125 }
 126
 127 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
 128 {
 129         if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
 130                 pr_warn("%s: Invalid PE %d on PHB#%x\n",
 131                         __func__, pe_no, phb->hose->global_number);
 132                 return;
 133         }
 134
 135         if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
 136                 pr_warn("%s: PE %d was assigned on PHB#%x\n",
 137                         __func__, pe_no, phb->hose->global_number);
 138                 return;
 139         }
 140
 141         phb->ioda.pe_array[pe_no].phb = phb;
 142         phb->ioda.pe_array[pe_no].pe_number = pe_no;
 143 }
 144
 145 static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
 146 {
 147         unsigned long pe;
 148
 149         do {
 150                 pe = find_next_zero_bit(phb->ioda.pe_alloc,
 151                                         phb->ioda.total_pe, 0);
 152                 if (pe >= phb->ioda.total_pe)
 153                         return IODA_INVALID_PE;
 154         } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
 155
 156         phb->ioda.pe_array[pe].phb = phb;
 157         phb->ioda.pe_array[pe].pe_number = pe;
 158         return pe;
 159 }
 160
 161 static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
 162 {
 163         WARN_ON(phb->ioda.pe_array[pe].pdev);
 164
 165         memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
 166         clear_bit(pe, phb->ioda.pe_alloc);
 167 }
 168
 169 /* The default M64 BAR is shared by all PEs */
 170 static int pnv_ioda2_init_m64(struct pnv_phb *phb)
 171 {
 172         const char *desc;
 173         struct resource *r;
 174         s64 rc;
 175
 176         /* Configure the default M64 BAR */
 177         rc = opal_pci_set_phb_mem_window(phb->opal_id,
 178                                          OPAL_M64_WINDOW_TYPE,
 179                                          phb->ioda.m64_bar_idx,
 180                                          phb->ioda.m64_base,
 181                                          0, /* unused */
 182                                          phb->ioda.m64_size);
 183         if (rc != OPAL_SUCCESS) {
 184                 desc = "configuring";
 185                 goto fail;
 186         }
 187
 188         /* Enable the default M64 BAR */
 189         rc = opal_pci_phb_mmio_enable(phb->opal_id,
 190                                       OPAL_M64_WINDOW_TYPE,
 191                                       phb->ioda.m64_bar_idx,
 192                                       OPAL_ENABLE_M64_SPLIT);
 193         if (rc != OPAL_SUCCESS) {
 194                 desc = "enabling";
 195                 goto fail;
 196         }
 197
 198         /* Mark the M64 BAR assigned */
 199         set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
 200
 201         /*
 202          * Strip off the segment used by the reserved PE, which is
 203          * expected to be 0 or last one of PE capabicity.
 204          */
 205         r = &phb->hose->mem_resources[1];
 206         if (phb->ioda.reserved_pe == 0)
 207                 r->start += phb->ioda.m64_segsize;
 208         else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
 209                 r->end -= phb->ioda.m64_segsize;
 210         else
 211                 pr_warn("  Cannot strip M64 segment for reserved PE#%d\n",
 212                         phb->ioda.reserved_pe);
 213
 214         return 0;
 215
 216 fail:
 217         pr_warn("  Failure %lld %s M64 BAR#%d\n",
 218                 rc, desc, phb->ioda.m64_bar_idx);
 219         opal_pci_phb_mmio_enable(phb->opal_id,
 220                                  OPAL_M64_WINDOW_TYPE,
 221                                  phb->ioda.m64_bar_idx,
 222                                  OPAL_DISABLE_M64);
 223         return -EIO;
 224 }
 225
 226 static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
 227 {
 228         resource_size_t sgsz = phb->ioda.m64_segsize;
 229         struct pci_dev *pdev;
 230         struct resource *r;
 231         int base, step, i;
 232
 233         /*
 234          * Root bus always has full M64 range and root port has
 235          * M64 range used in reality. So we're checking root port
 236          * instead of root bus.
 237          */
 238         list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
 239                 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
 240                         r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
 241                         if (!r->parent ||
 242                             !pnv_pci_is_mem_pref_64(r->flags))
 243                                 continue;
 244
 245                         base = (r->start - phb->ioda.m64_base) / sgsz;
 246                         for (step = 0; step < resource_size(r) / sgsz; step++)
 247                                 pnv_ioda_reserve_pe(phb, base + step);
 248                 }
 249         }
 250 }
 251
 252 static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
 253                                  struct pci_bus *bus, int all)
 254 {
 255         resource_size_t segsz = phb->ioda.m64_segsize;
 256         struct pci_dev *pdev;
 257         struct resource *r;
 258         struct pnv_ioda_pe *master_pe, *pe;
 259         unsigned long size, *pe_alloc;
 260         bool found;
 261         int start, i, j;
 262
 263         /* Root bus shouldn't use M64 */
 264         if (pci_is_root_bus(bus))
 265                 return IODA_INVALID_PE;
 266
 267         /* We support only one M64 window on each bus */
 268         found = false;
 269         pci_bus_for_each_resource(bus, r, i) {
 270                 if (r && r->parent &&
 271                     pnv_pci_is_mem_pref_64(r->flags)) {
 272                         found = true;
 273                         break;
 274                 }
 275         }
 276
 277         /* No M64 window found ? */
 278         if (!found)
 279                 return IODA_INVALID_PE;
 280
 281         /* Allocate bitmap */
 282         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
 283         pe_alloc = kzalloc(size, GFP_KERNEL);
 284         if (!pe_alloc) {
 285                 pr_warn("%s: Out of memory !\n",
 286                         __func__);
 287                 return IODA_INVALID_PE;
 288         }
 289
 290         /*
 291          * Figure out reserved PE numbers by the PE
 292          * the its child PEs.
 293          */
 294         start = (r->start - phb->ioda.m64_base) / segsz;
 295         for (i = 0; i < resource_size(r) / segsz; i++)
 296                 set_bit(start + i, pe_alloc);
 297
 298         if (all)
 299                 goto done;
 300
 301         /*
 302          * If the PE doesn't cover all subordinate buses,
 303          * we need subtract from reserved PEs for children.
 304          */
 305         list_for_each_entry(pdev, &bus->devices, bus_list) {
 306                 if (!pdev->subordinate)
 307                         continue;
 308
 309                 pci_bus_for_each_resource(pdev->subordinate, r, i) {
 310                         if (!r || !r->parent ||
 311                             !pnv_pci_is_mem_pref_64(r->flags))
 312                                 continue;
 313
 314                         start = (r->start - phb->ioda.m64_base) / segsz;
 315                         for (j = 0; j < resource_size(r) / segsz ; j++)
 316                                 clear_bit(start + j, pe_alloc);
 317                 }
 318         }
 319
 320         /*
 321          * the current bus might not own M64 window and that's all
 322          * contributed by its child buses. For the case, we needn't
 323          * pick M64 dependent PE#.
 324          */
 325         if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
 326                 kfree(pe_alloc);
 327                 return IODA_INVALID_PE;
 328         }
 329
 330         /*
 331          * Figure out the master PE and put all slave PEs to master
 332          * PE's list to form compound PE.
 333          */
 334 done:
 335         master_pe = NULL;
 336         i = -1;
 337         while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
 338                 phb->ioda.total_pe) {
 339                 pe = &phb->ioda.pe_array[i];
 340
 341                 if (!master_pe) {
 342                         pe->flags |= PNV_IODA_PE_MASTER;
 343                         INIT_LIST_HEAD(&pe->slaves);
 344                         master_pe = pe;
 345                 } else {
 346                         pe->flags |= PNV_IODA_PE_SLAVE;
 347                         pe->master = master_pe;
 348                         list_add_tail(&pe->list, &master_pe->slaves);
 349                 }
 350         }
 351
 352         kfree(pe_alloc);
 353         return master_pe->pe_number;
 354 }
 355
 356 static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
 357 {
 358         struct pci_controller *hose = phb->hose;
 359         struct device_node *dn = hose->dn;
 360         struct resource *res;
 361         const u32 *r;
 362         u64 pci_addr;
 363
 364         /* FIXME: Support M64 for P7IOC */
 365         if (phb->type != PNV_PHB_IODA2) {
 366                 pr_info("  Not support M64 window\n");
 367                 return;
 368         }
 369
 370         if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
 371                 pr_info("  Firmware too old to support M64 window\n");
 372                 return;
 373         }
 374
 375         r = of_get_property(dn, "ibm,opal-m64-window", NULL);
 376         if (!r) {
 377                 pr_info("  No <ibm,opal-m64-window> on %s\n",
 378                         dn->full_name);
 379                 return;
 380         }
 381
 382         res = &hose->mem_resources[1];
 383         res->start = of_translate_address(dn, r + 2);
 384         res->end = res->start + of_read_number(r + 4, 2) - 1;
 385         res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
 386         pci_addr = of_read_number(r, 2);
 387         hose->mem_offset[1] = res->start - pci_addr;
 388
 389         phb->ioda.m64_size = resource_size(res);
 390         phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
 391         phb->ioda.m64_base = pci_addr;
 392
 393         pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
 394                         res->start, res->end, pci_addr);
 395
 396         /* Use last M64 BAR to cover M64 window */
 397         phb->ioda.m64_bar_idx = 15;
 398         phb->init_m64 = pnv_ioda2_init_m64;
 399         phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
 400         phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
 401 }
 402
 403 static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
 404 {
 405         struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
 406         struct pnv_ioda_pe *slave;
 407         s64 rc;
 408
 409         /* Fetch master PE */
 410         if (pe->flags & PNV_IODA_PE_SLAVE) {
 411                 pe = pe->master;
 412                 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
 413                         return;
 414
 415                 pe_no = pe->pe_number;
 416         }
 417
 418         /* Freeze master PE */
 419         rc = opal_pci_eeh_freeze_set(phb->opal_id,
 420                                      pe_no,
 421                                      OPAL_EEH_ACTION_SET_FREEZE_ALL);
 422         if (rc != OPAL_SUCCESS) {
 423                 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 424                         __func__, rc, phb->hose->global_number, pe_no);
 425                 return;
 426         }
 427
 428         /* Freeze slave PEs */
 429         if (!(pe->flags & PNV_IODA_PE_MASTER))
 430                 return;
 431
 432         list_for_each_entry(slave, &pe->slaves, list) {
 433                 rc = opal_pci_eeh_freeze_set(phb->opal_id,
 434                                              slave->pe_number,
 435                                              OPAL_EEH_ACTION_SET_FREEZE_ALL);
 436                 if (rc != OPAL_SUCCESS)
 437                         pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 438                                 __func__, rc, phb->hose->global_number,
 439                                 slave->pe_number);
 440         }
 441 }
 442
 443 static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
 444 {
 445         struct pnv_ioda_pe *pe, *slave;
 446         s64 rc;
 447
 448         /* Find master PE */
 449         pe = &phb->ioda.pe_array[pe_no];
 450         if (pe->flags & PNV_IODA_PE_SLAVE) {
 451                 pe = pe->master;
 452                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 453                 pe_no = pe->pe_number;
 454         }
 455
 456         /* Clear frozen state for master PE */
 457         rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
 458         if (rc != OPAL_SUCCESS) {
 459                 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 460                         __func__, rc, opt, phb->hose->global_number, pe_no);
 461                 return -EIO;
 462         }
 463
 464         if (!(pe->flags & PNV_IODA_PE_MASTER))
 465                 return 0;
 466
 467         /* Clear frozen state for slave PEs */
 468         list_for_each_entry(slave, &pe->slaves, list) {
 469                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 470                                              slave->pe_number,
 471                                              opt);
 472                 if (rc != OPAL_SUCCESS) {
 473                         pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 474                                 __func__, rc, opt, phb->hose->global_number,
 475                                 slave->pe_number);
 476                         return -EIO;
 477                 }
 478         }
 479
 480         return 0;
 481 }
 482
 483 static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
 484 {
 485         struct pnv_ioda_pe *slave, *pe;
 486         u8 fstate, state;
 487         __be16 pcierr;
 488         s64 rc;
 489
 490         /* Sanity check on PE number */
 491         if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
 492                 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
 493
 494         /*
 495          * Fetch the master PE and the PE instance might be
 496          * not initialized yet.
 497          */
 498         pe = &phb->ioda.pe_array[pe_no];
 499         if (pe->flags & PNV_IODA_PE_SLAVE) {
 500                 pe = pe->master;
 501                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 502                 pe_no = pe->pe_number;
 503         }
 504
 505         /* Check the master PE */
 506         rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
 507                                         &state, &pcierr, NULL);
 508         if (rc != OPAL_SUCCESS) {
 509                 pr_warn("%s: Failure %lld getting "
 510                         "PHB#%x-PE#%x state\n",
 511                         __func__, rc,
 512                         phb->hose->global_number, pe_no);
 513                 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 514         }
 515
 516         /* Check the slave PE */
 517         if (!(pe->flags & PNV_IODA_PE_MASTER))
 518                 return state;
 519
 520         list_for_each_entry(slave, &pe->slaves, list) {
 521                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 522                                                 slave->pe_number,
 523                                                 &fstate,
 524                                                 &pcierr,
 525                                                 NULL);
 526                 if (rc != OPAL_SUCCESS) {
 527                         pr_warn("%s: Failure %lld getting "
 528                                 "PHB#%x-PE#%x state\n",
 529                                 __func__, rc,
 530                                 phb->hose->global_number, slave->pe_number);
 531                         return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 532                 }
 533
 534                 /*
 535                  * Override the result based on the ascending
 536                  * priority.
 537                  */
 538                 if (fstate > state)
 539                         state = fstate;
 540         }
 541
 542         return state;
 543 }
 544
 545 /* Currently those 2 are only used when MSIs are enabled, this will change
 546  * but in the meantime, we need to protect them to avoid warnings
 547  */
 548 #ifdef CONFIG_PCI_MSI
 549 static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
 550 {
 551         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 552         struct pnv_phb *phb = hose->private_data;
 553         struct pci_dn *pdn = pci_get_pdn(dev);
 554
 555         if (!pdn)
 556                 return NULL;
 557         if (pdn->pe_number == IODA_INVALID_PE)
 558                 return NULL;
 559         return &phb->ioda.pe_array[pdn->pe_number];
 560 }
 561 #endif /* CONFIG_PCI_MSI */
 562
 563 static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
 564                                   struct pnv_ioda_pe *parent,
 565                                   struct pnv_ioda_pe *child,
 566                                   bool is_add)
 567 {
 568         const char *desc = is_add ? "adding" : "removing";
 569         uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
 570                               OPAL_REMOVE_PE_FROM_DOMAIN;
 571         struct pnv_ioda_pe *slave;
 572         long rc;
 573
 574         /* Parent PE affects child PE */
 575         rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 576                                 child->pe_number, op);
 577         if (rc != OPAL_SUCCESS) {
 578                 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
 579                         rc, desc);
 580                 return -ENXIO;
 581         }
 582
 583         if (!(child->flags & PNV_IODA_PE_MASTER))
 584                 return 0;
 585
 586         /* Compound case: parent PE affects slave PEs */
 587         list_for_each_entry(slave, &child->slaves, list) {
 588                 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 589                                         slave->pe_number, op);
 590                 if (rc != OPAL_SUCCESS) {
 591                         pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
 592                                 rc, desc);
 593                         return -ENXIO;
 594                 }
 595         }
 596
 597         return 0;
 598 }
 599
 600 static int pnv_ioda_set_peltv(struct pnv_phb *phb,
 601                               struct pnv_ioda_pe *pe,
 602                               bool is_add)
 603 {
 604         struct pnv_ioda_pe *slave;
 605         struct pci_dev *pdev = NULL;
 606         int ret;
 607
 608         /*
 609          * Clear PE frozen state. If it's master PE, we need
 610          * clear slave PE frozen state as well.
 611          */
 612         if (is_add) {
 613                 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 614                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 615                 if (pe->flags & PNV_IODA_PE_MASTER) {
 616                         list_for_each_entry(slave, &pe->slaves, list)
 617                                 opal_pci_eeh_freeze_clear(phb->opal_id,
 618                                                           slave->pe_number,
 619                                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 620                 }
 621         }
 622
 623         /*
 624          * Associate PE in PELT. We need add the PE into the
 625          * corresponding PELT-V as well. Otherwise, the error
 626          * originated from the PE might contribute to other
 627          * PEs.
 628          */
 629         ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
 630         if (ret)
 631                 return ret;
 632
 633         /* For compound PEs, any one affects all of them */
 634         if (pe->flags & PNV_IODA_PE_MASTER) {
 635                 list_for_each_entry(slave, &pe->slaves, list) {
 636                         ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
 637                         if (ret)
 638                                 return ret;
 639                 }
 640         }
 641
 642         if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
 643                 pdev = pe->pbus->self;
 644         else if (pe->flags & PNV_IODA_PE_DEV)
 645                 pdev = pe->pdev->bus->self;
 646 #ifdef CONFIG_PCI_IOV
 647         else if (pe->flags & PNV_IODA_PE_VF)
 648                 pdev = pe->parent_dev->bus->self;
 649 #endif /* CONFIG_PCI_IOV */
 650         while (pdev) {
 651                 struct pci_dn *pdn = pci_get_pdn(pdev);
 652                 struct pnv_ioda_pe *parent;
 653
 654                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 655                         parent = &phb->ioda.pe_array[pdn->pe_number];
 656                         ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
 657                         if (ret)
 658                                 return ret;
 659                 }
 660
 661                 pdev = pdev->bus->self;
 662         }
 663
 664         return 0;
 665 }
 666
 667 #ifdef CONFIG_PCI_IOV
 668 static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 669 {
 670         struct pci_dev *parent;
 671         uint8_t bcomp, dcomp, fcomp;
 672         int64_t rc;
 673         long rid_end, rid;
 674
 675         /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
 676         if (pe->pbus) {
 677                 int count;
 678
 679                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 680                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 681                 parent = pe->pbus->self;
 682                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 683                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 684                 else
 685                         count = 1;
 686
 687                 switch(count) {
 688                 case  1: bcomp = OpalPciBusAll;         break;
 689                 case  2: bcomp = OpalPciBus7Bits;       break;
 690                 case  4: bcomp = OpalPciBus6Bits;       break;
 691                 case  8: bcomp = OpalPciBus5Bits;       break;
 692                 case 16: bcomp = OpalPciBus4Bits;       break;
 693                 case 32: bcomp = OpalPciBus3Bits;       break;
 694                 default:
 695                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 696                                 count);
 697                         /* Do an exact match only */
 698                         bcomp = OpalPciBusAll;
 699                 }
 700                 rid_end = pe->rid + (count << 8);
 701         } else {
 702                 if (pe->flags & PNV_IODA_PE_VF)
 703                         parent = pe->parent_dev;
 704                 else
 705                         parent = pe->pdev->bus->self;
 706                 bcomp = OpalPciBusAll;
 707                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 708                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 709                 rid_end = pe->rid + 1;
 710         }
 711
 712         /* Clear the reverse map */
 713         for (rid = pe->rid; rid < rid_end; rid++)
 714                 phb->ioda.pe_rmap[rid] = 0;
 715
 716         /* Release from all parents PELT-V */
 717         while (parent) {
 718                 struct pci_dn *pdn = pci_get_pdn(parent);
 719                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 720                         rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
 721                                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 722                         /* XXX What to do in case of error ? */
 723                 }
 724                 parent = parent->bus->self;
 725         }
 726
 727         opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
 728                                   OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 729
 730         /* Disassociate PE in PELT */
 731         rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
 732                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 733         if (rc)
 734                 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
 735         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 736                              bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
 737         if (rc)
 738                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 739
 740         pe->pbus = NULL;
 741         pe->pdev = NULL;
 742         pe->parent_dev = NULL;
 743
 744         return 0;
 745 }
 746 #endif /* CONFIG_PCI_IOV */
 747
 748 static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 749 {
 750         struct pci_dev *parent;
 751         uint8_t bcomp, dcomp, fcomp;
 752         long rc, rid_end, rid;
 753
 754         /* Bus validation ? */
 755         if (pe->pbus) {
 756                 int count;
 757
 758                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 759                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 760                 parent = pe->pbus->self;
 761                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 762                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 763                 else
 764                         count = 1;
 765
 766                 switch(count) {
 767                 case  1: bcomp = OpalPciBusAll;         break;
 768                 case  2: bcomp = OpalPciBus7Bits;       break;
 769                 case  4: bcomp = OpalPciBus6Bits;       break;
 770                 case  8: bcomp = OpalPciBus5Bits;       break;
 771                 case 16: bcomp = OpalPciBus4Bits;       break;
 772                 case 32: bcomp = OpalPciBus3Bits;       break;
 773                 default:
 774                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 775                                 count);
 776                         /* Do an exact match only */
 777                         bcomp = OpalPciBusAll;
 778                 }
 779                 rid_end = pe->rid + (count << 8);
 780         } else {
 781 #ifdef CONFIG_PCI_IOV
 782                 if (pe->flags & PNV_IODA_PE_VF)
 783                         parent = pe->parent_dev;
 784                 else
 785 #endif /* CONFIG_PCI_IOV */
 786                         parent = pe->pdev->bus->self;
 787                 bcomp = OpalPciBusAll;
 788                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 789                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 790                 rid_end = pe->rid + 1;
 791         }
 792
 793         /*
 794          * Associate PE in PELT. We need add the PE into the
 795          * corresponding PELT-V as well. Otherwise, the error
 796          * originated from the PE might contribute to other
 797          * PEs.
 798          */
 799         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 800                              bcomp, dcomp, fcomp, OPAL_MAP_PE);
 801         if (rc) {
 802                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 803                 return -ENXIO;
 804         }
 805
 806         /* Configure PELTV */
 807         pnv_ioda_set_peltv(phb, pe, true);
 808
 809         /* Setup reverse map */
 810         for (rid = pe->rid; rid < rid_end; rid++)
 811                 phb->ioda.pe_rmap[rid] = pe->pe_number;
 812
 813         /* Setup one MVTs on IODA1 */
 814         if (phb->type != PNV_PHB_IODA1) {
 815                 pe->mve_number = 0;
 816                 goto out;
 817         }
 818
 819         pe->mve_number = pe->pe_number;
 820         rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
 821         if (rc != OPAL_SUCCESS) {
 822                 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
 823                        rc, pe->mve_number);
 824                 pe->mve_number = -1;
 825         } else {
 826                 rc = opal_pci_set_mve_enable(phb->opal_id,
 827                                              pe->mve_number, OPAL_ENABLE_MVE);
 828                 if (rc) {
 829                         pe_err(pe, "OPAL error %ld enabling MVE %d\n",
 830                                rc, pe->mve_number);
 831                         pe->mve_number = -1;
 832                 }
 833         }
 834
 835 out:
 836         return 0;
 837 }
 838
 839 static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
 840                                        struct pnv_ioda_pe *pe)
 841 {
 842         struct pnv_ioda_pe *lpe;
 843
 844         list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
 845                 if (lpe->dma_weight < pe->dma_weight) {
 846                         list_add_tail(&pe->dma_link, &lpe->dma_link);
 847                         return;
 848                 }
 849         }
 850         list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
 851 }
 852
 853 static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
 854 {
 855         /* This is quite simplistic. The "base" weight of a device
 856          * is 10. 0 means no DMA is to be accounted for it.
 857          */
 858
 859         /* If it's a bridge, no DMA */
 860         if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
 861                 return 0;
 862
 863         /* Reduce the weight of slow USB controllers */
 864         if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
 865             dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
 866             dev->class == PCI_CLASS_SERIAL_USB_EHCI)
 867                 return 3;
 868
 869         /* Increase the weight of RAID (includes Obsidian) */
 870         if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
 871                 return 15;
 872
 873         /* Default */
 874         return 10;
 875 }
 876
 877 #ifdef CONFIG_PCI_IOV
 878 static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
 879 {
 880         struct pci_dn *pdn = pci_get_pdn(dev);
 881         int i;
 882         struct resource *res, res2;
 883         resource_size_t size;
 884         u16 num_vfs;
 885
 886         if (!dev->is_physfn)
 887                 return -EINVAL;
 888
 889         /*
 890          * "offset" is in VFs.  The M64 windows are sized so that when they
 891          * are segmented, each segment is the same size as the IOV BAR.
 892          * Each segment is in a separate PE, and the high order bits of the
 893          * address are the PE number.  Therefore, each VF's BAR is in a
 894          * separate PE, and changing the IOV BAR start address changes the
 895          * range of PEs the VFs are in.
 896          */
 897         num_vfs = pdn->num_vfs;
 898         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 899                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 900                 if (!res->flags || !res->parent)
 901                         continue;
 902
 903                 if (!pnv_pci_is_mem_pref_64(res->flags))
 904                         continue;
 905
 906                 /*
 907                  * The actual IOV BAR range is determined by the start address
 908                  * and the actual size for num_vfs VFs BAR.  This check is to
 909                  * make sure that after shifting, the range will not overlap
 910                  * with another device.
 911                  */
 912                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 913                 res2.flags = res->flags;
 914                 res2.start = res->start + (size * offset);
 915                 res2.end = res2.start + (size * num_vfs) - 1;
 916
 917                 if (res2.end > res->end) {
 918                         dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
 919                                 i, &res2, res, num_vfs, offset);
 920                         return -EBUSY;
 921                 }
 922         }
 923
 924         /*
 925          * After doing so, there would be a "hole" in the /proc/iomem when
 926          * offset is a positive value. It looks like the device return some
 927          * mmio back to the system, which actually no one could use it.
 928          */
 929         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 930                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 931                 if (!res->flags || !res->parent)
 932                         continue;
 933
 934                 if (!pnv_pci_is_mem_pref_64(res->flags))
 935                         continue;
 936
 937                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 938                 res2 = *res;
 939                 res->start += size * offset;
 940
 941                 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
 942                          i, &res2, res, num_vfs, offset);
 943                 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
 944         }
 945         return 0;
 946 }
 947 #endif /* CONFIG_PCI_IOV */
 948
 949 #if 0
 950 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 951 {
 952         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 953         struct pnv_phb *phb = hose->private_data;
 954         struct pci_dn *pdn = pci_get_pdn(dev);
 955         struct pnv_ioda_pe *pe;
 956         int pe_num;
 957
 958         if (!pdn) {
 959                 pr_err("%s: Device tree node not associated properly\n",
 960                            pci_name(dev));
 961                 return NULL;
 962         }
 963         if (pdn->pe_number != IODA_INVALID_PE)
 964                 return NULL;
 965
 966         /* PE#0 has been pre-set */
 967         if (dev->bus->number == 0)
 968                 pe_num = 0;
 969         else
 970                 pe_num = pnv_ioda_alloc_pe(phb);
 971         if (pe_num == IODA_INVALID_PE) {
 972                 pr_warning("%s: Not enough PE# available, disabling device\n",
 973                            pci_name(dev));
 974                 return NULL;
 975         }
 976
 977         /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
 978          * pointer in the PE data structure, both should be destroyed at the
 979          * same time. However, this needs to be looked at more closely again
 980          * once we actually start removing things (Hotplug, SR-IOV, ...)
 981          *
 982          * At some point we want to remove the PDN completely anyways
 983          */
 984         pe = &phb->ioda.pe_array[pe_num];
 985         pci_dev_get(dev);
 986         pdn->pcidev = dev;
 987         pdn->pe_number = pe_num;
 988         pe->pdev = dev;
 989         pe->pbus = NULL;
 990         pe->tce32_seg = -1;
 991         pe->mve_number = -1;
 992         pe->rid = dev->bus->number << 8 | pdn->devfn;
 993
 994         pe_info(pe, "Associated device to PE\n");
 995
 996         if (pnv_ioda_configure_pe(phb, pe)) {
 997                 /* XXX What do we do here ? */
 998                 if (pe_num)
 999                         pnv_ioda_free_pe(phb, pe_num);
1000                 pdn->pe_number = IODA_INVALID_PE;
1001                 pe->pdev = NULL;
1002                 pci_dev_put(dev);
1003                 return NULL;
1004         }
1005
1006         /* Assign a DMA weight to the device */
1007         pe->dma_weight = pnv_ioda_dma_weight(dev);
1008         if (pe->dma_weight != 0) {
1009                 phb->ioda.dma_weight += pe->dma_weight;
1010                 phb->ioda.dma_pe_count++;
1011         }
1012
1013         /* Link the PE */
1014         pnv_ioda_link_pe_by_weight(phb, pe);
1015
1016         return pe;
1017 }
1018 #endif /* Useful for SRIOV case */
1019
1020 static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1021 {
1022         struct pci_dev *dev;
1023
1024         list_for_each_entry(dev, &bus->devices, bus_list) {
1025                 struct pci_dn *pdn = pci_get_pdn(dev);
1026
1027                 if (pdn == NULL) {
1028                         pr_warn("%s: No device node associated with device !\n",
1029                                 pci_name(dev));
1030                         continue;
1031                 }
1032                 pdn->pe_number = pe->pe_number;
1033                 pe->dma_weight += pnv_ioda_dma_weight(dev);
1034                 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1035                         pnv_ioda_setup_same_PE(dev->subordinate, pe);
1036         }
1037 }
1038
1039 /*
1040  * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1041  * single PCI bus. Another one that contains the primary PCI bus and its
1042  * subordinate PCI devices and buses. The second type of PE is normally
1043  * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1044  */
1045 static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
1046 {
1047         struct pci_controller *hose = pci_bus_to_host(bus);
1048         struct pnv_phb *phb = hose->private_data;
1049         struct pnv_ioda_pe *pe;
1050         int pe_num = IODA_INVALID_PE;
1051
1052         /* Check if PE is determined by M64 */
1053         if (phb->pick_m64_pe)
1054                 pe_num = phb->pick_m64_pe(phb, bus, all);
1055
1056         /* The PE number isn't pinned by M64 */
1057         if (pe_num == IODA_INVALID_PE)
1058                 pe_num = pnv_ioda_alloc_pe(phb);
1059
1060         if (pe_num == IODA_INVALID_PE) {
1061                 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1062                         __func__, pci_domain_nr(bus), bus->number);
1063                 return;
1064         }
1065
1066         pe = &phb->ioda.pe_array[pe_num];
1067         pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
1068         pe->pbus = bus;
1069         pe->pdev = NULL;
1070         pe->tce32_seg = -1;
1071         pe->mve_number = -1;
1072         pe->rid = bus->busn_res.start << 8;
1073         pe->dma_weight = 0;
1074
1075         if (all)
1076                 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1077                         bus->busn_res.start, bus->busn_res.end, pe_num);
1078         else
1079                 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1080                         bus->busn_res.start, pe_num);
1081
1082         if (pnv_ioda_configure_pe(phb, pe)) {
1083                 /* XXX What do we do here ? */
1084                 if (pe_num)
1085                         pnv_ioda_free_pe(phb, pe_num);
1086                 pe->pbus = NULL;
1087                 return;
1088         }
1089
1090         /* Associate it with all child devices */
1091         pnv_ioda_setup_same_PE(bus, pe);
1092
1093         /* Put PE to the list */
1094         list_add_tail(&pe->list, &phb->ioda.pe_list);
1095
1096         /* Account for one DMA PE if at least one DMA capable device exist
1097          * below the bridge
1098          */
1099         if (pe->dma_weight != 0) {
1100                 phb->ioda.dma_weight += pe->dma_weight;
1101                 phb->ioda.dma_pe_count++;
1102         }
1103
1104         /* Link the PE */
1105         pnv_ioda_link_pe_by_weight(phb, pe);
1106 }
1107
1108 static void pnv_ioda_setup_PEs(struct pci_bus *bus)
1109 {
1110         struct pci_dev *dev;
1111
1112         pnv_ioda_setup_bus_PE(bus, 0);
1113
1114         list_for_each_entry(dev, &bus->devices, bus_list) {
1115                 if (dev->subordinate) {
1116                         if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1117                                 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1118                         else
1119                                 pnv_ioda_setup_PEs(dev->subordinate);
1120                 }
1121         }
1122 }
1123
1124 /*
1125  * Configure PEs so that the downstream PCI buses and devices
1126  * could have their associated PE#. Unfortunately, we didn't
1127  * figure out the way to identify the PLX bridge yet. So we
1128  * simply put the PCI bus and the subordinate behind the root
1129  * port to PE# here. The game rule here is expected to be changed
1130  * as soon as we can detected PLX bridge correctly.
1131  */
1132 static void pnv_pci_ioda_setup_PEs(void)
1133 {
1134         struct pci_controller *hose, *tmp;
1135         struct pnv_phb *phb;
1136
1137         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
1138                 phb = hose->private_data;
1139
1140                 /* M64 layout might affect PE allocation */
1141                 if (phb->reserve_m64_pe)
1142                         phb->reserve_m64_pe(phb);
1143
1144                 pnv_ioda_setup_PEs(hose->bus);
1145         }
1146 }
1147
1148 #ifdef CONFIG_PCI_IOV
1149 static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1150 {
1151         struct pci_bus        *bus;
1152         struct pci_controller *hose;
1153         struct pnv_phb        *phb;
1154         struct pci_dn         *pdn;
1155         int                    i, j;
1156
1157         bus = pdev->bus;
1158         hose = pci_bus_to_host(bus);
1159         phb = hose->private_data;
1160         pdn = pci_get_pdn(pdev);
1161
1162         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1163                 for (j = 0; j < M64_PER_IOV; j++) {
1164                         if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1165                                 continue;
1166                         opal_pci_phb_mmio_enable(phb->opal_id,
1167                                 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1168                         clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1169                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1170                 }
1171
1172         return 0;
1173 }
1174
1175 static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
1176 {
1177         struct pci_bus        *bus;
1178         struct pci_controller *hose;
1179         struct pnv_phb        *phb;
1180         struct pci_dn         *pdn;
1181         unsigned int           win;
1182         struct resource       *res;
1183         int                    i, j;
1184         int64_t                rc;
1185         int                    total_vfs;
1186         resource_size_t        size, start;
1187         int                    pe_num;
1188         int                    vf_groups;
1189         int                    vf_per_group;
1190
1191         bus = pdev->bus;
1192         hose = pci_bus_to_host(bus);
1193         phb = hose->private_data;
1194         pdn = pci_get_pdn(pdev);
1195         total_vfs = pci_sriov_get_totalvfs(pdev);
1196
1197         /* Initialize the m64_wins to IODA_INVALID_M64 */
1198         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1199                 for (j = 0; j < M64_PER_IOV; j++)
1200                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1201
1202         if (pdn->m64_per_iov == M64_PER_IOV) {
1203                 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1204                 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1205                         roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1206         } else {
1207                 vf_groups = 1;
1208                 vf_per_group = 1;
1209         }
1210
1211         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1212                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1213                 if (!res->flags || !res->parent)
1214                         continue;
1215
1216                 if (!pnv_pci_is_mem_pref_64(res->flags))
1217                         continue;
1218
1219                 for (j = 0; j < vf_groups; j++) {
1220                         do {
1221                                 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1222                                                 phb->ioda.m64_bar_idx + 1, 0);
1223
1224                                 if (win >= phb->ioda.m64_bar_idx + 1)
1225                                         goto m64_failed;
1226                         } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1227
1228                         pdn->m64_wins[i][j] = win;
1229
1230                         if (pdn->m64_per_iov == M64_PER_IOV) {
1231                                 size = pci_iov_resource_size(pdev,
1232                                                         PCI_IOV_RESOURCES + i);
1233                                 size = size * vf_per_group;
1234                                 start = res->start + size * j;
1235                         } else {
1236                                 size = resource_size(res);
1237                                 start = res->start;
1238                         }
1239
1240                         /* Map the M64 here */
1241                         if (pdn->m64_per_iov == M64_PER_IOV) {
1242                                 pe_num = pdn->offset + j;
1243                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1244                                                 pe_num, OPAL_M64_WINDOW_TYPE,
1245                                                 pdn->m64_wins[i][j], 0);
1246                         }
1247
1248                         rc = opal_pci_set_phb_mem_window(phb->opal_id,
1249                                                  OPAL_M64_WINDOW_TYPE,
1250                                                  pdn->m64_wins[i][j],
1251                                                  start,
1252                                                  0, /* unused */
1253                                                  size);
1254
1255
1256                         if (rc != OPAL_SUCCESS) {
1257                                 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1258                                         win, rc);
1259                                 goto m64_failed;
1260                         }
1261
1262                         if (pdn->m64_per_iov == M64_PER_IOV)
1263                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1264                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1265                         else
1266                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1267                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
1268
1269                         if (rc != OPAL_SUCCESS) {
1270                                 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1271                                         win, rc);
1272                                 goto m64_failed;
1273                         }
1274                 }
1275         }
1276         return 0;
1277
1278 m64_failed:
1279         pnv_pci_vf_release_m64(pdev);
1280         return -EBUSY;
1281 }
1282
1283 static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1284 {
1285         struct pci_bus        *bus;
1286         struct pci_controller *hose;
1287         struct pnv_phb        *phb;
1288         struct iommu_table    *tbl;
1289         unsigned long         addr;
1290         int64_t               rc;
1291
1292         bus = dev->bus;
1293         hose = pci_bus_to_host(bus);
1294         phb = hose->private_data;
1295         tbl = pe->table_group.tables[0];
1296         addr = tbl->it_base;
1297
1298         opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1299                                    pe->pe_number << 1, 1, __pa(addr),
1300                                    0, 0x1000);
1301
1302         rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1303                                         pe->pe_number,
1304                                         (pe->pe_number << 1) + 1,
1305                                         pe->tce_bypass_base,
1306                                         0);
1307         if (rc)
1308                 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1309
1310         pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1311         if (pe->table_group.group) {
1312                 iommu_group_put(pe->table_group.group);
1313                 BUG_ON(pe->table_group.group);
1314         }
1315         iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
1316         free_pages(addr, get_order(TCE32_TABLE_SIZE));
1317 }
1318
1319 static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1320 {
1321         struct pci_bus        *bus;
1322         struct pci_controller *hose;
1323         struct pnv_phb        *phb;
1324         struct pnv_ioda_pe    *pe, *pe_n;
1325         struct pci_dn         *pdn;
1326         u16                    vf_index;
1327         int64_t                rc;
1328
1329         bus = pdev->bus;
1330         hose = pci_bus_to_host(bus);
1331         phb = hose->private_data;
1332         pdn = pci_get_pdn(pdev);
1333
1334         if (!pdev->is_physfn)
1335                 return;
1336
1337         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1338                 int   vf_group;
1339                 int   vf_per_group;
1340                 int   vf_index1;
1341
1342                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1343
1344                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1345                         for (vf_index = vf_group * vf_per_group;
1346                                 vf_index < (vf_group + 1) * vf_per_group &&
1347                                 vf_index < num_vfs;
1348                                 vf_index++)
1349                                 for (vf_index1 = vf_group * vf_per_group;
1350                                         vf_index1 < (vf_group + 1) * vf_per_group &&
1351                                         vf_index1 < num_vfs;
1352                                         vf_index1++){
1353
1354                                         rc = opal_pci_set_peltv(phb->opal_id,
1355                                                 pdn->offset + vf_index,
1356                                                 pdn->offset + vf_index1,
1357                                                 OPAL_REMOVE_PE_FROM_DOMAIN);
1358
1359                                         if (rc)
1360                                             dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1361                                                 __func__,
1362                                                 pdn->offset + vf_index1, rc);
1363                                 }
1364         }
1365
1366         list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1367                 if (pe->parent_dev != pdev)
1368                         continue;
1369
1370                 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1371
1372                 /* Remove from list */
1373                 mutex_lock(&phb->ioda.pe_list_mutex);
1374                 list_del(&pe->list);
1375                 mutex_unlock(&phb->ioda.pe_list_mutex);
1376
1377                 pnv_ioda_deconfigure_pe(phb, pe);
1378
1379                 pnv_ioda_free_pe(phb, pe->pe_number);
1380         }
1381 }
1382
1383 void pnv_pci_sriov_disable(struct pci_dev *pdev)
1384 {
1385         struct pci_bus        *bus;
1386         struct pci_controller *hose;
1387         struct pnv_phb        *phb;
1388         struct pci_dn         *pdn;
1389         struct pci_sriov      *iov;
1390         u16 num_vfs;
1391
1392         bus = pdev->bus;
1393         hose = pci_bus_to_host(bus);
1394         phb = hose->private_data;
1395         pdn = pci_get_pdn(pdev);
1396         iov = pdev->sriov;
1397         num_vfs = pdn->num_vfs;
1398
1399         /* Release VF PEs */
1400         pnv_ioda_release_vf_PE(pdev, num_vfs);
1401
1402         if (phb->type == PNV_PHB_IODA2) {
1403                 if (pdn->m64_per_iov == 1)
1404                         pnv_pci_vf_resource_shift(pdev, -pdn->offset);
1405
1406                 /* Release M64 windows */
1407                 pnv_pci_vf_release_m64(pdev);
1408
1409                 /* Release PE numbers */
1410                 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1411                 pdn->offset = 0;
1412         }
1413 }
1414
1415 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1416                                        struct pnv_ioda_pe *pe);
1417 static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1418 {
1419         struct pci_bus        *bus;
1420         struct pci_controller *hose;
1421         struct pnv_phb        *phb;
1422         struct pnv_ioda_pe    *pe;
1423         int                    pe_num;
1424         u16                    vf_index;
1425         struct pci_dn         *pdn;
1426         int64_t                rc;
1427
1428         bus = pdev->bus;
1429         hose = pci_bus_to_host(bus);
1430         phb = hose->private_data;
1431         pdn = pci_get_pdn(pdev);
1432
1433         if (!pdev->is_physfn)
1434                 return;
1435
1436         /* Reserve PE for each VF */
1437         for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1438                 pe_num = pdn->offset + vf_index;
1439
1440                 pe = &phb->ioda.pe_array[pe_num];
1441                 pe->pe_number = pe_num;
1442                 pe->phb = phb;
1443                 pe->flags = PNV_IODA_PE_VF;
1444                 pe->pbus = NULL;
1445                 pe->parent_dev = pdev;
1446                 pe->tce32_seg = -1;
1447                 pe->mve_number = -1;
1448                 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1449                            pci_iov_virtfn_devfn(pdev, vf_index);
1450
1451                 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1452                         hose->global_number, pdev->bus->number,
1453                         PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1454                         PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1455
1456                 if (pnv_ioda_configure_pe(phb, pe)) {
1457                         /* XXX What do we do here ? */
1458                         if (pe_num)
1459                                 pnv_ioda_free_pe(phb, pe_num);
1460                         pe->pdev = NULL;
1461                         continue;
1462                 }
1463
1464                 /* Put PE to the list */
1465                 mutex_lock(&phb->ioda.pe_list_mutex);
1466                 list_add_tail(&pe->list, &phb->ioda.pe_list);
1467                 mutex_unlock(&phb->ioda.pe_list_mutex);
1468
1469                 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1470         }
1471
1472         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1473                 int   vf_group;
1474                 int   vf_per_group;
1475                 int   vf_index1;
1476
1477                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1478
1479                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1480                         for (vf_index = vf_group * vf_per_group;
1481                              vf_index < (vf_group + 1) * vf_per_group &&
1482                              vf_index < num_vfs;
1483                              vf_index++) {
1484                                 for (vf_index1 = vf_group * vf_per_group;
1485                                      vf_index1 < (vf_group + 1) * vf_per_group &&
1486                                      vf_index1 < num_vfs;
1487                                      vf_index1++) {
1488
1489                                         rc = opal_pci_set_peltv(phb->opal_id,
1490                                                 pdn->offset + vf_index,
1491                                                 pdn->offset + vf_index1,
1492                                                 OPAL_ADD_PE_TO_DOMAIN);
1493
1494                                         if (rc)
1495                                             dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1496                                                 __func__,
1497                                                 pdn->offset + vf_index1, rc);
1498                                 }
1499                         }
1500                 }
1501         }
1502 }
1503
1504 int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1505 {
1506         struct pci_bus        *bus;
1507         struct pci_controller *hose;
1508         struct pnv_phb        *phb;
1509         struct pci_dn         *pdn;
1510         int                    ret;
1511
1512         bus = pdev->bus;
1513         hose = pci_bus_to_host(bus);
1514         phb = hose->private_data;
1515         pdn = pci_get_pdn(pdev);
1516
1517         if (phb->type == PNV_PHB_IODA2) {
1518                 /* Calculate available PE for required VFs */
1519                 mutex_lock(&phb->ioda.pe_alloc_mutex);
1520                 pdn->offset = bitmap_find_next_zero_area(
1521                         phb->ioda.pe_alloc, phb->ioda.total_pe,
1522                         0, num_vfs, 0);
1523                 if (pdn->offset >= phb->ioda.total_pe) {
1524                         mutex_unlock(&phb->ioda.pe_alloc_mutex);
1525                         dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1526                         pdn->offset = 0;
1527                         return -EBUSY;
1528                 }
1529                 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1530                 pdn->num_vfs = num_vfs;
1531                 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1532
1533                 /* Assign M64 window accordingly */
1534                 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
1535                 if (ret) {
1536                         dev_info(&pdev->dev, "Not enough M64 window resources\n");
1537                         goto m64_failed;
1538                 }
1539
1540                 /*
1541                  * When using one M64 BAR to map one IOV BAR, we need to shift
1542                  * the IOV BAR according to the PE# allocated to the VFs.
1543                  * Otherwise, the PE# for the VF will conflict with others.
1544                  */
1545                 if (pdn->m64_per_iov == 1) {
1546                         ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1547                         if (ret)
1548                                 goto m64_failed;
1549                 }
1550         }
1551
1552         /* Setup VF PEs */
1553         pnv_ioda_setup_vf_PE(pdev, num_vfs);
1554
1555         return 0;
1556
1557 m64_failed:
1558         bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1559         pdn->offset = 0;
1560
1561         return ret;
1562 }
1563
1564 int pcibios_sriov_disable(struct pci_dev *pdev)
1565 {
1566         pnv_pci_sriov_disable(pdev);
1567
1568         /* Release PCI data */
1569         remove_dev_pci_data(pdev);
1570         return 0;
1571 }
1572
1573 int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1574 {
1575         /* Allocate PCI data */
1576         add_dev_pci_data(pdev);
1577
1578         pnv_pci_sriov_enable(pdev, num_vfs);
1579         return 0;
1580 }
1581 #endif /* CONFIG_PCI_IOV */
1582
1583 static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
1584 {
1585         struct pci_dn *pdn = pci_get_pdn(pdev);
1586         struct pnv_ioda_pe *pe;
1587
1588         /*
1589          * The function can be called while the PE#
1590          * hasn't been assigned. Do nothing for the
1591          * case.
1592          */
1593         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1594                 return;
1595
1596         pe = &phb->ioda.pe_array[pdn->pe_number];
1597         WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1598         set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1599         /*
1600          * Note: iommu_add_device() will fail here as
1601          * for physical PE: the device is already added by now;
1602          * for virtual PE: sysfs entries are not ready yet and
1603          * tce_iommu_bus_notifier will add the device to a group later.
1604          */
1605 }
1606
1607 static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
1608 {
1609         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1610         struct pnv_phb *phb = hose->private_data;
1611         struct pci_dn *pdn = pci_get_pdn(pdev);
1612         struct pnv_ioda_pe *pe;
1613         uint64_t top;
1614         bool bypass = false;
1615
1616         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1617                 return -ENODEV;;
1618
1619         pe = &phb->ioda.pe_array[pdn->pe_number];
1620         if (pe->tce_bypass_enabled) {
1621                 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1622                 bypass = (dma_mask >= top);
1623         }
1624
1625         if (bypass) {
1626                 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1627                 set_dma_ops(&pdev->dev, &dma_direct_ops);
1628                 set_dma_offset(&pdev->dev, pe->tce_bypass_base);
1629         } else {
1630                 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1631                 set_dma_ops(&pdev->dev, &dma_iommu_ops);
1632                 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1633         }
1634         *pdev->dev.dma_mask = dma_mask;
1635         return 0;
1636 }
1637
1638 static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1639                                               struct pci_dev *pdev)
1640 {
1641         struct pci_dn *pdn = pci_get_pdn(pdev);
1642         struct pnv_ioda_pe *pe;
1643         u64 end, mask;
1644
1645         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1646                 return 0;
1647
1648         pe = &phb->ioda.pe_array[pdn->pe_number];
1649         if (!pe->tce_bypass_enabled)
1650                 return __dma_get_required_mask(&pdev->dev);
1651
1652
1653         end = pe->tce_bypass_base + memblock_end_of_DRAM();
1654         mask = 1ULL << (fls64(end) - 1);
1655         mask += mask - 1;
1656
1657         return mask;
1658 }
1659
1660 static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
1661                                    struct pci_bus *bus)
1662 {
1663         struct pci_dev *dev;
1664
1665         list_for_each_entry(dev, &bus->devices, bus_list) {
1666                 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
1667                 iommu_add_device(&dev->dev);
1668
1669                 if (dev->subordinate)
1670                         pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1671         }
1672 }
1673
1674 static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1675                 unsigned long index, unsigned long npages, bool rm)
1676 {
1677         struct iommu_table_group_link *tgl = list_first_entry_or_null(
1678                         &tbl->it_group_list, struct iommu_table_group_link,
1679                         next);
1680         struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1681                         struct pnv_ioda_pe, table_group);
1682         __be64 __iomem *invalidate = rm ?
1683                 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1684                 pe->phb->ioda.tce_inval_reg;
1685         unsigned long start, end, inc;
1686         const unsigned shift = tbl->it_page_shift;
1687
1688         start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1689         end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1690                         npages - 1);
1691
1692         /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1693         if (tbl->it_busno) {
1694                 start <<= shift;
1695                 end <<= shift;
1696                 inc = 128ull << shift;
1697                 start |= tbl->it_busno;
1698                 end |= tbl->it_busno;
1699         } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1700                 /* p7ioc-style invalidation, 2 TCEs per write */
1701                 start |= (1ull << 63);
1702                 end |= (1ull << 63);
1703                 inc = 16;
1704         } else {
1705                 /* Default (older HW) */
1706                 inc = 128;
1707         }
1708
1709         end |= inc - 1; /* round up end to be different than start */
1710
1711         mb(); /* Ensure above stores are visible */
1712         while (start <= end) {
1713                 if (rm)
1714                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1715                 else
1716                         __raw_writeq(cpu_to_be64(start), invalidate);
1717                 start += inc;
1718         }
1719
1720         /*
1721          * The iommu layer will do another mb() for us on build()
1722          * and we don't care on free()
1723          */
1724 }
1725
1726 static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1727                 long npages, unsigned long uaddr,
1728                 enum dma_data_direction direction,
1729                 struct dma_attrs *attrs)
1730 {
1731         int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1732                         attrs);
1733
1734         if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1735                 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1736
1737         return ret;
1738 }
1739
1740 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1741                 long npages)
1742 {
1743         pnv_tce_free(tbl, index, npages);
1744
1745         if (tbl->it_type & TCE_PCI_SWINV_FREE)
1746                 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1747 }
1748
1749 static struct iommu_table_ops pnv_ioda1_iommu_ops = {
1750         .set = pnv_ioda1_tce_build,
1751         .clear = pnv_ioda1_tce_free,
1752         .get = pnv_tce_get,
1753 };
1754
1755 static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1756 {
1757         /* 01xb - invalidate TCEs that match the specified PE# */
1758         unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1759         struct pnv_phb *phb = pe->phb;
1760
1761         if (!phb->ioda.tce_inval_reg)
1762                 return;
1763
1764         mb(); /* Ensure above stores are visible */
1765         __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1766 }
1767
1768 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1769                 unsigned long index, unsigned long npages, bool rm)
1770 {
1771         struct iommu_table_group_link *tgl = list_first_entry_or_null(
1772                         &tbl->it_group_list, struct iommu_table_group_link,
1773                         next);
1774         struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1775                         struct pnv_ioda_pe, table_group);
1776         unsigned long start, end, inc;
1777         __be64 __iomem *invalidate = rm ?
1778                 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1779                 pe->phb->ioda.tce_inval_reg;
1780         const unsigned shift = tbl->it_page_shift;
1781
1782         /* We'll invalidate DMA address in PE scope */
1783         start = 0x2ull << 60;
1784         start |= (pe->pe_number & 0xFF);
1785         end = start;
1786
1787         /* Figure out the start, end and step */
1788         start |= (index << shift);
1789         end |= ((index + npages - 1) << shift);
1790         inc = (0x1ull << shift);
1791         mb();
1792
1793         while (start <= end) {
1794                 if (rm)
1795                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1796                 else
1797                         __raw_writeq(cpu_to_be64(start), invalidate);
1798                 start += inc;
1799         }
1800 }
1801
1802 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1803                 long npages, unsigned long uaddr,
1804                 enum dma_data_direction direction,
1805                 struct dma_attrs *attrs)
1806 {
1807         int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1808                         attrs);
1809
1810         if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1811                 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1812
1813         return ret;
1814 }
1815
1816 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1817                 long npages)
1818 {
1819         pnv_tce_free(tbl, index, npages);
1820
1821         if (tbl->it_type & TCE_PCI_SWINV_FREE)
1822                 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1823 }
1824
1825 static struct iommu_table_ops pnv_ioda2_iommu_ops = {
1826         .set = pnv_ioda2_tce_build,
1827         .clear = pnv_ioda2_tce_free,
1828         .get = pnv_tce_get,
1829 };
1830
1831 static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1832                                       struct pnv_ioda_pe *pe, unsigned int base,
1833                                       unsigned int segs)
1834 {
1835
1836         struct page *tce_mem = NULL;
1837         struct iommu_table *tbl;
1838         unsigned int i;
1839         int64_t rc;
1840         void *addr;
1841
1842         /* XXX FIXME: Handle 64-bit only DMA devices */
1843         /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1844         /* XXX FIXME: Allocate multi-level tables on PHB3 */
1845
1846         /* We shouldn't already have a 32-bit DMA associated */
1847         if (WARN_ON(pe->tce32_seg >= 0))
1848                 return;
1849
1850         tbl = pnv_pci_table_alloc(phb->hose->node);
1851         iommu_register_group(&pe->table_group, phb->hose->global_number,
1852                         pe->pe_number);
1853         pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
1854
1855         /* Grab a 32-bit TCE table */
1856         pe->tce32_seg = base;
1857         pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1858                 (base << 28), ((base + segs) << 28) - 1);
1859
1860         /* XXX Currently, we allocate one big contiguous table for the
1861          * TCEs. We only really need one chunk per 256M of TCE space
1862          * (ie per segment) but that's an optimization for later, it
1863          * requires some added smarts with our get/put_tce implementation
1864          */
1865         tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1866                                    get_order(TCE32_TABLE_SIZE * segs));
1867         if (!tce_mem) {
1868                 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1869                 goto fail;
1870         }
1871         addr = page_address(tce_mem);
1872         memset(addr, 0, TCE32_TABLE_SIZE * segs);
1873
1874         /* Configure HW */
1875         for (i = 0; i < segs; i++) {
1876                 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1877                                               pe->pe_number,
1878                                               base + i, 1,
1879                                               __pa(addr) + TCE32_TABLE_SIZE * i,
1880                                               TCE32_TABLE_SIZE, 0x1000);
1881                 if (rc) {
1882                         pe_err(pe, " Failed to configure 32-bit TCE table,"
1883                                " err %ld\n", rc);
1884                         goto fail;
1885                 }
1886         }
1887
1888         /* Setup linux iommu table */
1889         pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
1890                                   base << 28, IOMMU_PAGE_SHIFT_4K);
1891
1892         /* OPAL variant of P7IOC SW invalidated TCEs */
1893         if (phb->ioda.tce_inval_reg)
1894                 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1895                                  TCE_PCI_SWINV_FREE   |
1896                                  TCE_PCI_SWINV_PAIR);
1897
1898         tbl->it_ops = &pnv_ioda1_iommu_ops;
1899         iommu_init_table(tbl, phb->hose->node);
1900
1901         if (pe->flags & PNV_IODA_PE_DEV) {
1902                 /*
1903                  * Setting table base here only for carrying iommu_group
1904                  * further down to let iommu_add_device() do the job.
1905                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1906                  */
1907                 set_iommu_table_base(&pe->pdev->dev, tbl);
1908                 iommu_add_device(&pe->pdev->dev);
1909         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
1910                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
1911
1912         return;
1913  fail:
1914         /* XXX Failure: Try to fallback to 64-bit only ? */
1915         if (pe->tce32_seg >= 0)
1916                 pe->tce32_seg = -1;
1917         if (tce_mem)
1918                 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
1919         if (tbl) {
1920                 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1921                 iommu_free_table(tbl, "pnv");
1922         }
1923 }
1924
1925 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
1926 {
1927         uint16_t window_id = (pe->pe_number << 1 ) + 1;
1928         int64_t rc;
1929
1930         pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
1931         if (enable) {
1932                 phys_addr_t top = memblock_end_of_DRAM();
1933
1934                 top = roundup_pow_of_two(top);
1935                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1936                                                      pe->pe_number,
1937                                                      window_id,
1938                                                      pe->tce_bypass_base,
1939                                                      top);
1940         } else {
1941                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1942                                                      pe->pe_number,
1943                                                      window_id,
1944                                                      pe->tce_bypass_base,
1945                                                      0);
1946         }
1947         if (rc)
1948                 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
1949         else
1950                 pe->tce_bypass_enabled = enable;
1951 }
1952
1953 #ifdef CONFIG_IOMMU_API
1954 static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
1955 {
1956         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1957                                                 table_group);
1958
1959         iommu_take_ownership(table_group->tables[0]);
1960         pnv_pci_ioda2_set_bypass(pe, false);
1961 }
1962
1963 static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
1964 {
1965         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1966                                                 table_group);
1967
1968         iommu_release_ownership(table_group->tables[0]);
1969         pnv_pci_ioda2_set_bypass(pe, true);
1970 }
1971
1972 static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
1973         .take_ownership = pnv_ioda2_take_ownership,
1974         .release_ownership = pnv_ioda2_release_ownership,
1975 };
1976 #endif
1977
1978 static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
1979 {
1980         const __be64 *swinvp;
1981
1982         /* OPAL variant of PHB3 invalidated TCEs */
1983         swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
1984         if (!swinvp)
1985                 return;
1986
1987         phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
1988         phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
1989 }
1990
1991 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1992                                        struct pnv_ioda_pe *pe)
1993 {
1994         struct page *tce_mem = NULL;
1995         void *addr;
1996         struct iommu_table *tbl;
1997         unsigned int tce_table_size, end;
1998         int64_t rc;
1999
2000         /* We shouldn't already have a 32-bit DMA associated */
2001         if (WARN_ON(pe->tce32_seg >= 0))
2002                 return;
2003
2004         /* TVE #1 is selected by PCI address bit 59 */
2005         pe->tce_bypass_base = 1ull << 59;
2006
2007         tbl = pnv_pci_table_alloc(phb->hose->node);
2008         iommu_register_group(&pe->table_group, phb->hose->global_number,
2009                         pe->pe_number);
2010         pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
2011
2012         /* The PE will reserve all possible 32-bits space */
2013         pe->tce32_seg = 0;
2014         end = (1 << ilog2(phb->ioda.m32_pci_base));
2015         tce_table_size = (end / 0x1000) * 8;
2016         pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
2017                 end);
2018
2019         /* Allocate TCE table */
2020         tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
2021                                    get_order(tce_table_size));
2022         if (!tce_mem) {
2023                 pe_err(pe, "Failed to allocate a 32-bit TCE memory\n");
2024                 goto fail;
2025         }
2026         addr = page_address(tce_mem);
2027         memset(addr, 0, tce_table_size);
2028
2029         /*
2030          * Map TCE table through TVT. The TVE index is the PE number
2031          * shifted by 1 bit for 32-bits DMA space.
2032          */
2033         rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2034                                         pe->pe_number << 1, 1, __pa(addr),
2035                                         tce_table_size, 0x1000);
2036         if (rc) {
2037                 pe_err(pe, "Failed to configure 32-bit TCE table,"
2038                        " err %ld\n", rc);
2039                 goto fail;
2040         }
2041
2042         pnv_pci_ioda2_tce_invalidate_entire(pe);
2043
2044         /* Setup linux iommu table */
2045         pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0,
2046                         IOMMU_PAGE_SHIFT_4K);
2047
2048         /* OPAL variant of PHB3 invalidated TCEs */
2049         if (phb->ioda.tce_inval_reg)
2050                 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2051
2052         tbl->it_ops = &pnv_ioda2_iommu_ops;
2053         iommu_init_table(tbl, phb->hose->node);
2054 #ifdef CONFIG_IOMMU_API
2055         pe->table_group.ops = &pnv_pci_ioda2_ops;
2056 #endif
2057
2058         if (pe->flags & PNV_IODA_PE_DEV) {
2059                 /*
2060                  * Setting table base here only for carrying iommu_group
2061                  * further down to let iommu_add_device() do the job.
2062                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2063                  */
2064                 set_iommu_table_base(&pe->pdev->dev, tbl);
2065                 iommu_add_device(&pe->pdev->dev);
2066         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
2067                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
2068
2069         /* Also create a bypass window */
2070         if (!pnv_iommu_bypass_disabled)
2071                 pnv_pci_ioda2_set_bypass(pe, true);
2072
2073         return;
2074 fail:
2075         if (pe->tce32_seg >= 0)
2076                 pe->tce32_seg = -1;
2077         if (tce_mem)
2078                 __free_pages(tce_mem, get_order(tce_table_size));
2079         if (tbl) {
2080                 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
2081                 iommu_free_table(tbl, "pnv");
2082         }
2083 }
2084
2085 static void pnv_ioda_setup_dma(struct pnv_phb *phb)
2086 {
2087         struct pci_controller *hose = phb->hose;
2088         unsigned int residual, remaining, segs, tw, base;
2089         struct pnv_ioda_pe *pe;
2090
2091         /* If we have more PE# than segments available, hand out one
2092          * per PE until we run out and let the rest fail. If not,
2093          * then we assign at least one segment per PE, plus more based
2094          * on the amount of devices under that PE
2095          */
2096         if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2097                 residual = 0;
2098         else
2099                 residual = phb->ioda.tce32_count -
2100                         phb->ioda.dma_pe_count;
2101
2102         pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2103                 hose->global_number, phb->ioda.tce32_count);
2104         pr_info("PCI: %d PE# for a total weight of %d\n",
2105                 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2106
2107         pnv_pci_ioda_setup_opal_tce_kill(phb);
2108
2109         /* Walk our PE list and configure their DMA segments, hand them
2110          * out one base segment plus any residual segments based on
2111          * weight
2112          */
2113         remaining = phb->ioda.tce32_count;
2114         tw = phb->ioda.dma_weight;
2115         base = 0;
2116         list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
2117                 if (!pe->dma_weight)
2118                         continue;
2119                 if (!remaining) {
2120                         pe_warn(pe, "No DMA32 resources available\n");
2121                         continue;
2122                 }
2123                 segs = 1;
2124                 if (residual) {
2125                         segs += ((pe->dma_weight * residual)  + (tw / 2)) / tw;
2126                         if (segs > remaining)
2127                                 segs = remaining;
2128                 }
2129
2130                 /*
2131                  * For IODA2 compliant PHB3, we needn't care about the weight.
2132                  * The all available 32-bits DMA space will be assigned to
2133                  * the specific PE.
2134                  */
2135                 if (phb->type == PNV_PHB_IODA1) {
2136                         pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2137                                 pe->dma_weight, segs);
2138                         pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2139                 } else {
2140                         pe_info(pe, "Assign DMA32 space\n");
2141                         segs = 0;
2142                         pnv_pci_ioda2_setup_dma_pe(phb, pe);
2143                 }
2144
2145                 remaining -= segs;
2146                 base += segs;
2147         }
2148 }
2149
2150 #ifdef CONFIG_PCI_MSI
2151 static void pnv_ioda2_msi_eoi(struct irq_data *d)
2152 {
2153         unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2154         struct irq_chip *chip = irq_data_get_irq_chip(d);
2155         struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2156                                            ioda.irq_chip);
2157         int64_t rc;
2158
2159         rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2160         WARN_ON_ONCE(rc);
2161
2162         icp_native_eoi(d);
2163 }
2164
2165
2166 static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2167 {
2168         struct irq_data *idata;
2169         struct irq_chip *ichip;
2170
2171         if (phb->type != PNV_PHB_IODA2)
2172                 return;
2173
2174         if (!phb->ioda.irq_chip_init) {
2175                 /*
2176                  * First time we setup an MSI IRQ, we need to setup the
2177                  * corresponding IRQ chip to route correctly.
2178                  */
2179                 idata = irq_get_irq_data(virq);
2180                 ichip = irq_data_get_irq_chip(idata);
2181                 phb->ioda.irq_chip_init = 1;
2182                 phb->ioda.irq_chip = *ichip;
2183                 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2184         }
2185         irq_set_chip(virq, &phb->ioda.irq_chip);
2186 }
2187
2188 #ifdef CONFIG_CXL_BASE
2189
2190 struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
2191 {
2192         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2193
2194         return of_node_get(hose->dn);
2195 }
2196 EXPORT_SYMBOL(pnv_pci_get_phb_node);
2197
2198 int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
2199 {
2200         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2201         struct pnv_phb *phb = hose->private_data;
2202         struct pnv_ioda_pe *pe;
2203         int rc;
2204
2205         pe = pnv_ioda_get_pe(dev);
2206         if (!pe)
2207                 return -ENODEV;
2208
2209         pe_info(pe, "Switching PHB to CXL\n");
2210
2211         rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
2212         if (rc)
2213                 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2214
2215         return rc;
2216 }
2217 EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
2218
2219 /* Find PHB for cxl dev and allocate MSI hwirqs?
2220  * Returns the absolute hardware IRQ number
2221  */
2222 int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2223 {
2224         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2225         struct pnv_phb *phb = hose->private_data;
2226         int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2227
2228         if (hwirq < 0) {
2229                 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2230                 return -ENOSPC;
2231         }
2232
2233         return phb->msi_base + hwirq;
2234 }
2235 EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2236
2237 void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2238 {
2239         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2240         struct pnv_phb *phb = hose->private_data;
2241
2242         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2243 }
2244 EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2245
2246 void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2247                                   struct pci_dev *dev)
2248 {
2249         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2250         struct pnv_phb *phb = hose->private_data;
2251         int i, hwirq;
2252
2253         for (i = 1; i < CXL_IRQ_RANGES; i++) {
2254                 if (!irqs->range[i])
2255                         continue;
2256                 pr_devel("cxl release irq range 0x%x: offset: 0x%lx  limit: %ld\n",
2257                          i, irqs->offset[i],
2258                          irqs->range[i]);
2259                 hwirq = irqs->offset[i] - phb->msi_base;
2260                 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2261                                        irqs->range[i]);
2262         }
2263 }
2264 EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2265
2266 int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2267                                struct pci_dev *dev, int num)
2268 {
2269         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2270         struct pnv_phb *phb = hose->private_data;
2271         int i, hwirq, try;
2272
2273         memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2274
2275         /* 0 is reserved for the multiplexed PSL DSI interrupt */
2276         for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2277                 try = num;
2278                 while (try) {
2279                         hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2280                         if (hwirq >= 0)
2281                                 break;
2282                         try /= 2;
2283                 }
2284                 if (!try)
2285                         goto fail;
2286
2287                 irqs->offset[i] = phb->msi_base + hwirq;
2288                 irqs->range[i] = try;
2289                 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx  limit: %li\n",
2290                          i, irqs->offset[i], irqs->range[i]);
2291                 num -= try;
2292         }
2293         if (num)
2294                 goto fail;
2295
2296         return 0;
2297 fail:
2298         pnv_cxl_release_hwirq_ranges(irqs, dev);
2299         return -ENOSPC;
2300 }
2301 EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2302
2303 int pnv_cxl_get_irq_count(struct pci_dev *dev)
2304 {
2305         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2306         struct pnv_phb *phb = hose->private_data;
2307
2308         return phb->msi_bmp.irq_count;
2309 }
2310 EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2311
2312 int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2313                            unsigned int virq)
2314 {
2315         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2316         struct pnv_phb *phb = hose->private_data;
2317         unsigned int xive_num = hwirq - phb->msi_base;
2318         struct pnv_ioda_pe *pe;
2319         int rc;
2320
2321         if (!(pe = pnv_ioda_get_pe(dev)))
2322                 return -ENODEV;
2323
2324         /* Assign XIVE to PE */
2325         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2326         if (rc) {
2327                 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2328                         "hwirq 0x%x XIVE 0x%x PE\n",
2329                         pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2330                 return -EIO;
2331         }
2332         set_msi_irq_chip(phb, virq);
2333
2334         return 0;
2335 }
2336 EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2337 #endif
2338
2339 static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
2340                                   unsigned int hwirq, unsigned int virq,
2341                                   unsigned int is_64, struct msi_msg *msg)
2342 {
2343         struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2344         unsigned int xive_num = hwirq - phb->msi_base;
2345         __be32 data;
2346         int rc;
2347
2348         /* No PE assigned ? bail out ... no MSI for you ! */
2349         if (pe == NULL)
2350                 return -ENXIO;
2351
2352         /* Check if we have an MVE */
2353         if (pe->mve_number < 0)
2354                 return -ENXIO;
2355
2356         /* Force 32-bit MSI on some broken devices */
2357         if (dev->no_64bit_msi)
2358                 is_64 = 0;
2359
2360         /* Assign XIVE to PE */
2361         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2362         if (rc) {
2363                 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2364                         pci_name(dev), rc, xive_num);
2365                 return -EIO;
2366         }
2367
2368         if (is_64) {
2369                 __be64 addr64;
2370
2371                 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2372                                      &addr64, &data);
2373                 if (rc) {
2374                         pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2375                                 pci_name(dev), rc);
2376                         return -EIO;
2377                 }
2378                 msg->address_hi = be64_to_cpu(addr64) >> 32;
2379                 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
2380         } else {
2381                 __be32 addr32;
2382
2383                 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2384                                      &addr32, &data);
2385                 if (rc) {
2386                         pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2387                                 pci_name(dev), rc);
2388                         return -EIO;
2389                 }
2390                 msg->address_hi = 0;
2391                 msg->address_lo = be32_to_cpu(addr32);
2392         }
2393         msg->data = be32_to_cpu(data);
2394
2395         set_msi_irq_chip(phb, virq);
2396
2397         pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2398                  " address=%x_%08x data=%x PE# %d\n",
2399                  pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2400                  msg->address_hi, msg->address_lo, data, pe->pe_number);
2401
2402         return 0;
2403 }
2404
2405 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2406 {
2407         unsigned int count;
2408         const __be32 *prop = of_get_property(phb->hose->dn,
2409                                              "ibm,opal-msi-ranges", NULL);
2410         if (!prop) {
2411                 /* BML Fallback */
2412                 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2413         }
2414         if (!prop)
2415                 return;
2416
2417         phb->msi_base = be32_to_cpup(prop);
2418         count = be32_to_cpup(prop + 1);
2419         if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
2420                 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2421                        phb->hose->global_number);
2422                 return;
2423         }
2424
2425         phb->msi_setup = pnv_pci_ioda_msi_setup;
2426         phb->msi32_support = 1;
2427         pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
2428                 count, phb->msi_base);
2429 }
2430 #else
2431 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2432 #endif /* CONFIG_PCI_MSI */
2433
2434 #ifdef CONFIG_PCI_IOV
2435 static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2436 {
2437         struct pci_controller *hose;
2438         struct pnv_phb *phb;
2439         struct resource *res;
2440         int i;
2441         resource_size_t size;
2442         struct pci_dn *pdn;
2443         int mul, total_vfs;
2444
2445         if (!pdev->is_physfn || pdev->is_added)
2446                 return;
2447
2448         hose = pci_bus_to_host(pdev->bus);
2449         phb = hose->private_data;
2450
2451         pdn = pci_get_pdn(pdev);
2452         pdn->vfs_expanded = 0;
2453
2454         total_vfs = pci_sriov_get_totalvfs(pdev);
2455         pdn->m64_per_iov = 1;
2456         mul = phb->ioda.total_pe;
2457
2458         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2459                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2460                 if (!res->flags || res->parent)
2461                         continue;
2462                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2463                         dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2464                                  i, res);
2465                         continue;
2466                 }
2467
2468                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2469
2470                 /* bigger than 64M */
2471                 if (size > (1 << 26)) {
2472                         dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2473                                  i, res);
2474                         pdn->m64_per_iov = M64_PER_IOV;
2475                         mul = roundup_pow_of_two(total_vfs);
2476                         break;
2477                 }
2478         }
2479
2480         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2481                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2482                 if (!res->flags || res->parent)
2483                         continue;
2484                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2485                         dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2486                                  i, res);
2487                         continue;
2488                 }
2489
2490                 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2491                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2492                 res->end = res->start + size * mul - 1;
2493                 dev_dbg(&pdev->dev, "                       %pR\n", res);
2494                 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
2495                          i, res, mul);
2496         }
2497         pdn->vfs_expanded = mul;
2498 }
2499 #endif /* CONFIG_PCI_IOV */
2500
2501 /*
2502  * This function is supposed to be called on basis of PE from top
2503  * to bottom style. So the the I/O or MMIO segment assigned to
2504  * parent PE could be overrided by its child PEs if necessary.
2505  */
2506 static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2507                                   struct pnv_ioda_pe *pe)
2508 {
2509         struct pnv_phb *phb = hose->private_data;
2510         struct pci_bus_region region;
2511         struct resource *res;
2512         int i, index;
2513         int rc;
2514
2515         /*
2516          * NOTE: We only care PCI bus based PE for now. For PCI
2517          * device based PE, for example SRIOV sensitive VF should
2518          * be figured out later.
2519          */
2520         BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2521
2522         pci_bus_for_each_resource(pe->pbus, res, i) {
2523                 if (!res || !res->flags ||
2524                     res->start > res->end)
2525                         continue;
2526
2527                 if (res->flags & IORESOURCE_IO) {
2528                         region.start = res->start - phb->ioda.io_pci_base;
2529                         region.end   = res->end - phb->ioda.io_pci_base;
2530                         index = region.start / phb->ioda.io_segsize;
2531
2532                         while (index < phb->ioda.total_pe &&
2533                                region.start <= region.end) {
2534                                 phb->ioda.io_segmap[index] = pe->pe_number;
2535                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2536                                         pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2537                                 if (rc != OPAL_SUCCESS) {
2538                                         pr_err("%s: OPAL error %d when mapping IO "
2539                                                "segment #%d to PE#%d\n",
2540                                                __func__, rc, index, pe->pe_number);
2541                                         break;
2542                                 }
2543
2544                                 region.start += phb->ioda.io_segsize;
2545                                 index++;
2546                         }
2547                 } else if ((res->flags & IORESOURCE_MEM) &&
2548                            !pnv_pci_is_mem_pref_64(res->flags)) {
2549                         region.start = res->start -
2550                                        hose->mem_offset[0] -
2551                                        phb->ioda.m32_pci_base;
2552                         region.end   = res->end -
2553                                        hose->mem_offset[0] -
2554                                        phb->ioda.m32_pci_base;
2555                         index = region.start / phb->ioda.m32_segsize;
2556
2557                         while (index < phb->ioda.total_pe &&
2558                                region.start <= region.end) {
2559                                 phb->ioda.m32_segmap[index] = pe->pe_number;
2560                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2561                                         pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2562                                 if (rc != OPAL_SUCCESS) {
2563                                         pr_err("%s: OPAL error %d when mapping M32 "
2564                                                "segment#%d to PE#%d",
2565                                                __func__, rc, index, pe->pe_number);
2566                                         break;
2567                                 }
2568
2569                                 region.start += phb->ioda.m32_segsize;
2570                                 index++;
2571                         }
2572                 }
2573         }
2574 }
2575
2576 static void pnv_pci_ioda_setup_seg(void)
2577 {
2578         struct pci_controller *tmp, *hose;
2579         struct pnv_phb *phb;
2580         struct pnv_ioda_pe *pe;
2581
2582         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2583                 phb = hose->private_data;
2584                 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2585                         pnv_ioda_setup_pe_seg(hose, pe);
2586                 }
2587         }
2588 }
2589
2590 static void pnv_pci_ioda_setup_DMA(void)
2591 {
2592         struct pci_controller *hose, *tmp;
2593         struct pnv_phb *phb;
2594
2595         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2596                 pnv_ioda_setup_dma(hose->private_data);
2597
2598                 /* Mark the PHB initialization done */
2599                 phb = hose->private_data;
2600                 phb->initialized = 1;
2601         }
2602 }
2603
2604 static void pnv_pci_ioda_create_dbgfs(void)
2605 {
2606 #ifdef CONFIG_DEBUG_FS
2607         struct pci_controller *hose, *tmp;
2608         struct pnv_phb *phb;
2609         char name[16];
2610
2611         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2612                 phb = hose->private_data;
2613
2614                 sprintf(name, "PCI%04x", hose->global_number);
2615                 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2616                 if (!phb->dbgfs)
2617                         pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2618                                 __func__, hose->global_number);
2619         }
2620 #endif /* CONFIG_DEBUG_FS */
2621 }
2622
2623 static void pnv_pci_ioda_fixup(void)
2624 {
2625         pnv_pci_ioda_setup_PEs();
2626         pnv_pci_ioda_setup_seg();
2627         pnv_pci_ioda_setup_DMA();
2628
2629         pnv_pci_ioda_create_dbgfs();
2630
2631 #ifdef CONFIG_EEH
2632         eeh_init();
2633         eeh_addr_cache_build();
2634 #endif
2635 }
2636
2637 /*
2638  * Returns the alignment for I/O or memory windows for P2P
2639  * bridges. That actually depends on how PEs are segmented.
2640  * For now, we return I/O or M32 segment size for PE sensitive
2641  * P2P bridges. Otherwise, the default values (4KiB for I/O,
2642  * 1MiB for memory) will be returned.
2643  *
2644  * The current PCI bus might be put into one PE, which was
2645  * create against the parent PCI bridge. For that case, we
2646  * needn't enlarge the alignment so that we can save some
2647  * resources.
2648  */
2649 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2650                                                 unsigned long type)
2651 {
2652         struct pci_dev *bridge;
2653         struct pci_controller *hose = pci_bus_to_host(bus);
2654         struct pnv_phb *phb = hose->private_data;
2655         int num_pci_bridges = 0;
2656
2657         bridge = bus->self;
2658         while (bridge) {
2659                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2660                         num_pci_bridges++;
2661                         if (num_pci_bridges >= 2)
2662                                 return 1;
2663                 }
2664
2665                 bridge = bridge->bus->self;
2666         }
2667
2668         /* We fail back to M32 if M64 isn't supported */
2669         if (phb->ioda.m64_segsize &&
2670             pnv_pci_is_mem_pref_64(type))
2671                 return phb->ioda.m64_segsize;
2672         if (type & IORESOURCE_MEM)
2673                 return phb->ioda.m32_segsize;
2674
2675         return phb->ioda.io_segsize;
2676 }
2677
2678 #ifdef CONFIG_PCI_IOV
2679 static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2680                                                       int resno)
2681 {
2682         struct pci_dn *pdn = pci_get_pdn(pdev);
2683         resource_size_t align, iov_align;
2684
2685         iov_align = resource_size(&pdev->resource[resno]);
2686         if (iov_align)
2687                 return iov_align;
2688
2689         align = pci_iov_resource_size(pdev, resno);
2690         if (pdn->vfs_expanded)
2691                 return pdn->vfs_expanded * align;
2692
2693         return align;
2694 }
2695 #endif /* CONFIG_PCI_IOV */
2696
2697 /* Prevent enabling devices for which we couldn't properly
2698  * assign a PE
2699  */
2700 static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
2701 {
2702         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2703         struct pnv_phb *phb = hose->private_data;
2704         struct pci_dn *pdn;
2705
2706         /* The function is probably called while the PEs have
2707          * not be created yet. For example, resource reassignment
2708          * during PCI probe period. We just skip the check if
2709          * PEs isn't ready.
2710          */
2711         if (!phb->initialized)
2712                 return true;
2713
2714         pdn = pci_get_pdn(dev);
2715         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
2716                 return false;
2717
2718         return true;
2719 }
2720
2721 static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
2722                                u32 devfn)
2723 {
2724         return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
2725 }
2726
2727 static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
2728 {
2729         struct pnv_phb *phb = hose->private_data;
2730
2731         opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
2732                        OPAL_ASSERT_RESET);
2733 }
2734
2735 static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2736        .dma_dev_setup = pnv_pci_dma_dev_setup,
2737 #ifdef CONFIG_PCI_MSI
2738        .setup_msi_irqs = pnv_setup_msi_irqs,
2739        .teardown_msi_irqs = pnv_teardown_msi_irqs,
2740 #endif
2741        .enable_device_hook = pnv_pci_enable_device_hook,
2742        .window_alignment = pnv_pci_window_alignment,
2743        .reset_secondary_bus = pnv_pci_reset_secondary_bus,
2744        .dma_set_mask = pnv_pci_ioda_dma_set_mask,
2745        .shutdown = pnv_pci_ioda_shutdown,
2746 };
2747
2748 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2749                                          u64 hub_id, int ioda_type)
2750 {
2751         struct pci_controller *hose;
2752         struct pnv_phb *phb;
2753         unsigned long size, m32map_off, pemap_off, iomap_off = 0;
2754         const __be64 *prop64;
2755         const __be32 *prop32;
2756         int len;
2757         u64 phb_id;
2758         void *aux;
2759         long rc;
2760
2761         pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
2762
2763         prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2764         if (!prop64) {
2765                 pr_err("  Missing \"ibm,opal-phbid\" property !\n");
2766                 return;
2767         }
2768         phb_id = be64_to_cpup(prop64);
2769         pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
2770
2771         phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
2772
2773         /* Allocate PCI controller */
2774         phb->hose = hose = pcibios_alloc_controller(np);
2775         if (!phb->hose) {
2776                 pr_err("  Can't allocate PCI controller for %s\n",
2777                        np->full_name);
2778                 memblock_free(__pa(phb), sizeof(struct pnv_phb));
2779                 return;
2780         }
2781
2782         spin_lock_init(&phb->lock);
2783         prop32 = of_get_property(np, "bus-range", &len);
2784         if (prop32 && len == 8) {
2785                 hose->first_busno = be32_to_cpu(prop32[0]);
2786                 hose->last_busno = be32_to_cpu(prop32[1]);
2787         } else {
2788                 pr_warn("  Broken <bus-range> on %s\n", np->full_name);
2789                 hose->first_busno = 0;
2790                 hose->last_busno = 0xff;
2791         }
2792         hose->private_data = phb;
2793         phb->hub_id = hub_id;
2794         phb->opal_id = phb_id;
2795         phb->type = ioda_type;
2796         mutex_init(&phb->ioda.pe_alloc_mutex);
2797
2798         /* Detect specific models for error handling */
2799         if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2800                 phb->model = PNV_PHB_MODEL_P7IOC;
2801         else if (of_device_is_compatible(np, "ibm,power8-pciex"))
2802                 phb->model = PNV_PHB_MODEL_PHB3;
2803         else
2804                 phb->model = PNV_PHB_MODEL_UNKNOWN;
2805
2806         /* Parse 32-bit and IO ranges (if any) */
2807         pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
2808
2809         /* Get registers */
2810         phb->regs = of_iomap(np, 0);
2811         if (phb->regs == NULL)
2812                 pr_err("  Failed to map registers !\n");
2813
2814         /* Initialize more IODA stuff */
2815         phb->ioda.total_pe = 1;
2816         prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
2817         if (prop32)
2818                 phb->ioda.total_pe = be32_to_cpup(prop32);
2819         prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
2820         if (prop32)
2821                 phb->ioda.reserved_pe = be32_to_cpup(prop32);
2822
2823         /* Parse 64-bit MMIO range */
2824         pnv_ioda_parse_m64_window(phb);
2825
2826         phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
2827         /* FW Has already off top 64k of M32 space (MSI space) */
2828         phb->ioda.m32_size += 0x10000;
2829
2830         phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
2831         phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
2832         phb->ioda.io_size = hose->pci_io_size;
2833         phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
2834         phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
2835
2836         /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
2837         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
2838         m32map_off = size;
2839         size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
2840         if (phb->type == PNV_PHB_IODA1) {
2841                 iomap_off = size;
2842                 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
2843         }
2844         pemap_off = size;
2845         size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
2846         aux = memblock_virt_alloc(size, 0);
2847         phb->ioda.pe_alloc = aux;
2848         phb->ioda.m32_segmap = aux + m32map_off;
2849         if (phb->type == PNV_PHB_IODA1)
2850                 phb->ioda.io_segmap = aux + iomap_off;
2851         phb->ioda.pe_array = aux + pemap_off;
2852         set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
2853
2854         INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
2855         INIT_LIST_HEAD(&phb->ioda.pe_list);
2856         mutex_init(&phb->ioda.pe_list_mutex);
2857
2858         /* Calculate how many 32-bit TCE segments we have */
2859         phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
2860
2861 #if 0 /* We should really do that ... */
2862         rc = opal_pci_set_phb_mem_window(opal->phb_id,
2863                                          window_type,
2864                                          window_num,
2865                                          starting_real_address,
2866                                          starting_pci_address,
2867                                          segment_size);
2868 #endif
2869
2870         pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
2871                 phb->ioda.total_pe, phb->ioda.reserved_pe,
2872                 phb->ioda.m32_size, phb->ioda.m32_segsize);
2873         if (phb->ioda.m64_size)
2874                 pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
2875                         phb->ioda.m64_size, phb->ioda.m64_segsize);
2876         if (phb->ioda.io_size)
2877                 pr_info("                  IO: 0x%x [segment=0x%x]\n",
2878                         phb->ioda.io_size, phb->ioda.io_segsize);
2879
2880
2881         phb->hose->ops = &pnv_pci_ops;
2882         phb->get_pe_state = pnv_ioda_get_pe_state;
2883         phb->freeze_pe = pnv_ioda_freeze_pe;
2884         phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
2885
2886         /* Setup RID -> PE mapping function */
2887         phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
2888
2889         /* Setup TCEs */
2890         phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
2891         phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
2892
2893         /* Setup MSI support */
2894         pnv_pci_init_ioda_msis(phb);
2895
2896         /*
2897          * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
2898          * to let the PCI core do resource assignment. It's supposed
2899          * that the PCI core will do correct I/O and MMIO alignment
2900          * for the P2P bridge bars so that each PCI bus (excluding
2901          * the child P2P bridges) can form individual PE.
2902          */
2903         ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
2904         hose->controller_ops = pnv_pci_ioda_controller_ops;
2905
2906 #ifdef CONFIG_PCI_IOV
2907         ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
2908         ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
2909 #endif
2910
2911         pci_add_flags(PCI_REASSIGN_ALL_RSRC);
2912
2913         /* Reset IODA tables to a clean state */
2914         rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
2915         if (rc)
2916                 pr_warning("  OPAL Error %ld performing IODA table reset !\n", rc);
2917
2918         /* If we're running in kdump kerenl, the previous kerenl never
2919          * shutdown PCI devices correctly. We already got IODA table
2920          * cleaned out. So we have to issue PHB reset to stop all PCI
2921          * transactions from previous kerenl.
2922          */
2923         if (is_kdump_kernel()) {
2924                 pr_info("  Issue PHB reset ...\n");
2925                 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
2926                 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
2927         }
2928
2929         /* Remove M64 resource if we can't configure it successfully */
2930         if (!phb->init_m64 || phb->init_m64(phb))
2931                 hose->mem_resources[1].flags = 0;
2932 }
2933
2934 void __init pnv_pci_init_ioda2_phb(struct device_node *np)
2935 {
2936         pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
2937 }
2938
2939 void __init pnv_pci_init_ioda_hub(struct device_node *np)
2940 {
2941         struct device_node *phbn;
2942         const __be64 *prop64;
2943         u64 hub_id;
2944
2945         pr_info("Probing IODA IO-Hub %s\n", np->full_name);
2946
2947         prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
2948         if (!prop64) {
2949                 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
2950                 return;
2951         }
2952         hub_id = be64_to_cpup(prop64);
2953         pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
2954
2955         /* Count child PHBs */
2956         for_each_child_of_node(np, phbn) {
2957                 /* Look for IODA1 PHBs */
2958                 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
2959                         pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
2960         }
2961 }