arch/powerpc/platforms/powernv/pci-ioda.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #undef DEBUG
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/crash_dump.h>
  17 #include <linux/debugfs.h>
  18 #include <linux/delay.h>
  19 #include <linux/string.h>
  20 #include <linux/init.h>
  21 #include <linux/bootmem.h>
  22 #include <linux/irq.h>
  23 #include <linux/io.h>
  24 #include <linux/msi.h>
  25 #include <linux/memblock.h>
  26 #include <linux/iommu.h>
  27 #include <linux/rculist.h>
  28
  29 #include <asm/sections.h>
  30 #include <asm/io.h>
  31 #include <asm/prom.h>
  32 #include <asm/pci-bridge.h>
  33 #include <asm/machdep.h>
  34 #include <asm/msi_bitmap.h>
  35 #include <asm/ppc-pci.h>
  36 #include <asm/opal.h>
  37 #include <asm/iommu.h>
  38 #include <asm/tce.h>
  39 #include <asm/xics.h>
  40 #include <asm/debug.h>
  41 #include <asm/firmware.h>
  42 #include <asm/pnv-pci.h>
  43 #include <asm/mmzone.h>
  44
  45 #include <misc/cxl-base.h>
  46
  47 #include "powernv.h"
  48 #include "pci.h"
  49
  50 /* 256M DMA window, 4K TCE pages, 8 bytes TCE */
  51 #define TCE32_TABLE_SIZE        ((0x10000000 / 0x1000) * 8)
  52
  53 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
  54
  55 static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  56                             const char *fmt, ...)
  57 {
  58         struct va_format vaf;
  59         va_list args;
  60         char pfix[32];
  61
  62         va_start(args, fmt);
  63
  64         vaf.fmt = fmt;
  65         vaf.va = &args;
  66
  67         if (pe->flags & PNV_IODA_PE_DEV)
  68                 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  69         else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  70                 sprintf(pfix, "%04x:%02x     ",
  71                         pci_domain_nr(pe->pbus), pe->pbus->number);
  72 #ifdef CONFIG_PCI_IOV
  73         else if (pe->flags & PNV_IODA_PE_VF)
  74                 sprintf(pfix, "%04x:%02x:%2x.%d",
  75                         pci_domain_nr(pe->parent_dev->bus),
  76                         (pe->rid & 0xff00) >> 8,
  77                         PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  78 #endif /* CONFIG_PCI_IOV*/
  79
  80         printk("%spci %s: [PE# %.3d] %pV",
  81                level, pfix, pe->pe_number, &vaf);
  82
  83         va_end(args);
  84 }
  85
  86 #define pe_err(pe, fmt, ...)                                    \
  87         pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
  88 #define pe_warn(pe, fmt, ...)                                   \
  89         pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
  90 #define pe_info(pe, fmt, ...)                                   \
  91         pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
  92
  93 static bool pnv_iommu_bypass_disabled __read_mostly;
  94
  95 static int __init iommu_setup(char *str)
  96 {
  97         if (!str)
  98                 return -EINVAL;
  99
 100         while (*str) {
 101                 if (!strncmp(str, "nobypass", 8)) {
 102                         pnv_iommu_bypass_disabled = true;
 103                         pr_info("PowerNV: IOMMU bypass window disabled.\n");
 104                         break;
 105                 }
 106                 str += strcspn(str, ",");
 107                 if (*str == ',')
 108                         str++;
 109         }
 110
 111         return 0;
 112 }
 113 early_param("iommu", iommu_setup);
 114
 115 /*
 116  * stdcix is only supposed to be used in hypervisor real mode as per
 117  * the architecture spec
 118  */
 119 static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
 120 {
 121         __asm__ __volatile__("stdcix %0,0,%1"
 122                 : : "r" (val), "r" (paddr) : "memory");
 123 }
 124
 125 static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
 126 {
 127         return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
 128                 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
 129 }
 130
 131 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
 132 {
 133         if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
 134                 pr_warn("%s: Invalid PE %d on PHB#%x\n",
 135                         __func__, pe_no, phb->hose->global_number);
 136                 return;
 137         }
 138
 139         if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
 140                 pr_warn("%s: PE %d was assigned on PHB#%x\n",
 141                         __func__, pe_no, phb->hose->global_number);
 142                 return;
 143         }
 144
 145         phb->ioda.pe_array[pe_no].phb = phb;
 146         phb->ioda.pe_array[pe_no].pe_number = pe_no;
 147 }
 148
 149 static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
 150 {
 151         unsigned long pe;
 152
 153         do {
 154                 pe = find_next_zero_bit(phb->ioda.pe_alloc,
 155                                         phb->ioda.total_pe, 0);
 156                 if (pe >= phb->ioda.total_pe)
 157                         return IODA_INVALID_PE;
 158         } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
 159
 160         phb->ioda.pe_array[pe].phb = phb;
 161         phb->ioda.pe_array[pe].pe_number = pe;
 162         return pe;
 163 }
 164
 165 static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
 166 {
 167         WARN_ON(phb->ioda.pe_array[pe].pdev);
 168
 169         memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
 170         clear_bit(pe, phb->ioda.pe_alloc);
 171 }
 172
 173 /* The default M64 BAR is shared by all PEs */
 174 static int pnv_ioda2_init_m64(struct pnv_phb *phb)
 175 {
 176         const char *desc;
 177         struct resource *r;
 178         s64 rc;
 179
 180         /* Configure the default M64 BAR */
 181         rc = opal_pci_set_phb_mem_window(phb->opal_id,
 182                                          OPAL_M64_WINDOW_TYPE,
 183                                          phb->ioda.m64_bar_idx,
 184                                          phb->ioda.m64_base,
 185                                          0, /* unused */
 186                                          phb->ioda.m64_size);
 187         if (rc != OPAL_SUCCESS) {
 188                 desc = "configuring";
 189                 goto fail;
 190         }
 191
 192         /* Enable the default M64 BAR */
 193         rc = opal_pci_phb_mmio_enable(phb->opal_id,
 194                                       OPAL_M64_WINDOW_TYPE,
 195                                       phb->ioda.m64_bar_idx,
 196                                       OPAL_ENABLE_M64_SPLIT);
 197         if (rc != OPAL_SUCCESS) {
 198                 desc = "enabling";
 199                 goto fail;
 200         }
 201
 202         /* Mark the M64 BAR assigned */
 203         set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
 204
 205         /*
 206          * Strip off the segment used by the reserved PE, which is
 207          * expected to be 0 or last one of PE capabicity.
 208          */
 209         r = &phb->hose->mem_resources[1];
 210         if (phb->ioda.reserved_pe == 0)
 211                 r->start += phb->ioda.m64_segsize;
 212         else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
 213                 r->end -= phb->ioda.m64_segsize;
 214         else
 215                 pr_warn("  Cannot strip M64 segment for reserved PE#%d\n",
 216                         phb->ioda.reserved_pe);
 217
 218         return 0;
 219
 220 fail:
 221         pr_warn("  Failure %lld %s M64 BAR#%d\n",
 222                 rc, desc, phb->ioda.m64_bar_idx);
 223         opal_pci_phb_mmio_enable(phb->opal_id,
 224                                  OPAL_M64_WINDOW_TYPE,
 225                                  phb->ioda.m64_bar_idx,
 226                                  OPAL_DISABLE_M64);
 227         return -EIO;
 228 }
 229
 230 static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
 231 {
 232         resource_size_t sgsz = phb->ioda.m64_segsize;
 233         struct pci_dev *pdev;
 234         struct resource *r;
 235         int base, step, i;
 236
 237         /*
 238          * Root bus always has full M64 range and root port has
 239          * M64 range used in reality. So we're checking root port
 240          * instead of root bus.
 241          */
 242         list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
 243                 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
 244                         r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
 245                         if (!r->parent ||
 246                             !pnv_pci_is_mem_pref_64(r->flags))
 247                                 continue;
 248
 249                         base = (r->start - phb->ioda.m64_base) / sgsz;
 250                         for (step = 0; step < resource_size(r) / sgsz; step++)
 251                                 pnv_ioda_reserve_pe(phb, base + step);
 252                 }
 253         }
 254 }
 255
 256 static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
 257                                  struct pci_bus *bus, int all)
 258 {
 259         resource_size_t segsz = phb->ioda.m64_segsize;
 260         struct pci_dev *pdev;
 261         struct resource *r;
 262         struct pnv_ioda_pe *master_pe, *pe;
 263         unsigned long size, *pe_alloc;
 264         bool found;
 265         int start, i, j;
 266
 267         /* Root bus shouldn't use M64 */
 268         if (pci_is_root_bus(bus))
 269                 return IODA_INVALID_PE;
 270
 271         /* We support only one M64 window on each bus */
 272         found = false;
 273         pci_bus_for_each_resource(bus, r, i) {
 274                 if (r && r->parent &&
 275                     pnv_pci_is_mem_pref_64(r->flags)) {
 276                         found = true;
 277                         break;
 278                 }
 279         }
 280
 281         /* No M64 window found ? */
 282         if (!found)
 283                 return IODA_INVALID_PE;
 284
 285         /* Allocate bitmap */
 286         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
 287         pe_alloc = kzalloc(size, GFP_KERNEL);
 288         if (!pe_alloc) {
 289                 pr_warn("%s: Out of memory !\n",
 290                         __func__);
 291                 return IODA_INVALID_PE;
 292         }
 293
 294         /*
 295          * Figure out reserved PE numbers by the PE
 296          * the its child PEs.
 297          */
 298         start = (r->start - phb->ioda.m64_base) / segsz;
 299         for (i = 0; i < resource_size(r) / segsz; i++)
 300                 set_bit(start + i, pe_alloc);
 301
 302         if (all)
 303                 goto done;
 304
 305         /*
 306          * If the PE doesn't cover all subordinate buses,
 307          * we need subtract from reserved PEs for children.
 308          */
 309         list_for_each_entry(pdev, &bus->devices, bus_list) {
 310                 if (!pdev->subordinate)
 311                         continue;
 312
 313                 pci_bus_for_each_resource(pdev->subordinate, r, i) {
 314                         if (!r || !r->parent ||
 315                             !pnv_pci_is_mem_pref_64(r->flags))
 316                                 continue;
 317
 318                         start = (r->start - phb->ioda.m64_base) / segsz;
 319                         for (j = 0; j < resource_size(r) / segsz ; j++)
 320                                 clear_bit(start + j, pe_alloc);
 321                 }
 322         }
 323
 324         /*
 325          * the current bus might not own M64 window and that's all
 326          * contributed by its child buses. For the case, we needn't
 327          * pick M64 dependent PE#.
 328          */
 329         if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
 330                 kfree(pe_alloc);
 331                 return IODA_INVALID_PE;
 332         }
 333
 334         /*
 335          * Figure out the master PE and put all slave PEs to master
 336          * PE's list to form compound PE.
 337          */
 338 done:
 339         master_pe = NULL;
 340         i = -1;
 341         while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
 342                 phb->ioda.total_pe) {
 343                 pe = &phb->ioda.pe_array[i];
 344
 345                 if (!master_pe) {
 346                         pe->flags |= PNV_IODA_PE_MASTER;
 347                         INIT_LIST_HEAD(&pe->slaves);
 348                         master_pe = pe;
 349                 } else {
 350                         pe->flags |= PNV_IODA_PE_SLAVE;
 351                         pe->master = master_pe;
 352                         list_add_tail(&pe->list, &master_pe->slaves);
 353                 }
 354         }
 355
 356         kfree(pe_alloc);
 357         return master_pe->pe_number;
 358 }
 359
 360 static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
 361 {
 362         struct pci_controller *hose = phb->hose;
 363         struct device_node *dn = hose->dn;
 364         struct resource *res;
 365         const u32 *r;
 366         u64 pci_addr;
 367
 368         /* FIXME: Support M64 for P7IOC */
 369         if (phb->type != PNV_PHB_IODA2) {
 370                 pr_info("  Not support M64 window\n");
 371                 return;
 372         }
 373
 374         if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
 375                 pr_info("  Firmware too old to support M64 window\n");
 376                 return;
 377         }
 378
 379         r = of_get_property(dn, "ibm,opal-m64-window", NULL);
 380         if (!r) {
 381                 pr_info("  No <ibm,opal-m64-window> on %s\n",
 382                         dn->full_name);
 383                 return;
 384         }
 385
 386         res = &hose->mem_resources[1];
 387         res->start = of_translate_address(dn, r + 2);
 388         res->end = res->start + of_read_number(r + 4, 2) - 1;
 389         res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
 390         pci_addr = of_read_number(r, 2);
 391         hose->mem_offset[1] = res->start - pci_addr;
 392
 393         phb->ioda.m64_size = resource_size(res);
 394         phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
 395         phb->ioda.m64_base = pci_addr;
 396
 397         pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
 398                         res->start, res->end, pci_addr);
 399
 400         /* Use last M64 BAR to cover M64 window */
 401         phb->ioda.m64_bar_idx = 15;
 402         phb->init_m64 = pnv_ioda2_init_m64;
 403         phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
 404         phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
 405 }
 406
 407 static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
 408 {
 409         struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
 410         struct pnv_ioda_pe *slave;
 411         s64 rc;
 412
 413         /* Fetch master PE */
 414         if (pe->flags & PNV_IODA_PE_SLAVE) {
 415                 pe = pe->master;
 416                 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
 417                         return;
 418
 419                 pe_no = pe->pe_number;
 420         }
 421
 422         /* Freeze master PE */
 423         rc = opal_pci_eeh_freeze_set(phb->opal_id,
 424                                      pe_no,
 425                                      OPAL_EEH_ACTION_SET_FREEZE_ALL);
 426         if (rc != OPAL_SUCCESS) {
 427                 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 428                         __func__, rc, phb->hose->global_number, pe_no);
 429                 return;
 430         }
 431
 432         /* Freeze slave PEs */
 433         if (!(pe->flags & PNV_IODA_PE_MASTER))
 434                 return;
 435
 436         list_for_each_entry(slave, &pe->slaves, list) {
 437                 rc = opal_pci_eeh_freeze_set(phb->opal_id,
 438                                              slave->pe_number,
 439                                              OPAL_EEH_ACTION_SET_FREEZE_ALL);
 440                 if (rc != OPAL_SUCCESS)
 441                         pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
 442                                 __func__, rc, phb->hose->global_number,
 443                                 slave->pe_number);
 444         }
 445 }
 446
 447 static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
 448 {
 449         struct pnv_ioda_pe *pe, *slave;
 450         s64 rc;
 451
 452         /* Find master PE */
 453         pe = &phb->ioda.pe_array[pe_no];
 454         if (pe->flags & PNV_IODA_PE_SLAVE) {
 455                 pe = pe->master;
 456                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 457                 pe_no = pe->pe_number;
 458         }
 459
 460         /* Clear frozen state for master PE */
 461         rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
 462         if (rc != OPAL_SUCCESS) {
 463                 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 464                         __func__, rc, opt, phb->hose->global_number, pe_no);
 465                 return -EIO;
 466         }
 467
 468         if (!(pe->flags & PNV_IODA_PE_MASTER))
 469                 return 0;
 470
 471         /* Clear frozen state for slave PEs */
 472         list_for_each_entry(slave, &pe->slaves, list) {
 473                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 474                                              slave->pe_number,
 475                                              opt);
 476                 if (rc != OPAL_SUCCESS) {
 477                         pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
 478                                 __func__, rc, opt, phb->hose->global_number,
 479                                 slave->pe_number);
 480                         return -EIO;
 481                 }
 482         }
 483
 484         return 0;
 485 }
 486
 487 static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
 488 {
 489         struct pnv_ioda_pe *slave, *pe;
 490         u8 fstate, state;
 491         __be16 pcierr;
 492         s64 rc;
 493
 494         /* Sanity check on PE number */
 495         if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
 496                 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
 497
 498         /*
 499          * Fetch the master PE and the PE instance might be
 500          * not initialized yet.
 501          */
 502         pe = &phb->ioda.pe_array[pe_no];
 503         if (pe->flags & PNV_IODA_PE_SLAVE) {
 504                 pe = pe->master;
 505                 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
 506                 pe_no = pe->pe_number;
 507         }
 508
 509         /* Check the master PE */
 510         rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
 511                                         &state, &pcierr, NULL);
 512         if (rc != OPAL_SUCCESS) {
 513                 pr_warn("%s: Failure %lld getting "
 514                         "PHB#%x-PE#%x state\n",
 515                         __func__, rc,
 516                         phb->hose->global_number, pe_no);
 517                 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 518         }
 519
 520         /* Check the slave PE */
 521         if (!(pe->flags & PNV_IODA_PE_MASTER))
 522                 return state;
 523
 524         list_for_each_entry(slave, &pe->slaves, list) {
 525                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 526                                                 slave->pe_number,
 527                                                 &fstate,
 528                                                 &pcierr,
 529                                                 NULL);
 530                 if (rc != OPAL_SUCCESS) {
 531                         pr_warn("%s: Failure %lld getting "
 532                                 "PHB#%x-PE#%x state\n",
 533                                 __func__, rc,
 534                                 phb->hose->global_number, slave->pe_number);
 535                         return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
 536                 }
 537
 538                 /*
 539                  * Override the result based on the ascending
 540                  * priority.
 541                  */
 542                 if (fstate > state)
 543                         state = fstate;
 544         }
 545
 546         return state;
 547 }
 548
 549 /* Currently those 2 are only used when MSIs are enabled, this will change
 550  * but in the meantime, we need to protect them to avoid warnings
 551  */
 552 #ifdef CONFIG_PCI_MSI
 553 static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
 554 {
 555         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 556         struct pnv_phb *phb = hose->private_data;
 557         struct pci_dn *pdn = pci_get_pdn(dev);
 558
 559         if (!pdn)
 560                 return NULL;
 561         if (pdn->pe_number == IODA_INVALID_PE)
 562                 return NULL;
 563         return &phb->ioda.pe_array[pdn->pe_number];
 564 }
 565 #endif /* CONFIG_PCI_MSI */
 566
 567 static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
 568                                   struct pnv_ioda_pe *parent,
 569                                   struct pnv_ioda_pe *child,
 570                                   bool is_add)
 571 {
 572         const char *desc = is_add ? "adding" : "removing";
 573         uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
 574                               OPAL_REMOVE_PE_FROM_DOMAIN;
 575         struct pnv_ioda_pe *slave;
 576         long rc;
 577
 578         /* Parent PE affects child PE */
 579         rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 580                                 child->pe_number, op);
 581         if (rc != OPAL_SUCCESS) {
 582                 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
 583                         rc, desc);
 584                 return -ENXIO;
 585         }
 586
 587         if (!(child->flags & PNV_IODA_PE_MASTER))
 588                 return 0;
 589
 590         /* Compound case: parent PE affects slave PEs */
 591         list_for_each_entry(slave, &child->slaves, list) {
 592                 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
 593                                         slave->pe_number, op);
 594                 if (rc != OPAL_SUCCESS) {
 595                         pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
 596                                 rc, desc);
 597                         return -ENXIO;
 598                 }
 599         }
 600
 601         return 0;
 602 }
 603
 604 static int pnv_ioda_set_peltv(struct pnv_phb *phb,
 605                               struct pnv_ioda_pe *pe,
 606                               bool is_add)
 607 {
 608         struct pnv_ioda_pe *slave;
 609         struct pci_dev *pdev = NULL;
 610         int ret;
 611
 612         /*
 613          * Clear PE frozen state. If it's master PE, we need
 614          * clear slave PE frozen state as well.
 615          */
 616         if (is_add) {
 617                 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
 618                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 619                 if (pe->flags & PNV_IODA_PE_MASTER) {
 620                         list_for_each_entry(slave, &pe->slaves, list)
 621                                 opal_pci_eeh_freeze_clear(phb->opal_id,
 622                                                           slave->pe_number,
 623                                                           OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 624                 }
 625         }
 626
 627         /*
 628          * Associate PE in PELT. We need add the PE into the
 629          * corresponding PELT-V as well. Otherwise, the error
 630          * originated from the PE might contribute to other
 631          * PEs.
 632          */
 633         ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
 634         if (ret)
 635                 return ret;
 636
 637         /* For compound PEs, any one affects all of them */
 638         if (pe->flags & PNV_IODA_PE_MASTER) {
 639                 list_for_each_entry(slave, &pe->slaves, list) {
 640                         ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
 641                         if (ret)
 642                                 return ret;
 643                 }
 644         }
 645
 646         if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
 647                 pdev = pe->pbus->self;
 648         else if (pe->flags & PNV_IODA_PE_DEV)
 649                 pdev = pe->pdev->bus->self;
 650 #ifdef CONFIG_PCI_IOV
 651         else if (pe->flags & PNV_IODA_PE_VF)
 652                 pdev = pe->parent_dev->bus->self;
 653 #endif /* CONFIG_PCI_IOV */
 654         while (pdev) {
 655                 struct pci_dn *pdn = pci_get_pdn(pdev);
 656                 struct pnv_ioda_pe *parent;
 657
 658                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 659                         parent = &phb->ioda.pe_array[pdn->pe_number];
 660                         ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
 661                         if (ret)
 662                                 return ret;
 663                 }
 664
 665                 pdev = pdev->bus->self;
 666         }
 667
 668         return 0;
 669 }
 670
 671 #ifdef CONFIG_PCI_IOV
 672 static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 673 {
 674         struct pci_dev *parent;
 675         uint8_t bcomp, dcomp, fcomp;
 676         int64_t rc;
 677         long rid_end, rid;
 678
 679         /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
 680         if (pe->pbus) {
 681                 int count;
 682
 683                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 684                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 685                 parent = pe->pbus->self;
 686                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 687                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 688                 else
 689                         count = 1;
 690
 691                 switch(count) {
 692                 case  1: bcomp = OpalPciBusAll;         break;
 693                 case  2: bcomp = OpalPciBus7Bits;       break;
 694                 case  4: bcomp = OpalPciBus6Bits;       break;
 695                 case  8: bcomp = OpalPciBus5Bits;       break;
 696                 case 16: bcomp = OpalPciBus4Bits;       break;
 697                 case 32: bcomp = OpalPciBus3Bits;       break;
 698                 default:
 699                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 700                                 count);
 701                         /* Do an exact match only */
 702                         bcomp = OpalPciBusAll;
 703                 }
 704                 rid_end = pe->rid + (count << 8);
 705         } else {
 706                 if (pe->flags & PNV_IODA_PE_VF)
 707                         parent = pe->parent_dev;
 708                 else
 709                         parent = pe->pdev->bus->self;
 710                 bcomp = OpalPciBusAll;
 711                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 712                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 713                 rid_end = pe->rid + 1;
 714         }
 715
 716         /* Clear the reverse map */
 717         for (rid = pe->rid; rid < rid_end; rid++)
 718                 phb->ioda.pe_rmap[rid] = 0;
 719
 720         /* Release from all parents PELT-V */
 721         while (parent) {
 722                 struct pci_dn *pdn = pci_get_pdn(parent);
 723                 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
 724                         rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
 725                                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 726                         /* XXX What to do in case of error ? */
 727                 }
 728                 parent = parent->bus->self;
 729         }
 730
 731         opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
 732                                   OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 733
 734         /* Disassociate PE in PELT */
 735         rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
 736                                 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
 737         if (rc)
 738                 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
 739         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 740                              bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
 741         if (rc)
 742                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 743
 744         pe->pbus = NULL;
 745         pe->pdev = NULL;
 746         pe->parent_dev = NULL;
 747
 748         return 0;
 749 }
 750 #endif /* CONFIG_PCI_IOV */
 751
 752 static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 753 {
 754         struct pci_dev *parent;
 755         uint8_t bcomp, dcomp, fcomp;
 756         long rc, rid_end, rid;
 757
 758         /* Bus validation ? */
 759         if (pe->pbus) {
 760                 int count;
 761
 762                 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
 763                 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
 764                 parent = pe->pbus->self;
 765                 if (pe->flags & PNV_IODA_PE_BUS_ALL)
 766                         count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
 767                 else
 768                         count = 1;
 769
 770                 switch(count) {
 771                 case  1: bcomp = OpalPciBusAll;         break;
 772                 case  2: bcomp = OpalPciBus7Bits;       break;
 773                 case  4: bcomp = OpalPciBus6Bits;       break;
 774                 case  8: bcomp = OpalPciBus5Bits;       break;
 775                 case 16: bcomp = OpalPciBus4Bits;       break;
 776                 case 32: bcomp = OpalPciBus3Bits;       break;
 777                 default:
 778                         dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
 779                                 count);
 780                         /* Do an exact match only */
 781                         bcomp = OpalPciBusAll;
 782                 }
 783                 rid_end = pe->rid + (count << 8);
 784         } else {
 785 #ifdef CONFIG_PCI_IOV
 786                 if (pe->flags & PNV_IODA_PE_VF)
 787                         parent = pe->parent_dev;
 788                 else
 789 #endif /* CONFIG_PCI_IOV */
 790                         parent = pe->pdev->bus->self;
 791                 bcomp = OpalPciBusAll;
 792                 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
 793                 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
 794                 rid_end = pe->rid + 1;
 795         }
 796
 797         /*
 798          * Associate PE in PELT. We need add the PE into the
 799          * corresponding PELT-V as well. Otherwise, the error
 800          * originated from the PE might contribute to other
 801          * PEs.
 802          */
 803         rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
 804                              bcomp, dcomp, fcomp, OPAL_MAP_PE);
 805         if (rc) {
 806                 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
 807                 return -ENXIO;
 808         }
 809
 810         /* Configure PELTV */
 811         pnv_ioda_set_peltv(phb, pe, true);
 812
 813         /* Setup reverse map */
 814         for (rid = pe->rid; rid < rid_end; rid++)
 815                 phb->ioda.pe_rmap[rid] = pe->pe_number;
 816
 817         /* Setup one MVTs on IODA1 */
 818         if (phb->type != PNV_PHB_IODA1) {
 819                 pe->mve_number = 0;
 820                 goto out;
 821         }
 822
 823         pe->mve_number = pe->pe_number;
 824         rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
 825         if (rc != OPAL_SUCCESS) {
 826                 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
 827                        rc, pe->mve_number);
 828                 pe->mve_number = -1;
 829         } else {
 830                 rc = opal_pci_set_mve_enable(phb->opal_id,
 831                                              pe->mve_number, OPAL_ENABLE_MVE);
 832                 if (rc) {
 833                         pe_err(pe, "OPAL error %ld enabling MVE %d\n",
 834                                rc, pe->mve_number);
 835                         pe->mve_number = -1;
 836                 }
 837         }
 838
 839 out:
 840         return 0;
 841 }
 842
 843 static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
 844                                        struct pnv_ioda_pe *pe)
 845 {
 846         struct pnv_ioda_pe *lpe;
 847
 848         list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
 849                 if (lpe->dma_weight < pe->dma_weight) {
 850                         list_add_tail(&pe->dma_link, &lpe->dma_link);
 851                         return;
 852                 }
 853         }
 854         list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
 855 }
 856
 857 static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
 858 {
 859         /* This is quite simplistic. The "base" weight of a device
 860          * is 10. 0 means no DMA is to be accounted for it.
 861          */
 862
 863         /* If it's a bridge, no DMA */
 864         if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
 865                 return 0;
 866
 867         /* Reduce the weight of slow USB controllers */
 868         if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
 869             dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
 870             dev->class == PCI_CLASS_SERIAL_USB_EHCI)
 871                 return 3;
 872
 873         /* Increase the weight of RAID (includes Obsidian) */
 874         if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
 875                 return 15;
 876
 877         /* Default */
 878         return 10;
 879 }
 880
 881 #ifdef CONFIG_PCI_IOV
 882 static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
 883 {
 884         struct pci_dn *pdn = pci_get_pdn(dev);
 885         int i;
 886         struct resource *res, res2;
 887         resource_size_t size;
 888         u16 num_vfs;
 889
 890         if (!dev->is_physfn)
 891                 return -EINVAL;
 892
 893         /*
 894          * "offset" is in VFs.  The M64 windows are sized so that when they
 895          * are segmented, each segment is the same size as the IOV BAR.
 896          * Each segment is in a separate PE, and the high order bits of the
 897          * address are the PE number.  Therefore, each VF's BAR is in a
 898          * separate PE, and changing the IOV BAR start address changes the
 899          * range of PEs the VFs are in.
 900          */
 901         num_vfs = pdn->num_vfs;
 902         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 903                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 904                 if (!res->flags || !res->parent)
 905                         continue;
 906
 907                 if (!pnv_pci_is_mem_pref_64(res->flags))
 908                         continue;
 909
 910                 /*
 911                  * The actual IOV BAR range is determined by the start address
 912                  * and the actual size for num_vfs VFs BAR.  This check is to
 913                  * make sure that after shifting, the range will not overlap
 914                  * with another device.
 915                  */
 916                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 917                 res2.flags = res->flags;
 918                 res2.start = res->start + (size * offset);
 919                 res2.end = res2.start + (size * num_vfs) - 1;
 920
 921                 if (res2.end > res->end) {
 922                         dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
 923                                 i, &res2, res, num_vfs, offset);
 924                         return -EBUSY;
 925                 }
 926         }
 927
 928         /*
 929          * After doing so, there would be a "hole" in the /proc/iomem when
 930          * offset is a positive value. It looks like the device return some
 931          * mmio back to the system, which actually no one could use it.
 932          */
 933         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
 934                 res = &dev->resource[i + PCI_IOV_RESOURCES];
 935                 if (!res->flags || !res->parent)
 936                         continue;
 937
 938                 if (!pnv_pci_is_mem_pref_64(res->flags))
 939                         continue;
 940
 941                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
 942                 res2 = *res;
 943                 res->start += size * offset;
 944
 945                 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
 946                          i, &res2, res, num_vfs, offset);
 947                 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
 948         }
 949         return 0;
 950 }
 951 #endif /* CONFIG_PCI_IOV */
 952
 953 #if 0
 954 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 955 {
 956         struct pci_controller *hose = pci_bus_to_host(dev->bus);
 957         struct pnv_phb *phb = hose->private_data;
 958         struct pci_dn *pdn = pci_get_pdn(dev);
 959         struct pnv_ioda_pe *pe;
 960         int pe_num;
 961
 962         if (!pdn) {
 963                 pr_err("%s: Device tree node not associated properly\n",
 964                            pci_name(dev));
 965                 return NULL;
 966         }
 967         if (pdn->pe_number != IODA_INVALID_PE)
 968                 return NULL;
 969
 970         /* PE#0 has been pre-set */
 971         if (dev->bus->number == 0)
 972                 pe_num = 0;
 973         else
 974                 pe_num = pnv_ioda_alloc_pe(phb);
 975         if (pe_num == IODA_INVALID_PE) {
 976                 pr_warning("%s: Not enough PE# available, disabling device\n",
 977                            pci_name(dev));
 978                 return NULL;
 979         }
 980
 981         /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
 982          * pointer in the PE data structure, both should be destroyed at the
 983          * same time. However, this needs to be looked at more closely again
 984          * once we actually start removing things (Hotplug, SR-IOV, ...)
 985          *
 986          * At some point we want to remove the PDN completely anyways
 987          */
 988         pe = &phb->ioda.pe_array[pe_num];
 989         pci_dev_get(dev);
 990         pdn->pcidev = dev;
 991         pdn->pe_number = pe_num;
 992         pe->pdev = dev;
 993         pe->pbus = NULL;
 994         pe->tce32_seg = -1;
 995         pe->mve_number = -1;
 996         pe->rid = dev->bus->number << 8 | pdn->devfn;
 997
 998         pe_info(pe, "Associated device to PE\n");
 999
1000         if (pnv_ioda_configure_pe(phb, pe)) {
1001                 /* XXX What do we do here ? */
1002                 if (pe_num)
1003                         pnv_ioda_free_pe(phb, pe_num);
1004                 pdn->pe_number = IODA_INVALID_PE;
1005                 pe->pdev = NULL;
1006                 pci_dev_put(dev);
1007                 return NULL;
1008         }
1009
1010         /* Assign a DMA weight to the device */
1011         pe->dma_weight = pnv_ioda_dma_weight(dev);
1012         if (pe->dma_weight != 0) {
1013                 phb->ioda.dma_weight += pe->dma_weight;
1014                 phb->ioda.dma_pe_count++;
1015         }
1016
1017         /* Link the PE */
1018         pnv_ioda_link_pe_by_weight(phb, pe);
1019
1020         return pe;
1021 }
1022 #endif /* Useful for SRIOV case */
1023
1024 static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1025 {
1026         struct pci_dev *dev;
1027
1028         list_for_each_entry(dev, &bus->devices, bus_list) {
1029                 struct pci_dn *pdn = pci_get_pdn(dev);
1030
1031                 if (pdn == NULL) {
1032                         pr_warn("%s: No device node associated with device !\n",
1033                                 pci_name(dev));
1034                         continue;
1035                 }
1036                 pdn->pe_number = pe->pe_number;
1037                 pe->dma_weight += pnv_ioda_dma_weight(dev);
1038                 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1039                         pnv_ioda_setup_same_PE(dev->subordinate, pe);
1040         }
1041 }
1042
1043 /*
1044  * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1045  * single PCI bus. Another one that contains the primary PCI bus and its
1046  * subordinate PCI devices and buses. The second type of PE is normally
1047  * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1048  */
1049 static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
1050 {
1051         struct pci_controller *hose = pci_bus_to_host(bus);
1052         struct pnv_phb *phb = hose->private_data;
1053         struct pnv_ioda_pe *pe;
1054         int pe_num = IODA_INVALID_PE;
1055
1056         /* Check if PE is determined by M64 */
1057         if (phb->pick_m64_pe)
1058                 pe_num = phb->pick_m64_pe(phb, bus, all);
1059
1060         /* The PE number isn't pinned by M64 */
1061         if (pe_num == IODA_INVALID_PE)
1062                 pe_num = pnv_ioda_alloc_pe(phb);
1063
1064         if (pe_num == IODA_INVALID_PE) {
1065                 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1066                         __func__, pci_domain_nr(bus), bus->number);
1067                 return;
1068         }
1069
1070         pe = &phb->ioda.pe_array[pe_num];
1071         pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
1072         pe->pbus = bus;
1073         pe->pdev = NULL;
1074         pe->tce32_seg = -1;
1075         pe->mve_number = -1;
1076         pe->rid = bus->busn_res.start << 8;
1077         pe->dma_weight = 0;
1078
1079         if (all)
1080                 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1081                         bus->busn_res.start, bus->busn_res.end, pe_num);
1082         else
1083                 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1084                         bus->busn_res.start, pe_num);
1085
1086         if (pnv_ioda_configure_pe(phb, pe)) {
1087                 /* XXX What do we do here ? */
1088                 if (pe_num)
1089                         pnv_ioda_free_pe(phb, pe_num);
1090                 pe->pbus = NULL;
1091                 return;
1092         }
1093
1094         /* Associate it with all child devices */
1095         pnv_ioda_setup_same_PE(bus, pe);
1096
1097         /* Put PE to the list */
1098         list_add_tail(&pe->list, &phb->ioda.pe_list);
1099
1100         /* Account for one DMA PE if at least one DMA capable device exist
1101          * below the bridge
1102          */
1103         if (pe->dma_weight != 0) {
1104                 phb->ioda.dma_weight += pe->dma_weight;
1105                 phb->ioda.dma_pe_count++;
1106         }
1107
1108         /* Link the PE */
1109         pnv_ioda_link_pe_by_weight(phb, pe);
1110 }
1111
1112 static void pnv_ioda_setup_PEs(struct pci_bus *bus)
1113 {
1114         struct pci_dev *dev;
1115
1116         pnv_ioda_setup_bus_PE(bus, 0);
1117
1118         list_for_each_entry(dev, &bus->devices, bus_list) {
1119                 if (dev->subordinate) {
1120                         if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1121                                 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1122                         else
1123                                 pnv_ioda_setup_PEs(dev->subordinate);
1124                 }
1125         }
1126 }
1127
1128 /*
1129  * Configure PEs so that the downstream PCI buses and devices
1130  * could have their associated PE#. Unfortunately, we didn't
1131  * figure out the way to identify the PLX bridge yet. So we
1132  * simply put the PCI bus and the subordinate behind the root
1133  * port to PE# here. The game rule here is expected to be changed
1134  * as soon as we can detected PLX bridge correctly.
1135  */
1136 static void pnv_pci_ioda_setup_PEs(void)
1137 {
1138         struct pci_controller *hose, *tmp;
1139         struct pnv_phb *phb;
1140
1141         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
1142                 phb = hose->private_data;
1143
1144                 /* M64 layout might affect PE allocation */
1145                 if (phb->reserve_m64_pe)
1146                         phb->reserve_m64_pe(phb);
1147
1148                 pnv_ioda_setup_PEs(hose->bus);
1149         }
1150 }
1151
1152 #ifdef CONFIG_PCI_IOV
1153 static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1154 {
1155         struct pci_bus        *bus;
1156         struct pci_controller *hose;
1157         struct pnv_phb        *phb;
1158         struct pci_dn         *pdn;
1159         int                    i, j;
1160
1161         bus = pdev->bus;
1162         hose = pci_bus_to_host(bus);
1163         phb = hose->private_data;
1164         pdn = pci_get_pdn(pdev);
1165
1166         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1167                 for (j = 0; j < M64_PER_IOV; j++) {
1168                         if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1169                                 continue;
1170                         opal_pci_phb_mmio_enable(phb->opal_id,
1171                                 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1172                         clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1173                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1174                 }
1175
1176         return 0;
1177 }
1178
1179 static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
1180 {
1181         struct pci_bus        *bus;
1182         struct pci_controller *hose;
1183         struct pnv_phb        *phb;
1184         struct pci_dn         *pdn;
1185         unsigned int           win;
1186         struct resource       *res;
1187         int                    i, j;
1188         int64_t                rc;
1189         int                    total_vfs;
1190         resource_size_t        size, start;
1191         int                    pe_num;
1192         int                    vf_groups;
1193         int                    vf_per_group;
1194
1195         bus = pdev->bus;
1196         hose = pci_bus_to_host(bus);
1197         phb = hose->private_data;
1198         pdn = pci_get_pdn(pdev);
1199         total_vfs = pci_sriov_get_totalvfs(pdev);
1200
1201         /* Initialize the m64_wins to IODA_INVALID_M64 */
1202         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1203                 for (j = 0; j < M64_PER_IOV; j++)
1204                         pdn->m64_wins[i][j] = IODA_INVALID_M64;
1205
1206         if (pdn->m64_per_iov == M64_PER_IOV) {
1207                 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1208                 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1209                         roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1210         } else {
1211                 vf_groups = 1;
1212                 vf_per_group = 1;
1213         }
1214
1215         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1216                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1217                 if (!res->flags || !res->parent)
1218                         continue;
1219
1220                 if (!pnv_pci_is_mem_pref_64(res->flags))
1221                         continue;
1222
1223                 for (j = 0; j < vf_groups; j++) {
1224                         do {
1225                                 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1226                                                 phb->ioda.m64_bar_idx + 1, 0);
1227
1228                                 if (win >= phb->ioda.m64_bar_idx + 1)
1229                                         goto m64_failed;
1230                         } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1231
1232                         pdn->m64_wins[i][j] = win;
1233
1234                         if (pdn->m64_per_iov == M64_PER_IOV) {
1235                                 size = pci_iov_resource_size(pdev,
1236                                                         PCI_IOV_RESOURCES + i);
1237                                 size = size * vf_per_group;
1238                                 start = res->start + size * j;
1239                         } else {
1240                                 size = resource_size(res);
1241                                 start = res->start;
1242                         }
1243
1244                         /* Map the M64 here */
1245                         if (pdn->m64_per_iov == M64_PER_IOV) {
1246                                 pe_num = pdn->offset + j;
1247                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1248                                                 pe_num, OPAL_M64_WINDOW_TYPE,
1249                                                 pdn->m64_wins[i][j], 0);
1250                         }
1251
1252                         rc = opal_pci_set_phb_mem_window(phb->opal_id,
1253                                                  OPAL_M64_WINDOW_TYPE,
1254                                                  pdn->m64_wins[i][j],
1255                                                  start,
1256                                                  0, /* unused */
1257                                                  size);
1258
1259
1260                         if (rc != OPAL_SUCCESS) {
1261                                 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1262                                         win, rc);
1263                                 goto m64_failed;
1264                         }
1265
1266                         if (pdn->m64_per_iov == M64_PER_IOV)
1267                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1268                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1269                         else
1270                                 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1271                                      OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
1272
1273                         if (rc != OPAL_SUCCESS) {
1274                                 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1275                                         win, rc);
1276                                 goto m64_failed;
1277                         }
1278                 }
1279         }
1280         return 0;
1281
1282 m64_failed:
1283         pnv_pci_vf_release_m64(pdev);
1284         return -EBUSY;
1285 }
1286
1287 static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1288 {
1289         struct pci_bus        *bus;
1290         struct pci_controller *hose;
1291         struct pnv_phb        *phb;
1292         struct iommu_table    *tbl;
1293         unsigned long         addr;
1294         int64_t               rc;
1295
1296         bus = dev->bus;
1297         hose = pci_bus_to_host(bus);
1298         phb = hose->private_data;
1299         tbl = pe->table_group.tables[0];
1300         addr = tbl->it_base;
1301
1302         opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1303                                    pe->pe_number << 1, 1, __pa(addr),
1304                                    0, 0x1000);
1305
1306         rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1307                                         pe->pe_number,
1308                                         (pe->pe_number << 1) + 1,
1309                                         pe->tce_bypass_base,
1310                                         0);
1311         if (rc)
1312                 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1313
1314         pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1315         if (pe->table_group.group) {
1316                 iommu_group_put(pe->table_group.group);
1317                 BUG_ON(pe->table_group.group);
1318         }
1319         pnv_pci_ioda2_table_free_pages(tbl);
1320         iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
1321 }
1322
1323 static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1324 {
1325         struct pci_bus        *bus;
1326         struct pci_controller *hose;
1327         struct pnv_phb        *phb;
1328         struct pnv_ioda_pe    *pe, *pe_n;
1329         struct pci_dn         *pdn;
1330         u16                    vf_index;
1331         int64_t                rc;
1332
1333         bus = pdev->bus;
1334         hose = pci_bus_to_host(bus);
1335         phb = hose->private_data;
1336         pdn = pci_get_pdn(pdev);
1337
1338         if (!pdev->is_physfn)
1339                 return;
1340
1341         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1342                 int   vf_group;
1343                 int   vf_per_group;
1344                 int   vf_index1;
1345
1346                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1347
1348                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1349                         for (vf_index = vf_group * vf_per_group;
1350                                 vf_index < (vf_group + 1) * vf_per_group &&
1351                                 vf_index < num_vfs;
1352                                 vf_index++)
1353                                 for (vf_index1 = vf_group * vf_per_group;
1354                                         vf_index1 < (vf_group + 1) * vf_per_group &&
1355                                         vf_index1 < num_vfs;
1356                                         vf_index1++){
1357
1358                                         rc = opal_pci_set_peltv(phb->opal_id,
1359                                                 pdn->offset + vf_index,
1360                                                 pdn->offset + vf_index1,
1361                                                 OPAL_REMOVE_PE_FROM_DOMAIN);
1362
1363                                         if (rc)
1364                                             dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1365                                                 __func__,
1366                                                 pdn->offset + vf_index1, rc);
1367                                 }
1368         }
1369
1370         list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1371                 if (pe->parent_dev != pdev)
1372                         continue;
1373
1374                 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1375
1376                 /* Remove from list */
1377                 mutex_lock(&phb->ioda.pe_list_mutex);
1378                 list_del(&pe->list);
1379                 mutex_unlock(&phb->ioda.pe_list_mutex);
1380
1381                 pnv_ioda_deconfigure_pe(phb, pe);
1382
1383                 pnv_ioda_free_pe(phb, pe->pe_number);
1384         }
1385 }
1386
1387 void pnv_pci_sriov_disable(struct pci_dev *pdev)
1388 {
1389         struct pci_bus        *bus;
1390         struct pci_controller *hose;
1391         struct pnv_phb        *phb;
1392         struct pci_dn         *pdn;
1393         struct pci_sriov      *iov;
1394         u16 num_vfs;
1395
1396         bus = pdev->bus;
1397         hose = pci_bus_to_host(bus);
1398         phb = hose->private_data;
1399         pdn = pci_get_pdn(pdev);
1400         iov = pdev->sriov;
1401         num_vfs = pdn->num_vfs;
1402
1403         /* Release VF PEs */
1404         pnv_ioda_release_vf_PE(pdev, num_vfs);
1405
1406         if (phb->type == PNV_PHB_IODA2) {
1407                 if (pdn->m64_per_iov == 1)
1408                         pnv_pci_vf_resource_shift(pdev, -pdn->offset);
1409
1410                 /* Release M64 windows */
1411                 pnv_pci_vf_release_m64(pdev);
1412
1413                 /* Release PE numbers */
1414                 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1415                 pdn->offset = 0;
1416         }
1417 }
1418
1419 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1420                                        struct pnv_ioda_pe *pe);
1421 static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1422 {
1423         struct pci_bus        *bus;
1424         struct pci_controller *hose;
1425         struct pnv_phb        *phb;
1426         struct pnv_ioda_pe    *pe;
1427         int                    pe_num;
1428         u16                    vf_index;
1429         struct pci_dn         *pdn;
1430         int64_t                rc;
1431
1432         bus = pdev->bus;
1433         hose = pci_bus_to_host(bus);
1434         phb = hose->private_data;
1435         pdn = pci_get_pdn(pdev);
1436
1437         if (!pdev->is_physfn)
1438                 return;
1439
1440         /* Reserve PE for each VF */
1441         for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1442                 pe_num = pdn->offset + vf_index;
1443
1444                 pe = &phb->ioda.pe_array[pe_num];
1445                 pe->pe_number = pe_num;
1446                 pe->phb = phb;
1447                 pe->flags = PNV_IODA_PE_VF;
1448                 pe->pbus = NULL;
1449                 pe->parent_dev = pdev;
1450                 pe->tce32_seg = -1;
1451                 pe->mve_number = -1;
1452                 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1453                            pci_iov_virtfn_devfn(pdev, vf_index);
1454
1455                 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1456                         hose->global_number, pdev->bus->number,
1457                         PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1458                         PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1459
1460                 if (pnv_ioda_configure_pe(phb, pe)) {
1461                         /* XXX What do we do here ? */
1462                         if (pe_num)
1463                                 pnv_ioda_free_pe(phb, pe_num);
1464                         pe->pdev = NULL;
1465                         continue;
1466                 }
1467
1468                 /* Put PE to the list */
1469                 mutex_lock(&phb->ioda.pe_list_mutex);
1470                 list_add_tail(&pe->list, &phb->ioda.pe_list);
1471                 mutex_unlock(&phb->ioda.pe_list_mutex);
1472
1473                 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1474         }
1475
1476         if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1477                 int   vf_group;
1478                 int   vf_per_group;
1479                 int   vf_index1;
1480
1481                 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1482
1483                 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1484                         for (vf_index = vf_group * vf_per_group;
1485                              vf_index < (vf_group + 1) * vf_per_group &&
1486                              vf_index < num_vfs;
1487                              vf_index++) {
1488                                 for (vf_index1 = vf_group * vf_per_group;
1489                                      vf_index1 < (vf_group + 1) * vf_per_group &&
1490                                      vf_index1 < num_vfs;
1491                                      vf_index1++) {
1492
1493                                         rc = opal_pci_set_peltv(phb->opal_id,
1494                                                 pdn->offset + vf_index,
1495                                                 pdn->offset + vf_index1,
1496                                                 OPAL_ADD_PE_TO_DOMAIN);
1497
1498                                         if (rc)
1499                                             dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1500                                                 __func__,
1501                                                 pdn->offset + vf_index1, rc);
1502                                 }
1503                         }
1504                 }
1505         }
1506 }
1507
1508 int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1509 {
1510         struct pci_bus        *bus;
1511         struct pci_controller *hose;
1512         struct pnv_phb        *phb;
1513         struct pci_dn         *pdn;
1514         int                    ret;
1515
1516         bus = pdev->bus;
1517         hose = pci_bus_to_host(bus);
1518         phb = hose->private_data;
1519         pdn = pci_get_pdn(pdev);
1520
1521         if (phb->type == PNV_PHB_IODA2) {
1522                 /* Calculate available PE for required VFs */
1523                 mutex_lock(&phb->ioda.pe_alloc_mutex);
1524                 pdn->offset = bitmap_find_next_zero_area(
1525                         phb->ioda.pe_alloc, phb->ioda.total_pe,
1526                         0, num_vfs, 0);
1527                 if (pdn->offset >= phb->ioda.total_pe) {
1528                         mutex_unlock(&phb->ioda.pe_alloc_mutex);
1529                         dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1530                         pdn->offset = 0;
1531                         return -EBUSY;
1532                 }
1533                 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1534                 pdn->num_vfs = num_vfs;
1535                 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1536
1537                 /* Assign M64 window accordingly */
1538                 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
1539                 if (ret) {
1540                         dev_info(&pdev->dev, "Not enough M64 window resources\n");
1541                         goto m64_failed;
1542                 }
1543
1544                 /*
1545                  * When using one M64 BAR to map one IOV BAR, we need to shift
1546                  * the IOV BAR according to the PE# allocated to the VFs.
1547                  * Otherwise, the PE# for the VF will conflict with others.
1548                  */
1549                 if (pdn->m64_per_iov == 1) {
1550                         ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1551                         if (ret)
1552                                 goto m64_failed;
1553                 }
1554         }
1555
1556         /* Setup VF PEs */
1557         pnv_ioda_setup_vf_PE(pdev, num_vfs);
1558
1559         return 0;
1560
1561 m64_failed:
1562         bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1563         pdn->offset = 0;
1564
1565         return ret;
1566 }
1567
1568 int pcibios_sriov_disable(struct pci_dev *pdev)
1569 {
1570         pnv_pci_sriov_disable(pdev);
1571
1572         /* Release PCI data */
1573         remove_dev_pci_data(pdev);
1574         return 0;
1575 }
1576
1577 int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1578 {
1579         /* Allocate PCI data */
1580         add_dev_pci_data(pdev);
1581
1582         pnv_pci_sriov_enable(pdev, num_vfs);
1583         return 0;
1584 }
1585 #endif /* CONFIG_PCI_IOV */
1586
1587 static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
1588 {
1589         struct pci_dn *pdn = pci_get_pdn(pdev);
1590         struct pnv_ioda_pe *pe;
1591
1592         /*
1593          * The function can be called while the PE#
1594          * hasn't been assigned. Do nothing for the
1595          * case.
1596          */
1597         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1598                 return;
1599
1600         pe = &phb->ioda.pe_array[pdn->pe_number];
1601         WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1602         set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1603         /*
1604          * Note: iommu_add_device() will fail here as
1605          * for physical PE: the device is already added by now;
1606          * for virtual PE: sysfs entries are not ready yet and
1607          * tce_iommu_bus_notifier will add the device to a group later.
1608          */
1609 }
1610
1611 static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
1612 {
1613         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1614         struct pnv_phb *phb = hose->private_data;
1615         struct pci_dn *pdn = pci_get_pdn(pdev);
1616         struct pnv_ioda_pe *pe;
1617         uint64_t top;
1618         bool bypass = false;
1619
1620         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1621                 return -ENODEV;;
1622
1623         pe = &phb->ioda.pe_array[pdn->pe_number];
1624         if (pe->tce_bypass_enabled) {
1625                 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1626                 bypass = (dma_mask >= top);
1627         }
1628
1629         if (bypass) {
1630                 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1631                 set_dma_ops(&pdev->dev, &dma_direct_ops);
1632                 set_dma_offset(&pdev->dev, pe->tce_bypass_base);
1633         } else {
1634                 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1635                 set_dma_ops(&pdev->dev, &dma_iommu_ops);
1636                 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1637         }
1638         *pdev->dev.dma_mask = dma_mask;
1639         return 0;
1640 }
1641
1642 static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1643                                               struct pci_dev *pdev)
1644 {
1645         struct pci_dn *pdn = pci_get_pdn(pdev);
1646         struct pnv_ioda_pe *pe;
1647         u64 end, mask;
1648
1649         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1650                 return 0;
1651
1652         pe = &phb->ioda.pe_array[pdn->pe_number];
1653         if (!pe->tce_bypass_enabled)
1654                 return __dma_get_required_mask(&pdev->dev);
1655
1656
1657         end = pe->tce_bypass_base + memblock_end_of_DRAM();
1658         mask = 1ULL << (fls64(end) - 1);
1659         mask += mask - 1;
1660
1661         return mask;
1662 }
1663
1664 static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
1665                                    struct pci_bus *bus)
1666 {
1667         struct pci_dev *dev;
1668
1669         list_for_each_entry(dev, &bus->devices, bus_list) {
1670                 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
1671                 iommu_add_device(&dev->dev);
1672
1673                 if (dev->subordinate)
1674                         pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1675         }
1676 }
1677
1678 static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1679                 unsigned long index, unsigned long npages, bool rm)
1680 {
1681         struct iommu_table_group_link *tgl = list_first_entry_or_null(
1682                         &tbl->it_group_list, struct iommu_table_group_link,
1683                         next);
1684         struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1685                         struct pnv_ioda_pe, table_group);
1686         __be64 __iomem *invalidate = rm ?
1687                 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1688                 pe->phb->ioda.tce_inval_reg;
1689         unsigned long start, end, inc;
1690         const unsigned shift = tbl->it_page_shift;
1691
1692         start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1693         end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1694                         npages - 1);
1695
1696         /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1697         if (tbl->it_busno) {
1698                 start <<= shift;
1699                 end <<= shift;
1700                 inc = 128ull << shift;
1701                 start |= tbl->it_busno;
1702                 end |= tbl->it_busno;
1703         } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1704                 /* p7ioc-style invalidation, 2 TCEs per write */
1705                 start |= (1ull << 63);
1706                 end |= (1ull << 63);
1707                 inc = 16;
1708         } else {
1709                 /* Default (older HW) */
1710                 inc = 128;
1711         }
1712
1713         end |= inc - 1; /* round up end to be different than start */
1714
1715         mb(); /* Ensure above stores are visible */
1716         while (start <= end) {
1717                 if (rm)
1718                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1719                 else
1720                         __raw_writeq(cpu_to_be64(start), invalidate);
1721                 start += inc;
1722         }
1723
1724         /*
1725          * The iommu layer will do another mb() for us on build()
1726          * and we don't care on free()
1727          */
1728 }
1729
1730 static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1731                 long npages, unsigned long uaddr,
1732                 enum dma_data_direction direction,
1733                 struct dma_attrs *attrs)
1734 {
1735         int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1736                         attrs);
1737
1738         if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1739                 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1740
1741         return ret;
1742 }
1743
1744 #ifdef CONFIG_IOMMU_API
1745 static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
1746                 unsigned long *hpa, enum dma_data_direction *direction)
1747 {
1748         long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1749
1750         if (!ret && (tbl->it_type &
1751                         (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1752                 pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
1753
1754         return ret;
1755 }
1756 #endif
1757
1758 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1759                 long npages)
1760 {
1761         pnv_tce_free(tbl, index, npages);
1762
1763         if (tbl->it_type & TCE_PCI_SWINV_FREE)
1764                 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1765 }
1766
1767 static struct iommu_table_ops pnv_ioda1_iommu_ops = {
1768         .set = pnv_ioda1_tce_build,
1769 #ifdef CONFIG_IOMMU_API
1770         .exchange = pnv_ioda1_tce_xchg,
1771 #endif
1772         .clear = pnv_ioda1_tce_free,
1773         .get = pnv_tce_get,
1774 };
1775
1776 static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1777 {
1778         /* 01xb - invalidate TCEs that match the specified PE# */
1779         unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1780         struct pnv_phb *phb = pe->phb;
1781
1782         if (!phb->ioda.tce_inval_reg)
1783                 return;
1784
1785         mb(); /* Ensure above stores are visible */
1786         __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1787 }
1788
1789 static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
1790                 __be64 __iomem *invalidate, unsigned shift,
1791                 unsigned long index, unsigned long npages)
1792 {
1793         unsigned long start, end, inc;
1794
1795         /* We'll invalidate DMA address in PE scope */
1796         start = 0x2ull << 60;
1797         start |= (pe_number & 0xFF);
1798         end = start;
1799
1800         /* Figure out the start, end and step */
1801         start |= (index << shift);
1802         end |= ((index + npages - 1) << shift);
1803         inc = (0x1ull << shift);
1804         mb();
1805
1806         while (start <= end) {
1807                 if (rm)
1808                         __raw_rm_writeq(cpu_to_be64(start), invalidate);
1809                 else
1810                         __raw_writeq(cpu_to_be64(start), invalidate);
1811                 start += inc;
1812         }
1813 }
1814
1815 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1816                 unsigned long index, unsigned long npages, bool rm)
1817 {
1818         struct iommu_table_group_link *tgl;
1819
1820         list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1821                 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1822                                 struct pnv_ioda_pe, table_group);
1823                 __be64 __iomem *invalidate = rm ?
1824                         (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1825                         pe->phb->ioda.tce_inval_reg;
1826
1827                 pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
1828                         invalidate, tbl->it_page_shift,
1829                         index, npages);
1830         }
1831 }
1832
1833 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1834                 long npages, unsigned long uaddr,
1835                 enum dma_data_direction direction,
1836                 struct dma_attrs *attrs)
1837 {
1838         int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1839                         attrs);
1840
1841         if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1842                 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1843
1844         return ret;
1845 }
1846
1847 #ifdef CONFIG_IOMMU_API
1848 static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
1849                 unsigned long *hpa, enum dma_data_direction *direction)
1850 {
1851         long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1852
1853         if (!ret && (tbl->it_type &
1854                         (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1855                 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
1856
1857         return ret;
1858 }
1859 #endif
1860
1861 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1862                 long npages)
1863 {
1864         pnv_tce_free(tbl, index, npages);
1865
1866         if (tbl->it_type & TCE_PCI_SWINV_FREE)
1867                 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1868 }
1869
1870 static struct iommu_table_ops pnv_ioda2_iommu_ops = {
1871         .set = pnv_ioda2_tce_build,
1872 #ifdef CONFIG_IOMMU_API
1873         .exchange = pnv_ioda2_tce_xchg,
1874 #endif
1875         .clear = pnv_ioda2_tce_free,
1876         .get = pnv_tce_get,
1877 };
1878
1879 static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1880                                       struct pnv_ioda_pe *pe, unsigned int base,
1881                                       unsigned int segs)
1882 {
1883
1884         struct page *tce_mem = NULL;
1885         struct iommu_table *tbl;
1886         unsigned int i;
1887         int64_t rc;
1888         void *addr;
1889
1890         /* XXX FIXME: Handle 64-bit only DMA devices */
1891         /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1892         /* XXX FIXME: Allocate multi-level tables on PHB3 */
1893
1894         /* We shouldn't already have a 32-bit DMA associated */
1895         if (WARN_ON(pe->tce32_seg >= 0))
1896                 return;
1897
1898         tbl = pnv_pci_table_alloc(phb->hose->node);
1899         iommu_register_group(&pe->table_group, phb->hose->global_number,
1900                         pe->pe_number);
1901         pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
1902
1903         /* Grab a 32-bit TCE table */
1904         pe->tce32_seg = base;
1905         pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1906                 (base << 28), ((base + segs) << 28) - 1);
1907
1908         /* XXX Currently, we allocate one big contiguous table for the
1909          * TCEs. We only really need one chunk per 256M of TCE space
1910          * (ie per segment) but that's an optimization for later, it
1911          * requires some added smarts with our get/put_tce implementation
1912          */
1913         tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1914                                    get_order(TCE32_TABLE_SIZE * segs));
1915         if (!tce_mem) {
1916                 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1917                 goto fail;
1918         }
1919         addr = page_address(tce_mem);
1920         memset(addr, 0, TCE32_TABLE_SIZE * segs);
1921
1922         /* Configure HW */
1923         for (i = 0; i < segs; i++) {
1924                 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1925                                               pe->pe_number,
1926                                               base + i, 1,
1927                                               __pa(addr) + TCE32_TABLE_SIZE * i,
1928                                               TCE32_TABLE_SIZE, 0x1000);
1929                 if (rc) {
1930                         pe_err(pe, " Failed to configure 32-bit TCE table,"
1931                                " err %ld\n", rc);
1932                         goto fail;
1933                 }
1934         }
1935
1936         /* Setup linux iommu table */
1937         pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
1938                                   base << 28, IOMMU_PAGE_SHIFT_4K);
1939
1940         /* OPAL variant of P7IOC SW invalidated TCEs */
1941         if (phb->ioda.tce_inval_reg)
1942                 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1943                                  TCE_PCI_SWINV_FREE   |
1944                                  TCE_PCI_SWINV_PAIR);
1945
1946         tbl->it_ops = &pnv_ioda1_iommu_ops;
1947         iommu_init_table(tbl, phb->hose->node);
1948
1949         if (pe->flags & PNV_IODA_PE_DEV) {
1950                 /*
1951                  * Setting table base here only for carrying iommu_group
1952                  * further down to let iommu_add_device() do the job.
1953                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1954                  */
1955                 set_iommu_table_base(&pe->pdev->dev, tbl);
1956                 iommu_add_device(&pe->pdev->dev);
1957         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
1958                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
1959
1960         return;
1961  fail:
1962         /* XXX Failure: Try to fallback to 64-bit only ? */
1963         if (pe->tce32_seg >= 0)
1964                 pe->tce32_seg = -1;
1965         if (tce_mem)
1966                 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
1967         if (tbl) {
1968                 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1969                 iommu_free_table(tbl, "pnv");
1970         }
1971 }
1972
1973 static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1974                 int num, struct iommu_table *tbl)
1975 {
1976         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1977                         table_group);
1978         struct pnv_phb *phb = pe->phb;
1979         int64_t rc;
1980         const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1981         const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1982
1983         pe_info(pe, "Setting up window %llx..%llx pg=%x\n",
1984                         start_addr, start_addr + win_size - 1,
1985                         IOMMU_PAGE_SIZE(tbl));
1986
1987         /*
1988          * Map TCE table through TVT. The TVE index is the PE number
1989          * shifted by 1 bit for 32-bits DMA space.
1990          */
1991         rc = opal_pci_map_pe_dma_window(phb->opal_id,
1992                         pe->pe_number,
1993                         pe->pe_number << 1,
1994                         1,
1995                         __pa(tbl->it_base),
1996                         tbl->it_size << 3,
1997                         IOMMU_PAGE_SIZE(tbl));
1998         if (rc) {
1999                 pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
2000                 return rc;
2001         }
2002
2003         pnv_pci_link_table_and_group(phb->hose->node, num,
2004                         tbl, &pe->table_group);
2005         pnv_pci_ioda2_tce_invalidate_entire(pe);
2006
2007         return 0;
2008 }
2009
2010 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
2011 {
2012         uint16_t window_id = (pe->pe_number << 1 ) + 1;
2013         int64_t rc;
2014
2015         pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
2016         if (enable) {
2017                 phys_addr_t top = memblock_end_of_DRAM();
2018
2019                 top = roundup_pow_of_two(top);
2020                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2021                                                      pe->pe_number,
2022                                                      window_id,
2023                                                      pe->tce_bypass_base,
2024                                                      top);
2025         } else {
2026                 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2027                                                      pe->pe_number,
2028                                                      window_id,
2029                                                      pe->tce_bypass_base,
2030                                                      0);
2031         }
2032         if (rc)
2033                 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
2034         else
2035                 pe->tce_bypass_enabled = enable;
2036 }
2037
2038 #ifdef CONFIG_IOMMU_API
2039 static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
2040 {
2041         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2042                                                 table_group);
2043
2044         iommu_take_ownership(table_group->tables[0]);
2045         pnv_pci_ioda2_set_bypass(pe, false);
2046 }
2047
2048 static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2049 {
2050         struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2051                                                 table_group);
2052
2053         iommu_release_ownership(table_group->tables[0]);
2054         pnv_pci_ioda2_set_bypass(pe, true);
2055 }
2056
2057 static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
2058         .take_ownership = pnv_ioda2_take_ownership,
2059         .release_ownership = pnv_ioda2_release_ownership,
2060 };
2061 #endif
2062
2063 static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
2064 {
2065         const __be64 *swinvp;
2066
2067         /* OPAL variant of PHB3 invalidated TCEs */
2068         swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
2069         if (!swinvp)
2070                 return;
2071
2072         phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
2073         phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
2074 }
2075
2076 static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift)
2077 {
2078         struct page *tce_mem = NULL;
2079         __be64 *addr;
2080         unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
2081
2082         tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
2083         if (!tce_mem) {
2084                 pr_err("Failed to allocate a TCE memory, order=%d\n", order);
2085                 return NULL;
2086         }
2087         addr = page_address(tce_mem);
2088         memset(addr, 0, 1UL << (order + PAGE_SHIFT));
2089
2090         return addr;
2091 }
2092
2093 static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2094                 __u32 page_shift, __u64 window_size, struct iommu_table *tbl)
2095 {
2096         void *addr;
2097         const unsigned window_shift = ilog2(window_size);
2098         unsigned entries_shift = window_shift - page_shift;
2099         unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
2100         const unsigned long tce_table_size = 1UL << table_shift;
2101
2102         if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
2103                 return -EINVAL;
2104
2105         /* Allocate TCE table */
2106         addr = pnv_pci_ioda2_table_do_alloc_pages(nid, table_shift);
2107         if (!addr)
2108                 return -ENOMEM;
2109
2110         /* Setup linux iommu table */
2111         pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
2112                         page_shift);
2113
2114         pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
2115                         window_size, tce_table_size, bus_offset);
2116
2117         return 0;
2118 }
2119
2120 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
2121 {
2122         if (!tbl->it_size)
2123                 return;
2124
2125         free_pages(tbl->it_base, get_order(tbl->it_size << 3));
2126 }
2127
2128 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2129                                        struct pnv_ioda_pe *pe)
2130 {
2131         struct iommu_table *tbl;
2132         int64_t rc;
2133
2134         /* We shouldn't already have a 32-bit DMA associated */
2135         if (WARN_ON(pe->tce32_seg >= 0))
2136                 return;
2137
2138         /* TVE #1 is selected by PCI address bit 59 */
2139         pe->tce_bypass_base = 1ull << 59;
2140
2141         tbl = pnv_pci_table_alloc(phb->hose->node);
2142         iommu_register_group(&pe->table_group, phb->hose->global_number,
2143                         pe->pe_number);
2144         pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
2145
2146         /* The PE will reserve all possible 32-bits space */
2147         pe->tce32_seg = 0;
2148         pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
2149                 phb->ioda.m32_pci_base);
2150
2151         /* Setup linux iommu table */
2152         rc = pnv_pci_ioda2_table_alloc_pages(pe->phb->hose->node,
2153                         0, IOMMU_PAGE_SHIFT_4K, phb->ioda.m32_pci_base, tbl);
2154         if (rc) {
2155                 pe_err(pe, "Failed to create 32-bit TCE table, err %ld", rc);
2156                 goto fail;
2157         }
2158
2159         tbl->it_ops = &pnv_ioda2_iommu_ops;
2160         iommu_init_table(tbl, phb->hose->node);
2161 #ifdef CONFIG_IOMMU_API
2162         pe->table_group.ops = &pnv_pci_ioda2_ops;
2163 #endif
2164
2165         rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
2166         if (rc) {
2167                 pe_err(pe, "Failed to configure 32-bit TCE table,"
2168                        " err %ld\n", rc);
2169                 goto fail;
2170         }
2171
2172         /* OPAL variant of PHB3 invalidated TCEs */
2173         if (phb->ioda.tce_inval_reg)
2174                 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
2175
2176         if (pe->flags & PNV_IODA_PE_DEV) {
2177                 /*
2178                  * Setting table base here only for carrying iommu_group
2179                  * further down to let iommu_add_device() do the job.
2180                  * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2181                  */
2182                 set_iommu_table_base(&pe->pdev->dev, tbl);
2183                 iommu_add_device(&pe->pdev->dev);
2184         } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
2185                 pnv_ioda_setup_bus_dma(pe, pe->pbus);
2186
2187         /* Also create a bypass window */
2188         if (!pnv_iommu_bypass_disabled)
2189                 pnv_pci_ioda2_set_bypass(pe, true);
2190
2191         return;
2192 fail:
2193         if (pe->tce32_seg >= 0)
2194                 pe->tce32_seg = -1;
2195         if (tbl) {
2196                 pnv_pci_ioda2_table_free_pages(tbl);
2197                 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
2198                 iommu_free_table(tbl, "pnv");
2199         }
2200 }
2201
2202 static void pnv_ioda_setup_dma(struct pnv_phb *phb)
2203 {
2204         struct pci_controller *hose = phb->hose;
2205         unsigned int residual, remaining, segs, tw, base;
2206         struct pnv_ioda_pe *pe;
2207
2208         /* If we have more PE# than segments available, hand out one
2209          * per PE until we run out and let the rest fail. If not,
2210          * then we assign at least one segment per PE, plus more based
2211          * on the amount of devices under that PE
2212          */
2213         if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2214                 residual = 0;
2215         else
2216                 residual = phb->ioda.tce32_count -
2217                         phb->ioda.dma_pe_count;
2218
2219         pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2220                 hose->global_number, phb->ioda.tce32_count);
2221         pr_info("PCI: %d PE# for a total weight of %d\n",
2222                 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2223
2224         pnv_pci_ioda_setup_opal_tce_kill(phb);
2225
2226         /* Walk our PE list and configure their DMA segments, hand them
2227          * out one base segment plus any residual segments based on
2228          * weight
2229          */
2230         remaining = phb->ioda.tce32_count;
2231         tw = phb->ioda.dma_weight;
2232         base = 0;
2233         list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
2234                 if (!pe->dma_weight)
2235                         continue;
2236                 if (!remaining) {
2237                         pe_warn(pe, "No DMA32 resources available\n");
2238                         continue;
2239                 }
2240                 segs = 1;
2241                 if (residual) {
2242                         segs += ((pe->dma_weight * residual)  + (tw / 2)) / tw;
2243                         if (segs > remaining)
2244                                 segs = remaining;
2245                 }
2246
2247                 /*
2248                  * For IODA2 compliant PHB3, we needn't care about the weight.
2249                  * The all available 32-bits DMA space will be assigned to
2250                  * the specific PE.
2251                  */
2252                 if (phb->type == PNV_PHB_IODA1) {
2253                         pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2254                                 pe->dma_weight, segs);
2255                         pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2256                 } else {
2257                         pe_info(pe, "Assign DMA32 space\n");
2258                         segs = 0;
2259                         pnv_pci_ioda2_setup_dma_pe(phb, pe);
2260                 }
2261
2262                 remaining -= segs;
2263                 base += segs;
2264         }
2265 }
2266
2267 #ifdef CONFIG_PCI_MSI
2268 static void pnv_ioda2_msi_eoi(struct irq_data *d)
2269 {
2270         unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2271         struct irq_chip *chip = irq_data_get_irq_chip(d);
2272         struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2273                                            ioda.irq_chip);
2274         int64_t rc;
2275
2276         rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2277         WARN_ON_ONCE(rc);
2278
2279         icp_native_eoi(d);
2280 }
2281
2282
2283 static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2284 {
2285         struct irq_data *idata;
2286         struct irq_chip *ichip;
2287
2288         if (phb->type != PNV_PHB_IODA2)
2289                 return;
2290
2291         if (!phb->ioda.irq_chip_init) {
2292                 /*
2293                  * First time we setup an MSI IRQ, we need to setup the
2294                  * corresponding IRQ chip to route correctly.
2295                  */
2296                 idata = irq_get_irq_data(virq);
2297                 ichip = irq_data_get_irq_chip(idata);
2298                 phb->ioda.irq_chip_init = 1;
2299                 phb->ioda.irq_chip = *ichip;
2300                 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2301         }
2302         irq_set_chip(virq, &phb->ioda.irq_chip);
2303 }
2304
2305 #ifdef CONFIG_CXL_BASE
2306
2307 struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
2308 {
2309         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2310
2311         return of_node_get(hose->dn);
2312 }
2313 EXPORT_SYMBOL(pnv_pci_get_phb_node);
2314
2315 int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
2316 {
2317         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2318         struct pnv_phb *phb = hose->private_data;
2319         struct pnv_ioda_pe *pe;
2320         int rc;
2321
2322         pe = pnv_ioda_get_pe(dev);
2323         if (!pe)
2324                 return -ENODEV;
2325
2326         pe_info(pe, "Switching PHB to CXL\n");
2327
2328         rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
2329         if (rc)
2330                 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2331
2332         return rc;
2333 }
2334 EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
2335
2336 /* Find PHB for cxl dev and allocate MSI hwirqs?
2337  * Returns the absolute hardware IRQ number
2338  */
2339 int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2340 {
2341         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2342         struct pnv_phb *phb = hose->private_data;
2343         int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2344
2345         if (hwirq < 0) {
2346                 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2347                 return -ENOSPC;
2348         }
2349
2350         return phb->msi_base + hwirq;
2351 }
2352 EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2353
2354 void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2355 {
2356         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2357         struct pnv_phb *phb = hose->private_data;
2358
2359         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2360 }
2361 EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2362
2363 void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2364                                   struct pci_dev *dev)
2365 {
2366         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2367         struct pnv_phb *phb = hose->private_data;
2368         int i, hwirq;
2369
2370         for (i = 1; i < CXL_IRQ_RANGES; i++) {
2371                 if (!irqs->range[i])
2372                         continue;
2373                 pr_devel("cxl release irq range 0x%x: offset: 0x%lx  limit: %ld\n",
2374                          i, irqs->offset[i],
2375                          irqs->range[i]);
2376                 hwirq = irqs->offset[i] - phb->msi_base;
2377                 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2378                                        irqs->range[i]);
2379         }
2380 }
2381 EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2382
2383 int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2384                                struct pci_dev *dev, int num)
2385 {
2386         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2387         struct pnv_phb *phb = hose->private_data;
2388         int i, hwirq, try;
2389
2390         memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2391
2392         /* 0 is reserved for the multiplexed PSL DSI interrupt */
2393         for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2394                 try = num;
2395                 while (try) {
2396                         hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2397                         if (hwirq >= 0)
2398                                 break;
2399                         try /= 2;
2400                 }
2401                 if (!try)
2402                         goto fail;
2403
2404                 irqs->offset[i] = phb->msi_base + hwirq;
2405                 irqs->range[i] = try;
2406                 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx  limit: %li\n",
2407                          i, irqs->offset[i], irqs->range[i]);
2408                 num -= try;
2409         }
2410         if (num)
2411                 goto fail;
2412
2413         return 0;
2414 fail:
2415         pnv_cxl_release_hwirq_ranges(irqs, dev);
2416         return -ENOSPC;
2417 }
2418 EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2419
2420 int pnv_cxl_get_irq_count(struct pci_dev *dev)
2421 {
2422         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2423         struct pnv_phb *phb = hose->private_data;
2424
2425         return phb->msi_bmp.irq_count;
2426 }
2427 EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2428
2429 int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2430                            unsigned int virq)
2431 {
2432         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2433         struct pnv_phb *phb = hose->private_data;
2434         unsigned int xive_num = hwirq - phb->msi_base;
2435         struct pnv_ioda_pe *pe;
2436         int rc;
2437
2438         if (!(pe = pnv_ioda_get_pe(dev)))
2439                 return -ENODEV;
2440
2441         /* Assign XIVE to PE */
2442         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2443         if (rc) {
2444                 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2445                         "hwirq 0x%x XIVE 0x%x PE\n",
2446                         pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2447                 return -EIO;
2448         }
2449         set_msi_irq_chip(phb, virq);
2450
2451         return 0;
2452 }
2453 EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2454 #endif
2455
2456 static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
2457                                   unsigned int hwirq, unsigned int virq,
2458                                   unsigned int is_64, struct msi_msg *msg)
2459 {
2460         struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2461         unsigned int xive_num = hwirq - phb->msi_base;
2462         __be32 data;
2463         int rc;
2464
2465         /* No PE assigned ? bail out ... no MSI for you ! */
2466         if (pe == NULL)
2467                 return -ENXIO;
2468
2469         /* Check if we have an MVE */
2470         if (pe->mve_number < 0)
2471                 return -ENXIO;
2472
2473         /* Force 32-bit MSI on some broken devices */
2474         if (dev->no_64bit_msi)
2475                 is_64 = 0;
2476
2477         /* Assign XIVE to PE */
2478         rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2479         if (rc) {
2480                 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2481                         pci_name(dev), rc, xive_num);
2482                 return -EIO;
2483         }
2484
2485         if (is_64) {
2486                 __be64 addr64;
2487
2488                 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2489                                      &addr64, &data);
2490                 if (rc) {
2491                         pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2492                                 pci_name(dev), rc);
2493                         return -EIO;
2494                 }
2495                 msg->address_hi = be64_to_cpu(addr64) >> 32;
2496                 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
2497         } else {
2498                 __be32 addr32;
2499
2500                 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2501                                      &addr32, &data);
2502                 if (rc) {
2503                         pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2504                                 pci_name(dev), rc);
2505                         return -EIO;
2506                 }
2507                 msg->address_hi = 0;
2508                 msg->address_lo = be32_to_cpu(addr32);
2509         }
2510         msg->data = be32_to_cpu(data);
2511
2512         set_msi_irq_chip(phb, virq);
2513
2514         pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2515                  " address=%x_%08x data=%x PE# %d\n",
2516                  pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2517                  msg->address_hi, msg->address_lo, data, pe->pe_number);
2518
2519         return 0;
2520 }
2521
2522 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2523 {
2524         unsigned int count;
2525         const __be32 *prop = of_get_property(phb->hose->dn,
2526                                              "ibm,opal-msi-ranges", NULL);
2527         if (!prop) {
2528                 /* BML Fallback */
2529                 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2530         }
2531         if (!prop)
2532                 return;
2533
2534         phb->msi_base = be32_to_cpup(prop);
2535         count = be32_to_cpup(prop + 1);
2536         if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
2537                 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2538                        phb->hose->global_number);
2539                 return;
2540         }
2541
2542         phb->msi_setup = pnv_pci_ioda_msi_setup;
2543         phb->msi32_support = 1;
2544         pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
2545                 count, phb->msi_base);
2546 }
2547 #else
2548 static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2549 #endif /* CONFIG_PCI_MSI */
2550
2551 #ifdef CONFIG_PCI_IOV
2552 static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2553 {
2554         struct pci_controller *hose;
2555         struct pnv_phb *phb;
2556         struct resource *res;
2557         int i;
2558         resource_size_t size;
2559         struct pci_dn *pdn;
2560         int mul, total_vfs;
2561
2562         if (!pdev->is_physfn || pdev->is_added)
2563                 return;
2564
2565         hose = pci_bus_to_host(pdev->bus);
2566         phb = hose->private_data;
2567
2568         pdn = pci_get_pdn(pdev);
2569         pdn->vfs_expanded = 0;
2570
2571         total_vfs = pci_sriov_get_totalvfs(pdev);
2572         pdn->m64_per_iov = 1;
2573         mul = phb->ioda.total_pe;
2574
2575         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2576                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2577                 if (!res->flags || res->parent)
2578                         continue;
2579                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2580                         dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2581                                  i, res);
2582                         continue;
2583                 }
2584
2585                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2586
2587                 /* bigger than 64M */
2588                 if (size > (1 << 26)) {
2589                         dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2590                                  i, res);
2591                         pdn->m64_per_iov = M64_PER_IOV;
2592                         mul = roundup_pow_of_two(total_vfs);
2593                         break;
2594                 }
2595         }
2596
2597         for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2598                 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2599                 if (!res->flags || res->parent)
2600                         continue;
2601                 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2602                         dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2603                                  i, res);
2604                         continue;
2605                 }
2606
2607                 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2608                 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2609                 res->end = res->start + size * mul - 1;
2610                 dev_dbg(&pdev->dev, "                       %pR\n", res);
2611                 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
2612                          i, res, mul);
2613         }
2614         pdn->vfs_expanded = mul;
2615 }
2616 #endif /* CONFIG_PCI_IOV */
2617
2618 /*
2619  * This function is supposed to be called on basis of PE from top
2620  * to bottom style. So the the I/O or MMIO segment assigned to
2621  * parent PE could be overrided by its child PEs if necessary.
2622  */
2623 static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2624                                   struct pnv_ioda_pe *pe)
2625 {
2626         struct pnv_phb *phb = hose->private_data;
2627         struct pci_bus_region region;
2628         struct resource *res;
2629         int i, index;
2630         int rc;
2631
2632         /*
2633          * NOTE: We only care PCI bus based PE for now. For PCI
2634          * device based PE, for example SRIOV sensitive VF should
2635          * be figured out later.
2636          */
2637         BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2638
2639         pci_bus_for_each_resource(pe->pbus, res, i) {
2640                 if (!res || !res->flags ||
2641                     res->start > res->end)
2642                         continue;
2643
2644                 if (res->flags & IORESOURCE_IO) {
2645                         region.start = res->start - phb->ioda.io_pci_base;
2646                         region.end   = res->end - phb->ioda.io_pci_base;
2647                         index = region.start / phb->ioda.io_segsize;
2648
2649                         while (index < phb->ioda.total_pe &&
2650                                region.start <= region.end) {
2651                                 phb->ioda.io_segmap[index] = pe->pe_number;
2652                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2653                                         pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2654                                 if (rc != OPAL_SUCCESS) {
2655                                         pr_err("%s: OPAL error %d when mapping IO "
2656                                                "segment #%d to PE#%d\n",
2657                                                __func__, rc, index, pe->pe_number);
2658                                         break;
2659                                 }
2660
2661                                 region.start += phb->ioda.io_segsize;
2662                                 index++;
2663                         }
2664                 } else if ((res->flags & IORESOURCE_MEM) &&
2665                            !pnv_pci_is_mem_pref_64(res->flags)) {
2666                         region.start = res->start -
2667                                        hose->mem_offset[0] -
2668                                        phb->ioda.m32_pci_base;
2669                         region.end   = res->end -
2670                                        hose->mem_offset[0] -
2671                                        phb->ioda.m32_pci_base;
2672                         index = region.start / phb->ioda.m32_segsize;
2673
2674                         while (index < phb->ioda.total_pe &&
2675                                region.start <= region.end) {
2676                                 phb->ioda.m32_segmap[index] = pe->pe_number;
2677                                 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2678                                         pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2679                                 if (rc != OPAL_SUCCESS) {
2680                                         pr_err("%s: OPAL error %d when mapping M32 "
2681                                                "segment#%d to PE#%d",
2682                                                __func__, rc, index, pe->pe_number);
2683                                         break;
2684                                 }
2685
2686                                 region.start += phb->ioda.m32_segsize;
2687                                 index++;
2688                         }
2689                 }
2690         }
2691 }
2692
2693 static void pnv_pci_ioda_setup_seg(void)
2694 {
2695         struct pci_controller *tmp, *hose;
2696         struct pnv_phb *phb;
2697         struct pnv_ioda_pe *pe;
2698
2699         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2700                 phb = hose->private_data;
2701                 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2702                         pnv_ioda_setup_pe_seg(hose, pe);
2703                 }
2704         }
2705 }
2706
2707 static void pnv_pci_ioda_setup_DMA(void)
2708 {
2709         struct pci_controller *hose, *tmp;
2710         struct pnv_phb *phb;
2711
2712         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2713                 pnv_ioda_setup_dma(hose->private_data);
2714
2715                 /* Mark the PHB initialization done */
2716                 phb = hose->private_data;
2717                 phb->initialized = 1;
2718         }
2719 }
2720
2721 static void pnv_pci_ioda_create_dbgfs(void)
2722 {
2723 #ifdef CONFIG_DEBUG_FS
2724         struct pci_controller *hose, *tmp;
2725         struct pnv_phb *phb;
2726         char name[16];
2727
2728         list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2729                 phb = hose->private_data;
2730
2731                 sprintf(name, "PCI%04x", hose->global_number);
2732                 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2733                 if (!phb->dbgfs)
2734                         pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2735                                 __func__, hose->global_number);
2736         }
2737 #endif /* CONFIG_DEBUG_FS */
2738 }
2739
2740 static void pnv_pci_ioda_fixup(void)
2741 {
2742         pnv_pci_ioda_setup_PEs();
2743         pnv_pci_ioda_setup_seg();
2744         pnv_pci_ioda_setup_DMA();
2745
2746         pnv_pci_ioda_create_dbgfs();
2747
2748 #ifdef CONFIG_EEH
2749         eeh_init();
2750         eeh_addr_cache_build();
2751 #endif
2752 }
2753
2754 /*
2755  * Returns the alignment for I/O or memory windows for P2P
2756  * bridges. That actually depends on how PEs are segmented.
2757  * For now, we return I/O or M32 segment size for PE sensitive
2758  * P2P bridges. Otherwise, the default values (4KiB for I/O,
2759  * 1MiB for memory) will be returned.
2760  *
2761  * The current PCI bus might be put into one PE, which was
2762  * create against the parent PCI bridge. For that case, we
2763  * needn't enlarge the alignment so that we can save some
2764  * resources.
2765  */
2766 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2767                                                 unsigned long type)
2768 {
2769         struct pci_dev *bridge;
2770         struct pci_controller *hose = pci_bus_to_host(bus);
2771         struct pnv_phb *phb = hose->private_data;
2772         int num_pci_bridges = 0;
2773
2774         bridge = bus->self;
2775         while (bridge) {
2776                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2777                         num_pci_bridges++;
2778                         if (num_pci_bridges >= 2)
2779                                 return 1;
2780                 }
2781
2782                 bridge = bridge->bus->self;
2783         }
2784
2785         /* We fail back to M32 if M64 isn't supported */
2786         if (phb->ioda.m64_segsize &&
2787             pnv_pci_is_mem_pref_64(type))
2788                 return phb->ioda.m64_segsize;
2789         if (type & IORESOURCE_MEM)
2790                 return phb->ioda.m32_segsize;
2791
2792         return phb->ioda.io_segsize;
2793 }
2794
2795 #ifdef CONFIG_PCI_IOV
2796 static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2797                                                       int resno)
2798 {
2799         struct pci_dn *pdn = pci_get_pdn(pdev);
2800         resource_size_t align, iov_align;
2801
2802         iov_align = resource_size(&pdev->resource[resno]);
2803         if (iov_align)
2804                 return iov_align;
2805
2806         align = pci_iov_resource_size(pdev, resno);
2807         if (pdn->vfs_expanded)
2808                 return pdn->vfs_expanded * align;
2809
2810         return align;
2811 }
2812 #endif /* CONFIG_PCI_IOV */
2813
2814 /* Prevent enabling devices for which we couldn't properly
2815  * assign a PE
2816  */
2817 static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
2818 {
2819         struct pci_controller *hose = pci_bus_to_host(dev->bus);
2820         struct pnv_phb *phb = hose->private_data;
2821         struct pci_dn *pdn;
2822
2823         /* The function is probably called while the PEs have
2824          * not be created yet. For example, resource reassignment
2825          * during PCI probe period. We just skip the check if
2826          * PEs isn't ready.
2827          */
2828         if (!phb->initialized)
2829                 return true;
2830
2831         pdn = pci_get_pdn(dev);
2832         if (!pdn || pdn->pe_number == IODA_INVALID_PE)
2833                 return false;
2834
2835         return true;
2836 }
2837
2838 static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
2839                                u32 devfn)
2840 {
2841         return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
2842 }
2843
2844 static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
2845 {
2846         struct pnv_phb *phb = hose->private_data;
2847
2848         opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
2849                        OPAL_ASSERT_RESET);
2850 }
2851
2852 static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2853        .dma_dev_setup = pnv_pci_dma_dev_setup,
2854 #ifdef CONFIG_PCI_MSI
2855        .setup_msi_irqs = pnv_setup_msi_irqs,
2856        .teardown_msi_irqs = pnv_teardown_msi_irqs,
2857 #endif
2858        .enable_device_hook = pnv_pci_enable_device_hook,
2859        .window_alignment = pnv_pci_window_alignment,
2860        .reset_secondary_bus = pnv_pci_reset_secondary_bus,
2861        .dma_set_mask = pnv_pci_ioda_dma_set_mask,
2862        .shutdown = pnv_pci_ioda_shutdown,
2863 };
2864
2865 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2866                                          u64 hub_id, int ioda_type)
2867 {
2868         struct pci_controller *hose;
2869         struct pnv_phb *phb;
2870         unsigned long size, m32map_off, pemap_off, iomap_off = 0;
2871         const __be64 *prop64;
2872         const __be32 *prop32;
2873         int len;
2874         u64 phb_id;
2875         void *aux;
2876         long rc;
2877
2878         pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
2879
2880         prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2881         if (!prop64) {
2882                 pr_err("  Missing \"ibm,opal-phbid\" property !\n");
2883                 return;
2884         }
2885         phb_id = be64_to_cpup(prop64);
2886         pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
2887
2888         phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
2889
2890         /* Allocate PCI controller */
2891         phb->hose = hose = pcibios_alloc_controller(np);
2892         if (!phb->hose) {
2893                 pr_err("  Can't allocate PCI controller for %s\n",
2894                        np->full_name);
2895                 memblock_free(__pa(phb), sizeof(struct pnv_phb));
2896                 return;
2897         }
2898
2899         spin_lock_init(&phb->lock);
2900         prop32 = of_get_property(np, "bus-range", &len);
2901         if (prop32 && len == 8) {
2902                 hose->first_busno = be32_to_cpu(prop32[0]);
2903                 hose->last_busno = be32_to_cpu(prop32[1]);
2904         } else {
2905                 pr_warn("  Broken <bus-range> on %s\n", np->full_name);
2906                 hose->first_busno = 0;
2907                 hose->last_busno = 0xff;
2908         }
2909         hose->private_data = phb;
2910         phb->hub_id = hub_id;
2911         phb->opal_id = phb_id;
2912         phb->type = ioda_type;
2913         mutex_init(&phb->ioda.pe_alloc_mutex);
2914
2915         /* Detect specific models for error handling */
2916         if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2917                 phb->model = PNV_PHB_MODEL_P7IOC;
2918         else if (of_device_is_compatible(np, "ibm,power8-pciex"))
2919                 phb->model = PNV_PHB_MODEL_PHB3;
2920         else
2921                 phb->model = PNV_PHB_MODEL_UNKNOWN;
2922
2923         /* Parse 32-bit and IO ranges (if any) */
2924         pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
2925
2926         /* Get registers */
2927         phb->regs = of_iomap(np, 0);
2928         if (phb->regs == NULL)
2929                 pr_err("  Failed to map registers !\n");
2930
2931         /* Initialize more IODA stuff */
2932         phb->ioda.total_pe = 1;
2933         prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
2934         if (prop32)
2935                 phb->ioda.total_pe = be32_to_cpup(prop32);
2936         prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
2937         if (prop32)
2938                 phb->ioda.reserved_pe = be32_to_cpup(prop32);
2939
2940         /* Parse 64-bit MMIO range */
2941         pnv_ioda_parse_m64_window(phb);
2942
2943         phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
2944         /* FW Has already off top 64k of M32 space (MSI space) */
2945         phb->ioda.m32_size += 0x10000;
2946
2947         phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
2948         phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
2949         phb->ioda.io_size = hose->pci_io_size;
2950         phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
2951         phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
2952
2953         /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
2954         size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
2955         m32map_off = size;
2956         size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
2957         if (phb->type == PNV_PHB_IODA1) {
2958                 iomap_off = size;
2959                 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
2960         }
2961         pemap_off = size;
2962         size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
2963         aux = memblock_virt_alloc(size, 0);
2964         phb->ioda.pe_alloc = aux;
2965         phb->ioda.m32_segmap = aux + m32map_off;
2966         if (phb->type == PNV_PHB_IODA1)
2967                 phb->ioda.io_segmap = aux + iomap_off;
2968         phb->ioda.pe_array = aux + pemap_off;
2969         set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
2970
2971         INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
2972         INIT_LIST_HEAD(&phb->ioda.pe_list);
2973         mutex_init(&phb->ioda.pe_list_mutex);
2974
2975         /* Calculate how many 32-bit TCE segments we have */
2976         phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
2977
2978 #if 0 /* We should really do that ... */
2979         rc = opal_pci_set_phb_mem_window(opal->phb_id,
2980                                          window_type,
2981                                          window_num,
2982                                          starting_real_address,
2983                                          starting_pci_address,
2984                                          segment_size);
2985 #endif
2986
2987         pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
2988                 phb->ioda.total_pe, phb->ioda.reserved_pe,
2989                 phb->ioda.m32_size, phb->ioda.m32_segsize);
2990         if (phb->ioda.m64_size)
2991                 pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
2992                         phb->ioda.m64_size, phb->ioda.m64_segsize);
2993         if (phb->ioda.io_size)
2994                 pr_info("                  IO: 0x%x [segment=0x%x]\n",
2995                         phb->ioda.io_size, phb->ioda.io_segsize);
2996
2997
2998         phb->hose->ops = &pnv_pci_ops;
2999         phb->get_pe_state = pnv_ioda_get_pe_state;
3000         phb->freeze_pe = pnv_ioda_freeze_pe;
3001         phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
3002
3003         /* Setup RID -> PE mapping function */
3004         phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
3005
3006         /* Setup TCEs */
3007         phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
3008         phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
3009
3010         /* Setup MSI support */
3011         pnv_pci_init_ioda_msis(phb);
3012
3013         /*
3014          * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3015          * to let the PCI core do resource assignment. It's supposed
3016          * that the PCI core will do correct I/O and MMIO alignment
3017          * for the P2P bridge bars so that each PCI bus (excluding
3018          * the child P2P bridges) can form individual PE.
3019          */
3020         ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
3021         hose->controller_ops = pnv_pci_ioda_controller_ops;
3022
3023 #ifdef CONFIG_PCI_IOV
3024         ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
3025         ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
3026 #endif
3027
3028         pci_add_flags(PCI_REASSIGN_ALL_RSRC);
3029
3030         /* Reset IODA tables to a clean state */
3031         rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
3032         if (rc)
3033                 pr_warning("  OPAL Error %ld performing IODA table reset !\n", rc);
3034
3035         /* If we're running in kdump kerenl, the previous kerenl never
3036          * shutdown PCI devices correctly. We already got IODA table
3037          * cleaned out. So we have to issue PHB reset to stop all PCI
3038          * transactions from previous kerenl.
3039          */
3040         if (is_kdump_kernel()) {
3041                 pr_info("  Issue PHB reset ...\n");
3042                 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3043                 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
3044         }
3045
3046         /* Remove M64 resource if we can't configure it successfully */
3047         if (!phb->init_m64 || phb->init_m64(phb))
3048                 hose->mem_resources[1].flags = 0;
3049 }
3050
3051 void __init pnv_pci_init_ioda2_phb(struct device_node *np)
3052 {
3053         pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
3054 }
3055
3056 void __init pnv_pci_init_ioda_hub(struct device_node *np)
3057 {
3058         struct device_node *phbn;
3059         const __be64 *prop64;
3060         u64 hub_id;
3061
3062         pr_info("Probing IODA IO-Hub %s\n", np->full_name);
3063
3064         prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3065         if (!prop64) {
3066                 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3067                 return;
3068         }
3069         hub_id = be64_to_cpup(prop64);
3070         pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3071
3072         /* Count child PHBs */
3073         for_each_child_of_node(np, phbn) {
3074                 /* Look for IODA1 PHBs */
3075                 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
3076                         pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
3077         }
3078 }