2 * Derived from arch/powerpc/kernel/iommu.c
4 * Copyright IBM Corporation, 2006-2007
5 * Copyright (C) 2006 Jon Mason <jdmason@kudzu.us>
7 * Author: Jon Mason <jdmason@kudzu.us>
8 * Author: Muli Ben-Yehuda <muli@il.ibm.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #include <linux/kernel.h>
26 #include <linux/init.h>
27 #include <linux/types.h>
28 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/string.h>
32 #include <linux/crash_dump.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/bitmap.h>
35 #include <linux/pci_ids.h>
36 #include <linux/pci.h>
37 #include <linux/delay.h>
38 #include <linux/scatterlist.h>
39 #include <linux/iommu-helper.h>
41 #include <asm/iommu.h>
42 #include <asm/calgary.h>
44 #include <asm/pci-direct.h>
45 #include <asm/system.h>
48 #include <asm/bios_ebda.h>
49 #include <asm/x86_init.h>
51 #ifdef CONFIG_CALGARY_IOMMU_ENABLED_BY_DEFAULT
52 int use_calgary __read_mostly = 1;
54 int use_calgary __read_mostly = 0;
55 #endif /* CONFIG_CALGARY_DEFAULT_ENABLED */
57 #define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
58 #define PCI_DEVICE_ID_IBM_CALIOC2 0x0308
60 /* register offsets inside the host bridge space */
61 #define CALGARY_CONFIG_REG 0x0108
62 #define PHB_CSR_OFFSET 0x0110 /* Channel Status */
63 #define PHB_PLSSR_OFFSET 0x0120
64 #define PHB_CONFIG_RW_OFFSET 0x0160
65 #define PHB_IOBASE_BAR_LOW 0x0170
66 #define PHB_IOBASE_BAR_HIGH 0x0180
67 #define PHB_MEM_1_LOW 0x0190
68 #define PHB_MEM_1_HIGH 0x01A0
69 #define PHB_IO_ADDR_SIZE 0x01B0
70 #define PHB_MEM_1_SIZE 0x01C0
71 #define PHB_MEM_ST_OFFSET 0x01D0
72 #define PHB_AER_OFFSET 0x0200
73 #define PHB_CONFIG_0_HIGH 0x0220
74 #define PHB_CONFIG_0_LOW 0x0230
75 #define PHB_CONFIG_0_END 0x0240
76 #define PHB_MEM_2_LOW 0x02B0
77 #define PHB_MEM_2_HIGH 0x02C0
78 #define PHB_MEM_2_SIZE_HIGH 0x02D0
79 #define PHB_MEM_2_SIZE_LOW 0x02E0
80 #define PHB_DOSHOLE_OFFSET 0x08E0
82 /* CalIOC2 specific */
83 #define PHB_SAVIOR_L2 0x0DB0
84 #define PHB_PAGE_MIG_CTRL 0x0DA8
85 #define PHB_PAGE_MIG_DEBUG 0x0DA0
86 #define PHB_ROOT_COMPLEX_STATUS 0x0CB0
89 #define PHB_TCE_ENABLE 0x20000000
90 #define PHB_SLOT_DISABLE 0x1C000000
91 #define PHB_DAC_DISABLE 0x01000000
92 #define PHB_MEM2_ENABLE 0x00400000
93 #define PHB_MCSR_ENABLE 0x00100000
94 /* TAR (Table Address Register) */
95 #define TAR_SW_BITS 0x0000ffffffff800fUL
96 #define TAR_VALID 0x0000000000000008UL
97 /* CSR (Channel/DMA Status Register) */
98 #define CSR_AGENT_MASK 0xffe0ffff
99 /* CCR (Calgary Configuration Register) */
100 #define CCR_2SEC_TIMEOUT 0x000000000000000EUL
101 /* PMCR/PMDR (Page Migration Control/Debug Registers */
102 #define PMR_SOFTSTOP 0x80000000
103 #define PMR_SOFTSTOPFAULT 0x40000000
104 #define PMR_HARDSTOP 0x20000000
107 * The maximum PHB bus number.
108 * x3950M2 (rare): 8 chassis, 48 PHBs per chassis = 384
109 * x3950M2: 4 chassis, 48 PHBs per chassis = 192
110 * x3950 (PCIE): 8 chassis, 32 PHBs per chassis = 256
111 * x3950 (PCIX): 8 chassis, 16 PHBs per chassis = 128
113 #define MAX_PHB_BUS_NUM 384
115 #define PHBS_PER_CALGARY 4
117 /* register offsets in Calgary's internal register space */
118 static const unsigned long tar_offsets[] = {
125 static const unsigned long split_queue_offsets[] = {
126 0x4870 /* SPLIT QUEUE 0 */,
127 0x5870 /* SPLIT QUEUE 1 */,
128 0x6870 /* SPLIT QUEUE 2 */,
129 0x7870 /* SPLIT QUEUE 3 */
132 static const unsigned long phb_offsets[] = {
139 /* PHB debug registers */
141 static const unsigned long phb_debug_offsets[] = {
142 0x4000 /* PHB 0 DEBUG */,
143 0x5000 /* PHB 1 DEBUG */,
144 0x6000 /* PHB 2 DEBUG */,
145 0x7000 /* PHB 3 DEBUG */
149 * STUFF register for each debug PHB,
150 * byte 1 = start bus number, byte 2 = end bus number
153 #define PHB_DEBUG_STUFF_OFFSET 0x0020
155 #define EMERGENCY_PAGES 32 /* = 128KB */
157 unsigned int specified_table_size = TCE_TABLE_SIZE_UNSPECIFIED;
158 static int translate_empty_slots __read_mostly = 0;
159 static int calgary_detected __read_mostly = 0;
161 static struct rio_table_hdr *rio_table_hdr __initdata;
162 static struct scal_detail *scal_devs[MAX_NUMNODES] __initdata;
163 static struct rio_detail *rio_devs[MAX_NUMNODES * 4] __initdata;
165 struct calgary_bus_info {
167 unsigned char translation_disabled;
172 static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
173 static void calgary_tce_cache_blast(struct iommu_table *tbl);
174 static void calgary_dump_error_regs(struct iommu_table *tbl);
175 static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
176 static void calioc2_tce_cache_blast(struct iommu_table *tbl);
177 static void calioc2_dump_error_regs(struct iommu_table *tbl);
178 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl);
179 static void get_tce_space_from_tar(void);
181 static struct cal_chipset_ops calgary_chip_ops = {
182 .handle_quirks = calgary_handle_quirks,
183 .tce_cache_blast = calgary_tce_cache_blast,
184 .dump_error_regs = calgary_dump_error_regs
187 static struct cal_chipset_ops calioc2_chip_ops = {
188 .handle_quirks = calioc2_handle_quirks,
189 .tce_cache_blast = calioc2_tce_cache_blast,
190 .dump_error_regs = calioc2_dump_error_regs
193 static struct calgary_bus_info bus_info[MAX_PHB_BUS_NUM] = { { NULL, 0, 0 }, };
195 static inline int translation_enabled(struct iommu_table *tbl)
197 /* only PHBs with translation enabled have an IOMMU table */
198 return (tbl != NULL);
201 static void iommu_range_reserve(struct iommu_table *tbl,
202 unsigned long start_addr, unsigned int npages)
208 index = start_addr >> PAGE_SHIFT;
210 /* bail out if we're asked to reserve a region we don't cover */
211 if (index >= tbl->it_size)
214 end = index + npages;
215 if (end > tbl->it_size) /* don't go off the table */
218 spin_lock_irqsave(&tbl->it_lock, flags);
220 bitmap_set(tbl->it_map, index, npages);
222 spin_unlock_irqrestore(&tbl->it_lock, flags);
225 static unsigned long iommu_range_alloc(struct device *dev,
226 struct iommu_table *tbl,
230 unsigned long offset;
231 unsigned long boundary_size;
233 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
234 PAGE_SIZE) >> PAGE_SHIFT;
238 spin_lock_irqsave(&tbl->it_lock, flags);
240 offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
241 npages, 0, boundary_size, 0);
242 if (offset == ~0UL) {
243 tbl->chip_ops->tce_cache_blast(tbl);
245 offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
246 npages, 0, boundary_size, 0);
247 if (offset == ~0UL) {
248 printk(KERN_WARNING "Calgary: IOMMU full.\n");
249 spin_unlock_irqrestore(&tbl->it_lock, flags);
250 if (panic_on_overflow)
251 panic("Calgary: fix the allocator.\n");
253 return DMA_ERROR_CODE;
257 tbl->it_hint = offset + npages;
258 BUG_ON(tbl->it_hint > tbl->it_size);
260 spin_unlock_irqrestore(&tbl->it_lock, flags);
265 static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
266 void *vaddr, unsigned int npages, int direction)
271 entry = iommu_range_alloc(dev, tbl, npages);
273 if (unlikely(entry == DMA_ERROR_CODE)) {
274 printk(KERN_WARNING "Calgary: failed to allocate %u pages in "
275 "iommu %p\n", npages, tbl);
276 return DMA_ERROR_CODE;
279 /* set the return dma address */
280 ret = (entry << PAGE_SHIFT) | ((unsigned long)vaddr & ~PAGE_MASK);
282 /* put the TCEs in the HW table */
283 tce_build(tbl, entry, npages, (unsigned long)vaddr & PAGE_MASK,
288 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
292 unsigned long badend;
295 /* were we called with bad_dma_address? */
296 badend = DMA_ERROR_CODE + (EMERGENCY_PAGES * PAGE_SIZE);
297 if (unlikely((dma_addr >= DMA_ERROR_CODE) && (dma_addr < badend))) {
298 WARN(1, KERN_ERR "Calgary: driver tried unmapping bad DMA "
299 "address 0x%Lx\n", dma_addr);
303 entry = dma_addr >> PAGE_SHIFT;
305 BUG_ON(entry + npages > tbl->it_size);
307 tce_free(tbl, entry, npages);
309 spin_lock_irqsave(&tbl->it_lock, flags);
311 bitmap_clear(tbl->it_map, entry, npages);
313 spin_unlock_irqrestore(&tbl->it_lock, flags);
316 static inline struct iommu_table *find_iommu_table(struct device *dev)
318 struct pci_dev *pdev;
319 struct pci_bus *pbus;
320 struct iommu_table *tbl;
322 pdev = to_pci_dev(dev);
324 /* search up the device tree for an iommu */
327 tbl = pci_iommu(pbus);
328 if (tbl && tbl->it_busno == pbus->number)
334 BUG_ON(tbl && (tbl->it_busno != pbus->number));
339 static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist,
340 int nelems,enum dma_data_direction dir,
341 struct dma_attrs *attrs)
343 struct iommu_table *tbl = find_iommu_table(dev);
344 struct scatterlist *s;
347 if (!translation_enabled(tbl))
350 for_each_sg(sglist, s, nelems, i) {
352 dma_addr_t dma = s->dma_address;
353 unsigned int dmalen = s->dma_length;
358 npages = iommu_num_pages(dma, dmalen, PAGE_SIZE);
359 iommu_free(tbl, dma, npages);
363 static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
364 int nelems, enum dma_data_direction dir,
365 struct dma_attrs *attrs)
367 struct iommu_table *tbl = find_iommu_table(dev);
368 struct scatterlist *s;
374 for_each_sg(sg, s, nelems, i) {
377 vaddr = (unsigned long) sg_virt(s);
378 npages = iommu_num_pages(vaddr, s->length, PAGE_SIZE);
380 entry = iommu_range_alloc(dev, tbl, npages);
381 if (entry == DMA_ERROR_CODE) {
382 /* makes sure unmap knows to stop */
387 s->dma_address = (entry << PAGE_SHIFT) | s->offset;
389 /* insert into HW table */
390 tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);
392 s->dma_length = s->length;
397 calgary_unmap_sg(dev, sg, nelems, dir, NULL);
398 for_each_sg(sg, s, nelems, i) {
399 sg->dma_address = DMA_ERROR_CODE;
405 static dma_addr_t calgary_map_page(struct device *dev, struct page *page,
406 unsigned long offset, size_t size,
407 enum dma_data_direction dir,
408 struct dma_attrs *attrs)
410 void *vaddr = page_address(page) + offset;
413 struct iommu_table *tbl = find_iommu_table(dev);
415 uaddr = (unsigned long)vaddr;
416 npages = iommu_num_pages(uaddr, size, PAGE_SIZE);
418 return iommu_alloc(dev, tbl, vaddr, npages, dir);
421 static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr,
422 size_t size, enum dma_data_direction dir,
423 struct dma_attrs *attrs)
425 struct iommu_table *tbl = find_iommu_table(dev);
428 npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
429 iommu_free(tbl, dma_addr, npages);
432 static void* calgary_alloc_coherent(struct device *dev, size_t size,
433 dma_addr_t *dma_handle, gfp_t flag)
437 unsigned int npages, order;
438 struct iommu_table *tbl = find_iommu_table(dev);
440 size = PAGE_ALIGN(size); /* size rounded up to full pages */
441 npages = size >> PAGE_SHIFT;
442 order = get_order(size);
444 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
446 /* alloc enough pages (and possibly more) */
447 ret = (void *)__get_free_pages(flag, order);
450 memset(ret, 0, size);
452 /* set up tces to cover the allocated range */
453 mapping = iommu_alloc(dev, tbl, ret, npages, DMA_BIDIRECTIONAL);
454 if (mapping == DMA_ERROR_CODE)
456 *dma_handle = mapping;
459 free_pages((unsigned long)ret, get_order(size));
465 static void calgary_free_coherent(struct device *dev, size_t size,
466 void *vaddr, dma_addr_t dma_handle)
469 struct iommu_table *tbl = find_iommu_table(dev);
471 size = PAGE_ALIGN(size);
472 npages = size >> PAGE_SHIFT;
474 iommu_free(tbl, dma_handle, npages);
475 free_pages((unsigned long)vaddr, get_order(size));
478 static struct dma_map_ops calgary_dma_ops = {
479 .alloc_coherent = calgary_alloc_coherent,
480 .free_coherent = calgary_free_coherent,
481 .map_sg = calgary_map_sg,
482 .unmap_sg = calgary_unmap_sg,
483 .map_page = calgary_map_page,
484 .unmap_page = calgary_unmap_page,
487 static inline void __iomem * busno_to_bbar(unsigned char num)
489 return bus_info[num].bbar;
492 static inline int busno_to_phbid(unsigned char num)
494 return bus_info[num].phbid;
497 static inline unsigned long split_queue_offset(unsigned char num)
499 size_t idx = busno_to_phbid(num);
501 return split_queue_offsets[idx];
504 static inline unsigned long tar_offset(unsigned char num)
506 size_t idx = busno_to_phbid(num);
508 return tar_offsets[idx];
511 static inline unsigned long phb_offset(unsigned char num)
513 size_t idx = busno_to_phbid(num);
515 return phb_offsets[idx];
518 static inline void __iomem* calgary_reg(void __iomem *bar, unsigned long offset)
520 unsigned long target = ((unsigned long)bar) | offset;
521 return (void __iomem*)target;
524 static inline int is_calioc2(unsigned short device)
526 return (device == PCI_DEVICE_ID_IBM_CALIOC2);
529 static inline int is_calgary(unsigned short device)
531 return (device == PCI_DEVICE_ID_IBM_CALGARY);
534 static inline int is_cal_pci_dev(unsigned short device)
536 return (is_calgary(device) || is_calioc2(device));
539 static void calgary_tce_cache_blast(struct iommu_table *tbl)
544 void __iomem *bbar = tbl->bbar;
545 void __iomem *target;
547 /* disable arbitration on the bus */
548 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
552 /* read plssr to ensure it got there */
553 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
556 /* poll split queues until all DMA activity is done */
557 target = calgary_reg(bbar, split_queue_offset(tbl->it_busno));
561 } while ((val & 0xff) != 0xff && i < 100);
563 printk(KERN_WARNING "Calgary: PCI bus not quiesced, "
564 "continuing anyway\n");
566 /* invalidate TCE cache */
567 target = calgary_reg(bbar, tar_offset(tbl->it_busno));
568 writeq(tbl->tar_val, target);
570 /* enable arbitration */
571 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
573 (void)readl(target); /* flush */
576 static void calioc2_tce_cache_blast(struct iommu_table *tbl)
578 void __iomem *bbar = tbl->bbar;
579 void __iomem *target;
584 unsigned char bus = tbl->it_busno;
587 printk(KERN_DEBUG "Calgary: CalIOC2 bus 0x%x entering tce cache blast "
588 "sequence - count %d\n", bus, count);
590 /* 1. using the Page Migration Control reg set SoftStop */
591 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
592 val = be32_to_cpu(readl(target));
593 printk(KERN_DEBUG "1a. read 0x%x [LE] from %p\n", val, target);
595 printk(KERN_DEBUG "1b. writing 0x%x [LE] to %p\n", val, target);
596 writel(cpu_to_be32(val), target);
598 /* 2. poll split queues until all DMA activity is done */
599 printk(KERN_DEBUG "2a. starting to poll split queues\n");
600 target = calgary_reg(bbar, split_queue_offset(bus));
602 val64 = readq(target);
604 } while ((val64 & 0xff) != 0xff && i < 100);
606 printk(KERN_WARNING "CalIOC2: PCI bus not quiesced, "
607 "continuing anyway\n");
609 /* 3. poll Page Migration DEBUG for SoftStopFault */
610 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
611 val = be32_to_cpu(readl(target));
612 printk(KERN_DEBUG "3. read 0x%x [LE] from %p\n", val, target);
614 /* 4. if SoftStopFault - goto (1) */
615 if (val & PMR_SOFTSTOPFAULT) {
619 printk(KERN_WARNING "CalIOC2: too many SoftStopFaults, "
620 "aborting TCE cache flush sequence!\n");
621 return; /* pray for the best */
625 /* 5. Slam into HardStop by reading PHB_PAGE_MIG_CTRL */
626 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
627 printk(KERN_DEBUG "5a. slamming into HardStop by reading %p\n", target);
628 val = be32_to_cpu(readl(target));
629 printk(KERN_DEBUG "5b. read 0x%x [LE] from %p\n", val, target);
630 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
631 val = be32_to_cpu(readl(target));
632 printk(KERN_DEBUG "5c. read 0x%x [LE] from %p (debug)\n", val, target);
634 /* 6. invalidate TCE cache */
635 printk(KERN_DEBUG "6. invalidating TCE cache\n");
636 target = calgary_reg(bbar, tar_offset(bus));
637 writeq(tbl->tar_val, target);
639 /* 7. Re-read PMCR */
640 printk(KERN_DEBUG "7a. Re-reading PMCR\n");
641 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
642 val = be32_to_cpu(readl(target));
643 printk(KERN_DEBUG "7b. read 0x%x [LE] from %p\n", val, target);
645 /* 8. Remove HardStop */
646 printk(KERN_DEBUG "8a. removing HardStop from PMCR\n");
647 target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
649 printk(KERN_DEBUG "8b. writing 0x%x [LE] to %p\n", val, target);
650 writel(cpu_to_be32(val), target);
651 val = be32_to_cpu(readl(target));
652 printk(KERN_DEBUG "8c. read 0x%x [LE] from %p\n", val, target);
655 static void __init calgary_reserve_mem_region(struct pci_dev *dev, u64 start,
658 unsigned int numpages;
660 limit = limit | 0xfffff;
663 numpages = ((limit - start) >> PAGE_SHIFT);
664 iommu_range_reserve(pci_iommu(dev->bus), start, numpages);
667 static void __init calgary_reserve_peripheral_mem_1(struct pci_dev *dev)
669 void __iomem *target;
670 u64 low, high, sizelow;
672 struct iommu_table *tbl = pci_iommu(dev->bus);
673 unsigned char busnum = dev->bus->number;
674 void __iomem *bbar = tbl->bbar;
676 /* peripheral MEM_1 region */
677 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_LOW);
678 low = be32_to_cpu(readl(target));
679 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_HIGH);
680 high = be32_to_cpu(readl(target));
681 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_SIZE);
682 sizelow = be32_to_cpu(readl(target));
684 start = (high << 32) | low;
687 calgary_reserve_mem_region(dev, start, limit);
690 static void __init calgary_reserve_peripheral_mem_2(struct pci_dev *dev)
692 void __iomem *target;
694 u64 low, high, sizelow, sizehigh;
696 struct iommu_table *tbl = pci_iommu(dev->bus);
697 unsigned char busnum = dev->bus->number;
698 void __iomem *bbar = tbl->bbar;
701 target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
702 val32 = be32_to_cpu(readl(target));
703 if (!(val32 & PHB_MEM2_ENABLE))
706 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_LOW);
707 low = be32_to_cpu(readl(target));
708 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_HIGH);
709 high = be32_to_cpu(readl(target));
710 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_LOW);
711 sizelow = be32_to_cpu(readl(target));
712 target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_HIGH);
713 sizehigh = be32_to_cpu(readl(target));
715 start = (high << 32) | low;
716 limit = (sizehigh << 32) | sizelow;
718 calgary_reserve_mem_region(dev, start, limit);
722 * some regions of the IO address space do not get translated, so we
723 * must not give devices IO addresses in those regions. The regions
724 * are the 640KB-1MB region and the two PCI peripheral memory holes.
725 * Reserve all of them in the IOMMU bitmap to avoid giving them out
728 static void __init calgary_reserve_regions(struct pci_dev *dev)
732 struct iommu_table *tbl = pci_iommu(dev->bus);
734 /* reserve EMERGENCY_PAGES from bad_dma_address and up */
735 iommu_range_reserve(tbl, DMA_ERROR_CODE, EMERGENCY_PAGES);
737 /* avoid the BIOS/VGA first 640KB-1MB region */
738 /* for CalIOC2 - avoid the entire first MB */
739 if (is_calgary(dev->device)) {
740 start = (640 * 1024);
741 npages = ((1024 - 640) * 1024) >> PAGE_SHIFT;
742 } else { /* calioc2 */
744 npages = (1 * 1024 * 1024) >> PAGE_SHIFT;
746 iommu_range_reserve(tbl, start, npages);
748 /* reserve the two PCI peripheral memory regions in IO space */
749 calgary_reserve_peripheral_mem_1(dev);
750 calgary_reserve_peripheral_mem_2(dev);
753 static int __init calgary_setup_tar(struct pci_dev *dev, void __iomem *bbar)
757 void __iomem *target;
759 struct iommu_table *tbl;
761 /* build TCE tables for each PHB */
762 ret = build_tce_table(dev, bbar);
766 tbl = pci_iommu(dev->bus);
767 tbl->it_base = (unsigned long)bus_info[dev->bus->number].tce_space;
769 if (is_kdump_kernel())
770 calgary_init_bitmap_from_tce_table(tbl);
772 tce_free(tbl, 0, tbl->it_size);
774 if (is_calgary(dev->device))
775 tbl->chip_ops = &calgary_chip_ops;
776 else if (is_calioc2(dev->device))
777 tbl->chip_ops = &calioc2_chip_ops;
781 calgary_reserve_regions(dev);
783 /* set TARs for each PHB */
784 target = calgary_reg(bbar, tar_offset(dev->bus->number));
785 val64 = be64_to_cpu(readq(target));
787 /* zero out all TAR bits under sw control */
788 val64 &= ~TAR_SW_BITS;
789 table_phys = (u64)__pa(tbl->it_base);
793 BUG_ON(specified_table_size > TCE_TABLE_SIZE_8M);
794 val64 |= (u64) specified_table_size;
796 tbl->tar_val = cpu_to_be64(val64);
798 writeq(tbl->tar_val, target);
799 readq(target); /* flush */
804 static void __init calgary_free_bus(struct pci_dev *dev)
807 struct iommu_table *tbl = pci_iommu(dev->bus);
808 void __iomem *target;
809 unsigned int bitmapsz;
811 target = calgary_reg(tbl->bbar, tar_offset(dev->bus->number));
812 val64 = be64_to_cpu(readq(target));
813 val64 &= ~TAR_SW_BITS;
814 writeq(cpu_to_be64(val64), target);
815 readq(target); /* flush */
817 bitmapsz = tbl->it_size / BITS_PER_BYTE;
818 free_pages((unsigned long)tbl->it_map, get_order(bitmapsz));
823 set_pci_iommu(dev->bus, NULL);
825 /* Can't free bootmem allocated memory after system is up :-( */
826 bus_info[dev->bus->number].tce_space = NULL;
829 static void calgary_dump_error_regs(struct iommu_table *tbl)
831 void __iomem *bbar = tbl->bbar;
832 void __iomem *target;
835 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
836 csr = be32_to_cpu(readl(target));
838 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
839 plssr = be32_to_cpu(readl(target));
841 /* If no error, the agent ID in the CSR is not valid */
842 printk(KERN_EMERG "Calgary: DMA error on Calgary PHB 0x%x, "
843 "0x%08x@CSR 0x%08x@PLSSR\n", tbl->it_busno, csr, plssr);
846 static void calioc2_dump_error_regs(struct iommu_table *tbl)
848 void __iomem *bbar = tbl->bbar;
849 u32 csr, csmr, plssr, mck, rcstat;
850 void __iomem *target;
851 unsigned long phboff = phb_offset(tbl->it_busno);
852 unsigned long erroff;
857 target = calgary_reg(bbar, phboff | PHB_CSR_OFFSET);
858 csr = be32_to_cpu(readl(target));
860 target = calgary_reg(bbar, phboff | PHB_PLSSR_OFFSET);
861 plssr = be32_to_cpu(readl(target));
863 target = calgary_reg(bbar, phboff | 0x290);
864 csmr = be32_to_cpu(readl(target));
866 target = calgary_reg(bbar, phboff | 0x800);
867 mck = be32_to_cpu(readl(target));
869 printk(KERN_EMERG "Calgary: DMA error on CalIOC2 PHB 0x%x\n",
872 printk(KERN_EMERG "Calgary: 0x%08x@CSR 0x%08x@PLSSR 0x%08x@CSMR 0x%08x@MCK\n",
873 csr, plssr, csmr, mck);
875 /* dump rest of error regs */
876 printk(KERN_EMERG "Calgary: ");
877 for (i = 0; i < ARRAY_SIZE(errregs); i++) {
878 /* err regs are at 0x810 - 0x870 */
879 erroff = (0x810 + (i * 0x10));
880 target = calgary_reg(bbar, phboff | erroff);
881 errregs[i] = be32_to_cpu(readl(target));
882 printk("0x%08x@0x%lx ", errregs[i], erroff);
886 /* root complex status */
887 target = calgary_reg(bbar, phboff | PHB_ROOT_COMPLEX_STATUS);
888 rcstat = be32_to_cpu(readl(target));
889 printk(KERN_EMERG "Calgary: 0x%08x@0x%x\n", rcstat,
890 PHB_ROOT_COMPLEX_STATUS);
893 static void calgary_watchdog(unsigned long data)
895 struct pci_dev *dev = (struct pci_dev *)data;
896 struct iommu_table *tbl = pci_iommu(dev->bus);
897 void __iomem *bbar = tbl->bbar;
899 void __iomem *target;
901 target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
902 val32 = be32_to_cpu(readl(target));
904 /* If no error, the agent ID in the CSR is not valid */
905 if (val32 & CSR_AGENT_MASK) {
906 tbl->chip_ops->dump_error_regs(tbl);
911 /* Disable bus that caused the error */
912 target = calgary_reg(bbar, phb_offset(tbl->it_busno) |
913 PHB_CONFIG_RW_OFFSET);
914 val32 = be32_to_cpu(readl(target));
915 val32 |= PHB_SLOT_DISABLE;
916 writel(cpu_to_be32(val32), target);
917 readl(target); /* flush */
919 /* Reset the timer */
920 mod_timer(&tbl->watchdog_timer, jiffies + 2 * HZ);
924 static void __init calgary_set_split_completion_timeout(void __iomem *bbar,
925 unsigned char busnum, unsigned long timeout)
928 void __iomem *target;
929 unsigned int phb_shift = ~0; /* silence gcc */
932 switch (busno_to_phbid(busnum)) {
933 case 0: phb_shift = (63 - 19);
935 case 1: phb_shift = (63 - 23);
937 case 2: phb_shift = (63 - 27);
939 case 3: phb_shift = (63 - 35);
942 BUG_ON(busno_to_phbid(busnum));
945 target = calgary_reg(bbar, CALGARY_CONFIG_REG);
946 val64 = be64_to_cpu(readq(target));
948 /* zero out this PHB's timer bits */
949 mask = ~(0xFUL << phb_shift);
951 val64 |= (timeout << phb_shift);
952 writeq(cpu_to_be64(val64), target);
953 readq(target); /* flush */
956 static void __init calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
958 unsigned char busnum = dev->bus->number;
959 void __iomem *bbar = tbl->bbar;
960 void __iomem *target;
964 * CalIOC2 designers recommend setting bit 8 in 0xnDB0 to 1
966 target = calgary_reg(bbar, phb_offset(busnum) | PHB_SAVIOR_L2);
967 val = cpu_to_be32(readl(target));
969 writel(cpu_to_be32(val), target);
972 static void __init calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
974 unsigned char busnum = dev->bus->number;
977 * Give split completion a longer timeout on bus 1 for aic94xx
978 * http://bugzilla.kernel.org/show_bug.cgi?id=7180
980 if (is_calgary(dev->device) && (busnum == 1))
981 calgary_set_split_completion_timeout(tbl->bbar, busnum,
985 static void __init calgary_enable_translation(struct pci_dev *dev)
988 unsigned char busnum;
989 void __iomem *target;
991 struct iommu_table *tbl;
993 busnum = dev->bus->number;
994 tbl = pci_iommu(dev->bus);
997 /* enable TCE in PHB Config Register */
998 target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
999 val32 = be32_to_cpu(readl(target));
1000 val32 |= PHB_TCE_ENABLE | PHB_DAC_DISABLE | PHB_MCSR_ENABLE;
1002 printk(KERN_INFO "Calgary: enabling translation on %s PHB %#x\n",
1003 (dev->device == PCI_DEVICE_ID_IBM_CALGARY) ?
1004 "Calgary" : "CalIOC2", busnum);
1005 printk(KERN_INFO "Calgary: errant DMAs will now be prevented on this "
1008 writel(cpu_to_be32(val32), target);
1009 readl(target); /* flush */
1011 init_timer(&tbl->watchdog_timer);
1012 tbl->watchdog_timer.function = &calgary_watchdog;
1013 tbl->watchdog_timer.data = (unsigned long)dev;
1014 mod_timer(&tbl->watchdog_timer, jiffies);
1017 static void __init calgary_disable_translation(struct pci_dev *dev)
1020 unsigned char busnum;
1021 void __iomem *target;
1023 struct iommu_table *tbl;
1025 busnum = dev->bus->number;
1026 tbl = pci_iommu(dev->bus);
1029 /* disable TCE in PHB Config Register */
1030 target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
1031 val32 = be32_to_cpu(readl(target));
1032 val32 &= ~(PHB_TCE_ENABLE | PHB_DAC_DISABLE | PHB_MCSR_ENABLE);
1034 printk(KERN_INFO "Calgary: disabling translation on PHB %#x!\n", busnum);
1035 writel(cpu_to_be32(val32), target);
1036 readl(target); /* flush */
1038 del_timer_sync(&tbl->watchdog_timer);
1041 static void __init calgary_init_one_nontraslated(struct pci_dev *dev)
1044 set_pci_iommu(dev->bus, NULL);
1046 /* is the device behind a bridge? */
1047 if (dev->bus->parent)
1048 dev->bus->parent->self = dev;
1050 dev->bus->self = dev;
1053 static int __init calgary_init_one(struct pci_dev *dev)
1056 struct iommu_table *tbl;
1059 BUG_ON(dev->bus->number >= MAX_PHB_BUS_NUM);
1061 bbar = busno_to_bbar(dev->bus->number);
1062 ret = calgary_setup_tar(dev, bbar);
1068 if (dev->bus->parent) {
1069 if (dev->bus->parent->self)
1070 printk(KERN_WARNING "Calgary: IEEEE, dev %p has "
1071 "bus->parent->self!\n", dev);
1072 dev->bus->parent->self = dev;
1074 dev->bus->self = dev;
1076 tbl = pci_iommu(dev->bus);
1077 tbl->chip_ops->handle_quirks(tbl, dev);
1079 calgary_enable_translation(dev);
1087 static int __init calgary_locate_bbars(void)
1090 int rioidx, phb, bus;
1092 void __iomem *target;
1093 unsigned long offset;
1094 u8 start_bus, end_bus;
1098 for (rioidx = 0; rioidx < rio_table_hdr->num_rio_dev; rioidx++) {
1099 struct rio_detail *rio = rio_devs[rioidx];
1101 if ((rio->type != COMPAT_CALGARY) && (rio->type != ALT_CALGARY))
1104 /* map entire 1MB of Calgary config space */
1105 bbar = ioremap_nocache(rio->BBAR, 1024 * 1024);
1109 for (phb = 0; phb < PHBS_PER_CALGARY; phb++) {
1110 offset = phb_debug_offsets[phb] | PHB_DEBUG_STUFF_OFFSET;
1111 target = calgary_reg(bbar, offset);
1113 val = be32_to_cpu(readl(target));
1115 start_bus = (u8)((val & 0x00FF0000) >> 16);
1116 end_bus = (u8)((val & 0x0000FF00) >> 8);
1119 for (bus = start_bus; bus <= end_bus; bus++) {
1120 bus_info[bus].bbar = bbar;
1121 bus_info[bus].phbid = phb;
1124 bus_info[start_bus].bbar = bbar;
1125 bus_info[start_bus].phbid = phb;
1133 /* scan bus_info and iounmap any bbars we previously ioremap'd */
1134 for (bus = 0; bus < ARRAY_SIZE(bus_info); bus++)
1135 if (bus_info[bus].bbar)
1136 iounmap(bus_info[bus].bbar);
1141 static int __init calgary_init(void)
1144 struct pci_dev *dev = NULL;
1145 struct calgary_bus_info *info;
1147 ret = calgary_locate_bbars();
1151 /* Purely for kdump kernel case */
1152 if (is_kdump_kernel())
1153 get_tce_space_from_tar();
1156 dev = pci_get_device(PCI_VENDOR_ID_IBM, PCI_ANY_ID, dev);
1159 if (!is_cal_pci_dev(dev->device))
1162 info = &bus_info[dev->bus->number];
1163 if (info->translation_disabled) {
1164 calgary_init_one_nontraslated(dev);
1168 if (!info->tce_space && !translate_empty_slots)
1171 ret = calgary_init_one(dev);
1177 for_each_pci_dev(dev) {
1178 struct iommu_table *tbl;
1180 tbl = find_iommu_table(&dev->dev);
1182 if (translation_enabled(tbl))
1183 dev->dev.archdata.dma_ops = &calgary_dma_ops;
1190 dev = pci_get_device(PCI_VENDOR_ID_IBM, PCI_ANY_ID, dev);
1193 if (!is_cal_pci_dev(dev->device))
1196 info = &bus_info[dev->bus->number];
1197 if (info->translation_disabled) {
1201 if (!info->tce_space && !translate_empty_slots)
1204 calgary_disable_translation(dev);
1205 calgary_free_bus(dev);
1206 pci_dev_put(dev); /* Undo calgary_init_one()'s pci_dev_get() */
1207 dev->dev.archdata.dma_ops = NULL;
1213 static inline int __init determine_tce_table_size(u64 ram)
1217 if (specified_table_size != TCE_TABLE_SIZE_UNSPECIFIED)
1218 return specified_table_size;
1221 * Table sizes are from 0 to 7 (TCE_TABLE_SIZE_64K to
1222 * TCE_TABLE_SIZE_8M). Table size 0 has 8K entries and each
1223 * larger table size has twice as many entries, so shift the
1224 * max ram address by 13 to divide by 8K and then look at the
1225 * order of the result to choose between 0-7.
1227 ret = get_order(ram >> 13);
1228 if (ret > TCE_TABLE_SIZE_8M)
1229 ret = TCE_TABLE_SIZE_8M;
1234 static int __init build_detail_arrays(void)
1237 unsigned numnodes, i;
1238 int scal_detail_size, rio_detail_size;
1240 numnodes = rio_table_hdr->num_scal_dev;
1241 if (numnodes > MAX_NUMNODES){
1243 "Calgary: MAX_NUMNODES too low! Defined as %d, "
1244 "but system has %d nodes.\n",
1245 MAX_NUMNODES, numnodes);
1249 switch (rio_table_hdr->version){
1251 scal_detail_size = 11;
1252 rio_detail_size = 13;
1255 scal_detail_size = 12;
1256 rio_detail_size = 15;
1260 "Calgary: Invalid Rio Grande Table Version: %d\n",
1261 rio_table_hdr->version);
1265 ptr = ((unsigned long)rio_table_hdr) + 3;
1266 for (i = 0; i < numnodes; i++, ptr += scal_detail_size)
1267 scal_devs[i] = (struct scal_detail *)ptr;
1269 for (i = 0; i < rio_table_hdr->num_rio_dev;
1270 i++, ptr += rio_detail_size)
1271 rio_devs[i] = (struct rio_detail *)ptr;
1276 static int __init calgary_bus_has_devices(int bus, unsigned short pci_dev)
1281 if (pci_dev == PCI_DEVICE_ID_IBM_CALIOC2) {
1283 * FIXME: properly scan for devices accross the
1284 * PCI-to-PCI bridge on every CalIOC2 port.
1289 for (dev = 1; dev < 8; dev++) {
1290 val = read_pci_config(bus, dev, 0, 0);
1291 if (val != 0xffffffff)
1294 return (val != 0xffffffff);
1298 * calgary_init_bitmap_from_tce_table():
1299 * Funtion for kdump case. In the second/kdump kernel initialize
1300 * the bitmap based on the tce table entries obtained from first kernel
1302 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl)
1306 tp = ((u64 *)tbl->it_base);
1307 for (index = 0 ; index < tbl->it_size; index++) {
1309 set_bit(index, tbl->it_map);
1315 * get_tce_space_from_tar():
1316 * Function for kdump case. Get the tce tables from first kernel
1317 * by reading the contents of the base address register of calgary iommu
1319 static void __init get_tce_space_from_tar(void)
1322 void __iomem *target;
1323 unsigned long tce_space;
1325 for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1326 struct calgary_bus_info *info = &bus_info[bus];
1327 unsigned short pci_device;
1330 val = read_pci_config(bus, 0, 0, 0);
1331 pci_device = (val & 0xFFFF0000) >> 16;
1333 if (!is_cal_pci_dev(pci_device))
1335 if (info->translation_disabled)
1338 if (calgary_bus_has_devices(bus, pci_device) ||
1339 translate_empty_slots) {
1340 target = calgary_reg(bus_info[bus].bbar,
1342 tce_space = be64_to_cpu(readq(target));
1343 tce_space = tce_space & TAR_SW_BITS;
1345 tce_space = tce_space & (~specified_table_size);
1346 info->tce_space = (u64 *)__va(tce_space);
1352 static int __init calgary_iommu_init(void)
1356 /* ok, we're trying to use Calgary - let's roll */
1357 printk(KERN_INFO "PCI-DMA: Using Calgary IOMMU\n");
1359 ret = calgary_init();
1361 printk(KERN_ERR "PCI-DMA: Calgary init failed %d, "
1362 "falling back to no_iommu\n", ret);
1369 void __init detect_calgary(void)
1373 int calgary_found = 0;
1375 unsigned int offset, prev_offset;
1379 * if the user specified iommu=off or iommu=soft or we found
1380 * another HW IOMMU already, bail out.
1382 if (no_iommu || iommu_detected)
1388 if (!early_pci_allowed())
1391 printk(KERN_DEBUG "Calgary: detecting Calgary via BIOS EBDA area\n");
1393 ptr = (unsigned long)phys_to_virt(get_bios_ebda());
1395 rio_table_hdr = NULL;
1399 * The next offset is stored in the 1st word.
1400 * Only parse up until the offset increases:
1402 while (offset > prev_offset) {
1403 /* The block id is stored in the 2nd word */
1404 if (*((unsigned short *)(ptr + offset + 2)) == 0x4752){
1405 /* set the pointer past the offset & block id */
1406 rio_table_hdr = (struct rio_table_hdr *)(ptr + offset + 4);
1409 prev_offset = offset;
1410 offset = *((unsigned short *)(ptr + offset));
1412 if (!rio_table_hdr) {
1413 printk(KERN_DEBUG "Calgary: Unable to locate Rio Grande table "
1414 "in EBDA - bailing!\n");
1418 ret = build_detail_arrays();
1420 printk(KERN_DEBUG "Calgary: build_detail_arrays ret %d\n", ret);
1424 specified_table_size = determine_tce_table_size((is_kdump_kernel() ?
1425 saved_max_pfn : max_pfn) * PAGE_SIZE);
1427 for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1428 struct calgary_bus_info *info = &bus_info[bus];
1429 unsigned short pci_device;
1432 val = read_pci_config(bus, 0, 0, 0);
1433 pci_device = (val & 0xFFFF0000) >> 16;
1435 if (!is_cal_pci_dev(pci_device))
1438 if (info->translation_disabled)
1441 if (calgary_bus_has_devices(bus, pci_device) ||
1442 translate_empty_slots) {
1444 * If it is kdump kernel, find and use tce tables
1445 * from first kernel, else allocate tce tables here
1447 if (!is_kdump_kernel()) {
1448 tbl = alloc_tce_table();
1451 info->tce_space = tbl;
1457 printk(KERN_DEBUG "Calgary: finished detection, Calgary %s\n",
1458 calgary_found ? "found" : "not found");
1460 if (calgary_found) {
1462 calgary_detected = 1;
1463 printk(KERN_INFO "PCI-DMA: Calgary IOMMU detected.\n");
1464 printk(KERN_INFO "PCI-DMA: Calgary TCE table spec is %d\n",
1465 specified_table_size);
1467 x86_init.iommu.iommu_init = calgary_iommu_init;
1472 for (--bus; bus >= 0; --bus) {
1473 struct calgary_bus_info *info = &bus_info[bus];
1475 if (info->tce_space)
1476 free_tce_table(info->tce_space);
1480 static int __init calgary_parse_options(char *p)
1482 unsigned int bridge;
1487 if (!strncmp(p, "64k", 3))
1488 specified_table_size = TCE_TABLE_SIZE_64K;
1489 else if (!strncmp(p, "128k", 4))
1490 specified_table_size = TCE_TABLE_SIZE_128K;
1491 else if (!strncmp(p, "256k", 4))
1492 specified_table_size = TCE_TABLE_SIZE_256K;
1493 else if (!strncmp(p, "512k", 4))
1494 specified_table_size = TCE_TABLE_SIZE_512K;
1495 else if (!strncmp(p, "1M", 2))
1496 specified_table_size = TCE_TABLE_SIZE_1M;
1497 else if (!strncmp(p, "2M", 2))
1498 specified_table_size = TCE_TABLE_SIZE_2M;
1499 else if (!strncmp(p, "4M", 2))
1500 specified_table_size = TCE_TABLE_SIZE_4M;
1501 else if (!strncmp(p, "8M", 2))
1502 specified_table_size = TCE_TABLE_SIZE_8M;
1504 len = strlen("translate_empty_slots");
1505 if (!strncmp(p, "translate_empty_slots", len))
1506 translate_empty_slots = 1;
1508 len = strlen("disable");
1509 if (!strncmp(p, "disable", len)) {
1515 bridge = simple_strtoul(p, &endp, 0);
1519 if (bridge < MAX_PHB_BUS_NUM) {
1520 printk(KERN_INFO "Calgary: disabling "
1521 "translation for PHB %#x\n", bridge);
1522 bus_info[bridge].translation_disabled = 1;
1526 p = strpbrk(p, ",");
1534 __setup("calgary=", calgary_parse_options);
1536 static void __init calgary_fixup_one_tce_space(struct pci_dev *dev)
1538 struct iommu_table *tbl;
1539 unsigned int npages;
1542 tbl = pci_iommu(dev->bus);
1544 for (i = 0; i < 4; i++) {
1545 struct resource *r = &dev->resource[PCI_BRIDGE_RESOURCES + i];
1547 /* Don't give out TCEs that map MEM resources */
1548 if (!(r->flags & IORESOURCE_MEM))
1551 /* 0-based? we reserve the whole 1st MB anyway */
1555 /* cover the whole region */
1556 npages = (r->end - r->start) >> PAGE_SHIFT;
1559 iommu_range_reserve(tbl, r->start, npages);
1563 static int __init calgary_fixup_tce_spaces(void)
1565 struct pci_dev *dev = NULL;
1566 struct calgary_bus_info *info;
1568 if (no_iommu || swiotlb || !calgary_detected)
1571 printk(KERN_DEBUG "Calgary: fixing up tce spaces\n");
1574 dev = pci_get_device(PCI_VENDOR_ID_IBM, PCI_ANY_ID, dev);
1577 if (!is_cal_pci_dev(dev->device))
1580 info = &bus_info[dev->bus->number];
1581 if (info->translation_disabled)
1584 if (!info->tce_space)
1587 calgary_fixup_one_tce_space(dev);
1595 * We need to be call after pcibios_assign_resources (fs_initcall level)
1596 * and before device_initcall.
1598 rootfs_initcall(calgary_fixup_tce_spaces);