2 * Meta version derived from arch/powerpc/lib/dma-noncoherent.c
3 * Copyright (C) 2008 Imagination Technologies Ltd.
5 * PowerPC version derived from arch/arm/mm/consistent.c
6 * Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
8 * Copyright (C) 2000 Russell King
10 * Consistent memory allocators. Used for DMA devices that want to
11 * share uncached memory with the processor core. The function return
12 * is the virtual address and 'dma_handle' is the physical address.
13 * Mostly stolen from the ARM port, with some changes for PowerPC.
16 * Reorganized to get rid of the arch-specific consistent_* functions
17 * and provide non-coherent implementations for the DMA API. -Matt
19 * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
20 * implementation. This is pulled straight from ARM and barely
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License version 2 as
25 * published by the Free Software Foundation.
28 #include <linux/sched.h>
29 #include <linux/kernel.h>
30 #include <linux/errno.h>
31 #include <linux/export.h>
32 #include <linux/string.h>
33 #include <linux/types.h>
34 #include <linux/highmem.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/slab.h>
38 #include <asm/tlbflush.h>
41 #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_START) \
44 static u64 get_coherent_dma_mask(struct device *dev)
49 mask = dev->coherent_dma_mask;
52 * Sanity check the DMA mask - it must be non-zero, and
53 * must be able to be satisfied by a DMA allocation.
56 dev_warn(dev, "coherent DMA mask is unset\n");
64 * This is the page table (2MB) covering uncached, DMA consistent allocations
66 static pte_t *consistent_pte;
67 static DEFINE_SPINLOCK(consistent_lock);
70 * VM region handling support.
72 * This should become something generic, handling VM region allocations for
73 * vmalloc and similar (ioremap, module space, etc).
75 * I envisage vmalloc()'s supporting vm_struct becoming:
78 * struct metag_vm_region region;
79 * unsigned long flags;
80 * struct page **pages;
81 * unsigned int nr_pages;
82 * unsigned long phys_addr;
85 * get_vm_area() would then call metag_vm_region_alloc with an appropriate
86 * struct metag_vm_region head (eg):
88 * struct metag_vm_region vmalloc_head = {
89 * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
90 * .vm_start = VMALLOC_START,
91 * .vm_end = VMALLOC_END,
94 * However, vmalloc_head.vm_start is variable (typically, it is dependent on
95 * the amount of RAM found at boot time.) I would imagine that get_vm_area()
96 * would have to initialise this each time prior to calling
97 * metag_vm_region_alloc().
99 struct metag_vm_region {
100 struct list_head vm_list;
101 unsigned long vm_start;
102 unsigned long vm_end;
103 struct page *vm_pages;
107 static struct metag_vm_region consistent_head = {
108 .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
109 .vm_start = CONSISTENT_START,
110 .vm_end = CONSISTENT_END,
113 static struct metag_vm_region *metag_vm_region_alloc(struct metag_vm_region
117 unsigned long addr = head->vm_start, end = head->vm_end - size;
119 struct metag_vm_region *c, *new;
121 new = kmalloc(sizeof(struct metag_vm_region), gfp);
125 spin_lock_irqsave(&consistent_lock, flags);
127 list_for_each_entry(c, &head->vm_list, vm_list) {
128 if ((addr + size) < addr)
130 if ((addr + size) <= c->vm_start)
139 * Insert this entry _before_ the one we found.
141 list_add_tail(&new->vm_list, &c->vm_list);
142 new->vm_start = addr;
143 new->vm_end = addr + size;
146 spin_unlock_irqrestore(&consistent_lock, flags);
150 spin_unlock_irqrestore(&consistent_lock, flags);
156 static struct metag_vm_region *metag_vm_region_find(struct metag_vm_region
157 *head, unsigned long addr)
159 struct metag_vm_region *c;
161 list_for_each_entry(c, &head->vm_list, vm_list) {
162 if (c->vm_active && c->vm_start == addr)
171 * Allocate DMA-coherent memory space and return both the kernel remapped
172 * virtual and bus address for that space.
174 void *dma_alloc_coherent(struct device *dev, size_t size,
175 dma_addr_t *handle, gfp_t gfp)
178 struct metag_vm_region *c;
180 u64 mask = get_coherent_dma_mask(dev);
183 if (!consistent_pte) {
184 pr_err("%s: not initialised\n", __func__);
191 size = PAGE_ALIGN(size);
192 limit = (mask + 1) & ~mask;
193 if ((limit && size >= limit)
194 || size >= (CONSISTENT_END - CONSISTENT_START)) {
195 pr_warn("coherent allocation too big (requested %#x mask %#Lx)\n",
200 order = get_order(size);
202 if (mask != 0xffffffff)
205 page = alloc_pages(gfp, order);
210 * Invalidate any data that might be lurking in the
211 * kernel direct-mapped region for device DMA.
214 void *kaddr = page_address(page);
215 memset(kaddr, 0, size);
216 flush_dcache_region(kaddr, size);
220 * Allocate a virtual address in the consistent mapping region.
222 c = metag_vm_region_alloc(&consistent_head, size,
223 gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
225 unsigned long vaddr = c->vm_start;
226 pte_t *pte = consistent_pte + CONSISTENT_OFFSET(vaddr);
227 struct page *end = page + (1 << order);
230 split_page(page, order);
233 * Set the "dma handle"
235 *handle = page_to_bus(page);
238 BUG_ON(!pte_none(*pte));
240 SetPageReserved(page);
241 set_pte_at(&init_mm, vaddr,
248 } while (size -= PAGE_SIZE);
251 * Free the otherwise unused pages.
258 return (void *)c->vm_start;
262 __free_pages(page, order);
266 EXPORT_SYMBOL(dma_alloc_coherent);
269 * free a page as defined by the above mapping.
271 void dma_free_coherent(struct device *dev, size_t size,
272 void *vaddr, dma_addr_t dma_handle)
274 struct metag_vm_region *c;
275 unsigned long flags, addr;
278 size = PAGE_ALIGN(size);
280 spin_lock_irqsave(&consistent_lock, flags);
282 c = metag_vm_region_find(&consistent_head, (unsigned long)vaddr);
287 if ((c->vm_end - c->vm_start) != size) {
288 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
289 __func__, c->vm_end - c->vm_start, size);
291 size = c->vm_end - c->vm_start;
294 ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
297 pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
303 if (!pte_none(pte) && pte_present(pte)) {
306 if (pfn_valid(pfn)) {
307 struct page *page = pfn_to_page(pfn);
308 ClearPageReserved(page);
315 pr_crit("%s: bad page in kernel page table\n",
317 } while (size -= PAGE_SIZE);
319 flush_tlb_kernel_range(c->vm_start, c->vm_end);
321 list_del(&c->vm_list);
323 spin_unlock_irqrestore(&consistent_lock, flags);
329 spin_unlock_irqrestore(&consistent_lock, flags);
330 pr_err("%s: trying to free invalid coherent area: %p\n",
334 EXPORT_SYMBOL(dma_free_coherent);
337 static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
338 void *cpu_addr, dma_addr_t dma_addr, size_t size)
342 unsigned long flags, user_size, kern_size;
343 struct metag_vm_region *c;
345 user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
347 spin_lock_irqsave(&consistent_lock, flags);
348 c = metag_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
349 spin_unlock_irqrestore(&consistent_lock, flags);
352 unsigned long off = vma->vm_pgoff;
354 kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
356 if (off < kern_size &&
357 user_size <= (kern_size - off)) {
358 ret = remap_pfn_range(vma, vma->vm_start,
359 page_to_pfn(c->vm_pages) + off,
360 user_size << PAGE_SHIFT,
369 int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
370 void *cpu_addr, dma_addr_t dma_addr, size_t size)
372 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
373 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
375 EXPORT_SYMBOL(dma_mmap_coherent);
377 int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
378 void *cpu_addr, dma_addr_t dma_addr, size_t size)
380 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
381 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
383 EXPORT_SYMBOL(dma_mmap_writecombine);
389 * Initialise the consistent memory allocation.
391 static int __init dma_alloc_init(void)
400 int offset = pgd_index(CONSISTENT_START);
401 pgd = pgd_offset(&init_mm, CONSISTENT_START);
402 pud = pud_alloc(&init_mm, pgd, CONSISTENT_START);
403 pmd = pmd_alloc(&init_mm, pud, CONSISTENT_START);
405 pr_err("%s: no pmd tables\n", __func__);
409 WARN_ON(!pmd_none(*pmd));
411 pte = pte_alloc_kernel(pmd, CONSISTENT_START);
413 pr_err("%s: no pte tables\n", __func__);
418 pgd_k = ((pgd_t *) mmu_get_base()) + offset;
419 pud_k = pud_offset(pgd_k, CONSISTENT_START);
420 pmd_k = pmd_offset(pud_k, CONSISTENT_START);
421 set_pmd(pmd_k, *pmd);
423 consistent_pte = pte;
428 early_initcall(dma_alloc_init);
431 * make an area consistent to devices.
433 void dma_sync_for_device(void *vaddr, size_t size, int dma_direction)
436 * Ensure any writes get through the write combiner. This is necessary
437 * even with DMA_FROM_DEVICE, or the write may dirty the cache after
438 * we've invalidated it and get written back during the DMA.
443 switch (dma_direction) {
444 case DMA_BIDIRECTIONAL:
446 * Writeback to ensure the device can see our latest changes and
447 * so that we have no dirty lines, and invalidate the cache
448 * lines too in preparation for receiving the buffer back
449 * (dma_sync_for_cpu) later.
451 flush_dcache_region(vaddr, size);
455 * Writeback to ensure the device can see our latest changes.
456 * There's no need to invalidate as the device shouldn't write
459 writeback_dcache_region(vaddr, size);
461 case DMA_FROM_DEVICE:
463 * Invalidate to ensure we have no dirty lines that could get
464 * written back during the DMA. It's also safe to flush
465 * (writeback) here if necessary.
467 invalidate_dcache_region(vaddr, size);
475 EXPORT_SYMBOL(dma_sync_for_device);
478 * make an area consistent to the core.
480 void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction)
483 * Hardware L2 cache prefetch doesn't occur across 4K physical
484 * boundaries, however according to Documentation/DMA-API-HOWTO.txt
485 * kmalloc'd memory is DMA'able, so accesses in nearby memory could
486 * trigger a cache fill in the DMA buffer.
488 * This should never cause dirty lines, so a flush or invalidate should
489 * be safe to allow us to see data from the device.
491 if (_meta_l2c_pf_is_enabled()) {
492 switch (dma_direction) {
493 case DMA_BIDIRECTIONAL:
494 case DMA_FROM_DEVICE:
495 invalidate_dcache_region(vaddr, size);
498 /* The device shouldn't have written to the buffer */
507 EXPORT_SYMBOL(dma_sync_for_cpu);