2 * omap iommu: simple virtual address space management
4 * Copyright (C) 2008-2009 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/err.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/device.h>
17 #include <linux/scatterlist.h>
19 #include <asm/cacheflush.h>
20 #include <asm/mach/map.h>
22 #include <plat/iommu.h>
23 #include <plat/iovmm.h>
25 #include "iopgtable.h"
28 * A device driver needs to create address mappings between:
30 * - iommu/device address
32 * - mpu virtual address
34 * There are 4 possible patterns for them:
36 * |iova/ mapping iommu_ page
37 * | da pa va (d)-(p)-(v) function type
38 * ---------------------------------------------------------------------------
39 * 1 | c c c 1 - 1 - 1 _kmap() / _kunmap() s
40 * 2 | c c,a c 1 - 1 - 1 _kmalloc()/ _kfree() s
41 * 3 | c d c 1 - n - 1 _vmap() / _vunmap() s
42 * 4 | c d,a c 1 - n - 1 _vmalloc()/ _vfree() n*
45 * 'iova': device iommu virtual address
46 * 'da': alias of 'iova'
47 * 'pa': physical address
48 * 'va': mpu virtual address
50 * 'c': contiguous memory area
51 * 'd': discontiguous memory area
52 * 'a': anonymous memory allocation
53 * '()': optional feature
55 * 'n': a normal page(4KB) size is used.
56 * 's': multiple iommu superpage(16MB, 1MB, 64KB, 4KB) size is used.
58 * '*': not yet, but feasible.
61 static struct kmem_cache *iovm_area_cachep;
63 /* return total bytes of sg buffers */
64 static size_t sgtable_len(const struct sg_table *sgt)
66 unsigned int i, total = 0;
67 struct scatterlist *sg;
72 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
75 bytes = sg_dma_len(sg);
77 if (!iopgsz_ok(bytes)) {
78 pr_err("%s: sg[%d] not iommu pagesize(%x)\n",
88 #define sgtable_ok(x) (!!sgtable_len(x))
91 * calculate the optimal number sg elements from total bytes based on
94 static unsigned int sgtable_nents(size_t bytes)
97 unsigned int nr_entries;
98 const unsigned long pagesize[] = { SZ_16M, SZ_1M, SZ_64K, SZ_4K, };
100 if (!IS_ALIGNED(bytes, PAGE_SIZE)) {
101 pr_err("%s: wrong size %08x\n", __func__, bytes);
106 for (i = 0; i < ARRAY_SIZE(pagesize); i++) {
107 if (bytes >= pagesize[i]) {
108 nr_entries += (bytes / pagesize[i]);
109 bytes %= pagesize[i];
117 /* allocate and initialize sg_table header(a kind of 'superblock') */
118 static struct sg_table *sgtable_alloc(const size_t bytes, u32 flags)
120 unsigned int nr_entries;
122 struct sg_table *sgt;
125 return ERR_PTR(-EINVAL);
127 if (!IS_ALIGNED(bytes, PAGE_SIZE))
128 return ERR_PTR(-EINVAL);
130 /* FIXME: IOVMF_DA_FIXED should support 'superpages' */
131 if ((flags & IOVMF_LINEAR) && (flags & IOVMF_DA_ANON)) {
132 nr_entries = sgtable_nents(bytes);
134 return ERR_PTR(-EINVAL);
136 nr_entries = bytes / PAGE_SIZE;
138 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
140 return ERR_PTR(-ENOMEM);
142 err = sg_alloc_table(sgt, nr_entries, GFP_KERNEL);
148 pr_debug("%s: sgt:%p(%d entries)\n", __func__, sgt, nr_entries);
153 /* free sg_table header(a kind of superblock) */
154 static void sgtable_free(struct sg_table *sgt)
162 pr_debug("%s: sgt:%p\n", __func__, sgt);
165 /* map 'sglist' to a contiguous mpu virtual area and return 'va' */
166 static void *vmap_sg(const struct sg_table *sgt)
171 struct scatterlist *sg;
172 struct vm_struct *new;
173 const struct mem_type *mtype;
175 mtype = get_mem_type(MT_DEVICE);
177 return ERR_PTR(-EINVAL);
179 total = sgtable_len(sgt);
181 return ERR_PTR(-EINVAL);
183 new = __get_vm_area(total, VM_IOREMAP, VMALLOC_START, VMALLOC_END);
185 return ERR_PTR(-ENOMEM);
188 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
194 bytes = sg_dma_len(sg);
196 BUG_ON(bytes != PAGE_SIZE);
198 err = ioremap_page(va, pa, mtype);
205 flush_cache_vmap((unsigned long)new->addr,
206 (unsigned long)(new->addr + total));
210 WARN_ON(1); /* FIXME: cleanup some mpu mappings */
212 return ERR_PTR(-EAGAIN);
215 static inline void vunmap_sg(const void *va)
220 static struct iovm_struct *__find_iovm_area(struct iommu *obj, const u32 da)
222 struct iovm_struct *tmp;
224 list_for_each_entry(tmp, &obj->mmap, list) {
225 if ((da >= tmp->da_start) && (da < tmp->da_end)) {
228 len = tmp->da_end - tmp->da_start;
230 dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n",
231 __func__, tmp->da_start, da, tmp->da_end, len,
242 * find_iovm_area - find iovma which includes @da
243 * @da: iommu device virtual address
245 * Find the existing iovma starting at @da
247 struct iovm_struct *find_iovm_area(struct iommu *obj, u32 da)
249 struct iovm_struct *area;
251 mutex_lock(&obj->mmap_lock);
252 area = __find_iovm_area(obj, da);
253 mutex_unlock(&obj->mmap_lock);
257 EXPORT_SYMBOL_GPL(find_iovm_area);
260 * This finds the hole(area) which fits the requested address and len
261 * in iovmas mmap, and returns the new allocated iovma.
263 static struct iovm_struct *alloc_iovm_area(struct iommu *obj, u32 da,
264 size_t bytes, u32 flags)
266 struct iovm_struct *new, *tmp;
267 u32 start, prev_end, alignement;
270 return ERR_PTR(-EINVAL);
273 alignement = PAGE_SIZE;
275 if (flags & IOVMF_DA_ANON) {
277 * Reserve the first page for NULL
280 if (flags & IOVMF_LINEAR)
281 alignement = iopgsz_max(bytes);
282 start = roundup(start, alignement);
286 if (list_empty(&obj->mmap))
290 list_for_each_entry(tmp, &obj->mmap, list) {
292 if (prev_end >= start)
295 if (start + bytes < tmp->da_start)
298 if (flags & IOVMF_DA_ANON)
299 start = roundup(tmp->da_end + 1, alignement);
301 prev_end = tmp->da_end;
304 if ((start > prev_end) && (ULONG_MAX - start >= bytes))
307 dev_dbg(obj->dev, "%s: no space to fit %08x(%x) flags: %08x\n",
308 __func__, da, bytes, flags);
310 return ERR_PTR(-EINVAL);
313 new = kmem_cache_zalloc(iovm_area_cachep, GFP_KERNEL);
315 return ERR_PTR(-ENOMEM);
318 new->da_start = start;
319 new->da_end = start + bytes;
323 * keep ascending order of iovmas
326 list_add_tail(&new->list, &tmp->list);
328 list_add(&new->list, &obj->mmap);
330 dev_dbg(obj->dev, "%s: found %08x-%08x-%08x(%x) %08x\n",
331 __func__, new->da_start, start, new->da_end, bytes, flags);
336 static void free_iovm_area(struct iommu *obj, struct iovm_struct *area)
340 BUG_ON(!obj || !area);
342 bytes = area->da_end - area->da_start;
344 dev_dbg(obj->dev, "%s: %08x-%08x(%x) %08x\n",
345 __func__, area->da_start, area->da_end, bytes, area->flags);
347 list_del(&area->list);
348 kmem_cache_free(iovm_area_cachep, area);
352 * da_to_va - convert (d) to (v)
353 * @obj: objective iommu
354 * @da: iommu device virtual address
355 * @va: mpu virtual address
357 * Returns mpu virtual addr which corresponds to a given device virtual addr
359 void *da_to_va(struct iommu *obj, u32 da)
362 struct iovm_struct *area;
364 mutex_lock(&obj->mmap_lock);
366 area = __find_iovm_area(obj, da);
368 dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da);
373 mutex_unlock(&obj->mmap_lock);
377 EXPORT_SYMBOL_GPL(da_to_va);
379 static void sgtable_fill_vmalloc(struct sg_table *sgt, void *_va)
382 struct scatterlist *sg;
386 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
388 const size_t bytes = PAGE_SIZE;
391 * iommu 'superpage' isn't supported with 'iommu_vmalloc()'
393 pg = vmalloc_to_page(va);
395 sg_set_page(sg, pg, bytes, 0);
400 va_end = _va + PAGE_SIZE * i;
403 static inline void sgtable_drain_vmalloc(struct sg_table *sgt)
406 * Actually this is not necessary at all, just exists for
407 * consistency of the code readability.
412 static void sgtable_fill_kmalloc(struct sg_table *sgt, u32 pa, size_t len)
415 struct scatterlist *sg;
418 va = phys_to_virt(pa);
420 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
423 bytes = iopgsz_max(len);
425 BUG_ON(!iopgsz_ok(bytes));
427 sg_set_buf(sg, phys_to_virt(pa), bytes);
429 * 'pa' is cotinuous(linear).
437 static inline void sgtable_drain_kmalloc(struct sg_table *sgt)
440 * Actually this is not necessary at all, just exists for
441 * consistency of the code readability
446 /* create 'da' <-> 'pa' mapping from 'sgt' */
447 static int map_iovm_area(struct iommu *obj, struct iovm_struct *new,
448 const struct sg_table *sgt, u32 flags)
452 struct scatterlist *sg;
453 u32 da = new->da_start;
458 BUG_ON(!sgtable_ok(sgt));
460 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
464 struct iotlb_entry e;
467 bytes = sg_dma_len(sg);
469 flags &= ~IOVMF_PGSZ_MASK;
470 pgsz = bytes_to_iopgsz(bytes);
475 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__,
478 iotlb_init_entry(&e, da, pa, flags);
479 err = iopgtable_store_entry(obj, &e);
490 for_each_sg(sgt->sgl, sg, i, j) {
493 bytes = iopgtable_clear_entry(obj, da);
495 BUG_ON(!iopgsz_ok(bytes));
502 /* release 'da' <-> 'pa' mapping */
503 static void unmap_iovm_area(struct iommu *obj, struct iovm_struct *area)
506 size_t total = area->da_end - area->da_start;
508 BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE));
510 start = area->da_start;
514 bytes = iopgtable_clear_entry(obj, start);
518 dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n",
519 __func__, start, bytes, area->flags);
521 BUG_ON(!IS_ALIGNED(bytes, PAGE_SIZE));
529 /* template function for all unmapping */
530 static struct sg_table *unmap_vm_area(struct iommu *obj, const u32 da,
531 void (*fn)(const void *), u32 flags)
533 struct sg_table *sgt = NULL;
534 struct iovm_struct *area;
536 if (!IS_ALIGNED(da, PAGE_SIZE)) {
537 dev_err(obj->dev, "%s: alignment err(%08x)\n", __func__, da);
541 mutex_lock(&obj->mmap_lock);
543 area = __find_iovm_area(obj, da);
545 dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da);
549 if ((area->flags & flags) != flags) {
550 dev_err(obj->dev, "%s: wrong flags(%08x)\n", __func__,
554 sgt = (struct sg_table *)area->sgt;
556 unmap_iovm_area(obj, area);
560 dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n", __func__,
561 area->da_start, da, area->da_end,
562 area->da_end - area->da_start, area->flags);
564 free_iovm_area(obj, area);
566 mutex_unlock(&obj->mmap_lock);
571 static u32 map_iommu_region(struct iommu *obj, u32 da,
572 const struct sg_table *sgt, void *va, size_t bytes, u32 flags)
575 struct iovm_struct *new;
577 mutex_lock(&obj->mmap_lock);
579 new = alloc_iovm_area(obj, da, bytes, flags);
582 goto err_alloc_iovma;
587 if (map_iovm_area(obj, new, sgt, new->flags))
590 mutex_unlock(&obj->mmap_lock);
592 dev_dbg(obj->dev, "%s: da:%08x(%x) flags:%08x va:%p\n",
593 __func__, new->da_start, bytes, new->flags, va);
595 return new->da_start;
598 free_iovm_area(obj, new);
600 mutex_unlock(&obj->mmap_lock);
604 static inline u32 __iommu_vmap(struct iommu *obj, u32 da,
605 const struct sg_table *sgt, void *va, size_t bytes, u32 flags)
607 return map_iommu_region(obj, da, sgt, va, bytes, flags);
611 * iommu_vmap - (d)-(p)-(v) address mapper
612 * @obj: objective iommu
613 * @sgt: address of scatter gather table
614 * @flags: iovma and page property
616 * Creates 1-n-1 mapping with given @sgt and returns @da.
617 * All @sgt element must be io page size aligned.
619 u32 iommu_vmap(struct iommu *obj, u32 da, const struct sg_table *sgt,
625 if (!obj || !obj->dev || !sgt)
628 bytes = sgtable_len(sgt);
631 bytes = PAGE_ALIGN(bytes);
633 if (flags & IOVMF_MMIO) {
639 flags &= IOVMF_HW_MASK;
640 flags |= IOVMF_DISCONT;
642 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
644 da = __iommu_vmap(obj, da, sgt, va, bytes, flags);
645 if (IS_ERR_VALUE(da))
650 EXPORT_SYMBOL_GPL(iommu_vmap);
653 * iommu_vunmap - release virtual mapping obtained by 'iommu_vmap()'
654 * @obj: objective iommu
655 * @da: iommu device virtual address
657 * Free the iommu virtually contiguous memory area starting at
658 * @da, which was returned by 'iommu_vmap()'.
660 struct sg_table *iommu_vunmap(struct iommu *obj, u32 da)
662 struct sg_table *sgt;
664 * 'sgt' is allocated before 'iommu_vmalloc()' is called.
665 * Just returns 'sgt' to the caller to free
667 sgt = unmap_vm_area(obj, da, vunmap_sg, IOVMF_DISCONT | IOVMF_MMIO);
669 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
672 EXPORT_SYMBOL_GPL(iommu_vunmap);
675 * iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
676 * @obj: objective iommu
677 * @da: contiguous iommu virtual memory
678 * @bytes: allocation size
679 * @flags: iovma and page property
681 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
682 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
684 u32 iommu_vmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags)
687 struct sg_table *sgt;
689 if (!obj || !obj->dev || !bytes)
692 bytes = PAGE_ALIGN(bytes);
698 sgt = sgtable_alloc(bytes, flags);
703 sgtable_fill_vmalloc(sgt, va);
705 flags &= IOVMF_HW_MASK;
706 flags |= IOVMF_DISCONT;
707 flags |= IOVMF_ALLOC;
708 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
710 da = __iommu_vmap(obj, da, sgt, va, bytes, flags);
711 if (IS_ERR_VALUE(da))
717 sgtable_drain_vmalloc(sgt);
723 EXPORT_SYMBOL_GPL(iommu_vmalloc);
726 * iommu_vfree - release memory allocated by 'iommu_vmalloc()'
727 * @obj: objective iommu
728 * @da: iommu device virtual address
730 * Frees the iommu virtually continuous memory area starting at
731 * @da, as obtained from 'iommu_vmalloc()'.
733 void iommu_vfree(struct iommu *obj, const u32 da)
735 struct sg_table *sgt;
737 sgt = unmap_vm_area(obj, da, vfree, IOVMF_DISCONT | IOVMF_ALLOC);
739 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
742 EXPORT_SYMBOL_GPL(iommu_vfree);
744 static u32 __iommu_kmap(struct iommu *obj, u32 da, u32 pa, void *va,
745 size_t bytes, u32 flags)
747 struct sg_table *sgt;
749 sgt = sgtable_alloc(bytes, flags);
753 sgtable_fill_kmalloc(sgt, pa, bytes);
755 da = map_iommu_region(obj, da, sgt, va, bytes, flags);
756 if (IS_ERR_VALUE(da)) {
757 sgtable_drain_kmalloc(sgt);
765 * iommu_kmap - (d)-(p)-(v) address mapper
766 * @obj: objective iommu
767 * @da: contiguous iommu virtual memory
768 * @pa: contiguous physical memory
769 * @flags: iovma and page property
771 * Creates 1-1-1 mapping and returns @da again, which can be
772 * adjusted if 'IOVMF_DA_ANON' is set.
774 u32 iommu_kmap(struct iommu *obj, u32 da, u32 pa, size_t bytes,
779 if (!obj || !obj->dev || !bytes)
782 bytes = PAGE_ALIGN(bytes);
784 va = ioremap(pa, bytes);
788 flags &= IOVMF_HW_MASK;
789 flags |= IOVMF_LINEAR;
791 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
793 da = __iommu_kmap(obj, da, pa, va, bytes, flags);
794 if (IS_ERR_VALUE(da))
799 EXPORT_SYMBOL_GPL(iommu_kmap);
802 * iommu_kunmap - release virtual mapping obtained by 'iommu_kmap()'
803 * @obj: objective iommu
804 * @da: iommu device virtual address
806 * Frees the iommu virtually contiguous memory area starting at
807 * @da, which was passed to and was returned by'iommu_kmap()'.
809 void iommu_kunmap(struct iommu *obj, u32 da)
811 struct sg_table *sgt;
812 typedef void (*func_t)(const void *);
814 sgt = unmap_vm_area(obj, da, (func_t)__iounmap,
815 IOVMF_LINEAR | IOVMF_MMIO);
817 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
820 EXPORT_SYMBOL_GPL(iommu_kunmap);
823 * iommu_kmalloc - (d)-(p)-(v) address allocator and mapper
824 * @obj: objective iommu
825 * @da: contiguous iommu virtual memory
826 * @bytes: bytes for allocation
827 * @flags: iovma and page property
829 * Allocate @bytes linearly and creates 1-1-1 mapping and returns
830 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
832 u32 iommu_kmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags)
837 if (!obj || !obj->dev || !bytes)
840 bytes = PAGE_ALIGN(bytes);
842 va = kmalloc(bytes, GFP_KERNEL | GFP_DMA);
845 pa = virt_to_phys(va);
847 flags &= IOVMF_HW_MASK;
848 flags |= IOVMF_LINEAR;
849 flags |= IOVMF_ALLOC;
850 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
852 da = __iommu_kmap(obj, da, pa, va, bytes, flags);
853 if (IS_ERR_VALUE(da))
858 EXPORT_SYMBOL_GPL(iommu_kmalloc);
861 * iommu_kfree - release virtual mapping obtained by 'iommu_kmalloc()'
862 * @obj: objective iommu
863 * @da: iommu device virtual address
865 * Frees the iommu virtually contiguous memory area starting at
866 * @da, which was passed to and was returned by'iommu_kmalloc()'.
868 void iommu_kfree(struct iommu *obj, u32 da)
870 struct sg_table *sgt;
872 sgt = unmap_vm_area(obj, da, kfree, IOVMF_LINEAR | IOVMF_ALLOC);
874 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
877 EXPORT_SYMBOL_GPL(iommu_kfree);
880 static int __init iovmm_init(void)
882 const unsigned long flags = SLAB_HWCACHE_ALIGN;
883 struct kmem_cache *p;
885 p = kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct), 0,
889 iovm_area_cachep = p;
893 module_init(iovmm_init);
895 static void __exit iovmm_exit(void)
897 kmem_cache_destroy(iovm_area_cachep);
899 module_exit(iovmm_exit);
901 MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
902 MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
903 MODULE_LICENSE("GPL v2");