* Licensed under the GPL
*/
-#include "linux/mm.h"
-#include "asm/pgtable.h"
-#include "asm/tlbflush.h"
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
#include "as-layout.h"
#include "mem_user.h"
#include "os.h"
#include "skas.h"
#include "tlb.h"
+struct host_vm_change {
+ struct host_vm_op {
+ enum { NONE, MMAP, MUNMAP, MPROTECT } type;
+ union {
+ struct {
+ unsigned long addr;
+ unsigned long len;
+ unsigned int prot;
+ int fd;
+ __u64 offset;
+ } mmap;
+ struct {
+ unsigned long addr;
+ unsigned long len;
+ } munmap;
+ struct {
+ unsigned long addr;
+ unsigned long len;
+ unsigned int prot;
+ } mprotect;
+ } u;
+ } ops[1];
+ int index;
+ struct mm_id *id;
+ void *data;
+ int force;
+};
+
+#define INIT_HVC(mm, force) \
+ ((struct host_vm_change) \
+ { .ops = { { .type = NONE } }, \
+ .id = &mm->context.id, \
+ .data = NULL, \
+ .index = 0, \
+ .force = force })
+
+static int do_ops(struct host_vm_change *hvc, int end,
+ int finished)
+{
+ struct host_vm_op *op;
+ int i, ret = 0;
+
+ for (i = 0; i < end && !ret; i++) {
+ op = &hvc->ops[i];
+ switch (op->type) {
+ case MMAP:
+ ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len,
+ op->u.mmap.prot, op->u.mmap.fd,
+ op->u.mmap.offset, finished, &hvc->data);
+ break;
+ case MUNMAP:
+ ret = unmap(hvc->id, op->u.munmap.addr,
+ op->u.munmap.len, finished, &hvc->data);
+ break;
+ case MPROTECT:
+ ret = protect(hvc->id, op->u.mprotect.addr,
+ op->u.mprotect.len, op->u.mprotect.prot,
+ finished, &hvc->data);
+ break;
+ default:
+ printk(KERN_ERR "Unknown op type %d in do_ops\n",
+ op->type);
+ break;
+ }
+ }
+
+ return ret;
+}
+
static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
- unsigned int prot, struct host_vm_op *ops, int *index,
- int last_filled, union mm_context *mmu, void **flush,
- int (*do_ops)(union mm_context *, struct host_vm_op *,
- int, int, void **))
+ unsigned int prot, struct host_vm_change *hvc)
{
__u64 offset;
struct host_vm_op *last;
int fd, ret = 0;
fd = phys_mapping(phys, &offset);
- if (*index != -1) {
- last = &ops[*index];
+ if (hvc->index != 0) {
+ last = &hvc->ops[hvc->index - 1];
if ((last->type == MMAP) &&
(last->u.mmap.addr + last->u.mmap.len == virt) &&
(last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
}
}
- if (*index == last_filled) {
- ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
- *index = -1;
+ if (hvc->index == ARRAY_SIZE(hvc->ops)) {
+ ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
+ hvc->index = 0;
}
- ops[++*index] = ((struct host_vm_op) { .type = MMAP,
- .u = { .mmap = {
- .addr = virt,
- .len = len,
- .prot = prot,
- .fd = fd,
- .offset = offset }
+ hvc->ops[hvc->index++] = ((struct host_vm_op)
+ { .type = MMAP,
+ .u = { .mmap = { .addr = virt,
+ .len = len,
+ .prot = prot,
+ .fd = fd,
+ .offset = offset }
} });
return ret;
}
static int add_munmap(unsigned long addr, unsigned long len,
- struct host_vm_op *ops, int *index, int last_filled,
- union mm_context *mmu, void **flush,
- int (*do_ops)(union mm_context *, struct host_vm_op *,
- int, int, void **))
+ struct host_vm_change *hvc)
{
struct host_vm_op *last;
int ret = 0;
- if (*index != -1) {
- last = &ops[*index];
+ if (hvc->index != 0) {
+ last = &hvc->ops[hvc->index - 1];
if ((last->type == MUNMAP) &&
(last->u.munmap.addr + last->u.mmap.len == addr)) {
last->u.munmap.len += len;
}
}
- if (*index == last_filled) {
- ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
- *index = -1;
+ if (hvc->index == ARRAY_SIZE(hvc->ops)) {
+ ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
+ hvc->index = 0;
}
- ops[++*index] = ((struct host_vm_op) { .type = MUNMAP,
- .u = { .munmap = {
- .addr = addr,
- .len = len } } });
+ hvc->ops[hvc->index++] = ((struct host_vm_op)
+ { .type = MUNMAP,
+ .u = { .munmap = { .addr = addr,
+ .len = len } } });
return ret;
}
static int add_mprotect(unsigned long addr, unsigned long len,
- unsigned int prot, struct host_vm_op *ops, int *index,
- int last_filled, union mm_context *mmu, void **flush,
- int (*do_ops)(union mm_context *, struct host_vm_op *,
- int, int, void **))
+ unsigned int prot, struct host_vm_change *hvc)
{
struct host_vm_op *last;
int ret = 0;
- if (*index != -1) {
- last = &ops[*index];
+ if (hvc->index != 0) {
+ last = &hvc->ops[hvc->index - 1];
if ((last->type == MPROTECT) &&
(last->u.mprotect.addr + last->u.mprotect.len == addr) &&
(last->u.mprotect.prot == prot)) {
}
}
- if (*index == last_filled) {
- ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
- *index = -1;
+ if (hvc->index == ARRAY_SIZE(hvc->ops)) {
+ ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
+ hvc->index = 0;
}
- ops[++*index] = ((struct host_vm_op) { .type = MPROTECT,
- .u = { .mprotect = {
- .addr = addr,
- .len = len,
- .prot = prot } } });
+ hvc->ops[hvc->index++] = ((struct host_vm_op)
+ { .type = MPROTECT,
+ .u = { .mprotect = { .addr = addr,
+ .len = len,
+ .prot = prot } } });
return ret;
}
#define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
- unsigned long end, struct host_vm_op *ops,
- int last_op, int *op_index, int force,
- union mm_context *mmu, void **flush,
- int (*do_ops)(union mm_context *,
- struct host_vm_op *, int, int,
- void **))
+ unsigned long end,
+ struct host_vm_change *hvc)
{
pte_t *pte;
int r, w, x, prot, ret = 0;
pte = pte_offset_kernel(pmd, addr);
do {
+ if ((addr >= STUB_START) && (addr < STUB_END))
+ continue;
+
r = pte_read(*pte);
w = pte_write(*pte);
x = pte_exec(*pte);
if (!pte_young(*pte)) {
r = 0;
w = 0;
- } else if (!pte_dirty(*pte)) {
+ } else if (!pte_dirty(*pte))
w = 0;
- }
+
prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
(x ? UM_PROT_EXEC : 0));
- if (force || pte_newpage(*pte)) {
+ if (hvc->force || pte_newpage(*pte)) {
if (pte_present(*pte))
ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
- PAGE_SIZE, prot, ops, op_index,
- last_op, mmu, flush, do_ops);
- else ret = add_munmap(addr, PAGE_SIZE, ops, op_index,
- last_op, mmu, flush, do_ops);
- }
- else if (pte_newprot(*pte))
- ret = add_mprotect(addr, PAGE_SIZE, prot, ops, op_index,
- last_op, mmu, flush, do_ops);
+ PAGE_SIZE, prot, hvc);
+ else
+ ret = add_munmap(addr, PAGE_SIZE, hvc);
+ } else if (pte_newprot(*pte))
+ ret = add_mprotect(addr, PAGE_SIZE, prot, hvc);
*pte = pte_mkuptodate(*pte);
- } while (pte++, addr += PAGE_SIZE, ((addr != end) && !ret));
+ } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
return ret;
}
static inline int update_pmd_range(pud_t *pud, unsigned long addr,
- unsigned long end, struct host_vm_op *ops,
- int last_op, int *op_index, int force,
- union mm_context *mmu, void **flush,
- int (*do_ops)(union mm_context *,
- struct host_vm_op *, int, int,
- void **))
+ unsigned long end,
+ struct host_vm_change *hvc)
{
pmd_t *pmd;
unsigned long next;
do {
next = pmd_addr_end(addr, end);
if (!pmd_present(*pmd)) {
- if (force || pmd_newpage(*pmd)) {
- ret = add_munmap(addr, next - addr, ops,
- op_index, last_op, mmu,
- flush, do_ops);
+ if (hvc->force || pmd_newpage(*pmd)) {
+ ret = add_munmap(addr, next - addr, hvc);
pmd_mkuptodate(*pmd);
}
}
- else ret = update_pte_range(pmd, addr, next, ops, last_op,
- op_index, force, mmu, flush,
- do_ops);
- } while (pmd++, addr = next, ((addr != end) && !ret));
+ else ret = update_pte_range(pmd, addr, next, hvc);
+ } while (pmd++, addr = next, ((addr < end) && !ret));
return ret;
}
static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
- unsigned long end, struct host_vm_op *ops,
- int last_op, int *op_index, int force,
- union mm_context *mmu, void **flush,
- int (*do_ops)(union mm_context *,
- struct host_vm_op *, int, int,
- void **))
+ unsigned long end,
+ struct host_vm_change *hvc)
{
pud_t *pud;
unsigned long next;
do {
next = pud_addr_end(addr, end);
if (!pud_present(*pud)) {
- if (force || pud_newpage(*pud)) {
- ret = add_munmap(addr, next - addr, ops,
- op_index, last_op, mmu,
- flush, do_ops);
+ if (hvc->force || pud_newpage(*pud)) {
+ ret = add_munmap(addr, next - addr, hvc);
pud_mkuptodate(*pud);
}
}
- else ret = update_pmd_range(pud, addr, next, ops, last_op,
- op_index, force, mmu, flush,
- do_ops);
- } while (pud++, addr = next, ((addr != end) && !ret));
+ else ret = update_pmd_range(pud, addr, next, hvc);
+ } while (pud++, addr = next, ((addr < end) && !ret));
return ret;
}
void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
- unsigned long end_addr, int force,
- int (*do_ops)(union mm_context *, struct host_vm_op *,
- int, int, void **))
+ unsigned long end_addr, int force)
{
pgd_t *pgd;
- union mm_context *mmu = &mm->context;
- struct host_vm_op ops[1];
+ struct host_vm_change hvc;
unsigned long addr = start_addr, next;
- int ret = 0, last_op = ARRAY_SIZE(ops) - 1, op_index = -1;
- void *flush = NULL;
+ int ret = 0;
- ops[0].type = NONE;
+ hvc = INIT_HVC(mm, force);
pgd = pgd_offset(mm, addr);
do {
next = pgd_addr_end(addr, end_addr);
if (!pgd_present(*pgd)) {
if (force || pgd_newpage(*pgd)) {
- ret = add_munmap(addr, next - addr, ops,
- &op_index, last_op, mmu,
- &flush, do_ops);
+ ret = add_munmap(addr, next - addr, &hvc);
pgd_mkuptodate(*pgd);
}
}
- else ret = update_pud_range(pgd, addr, next, ops, last_op,
- &op_index, force, mmu, &flush,
- do_ops);
- } while (pgd++, addr = next, ((addr != end_addr) && !ret));
+ else ret = update_pud_range(pgd, addr, next, &hvc);
+ } while (pgd++, addr = next, ((addr < end_addr) && !ret));
if (!ret)
- ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
+ ret = do_ops(&hvc, hvc.index, 1);
/* This is not an else because ret is modified above */
if (ret) {
w = 0;
}
- mm_id = &mm->context.skas.id;
+ mm_id = &mm->context.id;
prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
(x ? UM_PROT_EXEC : 0));
if (pte_newpage(*pte)) {
flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
}
-static int do_ops(union mm_context *mmu, struct host_vm_op *ops, int last,
- int finished, void **flush)
-{
- struct host_vm_op *op;
- int i, ret = 0;
-
- for (i = 0; i <= last && !ret; i++) {
- op = &ops[i];
- switch(op->type) {
- case MMAP:
- ret = map(&mmu->skas.id, op->u.mmap.addr,
- op->u.mmap.len, op->u.mmap.prot,
- op->u.mmap.fd, op->u.mmap.offset, finished,
- flush);
- break;
- case MUNMAP:
- ret = unmap(&mmu->skas.id, op->u.munmap.addr,
- op->u.munmap.len, finished, flush);
- break;
- case MPROTECT:
- ret = protect(&mmu->skas.id, op->u.mprotect.addr,
- op->u.mprotect.len, op->u.mprotect.prot,
- finished, flush);
- break;
- default:
- printk(KERN_ERR "Unknown op type %d in do_ops\n",
- op->type);
- break;
- }
- }
-
- return ret;
-}
-
static void fix_range(struct mm_struct *mm, unsigned long start_addr,
unsigned long end_addr, int force)
{
- if (!proc_mm && (end_addr > CONFIG_STUB_START))
- end_addr = CONFIG_STUB_START;
-
- fix_range_common(mm, start_addr, end_addr, force, do_ops);
+ fix_range_common(mm, start_addr, end_addr, force);
}
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
else fix_range(vma->vm_mm, start, end, 0);
}
-void flush_tlb_mm(struct mm_struct *mm)
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
- unsigned long end;
-
/*
* Don't bother flushing if this address space is about to be
* destroyed.
if (atomic_read(&mm->mm_users) == 0)
return;
- end = proc_mm ? task_size : CONFIG_STUB_START;
- fix_range(mm, 0, end, 0);
+ fix_range(mm, start, end, 0);
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma = mm->mmap;
+
+ while (vma != NULL) {
+ fix_range(mm, vma->vm_start, vma->vm_end, 0);
+ vma = vma->vm_next;
+ }
}
void force_flush_all(void)
projects / mv-sheeva.git / blobdiff