2 * linux/arch/arm/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Modifications for ARM processor (c) 1995-2004 Russell King
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 #include <linux/module.h>
12 #include <linux/signal.h>
14 #include <linux/hardirq.h>
15 #include <linux/init.h>
16 #include <linux/kprobes.h>
17 #include <linux/uaccess.h>
18 #include <linux/page-flags.h>
19 #include <linux/sched.h>
20 #include <linux/highmem.h>
21 #include <linux/perf_event.h>
23 #include <asm/exception.h>
24 #include <asm/system.h>
25 #include <asm/pgtable.h>
26 #include <asm/tlbflush.h>
31 * Fault status register encodings. We steal bit 31 for our own purposes.
33 #define FSR_LNX_PF (1 << 31)
34 #define FSR_WRITE (1 << 11)
35 #define FSR_FS4 (1 << 10)
36 #define FSR_FS3_0 (15)
38 static inline int fsr_fs(unsigned int fsr)
40 return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
46 static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr)
50 if (!user_mode(regs)) {
51 /* kprobe_running() needs smp_processor_id() */
53 if (kprobe_running() && kprobe_fault_handler(regs, fsr))
61 static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr)
68 * This is useful to dump out the page tables associated with
71 void show_pte(struct mm_struct *mm, unsigned long addr)
78 printk(KERN_ALERT "pgd = %p\n", mm->pgd);
79 pgd = pgd_offset(mm, addr);
80 printk(KERN_ALERT "[%08lx] *pgd=%08llx",
81 addr, (long long)pgd_val(*pgd));
96 pud = pud_offset(pgd, addr);
97 if (PTRS_PER_PUD != 1)
98 printk(", *pud=%08llx", (long long)pud_val(*pud));
108 pmd = pmd_offset(pud, addr);
109 if (PTRS_PER_PMD != 1)
110 printk(", *pmd=%08llx", (long long)pmd_val(*pmd));
120 /* We must not map this if we have highmem enabled */
121 if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
124 pte = pte_offset_map(pmd, addr);
125 printk(", *pte=%08llx", (long long)pte_val(*pte));
126 printk(", *ppte=%08llx",
127 (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
133 #else /* CONFIG_MMU */
134 void show_pte(struct mm_struct *mm, unsigned long addr)
136 #endif /* CONFIG_MMU */
139 * Oops. The kernel tried to access some page that wasn't present.
142 __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
143 struct pt_regs *regs)
146 * Are we prepared to handle this kernel fault?
148 if (fixup_exception(regs))
152 * No handler, we'll have to terminate things with extreme prejudice.
156 "Unable to handle kernel %s at virtual address %08lx\n",
157 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
158 "paging request", addr);
161 die("Oops", regs, fsr);
167 * Something tried to access memory that isn't in our memory map..
168 * User mode accesses just cause a SIGSEGV
171 __do_user_fault(struct task_struct *tsk, unsigned long addr,
172 unsigned int fsr, unsigned int sig, int code,
173 struct pt_regs *regs)
177 #ifdef CONFIG_DEBUG_USER
178 if (user_debug & UDBG_SEGV) {
179 printk(KERN_DEBUG "%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
180 tsk->comm, sig, addr, fsr);
181 show_pte(tsk->mm, addr);
186 tsk->thread.address = addr;
187 tsk->thread.error_code = fsr;
188 tsk->thread.trap_no = 14;
192 si.si_addr = (void __user *)addr;
193 force_sig_info(sig, &si, tsk);
196 void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
198 struct task_struct *tsk = current;
199 struct mm_struct *mm = tsk->active_mm;
202 * If we are in kernel mode at this point, we
203 * have no context to handle this fault with.
206 __do_user_fault(tsk, addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
208 __do_kernel_fault(mm, addr, fsr, regs);
212 #define VM_FAULT_BADMAP 0x010000
213 #define VM_FAULT_BADACCESS 0x020000
216 * Check that the permissions on the VMA allow for the fault which occurred.
217 * If we encountered a write fault, we must have write permission, otherwise
218 * we allow any permission.
220 static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
222 unsigned int mask = VM_READ | VM_WRITE | VM_EXEC;
226 if (fsr & FSR_LNX_PF)
229 return vma->vm_flags & mask ? false : true;
233 __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
234 unsigned int flags, struct task_struct *tsk)
236 struct vm_area_struct *vma;
239 vma = find_vma(mm, addr);
240 fault = VM_FAULT_BADMAP;
243 if (unlikely(vma->vm_start > addr))
247 * Ok, we have a good vm_area for this
248 * memory access, so we can handle it.
251 if (access_error(fsr, vma)) {
252 fault = VM_FAULT_BADACCESS;
256 return handle_mm_fault(mm, vma, addr & PAGE_MASK, flags);
259 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
266 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
268 struct task_struct *tsk;
269 struct mm_struct *mm;
270 int fault, sig, code;
271 int write = fsr & FSR_WRITE;
272 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
273 (write ? FAULT_FLAG_WRITE : 0);
275 if (notify_page_fault(regs, fsr))
281 /* Enable interrupts if they were enabled in the parent context. */
282 if (interrupts_enabled(regs))
286 * If we're in an interrupt or have no user
287 * context, we must not take the fault..
289 if (in_atomic() || !mm)
293 * As per x86, we may deadlock here. However, since the kernel only
294 * validly references user space from well defined areas of the code,
295 * we can bug out early if this is from code which shouldn't.
297 if (!down_read_trylock(&mm->mmap_sem)) {
298 if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc))
301 down_read(&mm->mmap_sem);
304 * The above down_read_trylock() might have succeeded in
305 * which case, we'll have missed the might_sleep() from
309 #ifdef CONFIG_DEBUG_VM
310 if (!user_mode(regs) &&
311 !search_exception_tables(regs->ARM_pc))
316 fault = __do_page_fault(mm, addr, fsr, flags, tsk);
318 /* If we need to retry but a fatal signal is pending, handle the
319 * signal first. We do not need to release the mmap_sem because
320 * it would already be released in __lock_page_or_retry in
322 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
326 * Major/minor page fault accounting is only done on the
327 * initial attempt. If we go through a retry, it is extremely
328 * likely that the page will be found in page cache at that point.
331 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
332 if (flags & FAULT_FLAG_ALLOW_RETRY) {
333 if (fault & VM_FAULT_MAJOR) {
335 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
339 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
342 if (fault & VM_FAULT_RETRY) {
343 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
345 flags &= ~FAULT_FLAG_ALLOW_RETRY;
350 up_read(&mm->mmap_sem);
353 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
355 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
358 if (fault & VM_FAULT_OOM) {
360 * We ran out of memory, call the OOM killer, and return to
361 * userspace (which will retry the fault, or kill us if we
364 pagefault_out_of_memory();
369 * If we are in kernel mode at this point, we
370 * have no context to handle this fault with.
372 if (!user_mode(regs))
375 if (fault & VM_FAULT_SIGBUS) {
377 * We had some memory, but were unable to
378 * successfully fix up this page fault.
384 * Something tried to access memory that
385 * isn't in our memory map..
388 code = fault == VM_FAULT_BADACCESS ?
389 SEGV_ACCERR : SEGV_MAPERR;
392 __do_user_fault(tsk, addr, fsr, sig, code, regs);
396 __do_kernel_fault(mm, addr, fsr, regs);
399 #else /* CONFIG_MMU */
401 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
405 #endif /* CONFIG_MMU */
408 * First Level Translation Fault Handler
410 * We enter here because the first level page table doesn't contain
411 * a valid entry for the address.
413 * If the address is in kernel space (>= TASK_SIZE), then we are
414 * probably faulting in the vmalloc() area.
416 * If the init_task's first level page tables contains the relevant
417 * entry, we copy the it to this task. If not, we send the process
418 * a signal, fixup the exception, or oops the kernel.
420 * NOTE! We MUST NOT take any locks for this case. We may be in an
421 * interrupt or a critical region, and should only copy the information
422 * from the master page table, nothing more.
426 do_translation_fault(unsigned long addr, unsigned int fsr,
427 struct pt_regs *regs)
434 if (addr < TASK_SIZE)
435 return do_page_fault(addr, fsr, regs);
440 index = pgd_index(addr);
443 * FIXME: CP15 C1 is write only on ARMv3 architectures.
445 pgd = cpu_get_pgd() + index;
446 pgd_k = init_mm.pgd + index;
448 if (pgd_none(*pgd_k))
450 if (!pgd_present(*pgd))
451 set_pgd(pgd, *pgd_k);
453 pud = pud_offset(pgd, addr);
454 pud_k = pud_offset(pgd_k, addr);
456 if (pud_none(*pud_k))
458 if (!pud_present(*pud))
459 set_pud(pud, *pud_k);
461 pmd = pmd_offset(pud, addr);
462 pmd_k = pmd_offset(pud_k, addr);
465 * On ARM one Linux PGD entry contains two hardware entries (see page
466 * tables layout in pgtable.h). We normally guarantee that we always
467 * fill both L1 entries. But create_mapping() doesn't follow the rule.
468 * It can create inidividual L1 entries, so here we have to call
469 * pmd_none() check for the entry really corresponded to address, not
470 * for the first of pair.
472 index = (addr >> SECTION_SHIFT) & 1;
473 if (pmd_none(pmd_k[index]))
476 copy_pmd(pmd, pmd_k);
480 do_bad_area(addr, fsr, regs);
483 #else /* CONFIG_MMU */
485 do_translation_fault(unsigned long addr, unsigned int fsr,
486 struct pt_regs *regs)
490 #endif /* CONFIG_MMU */
493 * Some section permission faults need to be handled gracefully.
494 * They can happen due to a __{get,put}_user during an oops.
497 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
499 do_bad_area(addr, fsr, regs);
504 * This abort handler always returns "fault".
507 do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
512 static struct fsr_info {
513 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
519 * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
520 * defines these to be "precise" aborts.
522 { do_bad, SIGSEGV, 0, "vector exception" },
523 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
524 { do_bad, SIGKILL, 0, "terminal exception" },
525 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
526 { do_bad, SIGBUS, 0, "external abort on linefetch" },
527 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
528 { do_bad, SIGBUS, 0, "external abort on linefetch" },
529 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
530 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
531 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
532 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
533 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
534 { do_bad, SIGBUS, 0, "external abort on translation" },
535 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
536 { do_bad, SIGBUS, 0, "external abort on translation" },
537 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
539 * The following are "imprecise" aborts, which are signalled by bit
540 * 10 of the FSR, and may not be recoverable. These are only
541 * supported if the CPU abort handler supports bit 10.
543 { do_bad, SIGBUS, 0, "unknown 16" },
544 { do_bad, SIGBUS, 0, "unknown 17" },
545 { do_bad, SIGBUS, 0, "unknown 18" },
546 { do_bad, SIGBUS, 0, "unknown 19" },
547 { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */
548 { do_bad, SIGBUS, 0, "unknown 21" },
549 { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */
550 { do_bad, SIGBUS, 0, "unknown 23" },
551 { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */
552 { do_bad, SIGBUS, 0, "unknown 25" },
553 { do_bad, SIGBUS, 0, "unknown 26" },
554 { do_bad, SIGBUS, 0, "unknown 27" },
555 { do_bad, SIGBUS, 0, "unknown 28" },
556 { do_bad, SIGBUS, 0, "unknown 29" },
557 { do_bad, SIGBUS, 0, "unknown 30" },
558 { do_bad, SIGBUS, 0, "unknown 31" }
562 hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
563 int sig, int code, const char *name)
565 if (nr < 0 || nr >= ARRAY_SIZE(fsr_info))
568 fsr_info[nr].fn = fn;
569 fsr_info[nr].sig = sig;
570 fsr_info[nr].code = code;
571 fsr_info[nr].name = name;
575 * Dispatch a data abort to the relevant handler.
577 asmlinkage void __exception
578 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
580 const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
583 if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
586 printk(KERN_ALERT "Unhandled fault: %s (0x%03x) at 0x%08lx\n",
587 inf->name, fsr, addr);
589 info.si_signo = inf->sig;
591 info.si_code = inf->code;
592 info.si_addr = (void __user *)addr;
593 arm_notify_die("", regs, &info, fsr, 0);
597 static struct fsr_info ifsr_info[] = {
598 { do_bad, SIGBUS, 0, "unknown 0" },
599 { do_bad, SIGBUS, 0, "unknown 1" },
600 { do_bad, SIGBUS, 0, "debug event" },
601 { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" },
602 { do_bad, SIGBUS, 0, "unknown 4" },
603 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
604 { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" },
605 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
606 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
607 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
608 { do_bad, SIGBUS, 0, "unknown 10" },
609 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
610 { do_bad, SIGBUS, 0, "external abort on translation" },
611 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
612 { do_bad, SIGBUS, 0, "external abort on translation" },
613 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
614 { do_bad, SIGBUS, 0, "unknown 16" },
615 { do_bad, SIGBUS, 0, "unknown 17" },
616 { do_bad, SIGBUS, 0, "unknown 18" },
617 { do_bad, SIGBUS, 0, "unknown 19" },
618 { do_bad, SIGBUS, 0, "unknown 20" },
619 { do_bad, SIGBUS, 0, "unknown 21" },
620 { do_bad, SIGBUS, 0, "unknown 22" },
621 { do_bad, SIGBUS, 0, "unknown 23" },
622 { do_bad, SIGBUS, 0, "unknown 24" },
623 { do_bad, SIGBUS, 0, "unknown 25" },
624 { do_bad, SIGBUS, 0, "unknown 26" },
625 { do_bad, SIGBUS, 0, "unknown 27" },
626 { do_bad, SIGBUS, 0, "unknown 28" },
627 { do_bad, SIGBUS, 0, "unknown 29" },
628 { do_bad, SIGBUS, 0, "unknown 30" },
629 { do_bad, SIGBUS, 0, "unknown 31" },
633 hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
634 int sig, int code, const char *name)
636 if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
639 ifsr_info[nr].fn = fn;
640 ifsr_info[nr].sig = sig;
641 ifsr_info[nr].code = code;
642 ifsr_info[nr].name = name;
645 asmlinkage void __exception
646 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
648 const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
651 if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
654 printk(KERN_ALERT "Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
655 inf->name, ifsr, addr);
657 info.si_signo = inf->sig;
659 info.si_code = inf->code;
660 info.si_addr = (void __user *)addr;
661 arm_notify_die("", regs, &info, ifsr, 0);
664 static int __init exceptions_init(void)
666 if (cpu_architecture() >= CPU_ARCH_ARMv6) {
667 hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR,
668 "I-cache maintenance fault");
671 if (cpu_architecture() >= CPU_ARCH_ARMv7) {
673 * TODO: Access flag faults introduced in ARMv6K.
674 * Runtime check for 'K' extension is needed
676 hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR,
677 "section access flag fault");
678 hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR,
679 "section access flag fault");
685 arch_initcall(exceptions_init);