2 * Handle unaligned accesses by emulation.
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
8 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
11 * This file contains exception handler for address error exception with the
12 * special capability to execute faulting instructions in software. The
13 * handler does not try to handle the case when the program counter points
14 * to an address not aligned to a word boundary.
16 * Putting data to unaligned addresses is a bad practice even on Intel where
17 * only the performance is affected. Much worse is that such code is non-
18 * portable. Due to several programs that die on MIPS due to alignment
19 * problems I decided to implement this handler anyway though I originally
20 * didn't intend to do this at all for user code.
22 * For now I enable fixing of address errors by default to make life easier.
23 * I however intend to disable this somewhen in the future when the alignment
24 * problems with user programs have been fixed. For programmers this is the
27 * Fixing address errors is a per process option. The option is inherited
28 * across fork(2) and execve(2) calls. If you really want to use the
29 * option in your user programs - I discourage the use of the software
30 * emulation strongly - use the following code in your userland stuff:
32 * #include <sys/sysmips.h>
35 * sysmips(MIPS_FIXADE, x);
38 * The argument x is 0 for disabling software emulation, enabled otherwise.
40 * Below a little program to play around with this feature.
43 * #include <sys/sysmips.h>
46 * unsigned char bar[8];
49 * main(int argc, char *argv[])
51 * struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
52 * unsigned int *p = (unsigned int *) (x.bar + 3);
56 * sysmips(MIPS_FIXADE, atoi(argv[1]));
58 * printf("*p = %08lx\n", *p);
62 * for(i = 0; i <= 7; i++)
63 * printf("%02x ", x.bar[i]);
67 * Coprocessor loads are not supported; I think this case is unimportant
70 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
71 * exception for the R6000.
72 * A store crossing a page boundary might be executed only partially.
73 * Undo the partial store in this case.
75 #include <linux/context_tracking.h>
77 #include <linux/signal.h>
78 #include <linux/smp.h>
79 #include <linux/sched.h>
80 #include <linux/debugfs.h>
81 #include <linux/perf_event.h>
84 #include <asm/branch.h>
85 #include <asm/byteorder.h>
88 #include <asm/fpu_emulator.h>
90 #include <asm/uaccess.h>
92 #include <asm/fpu_emulator.h>
94 #define STR(x) __STR(x)
98 UNALIGNED_ACTION_QUIET,
99 UNALIGNED_ACTION_SIGNAL,
100 UNALIGNED_ACTION_SHOW,
102 #ifdef CONFIG_DEBUG_FS
103 static u32 unaligned_instructions;
104 static u32 unaligned_action;
106 #define unaligned_action UNALIGNED_ACTION_QUIET
108 extern void show_registers(struct pt_regs *regs);
111 #define LoadHW(addr, value, res) \
112 __asm__ __volatile__ (".set\tnoat\n" \
113 "1:\tlb\t%0, 0(%2)\n" \
114 "2:\tlbu\t$1, 1(%2)\n\t" \
120 ".section\t.fixup,\"ax\"\n\t" \
121 "4:\tli\t%1, %3\n\t" \
124 ".section\t__ex_table,\"a\"\n\t" \
125 STR(PTR)"\t1b, 4b\n\t" \
126 STR(PTR)"\t2b, 4b\n\t" \
128 : "=&r" (value), "=r" (res) \
129 : "r" (addr), "i" (-EFAULT));
131 #define LoadW(addr, value, res) \
132 __asm__ __volatile__ ( \
133 "1:\tlwl\t%0, (%2)\n" \
134 "2:\tlwr\t%0, 3(%2)\n\t" \
138 ".section\t.fixup,\"ax\"\n\t" \
139 "4:\tli\t%1, %3\n\t" \
142 ".section\t__ex_table,\"a\"\n\t" \
143 STR(PTR)"\t1b, 4b\n\t" \
144 STR(PTR)"\t2b, 4b\n\t" \
146 : "=&r" (value), "=r" (res) \
147 : "r" (addr), "i" (-EFAULT));
149 #define LoadHWU(addr, value, res) \
150 __asm__ __volatile__ ( \
152 "1:\tlbu\t%0, 0(%2)\n" \
153 "2:\tlbu\t$1, 1(%2)\n\t" \
160 ".section\t.fixup,\"ax\"\n\t" \
161 "4:\tli\t%1, %3\n\t" \
164 ".section\t__ex_table,\"a\"\n\t" \
165 STR(PTR)"\t1b, 4b\n\t" \
166 STR(PTR)"\t2b, 4b\n\t" \
168 : "=&r" (value), "=r" (res) \
169 : "r" (addr), "i" (-EFAULT));
171 #define LoadWU(addr, value, res) \
172 __asm__ __volatile__ ( \
173 "1:\tlwl\t%0, (%2)\n" \
174 "2:\tlwr\t%0, 3(%2)\n\t" \
175 "dsll\t%0, %0, 32\n\t" \
176 "dsrl\t%0, %0, 32\n\t" \
180 "\t.section\t.fixup,\"ax\"\n\t" \
181 "4:\tli\t%1, %3\n\t" \
184 ".section\t__ex_table,\"a\"\n\t" \
185 STR(PTR)"\t1b, 4b\n\t" \
186 STR(PTR)"\t2b, 4b\n\t" \
188 : "=&r" (value), "=r" (res) \
189 : "r" (addr), "i" (-EFAULT));
191 #define LoadDW(addr, value, res) \
192 __asm__ __volatile__ ( \
193 "1:\tldl\t%0, (%2)\n" \
194 "2:\tldr\t%0, 7(%2)\n\t" \
198 "\t.section\t.fixup,\"ax\"\n\t" \
199 "4:\tli\t%1, %3\n\t" \
202 ".section\t__ex_table,\"a\"\n\t" \
203 STR(PTR)"\t1b, 4b\n\t" \
204 STR(PTR)"\t2b, 4b\n\t" \
206 : "=&r" (value), "=r" (res) \
207 : "r" (addr), "i" (-EFAULT));
209 #define StoreHW(addr, value, res) \
210 __asm__ __volatile__ ( \
212 "1:\tsb\t%1, 1(%2)\n\t" \
213 "srl\t$1, %1, 0x8\n" \
214 "2:\tsb\t$1, 0(%2)\n\t" \
219 ".section\t.fixup,\"ax\"\n\t" \
220 "4:\tli\t%0, %3\n\t" \
223 ".section\t__ex_table,\"a\"\n\t" \
224 STR(PTR)"\t1b, 4b\n\t" \
225 STR(PTR)"\t2b, 4b\n\t" \
228 : "r" (value), "r" (addr), "i" (-EFAULT));
230 #define StoreW(addr, value, res) \
231 __asm__ __volatile__ ( \
232 "1:\tswl\t%1,(%2)\n" \
233 "2:\tswr\t%1, 3(%2)\n\t" \
237 ".section\t.fixup,\"ax\"\n\t" \
238 "4:\tli\t%0, %3\n\t" \
241 ".section\t__ex_table,\"a\"\n\t" \
242 STR(PTR)"\t1b, 4b\n\t" \
243 STR(PTR)"\t2b, 4b\n\t" \
246 : "r" (value), "r" (addr), "i" (-EFAULT));
248 #define StoreDW(addr, value, res) \
249 __asm__ __volatile__ ( \
250 "1:\tsdl\t%1,(%2)\n" \
251 "2:\tsdr\t%1, 7(%2)\n\t" \
255 ".section\t.fixup,\"ax\"\n\t" \
256 "4:\tli\t%0, %3\n\t" \
259 ".section\t__ex_table,\"a\"\n\t" \
260 STR(PTR)"\t1b, 4b\n\t" \
261 STR(PTR)"\t2b, 4b\n\t" \
264 : "r" (value), "r" (addr), "i" (-EFAULT));
267 #ifdef __LITTLE_ENDIAN
268 #define LoadHW(addr, value, res) \
269 __asm__ __volatile__ (".set\tnoat\n" \
270 "1:\tlb\t%0, 1(%2)\n" \
271 "2:\tlbu\t$1, 0(%2)\n\t" \
277 ".section\t.fixup,\"ax\"\n\t" \
278 "4:\tli\t%1, %3\n\t" \
281 ".section\t__ex_table,\"a\"\n\t" \
282 STR(PTR)"\t1b, 4b\n\t" \
283 STR(PTR)"\t2b, 4b\n\t" \
285 : "=&r" (value), "=r" (res) \
286 : "r" (addr), "i" (-EFAULT));
288 #define LoadW(addr, value, res) \
289 __asm__ __volatile__ ( \
290 "1:\tlwl\t%0, 3(%2)\n" \
291 "2:\tlwr\t%0, (%2)\n\t" \
295 ".section\t.fixup,\"ax\"\n\t" \
296 "4:\tli\t%1, %3\n\t" \
299 ".section\t__ex_table,\"a\"\n\t" \
300 STR(PTR)"\t1b, 4b\n\t" \
301 STR(PTR)"\t2b, 4b\n\t" \
303 : "=&r" (value), "=r" (res) \
304 : "r" (addr), "i" (-EFAULT));
306 #define LoadHWU(addr, value, res) \
307 __asm__ __volatile__ ( \
309 "1:\tlbu\t%0, 1(%2)\n" \
310 "2:\tlbu\t$1, 0(%2)\n\t" \
317 ".section\t.fixup,\"ax\"\n\t" \
318 "4:\tli\t%1, %3\n\t" \
321 ".section\t__ex_table,\"a\"\n\t" \
322 STR(PTR)"\t1b, 4b\n\t" \
323 STR(PTR)"\t2b, 4b\n\t" \
325 : "=&r" (value), "=r" (res) \
326 : "r" (addr), "i" (-EFAULT));
328 #define LoadWU(addr, value, res) \
329 __asm__ __volatile__ ( \
330 "1:\tlwl\t%0, 3(%2)\n" \
331 "2:\tlwr\t%0, (%2)\n\t" \
332 "dsll\t%0, %0, 32\n\t" \
333 "dsrl\t%0, %0, 32\n\t" \
337 "\t.section\t.fixup,\"ax\"\n\t" \
338 "4:\tli\t%1, %3\n\t" \
341 ".section\t__ex_table,\"a\"\n\t" \
342 STR(PTR)"\t1b, 4b\n\t" \
343 STR(PTR)"\t2b, 4b\n\t" \
345 : "=&r" (value), "=r" (res) \
346 : "r" (addr), "i" (-EFAULT));
348 #define LoadDW(addr, value, res) \
349 __asm__ __volatile__ ( \
350 "1:\tldl\t%0, 7(%2)\n" \
351 "2:\tldr\t%0, (%2)\n\t" \
355 "\t.section\t.fixup,\"ax\"\n\t" \
356 "4:\tli\t%1, %3\n\t" \
359 ".section\t__ex_table,\"a\"\n\t" \
360 STR(PTR)"\t1b, 4b\n\t" \
361 STR(PTR)"\t2b, 4b\n\t" \
363 : "=&r" (value), "=r" (res) \
364 : "r" (addr), "i" (-EFAULT));
366 #define StoreHW(addr, value, res) \
367 __asm__ __volatile__ ( \
369 "1:\tsb\t%1, 0(%2)\n\t" \
370 "srl\t$1,%1, 0x8\n" \
371 "2:\tsb\t$1, 1(%2)\n\t" \
376 ".section\t.fixup,\"ax\"\n\t" \
377 "4:\tli\t%0, %3\n\t" \
380 ".section\t__ex_table,\"a\"\n\t" \
381 STR(PTR)"\t1b, 4b\n\t" \
382 STR(PTR)"\t2b, 4b\n\t" \
385 : "r" (value), "r" (addr), "i" (-EFAULT));
387 #define StoreW(addr, value, res) \
388 __asm__ __volatile__ ( \
389 "1:\tswl\t%1, 3(%2)\n" \
390 "2:\tswr\t%1, (%2)\n\t" \
394 ".section\t.fixup,\"ax\"\n\t" \
395 "4:\tli\t%0, %3\n\t" \
398 ".section\t__ex_table,\"a\"\n\t" \
399 STR(PTR)"\t1b, 4b\n\t" \
400 STR(PTR)"\t2b, 4b\n\t" \
403 : "r" (value), "r" (addr), "i" (-EFAULT));
405 #define StoreDW(addr, value, res) \
406 __asm__ __volatile__ ( \
407 "1:\tsdl\t%1, 7(%2)\n" \
408 "2:\tsdr\t%1, (%2)\n\t" \
412 ".section\t.fixup,\"ax\"\n\t" \
413 "4:\tli\t%0, %3\n\t" \
416 ".section\t__ex_table,\"a\"\n\t" \
417 STR(PTR)"\t1b, 4b\n\t" \
418 STR(PTR)"\t2b, 4b\n\t" \
421 : "r" (value), "r" (addr), "i" (-EFAULT));
424 static void emulate_load_store_insn(struct pt_regs *regs,
425 void __user *addr, unsigned int __user *pc)
427 union mips_instruction insn;
430 unsigned long origpc;
431 unsigned long orig31;
432 void __user *fault_addr = NULL;
434 origpc = (unsigned long)pc;
435 orig31 = regs->regs[31];
437 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
440 * This load never faults.
442 __get_user(insn.word, pc);
444 switch (insn.i_format.opcode) {
446 * These are instructions that a compiler doesn't generate. We
447 * can assume therefore that the code is MIPS-aware and
448 * really buggy. Emulating these instructions would break the
457 * For these instructions the only way to create an address
458 * error is an attempted access to kernel/supervisor address
475 * The remaining opcodes are the ones that are really of
479 if (!access_ok(VERIFY_READ, addr, 2))
482 LoadHW(addr, value, res);
485 compute_return_epc(regs);
486 regs->regs[insn.i_format.rt] = value;
490 if (!access_ok(VERIFY_READ, addr, 4))
493 LoadW(addr, value, res);
496 compute_return_epc(regs);
497 regs->regs[insn.i_format.rt] = value;
501 if (!access_ok(VERIFY_READ, addr, 2))
504 LoadHWU(addr, value, res);
507 compute_return_epc(regs);
508 regs->regs[insn.i_format.rt] = value;
514 * A 32-bit kernel might be running on a 64-bit processor. But
515 * if we're on a 32-bit processor and an i-cache incoherency
516 * or race makes us see a 64-bit instruction here the sdl/sdr
517 * would blow up, so for now we don't handle unaligned 64-bit
518 * instructions on 32-bit kernels.
520 if (!access_ok(VERIFY_READ, addr, 4))
523 LoadWU(addr, value, res);
526 compute_return_epc(regs);
527 regs->regs[insn.i_format.rt] = value;
529 #endif /* CONFIG_64BIT */
531 /* Cannot handle 64-bit instructions in 32-bit kernel */
537 * A 32-bit kernel might be running on a 64-bit processor. But
538 * if we're on a 32-bit processor and an i-cache incoherency
539 * or race makes us see a 64-bit instruction here the sdl/sdr
540 * would blow up, so for now we don't handle unaligned 64-bit
541 * instructions on 32-bit kernels.
543 if (!access_ok(VERIFY_READ, addr, 8))
546 LoadDW(addr, value, res);
549 compute_return_epc(regs);
550 regs->regs[insn.i_format.rt] = value;
552 #endif /* CONFIG_64BIT */
554 /* Cannot handle 64-bit instructions in 32-bit kernel */
558 if (!access_ok(VERIFY_WRITE, addr, 2))
561 compute_return_epc(regs);
562 value = regs->regs[insn.i_format.rt];
563 StoreHW(addr, value, res);
569 if (!access_ok(VERIFY_WRITE, addr, 4))
572 compute_return_epc(regs);
573 value = regs->regs[insn.i_format.rt];
574 StoreW(addr, value, res);
582 * A 32-bit kernel might be running on a 64-bit processor. But
583 * if we're on a 32-bit processor and an i-cache incoherency
584 * or race makes us see a 64-bit instruction here the sdl/sdr
585 * would blow up, so for now we don't handle unaligned 64-bit
586 * instructions on 32-bit kernels.
588 if (!access_ok(VERIFY_WRITE, addr, 8))
591 compute_return_epc(regs);
592 value = regs->regs[insn.i_format.rt];
593 StoreDW(addr, value, res);
597 #endif /* CONFIG_64BIT */
599 /* Cannot handle 64-bit instructions in 32-bit kernel */
606 die_if_kernel("Unaligned FP access in kernel code", regs);
607 BUG_ON(!used_math());
608 BUG_ON(!is_fpu_owner());
610 lose_fpu(1); /* Save FPU state for the emulator. */
611 res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1,
613 own_fpu(1); /* Restore FPU state. */
615 /* Signal if something went wrong. */
616 process_fpemu_return(res, fault_addr);
623 * COP2 is available to implementor for application specific use.
624 * It's up to applications to register a notifier chain and do
625 * whatever they have to do, including possible sending of signals.
628 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
632 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
636 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
640 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
645 * Pheeee... We encountered an yet unknown instruction or
646 * cache coherence problem. Die sucker, die ...
651 #ifdef CONFIG_DEBUG_FS
652 unaligned_instructions++;
658 /* roll back jump/branch */
659 regs->cp0_epc = origpc;
660 regs->regs[31] = orig31;
661 /* Did we have an exception handler installed? */
662 if (fixup_exception(regs))
665 die_if_kernel("Unhandled kernel unaligned access", regs);
666 force_sig(SIGSEGV, current);
671 die_if_kernel("Unhandled kernel unaligned access", regs);
672 force_sig(SIGBUS, current);
678 ("Unhandled kernel unaligned access or invalid instruction", regs);
679 force_sig(SIGILL, current);
682 /* Recode table from 16-bit register notation to 32-bit GPR. */
683 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
685 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
686 const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
688 static void emulate_load_store_microMIPS(struct pt_regs *regs,
694 unsigned int reg = 0, rvar;
695 unsigned long orig31;
699 unsigned long origpc, contpc;
700 union mips_instruction insn;
701 struct mm_decoded_insn mminsn;
702 void __user *fault_addr = NULL;
704 origpc = regs->cp0_epc;
705 orig31 = regs->regs[31];
707 mminsn.micro_mips_mode = 1;
710 * This load never faults.
712 pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
713 __get_user(halfword, pc16);
715 contpc = regs->cp0_epc + 2;
716 word = ((unsigned int)halfword << 16);
719 if (!mm_insn_16bit(halfword)) {
720 __get_user(halfword, pc16);
722 contpc = regs->cp0_epc + 4;
728 if (get_user(halfword, pc16))
730 mminsn.next_pc_inc = 2;
731 word = ((unsigned int)halfword << 16);
733 if (!mm_insn_16bit(halfword)) {
735 if (get_user(halfword, pc16))
737 mminsn.next_pc_inc = 4;
740 mminsn.next_insn = word;
742 insn = (union mips_instruction)(mminsn.insn);
743 if (mm_isBranchInstr(regs, mminsn, &contpc))
744 insn = (union mips_instruction)(mminsn.next_insn);
746 /* Parse instruction to find what to do */
748 switch (insn.mm_i_format.opcode) {
751 switch (insn.mm_x_format.func) {
753 reg = insn.mm_x_format.rd;
760 switch (insn.mm_m_format.func) {
762 reg = insn.mm_m_format.rd;
766 if (!access_ok(VERIFY_READ, addr, 8))
769 LoadW(addr, value, res);
772 regs->regs[reg] = value;
774 LoadW(addr, value, res);
777 regs->regs[reg + 1] = value;
781 reg = insn.mm_m_format.rd;
785 if (!access_ok(VERIFY_WRITE, addr, 8))
788 value = regs->regs[reg];
789 StoreW(addr, value, res);
793 value = regs->regs[reg + 1];
794 StoreW(addr, value, res);
801 reg = insn.mm_m_format.rd;
805 if (!access_ok(VERIFY_READ, addr, 16))
808 LoadDW(addr, value, res);
811 regs->regs[reg] = value;
813 LoadDW(addr, value, res);
816 regs->regs[reg + 1] = value;
818 #endif /* CONFIG_64BIT */
824 reg = insn.mm_m_format.rd;
828 if (!access_ok(VERIFY_WRITE, addr, 16))
831 value = regs->regs[reg];
832 StoreDW(addr, value, res);
836 value = regs->regs[reg + 1];
837 StoreDW(addr, value, res);
841 #endif /* CONFIG_64BIT */
846 reg = insn.mm_m_format.rd;
848 if ((rvar > 9) || !reg)
852 (VERIFY_READ, addr, 4 * (rvar + 1)))
855 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
860 for (i = 16; rvar; rvar--, i++) {
861 LoadW(addr, value, res);
865 regs->regs[i] = value;
867 if ((reg & 0xf) == 9) {
868 LoadW(addr, value, res);
872 regs->regs[30] = value;
875 LoadW(addr, value, res);
878 regs->regs[31] = value;
883 reg = insn.mm_m_format.rd;
885 if ((rvar > 9) || !reg)
889 (VERIFY_WRITE, addr, 4 * (rvar + 1)))
892 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
897 for (i = 16; rvar; rvar--, i++) {
898 value = regs->regs[i];
899 StoreW(addr, value, res);
904 if ((reg & 0xf) == 9) {
905 value = regs->regs[30];
906 StoreW(addr, value, res);
912 value = regs->regs[31];
913 StoreW(addr, value, res);
921 reg = insn.mm_m_format.rd;
923 if ((rvar > 9) || !reg)
927 (VERIFY_READ, addr, 8 * (rvar + 1)))
930 if (!access_ok(VERIFY_READ, addr, 8 * rvar))
936 for (i = 16; rvar; rvar--, i++) {
937 LoadDW(addr, value, res);
941 regs->regs[i] = value;
943 if ((reg & 0xf) == 9) {
944 LoadDW(addr, value, res);
948 regs->regs[30] = value;
951 LoadDW(addr, value, res);
954 regs->regs[31] = value;
957 #endif /* CONFIG_64BIT */
963 reg = insn.mm_m_format.rd;
965 if ((rvar > 9) || !reg)
969 (VERIFY_WRITE, addr, 8 * (rvar + 1)))
972 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
978 for (i = 16; rvar; rvar--, i++) {
979 value = regs->regs[i];
980 StoreDW(addr, value, res);
985 if ((reg & 0xf) == 9) {
986 value = regs->regs[30];
987 StoreDW(addr, value, res);
993 value = regs->regs[31];
994 StoreDW(addr, value, res);
999 #endif /* CONFIG_64BIT */
1003 /* LWC2, SWC2, LDC2, SDC2 are not serviced */
1009 switch (insn.mm_m_format.func) {
1011 reg = insn.mm_m_format.rd;
1015 /* LL,SC,LLD,SCD are not serviced */
1019 switch (insn.mm_x_format.func) {
1034 /* roll back jump/branch */
1035 regs->cp0_epc = origpc;
1036 regs->regs[31] = orig31;
1038 die_if_kernel("Unaligned FP access in kernel code", regs);
1039 BUG_ON(!used_math());
1040 BUG_ON(!is_fpu_owner());
1042 lose_fpu(1); /* save the FPU state for the emulator */
1043 res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1,
1045 own_fpu(1); /* restore FPU state */
1047 /* If something went wrong, signal */
1048 process_fpemu_return(res, fault_addr);
1055 reg = insn.mm_i_format.rt;
1059 reg = insn.mm_i_format.rt;
1063 reg = insn.mm_i_format.rt;
1067 reg = insn.mm_i_format.rt;
1071 reg = insn.mm_i_format.rt;
1075 reg = insn.mm_i_format.rt;
1079 reg = insn.mm_i_format.rt;
1083 switch (insn.mm16_m_format.func) {
1085 reg = insn.mm16_m_format.rlist;
1087 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1090 for (i = 16; rvar; rvar--, i++) {
1091 LoadW(addr, value, res);
1095 regs->regs[i] = value;
1097 LoadW(addr, value, res);
1100 regs->regs[31] = value;
1105 reg = insn.mm16_m_format.rlist;
1107 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1110 for (i = 16; rvar; rvar--, i++) {
1111 value = regs->regs[i];
1112 StoreW(addr, value, res);
1117 value = regs->regs[31];
1118 StoreW(addr, value, res);
1129 reg = reg16to32[insn.mm16_rb_format.rt];
1133 reg = reg16to32[insn.mm16_rb_format.rt];
1137 reg = reg16to32st[insn.mm16_rb_format.rt];
1141 reg = reg16to32st[insn.mm16_rb_format.rt];
1145 reg = insn.mm16_r5_format.rt;
1149 reg = insn.mm16_r5_format.rt;
1153 reg = reg16to32[insn.mm16_r3_format.rt];
1161 if (!access_ok(VERIFY_READ, addr, 2))
1164 LoadHW(addr, value, res);
1167 regs->regs[reg] = value;
1171 if (!access_ok(VERIFY_READ, addr, 2))
1174 LoadHWU(addr, value, res);
1177 regs->regs[reg] = value;
1181 if (!access_ok(VERIFY_READ, addr, 4))
1184 LoadW(addr, value, res);
1187 regs->regs[reg] = value;
1193 * A 32-bit kernel might be running on a 64-bit processor. But
1194 * if we're on a 32-bit processor and an i-cache incoherency
1195 * or race makes us see a 64-bit instruction here the sdl/sdr
1196 * would blow up, so for now we don't handle unaligned 64-bit
1197 * instructions on 32-bit kernels.
1199 if (!access_ok(VERIFY_READ, addr, 4))
1202 LoadWU(addr, value, res);
1205 regs->regs[reg] = value;
1207 #endif /* CONFIG_64BIT */
1209 /* Cannot handle 64-bit instructions in 32-bit kernel */
1215 * A 32-bit kernel might be running on a 64-bit processor. But
1216 * if we're on a 32-bit processor and an i-cache incoherency
1217 * or race makes us see a 64-bit instruction here the sdl/sdr
1218 * would blow up, so for now we don't handle unaligned 64-bit
1219 * instructions on 32-bit kernels.
1221 if (!access_ok(VERIFY_READ, addr, 8))
1224 LoadDW(addr, value, res);
1227 regs->regs[reg] = value;
1229 #endif /* CONFIG_64BIT */
1231 /* Cannot handle 64-bit instructions in 32-bit kernel */
1235 if (!access_ok(VERIFY_WRITE, addr, 2))
1238 value = regs->regs[reg];
1239 StoreHW(addr, value, res);
1245 if (!access_ok(VERIFY_WRITE, addr, 4))
1248 value = regs->regs[reg];
1249 StoreW(addr, value, res);
1257 * A 32-bit kernel might be running on a 64-bit processor. But
1258 * if we're on a 32-bit processor and an i-cache incoherency
1259 * or race makes us see a 64-bit instruction here the sdl/sdr
1260 * would blow up, so for now we don't handle unaligned 64-bit
1261 * instructions on 32-bit kernels.
1263 if (!access_ok(VERIFY_WRITE, addr, 8))
1266 value = regs->regs[reg];
1267 StoreDW(addr, value, res);
1271 #endif /* CONFIG_64BIT */
1273 /* Cannot handle 64-bit instructions in 32-bit kernel */
1277 regs->cp0_epc = contpc; /* advance or branch */
1279 #ifdef CONFIG_DEBUG_FS
1280 unaligned_instructions++;
1285 /* roll back jump/branch */
1286 regs->cp0_epc = origpc;
1287 regs->regs[31] = orig31;
1288 /* Did we have an exception handler installed? */
1289 if (fixup_exception(regs))
1292 die_if_kernel("Unhandled kernel unaligned access", regs);
1293 force_sig(SIGSEGV, current);
1298 die_if_kernel("Unhandled kernel unaligned access", regs);
1299 force_sig(SIGBUS, current);
1305 ("Unhandled kernel unaligned access or invalid instruction", regs);
1306 force_sig(SIGILL, current);
1309 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
1311 unsigned long value;
1314 unsigned long orig31;
1316 unsigned long origpc;
1317 union mips16e_instruction mips16inst, oldinst;
1319 origpc = regs->cp0_epc;
1320 orig31 = regs->regs[31];
1321 pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
1323 * This load never faults.
1325 __get_user(mips16inst.full, pc16);
1326 oldinst = mips16inst;
1328 /* skip EXTEND instruction */
1329 if (mips16inst.ri.opcode == MIPS16e_extend_op) {
1331 __get_user(mips16inst.full, pc16);
1332 } else if (delay_slot(regs)) {
1333 /* skip jump instructions */
1334 /* JAL/JALX are 32 bits but have OPCODE in first short int */
1335 if (mips16inst.ri.opcode == MIPS16e_jal_op)
1338 if (get_user(mips16inst.full, pc16))
1342 switch (mips16inst.ri.opcode) {
1343 case MIPS16e_i64_op: /* I64 or RI64 instruction */
1344 switch (mips16inst.i64.func) { /* I64/RI64 func field check */
1345 case MIPS16e_ldpc_func:
1346 case MIPS16e_ldsp_func:
1347 reg = reg16to32[mips16inst.ri64.ry];
1350 case MIPS16e_sdsp_func:
1351 reg = reg16to32[mips16inst.ri64.ry];
1354 case MIPS16e_sdrasp_func:
1355 reg = 29; /* GPRSP */
1361 case MIPS16e_swsp_op:
1362 case MIPS16e_lwpc_op:
1363 case MIPS16e_lwsp_op:
1364 reg = reg16to32[mips16inst.ri.rx];
1368 if (mips16inst.i8.func != MIPS16e_swrasp_func)
1370 reg = 29; /* GPRSP */
1374 reg = reg16to32[mips16inst.rri.ry];
1378 switch (mips16inst.ri.opcode) {
1381 case MIPS16e_lbu_op:
1386 if (!access_ok(VERIFY_READ, addr, 2))
1389 LoadHW(addr, value, res);
1392 MIPS16e_compute_return_epc(regs, &oldinst);
1393 regs->regs[reg] = value;
1396 case MIPS16e_lhu_op:
1397 if (!access_ok(VERIFY_READ, addr, 2))
1400 LoadHWU(addr, value, res);
1403 MIPS16e_compute_return_epc(regs, &oldinst);
1404 regs->regs[reg] = value;
1408 case MIPS16e_lwpc_op:
1409 case MIPS16e_lwsp_op:
1410 if (!access_ok(VERIFY_READ, addr, 4))
1413 LoadW(addr, value, res);
1416 MIPS16e_compute_return_epc(regs, &oldinst);
1417 regs->regs[reg] = value;
1420 case MIPS16e_lwu_op:
1423 * A 32-bit kernel might be running on a 64-bit processor. But
1424 * if we're on a 32-bit processor and an i-cache incoherency
1425 * or race makes us see a 64-bit instruction here the sdl/sdr
1426 * would blow up, so for now we don't handle unaligned 64-bit
1427 * instructions on 32-bit kernels.
1429 if (!access_ok(VERIFY_READ, addr, 4))
1432 LoadWU(addr, value, res);
1435 MIPS16e_compute_return_epc(regs, &oldinst);
1436 regs->regs[reg] = value;
1438 #endif /* CONFIG_64BIT */
1440 /* Cannot handle 64-bit instructions in 32-bit kernel */
1447 * A 32-bit kernel might be running on a 64-bit processor. But
1448 * if we're on a 32-bit processor and an i-cache incoherency
1449 * or race makes us see a 64-bit instruction here the sdl/sdr
1450 * would blow up, so for now we don't handle unaligned 64-bit
1451 * instructions on 32-bit kernels.
1453 if (!access_ok(VERIFY_READ, addr, 8))
1456 LoadDW(addr, value, res);
1459 MIPS16e_compute_return_epc(regs, &oldinst);
1460 regs->regs[reg] = value;
1462 #endif /* CONFIG_64BIT */
1464 /* Cannot handle 64-bit instructions in 32-bit kernel */
1468 if (!access_ok(VERIFY_WRITE, addr, 2))
1471 MIPS16e_compute_return_epc(regs, &oldinst);
1472 value = regs->regs[reg];
1473 StoreHW(addr, value, res);
1479 case MIPS16e_swsp_op:
1480 case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */
1481 if (!access_ok(VERIFY_WRITE, addr, 4))
1484 MIPS16e_compute_return_epc(regs, &oldinst);
1485 value = regs->regs[reg];
1486 StoreW(addr, value, res);
1495 * A 32-bit kernel might be running on a 64-bit processor. But
1496 * if we're on a 32-bit processor and an i-cache incoherency
1497 * or race makes us see a 64-bit instruction here the sdl/sdr
1498 * would blow up, so for now we don't handle unaligned 64-bit
1499 * instructions on 32-bit kernels.
1501 if (!access_ok(VERIFY_WRITE, addr, 8))
1504 MIPS16e_compute_return_epc(regs, &oldinst);
1505 value = regs->regs[reg];
1506 StoreDW(addr, value, res);
1510 #endif /* CONFIG_64BIT */
1512 /* Cannot handle 64-bit instructions in 32-bit kernel */
1517 * Pheeee... We encountered an yet unknown instruction or
1518 * cache coherence problem. Die sucker, die ...
1523 #ifdef CONFIG_DEBUG_FS
1524 unaligned_instructions++;
1530 /* roll back jump/branch */
1531 regs->cp0_epc = origpc;
1532 regs->regs[31] = orig31;
1533 /* Did we have an exception handler installed? */
1534 if (fixup_exception(regs))
1537 die_if_kernel("Unhandled kernel unaligned access", regs);
1538 force_sig(SIGSEGV, current);
1543 die_if_kernel("Unhandled kernel unaligned access", regs);
1544 force_sig(SIGBUS, current);
1550 ("Unhandled kernel unaligned access or invalid instruction", regs);
1551 force_sig(SIGILL, current);
1554 asmlinkage void do_ade(struct pt_regs *regs)
1556 enum ctx_state prev_state;
1557 unsigned int __user *pc;
1560 prev_state = exception_enter();
1561 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
1562 1, regs, regs->cp0_badvaddr);
1564 * Did we catch a fault trying to load an instruction?
1566 if (regs->cp0_badvaddr == regs->cp0_epc)
1569 if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
1571 if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
1575 * Do branch emulation only if we didn't forward the exception.
1576 * This is all so but ugly ...
1580 * Are we running in microMIPS mode?
1582 if (get_isa16_mode(regs->cp0_epc)) {
1584 * Did we catch a fault trying to load an instruction in
1587 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
1589 if (unaligned_action == UNALIGNED_ACTION_SHOW)
1590 show_registers(regs);
1592 if (cpu_has_mmips) {
1594 if (!user_mode(regs))
1596 emulate_load_store_microMIPS(regs,
1597 (void __user *)regs->cp0_badvaddr);
1603 if (cpu_has_mips16) {
1605 if (!user_mode(regs))
1607 emulate_load_store_MIPS16e(regs,
1608 (void __user *)regs->cp0_badvaddr);
1617 if (unaligned_action == UNALIGNED_ACTION_SHOW)
1618 show_registers(regs);
1619 pc = (unsigned int __user *)exception_epc(regs);
1622 if (!user_mode(regs))
1624 emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
1630 die_if_kernel("Kernel unaligned instruction access", regs);
1631 force_sig(SIGBUS, current);
1634 * XXX On return from the signal handler we should advance the epc
1636 exception_exit(prev_state);
1639 #ifdef CONFIG_DEBUG_FS
1640 extern struct dentry *mips_debugfs_dir;
1641 static int __init debugfs_unaligned(void)
1645 if (!mips_debugfs_dir)
1647 d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
1648 mips_debugfs_dir, &unaligned_instructions);
1651 d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
1652 mips_debugfs_dir, &unaligned_action);
1657 __initcall(debugfs_unaligned);