arch/mips/kernel/unaligned.c

   1 /*
   2  * Handle unaligned accesses by emulation.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
   9  * Copyright (C) 1999 Silicon Graphics, Inc.
  10  * Copyright (C) 2014 Imagination Technologies Ltd.
  11  *
  12  * This file contains exception handler for address error exception with the
  13  * special capability to execute faulting instructions in software.  The
  14  * handler does not try to handle the case when the program counter points
  15  * to an address not aligned to a word boundary.
  16  *
  17  * Putting data to unaligned addresses is a bad practice even on Intel where
  18  * only the performance is affected.  Much worse is that such code is non-
  19  * portable.  Due to several programs that die on MIPS due to alignment
  20  * problems I decided to implement this handler anyway though I originally
  21  * didn't intend to do this at all for user code.
  22  *
  23  * For now I enable fixing of address errors by default to make life easier.
  24  * I however intend to disable this somewhen in the future when the alignment
  25  * problems with user programs have been fixed.  For programmers this is the
  26  * right way to go.
  27  *
  28  * Fixing address errors is a per process option.  The option is inherited
  29  * across fork(2) and execve(2) calls.  If you really want to use the
  30  * option in your user programs - I discourage the use of the software
  31  * emulation strongly - use the following code in your userland stuff:
  32  *
  33  * #include <sys/sysmips.h>
  34  *
  35  * ...
  36  * sysmips(MIPS_FIXADE, x);
  37  * ...
  38  *
  39  * The argument x is 0 for disabling software emulation, enabled otherwise.
  40  *
  41  * Below a little program to play around with this feature.
  42  *
  43  * #include <stdio.h>
  44  * #include <sys/sysmips.h>
  45  *
  46  * struct foo {
  47  *         unsigned char bar[8];
  48  * };
  49  *
  50  * main(int argc, char *argv[])
  51  * {
  52  *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
  53  *         unsigned int *p = (unsigned int *) (x.bar + 3);
  54  *         int i;
  55  *
  56  *         if (argc > 1)
  57  *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
  58  *
  59  *         printf("*p = %08lx\n", *p);
  60  *
  61  *         *p = 0xdeadface;
  62  *
  63  *         for(i = 0; i <= 7; i++)
  64  *         printf("%02x ", x.bar[i]);
  65  *         printf("\n");
  66  * }
  67  *
  68  * Coprocessor loads are not supported; I think this case is unimportant
  69  * in the practice.
  70  *
  71  * TODO: Handle ndc (attempted store to doubleword in uncached memory)
  72  *       exception for the R6000.
  73  *       A store crossing a page boundary might be executed only partially.
  74  *       Undo the partial store in this case.
  75  */
  76 #include <linux/context_tracking.h>
  77 #include <linux/mm.h>
  78 #include <linux/signal.h>
  79 #include <linux/smp.h>
  80 #include <linux/sched.h>
  81 #include <linux/debugfs.h>
  82 #include <linux/perf_event.h>
  83
  84 #include <asm/asm.h>
  85 #include <asm/branch.h>
  86 #include <asm/byteorder.h>
  87 #include <asm/cop2.h>
  88 #include <asm/fpu.h>
  89 #include <asm/fpu_emulator.h>
  90 #include <asm/inst.h>
  91 #include <asm/uaccess.h>
  92 #include <asm/fpu.h>
  93 #include <asm/fpu_emulator.h>
  94
  95 #define STR(x)  __STR(x)
  96 #define __STR(x)  #x
  97
  98 enum {
  99         UNALIGNED_ACTION_QUIET,
 100         UNALIGNED_ACTION_SIGNAL,
 101         UNALIGNED_ACTION_SHOW,
 102 };
 103 #ifdef CONFIG_DEBUG_FS
 104 static u32 unaligned_instructions;
 105 static u32 unaligned_action;
 106 #else
 107 #define unaligned_action UNALIGNED_ACTION_QUIET
 108 #endif
 109 extern void show_registers(struct pt_regs *regs);
 110
 111 #ifdef __BIG_ENDIAN
 112 #define     _LoadHW(addr, value, res, type)  \
 113 do {                                                        \
 114                 __asm__ __volatile__ (".set\tnoat\n"        \
 115                         "1:\t"type##_lb("%0", "0(%2)")"\n"  \
 116                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 117                         "sll\t%0, 0x8\n\t"                  \
 118                         "or\t%0, $1\n\t"                    \
 119                         "li\t%1, 0\n"                       \
 120                         "3:\t.set\tat\n\t"                  \
 121                         ".insn\n\t"                         \
 122                         ".section\t.fixup,\"ax\"\n\t"       \
 123                         "4:\tli\t%1, %3\n\t"                \
 124                         "j\t3b\n\t"                         \
 125                         ".previous\n\t"                     \
 126                         ".section\t__ex_table,\"a\"\n\t"    \
 127                         STR(PTR)"\t1b, 4b\n\t"              \
 128                         STR(PTR)"\t2b, 4b\n\t"              \
 129                         ".previous"                         \
 130                         : "=&r" (value), "=r" (res)         \
 131                         : "r" (addr), "i" (-EFAULT));       \
 132 } while(0)
 133
 134 #ifndef CONFIG_CPU_MIPSR6
 135 #define     _LoadW(addr, value, res, type)   \
 136 do {                                                        \
 137                 __asm__ __volatile__ (                      \
 138                         "1:\t"type##_lwl("%0", "(%2)")"\n"   \
 139                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 140                         "li\t%1, 0\n"                       \
 141                         "3:\n\t"                            \
 142                         ".insn\n\t"                         \
 143                         ".section\t.fixup,\"ax\"\n\t"       \
 144                         "4:\tli\t%1, %3\n\t"                \
 145                         "j\t3b\n\t"                         \
 146                         ".previous\n\t"                     \
 147                         ".section\t__ex_table,\"a\"\n\t"    \
 148                         STR(PTR)"\t1b, 4b\n\t"              \
 149                         STR(PTR)"\t2b, 4b\n\t"              \
 150                         ".previous"                         \
 151                         : "=&r" (value), "=r" (res)         \
 152                         : "r" (addr), "i" (-EFAULT));       \
 153 } while(0)
 154
 155 #else
 156 /* MIPSR6 has no lwl instruction */
 157 #define     _LoadW(addr, value, res, type) \
 158 do {                                                        \
 159                 __asm__ __volatile__ (                      \
 160                         ".set\tpush\n"                      \
 161                         ".set\tnoat\n\t"                    \
 162                         "1:"type##_lb("%0", "0(%2)")"\n\t"  \
 163                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
 164                         "sll\t%0, 0x8\n\t"                  \
 165                         "or\t%0, $1\n\t"                    \
 166                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
 167                         "sll\t%0, 0x8\n\t"                  \
 168                         "or\t%0, $1\n\t"                    \
 169                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
 170                         "sll\t%0, 0x8\n\t"                  \
 171                         "or\t%0, $1\n\t"                    \
 172                         "li\t%1, 0\n"                       \
 173                         ".set\tpop\n"                       \
 174                         "10:\n\t"                           \
 175                         ".insn\n\t"                         \
 176                         ".section\t.fixup,\"ax\"\n\t"       \
 177                         "11:\tli\t%1, %3\n\t"               \
 178                         "j\t10b\n\t"                        \
 179                         ".previous\n\t"                     \
 180                         ".section\t__ex_table,\"a\"\n\t"    \
 181                         STR(PTR)"\t1b, 11b\n\t"             \
 182                         STR(PTR)"\t2b, 11b\n\t"             \
 183                         STR(PTR)"\t3b, 11b\n\t"             \
 184                         STR(PTR)"\t4b, 11b\n\t"             \
 185                         ".previous"                         \
 186                         : "=&r" (value), "=r" (res)         \
 187                         : "r" (addr), "i" (-EFAULT));       \
 188 } while(0)
 189
 190 #endif /* CONFIG_CPU_MIPSR6 */
 191
 192 #define     _LoadHWU(addr, value, res, type) \
 193 do {                                                        \
 194                 __asm__ __volatile__ (                      \
 195                         ".set\tnoat\n"                      \
 196                         "1:\t"type##_lbu("%0", "0(%2)")"\n" \
 197                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 198                         "sll\t%0, 0x8\n\t"                  \
 199                         "or\t%0, $1\n\t"                    \
 200                         "li\t%1, 0\n"                       \
 201                         "3:\n\t"                            \
 202                         ".insn\n\t"                         \
 203                         ".set\tat\n\t"                      \
 204                         ".section\t.fixup,\"ax\"\n\t"       \
 205                         "4:\tli\t%1, %3\n\t"                \
 206                         "j\t3b\n\t"                         \
 207                         ".previous\n\t"                     \
 208                         ".section\t__ex_table,\"a\"\n\t"    \
 209                         STR(PTR)"\t1b, 4b\n\t"              \
 210                         STR(PTR)"\t2b, 4b\n\t"              \
 211                         ".previous"                         \
 212                         : "=&r" (value), "=r" (res)         \
 213                         : "r" (addr), "i" (-EFAULT));       \
 214 } while(0)
 215
 216 #ifndef CONFIG_CPU_MIPSR6
 217 #define     _LoadWU(addr, value, res, type)  \
 218 do {                                                        \
 219                 __asm__ __volatile__ (                      \
 220                         "1:\t"type##_lwl("%0", "(%2)")"\n"  \
 221                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 222                         "dsll\t%0, %0, 32\n\t"              \
 223                         "dsrl\t%0, %0, 32\n\t"              \
 224                         "li\t%1, 0\n"                       \
 225                         "3:\n\t"                            \
 226                         ".insn\n\t"                         \
 227                         "\t.section\t.fixup,\"ax\"\n\t"     \
 228                         "4:\tli\t%1, %3\n\t"                \
 229                         "j\t3b\n\t"                         \
 230                         ".previous\n\t"                     \
 231                         ".section\t__ex_table,\"a\"\n\t"    \
 232                         STR(PTR)"\t1b, 4b\n\t"              \
 233                         STR(PTR)"\t2b, 4b\n\t"              \
 234                         ".previous"                         \
 235                         : "=&r" (value), "=r" (res)         \
 236                         : "r" (addr), "i" (-EFAULT));       \
 237 } while(0)
 238
 239 #define     _LoadDW(addr, value, res)  \
 240 do {                                                        \
 241                 __asm__ __volatile__ (                      \
 242                         "1:\tldl\t%0, (%2)\n"               \
 243                         "2:\tldr\t%0, 7(%2)\n\t"            \
 244                         "li\t%1, 0\n"                       \
 245                         "3:\n\t"                            \
 246                         ".insn\n\t"                         \
 247                         "\t.section\t.fixup,\"ax\"\n\t"     \
 248                         "4:\tli\t%1, %3\n\t"                \
 249                         "j\t3b\n\t"                         \
 250                         ".previous\n\t"                     \
 251                         ".section\t__ex_table,\"a\"\n\t"    \
 252                         STR(PTR)"\t1b, 4b\n\t"              \
 253                         STR(PTR)"\t2b, 4b\n\t"              \
 254                         ".previous"                         \
 255                         : "=&r" (value), "=r" (res)         \
 256                         : "r" (addr), "i" (-EFAULT));       \
 257 } while(0)
 258
 259 #else
 260 /* MIPSR6 has not lwl and ldl instructions */
 261 #define     _LoadWU(addr, value, res, type) \
 262 do {                                                        \
 263                 __asm__ __volatile__ (                      \
 264                         ".set\tpush\n\t"                    \
 265                         ".set\tnoat\n\t"                    \
 266                         "1:"type##_lbu("%0", "0(%2)")"\n\t" \
 267                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
 268                         "sll\t%0, 0x8\n\t"                  \
 269                         "or\t%0, $1\n\t"                    \
 270                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
 271                         "sll\t%0, 0x8\n\t"                  \
 272                         "or\t%0, $1\n\t"                    \
 273                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
 274                         "sll\t%0, 0x8\n\t"                  \
 275                         "or\t%0, $1\n\t"                    \
 276                         "li\t%1, 0\n"                       \
 277                         ".set\tpop\n"                       \
 278                         "10:\n\t"                           \
 279                         ".insn\n\t"                         \
 280                         ".section\t.fixup,\"ax\"\n\t"       \
 281                         "11:\tli\t%1, %3\n\t"               \
 282                         "j\t10b\n\t"                        \
 283                         ".previous\n\t"                     \
 284                         ".section\t__ex_table,\"a\"\n\t"    \
 285                         STR(PTR)"\t1b, 11b\n\t"             \
 286                         STR(PTR)"\t2b, 11b\n\t"             \
 287                         STR(PTR)"\t3b, 11b\n\t"             \
 288                         STR(PTR)"\t4b, 11b\n\t"             \
 289                         ".previous"                         \
 290                         : "=&r" (value), "=r" (res)         \
 291                         : "r" (addr), "i" (-EFAULT));       \
 292 } while(0)
 293
 294 #define     _LoadDW(addr, value, res)  \
 295 do {                                                        \
 296                 __asm__ __volatile__ (                      \
 297                         ".set\tpush\n\t"                    \
 298                         ".set\tnoat\n\t"                    \
 299                         "1:lb\t%0, 0(%2)\n\t"               \
 300                         "2:lbu\t $1, 1(%2)\n\t"             \
 301                         "dsll\t%0, 0x8\n\t"                 \
 302                         "or\t%0, $1\n\t"                    \
 303                         "3:lbu\t$1, 2(%2)\n\t"              \
 304                         "dsll\t%0, 0x8\n\t"                 \
 305                         "or\t%0, $1\n\t"                    \
 306                         "4:lbu\t$1, 3(%2)\n\t"              \
 307                         "dsll\t%0, 0x8\n\t"                 \
 308                         "or\t%0, $1\n\t"                    \
 309                         "5:lbu\t$1, 4(%2)\n\t"              \
 310                         "dsll\t%0, 0x8\n\t"                 \
 311                         "or\t%0, $1\n\t"                    \
 312                         "6:lbu\t$1, 5(%2)\n\t"              \
 313                         "dsll\t%0, 0x8\n\t"                 \
 314                         "or\t%0, $1\n\t"                    \
 315                         "7:lbu\t$1, 6(%2)\n\t"              \
 316                         "dsll\t%0, 0x8\n\t"                 \
 317                         "or\t%0, $1\n\t"                    \
 318                         "8:lbu\t$1, 7(%2)\n\t"              \
 319                         "dsll\t%0, 0x8\n\t"                 \
 320                         "or\t%0, $1\n\t"                    \
 321                         "li\t%1, 0\n"                       \
 322                         ".set\tpop\n\t"                     \
 323                         "10:\n\t"                           \
 324                         ".insn\n\t"                         \
 325                         ".section\t.fixup,\"ax\"\n\t"       \
 326                         "11:\tli\t%1, %3\n\t"               \
 327                         "j\t10b\n\t"                        \
 328                         ".previous\n\t"                     \
 329                         ".section\t__ex_table,\"a\"\n\t"    \
 330                         STR(PTR)"\t1b, 11b\n\t"             \
 331                         STR(PTR)"\t2b, 11b\n\t"             \
 332                         STR(PTR)"\t3b, 11b\n\t"             \
 333                         STR(PTR)"\t4b, 11b\n\t"             \
 334                         STR(PTR)"\t5b, 11b\n\t"             \
 335                         STR(PTR)"\t6b, 11b\n\t"             \
 336                         STR(PTR)"\t7b, 11b\n\t"             \
 337                         STR(PTR)"\t8b, 11b\n\t"             \
 338                         ".previous"                         \
 339                         : "=&r" (value), "=r" (res)         \
 340                         : "r" (addr), "i" (-EFAULT));       \
 341 } while(0)
 342
 343 #endif /* CONFIG_CPU_MIPSR6 */
 344
 345
 346 #define     _StoreHW(addr, value, res, type) \
 347 do {                                                        \
 348                 __asm__ __volatile__ (                      \
 349                         ".set\tnoat\n"                      \
 350                         "1:\t"type##_sb("%1", "1(%2)")"\n"  \
 351                         "srl\t$1, %1, 0x8\n"                \
 352                         "2:\t"type##_sb("$1", "0(%2)")"\n"  \
 353                         ".set\tat\n\t"                      \
 354                         "li\t%0, 0\n"                       \
 355                         "3:\n\t"                            \
 356                         ".insn\n\t"                         \
 357                         ".section\t.fixup,\"ax\"\n\t"       \
 358                         "4:\tli\t%0, %3\n\t"                \
 359                         "j\t3b\n\t"                         \
 360                         ".previous\n\t"                     \
 361                         ".section\t__ex_table,\"a\"\n\t"    \
 362                         STR(PTR)"\t1b, 4b\n\t"              \
 363                         STR(PTR)"\t2b, 4b\n\t"              \
 364                         ".previous"                         \
 365                         : "=r" (res)                        \
 366                         : "r" (value), "r" (addr), "i" (-EFAULT));\
 367 } while(0)
 368
 369 #ifndef CONFIG_CPU_MIPSR6
 370 #define     _StoreW(addr, value, res, type)  \
 371 do {                                                        \
 372                 __asm__ __volatile__ (                      \
 373                         "1:\t"type##_swl("%1", "(%2)")"\n"  \
 374                         "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
 375                         "li\t%0, 0\n"                       \
 376                         "3:\n\t"                            \
 377                         ".insn\n\t"                         \
 378                         ".section\t.fixup,\"ax\"\n\t"       \
 379                         "4:\tli\t%0, %3\n\t"                \
 380                         "j\t3b\n\t"                         \
 381                         ".previous\n\t"                     \
 382                         ".section\t__ex_table,\"a\"\n\t"    \
 383                         STR(PTR)"\t1b, 4b\n\t"              \
 384                         STR(PTR)"\t2b, 4b\n\t"              \
 385                         ".previous"                         \
 386                 : "=r" (res)                                \
 387                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 388 } while(0)
 389
 390 #define     _StoreDW(addr, value, res) \
 391 do {                                                        \
 392                 __asm__ __volatile__ (                      \
 393                         "1:\tsdl\t%1,(%2)\n"                \
 394                         "2:\tsdr\t%1, 7(%2)\n\t"            \
 395                         "li\t%0, 0\n"                       \
 396                         "3:\n\t"                            \
 397                         ".insn\n\t"                         \
 398                         ".section\t.fixup,\"ax\"\n\t"       \
 399                         "4:\tli\t%0, %3\n\t"                \
 400                         "j\t3b\n\t"                         \
 401                         ".previous\n\t"                     \
 402                         ".section\t__ex_table,\"a\"\n\t"    \
 403                         STR(PTR)"\t1b, 4b\n\t"              \
 404                         STR(PTR)"\t2b, 4b\n\t"              \
 405                         ".previous"                         \
 406                 : "=r" (res)                                \
 407                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 408 } while(0)
 409
 410 #else
 411 /* MIPSR6 has no swl and sdl instructions */
 412 #define     _StoreW(addr, value, res, type)  \
 413 do {                                                        \
 414                 __asm__ __volatile__ (                      \
 415                         ".set\tpush\n\t"                    \
 416                         ".set\tnoat\n\t"                    \
 417                         "1:"type##_sb("%1", "3(%2)")"\n\t"  \
 418                         "srl\t$1, %1, 0x8\n\t"              \
 419                         "2:"type##_sb("$1", "2(%2)")"\n\t"  \
 420                         "srl\t$1, $1,  0x8\n\t"             \
 421                         "3:"type##_sb("$1", "1(%2)")"\n\t"  \
 422                         "srl\t$1, $1, 0x8\n\t"              \
 423                         "4:"type##_sb("$1", "0(%2)")"\n\t"  \
 424                         ".set\tpop\n\t"                     \
 425                         "li\t%0, 0\n"                       \
 426                         "10:\n\t"                           \
 427                         ".insn\n\t"                         \
 428                         ".section\t.fixup,\"ax\"\n\t"       \
 429                         "11:\tli\t%0, %3\n\t"               \
 430                         "j\t10b\n\t"                        \
 431                         ".previous\n\t"                     \
 432                         ".section\t__ex_table,\"a\"\n\t"    \
 433                         STR(PTR)"\t1b, 11b\n\t"             \
 434                         STR(PTR)"\t2b, 11b\n\t"             \
 435                         STR(PTR)"\t3b, 11b\n\t"             \
 436                         STR(PTR)"\t4b, 11b\n\t"             \
 437                         ".previous"                         \
 438                 : "=&r" (res)                               \
 439                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 440                 : "memory");                                \
 441 } while(0)
 442
 443 #define     StoreDW(addr, value, res) \
 444 do {                                                        \
 445                 __asm__ __volatile__ (                      \
 446                         ".set\tpush\n\t"                    \
 447                         ".set\tnoat\n\t"                    \
 448                         "1:sb\t%1, 7(%2)\n\t"               \
 449                         "dsrl\t$1, %1, 0x8\n\t"             \
 450                         "2:sb\t$1, 6(%2)\n\t"               \
 451                         "dsrl\t$1, $1, 0x8\n\t"             \
 452                         "3:sb\t$1, 5(%2)\n\t"               \
 453                         "dsrl\t$1, $1, 0x8\n\t"             \
 454                         "4:sb\t$1, 4(%2)\n\t"               \
 455                         "dsrl\t$1, $1, 0x8\n\t"             \
 456                         "5:sb\t$1, 3(%2)\n\t"               \
 457                         "dsrl\t$1, $1, 0x8\n\t"             \
 458                         "6:sb\t$1, 2(%2)\n\t"               \
 459                         "dsrl\t$1, $1, 0x8\n\t"             \
 460                         "7:sb\t$1, 1(%2)\n\t"               \
 461                         "dsrl\t$1, $1, 0x8\n\t"             \
 462                         "8:sb\t$1, 0(%2)\n\t"               \
 463                         "dsrl\t$1, $1, 0x8\n\t"             \
 464                         ".set\tpop\n\t"                     \
 465                         "li\t%0, 0\n"                       \
 466                         "10:\n\t"                           \
 467                         ".insn\n\t"                         \
 468                         ".section\t.fixup,\"ax\"\n\t"       \
 469                         "11:\tli\t%0, %3\n\t"               \
 470                         "j\t10b\n\t"                        \
 471                         ".previous\n\t"                     \
 472                         ".section\t__ex_table,\"a\"\n\t"    \
 473                         STR(PTR)"\t1b, 11b\n\t"             \
 474                         STR(PTR)"\t2b, 11b\n\t"             \
 475                         STR(PTR)"\t3b, 11b\n\t"             \
 476                         STR(PTR)"\t4b, 11b\n\t"             \
 477                         STR(PTR)"\t5b, 11b\n\t"             \
 478                         STR(PTR)"\t6b, 11b\n\t"             \
 479                         STR(PTR)"\t7b, 11b\n\t"             \
 480                         STR(PTR)"\t8b, 11b\n\t"             \
 481                         ".previous"                         \
 482                 : "=&r" (res)                               \
 483                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 484                 : "memory");                                \
 485 } while(0)
 486
 487 #endif /* CONFIG_CPU_MIPSR6 */
 488
 489 #else /* __BIG_ENDIAN */
 490
 491 #define     _LoadHW(addr, value, res, type)  \
 492 do {                                                        \
 493                 __asm__ __volatile__ (".set\tnoat\n"        \
 494                         "1:\t"type##_lb("%0", "1(%2)")"\n"  \
 495                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 496                         "sll\t%0, 0x8\n\t"                  \
 497                         "or\t%0, $1\n\t"                    \
 498                         "li\t%1, 0\n"                       \
 499                         "3:\t.set\tat\n\t"                  \
 500                         ".insn\n\t"                         \
 501                         ".section\t.fixup,\"ax\"\n\t"       \
 502                         "4:\tli\t%1, %3\n\t"                \
 503                         "j\t3b\n\t"                         \
 504                         ".previous\n\t"                     \
 505                         ".section\t__ex_table,\"a\"\n\t"    \
 506                         STR(PTR)"\t1b, 4b\n\t"              \
 507                         STR(PTR)"\t2b, 4b\n\t"              \
 508                         ".previous"                         \
 509                         : "=&r" (value), "=r" (res)         \
 510                         : "r" (addr), "i" (-EFAULT));       \
 511 } while(0)
 512
 513 #ifndef CONFIG_CPU_MIPSR6
 514 #define     _LoadW(addr, value, res, type)   \
 515 do {                                                        \
 516                 __asm__ __volatile__ (                      \
 517                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
 518                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 519                         "li\t%1, 0\n"                       \
 520                         "3:\n\t"                            \
 521                         ".insn\n\t"                         \
 522                         ".section\t.fixup,\"ax\"\n\t"       \
 523                         "4:\tli\t%1, %3\n\t"                \
 524                         "j\t3b\n\t"                         \
 525                         ".previous\n\t"                     \
 526                         ".section\t__ex_table,\"a\"\n\t"    \
 527                         STR(PTR)"\t1b, 4b\n\t"              \
 528                         STR(PTR)"\t2b, 4b\n\t"              \
 529                         ".previous"                         \
 530                         : "=&r" (value), "=r" (res)         \
 531                         : "r" (addr), "i" (-EFAULT));       \
 532 } while(0)
 533
 534 #else
 535 /* MIPSR6 has no lwl instruction */
 536 #define     _LoadW(addr, value, res, type) \
 537 do {                                                        \
 538                 __asm__ __volatile__ (                      \
 539                         ".set\tpush\n"                      \
 540                         ".set\tnoat\n\t"                    \
 541                         "1:"type##_lb("%0", "3(%2)")"\n\t"  \
 542                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
 543                         "sll\t%0, 0x8\n\t"                  \
 544                         "or\t%0, $1\n\t"                    \
 545                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
 546                         "sll\t%0, 0x8\n\t"                  \
 547                         "or\t%0, $1\n\t"                    \
 548                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
 549                         "sll\t%0, 0x8\n\t"                  \
 550                         "or\t%0, $1\n\t"                    \
 551                         "li\t%1, 0\n"                       \
 552                         ".set\tpop\n"                       \
 553                         "10:\n\t"                           \
 554                         ".insn\n\t"                         \
 555                         ".section\t.fixup,\"ax\"\n\t"       \
 556                         "11:\tli\t%1, %3\n\t"               \
 557                         "j\t10b\n\t"                        \
 558                         ".previous\n\t"                     \
 559                         ".section\t__ex_table,\"a\"\n\t"    \
 560                         STR(PTR)"\t1b, 11b\n\t"             \
 561                         STR(PTR)"\t2b, 11b\n\t"             \
 562                         STR(PTR)"\t3b, 11b\n\t"             \
 563                         STR(PTR)"\t4b, 11b\n\t"             \
 564                         ".previous"                         \
 565                         : "=&r" (value), "=r" (res)         \
 566                         : "r" (addr), "i" (-EFAULT));       \
 567 } while(0)
 568
 569 #endif /* CONFIG_CPU_MIPSR6 */
 570
 571
 572 #define     _LoadHWU(addr, value, res, type) \
 573 do {                                                        \
 574                 __asm__ __volatile__ (                      \
 575                         ".set\tnoat\n"                      \
 576                         "1:\t"type##_lbu("%0", "1(%2)")"\n" \
 577                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 578                         "sll\t%0, 0x8\n\t"                  \
 579                         "or\t%0, $1\n\t"                    \
 580                         "li\t%1, 0\n"                       \
 581                         "3:\n\t"                            \
 582                         ".insn\n\t"                         \
 583                         ".set\tat\n\t"                      \
 584                         ".section\t.fixup,\"ax\"\n\t"       \
 585                         "4:\tli\t%1, %3\n\t"                \
 586                         "j\t3b\n\t"                         \
 587                         ".previous\n\t"                     \
 588                         ".section\t__ex_table,\"a\"\n\t"    \
 589                         STR(PTR)"\t1b, 4b\n\t"              \
 590                         STR(PTR)"\t2b, 4b\n\t"              \
 591                         ".previous"                         \
 592                         : "=&r" (value), "=r" (res)         \
 593                         : "r" (addr), "i" (-EFAULT));       \
 594 } while(0)
 595
 596 #ifndef CONFIG_CPU_MIPSR6
 597 #define     _LoadWU(addr, value, res, type)  \
 598 do {                                                        \
 599                 __asm__ __volatile__ (                      \
 600                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
 601                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 602                         "dsll\t%0, %0, 32\n\t"              \
 603                         "dsrl\t%0, %0, 32\n\t"              \
 604                         "li\t%1, 0\n"                       \
 605                         "3:\n\t"                            \
 606                         ".insn\n\t"                         \
 607                         "\t.section\t.fixup,\"ax\"\n\t"     \
 608                         "4:\tli\t%1, %3\n\t"                \
 609                         "j\t3b\n\t"                         \
 610                         ".previous\n\t"                     \
 611                         ".section\t__ex_table,\"a\"\n\t"    \
 612                         STR(PTR)"\t1b, 4b\n\t"              \
 613                         STR(PTR)"\t2b, 4b\n\t"              \
 614                         ".previous"                         \
 615                         : "=&r" (value), "=r" (res)         \
 616                         : "r" (addr), "i" (-EFAULT));       \
 617 } while(0)
 618
 619 #define     _LoadDW(addr, value, res)  \
 620 do {                                                        \
 621                 __asm__ __volatile__ (                      \
 622                         "1:\tldl\t%0, 7(%2)\n"              \
 623                         "2:\tldr\t%0, (%2)\n\t"             \
 624                         "li\t%1, 0\n"                       \
 625                         "3:\n\t"                            \
 626                         ".insn\n\t"                         \
 627                         "\t.section\t.fixup,\"ax\"\n\t"     \
 628                         "4:\tli\t%1, %3\n\t"                \
 629                         "j\t3b\n\t"                         \
 630                         ".previous\n\t"                     \
 631                         ".section\t__ex_table,\"a\"\n\t"    \
 632                         STR(PTR)"\t1b, 4b\n\t"              \
 633                         STR(PTR)"\t2b, 4b\n\t"              \
 634                         ".previous"                         \
 635                         : "=&r" (value), "=r" (res)         \
 636                         : "r" (addr), "i" (-EFAULT));       \
 637 } while(0)
 638
 639 #else
 640 /* MIPSR6 has not lwl and ldl instructions */
 641 #define     _LoadWU(addr, value, res, type) \
 642 do {                                                        \
 643                 __asm__ __volatile__ (                      \
 644                         ".set\tpush\n\t"                    \
 645                         ".set\tnoat\n\t"                    \
 646                         "1:"type##_lbu("%0", "3(%2)")"\n\t" \
 647                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
 648                         "sll\t%0, 0x8\n\t"                  \
 649                         "or\t%0, $1\n\t"                    \
 650                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
 651                         "sll\t%0, 0x8\n\t"                  \
 652                         "or\t%0, $1\n\t"                    \
 653                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
 654                         "sll\t%0, 0x8\n\t"                  \
 655                         "or\t%0, $1\n\t"                    \
 656                         "li\t%1, 0\n"                       \
 657                         ".set\tpop\n"                       \
 658                         "10:\n\t"                           \
 659                         ".insn\n\t"                         \
 660                         ".section\t.fixup,\"ax\"\n\t"       \
 661                         "11:\tli\t%1, %3\n\t"               \
 662                         "j\t10b\n\t"                        \
 663                         ".previous\n\t"                     \
 664                         ".section\t__ex_table,\"a\"\n\t"    \
 665                         STR(PTR)"\t1b, 11b\n\t"             \
 666                         STR(PTR)"\t2b, 11b\n\t"             \
 667                         STR(PTR)"\t3b, 11b\n\t"             \
 668                         STR(PTR)"\t4b, 11b\n\t"             \
 669                         ".previous"                         \
 670                         : "=&r" (value), "=r" (res)         \
 671                         : "r" (addr), "i" (-EFAULT));       \
 672 } while(0)
 673
 674 #define     _LoadDW(addr, value, res)  \
 675 do {                                                        \
 676                 __asm__ __volatile__ (                      \
 677                         ".set\tpush\n\t"                    \
 678                         ".set\tnoat\n\t"                    \
 679                         "1:lb\t%0, 7(%2)\n\t"               \
 680                         "2:lbu\t$1, 6(%2)\n\t"              \
 681                         "dsll\t%0, 0x8\n\t"                 \
 682                         "or\t%0, $1\n\t"                    \
 683                         "3:lbu\t$1, 5(%2)\n\t"              \
 684                         "dsll\t%0, 0x8\n\t"                 \
 685                         "or\t%0, $1\n\t"                    \
 686                         "4:lbu\t$1, 4(%2)\n\t"              \
 687                         "dsll\t%0, 0x8\n\t"                 \
 688                         "or\t%0, $1\n\t"                    \
 689                         "5:lbu\t$1, 3(%2)\n\t"              \
 690                         "dsll\t%0, 0x8\n\t"                 \
 691                         "or\t%0, $1\n\t"                    \
 692                         "6:lbu\t$1, 2(%2)\n\t"              \
 693                         "dsll\t%0, 0x8\n\t"                 \
 694                         "or\t%0, $1\n\t"                    \
 695                         "7:lbu\t$1, 1(%2)\n\t"              \
 696                         "dsll\t%0, 0x8\n\t"                 \
 697                         "or\t%0, $1\n\t"                    \
 698                         "8:lbu\t$1, 0(%2)\n\t"              \
 699                         "dsll\t%0, 0x8\n\t"                 \
 700                         "or\t%0, $1\n\t"                    \
 701                         "li\t%1, 0\n"                       \
 702                         ".set\tpop\n\t"                     \
 703                         "10:\n\t"                           \
 704                         ".insn\n\t"                         \
 705                         ".section\t.fixup,\"ax\"\n\t"       \
 706                         "11:\tli\t%1, %3\n\t"               \
 707                         "j\t10b\n\t"                        \
 708                         ".previous\n\t"                     \
 709                         ".section\t__ex_table,\"a\"\n\t"    \
 710                         STR(PTR)"\t1b, 11b\n\t"             \
 711                         STR(PTR)"\t2b, 11b\n\t"             \
 712                         STR(PTR)"\t3b, 11b\n\t"             \
 713                         STR(PTR)"\t4b, 11b\n\t"             \
 714                         STR(PTR)"\t5b, 11b\n\t"             \
 715                         STR(PTR)"\t6b, 11b\n\t"             \
 716                         STR(PTR)"\t7b, 11b\n\t"             \
 717                         STR(PTR)"\t8b, 11b\n\t"             \
 718                         ".previous"                         \
 719                         : "=&r" (value), "=r" (res)         \
 720                         : "r" (addr), "i" (-EFAULT));       \
 721 } while(0)
 722 #endif /* CONFIG_CPU_MIPSR6 */
 723
 724 #define     _StoreHW(addr, value, res, type) \
 725 do {                                                        \
 726                 __asm__ __volatile__ (                      \
 727                         ".set\tnoat\n"                      \
 728                         "1:\t"type##_sb("%1", "0(%2)")"\n"  \
 729                         "srl\t$1,%1, 0x8\n"                 \
 730                         "2:\t"type##_sb("$1", "1(%2)")"\n"  \
 731                         ".set\tat\n\t"                      \
 732                         "li\t%0, 0\n"                       \
 733                         "3:\n\t"                            \
 734                         ".insn\n\t"                         \
 735                         ".section\t.fixup,\"ax\"\n\t"       \
 736                         "4:\tli\t%0, %3\n\t"                \
 737                         "j\t3b\n\t"                         \
 738                         ".previous\n\t"                     \
 739                         ".section\t__ex_table,\"a\"\n\t"    \
 740                         STR(PTR)"\t1b, 4b\n\t"              \
 741                         STR(PTR)"\t2b, 4b\n\t"              \
 742                         ".previous"                         \
 743                         : "=r" (res)                        \
 744                         : "r" (value), "r" (addr), "i" (-EFAULT));\
 745 } while(0)
 746
 747 #ifndef CONFIG_CPU_MIPSR6
 748 #define     _StoreW(addr, value, res, type)  \
 749 do {                                                        \
 750                 __asm__ __volatile__ (                      \
 751                         "1:\t"type##_swl("%1", "3(%2)")"\n" \
 752                         "2:\t"type##_swr("%1", "(%2)")"\n\t"\
 753                         "li\t%0, 0\n"                       \
 754                         "3:\n\t"                            \
 755                         ".insn\n\t"                         \
 756                         ".section\t.fixup,\"ax\"\n\t"       \
 757                         "4:\tli\t%0, %3\n\t"                \
 758                         "j\t3b\n\t"                         \
 759                         ".previous\n\t"                     \
 760                         ".section\t__ex_table,\"a\"\n\t"    \
 761                         STR(PTR)"\t1b, 4b\n\t"              \
 762                         STR(PTR)"\t2b, 4b\n\t"              \
 763                         ".previous"                         \
 764                 : "=r" (res)                                \
 765                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 766 } while(0)
 767
 768 #define     _StoreDW(addr, value, res) \
 769 do {                                                        \
 770                 __asm__ __volatile__ (                      \
 771                         "1:\tsdl\t%1, 7(%2)\n"              \
 772                         "2:\tsdr\t%1, (%2)\n\t"             \
 773                         "li\t%0, 0\n"                       \
 774                         "3:\n\t"                            \
 775                         ".insn\n\t"                         \
 776                         ".section\t.fixup,\"ax\"\n\t"       \
 777                         "4:\tli\t%0, %3\n\t"                \
 778                         "j\t3b\n\t"                         \
 779                         ".previous\n\t"                     \
 780                         ".section\t__ex_table,\"a\"\n\t"    \
 781                         STR(PTR)"\t1b, 4b\n\t"              \
 782                         STR(PTR)"\t2b, 4b\n\t"              \
 783                         ".previous"                         \
 784                 : "=r" (res)                                \
 785                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 786 } while(0)
 787
 788 #else
 789 /* MIPSR6 has no swl and sdl instructions */
 790 #define     _StoreW(addr, value, res, type)  \
 791 do {                                                        \
 792                 __asm__ __volatile__ (                      \
 793                         ".set\tpush\n\t"                    \
 794                         ".set\tnoat\n\t"                    \
 795                         "1:"type##_sb("%1", "0(%2)")"\n\t"  \
 796                         "srl\t$1, %1, 0x8\n\t"              \
 797                         "2:"type##_sb("$1", "1(%2)")"\n\t"  \
 798                         "srl\t$1, $1,  0x8\n\t"             \
 799                         "3:"type##_sb("$1", "2(%2)")"\n\t"  \
 800                         "srl\t$1, $1, 0x8\n\t"              \
 801                         "4:"type##_sb("$1", "3(%2)")"\n\t"  \
 802                         ".set\tpop\n\t"                     \
 803                         "li\t%0, 0\n"                       \
 804                         "10:\n\t"                           \
 805                         ".insn\n\t"                         \
 806                         ".section\t.fixup,\"ax\"\n\t"       \
 807                         "11:\tli\t%0, %3\n\t"               \
 808                         "j\t10b\n\t"                        \
 809                         ".previous\n\t"                     \
 810                         ".section\t__ex_table,\"a\"\n\t"    \
 811                         STR(PTR)"\t1b, 11b\n\t"             \
 812                         STR(PTR)"\t2b, 11b\n\t"             \
 813                         STR(PTR)"\t3b, 11b\n\t"             \
 814                         STR(PTR)"\t4b, 11b\n\t"             \
 815                         ".previous"                         \
 816                 : "=&r" (res)                               \
 817                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 818                 : "memory");                                \
 819 } while(0)
 820
 821 #define     _StoreDW(addr, value, res) \
 822 do {                                                        \
 823                 __asm__ __volatile__ (                      \
 824                         ".set\tpush\n\t"                    \
 825                         ".set\tnoat\n\t"                    \
 826                         "1:sb\t%1, 0(%2)\n\t"               \
 827                         "dsrl\t$1, %1, 0x8\n\t"             \
 828                         "2:sb\t$1, 1(%2)\n\t"               \
 829                         "dsrl\t$1, $1, 0x8\n\t"             \
 830                         "3:sb\t$1, 2(%2)\n\t"               \
 831                         "dsrl\t$1, $1, 0x8\n\t"             \
 832                         "4:sb\t$1, 3(%2)\n\t"               \
 833                         "dsrl\t$1, $1, 0x8\n\t"             \
 834                         "5:sb\t$1, 4(%2)\n\t"               \
 835                         "dsrl\t$1, $1, 0x8\n\t"             \
 836                         "6:sb\t$1, 5(%2)\n\t"               \
 837                         "dsrl\t$1, $1, 0x8\n\t"             \
 838                         "7:sb\t$1, 6(%2)\n\t"               \
 839                         "dsrl\t$1, $1, 0x8\n\t"             \
 840                         "8:sb\t$1, 7(%2)\n\t"               \
 841                         "dsrl\t$1, $1, 0x8\n\t"             \
 842                         ".set\tpop\n\t"                     \
 843                         "li\t%0, 0\n"                       \
 844                         "10:\n\t"                           \
 845                         ".insn\n\t"                         \
 846                         ".section\t.fixup,\"ax\"\n\t"       \
 847                         "11:\tli\t%0, %3\n\t"               \
 848                         "j\t10b\n\t"                        \
 849                         ".previous\n\t"                     \
 850                         ".section\t__ex_table,\"a\"\n\t"    \
 851                         STR(PTR)"\t1b, 11b\n\t"             \
 852                         STR(PTR)"\t2b, 11b\n\t"             \
 853                         STR(PTR)"\t3b, 11b\n\t"             \
 854                         STR(PTR)"\t4b, 11b\n\t"             \
 855                         STR(PTR)"\t5b, 11b\n\t"             \
 856                         STR(PTR)"\t6b, 11b\n\t"             \
 857                         STR(PTR)"\t7b, 11b\n\t"             \
 858                         STR(PTR)"\t8b, 11b\n\t"             \
 859                         ".previous"                         \
 860                 : "=&r" (res)                               \
 861                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 862                 : "memory");                                \
 863 } while(0)
 864
 865 #endif /* CONFIG_CPU_MIPSR6 */
 866 #endif
 867
 868 #define LoadHWU(addr, value, res)       _LoadHWU(addr, value, res, kernel)
 869 #define LoadHWUE(addr, value, res)      _LoadHWU(addr, value, res, user)
 870 #define LoadWU(addr, value, res)        _LoadWU(addr, value, res, kernel)
 871 #define LoadWUE(addr, value, res)       _LoadWU(addr, value, res, user)
 872 #define LoadHW(addr, value, res)        _LoadHW(addr, value, res, kernel)
 873 #define LoadHWE(addr, value, res)       _LoadHW(addr, value, res, user)
 874 #define LoadW(addr, value, res)         _LoadW(addr, value, res, kernel)
 875 #define LoadWE(addr, value, res)        _LoadW(addr, value, res, user)
 876 #define LoadDW(addr, value, res)        _LoadDW(addr, value, res)
 877
 878 #define StoreHW(addr, value, res)       _StoreHW(addr, value, res, kernel)
 879 #define StoreHWE(addr, value, res)      _StoreHW(addr, value, res, user)
 880 #define StoreW(addr, value, res)        _StoreW(addr, value, res, kernel)
 881 #define StoreWE(addr, value, res)       _StoreW(addr, value, res, user)
 882 #define StoreDW(addr, value, res)       _StoreDW(addr, value, res)
 883
 884 static void emulate_load_store_insn(struct pt_regs *regs,
 885         void __user *addr, unsigned int __user *pc)
 886 {
 887         union mips_instruction insn;
 888         unsigned long value;
 889         unsigned int res;
 890         unsigned long origpc;
 891         unsigned long orig31;
 892         void __user *fault_addr = NULL;
 893 #ifdef  CONFIG_EVA
 894         mm_segment_t seg;
 895 #endif
 896         origpc = (unsigned long)pc;
 897         orig31 = regs->regs[31];
 898
 899         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 900
 901         /*
 902          * This load never faults.
 903          */
 904         __get_user(insn.word, pc);
 905
 906         switch (insn.i_format.opcode) {
 907                 /*
 908                  * These are instructions that a compiler doesn't generate.  We
 909                  * can assume therefore that the code is MIPS-aware and
 910                  * really buggy.  Emulating these instructions would break the
 911                  * semantics anyway.
 912                  */
 913         case ll_op:
 914         case lld_op:
 915         case sc_op:
 916         case scd_op:
 917
 918                 /*
 919                  * For these instructions the only way to create an address
 920                  * error is an attempted access to kernel/supervisor address
 921                  * space.
 922                  */
 923         case ldl_op:
 924         case ldr_op:
 925         case lwl_op:
 926         case lwr_op:
 927         case sdl_op:
 928         case sdr_op:
 929         case swl_op:
 930         case swr_op:
 931         case lb_op:
 932         case lbu_op:
 933         case sb_op:
 934                 goto sigbus;
 935
 936                 /*
 937                  * The remaining opcodes are the ones that are really of
 938                  * interest.
 939                  */
 940 #ifdef CONFIG_EVA
 941         case spec3_op:
 942                 /*
 943                  * we can land here only from kernel accessing user memory,
 944                  * so we need to "switch" the address limit to user space, so
 945                  * address check can work properly.
 946                  */
 947                 seg = get_fs();
 948                 set_fs(USER_DS);
 949                 switch (insn.spec3_format.func) {
 950                 case lhe_op:
 951                         if (!access_ok(VERIFY_READ, addr, 2)) {
 952                                 set_fs(seg);
 953                                 goto sigbus;
 954                         }
 955                         LoadHWE(addr, value, res);
 956                         if (res) {
 957                                 set_fs(seg);
 958                                 goto fault;
 959                         }
 960                         compute_return_epc(regs);
 961                         regs->regs[insn.spec3_format.rt] = value;
 962                         break;
 963                 case lwe_op:
 964                         if (!access_ok(VERIFY_READ, addr, 4)) {
 965                                 set_fs(seg);
 966                                 goto sigbus;
 967                         }
 968                                 LoadWE(addr, value, res);
 969                         if (res) {
 970                                 set_fs(seg);
 971                                 goto fault;
 972                         }
 973                         compute_return_epc(regs);
 974                         regs->regs[insn.spec3_format.rt] = value;
 975                         break;
 976                 case lhue_op:
 977                         if (!access_ok(VERIFY_READ, addr, 2)) {
 978                                 set_fs(seg);
 979                                 goto sigbus;
 980                         }
 981                         LoadHWUE(addr, value, res);
 982                         if (res) {
 983                                 set_fs(seg);
 984                                 goto fault;
 985                         }
 986                         compute_return_epc(regs);
 987                         regs->regs[insn.spec3_format.rt] = value;
 988                         break;
 989                 case she_op:
 990                         if (!access_ok(VERIFY_WRITE, addr, 2)) {
 991                                 set_fs(seg);
 992                                 goto sigbus;
 993                         }
 994                         compute_return_epc(regs);
 995                         value = regs->regs[insn.spec3_format.rt];
 996                         StoreHWE(addr, value, res);
 997                         if (res) {
 998                                 set_fs(seg);
 999                                 goto fault;
1000                         }
1001                         break;
1002                 case swe_op:
1003                         if (!access_ok(VERIFY_WRITE, addr, 4)) {
1004                                 set_fs(seg);
1005                                 goto sigbus;
1006                         }
1007                         compute_return_epc(regs);
1008                         value = regs->regs[insn.spec3_format.rt];
1009                         StoreWE(addr, value, res);
1010                         if (res) {
1011                                 set_fs(seg);
1012                                 goto fault;
1013                         }
1014                         break;
1015                 default:
1016                         set_fs(seg);
1017                         goto sigill;
1018                 }
1019                 set_fs(seg);
1020                 break;
1021 #endif
1022         case lh_op:
1023                 if (!access_ok(VERIFY_READ, addr, 2))
1024                         goto sigbus;
1025
1026                 LoadHW(addr, value, res);
1027                 if (res)
1028                         goto fault;
1029                 compute_return_epc(regs);
1030                 regs->regs[insn.i_format.rt] = value;
1031                 break;
1032
1033         case lw_op:
1034                 if (!access_ok(VERIFY_READ, addr, 4))
1035                         goto sigbus;
1036
1037                 LoadW(addr, value, res);
1038                 if (res)
1039                         goto fault;
1040                 compute_return_epc(regs);
1041                 regs->regs[insn.i_format.rt] = value;
1042                 break;
1043
1044         case lhu_op:
1045                 if (!access_ok(VERIFY_READ, addr, 2))
1046                         goto sigbus;
1047
1048                 LoadHWU(addr, value, res);
1049                 if (res)
1050                         goto fault;
1051                 compute_return_epc(regs);
1052                 regs->regs[insn.i_format.rt] = value;
1053                 break;
1054
1055         case lwu_op:
1056 #ifdef CONFIG_64BIT
1057                 /*
1058                  * A 32-bit kernel might be running on a 64-bit processor.  But
1059                  * if we're on a 32-bit processor and an i-cache incoherency
1060                  * or race makes us see a 64-bit instruction here the sdl/sdr
1061                  * would blow up, so for now we don't handle unaligned 64-bit
1062                  * instructions on 32-bit kernels.
1063                  */
1064                 if (!access_ok(VERIFY_READ, addr, 4))
1065                         goto sigbus;
1066
1067                 LoadWU(addr, value, res);
1068                 if (res)
1069                         goto fault;
1070                 compute_return_epc(regs);
1071                 regs->regs[insn.i_format.rt] = value;
1072                 break;
1073 #endif /* CONFIG_64BIT */
1074
1075                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1076                 goto sigill;
1077
1078         case ld_op:
1079 #ifdef CONFIG_64BIT
1080                 /*
1081                  * A 32-bit kernel might be running on a 64-bit processor.  But
1082                  * if we're on a 32-bit processor and an i-cache incoherency
1083                  * or race makes us see a 64-bit instruction here the sdl/sdr
1084                  * would blow up, so for now we don't handle unaligned 64-bit
1085                  * instructions on 32-bit kernels.
1086                  */
1087                 if (!access_ok(VERIFY_READ, addr, 8))
1088                         goto sigbus;
1089
1090                 LoadDW(addr, value, res);
1091                 if (res)
1092                         goto fault;
1093                 compute_return_epc(regs);
1094                 regs->regs[insn.i_format.rt] = value;
1095                 break;
1096 #endif /* CONFIG_64BIT */
1097
1098                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1099                 goto sigill;
1100
1101         case sh_op:
1102                 if (!access_ok(VERIFY_WRITE, addr, 2))
1103                         goto sigbus;
1104
1105                 compute_return_epc(regs);
1106                 value = regs->regs[insn.i_format.rt];
1107                 StoreHW(addr, value, res);
1108                 if (res)
1109                         goto fault;
1110                 break;
1111
1112         case sw_op:
1113                 if (!access_ok(VERIFY_WRITE, addr, 4))
1114                         goto sigbus;
1115
1116                 compute_return_epc(regs);
1117                 value = regs->regs[insn.i_format.rt];
1118                 StoreW(addr, value, res);
1119                 if (res)
1120                         goto fault;
1121                 break;
1122
1123         case sd_op:
1124 #ifdef CONFIG_64BIT
1125                 /*
1126                  * A 32-bit kernel might be running on a 64-bit processor.  But
1127                  * if we're on a 32-bit processor and an i-cache incoherency
1128                  * or race makes us see a 64-bit instruction here the sdl/sdr
1129                  * would blow up, so for now we don't handle unaligned 64-bit
1130                  * instructions on 32-bit kernels.
1131                  */
1132                 if (!access_ok(VERIFY_WRITE, addr, 8))
1133                         goto sigbus;
1134
1135                 compute_return_epc(regs);
1136                 value = regs->regs[insn.i_format.rt];
1137                 StoreDW(addr, value, res);
1138                 if (res)
1139                         goto fault;
1140                 break;
1141 #endif /* CONFIG_64BIT */
1142
1143                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1144                 goto sigill;
1145
1146         case lwc1_op:
1147         case ldc1_op:
1148         case swc1_op:
1149         case sdc1_op:
1150                 die_if_kernel("Unaligned FP access in kernel code", regs);
1151                 BUG_ON(!used_math());
1152
1153                 lose_fpu(1);    /* Save FPU state for the emulator. */
1154                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1155                                                &fault_addr);
1156                 own_fpu(1);     /* Restore FPU state. */
1157
1158                 /* Signal if something went wrong. */
1159                 process_fpemu_return(res, fault_addr);
1160
1161                 if (res == 0)
1162                         break;
1163                 return;
1164
1165 #ifndef CONFIG_CPU_MIPSR6
1166         /*
1167          * COP2 is available to implementor for application specific use.
1168          * It's up to applications to register a notifier chain and do
1169          * whatever they have to do, including possible sending of signals.
1170          *
1171          * This instruction has been reallocated in Release 6
1172          */
1173         case lwc2_op:
1174                 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1175                 break;
1176
1177         case ldc2_op:
1178                 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1179                 break;
1180
1181         case swc2_op:
1182                 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1183                 break;
1184
1185         case sdc2_op:
1186                 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1187                 break;
1188 #endif
1189         default:
1190                 /*
1191                  * Pheeee...  We encountered an yet unknown instruction or
1192                  * cache coherence problem.  Die sucker, die ...
1193                  */
1194                 goto sigill;
1195         }
1196
1197 #ifdef CONFIG_DEBUG_FS
1198         unaligned_instructions++;
1199 #endif
1200
1201         return;
1202
1203 fault:
1204         /* roll back jump/branch */
1205         regs->cp0_epc = origpc;
1206         regs->regs[31] = orig31;
1207         /* Did we have an exception handler installed? */
1208         if (fixup_exception(regs))
1209                 return;
1210
1211         die_if_kernel("Unhandled kernel unaligned access", regs);
1212         force_sig(SIGSEGV, current);
1213
1214         return;
1215
1216 sigbus:
1217         die_if_kernel("Unhandled kernel unaligned access", regs);
1218         force_sig(SIGBUS, current);
1219
1220         return;
1221
1222 sigill:
1223         die_if_kernel
1224             ("Unhandled kernel unaligned access or invalid instruction", regs);
1225         force_sig(SIGILL, current);
1226 }
1227
1228 /* Recode table from 16-bit register notation to 32-bit GPR. */
1229 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1230
1231 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1232 const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1233
1234 static void emulate_load_store_microMIPS(struct pt_regs *regs,
1235                                          void __user *addr)
1236 {
1237         unsigned long value;
1238         unsigned int res;
1239         int i;
1240         unsigned int reg = 0, rvar;
1241         unsigned long orig31;
1242         u16 __user *pc16;
1243         u16 halfword;
1244         unsigned int word;
1245         unsigned long origpc, contpc;
1246         union mips_instruction insn;
1247         struct mm_decoded_insn mminsn;
1248         void __user *fault_addr = NULL;
1249
1250         origpc = regs->cp0_epc;
1251         orig31 = regs->regs[31];
1252
1253         mminsn.micro_mips_mode = 1;
1254
1255         /*
1256          * This load never faults.
1257          */
1258         pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1259         __get_user(halfword, pc16);
1260         pc16++;
1261         contpc = regs->cp0_epc + 2;
1262         word = ((unsigned int)halfword << 16);
1263         mminsn.pc_inc = 2;
1264
1265         if (!mm_insn_16bit(halfword)) {
1266                 __get_user(halfword, pc16);
1267                 pc16++;
1268                 contpc = regs->cp0_epc + 4;
1269                 mminsn.pc_inc = 4;
1270                 word |= halfword;
1271         }
1272         mminsn.insn = word;
1273
1274         if (get_user(halfword, pc16))
1275                 goto fault;
1276         mminsn.next_pc_inc = 2;
1277         word = ((unsigned int)halfword << 16);
1278
1279         if (!mm_insn_16bit(halfword)) {
1280                 pc16++;
1281                 if (get_user(halfword, pc16))
1282                         goto fault;
1283                 mminsn.next_pc_inc = 4;
1284                 word |= halfword;
1285         }
1286         mminsn.next_insn = word;
1287
1288         insn = (union mips_instruction)(mminsn.insn);
1289         if (mm_isBranchInstr(regs, mminsn, &contpc))
1290                 insn = (union mips_instruction)(mminsn.next_insn);
1291
1292         /*  Parse instruction to find what to do */
1293
1294         switch (insn.mm_i_format.opcode) {
1295
1296         case mm_pool32a_op:
1297                 switch (insn.mm_x_format.func) {
1298                 case mm_lwxs_op:
1299                         reg = insn.mm_x_format.rd;
1300                         goto loadW;
1301                 }
1302
1303                 goto sigbus;
1304
1305         case mm_pool32b_op:
1306                 switch (insn.mm_m_format.func) {
1307                 case mm_lwp_func:
1308                         reg = insn.mm_m_format.rd;
1309                         if (reg == 31)
1310                                 goto sigbus;
1311
1312                         if (!access_ok(VERIFY_READ, addr, 8))
1313                                 goto sigbus;
1314
1315                         LoadW(addr, value, res);
1316                         if (res)
1317                                 goto fault;
1318                         regs->regs[reg] = value;
1319                         addr += 4;
1320                         LoadW(addr, value, res);
1321                         if (res)
1322                                 goto fault;
1323                         regs->regs[reg + 1] = value;
1324                         goto success;
1325
1326                 case mm_swp_func:
1327                         reg = insn.mm_m_format.rd;
1328                         if (reg == 31)
1329                                 goto sigbus;
1330
1331                         if (!access_ok(VERIFY_WRITE, addr, 8))
1332                                 goto sigbus;
1333
1334                         value = regs->regs[reg];
1335                         StoreW(addr, value, res);
1336                         if (res)
1337                                 goto fault;
1338                         addr += 4;
1339                         value = regs->regs[reg + 1];
1340                         StoreW(addr, value, res);
1341                         if (res)
1342                                 goto fault;
1343                         goto success;
1344
1345                 case mm_ldp_func:
1346 #ifdef CONFIG_64BIT
1347                         reg = insn.mm_m_format.rd;
1348                         if (reg == 31)
1349                                 goto sigbus;
1350
1351                         if (!access_ok(VERIFY_READ, addr, 16))
1352                                 goto sigbus;
1353
1354                         LoadDW(addr, value, res);
1355                         if (res)
1356                                 goto fault;
1357                         regs->regs[reg] = value;
1358                         addr += 8;
1359                         LoadDW(addr, value, res);
1360                         if (res)
1361                                 goto fault;
1362                         regs->regs[reg + 1] = value;
1363                         goto success;
1364 #endif /* CONFIG_64BIT */
1365
1366                         goto sigill;
1367
1368                 case mm_sdp_func:
1369 #ifdef CONFIG_64BIT
1370                         reg = insn.mm_m_format.rd;
1371                         if (reg == 31)
1372                                 goto sigbus;
1373
1374                         if (!access_ok(VERIFY_WRITE, addr, 16))
1375                                 goto sigbus;
1376
1377                         value = regs->regs[reg];
1378                         StoreDW(addr, value, res);
1379                         if (res)
1380                                 goto fault;
1381                         addr += 8;
1382                         value = regs->regs[reg + 1];
1383                         StoreDW(addr, value, res);
1384                         if (res)
1385                                 goto fault;
1386                         goto success;
1387 #endif /* CONFIG_64BIT */
1388
1389                         goto sigill;
1390
1391                 case mm_lwm32_func:
1392                         reg = insn.mm_m_format.rd;
1393                         rvar = reg & 0xf;
1394                         if ((rvar > 9) || !reg)
1395                                 goto sigill;
1396                         if (reg & 0x10) {
1397                                 if (!access_ok
1398                                     (VERIFY_READ, addr, 4 * (rvar + 1)))
1399                                         goto sigbus;
1400                         } else {
1401                                 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1402                                         goto sigbus;
1403                         }
1404                         if (rvar == 9)
1405                                 rvar = 8;
1406                         for (i = 16; rvar; rvar--, i++) {
1407                                 LoadW(addr, value, res);
1408                                 if (res)
1409                                         goto fault;
1410                                 addr += 4;
1411                                 regs->regs[i] = value;
1412                         }
1413                         if ((reg & 0xf) == 9) {
1414                                 LoadW(addr, value, res);
1415                                 if (res)
1416                                         goto fault;
1417                                 addr += 4;
1418                                 regs->regs[30] = value;
1419                         }
1420                         if (reg & 0x10) {
1421                                 LoadW(addr, value, res);
1422                                 if (res)
1423                                         goto fault;
1424                                 regs->regs[31] = value;
1425                         }
1426                         goto success;
1427
1428                 case mm_swm32_func:
1429                         reg = insn.mm_m_format.rd;
1430                         rvar = reg & 0xf;
1431                         if ((rvar > 9) || !reg)
1432                                 goto sigill;
1433                         if (reg & 0x10) {
1434                                 if (!access_ok
1435                                     (VERIFY_WRITE, addr, 4 * (rvar + 1)))
1436                                         goto sigbus;
1437                         } else {
1438                                 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1439                                         goto sigbus;
1440                         }
1441                         if (rvar == 9)
1442                                 rvar = 8;
1443                         for (i = 16; rvar; rvar--, i++) {
1444                                 value = regs->regs[i];
1445                                 StoreW(addr, value, res);
1446                                 if (res)
1447                                         goto fault;
1448                                 addr += 4;
1449                         }
1450                         if ((reg & 0xf) == 9) {
1451                                 value = regs->regs[30];
1452                                 StoreW(addr, value, res);
1453                                 if (res)
1454                                         goto fault;
1455                                 addr += 4;
1456                         }
1457                         if (reg & 0x10) {
1458                                 value = regs->regs[31];
1459                                 StoreW(addr, value, res);
1460                                 if (res)
1461                                         goto fault;
1462                         }
1463                         goto success;
1464
1465                 case mm_ldm_func:
1466 #ifdef CONFIG_64BIT
1467                         reg = insn.mm_m_format.rd;
1468                         rvar = reg & 0xf;
1469                         if ((rvar > 9) || !reg)
1470                                 goto sigill;
1471                         if (reg & 0x10) {
1472                                 if (!access_ok
1473                                     (VERIFY_READ, addr, 8 * (rvar + 1)))
1474                                         goto sigbus;
1475                         } else {
1476                                 if (!access_ok(VERIFY_READ, addr, 8 * rvar))
1477                                         goto sigbus;
1478                         }
1479                         if (rvar == 9)
1480                                 rvar = 8;
1481
1482                         for (i = 16; rvar; rvar--, i++) {
1483                                 LoadDW(addr, value, res);
1484                                 if (res)
1485                                         goto fault;
1486                                 addr += 4;
1487                                 regs->regs[i] = value;
1488                         }
1489                         if ((reg & 0xf) == 9) {
1490                                 LoadDW(addr, value, res);
1491                                 if (res)
1492                                         goto fault;
1493                                 addr += 8;
1494                                 regs->regs[30] = value;
1495                         }
1496                         if (reg & 0x10) {
1497                                 LoadDW(addr, value, res);
1498                                 if (res)
1499                                         goto fault;
1500                                 regs->regs[31] = value;
1501                         }
1502                         goto success;
1503 #endif /* CONFIG_64BIT */
1504
1505                         goto sigill;
1506
1507                 case mm_sdm_func:
1508 #ifdef CONFIG_64BIT
1509                         reg = insn.mm_m_format.rd;
1510                         rvar = reg & 0xf;
1511                         if ((rvar > 9) || !reg)
1512                                 goto sigill;
1513                         if (reg & 0x10) {
1514                                 if (!access_ok
1515                                     (VERIFY_WRITE, addr, 8 * (rvar + 1)))
1516                                         goto sigbus;
1517                         } else {
1518                                 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
1519                                         goto sigbus;
1520                         }
1521                         if (rvar == 9)
1522                                 rvar = 8;
1523
1524                         for (i = 16; rvar; rvar--, i++) {
1525                                 value = regs->regs[i];
1526                                 StoreDW(addr, value, res);
1527                                 if (res)
1528                                         goto fault;
1529                                 addr += 8;
1530                         }
1531                         if ((reg & 0xf) == 9) {
1532                                 value = regs->regs[30];
1533                                 StoreDW(addr, value, res);
1534                                 if (res)
1535                                         goto fault;
1536                                 addr += 8;
1537                         }
1538                         if (reg & 0x10) {
1539                                 value = regs->regs[31];
1540                                 StoreDW(addr, value, res);
1541                                 if (res)
1542                                         goto fault;
1543                         }
1544                         goto success;
1545 #endif /* CONFIG_64BIT */
1546
1547                         goto sigill;
1548
1549                         /*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1550                 }
1551
1552                 goto sigbus;
1553
1554         case mm_pool32c_op:
1555                 switch (insn.mm_m_format.func) {
1556                 case mm_lwu_func:
1557                         reg = insn.mm_m_format.rd;
1558                         goto loadWU;
1559                 }
1560
1561                 /*  LL,SC,LLD,SCD are not serviced */
1562                 goto sigbus;
1563
1564         case mm_pool32f_op:
1565                 switch (insn.mm_x_format.func) {
1566                 case mm_lwxc1_func:
1567                 case mm_swxc1_func:
1568                 case mm_ldxc1_func:
1569                 case mm_sdxc1_func:
1570                         goto fpu_emul;
1571                 }
1572
1573                 goto sigbus;
1574
1575         case mm_ldc132_op:
1576         case mm_sdc132_op:
1577         case mm_lwc132_op:
1578         case mm_swc132_op:
1579 fpu_emul:
1580                 /* roll back jump/branch */
1581                 regs->cp0_epc = origpc;
1582                 regs->regs[31] = orig31;
1583
1584                 die_if_kernel("Unaligned FP access in kernel code", regs);
1585                 BUG_ON(!used_math());
1586                 BUG_ON(!is_fpu_owner());
1587
1588                 lose_fpu(1);    /* save the FPU state for the emulator */
1589                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1590                                                &fault_addr);
1591                 own_fpu(1);     /* restore FPU state */
1592
1593                 /* If something went wrong, signal */
1594                 process_fpemu_return(res, fault_addr);
1595
1596                 if (res == 0)
1597                         goto success;
1598                 return;
1599
1600         case mm_lh32_op:
1601                 reg = insn.mm_i_format.rt;
1602                 goto loadHW;
1603
1604         case mm_lhu32_op:
1605                 reg = insn.mm_i_format.rt;
1606                 goto loadHWU;
1607
1608         case mm_lw32_op:
1609                 reg = insn.mm_i_format.rt;
1610                 goto loadW;
1611
1612         case mm_sh32_op:
1613                 reg = insn.mm_i_format.rt;
1614                 goto storeHW;
1615
1616         case mm_sw32_op:
1617                 reg = insn.mm_i_format.rt;
1618                 goto storeW;
1619
1620         case mm_ld32_op:
1621                 reg = insn.mm_i_format.rt;
1622                 goto loadDW;
1623
1624         case mm_sd32_op:
1625                 reg = insn.mm_i_format.rt;
1626                 goto storeDW;
1627
1628         case mm_pool16c_op:
1629                 switch (insn.mm16_m_format.func) {
1630                 case mm_lwm16_op:
1631                         reg = insn.mm16_m_format.rlist;
1632                         rvar = reg + 1;
1633                         if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1634                                 goto sigbus;
1635
1636                         for (i = 16; rvar; rvar--, i++) {
1637                                 LoadW(addr, value, res);
1638                                 if (res)
1639                                         goto fault;
1640                                 addr += 4;
1641                                 regs->regs[i] = value;
1642                         }
1643                         LoadW(addr, value, res);
1644                         if (res)
1645                                 goto fault;
1646                         regs->regs[31] = value;
1647
1648                         goto success;
1649
1650                 case mm_swm16_op:
1651                         reg = insn.mm16_m_format.rlist;
1652                         rvar = reg + 1;
1653                         if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1654                                 goto sigbus;
1655
1656                         for (i = 16; rvar; rvar--, i++) {
1657                                 value = regs->regs[i];
1658                                 StoreW(addr, value, res);
1659                                 if (res)
1660                                         goto fault;
1661                                 addr += 4;
1662                         }
1663                         value = regs->regs[31];
1664                         StoreW(addr, value, res);
1665                         if (res)
1666                                 goto fault;
1667
1668                         goto success;
1669
1670                 }
1671
1672                 goto sigbus;
1673
1674         case mm_lhu16_op:
1675                 reg = reg16to32[insn.mm16_rb_format.rt];
1676                 goto loadHWU;
1677
1678         case mm_lw16_op:
1679                 reg = reg16to32[insn.mm16_rb_format.rt];
1680                 goto loadW;
1681
1682         case mm_sh16_op:
1683                 reg = reg16to32st[insn.mm16_rb_format.rt];
1684                 goto storeHW;
1685
1686         case mm_sw16_op:
1687                 reg = reg16to32st[insn.mm16_rb_format.rt];
1688                 goto storeW;
1689
1690         case mm_lwsp16_op:
1691                 reg = insn.mm16_r5_format.rt;
1692                 goto loadW;
1693
1694         case mm_swsp16_op:
1695                 reg = insn.mm16_r5_format.rt;
1696                 goto storeW;
1697
1698         case mm_lwgp16_op:
1699                 reg = reg16to32[insn.mm16_r3_format.rt];
1700                 goto loadW;
1701
1702         default:
1703                 goto sigill;
1704         }
1705
1706 loadHW:
1707         if (!access_ok(VERIFY_READ, addr, 2))
1708                 goto sigbus;
1709
1710         LoadHW(addr, value, res);
1711         if (res)
1712                 goto fault;
1713         regs->regs[reg] = value;
1714         goto success;
1715
1716 loadHWU:
1717         if (!access_ok(VERIFY_READ, addr, 2))
1718                 goto sigbus;
1719
1720         LoadHWU(addr, value, res);
1721         if (res)
1722                 goto fault;
1723         regs->regs[reg] = value;
1724         goto success;
1725
1726 loadW:
1727         if (!access_ok(VERIFY_READ, addr, 4))
1728                 goto sigbus;
1729
1730         LoadW(addr, value, res);
1731         if (res)
1732                 goto fault;
1733         regs->regs[reg] = value;
1734         goto success;
1735
1736 loadWU:
1737 #ifdef CONFIG_64BIT
1738         /*
1739          * A 32-bit kernel might be running on a 64-bit processor.  But
1740          * if we're on a 32-bit processor and an i-cache incoherency
1741          * or race makes us see a 64-bit instruction here the sdl/sdr
1742          * would blow up, so for now we don't handle unaligned 64-bit
1743          * instructions on 32-bit kernels.
1744          */
1745         if (!access_ok(VERIFY_READ, addr, 4))
1746                 goto sigbus;
1747
1748         LoadWU(addr, value, res);
1749         if (res)
1750                 goto fault;
1751         regs->regs[reg] = value;
1752         goto success;
1753 #endif /* CONFIG_64BIT */
1754
1755         /* Cannot handle 64-bit instructions in 32-bit kernel */
1756         goto sigill;
1757
1758 loadDW:
1759 #ifdef CONFIG_64BIT
1760         /*
1761          * A 32-bit kernel might be running on a 64-bit processor.  But
1762          * if we're on a 32-bit processor and an i-cache incoherency
1763          * or race makes us see a 64-bit instruction here the sdl/sdr
1764          * would blow up, so for now we don't handle unaligned 64-bit
1765          * instructions on 32-bit kernels.
1766          */
1767         if (!access_ok(VERIFY_READ, addr, 8))
1768                 goto sigbus;
1769
1770         LoadDW(addr, value, res);
1771         if (res)
1772                 goto fault;
1773         regs->regs[reg] = value;
1774         goto success;
1775 #endif /* CONFIG_64BIT */
1776
1777         /* Cannot handle 64-bit instructions in 32-bit kernel */
1778         goto sigill;
1779
1780 storeHW:
1781         if (!access_ok(VERIFY_WRITE, addr, 2))
1782                 goto sigbus;
1783
1784         value = regs->regs[reg];
1785         StoreHW(addr, value, res);
1786         if (res)
1787                 goto fault;
1788         goto success;
1789
1790 storeW:
1791         if (!access_ok(VERIFY_WRITE, addr, 4))
1792                 goto sigbus;
1793
1794         value = regs->regs[reg];
1795         StoreW(addr, value, res);
1796         if (res)
1797                 goto fault;
1798         goto success;
1799
1800 storeDW:
1801 #ifdef CONFIG_64BIT
1802         /*
1803          * A 32-bit kernel might be running on a 64-bit processor.  But
1804          * if we're on a 32-bit processor and an i-cache incoherency
1805          * or race makes us see a 64-bit instruction here the sdl/sdr
1806          * would blow up, so for now we don't handle unaligned 64-bit
1807          * instructions on 32-bit kernels.
1808          */
1809         if (!access_ok(VERIFY_WRITE, addr, 8))
1810                 goto sigbus;
1811
1812         value = regs->regs[reg];
1813         StoreDW(addr, value, res);
1814         if (res)
1815                 goto fault;
1816         goto success;
1817 #endif /* CONFIG_64BIT */
1818
1819         /* Cannot handle 64-bit instructions in 32-bit kernel */
1820         goto sigill;
1821
1822 success:
1823         regs->cp0_epc = contpc; /* advance or branch */
1824
1825 #ifdef CONFIG_DEBUG_FS
1826         unaligned_instructions++;
1827 #endif
1828         return;
1829
1830 fault:
1831         /* roll back jump/branch */
1832         regs->cp0_epc = origpc;
1833         regs->regs[31] = orig31;
1834         /* Did we have an exception handler installed? */
1835         if (fixup_exception(regs))
1836                 return;
1837
1838         die_if_kernel("Unhandled kernel unaligned access", regs);
1839         force_sig(SIGSEGV, current);
1840
1841         return;
1842
1843 sigbus:
1844         die_if_kernel("Unhandled kernel unaligned access", regs);
1845         force_sig(SIGBUS, current);
1846
1847         return;
1848
1849 sigill:
1850         die_if_kernel
1851             ("Unhandled kernel unaligned access or invalid instruction", regs);
1852         force_sig(SIGILL, current);
1853 }
1854
1855 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
1856 {
1857         unsigned long value;
1858         unsigned int res;
1859         int reg;
1860         unsigned long orig31;
1861         u16 __user *pc16;
1862         unsigned long origpc;
1863         union mips16e_instruction mips16inst, oldinst;
1864
1865         origpc = regs->cp0_epc;
1866         orig31 = regs->regs[31];
1867         pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
1868         /*
1869          * This load never faults.
1870          */
1871         __get_user(mips16inst.full, pc16);
1872         oldinst = mips16inst;
1873
1874         /* skip EXTEND instruction */
1875         if (mips16inst.ri.opcode == MIPS16e_extend_op) {
1876                 pc16++;
1877                 __get_user(mips16inst.full, pc16);
1878         } else if (delay_slot(regs)) {
1879                 /*  skip jump instructions */
1880                 /*  JAL/JALX are 32 bits but have OPCODE in first short int */
1881                 if (mips16inst.ri.opcode == MIPS16e_jal_op)
1882                         pc16++;
1883                 pc16++;
1884                 if (get_user(mips16inst.full, pc16))
1885                         goto sigbus;
1886         }
1887
1888         switch (mips16inst.ri.opcode) {
1889         case MIPS16e_i64_op:    /* I64 or RI64 instruction */
1890                 switch (mips16inst.i64.func) {  /* I64/RI64 func field check */
1891                 case MIPS16e_ldpc_func:
1892                 case MIPS16e_ldsp_func:
1893                         reg = reg16to32[mips16inst.ri64.ry];
1894                         goto loadDW;
1895
1896                 case MIPS16e_sdsp_func:
1897                         reg = reg16to32[mips16inst.ri64.ry];
1898                         goto writeDW;
1899
1900                 case MIPS16e_sdrasp_func:
1901                         reg = 29;       /* GPRSP */
1902                         goto writeDW;
1903                 }
1904
1905                 goto sigbus;
1906
1907         case MIPS16e_swsp_op:
1908         case MIPS16e_lwpc_op:
1909         case MIPS16e_lwsp_op:
1910                 reg = reg16to32[mips16inst.ri.rx];
1911                 break;
1912
1913         case MIPS16e_i8_op:
1914                 if (mips16inst.i8.func != MIPS16e_swrasp_func)
1915                         goto sigbus;
1916                 reg = 29;       /* GPRSP */
1917                 break;
1918
1919         default:
1920                 reg = reg16to32[mips16inst.rri.ry];
1921                 break;
1922         }
1923
1924         switch (mips16inst.ri.opcode) {
1925
1926         case MIPS16e_lb_op:
1927         case MIPS16e_lbu_op:
1928         case MIPS16e_sb_op:
1929                 goto sigbus;
1930
1931         case MIPS16e_lh_op:
1932                 if (!access_ok(VERIFY_READ, addr, 2))
1933                         goto sigbus;
1934
1935                 LoadHW(addr, value, res);
1936                 if (res)
1937                         goto fault;
1938                 MIPS16e_compute_return_epc(regs, &oldinst);
1939                 regs->regs[reg] = value;
1940                 break;
1941
1942         case MIPS16e_lhu_op:
1943                 if (!access_ok(VERIFY_READ, addr, 2))
1944                         goto sigbus;
1945
1946                 LoadHWU(addr, value, res);
1947                 if (res)
1948                         goto fault;
1949                 MIPS16e_compute_return_epc(regs, &oldinst);
1950                 regs->regs[reg] = value;
1951                 break;
1952
1953         case MIPS16e_lw_op:
1954         case MIPS16e_lwpc_op:
1955         case MIPS16e_lwsp_op:
1956                 if (!access_ok(VERIFY_READ, addr, 4))
1957                         goto sigbus;
1958
1959                 LoadW(addr, value, res);
1960                 if (res)
1961                         goto fault;
1962                 MIPS16e_compute_return_epc(regs, &oldinst);
1963                 regs->regs[reg] = value;
1964                 break;
1965
1966         case MIPS16e_lwu_op:
1967 #ifdef CONFIG_64BIT
1968                 /*
1969                  * A 32-bit kernel might be running on a 64-bit processor.  But
1970                  * if we're on a 32-bit processor and an i-cache incoherency
1971                  * or race makes us see a 64-bit instruction here the sdl/sdr
1972                  * would blow up, so for now we don't handle unaligned 64-bit
1973                  * instructions on 32-bit kernels.
1974                  */
1975                 if (!access_ok(VERIFY_READ, addr, 4))
1976                         goto sigbus;
1977
1978                 LoadWU(addr, value, res);
1979                 if (res)
1980                         goto fault;
1981                 MIPS16e_compute_return_epc(regs, &oldinst);
1982                 regs->regs[reg] = value;
1983                 break;
1984 #endif /* CONFIG_64BIT */
1985
1986                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1987                 goto sigill;
1988
1989         case MIPS16e_ld_op:
1990 loadDW:
1991 #ifdef CONFIG_64BIT
1992                 /*
1993                  * A 32-bit kernel might be running on a 64-bit processor.  But
1994                  * if we're on a 32-bit processor and an i-cache incoherency
1995                  * or race makes us see a 64-bit instruction here the sdl/sdr
1996                  * would blow up, so for now we don't handle unaligned 64-bit
1997                  * instructions on 32-bit kernels.
1998                  */
1999                 if (!access_ok(VERIFY_READ, addr, 8))
2000                         goto sigbus;
2001
2002                 LoadDW(addr, value, res);
2003                 if (res)
2004                         goto fault;
2005                 MIPS16e_compute_return_epc(regs, &oldinst);
2006                 regs->regs[reg] = value;
2007                 break;
2008 #endif /* CONFIG_64BIT */
2009
2010                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2011                 goto sigill;
2012
2013         case MIPS16e_sh_op:
2014                 if (!access_ok(VERIFY_WRITE, addr, 2))
2015                         goto sigbus;
2016
2017                 MIPS16e_compute_return_epc(regs, &oldinst);
2018                 value = regs->regs[reg];
2019                 StoreHW(addr, value, res);
2020                 if (res)
2021                         goto fault;
2022                 break;
2023
2024         case MIPS16e_sw_op:
2025         case MIPS16e_swsp_op:
2026         case MIPS16e_i8_op:     /* actually - MIPS16e_swrasp_func */
2027                 if (!access_ok(VERIFY_WRITE, addr, 4))
2028                         goto sigbus;
2029
2030                 MIPS16e_compute_return_epc(regs, &oldinst);
2031                 value = regs->regs[reg];
2032                 StoreW(addr, value, res);
2033                 if (res)
2034                         goto fault;
2035                 break;
2036
2037         case MIPS16e_sd_op:
2038 writeDW:
2039 #ifdef CONFIG_64BIT
2040                 /*
2041                  * A 32-bit kernel might be running on a 64-bit processor.  But
2042                  * if we're on a 32-bit processor and an i-cache incoherency
2043                  * or race makes us see a 64-bit instruction here the sdl/sdr
2044                  * would blow up, so for now we don't handle unaligned 64-bit
2045                  * instructions on 32-bit kernels.
2046                  */
2047                 if (!access_ok(VERIFY_WRITE, addr, 8))
2048                         goto sigbus;
2049
2050                 MIPS16e_compute_return_epc(regs, &oldinst);
2051                 value = regs->regs[reg];
2052                 StoreDW(addr, value, res);
2053                 if (res)
2054                         goto fault;
2055                 break;
2056 #endif /* CONFIG_64BIT */
2057
2058                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2059                 goto sigill;
2060
2061         default:
2062                 /*
2063                  * Pheeee...  We encountered an yet unknown instruction or
2064                  * cache coherence problem.  Die sucker, die ...
2065                  */
2066                 goto sigill;
2067         }
2068
2069 #ifdef CONFIG_DEBUG_FS
2070         unaligned_instructions++;
2071 #endif
2072
2073         return;
2074
2075 fault:
2076         /* roll back jump/branch */
2077         regs->cp0_epc = origpc;
2078         regs->regs[31] = orig31;
2079         /* Did we have an exception handler installed? */
2080         if (fixup_exception(regs))
2081                 return;
2082
2083         die_if_kernel("Unhandled kernel unaligned access", regs);
2084         force_sig(SIGSEGV, current);
2085
2086         return;
2087
2088 sigbus:
2089         die_if_kernel("Unhandled kernel unaligned access", regs);
2090         force_sig(SIGBUS, current);
2091
2092         return;
2093
2094 sigill:
2095         die_if_kernel
2096             ("Unhandled kernel unaligned access or invalid instruction", regs);
2097         force_sig(SIGILL, current);
2098 }
2099
2100 asmlinkage void do_ade(struct pt_regs *regs)
2101 {
2102         enum ctx_state prev_state;
2103         unsigned int __user *pc;
2104         mm_segment_t seg;
2105
2106         prev_state = exception_enter();
2107         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2108                         1, regs, regs->cp0_badvaddr);
2109         /*
2110          * Did we catch a fault trying to load an instruction?
2111          */
2112         if (regs->cp0_badvaddr == regs->cp0_epc)
2113                 goto sigbus;
2114
2115         if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2116                 goto sigbus;
2117         if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2118                 goto sigbus;
2119
2120         /*
2121          * Do branch emulation only if we didn't forward the exception.
2122          * This is all so but ugly ...
2123          */
2124
2125         /*
2126          * Are we running in microMIPS mode?
2127          */
2128         if (get_isa16_mode(regs->cp0_epc)) {
2129                 /*
2130                  * Did we catch a fault trying to load an instruction in
2131                  * 16-bit mode?
2132                  */
2133                 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2134                         goto sigbus;
2135                 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2136                         show_registers(regs);
2137
2138                 if (cpu_has_mmips) {
2139                         seg = get_fs();
2140                         if (!user_mode(regs))
2141                                 set_fs(KERNEL_DS);
2142                         emulate_load_store_microMIPS(regs,
2143                                 (void __user *)regs->cp0_badvaddr);
2144                         set_fs(seg);
2145
2146                         return;
2147                 }
2148
2149                 if (cpu_has_mips16) {
2150                         seg = get_fs();
2151                         if (!user_mode(regs))
2152                                 set_fs(KERNEL_DS);
2153                         emulate_load_store_MIPS16e(regs,
2154                                 (void __user *)regs->cp0_badvaddr);
2155                         set_fs(seg);
2156
2157                         return;
2158         }
2159
2160                 goto sigbus;
2161         }
2162
2163         if (unaligned_action == UNALIGNED_ACTION_SHOW)
2164                 show_registers(regs);
2165         pc = (unsigned int __user *)exception_epc(regs);
2166
2167         seg = get_fs();
2168         if (!user_mode(regs))
2169                 set_fs(KERNEL_DS);
2170         emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2171         set_fs(seg);
2172
2173         return;
2174
2175 sigbus:
2176         die_if_kernel("Kernel unaligned instruction access", regs);
2177         force_sig(SIGBUS, current);
2178
2179         /*
2180          * XXX On return from the signal handler we should advance the epc
2181          */
2182         exception_exit(prev_state);
2183 }
2184
2185 #ifdef CONFIG_DEBUG_FS
2186 extern struct dentry *mips_debugfs_dir;
2187 static int __init debugfs_unaligned(void)
2188 {
2189         struct dentry *d;
2190
2191         if (!mips_debugfs_dir)
2192                 return -ENODEV;
2193         d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
2194                                mips_debugfs_dir, &unaligned_instructions);
2195         if (!d)
2196                 return -ENOMEM;
2197         d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2198                                mips_debugfs_dir, &unaligned_action);
2199         if (!d)
2200                 return -ENOMEM;
2201         return 0;
2202 }
2203 __initcall(debugfs_unaligned);
2204 #endif