arm64: reduce stack use in irq_handler

[karo-tx-linux.git] / arch / arm64 / kernel / entry.S

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 7ed3d75f630418b56a1add8c91b308b48cd69774..0667fb7d8bb1f3d8d5eaa0ec8778b9f8fe4bb5c3 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -27,6 +27,7 @@
 #include <asm/cpufeature.h>
 #include <asm/errno.h>
 #include <asm/esr.h>
+#include <asm/irq.h>
 #include <asm/thread_info.h>
 #include <asm/unistd.h>
 
@@ -88,9 +89,12 @@
 
        .if     \el == 0
        mrs     x21, sp_el0
-       get_thread_info tsk                     // Ensure MDSCR_EL1.SS is clear,
+       mov     tsk, sp
+       and     tsk, tsk, #~(THREAD_SIZE - 1)   // Ensure MDSCR_EL1.SS is clear,
        ldr     x19, [tsk, #TI_FLAGS]           // since we can unmask debug
        disable_step_tsk x19, x20               // exceptions when scheduling.
+
+       mov     x29, xzr                        // fp pointed to user-space
        .else
        add     x21, sp, #S_FRAME_SIZE
        .endif
@@ -107,6 +111,13 @@
        str     x21, [sp, #S_SYSCALLNO]
        .endif
 
+       /*
+        * Set sp_el0 to current thread_info.
+        */
+       .if     \el == 0
+       msr     sp_el0, tsk
+       .endif
+
        /*
         * Registers that may be useful after this macro is invoked:
         *
@@ -164,8 +175,43 @@ alternative_endif
        .endm
 
        .macro  get_thread_info, rd
-       mov     \rd, sp
-       and     \rd, \rd, #~(THREAD_SIZE - 1)   // top of stack
+       mrs     \rd, sp_el0
+       .endm
+
+       .macro  irq_stack_entry
+       mov     x19, sp                 // preserve the original sp
+
+       this_cpu_ptr irq_stack, x25, x26
+
+       /*
+        * Check the lowest address on irq_stack for the irq_count value,
+        * incremented by do_softirq_own_stack if we have re-enabled irqs
+        * while on the irq_stack.
+        */
+       ldr     x26, [x25]
+       cbnz    x26, 9998f              // recursive use?
+
+       /* switch to the irq stack */
+       mov     x26, #IRQ_STACK_START_SP
+       add     x26, x25, x26
+       mov     sp, x26
+
+       /*
+        * Add a dummy stack frame, this non-standard format is fixed up
+        * by unwind_frame()
+        */
+       stp     x29, x19, [sp, #-16]!
+       mov     x29, sp
+
+9998:
+       .endm
+
+       /*
+        * x19 should be preserved between irq_stack_entry and
+        * irq_stack_exit.
+        */
+       .macro  irq_stack_exit
+       mov     sp, x19
        .endm
 
 /*
@@ -183,10 +229,11 @@ tsk       .req    x28             // current thread_info
  * Interrupt handling.
  */
        .macro  irq_handler
-       adrp    x1, handle_arch_irq
-       ldr     x1, [x1, #:lo12:handle_arch_irq]
+       ldr_l   x1, handle_arch_irq
        mov     x0, sp
+       irq_stack_entry
        blr     x1
+       irq_stack_exit
        .endm
 
        .text
@@ -599,6 +646,8 @@ ENTRY(cpu_switch_to)
        ldp     x29, x9, [x8], #16
        ldr     lr, [x8]
        mov     sp, x9
+       and     x9, x9, #~(THREAD_SIZE - 1)
+       msr     sp_el0, x9
        ret
 ENDPROC(cpu_switch_to)
 
@@ -634,6 +683,9 @@ work_pending:
        bl      do_notify_resume
        b       ret_to_user
 work_resched:
+#ifdef CONFIG_TRACE_IRQFLAGS
+       bl      trace_hardirqs_off              // the IRQs are off here, inform the tracing code
+#endif
        bl      schedule
 
 /*