x86: Speed up ___preempt_schedule*() by using THUNK helpers

___preempt_schedule() does SAVE_ALL/RESTORE_ALL but this is
suboptimal, we do not need to save/restore the callee-saved
register. And we already have arch/x86/lib/thunk_*.S which
implements the similar asm wrappers, so it makes sense to
redefine ___preempt_schedule() as "THUNK ..." and remove
preempt.S altogether.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Andy Lutomirski <luto@amacapital.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140921184153.GA23727@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index ada2e2d6be3e01335734016a2aae5dc677ab69af..8f1e77440b2bd65049b20a7a446213558b644f15 100644 (file)
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -39,8 +39,6 @@ obj-y                 += tsc.o tsc_msr.o io_delay.o rtc.o
 obj-y                  += pci-iommu_table.o
 obj-y                  += resource.o
 
-obj-$(CONFIG_PREEMPT)  += preempt.o
-
 obj-y                          += process.o
 obj-y                          += i387.o xsave.o
 obj-y                          += ptrace.o

diff --git a/arch/x86/kernel/preempt.S b/arch/x86/kernel/preempt.S
deleted file mode 100644 (file)
index ca7f0d5..0000000
--- a/arch/x86/kernel/preempt.S
+++ /dev/null
@@ -1,25 +0,0 @@
-
-#include <linux/linkage.h>
-#include <asm/dwarf2.h>
-#include <asm/asm.h>
-#include <asm/calling.h>
-
-ENTRY(___preempt_schedule)
-       CFI_STARTPROC
-       SAVE_ALL
-       call preempt_schedule
-       RESTORE_ALL
-       ret
-       CFI_ENDPROC
-
-#ifdef CONFIG_CONTEXT_TRACKING
-
-ENTRY(___preempt_schedule_context)
-       CFI_STARTPROC
-       SAVE_ALL
-       call preempt_schedule_context
-       RESTORE_ALL
-       ret
-       CFI_ENDPROC
-
-#endif

diff --git a/arch/x86/lib/thunk_32.S b/arch/x86/lib/thunk_32.S
index 28f85c916712232e951f30a0e9819b56357a87a9..e9acf5f4fc923caa2bd71cfa175462617130ed89 100644 (file)
--- a/arch/x86/lib/thunk_32.S
+++ b/arch/x86/lib/thunk_32.S
@@ -7,16 +7,19 @@
        #include <linux/linkage.h>
        #include <asm/asm.h>
 
-#ifdef CONFIG_TRACE_IRQFLAGS
        /* put return address in eax (arg1) */
-       .macro thunk_ra name,func
+       .macro THUNK name, func, put_ret_addr_in_eax=0
        .globl \name
 \name:
        pushl %eax
        pushl %ecx
        pushl %edx
+
+       .if \put_ret_addr_in_eax
        /* Place EIP in the arg1 */
        movl 3*4(%esp), %eax
+       .endif
+
        call \func
        popl %edx
        popl %ecx
@@ -25,6 +28,15 @@
        _ASM_NOKPROBE(\name)
        .endm
 
-       thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller
-       thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller
+#ifdef CONFIG_TRACE_IRQFLAGS
+       THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
+       THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
+#endif
+
+#ifdef CONFIG_PREEMPT
+       THUNK ___preempt_schedule, preempt_schedule
+#ifdef CONFIG_CONTEXT_TRACKING
+       THUNK ___preempt_schedule_context, preempt_schedule_context
 #endif
+#endif
+

diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 92d9feaff42b04fa0dd42f1c1f686cf3a43e7636..b30b5ebd614ada18d25b3fb7403f7a32e32a96d9 100644 (file)
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -38,6 +38,13 @@
        THUNK lockdep_sys_exit_thunk,lockdep_sys_exit
 #endif
 
+#ifdef CONFIG_PREEMPT
+       THUNK ___preempt_schedule, preempt_schedule
+#ifdef CONFIG_CONTEXT_TRACKING
+       THUNK ___preempt_schedule_context, preempt_schedule_context
+#endif
+#endif
+
        /* SAVE_ARGS below is used only for the .cfi directives it contains. */
        CFI_STARTPROC
        SAVE_ARGS