x86/asm/entry/64: Save R11 into pt_regs->flags on SYSCALL64 fastpath

Before this patch, R11 was saved in pt_regs->r11.

Which looks natural, but requires messy shuffling to/from iret
frame whenever ptrace or e.g. sys_iopl() wants to modify flags -
because that's how this register is used by SYSCALL/SYSRET.

This patch saves R11 in pt_regs->flags, and uses that value for
the SYSRET64 instruction. Shuffling is eliminated.

FIXUP/RESTORE_TOP_OF_STACK are simplified.

stub_iopl is no longer needed: pt_regs->flags needs no fixing up.

Testing shows that syscall fast path is ~54.3 ns before
and after the patch (on 2.7 GHz Sandy Bridge CPU).

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1425926364-9526-2-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h
index f1a962ff7ddfc5a5f30365edbfb02b0bdfb13ef8..4b5f7bf2b7809575646ed47012e84c4276785e6f 100644 (file)
--- a/arch/x86/include/asm/calling.h
+++ b/arch/x86/include/asm/calling.h
@@ -95,9 +95,11 @@ For 32-bit we have the following conventions - kernel is built with
        CFI_ADJUST_CFA_OFFSET 15*8+\addskip
        .endm
 
-       .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8plus=1
-       .if \r8plus
+       .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1
+       .if \r11
        movq_cfi r11, 6*8+\offset
+       .endif
+       .if \r8910
        movq_cfi r10, 7*8+\offset
        movq_cfi r9,  8*8+\offset
        movq_cfi r8,  9*8+\offset
@@ -113,16 +115,19 @@ For 32-bit we have the following conventions - kernel is built with
        movq_cfi rdi, 14*8+\offset
        .endm
        .macro SAVE_C_REGS offset=0
-       SAVE_C_REGS_HELPER \offset, 1, 1, 1
+       SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
        .endm
        .macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0
-       SAVE_C_REGS_HELPER \offset, 0, 0, 1
+       SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1
        .endm
        .macro SAVE_C_REGS_EXCEPT_R891011
-       SAVE_C_REGS_HELPER 0, 1, 1, 0
+       SAVE_C_REGS_HELPER 0, 1, 1, 0, 0
        .endm
        .macro SAVE_C_REGS_EXCEPT_RCX_R891011
-       SAVE_C_REGS_HELPER 0, 1, 0, 0
+       SAVE_C_REGS_HELPER 0, 1, 0, 0, 0
+       .endm
+       .macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11
+       SAVE_C_REGS_HELPER 0, 0, 0, 1, 0
        .endm
 
        .macro SAVE_EXTRA_REGS offset=0
@@ -179,6 +184,9 @@ For 32-bit we have the following conventions - kernel is built with
        .macro RESTORE_C_REGS_EXCEPT_R11
        RESTORE_C_REGS_HELPER 1,1,0,1,1
        .endm
+       .macro RESTORE_C_REGS_EXCEPT_RCX_R11
+       RESTORE_C_REGS_HELPER 1,0,0,1,1
+       .endm
        .macro RESTORE_RSI_RDI
        RESTORE_C_REGS_HELPER 0,0,0,0,0
        .endm

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 5117a2baefe92e50136a1e5495514531fba27503..324200aca431431bb2dac2fe9bf6cc7fe0fd6ae7 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64)
 #endif
 
 /*
- * C code is not supposed to know about undefined top of stack. Every time
- * a C function with an pt_regs argument is called from the SYSCALL based
- * fast path FIXUP_TOP_OF_STACK is needed.
+ * C code is not supposed to know that the iret frame is not populated.
+ * Every time a C function with an pt_regs argument is called from
+ * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed.
  * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
  * manipulation.
  */
-
-       /* %rsp:at FRAMEEND */
        .macro FIXUP_TOP_OF_STACK tmp offset=0
        movq PER_CPU_VAR(old_rsp),\tmp
        movq \tmp,RSP+\offset(%rsp)
@@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64)
        movq $__USER_CS,CS+\offset(%rsp)
        movq RIP+\offset(%rsp),\tmp  /* get rip */
        movq \tmp,RCX+\offset(%rsp)  /* copy it to rcx as sysret would do */
-       movq R11+\offset(%rsp),\tmp  /* get eflags */
-       movq \tmp,EFLAGS+\offset(%rsp)
+       movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */
+       movq \tmp,R11+\offset(%rsp)
        .endm
 
        .macro RESTORE_TOP_OF_STACK tmp offset=0
        movq RSP+\offset(%rsp),\tmp
        movq \tmp,PER_CPU_VAR(old_rsp)
-       movq EFLAGS+\offset(%rsp),\tmp
-       movq \tmp,R11+\offset(%rsp)
        .endm
 
 /*
@@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs)
         */
        ENABLE_INTERRUPTS(CLBR_NONE)
        ALLOC_PT_GPREGS_ON_STACK 8              /* +8: space for orig_ax */
-       SAVE_C_REGS_EXCEPT_RAX_RCX
+       SAVE_C_REGS_EXCEPT_RAX_RCX_R11
        movq    $-ENOSYS,RAX(%rsp)
        movq_cfi rax,ORIG_RAX
+       movq    %r11,EFLAGS(%rsp)
        movq    %rcx,RIP(%rsp)
        CFI_REL_OFFSET rip,RIP
        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
@@ -277,7 +274,7 @@ system_call_fastpath:
        movq %rax,RAX(%rsp)
 /*
  * Syscall return path ending with SYSRET (fast path)
- * Has incomplete stack frame and undefined top of stack.
+ * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
        testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
@@ -291,9 +288,10 @@ ret_from_sys_call:
         * sysretq will re-enable interrupts:
         */
        TRACE_IRQS_ON
-       RESTORE_C_REGS_EXCEPT_RCX
-       movq RIP(%rsp),%rcx
+       RESTORE_C_REGS_EXCEPT_RCX_R11
+       movq    RIP(%rsp),%rcx
        CFI_REGISTER    rip,rcx
+       movq    EFLAGS(%rsp),%r11
        /*CFI_REGISTER  rflags,r11*/
        movq    PER_CPU_VAR(old_rsp), %rsp
        /*