Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...

[~shefty/rdma-dev.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index b62aa298df7f336d060ccc8e6a1241d2673bfc2c..940ba711fc286510ddc20c441fd882fe7a632b62 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -221,7 +221,7 @@ ENDPROC(native_usergs_sysret64)
        /*CFI_REL_OFFSET        ss,0*/
        pushq_cfi %rax /* rsp */
        CFI_REL_OFFSET  rsp,0
        /*CFI_REL_OFFSET        ss,0*/
        pushq_cfi %rax /* rsp */
        CFI_REL_OFFSET  rsp,0

-       pushq_cfi $X86_EFLAGS_IF /* eflags - interrupts on */
+       pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_BIT1) /* eflags - interrupts on */

        /*CFI_REL_OFFSET        rflags,0*/
        pushq_cfi $__KERNEL_CS /* cs */
        /*CFI_REL_OFFSET        cs,0*/
        /*CFI_REL_OFFSET        rflags,0*/
        pushq_cfi $__KERNEL_CS /* cs */
        /*CFI_REL_OFFSET        cs,0*/


@@ -411,7 +411,7 @@ ENTRY(ret_from_fork)
        RESTORE_REST
 
        testl $3, CS-ARGOFFSET(%rsp)            # from kernel_thread?
        RESTORE_REST
 
        testl $3, CS-ARGOFFSET(%rsp)            # from kernel_thread?

-       je   int_ret_from_sys_call
+       jz   retint_restore_args

 
        testl $_TIF_IA32, TI_flags(%rcx)        # 32-bit compat task needs IRET
        jnz  int_ret_from_sys_call
 
        testl $_TIF_IA32, TI_flags(%rcx)        # 32-bit compat task needs IRET
        jnz  int_ret_from_sys_call


@@ -465,7 +465,7 @@ ENTRY(system_call)
         * after the swapgs, so that it can do the swapgs
         * for the guest and jump here on syscall.
         */
         * after the swapgs, so that it can do the swapgs
         * for the guest and jump here on syscall.
         */

-ENTRY(system_call_after_swapgs)
+GLOBAL(system_call_after_swapgs)

 
        movq    %rsp,PER_CPU_VAR(old_rsp)
        movq    PER_CPU_VAR(kernel_stack),%rsp
 
        movq    %rsp,PER_CPU_VAR(old_rsp)
        movq    PER_CPU_VAR(kernel_stack),%rsp


@@ -478,8 +478,7 @@ ENTRY(system_call_after_swapgs)
        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
        movq  %rcx,RIP-ARGOFFSET(%rsp)
        CFI_REL_OFFSET rip,RIP-ARGOFFSET
        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
        movq  %rcx,RIP-ARGOFFSET(%rsp)
        CFI_REL_OFFSET rip,RIP-ARGOFFSET

-       GET_THREAD_INFO(%rcx)
-       testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
+       testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

        jnz tracesys
 system_call_fastpath:
        cmpq $__NR_syscall_max,%rax
        jnz tracesys
 system_call_fastpath:
        cmpq $__NR_syscall_max,%rax


@@ -496,10 +495,9 @@ ret_from_sys_call:
        /* edi: flagmask */
 sysret_check:
        LOCKDEP_SYS_EXIT
        /* edi: flagmask */
 sysret_check:
        LOCKDEP_SYS_EXIT

-       GET_THREAD_INFO(%rcx)

        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF

-       movl TI_flags(%rcx),%edx
+       movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx

        andl %edi,%edx
        jnz  sysret_careful
        CFI_REMEMBER_STATE
        andl %edi,%edx
        jnz  sysret_careful
        CFI_REMEMBER_STATE


@@ -583,7 +581,7 @@ sysret_audit:
        /* Do syscall tracing */
 tracesys:
 #ifdef CONFIG_AUDITSYSCALL
        /* Do syscall tracing */
 tracesys:
 #ifdef CONFIG_AUDITSYSCALL

-       testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
+       testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)

        jz auditsys
 #endif
        SAVE_REST
        jz auditsys
 #endif
        SAVE_REST


@@ -612,8 +610,6 @@ tracesys:
 GLOBAL(int_ret_from_sys_call)
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
 GLOBAL(int_ret_from_sys_call)
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF

-       testl $3,CS-ARGOFFSET(%rsp)
-       je retint_restore_args

        movl $_TIF_ALLWORK_MASK,%edi
        /* edi: mask to check */
 GLOBAL(int_with_check)
        movl $_TIF_ALLWORK_MASK,%edi
        /* edi: mask to check */
 GLOBAL(int_with_check)


@@ -953,6 +949,7 @@ END(common_interrupt)
 ENTRY(\sym)
        INTR_FRAME
        pushq_cfi $~(\num)
 ENTRY(\sym)
        INTR_FRAME
        pushq_cfi $~(\num)

+.Lcommon_\sym:

        interrupt \do_sym
        jmp ret_from_intr
        CFI_ENDPROC
        interrupt \do_sym
        jmp ret_from_intr
        CFI_ENDPROC


@@ -976,13 +973,21 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR \
        x86_platform_ipi smp_x86_platform_ipi
 
 #ifdef CONFIG_SMP
        x86_platform_ipi smp_x86_platform_ipi
 
 #ifdef CONFIG_SMP

-.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
+       ALIGN
+       INTR_FRAME
+.irp idx,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \

        16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
 .if NUM_INVALIDATE_TLB_VECTORS > \idx
        16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
 .if NUM_INVALIDATE_TLB_VECTORS > \idx

-apicinterrupt (INVALIDATE_TLB_VECTOR_START)+\idx \
-       invalidate_interrupt\idx smp_invalidate_interrupt
+ENTRY(invalidate_interrupt\idx)
+       pushq_cfi $~(INVALIDATE_TLB_VECTOR_START+\idx)
+       jmp .Lcommon_invalidate_interrupt0
+       CFI_ADJUST_CFA_OFFSET -8
+END(invalidate_interrupt\idx)

 .endif
 .endr
 .endif
 .endr

+       CFI_ENDPROC
+apicinterrupt INVALIDATE_TLB_VECTOR_START, \
+       invalidate_interrupt0, smp_invalidate_interrupt

 #endif
 
 apicinterrupt THRESHOLD_APIC_VECTOR \
 #endif
 
 apicinterrupt THRESHOLD_APIC_VECTOR \