x86: use this_cpu_xxx to replace percpu_xxx funcs

Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.

And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.

BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine.  But it maybe other
platforms' performance.

[akpm@linux-foundation.org: fix build]
[sfr@canb.auug.org.au: arch/x86/include/asm/desc.h: fix smp_processor_id's need for this_cpu_read]
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@gentwo.org>
Acked-by: Tejun Heo <tj@kernel.org>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index e95822d683f4639f7211d4f3775975fcd87466ad..8bf1c06070d5655b3fcad7727ad53bd5e8ceaf8a 100644 (file)
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -6,6 +6,7 @@
 #include <asm/mmu.h>
 
 #include <linux/smp.h>
+#include <linux/percpu.h>
 
 static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info)
 {

diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 382f75d735f36723a40383fa9d71955ac9779eda..0c2b6d1b439e553871574ea840bb51ebfe36bf61 100644 (file)
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 
 #define __ARCH_IRQ_STAT
 
-#define inc_irq_stat(member)   percpu_inc(irq_stat.member)
+#define inc_irq_stat(member)   __this_cpu_inc(irq_stat.member)
 
-#define local_softirq_pending()        percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending()        __this_cpu_read(irq_stat.__softirq_pending)
 
 #define __ARCH_SET_SOFTIRQ_PENDING
 
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x)  percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) \
+               __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x)  __this_cpu_or(irq_stat.__softirq_pending, (x))
 
 extern void ack_bad_irq(unsigned int irq);
 

diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 77843225b7eac71e06331383106d7d962f332043..15639ed5d7ab91e1bd6368a74ebf11dd7a2048a3 100644 (file)
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
 
 static inline struct pt_regs *get_irq_regs(void)
 {
-       return percpu_read(irq_regs);
+       return __this_cpu_read(irq_regs);
 }
 
 static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
        struct pt_regs *old_regs;
 
        old_regs = get_irq_regs();
-       percpu_write(irq_regs, new_regs);
+       __this_cpu_write(irq_regs, new_regs);
 
        return old_regs;
 }

diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 69021528b43c240c2c67c4c4ab86b579b40c4ae5..02ca533554c41386e7f2cd1a9f9115ae847554e2 100644 (file)
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 #ifdef CONFIG_SMP
-       if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
-               percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+       if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+               __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
 #endif
 }
 
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 
        if (likely(prev != next)) {
 #ifdef CONFIG_SMP
-               percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
-               percpu_write(cpu_tlbstate.active_mm, next);
+               __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+               __this_cpu_write(cpu_tlbstate.active_mm, next);
 #endif
                cpumask_set_cpu(cpu, mm_cpumask(next));
 
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
        }
 #ifdef CONFIG_SMP
        else {
-               percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
-               BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+               __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+               BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
 
                if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
                        /* We were in lazy tlb mode and leave_mm disabled

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 7a11910a63c4d4857a5a951ebb56eaeb31599092..276bbc07f007231ad71edc2b43416188ab296885 100644 (file)
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
 
 #ifdef CONFIG_SMP
 #define __percpu_prefix                "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset                percpu_read(this_cpu_off)
+#define __my_cpu_offset                __this_cpu_read(this_cpu_off)
 
 /*
  * Compared to the generic __my_cpu_offset version, the following

diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 0434c400287cffa9c8d80075c5a31defa4097629..e276f6bb65244e116650d7c2a02156e5db42a56e 100644 (file)
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
  * from the initial startup. We map APIC_BASE very early in page_setup(),
  * so this is correct in the x86 case.
  */
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
 extern int safe_smp_processor_id(void);
 
 #elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
 
 #define stack_smp_processor_id()                                       \
 ({                                                             \

diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index b5d9533d2c38f4e78485256ec9ba07497e93057e..ac75993d53f3af4d845ee7a93e5920d036d5839b 100644 (file)
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -75,9 +75,9 @@ static __always_inline void boot_init_stack_canary(void)
 
        current->stack_canary = canary;
 #ifdef CONFIG_X86_64
-       percpu_write(irq_stack_union.stack_canary, canary);
+       __this_cpu_write(irq_stack_union.stack_canary, canary);
 #else
-       percpu_write(stack_canary.canary, canary);
+       __this_cpu_write(stack_canary.canary, canary);
 #endif
 }
 

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c0e108e08079e9a80ee075a5de44b31b4ba78d97..4ece077d17dcfb2dbd8b8370228ed5ad39ce0eb9 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
 
 static inline void reset_lazy_tlbstate(void)
 {
-       percpu_write(cpu_tlbstate.state, 0);
-       percpu_write(cpu_tlbstate.active_mm, &init_mm);
+       __this_cpu_write(cpu_tlbstate.state, 0);
+       __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
 }
 
 #endif /* SMP */

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 67e258362a3d56d7f94c675ec876705003469c3a..f4f0fbf84cfd784a8c9262e5ee01864a6e7770ec 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1194,7 +1194,7 @@ void __cpuinit cpu_init(void)
        oist = &per_cpu(orig_ist, cpu);
 
 #ifdef CONFIG_NUMA
-       if (cpu != 0 && percpu_read(numa_node) == 0 &&
+       if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
            early_cpu_to_node(cpu) != NUMA_NO_NODE)
                set_numa_node(early_cpu_to_node(cpu));
 #endif

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index d086a09c087d7fa3cf8d1ee3f20176623b1df9f6..b1fad5dd8f6a18a012d9d37d15f4ac79d5e538c1 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -583,7 +583,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
        struct mce m;
        int i;
 
-       percpu_inc(mce_poll_count);
+       __this_cpu_inc(mce_poll_count);
 
        mce_gather_info(&m, NULL);
 
@@ -1015,7 +1015,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
 
        atomic_inc(&mce_entry);
 
-       percpu_inc(mce_exception_count);
+       __this_cpu_inc(mce_exception_count);
 
        if (!banks)
                goto out;

diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index 7734bcbb5a3a3b21e11374f82747972d678e8999..32c3972ab60fef40b954b60b928f7d818f7a76ac 100644 (file)
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -88,7 +88,7 @@ void kernel_fpu_begin(void)
                __thread_clear_has_fpu(me);
                /* We do 'stts()' in kernel_fpu_end() */
        } else {
-               percpu_write(fpu_owner_task, NULL);
+               __this_cpu_write(fpu_owner_task, NULL);
                clts();
        }
 }

diff --git a/arch/x86/kernel/nmi_selftest.c b/arch/x86/kernel/nmi_selftest.c
index 2c39dcd510faee65feb37f1647578d2a0bb46975..ff3698625081506a7acf686f40aa8cd42ccde3c7 100644 (file)
--- a/arch/x86/kernel/nmi_selftest.c
+++ b/arch/x86/kernel/nmi_selftest.c
@@ -13,6 +13,7 @@
 #include <linux/cpumask.h>
 #include <linux/delay.h>
 #include <linux/init.h>
+#include <linux/percpu.h>
 
 #include <asm/apic.h>
 #include <asm/nmi.h>

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index ab137605e694d81ae9adae2d9483d8024936cd2d..f0e6bbdaa56dcc3130cde5e5dc5d0dec78b5ac95 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -241,16 +241,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
 
 static inline void enter_lazy(enum paravirt_lazy_mode mode)
 {
-       BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+       BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
 
-       percpu_write(paravirt_lazy_mode, mode);
+       __this_cpu_write(paravirt_lazy_mode, mode);
 }
 
 static void leave_lazy(enum paravirt_lazy_mode mode)
 {
-       BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+       BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
 
-       percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+       __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
 }
 
 void paravirt_enter_lazy_mmu(void)
@@ -267,7 +267,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
 {
        BUG_ON(preemptible());
 
-       if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+       if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
                arch_leave_lazy_mmu_mode();
                set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
        }
@@ -289,7 +289,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
        if (in_interrupt())
                return PARAVIRT_LAZY_NONE;
 
-       return percpu_read(paravirt_lazy_mode);
+       return __this_cpu_read(paravirt_lazy_mode);
 }
 
 void arch_flush_lazy_mmu_mode(void)

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index ae6847303e265c80c2199ab18884ee16e25bf386..6186d6cce91e27bf5612305b833ca0b187a40dfe 100644 (file)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -302,7 +302,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
        switch_fpu_finish(next_p, fpu);
 
-       percpu_write(current_task, next_p);
+       __this_cpu_write(current_task, next_p);
 
        return prev_p;
 }

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 733ca39f367ebcc222eaca07b87eb170f2831e1e..9ca04b36c8b9653bfb4725ba93d83fcfba39de31 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -237,7 +237,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
        current->thread.usersp  = new_sp;
        regs->ip                = new_ip;
        regs->sp                = new_sp;
-       percpu_write(old_rsp, new_sp);
+       this_cpu_write(old_rsp, new_sp);
        regs->cs                = _cs;
        regs->ss                = _ss;
        regs->flags             = X86_EFLAGS_IF;
@@ -359,11 +359,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        /*
         * Switch the PDA and FPU contexts.
         */
-       prev->usersp = percpu_read(old_rsp);
-       percpu_write(old_rsp, next->usersp);
-       percpu_write(current_task, next_p);
+       prev->usersp = __this_cpu_read(old_rsp);
+       __this_cpu_write(old_rsp, next->usersp);
+       __this_cpu_write(current_task, next_p);
 
-       percpu_write(kernel_stack,
+       __this_cpu_write(kernel_stack,
                  (unsigned long)task_stack_page(next_p) +
                  THREAD_SIZE - KERNEL_STACK_OFFSET);
 

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 125bcad1b757e37ac5682d6c4eec1a0184289fc8..fbd7b135e67dba77a688d1ab28a103433c48b784 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,8 +61,8 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
  */
 void leave_mm(int cpu)
 {
-       struct mm_struct *active_mm = percpu_read(cpu_tlbstate.active_mm);
-       if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+       struct mm_struct *active_mm = __this_cpu_read(cpu_tlbstate.active_mm);
+       if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                BUG();
        if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
                cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
@@ -154,8 +154,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
                 * BUG();
                 */
 
-       if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
-               if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+       if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+               if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
                        if (f->flush_va == TLB_FLUSH_ALL)
                                local_flush_tlb();
                        else
@@ -324,7 +324,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
 static void do_flush_tlb_all(void *info)
 {
        __flush_tlb_all();
-       if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+       if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
                leave_mm(smp_processor_id());
 }
 

diff --git a/include/linux/topology.h b/include/linux/topology.h
index e26db031303b69cee9954780f3ce3f8d276d8644..b4804032e0e1f8862c570dfac76272f93311bf1a 100644 (file)
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)
 #ifndef set_numa_node
 static inline void set_numa_node(int node)
 {
-       percpu_write(numa_node, node);
+       __this_cpu_write(numa_node, node);
 }
 #endif
 
@@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
 #ifndef set_numa_mem
 static inline void set_numa_mem(int node)
 {
-       percpu_write(_numa_mem_, node);
+       __this_cpu_write(_numa_mem_, node);
 }
 #endif